fix: 批量插入分片规避 SQLite 999 变量上限 (too many SQL variables)

log_entries 表 14 列 + id,500 行单 INSERT 产生约 7500 个绑定变量,
超出 SQLite 单语句 999 变量硬限。改为按列数自动计算每语句最大行数
(约 66 行) 分片提交,整批一次性 commit 保持事务性。
This commit is contained in:
Hermes Agent
2026-07-15 14:56:15 +08:00
parent a875d790b0
commit 6ca58f3dea
+32 -9
View File
@@ -198,20 +198,43 @@ def sync_log_to_db(
return 0, 0, f"{type(e).__name__}: {e}"
def _flush_batch(batch: list[dict]) -> None:
"""Insert a batch of rows in a single SQL statement.
def _column_count() -> int:
"""Return the number of columns in the :class:`~app.LogEntry` table.
We rely on the caller's in-memory ``(instance_id, log_time, client,
raw_line)`` set (built in :func:`sync_log_to_db`) for dedup since we
don't have a SQL-level UNIQUE constraint. This means inserts are
guaranteed to be new rows at insert time, so we use a plain
``insert().values()`` rather than ``on_conflict_do_nothing``.
Used to cap the per-statement row count so the single INSERT does not
exceed SQLite's 999-boundary-variable limit per SQL statement.
"""
return len(LogEntry.__table__.columns)
# SQLite hard limit: at most 999 ? placeholders per single SQL statement.
_SQLITE_MAX_VARIABLES = 999
def _max_rows_per_stmt() -> int:
"""Max rows that can safely fit in one parameterised INSERT."""
cols = _column_count()
return max(1, (_SQLITE_MAX_VARIABLES - 1) // cols)
def _flush_batch(batch: list[dict]) -> None:
"""Insert ``batch`` rows, splitting into chunks small enough for SQLite.
SQLite rejects a single parameterised INSERT when its bound-variable
count exceeds 999. Since :class:`~app.LogEntry` carries many columns, a
'large' batch (e.g. 500 rows) would blow past that limit. Split the
batch into sub-chunks of ``_max_rows_per_stmt()`` rows and execute each
sub-statement individually, then commit once the whole batch is in.
"""
if not batch:
return
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
stmt = sqlite_insert(LogEntry).values(batch)
db.session.execute(stmt)
chunk_size = _max_rows_per_stmt()
for start in range(0, len(batch), chunk_size):
chunk = batch[start:start + chunk_size]
stmt = sqlite_insert(LogEntry).values(chunk)
db.session.execute(stmt)
db.session.commit()