fix: 批量插入分片规避 SQLite 999 变量上限 (too many SQL variables)
log_entries 表 14 列 + id,500 行单 INSERT 产生约 7500 个绑定变量, 超出 SQLite 单语句 999 变量硬限。改为按列数自动计算每语句最大行数 (约 66 行) 分片提交,整批一次性 commit 保持事务性。
This commit is contained in:
+32
-9
@@ -198,20 +198,43 @@ def sync_log_to_db(
|
||||
return 0, 0, f"{type(e).__name__}: {e}"
|
||||
|
||||
|
||||
def _flush_batch(batch: list[dict]) -> None:
|
||||
"""Insert a batch of rows in a single SQL statement.
|
||||
def _column_count() -> int:
|
||||
"""Return the number of columns in the :class:`~app.LogEntry` table.
|
||||
|
||||
We rely on the caller's in-memory ``(instance_id, log_time, client,
|
||||
raw_line)`` set (built in :func:`sync_log_to_db`) for dedup since we
|
||||
don't have a SQL-level UNIQUE constraint. This means inserts are
|
||||
guaranteed to be new rows at insert time, so we use a plain
|
||||
``insert().values()`` rather than ``on_conflict_do_nothing``.
|
||||
Used to cap the per-statement row count so the single INSERT does not
|
||||
exceed SQLite's 999-boundary-variable limit per SQL statement.
|
||||
"""
|
||||
return len(LogEntry.__table__.columns)
|
||||
|
||||
|
||||
# SQLite hard limit: at most 999 ? placeholders per single SQL statement.
|
||||
_SQLITE_MAX_VARIABLES = 999
|
||||
|
||||
|
||||
def _max_rows_per_stmt() -> int:
|
||||
"""Max rows that can safely fit in one parameterised INSERT."""
|
||||
cols = _column_count()
|
||||
return max(1, (_SQLITE_MAX_VARIABLES - 1) // cols)
|
||||
|
||||
|
||||
def _flush_batch(batch: list[dict]) -> None:
|
||||
"""Insert ``batch`` rows, splitting into chunks small enough for SQLite.
|
||||
|
||||
SQLite rejects a single parameterised INSERT when its bound-variable
|
||||
count exceeds 999. Since :class:`~app.LogEntry` carries many columns, a
|
||||
'large' batch (e.g. 500 rows) would blow past that limit. Split the
|
||||
batch into sub-chunks of ``_max_rows_per_stmt()`` rows and execute each
|
||||
sub-statement individually, then commit once the whole batch is in.
|
||||
"""
|
||||
if not batch:
|
||||
return
|
||||
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
||||
stmt = sqlite_insert(LogEntry).values(batch)
|
||||
db.session.execute(stmt)
|
||||
|
||||
chunk_size = _max_rows_per_stmt()
|
||||
for start in range(0, len(batch), chunk_size):
|
||||
chunk = batch[start:start + chunk_size]
|
||||
stmt = sqlite_insert(LogEntry).values(chunk)
|
||||
db.session.execute(stmt)
|
||||
db.session.commit()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user