diff --git a/log_storage.py b/log_storage.py index 36738db..c9e5e05 100644 --- a/log_storage.py +++ b/log_storage.py @@ -198,20 +198,43 @@ def sync_log_to_db( return 0, 0, f"{type(e).__name__}: {e}" -def _flush_batch(batch: list[dict]) -> None: - """Insert a batch of rows in a single SQL statement. +def _column_count() -> int: + """Return the number of columns in the :class:`~app.LogEntry` table. - We rely on the caller's in-memory ``(instance_id, log_time, client, - raw_line)`` set (built in :func:`sync_log_to_db`) for dedup since we - don't have a SQL-level UNIQUE constraint. This means inserts are - guaranteed to be new rows at insert time, so we use a plain - ``insert().values()`` rather than ``on_conflict_do_nothing``. + Used to cap the per-statement row count so the single INSERT does not + exceed SQLite's 999-boundary-variable limit per SQL statement. + """ + return len(LogEntry.__table__.columns) + + +# SQLite hard limit: at most 999 ? placeholders per single SQL statement. +_SQLITE_MAX_VARIABLES = 999 + + +def _max_rows_per_stmt() -> int: + """Max rows that can safely fit in one parameterised INSERT.""" + cols = _column_count() + return max(1, (_SQLITE_MAX_VARIABLES - 1) // cols) + + +def _flush_batch(batch: list[dict]) -> None: + """Insert ``batch`` rows, splitting into chunks small enough for SQLite. + + SQLite rejects a single parameterised INSERT when its bound-variable + count exceeds 999. Since :class:`~app.LogEntry` carries many columns, a + 'large' batch (e.g. 500 rows) would blow past that limit. Split the + batch into sub-chunks of ``_max_rows_per_stmt()`` rows and execute each + sub-statement individually, then commit once the whole batch is in. """ if not batch: return from sqlalchemy.dialects.sqlite import insert as sqlite_insert - stmt = sqlite_insert(LogEntry).values(batch) - db.session.execute(stmt) + + chunk_size = _max_rows_per_stmt() + for start in range(0, len(batch), chunk_size): + chunk = batch[start:start + chunk_size] + stmt = sqlite_insert(LogEntry).values(chunk) + db.session.execute(stmt) db.session.commit()