From 6ca58f3dea4e7387a7090798a915837785ae9831 Mon Sep 17 00:00:00 2001
From: Hermes Agent <hermes@local>
Date: Wed, 15 Jul 2026 14:56:15 +0800
Subject: [PATCH] =?UTF-8?q?fix:=20=E6=89=B9=E9=87=8F=E6=8F=92=E5=85=A5?=
 =?UTF-8?q?=E5=88=86=E7=89=87=E8=A7=84=E9=81=BF=20SQLite=20999=20=E5=8F=98?=
 =?UTF-8?q?=E9=87=8F=E4=B8=8A=E9=99=90=20(too=20many=20SQL=20variables)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

log_entries 表 14 列 + id，500 行单 INSERT 产生约 7500 个绑定变量，
超出 SQLite 单语句 999 变量硬限。改为按列数自动计算每语句最大行数
(约 66 行) 分片提交，整批一次性 commit 保持事务性。
---
 log_storage.py | 41 ++++++++++++++++++++++++++++++++---------
 1 file changed, 32 insertions(+), 9 deletions(-)

diff --git a/log_storage.py b/log_storage.py
index 36738db..c9e5e05 100644
--- a/log_storage.py
+++ b/log_storage.py
@@ -198,20 +198,43 @@ def sync_log_to_db(
         return 0, 0, f"{type(e).__name__}: {e}"
 
 
-def _flush_batch(batch: list[dict]) -> None:
-    """Insert a batch of rows in a single SQL statement.
+def _column_count() -> int:
+    """Return the number of columns in the :class:`~app.LogEntry` table.
 
-    We rely on the caller's in-memory ``(instance_id, log_time, client,
-    raw_line)`` set (built in :func:`sync_log_to_db`) for dedup since we
-    don't have a SQL-level UNIQUE constraint. This means inserts are
-    guaranteed to be new rows at insert time, so we use a plain
-    ``insert().values()`` rather than ``on_conflict_do_nothing``.
+    Used to cap the per-statement row count so the single INSERT does not
+    exceed SQLite's 999-boundary-variable limit per SQL statement.
+    """
+    return len(LogEntry.__table__.columns)
+
+
+# SQLite hard limit: at most 999 ? placeholders per single SQL statement.
+_SQLITE_MAX_VARIABLES = 999
+
+
+def _max_rows_per_stmt() -> int:
+    """Max rows that can safely fit in one parameterised INSERT."""
+    cols = _column_count()
+    return max(1, (_SQLITE_MAX_VARIABLES - 1) // cols)
+
+
+def _flush_batch(batch: list[dict]) -> None:
+    """Insert ``batch`` rows, splitting into chunks small enough for SQLite.
+
+    SQLite rejects a single parameterised INSERT when its bound-variable
+    count exceeds 999. Since :class:`~app.LogEntry` carries many columns, a
+    'large' batch (e.g. 500 rows) would blow past that limit. Split the
+    batch into sub-chunks of ``_max_rows_per_stmt()`` rows and execute each
+    sub-statement individually, then commit once the whole batch is in.
     """
     if not batch:
         return
     from sqlalchemy.dialects.sqlite import insert as sqlite_insert
-    stmt = sqlite_insert(LogEntry).values(batch)
-    db.session.execute(stmt)
+
+    chunk_size = _max_rows_per_stmt()
+    for start in range(0, len(batch), chunk_size):
+        chunk = batch[start:start + chunk_size]
+        stmt = sqlite_insert(LogEntry).values(chunk)
+        db.session.execute(stmt)
     db.session.commit()