Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 057d72b264 | |||
| a7a637b0c6 | |||
| 0d413cfeaf | |||
| 43d00d1eda | |||
| 6ca58f3dea |
@@ -529,8 +529,23 @@ def backup_conf(reason: str = "manual") -> str | None:
|
||||
|
||||
|
||||
def _show_config_preview(pending_content: str, return_to: str):
|
||||
"""Store pending content in session, redirect to GET /config/preview for confirmation."""
|
||||
"""Store pending content in session, redirect to GET /config/preview for confirmation.
|
||||
|
||||
``return_to`` may be either an endpoint name (e.g. ``"config_cache"``)
|
||||
or a URL path (e.g. ``"/config/cache"``). Endpoint names get resolved
|
||||
via ``url_for`` so the preview page's "返回" link is always a valid URL.
|
||||
"""
|
||||
import base64
|
||||
# Normalize return_to: if it doesn't look like a URL, treat it as an
|
||||
# endpoint name and resolve to a URL. This was the root cause of the
|
||||
# 404 bug after saving a config - we stored "config_cache" and
|
||||
# ``redirect("config_cache")`` produced a 404.
|
||||
if return_to and not return_to.startswith("/") and not return_to.startswith("http"):
|
||||
try:
|
||||
return_to = url_for(return_to)
|
||||
except Exception:
|
||||
# unknown endpoint - fall back to dashboard
|
||||
return_to = url_for("dashboard")
|
||||
session["_pending_conf"] = base64.b64encode(pending_content.encode()).decode()
|
||||
session["_pending_return"] = return_to
|
||||
return redirect(url_for("config_preview"))
|
||||
@@ -648,7 +663,7 @@ def get_parsed_logs(force: bool = False) -> list[log_parser.LogEntry]:
|
||||
# Best-effort auto-sync (throttled by the file mtime + last sync stamp)
|
||||
if force or _should_auto_sync(instance_id):
|
||||
try:
|
||||
path = access_log_path()
|
||||
path = _resolve_access_log_path()
|
||||
if os.path.isfile(path):
|
||||
log_storage.sync_log_to_db(
|
||||
path=path, instance_id=instance_id,
|
||||
@@ -680,18 +695,46 @@ def get_parsed_logs(force: bool = False) -> list[log_parser.LogEntry]:
|
||||
return wrapped
|
||||
|
||||
|
||||
def _resolve_access_log_path() -> str:
|
||||
"""Look up the access log path WITHOUT going through session-bound code.
|
||||
|
||||
``get_config`` -> ``_resolve_active_instance`` -> ``session.get`` raises
|
||||
outside a request context, and ``_should_auto_sync`` is called from many
|
||||
places (background threads, app shell) that don't have a request context.
|
||||
Here we hit the DB directly using a DB-session that does not require
|
||||
session-binding.
|
||||
"""
|
||||
try:
|
||||
# 1) instance row, if any is active
|
||||
active = SquidInstance.query.filter_by(is_active=True).first()
|
||||
if active and active.access_log_path:
|
||||
return active.access_log_path
|
||||
# 2) AppConfig override
|
||||
row = AppConfig.query.filter_by(key="access_log_path").first()
|
||||
if row and row.value:
|
||||
return row.value
|
||||
except Exception:
|
||||
pass
|
||||
return DEFAULTS.get("access_log_path", "/var/log/squid/access.log")
|
||||
|
||||
|
||||
def _should_auto_sync(instance_id: int) -> bool:
|
||||
"""Decide whether to re-sync the access.log tail for ``instance_id``.
|
||||
|
||||
Cheap heuristic: check the file's mtime against the last sync stamp
|
||||
kept in :class:`AppConfig`. Always returns False when
|
||||
``log_sync_interval`` is 0 (manual sync only).
|
||||
|
||||
Must NOT depend on ``session`` (called outside request context).
|
||||
"""
|
||||
try:
|
||||
interval = int(get_config("log_sync_interval", "0"))
|
||||
# Use raw AppConfig instead of get_config() - get_config touches
|
||||
# session which raises outside request context.
|
||||
row = AppConfig.query.filter_by(key="log_sync_interval").first()
|
||||
interval = int(row.value) if (row and row.value) else 0
|
||||
if interval <= 0:
|
||||
return False
|
||||
path = access_log_path()
|
||||
path = _resolve_access_log_path()
|
||||
if not os.path.isfile(path):
|
||||
return False
|
||||
mtime = os.path.getmtime(path)
|
||||
@@ -709,7 +752,9 @@ def _should_auto_sync(instance_id: int) -> bool:
|
||||
return False
|
||||
_auto_sync_last_call = now
|
||||
return mtime > last_dt.timestamp()
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
# Don't use current_app here - this code can run outside app context
|
||||
# (e.g. in test scripts). Just swallow.
|
||||
return False
|
||||
|
||||
|
||||
@@ -818,8 +863,10 @@ def change_password():
|
||||
@app.route("/")
|
||||
@login_required
|
||||
def dashboard():
|
||||
stats = get_stats()
|
||||
# Trigger auto-sync first (get_parsed_logs checks _should_auto_sync),
|
||||
# then compute stats from the now-fresh SQLite data.
|
||||
entries = get_parsed_logs()
|
||||
stats = get_stats()
|
||||
cfg = get_all_config()
|
||||
# P1-2: surface any active alert rules at the top of the dashboard so
|
||||
# operators see them without having to navigate to /alerts.
|
||||
@@ -926,7 +973,7 @@ def logs_view():
|
||||
# best-effort sync before serving (skipped if the file is unchanged)
|
||||
try:
|
||||
if _should_auto_sync(instance_id):
|
||||
path = access_log_path()
|
||||
path = _resolve_access_log_path()
|
||||
if os.path.isfile(path):
|
||||
log_storage.sync_log_to_db(path=path, instance_id=instance_id)
|
||||
except Exception:
|
||||
@@ -939,8 +986,11 @@ def logs_view():
|
||||
"limit": per_page,
|
||||
"offset": (page - 1) * per_page,
|
||||
})
|
||||
# Wrap as LogEntry so the template's e.timestamp_local property
|
||||
# works (query_logs returns plain dicts, which would otherwise
|
||||
# silently fall through to the e.time branch in Jinja).
|
||||
entries = [log_parser.LogEntry(**r) for r in page_items]
|
||||
# query_logs returns newest-first, so no re-reverse needed
|
||||
entries = page_items
|
||||
else:
|
||||
entries = get_parsed_logs()
|
||||
f = dict(base_filters)
|
||||
@@ -970,7 +1020,7 @@ def logs_view():
|
||||
"start_time": f_start, "end_time": f_end,
|
||||
},
|
||||
cfg=cfg,
|
||||
last_sync=log_storage.get_last_sync_at(),
|
||||
last_sync=log_storage.get_last_sync_at_local(),
|
||||
last_sync_path=log_storage.get_last_sync_path(),
|
||||
use_persistent=use_persistent,
|
||||
format_bytes=log_parser.format_bytes,
|
||||
@@ -1060,10 +1110,14 @@ def logs_import():
|
||||
@app.route("/logs/live")
|
||||
@login_required
|
||||
def logs_live():
|
||||
"""Live tail view (auto-refresh)."""
|
||||
"""Live tail view (auto-refresh). Reads directly from file for real-time."""
|
||||
cfg = get_all_config()
|
||||
n = int(cfg.get("live_tail_lines", 100))
|
||||
entries = get_parsed_logs(force=False)[-n:]
|
||||
# Read directly from file - bypass SQLite cache for true real-time
|
||||
limit = max(n * 3, 500)
|
||||
text = read_access_log_tail(max_lines=limit)
|
||||
entries = log_parser.parse_lines(text)
|
||||
entries = entries[-n:]
|
||||
entries = list(reversed(entries))
|
||||
return render_template("logs_live.html", entries=entries,
|
||||
cfg=cfg, n=n,
|
||||
@@ -1264,7 +1318,7 @@ def config_acls():
|
||||
type=types[i] if i < len(types) else "src",
|
||||
values=vals, comment=cmnt))
|
||||
struct["acls"] = new_acls
|
||||
content_preview = msg if ok else ""
|
||||
ok, msg, content_preview = save_struct(struct)
|
||||
if not ok:
|
||||
flash(msg, "error")
|
||||
return redirect(url_for("config_acls"))
|
||||
@@ -1314,7 +1368,7 @@ def config_rules():
|
||||
idx = int(request.form.get("index", "-1"))
|
||||
if 0 <= idx < len(struct["rules"]):
|
||||
del struct["rules"][idx]
|
||||
content_preview = msg if ok else ""
|
||||
ok, msg, content_preview = save_struct(struct)
|
||||
if not ok:
|
||||
flash(msg, "error")
|
||||
return redirect(url_for("config_rules"))
|
||||
@@ -1364,7 +1418,7 @@ def config_cache():
|
||||
idx = int(request.form.get("index", "-1"))
|
||||
if 0 <= idx < len(struct["cache_dirs"]):
|
||||
del struct["cache_dirs"][idx]
|
||||
content_preview = msg if ok else ""
|
||||
ok, msg, content_preview = save_struct(struct)
|
||||
if not ok:
|
||||
flash(msg, "error")
|
||||
return redirect(url_for("config_cache"))
|
||||
@@ -1415,7 +1469,7 @@ def config_refresh():
|
||||
idx = int(request.form.get("index", "-1"))
|
||||
if 0 <= idx < len(struct["refresh_patterns"]):
|
||||
del struct["refresh_patterns"][idx]
|
||||
content_preview = msg if ok else ""
|
||||
ok, msg, content_preview = save_struct(struct)
|
||||
if not ok:
|
||||
flash(msg, "error")
|
||||
return redirect(url_for("config_refresh"))
|
||||
@@ -1468,7 +1522,7 @@ def config_auth():
|
||||
idx = int(request.form.get("index", "-1"))
|
||||
if 0 <= idx < len(struct["auth_params"]):
|
||||
del struct["auth_params"][idx]
|
||||
content_preview = msg if ok else ""
|
||||
ok, msg, content_preview = save_struct(struct)
|
||||
if not ok:
|
||||
flash(msg, "error")
|
||||
return redirect(url_for("config_auth"))
|
||||
@@ -1521,7 +1575,7 @@ def config_peers():
|
||||
idx = int(request.form.get("index", "-1"))
|
||||
if 0 <= idx < len(struct["cache_peers"]):
|
||||
del struct["cache_peers"][idx]
|
||||
content_preview = msg if ok else ""
|
||||
ok, msg, content_preview = save_struct(struct)
|
||||
if not ok:
|
||||
flash(msg, "error")
|
||||
return redirect(url_for("config_peers"))
|
||||
@@ -3109,7 +3163,7 @@ def _export_logs_payload():
|
||||
import log_storage
|
||||
try:
|
||||
if _should_auto_sync(instance_id):
|
||||
path = access_log_path()
|
||||
path = _resolve_access_log_path()
|
||||
if os.path.isfile(path):
|
||||
log_storage.sync_log_to_db(path=path, instance_id=instance_id)
|
||||
except Exception:
|
||||
|
||||
@@ -119,6 +119,17 @@ class LogEntry(dict):
|
||||
def result_description(self) -> str:
|
||||
return RESULT_CODE_DESCRIPTION.get(self.result_code, self.result_code)
|
||||
|
||||
@property
|
||||
def timestamp_local(self):
|
||||
"""Return a datetime converted to the server's local timezone (CST for cn)."""
|
||||
ts = self.get("timestamp")
|
||||
if ts is None:
|
||||
return None
|
||||
try:
|
||||
return ts.astimezone() # use system local tz
|
||||
except Exception:
|
||||
return ts
|
||||
|
||||
|
||||
def parse_line(line: str) -> LogEntry | None:
|
||||
"""Parse a single access.log line. Returns None on unparseable lines."""
|
||||
|
||||
+49
-9
@@ -198,20 +198,43 @@ def sync_log_to_db(
|
||||
return 0, 0, f"{type(e).__name__}: {e}"
|
||||
|
||||
|
||||
def _flush_batch(batch: list[dict]) -> None:
|
||||
"""Insert a batch of rows in a single SQL statement.
|
||||
def _column_count() -> int:
|
||||
"""Return the number of columns in the :class:`~app.LogEntry` table.
|
||||
|
||||
We rely on the caller's in-memory ``(instance_id, log_time, client,
|
||||
raw_line)`` set (built in :func:`sync_log_to_db`) for dedup since we
|
||||
don't have a SQL-level UNIQUE constraint. This means inserts are
|
||||
guaranteed to be new rows at insert time, so we use a plain
|
||||
``insert().values()`` rather than ``on_conflict_do_nothing``.
|
||||
Used to cap the per-statement row count so the single INSERT does not
|
||||
exceed SQLite's 999-boundary-variable limit per SQL statement.
|
||||
"""
|
||||
return len(LogEntry.__table__.columns)
|
||||
|
||||
|
||||
# SQLite hard limit: at most 999 ? placeholders per single SQL statement.
|
||||
_SQLITE_MAX_VARIABLES = 999
|
||||
|
||||
|
||||
def _max_rows_per_stmt() -> int:
|
||||
"""Max rows that can safely fit in one parameterised INSERT."""
|
||||
cols = _column_count()
|
||||
return max(1, (_SQLITE_MAX_VARIABLES - 1) // cols)
|
||||
|
||||
|
||||
def _flush_batch(batch: list[dict]) -> None:
|
||||
"""Insert ``batch`` rows, splitting into chunks small enough for SQLite.
|
||||
|
||||
SQLite rejects a single parameterised INSERT when its bound-variable
|
||||
count exceeds 999. Since :class:`~app.LogEntry` carries many columns, a
|
||||
'large' batch (e.g. 500 rows) would blow past that limit. Split the
|
||||
batch into sub-chunks of ``_max_rows_per_stmt()`` rows and execute each
|
||||
sub-statement individually, then commit once the whole batch is in.
|
||||
"""
|
||||
if not batch:
|
||||
return
|
||||
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
||||
stmt = sqlite_insert(LogEntry).values(batch)
|
||||
db.session.execute(stmt)
|
||||
|
||||
chunk_size = _max_rows_per_stmt()
|
||||
for start in range(0, len(batch), chunk_size):
|
||||
chunk = batch[start:start + chunk_size]
|
||||
stmt = sqlite_insert(LogEntry).values(chunk)
|
||||
db.session.execute(stmt)
|
||||
db.session.commit()
|
||||
|
||||
|
||||
@@ -774,6 +797,23 @@ def get_last_sync_path() -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def get_last_sync_at_local() -> str | None:
|
||||
"""Return the last-sync timestamp formatted in local time (not UTC).
|
||||
|
||||
:class:`~app.AppConfig` stores the sync time as an ISO-8601 UTC string
|
||||
(e.g. ``2026-07-16T07:43:36Z``). This converts it to the system's local
|
||||
timezone so the UI shows a human-friendly time to the operator.
|
||||
"""
|
||||
ts = get_last_sync_at()
|
||||
if not ts:
|
||||
return None
|
||||
try:
|
||||
dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
return dt.astimezone().strftime("%Y-%m-%d %H:%M:%S")
|
||||
except Exception:
|
||||
return ts
|
||||
|
||||
|
||||
def import_full_history(
|
||||
path: str,
|
||||
instance_id: int = 0,
|
||||
|
||||
+2
-2
@@ -611,8 +611,8 @@ def default_struct() -> dict:
|
||||
],
|
||||
"rules": [
|
||||
Rule(action="deny", acls=["manager"], directive="http_access"),
|
||||
Rule(action="deny", acls=["!Safe_ports"], directive="http_access"),
|
||||
Rule(action="deny", acls=["!SSL_ports"], directive="http_access"),
|
||||
Rule(action="deny", acls=["!safeports"], directive="http_access"),
|
||||
Rule(action="deny", acls=["!sslports"], directive="http_access"),
|
||||
Rule(action="allow", acls=["localhost"], directive="http_access"),
|
||||
Rule(action="deny", acls=["all"], directive="http_access"),
|
||||
],
|
||||
|
||||
+1
-1
@@ -129,7 +129,7 @@
|
||||
<tbody>
|
||||
{% for e in entries %}
|
||||
<tr>
|
||||
<td class="time-cell">{{ e.timestamp.strftime('%Y-%m-%d %H:%M:%S') if e.timestamp else e.time }}</td>
|
||||
<td class="time-cell">{{ e.timestamp_local.strftime('%Y-%m-%d %H:%M:%S') if e.timestamp_local else e.time }}</td>
|
||||
<td><a href="{{ url_for('client_detail', ip=e.client) }}">{{ e.client }}</a></td>
|
||||
<td>
|
||||
<span class="badge badge-{% if e.category == 'Hit' %}green{% elif e.category == 'Miss' %}blue{% elif e.category == 'Tunnel' %}cyan{% elif e.category == 'Denied' %}red{% elif e.category == 'Aborted' %}orange{% elif e.category == 'Error' %}red{% else %}gray{% endif %}" title="{{ e.result_description }}">{{ e.result_code }}</span>
|
||||
|
||||
@@ -109,7 +109,7 @@
|
||||
<tbody>
|
||||
{% for e in entries[:200] %}
|
||||
<tr>
|
||||
<td class="time-cell">{{ e.timestamp.strftime('%H:%M:%S') if e.timestamp else e.time }}</td>
|
||||
<td class="time-cell">{{ e.timestamp_local.strftime('%H:%M:%S') if e.timestamp_local else e.time }}</td>
|
||||
<td>{{ e.client }}</td>
|
||||
<td><span class="badge badge-{% if e.category == 'Hit' %}green{% elif e.category == 'Miss' %}blue{% elif e.category == 'Tunnel' %}cyan{% elif e.category == 'Denied' %}red{% elif e.category == 'Aborted' %}orange{% else %}gray{% endif %}">{{ e.result_code }}</span></td>
|
||||
<td>{{ e.http_code or '-' }}</td>
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
<tbody>
|
||||
{% for e in entries %}
|
||||
<tr>
|
||||
<td class="time-cell">{{ e.timestamp.strftime('%H:%M:%S') if e.timestamp else e.time }}</td>
|
||||
<td class="time-cell">{{ e.timestamp_local.strftime('%H:%M:%S') if e.timestamp_local else e.time }}</td>
|
||||
<td>{{ e.client }}</td>
|
||||
<td>
|
||||
<span class="badge badge-{% if e.category == 'Hit' %}green{% elif e.category == 'Miss' %}blue{% elif e.category == 'Tunnel' %}cyan{% elif e.category == 'Denied' %}red{% elif e.category == 'Aborted' %}orange{% elif e.category == 'Error' %}red{% else %}gray{% endif %}">{{ e.result_code }}</span>
|
||||
|
||||
Reference in New Issue
Block a user