feat: 完整 Squid Web Manager 平台 - P0-P3 全功能
完整 Web 管理平台,涵盖: P0 生产加固: - 会话超时(空闲+绝对) + CSRF 防护 + 登录失败限流 - HTTPS/TLS 证书管理(自签生成/上传/删除/过期提醒) - SSL Bump (HTTPS 透明代理) 可视化配置 - 配置文件 diff 预览 + 二次确认才写入 - 服务控制二次确认 + 操作期间按钮锁定 P1 运维能力: - 日志轮转管理(logrotate) + 日志状态监控 - 告警规则(5xx/命中率/磁盘/客户端流量) - 多 Squid 实例管理 + 一键切换 - squidclient mgr 实时性能指标 - 流量异常检测(突增/大文件/高频小包/新客户端) P2 日志分析增强: - 日志持久化到 SQLite + 历史时间范围查询 - 导出 CSV/JSON(日志/KPI/异常/审计) - GeoIP 地图(Leaflet + ip-api.com 离线可选 GeoLite2) - 每客户端 24h 趋势图 - 自定义 Squid logformat 解析器 P3 体验锦上添花: - 暗/亮主题切换(cookie 持久化) - WebSSH 终端(xterm.js + paramiko) - 完整审计日志 + 用户管理 技术栈: Flask 3.0 + SQLAlchemy 2.0 + SQLite + Chart.js + Leaflet 总代码量: ~16500 行 (15 Python 模块 + 34 模板 + CSS) 路由数: 73
This commit is contained in:
Hermes Agent
+800
@@ -0,0 +1,800 @@
|
||||
"""Persistent log storage for Squid access.log.
|
||||
|
||||
This module owns the bridge between raw access.log files on disk and the
|
||||
SQLite-backed :class:`~app.LogEntry` table. It exposes:
|
||||
|
||||
- :func:`sync_log_to_db` - incrementally import new lines from a log file
|
||||
- :func:`query_logs` - SQL-backed filter/search with time range support
|
||||
- :func:`get_log_stats` - aggregated statistics (hit/miss, top clients...)
|
||||
- :func:`get_distinct_hosts` / :func:`get_distinct_clients` /
|
||||
:func:`get_distinct_methods` - faceted browsing
|
||||
|
||||
Implementation notes:
|
||||
- We rely on the existing :mod:`log_parser` for line format handling so we do
|
||||
not duplicate the access.log grammar.
|
||||
- Bulk inserts are batched (default 500 rows) using
|
||||
:func:`sqlalchemy.orm.Session.execute` with a parameterised INSERT so we
|
||||
don't pay the round-trip cost of per-row commits.
|
||||
- De-duplication uses the tuple ``(instance_id, log_time, client, raw_line)``;
|
||||
on conflict (we surface those as ``skipped_count`` rather than re-inserting).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from typing import Iterable
|
||||
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
|
||||
import log_parser
|
||||
from app import db, LogEntry # imported lazily-friendly via app
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sync
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _iter_log_lines(path: str, max_lines: int | None = None):
|
||||
"""Yield raw lines from ``path`` as bytes-decoded strings.
|
||||
|
||||
When ``max_lines`` is set we walk backwards through the file with a
|
||||
bounded per-iteration buffer (Deque-style window) and yield lines in
|
||||
chronological order. For an unbounded scan (full-history import) we
|
||||
just stream forwards. Designed to be tolerant of mid-stream truncation
|
||||
or binary garbage.
|
||||
"""
|
||||
if not os.path.isfile(path):
|
||||
return
|
||||
|
||||
if not max_lines or max_lines <= 0:
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8", errors="replace") as fh:
|
||||
for raw in fh:
|
||||
line = raw.rstrip("\n").rstrip("\r")
|
||||
if line.strip():
|
||||
yield line
|
||||
except OSError:
|
||||
return
|
||||
return
|
||||
|
||||
# Bounded tail: read last ``max_lines`` lines efficiently
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
try:
|
||||
fh.seek(0, os.SEEK_END)
|
||||
size = fh.tell()
|
||||
chunks: list[bytes] = []
|
||||
lines_found = 0
|
||||
block = 65536
|
||||
pos = size
|
||||
while pos > 0 and lines_found <= max_lines:
|
||||
read_size = min(block, pos)
|
||||
pos -= read_size
|
||||
fh.seek(pos)
|
||||
chunks.append(fh.read(read_size))
|
||||
lines_found += chunks[-1].count(b"\n")
|
||||
data = b"".join(reversed(chunks))
|
||||
except OSError:
|
||||
fh.seek(0)
|
||||
data = fh.read()
|
||||
text = data.decode("utf-8", errors="replace")
|
||||
lines = text.splitlines()
|
||||
if len(lines) > max_lines:
|
||||
lines = lines[-max_lines:]
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if stripped:
|
||||
yield stripped
|
||||
except OSError:
|
||||
return
|
||||
|
||||
|
||||
def sync_log_to_db(
|
||||
path: str,
|
||||
instance_id: int = 0,
|
||||
batch_size: int = 500,
|
||||
max_lines: int = 100000,
|
||||
) -> tuple[int, int, str | None]:
|
||||
"""Incrementally import new lines from ``path`` into :class:`LogEntry`.
|
||||
|
||||
Steps:
|
||||
|
||||
1. Compute the high-water mark ``max(existing.log_time)`` for the
|
||||
instance so we can skip already-imported lines quickly.
|
||||
2. Stream the last ``max_lines`` lines of the file
|
||||
(or the entire file when ``max_lines`` is None/0) and parse them
|
||||
through :func:`log_parser.parse_line`.
|
||||
3. Insert new rows in batches of ``batch_size`` using
|
||||
:func:`sqlalchemy.dialects.sqlite.insert` with
|
||||
``OR (primary key uniqueness handled by auot-increment)`` semantics.
|
||||
Duplicate suppression uses an in-memory ``seen`` set keyed by
|
||||
``(client, log_time, raw_line)`` since the file itself can repeat
|
||||
a row at rotation boundaries.
|
||||
4. Commit once per batch and return a tuple ``(inserted, skipped, err)``.
|
||||
|
||||
All exceptions are caught and reported as the third tuple element so the
|
||||
caller can decide whether to surface them (e.g. on the dashboard).
|
||||
"""
|
||||
if batch_size <= 0:
|
||||
batch_size = 500
|
||||
try:
|
||||
# high-water mark: only lines with log_time > wm are new
|
||||
wm = (
|
||||
db.session.query(func.max(LogEntry.log_time))
|
||||
.filter(LogEntry.instance_id == instance_id)
|
||||
.scalar()
|
||||
)
|
||||
if wm is None:
|
||||
wm = 0.0
|
||||
|
||||
seen_keys: set[tuple[str, float, str]] = set()
|
||||
inserted = 0
|
||||
skipped = 0
|
||||
batch: list[dict] = []
|
||||
|
||||
for raw_line in _iter_log_lines(path, max_lines=max_lines):
|
||||
e = log_parser.parse_line(raw_line)
|
||||
if not e:
|
||||
skipped += 1
|
||||
continue
|
||||
try:
|
||||
ts = float(e.get("time") or 0)
|
||||
except (TypeError, ValueError):
|
||||
skipped += 1
|
||||
continue
|
||||
if ts <= wm:
|
||||
skipped += 1
|
||||
continue
|
||||
key = (e.get("client") or "", ts, raw_line)
|
||||
if key in seen_keys:
|
||||
skipped += 1
|
||||
continue
|
||||
seen_keys.add(key)
|
||||
|
||||
batch.append({
|
||||
"instance_id": instance_id,
|
||||
"log_time": ts,
|
||||
"elapsed_ms": int(e.get("elapsed_ms") or 0),
|
||||
"client": (e.get("client") or "")[:64],
|
||||
"result_code": (e.get("result_code") or "")[:32],
|
||||
"http_code": (e.get("http_code") or "")[:8],
|
||||
"size_bytes": int(e.get("size") or 0),
|
||||
"method": (e.get("method") or "")[:16],
|
||||
"url": (e.get("url") or "")[:65535],
|
||||
"host": (e.get("host") or "")[:256],
|
||||
"hier_code": (e.get("hier_code") or "")[:32],
|
||||
"content_type": (e.get("content_type") or "")[:128],
|
||||
"raw_line": raw_line[:65535],
|
||||
"created_at": datetime.now(timezone.utc),
|
||||
})
|
||||
if len(batch) >= batch_size:
|
||||
_flush_batch(batch)
|
||||
inserted += len(batch)
|
||||
batch = []
|
||||
|
||||
if batch:
|
||||
_flush_batch(batch)
|
||||
inserted += len(batch)
|
||||
|
||||
# record last sync time so the UI can display "上次同步"
|
||||
_record_last_sync(instance_id, path)
|
||||
return inserted, skipped, None
|
||||
except SQLAlchemyError as e:
|
||||
try:
|
||||
db.session.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return 0, 0, f"DB error: {type(e).__name__}: {e}"
|
||||
except Exception as e:
|
||||
try:
|
||||
db.session.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return 0, 0, f"{type(e).__name__}: {e}"
|
||||
|
||||
|
||||
def _flush_batch(batch: list[dict]) -> None:
|
||||
"""Insert a batch of rows in a single SQL statement.
|
||||
|
||||
We rely on the caller's in-memory ``(instance_id, log_time, client,
|
||||
raw_line)`` set (built in :func:`sync_log_to_db`) for dedup since we
|
||||
don't have a SQL-level UNIQUE constraint. This means inserts are
|
||||
guaranteed to be new rows at insert time, so we use a plain
|
||||
``insert().values()`` rather than ``on_conflict_do_nothing``.
|
||||
"""
|
||||
if not batch:
|
||||
return
|
||||
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
||||
stmt = sqlite_insert(LogEntry).values(batch)
|
||||
db.session.execute(stmt)
|
||||
db.session.commit()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Query
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _parse_time(value) -> float | None:
|
||||
"""Parse an ISO-ish datetime / unix timestamp into a unix float (UTC)."""
|
||||
if value is None or value == "":
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
if isinstance(value, datetime):
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
return value.timestamp()
|
||||
if isinstance(value, str):
|
||||
s = value.strip()
|
||||
if not s:
|
||||
return None
|
||||
# datetime-local HTML input sends strings like "2024-01-31T10:00"
|
||||
# or "2024-01-31 10:00:00". Python's fromisoformat handles both.
|
||||
try:
|
||||
# Support trailing Z
|
||||
if s.endswith("Z"):
|
||||
s = s[:-1] + "+00:00"
|
||||
dt = datetime.fromisoformat(s)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.timestamp()
|
||||
except ValueError:
|
||||
# fall through to float attempt
|
||||
pass
|
||||
try:
|
||||
return float(s)
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _entry_to_dict(row: LogEntry) -> dict:
|
||||
"""Convert a :class:`LogEntry` row to the dict shape used downstream.
|
||||
|
||||
Mirrors :class:`log_parser.LogEntry` (which is a dict subclass) so
|
||||
existing consumers - templates, alert rules, anomaly detection - keep
|
||||
working without modification.
|
||||
"""
|
||||
ts = float(row.log_time or 0)
|
||||
host = row.host or ""
|
||||
url = row.url or ""
|
||||
raw = row.raw_line or ""
|
||||
# split result_code (already stored without slash) - reconstruct full
|
||||
# Squid "result" string (e.g. "TCP_HIT/200") for backward compat
|
||||
result_full = row.result_code or ""
|
||||
if row.http_code:
|
||||
result_full = f"{result_full}/{row.http_code}"
|
||||
return {
|
||||
"id": row.id,
|
||||
"time": row.log_time, # float unix ts
|
||||
"timestamp": datetime.fromtimestamp(ts, tz=timezone.utc) if ts > 0 else None,
|
||||
"elapsed_ms": int(row.elapsed_ms or 0),
|
||||
"client": row.client or "",
|
||||
"result": result_full,
|
||||
"result_code": row.result_code or "",
|
||||
"http_code": row.http_code or "",
|
||||
"category": log_parser.RESULT_CATEGORIES.get(row.result_code or "", "Other"),
|
||||
"result_description": log_parser.RESULT_CODE_DESCRIPTION.get(row.result_code or "", row.result_code or ""),
|
||||
"size": int(row.size_bytes or 0),
|
||||
"method": row.method or "",
|
||||
"url": url,
|
||||
"host": host,
|
||||
"ident": "", # not persisted separately
|
||||
"hierarchy": row.hier_code or "",
|
||||
"hier_code": row.hier_code or "",
|
||||
"peer": "", # not persisted
|
||||
"content_type": row.content_type or "",
|
||||
"raw_line": raw,
|
||||
"formatted_size": log_parser.format_bytes(int(row.size_bytes or 0)),
|
||||
"formatted_duration": log_parser.format_duration(int(row.elapsed_ms or 0)),
|
||||
}
|
||||
|
||||
|
||||
def query_logs(filters: dict) -> list[dict]:
|
||||
"""Run a filtered SQL query against the persisted log rows.
|
||||
|
||||
Supported keys (all optional):
|
||||
|
||||
- ``instance_id`` (int)
|
||||
- ``client``, ``method``, ``result_code``, ``host`` (exact or substring)
|
||||
- ``host_filter_mode`` "exact" | "contains" (default "contains")
|
||||
- ``url`` substring LIKE
|
||||
- ``min_size``, ``max_size`` (int bytes)
|
||||
- ``min_elapsed``, ``max_elapsed`` (int milliseconds)
|
||||
- ``start_time``, ``end_time`` (datetime or unix timestamp, inclusive lo,
|
||||
inclusive hi for end so users don't accidentally exclude the minute
|
||||
they typed)
|
||||
- ``limit`` (default 1000), ``offset`` (default 0)
|
||||
|
||||
Returns a list of dicts with the same shape as
|
||||
:func:`log_parser.parse_lines`, plus ``formatted_size`` /
|
||||
``formatted_duration`` keys.
|
||||
"""
|
||||
try:
|
||||
q = db.session.query(LogEntry)
|
||||
# instance scoping
|
||||
iid = filters.get("instance_id")
|
||||
if iid is not None:
|
||||
q = q.filter(LogEntry.instance_id == iid)
|
||||
|
||||
c = (filters.get("client") or "").strip()
|
||||
if c:
|
||||
q = q.filter(LogEntry.client.like(f"%{c}%"))
|
||||
m = (filters.get("method") or "").strip()
|
||||
if m:
|
||||
q = q.filter(LogEntry.method == m)
|
||||
rc = (filters.get("result_code") or "").strip()
|
||||
if rc:
|
||||
q = q.filter(LogEntry.result_code.like(f"%{rc}%"))
|
||||
h = (filters.get("host") or "").strip()
|
||||
if h:
|
||||
mode = (filters.get("host_filter_mode") or "contains").lower()
|
||||
if mode == "exact":
|
||||
q = q.filter(LogEntry.host == h)
|
||||
else:
|
||||
q = q.filter(LogEntry.host.like(f"%{h}%"))
|
||||
u = (filters.get("url") or "").strip()
|
||||
if u:
|
||||
q = q.filter(LogEntry.url.like(f"%{u}%"))
|
||||
|
||||
mn = filters.get("min_size")
|
||||
if mn is not None:
|
||||
q = q.filter(LogEntry.size_bytes >= int(mn))
|
||||
mx = filters.get("max_size")
|
||||
if mx is not None:
|
||||
q = q.filter(LogEntry.size_bytes <= int(mx))
|
||||
mne = filters.get("min_elapsed")
|
||||
if mne is not None:
|
||||
q = q.filter(LogEntry.elapsed_ms >= int(mne))
|
||||
mxe = filters.get("max_elapsed")
|
||||
if mxe is not None:
|
||||
q = q.filter(LogEntry.elapsed_ms <= int(mxe))
|
||||
|
||||
st = _parse_time(filters.get("start_time"))
|
||||
if st is not None:
|
||||
q = q.filter(LogEntry.log_time >= st)
|
||||
et = _parse_time(filters.get("end_time"))
|
||||
if et is not None:
|
||||
q = q.filter(LogEntry.log_time <= et)
|
||||
|
||||
# default ordering: newest first
|
||||
q = q.order_by(LogEntry.log_time.desc(), LogEntry.id.desc())
|
||||
|
||||
limit = int(filters.get("limit") or 1000)
|
||||
offset = int(filters.get("offset") or 0)
|
||||
if limit > 0:
|
||||
q = q.limit(limit)
|
||||
if offset > 0:
|
||||
q = q.offset(offset)
|
||||
|
||||
rows = q.all()
|
||||
return [_entry_to_dict(r) for r in rows]
|
||||
except SQLAlchemyError as e:
|
||||
try:
|
||||
db.session.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def count_logs(filters: dict | None = None) -> int:
|
||||
"""Return the number of rows matching ``filters`` (same shape as ``query_logs``).
|
||||
|
||||
Used by ``logs_view`` to render total / pagination accurately without
|
||||
transferring every row over the wire.
|
||||
"""
|
||||
f = dict(filters or {})
|
||||
f["limit"] = 0
|
||||
f["offset"] = 0
|
||||
try:
|
||||
# rebuild only the WHERE portion - run the same filter logic but
|
||||
# count instead of fetch
|
||||
q = _build_base_query(f)
|
||||
return q.with_entities(func.count(LogEntry.id)).scalar() or 0
|
||||
except SQLAlchemyError:
|
||||
try:
|
||||
db.session.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
def _build_base_query(filters: dict):
|
||||
"""Internal: same WHERE-clause as :func:`query_logs` but returns the
|
||||
SQLAlchemy query for further composition (e.g. count)."""
|
||||
q = db.session.query(LogEntry)
|
||||
iid = filters.get("instance_id")
|
||||
if iid is not None:
|
||||
q = q.filter(LogEntry.instance_id == iid)
|
||||
c = (filters.get("client") or "").strip()
|
||||
if c:
|
||||
q = q.filter(LogEntry.client.like(f"%{c}%"))
|
||||
m = (filters.get("method") or "").strip()
|
||||
if m:
|
||||
q = q.filter(LogEntry.method == m)
|
||||
rc = (filters.get("result_code") or "").strip()
|
||||
if rc:
|
||||
q = q.filter(LogEntry.result_code.like(f"%{rc}%"))
|
||||
h = (filters.get("host") or "").strip()
|
||||
if h:
|
||||
q = q.filter(LogEntry.host.like(f"%{h}%"))
|
||||
u = (filters.get("url") or "").strip()
|
||||
if u:
|
||||
q = q.filter(LogEntry.url.like(f"%{u}%"))
|
||||
mn = filters.get("min_size")
|
||||
if mn is not None:
|
||||
q = q.filter(LogEntry.size_bytes >= int(mn))
|
||||
mx = filters.get("max_size")
|
||||
if mx is not None:
|
||||
q = q.filter(LogEntry.size_bytes <= int(mx))
|
||||
mne = filters.get("min_elapsed")
|
||||
if mne is not None:
|
||||
q = q.filter(LogEntry.elapsed_ms >= int(mne))
|
||||
mxe = filters.get("max_elapsed")
|
||||
if mxe is not None:
|
||||
q = q.filter(LogEntry.elapsed_ms <= int(mxe))
|
||||
st = _parse_time(filters.get("start_time"))
|
||||
if st is not None:
|
||||
q = q.filter(LogEntry.log_time >= st)
|
||||
et = _parse_time(filters.get("end_time"))
|
||||
if et is not None:
|
||||
q = q.filter(LogEntry.log_time <= et)
|
||||
return q
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Aggregate stats
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_log_stats(
|
||||
instance_id: int = 0,
|
||||
start_time: float | None = None,
|
||||
end_time: float | None = None,
|
||||
) -> dict:
|
||||
"""Compute aggregate statistics in SQL.
|
||||
|
||||
Compatible with :func:`log_parser.aggregate_stats`. When there are no
|
||||
rows in scope we return the same ``_empty_stats()`` structure so the
|
||||
caller doesn't need a special case.
|
||||
"""
|
||||
try:
|
||||
q = _build_base_query({
|
||||
"instance_id": instance_id,
|
||||
"start_time": start_time,
|
||||
"end_time": end_time,
|
||||
})
|
||||
total = q.with_entities(func.count(LogEntry.id)).scalar() or 0
|
||||
if total == 0:
|
||||
stats = _copy_empty_stats()
|
||||
else:
|
||||
agg = q.with_entities(
|
||||
func.coalesce(func.sum(LogEntry.size_bytes), 0),
|
||||
func.coalesce(func.sum(LogEntry.elapsed_ms), 0),
|
||||
func.min(LogEntry.log_time),
|
||||
func.max(LogEntry.log_time),
|
||||
).one()
|
||||
total_bytes, total_elapsed, ts_min, ts_max = agg
|
||||
|
||||
total = int(total)
|
||||
total_bytes = int(total_bytes)
|
||||
total_elapsed = int(total_elapsed)
|
||||
avg_elapsed = round(total_elapsed / total, 2) if total else 0.0
|
||||
avg_size = round(total_bytes / total, 2) if total else 0.0
|
||||
|
||||
# result_code distribution
|
||||
by_result = dict(
|
||||
q.with_entities(
|
||||
LogEntry.result_code,
|
||||
func.count(LogEntry.id),
|
||||
).group_by(LogEntry.result_code).all()
|
||||
)
|
||||
by_result = {k or "": v for k, v in by_result.items()}
|
||||
# http_code distribution (only rows with a value)
|
||||
by_http = dict(
|
||||
q.filter(LogEntry.http_code != "").with_entities(
|
||||
LogEntry.http_code,
|
||||
func.count(LogEntry.id),
|
||||
).group_by(LogEntry.http_code).all()
|
||||
)
|
||||
# method distribution
|
||||
by_method = dict(
|
||||
q.with_entities(
|
||||
LogEntry.method,
|
||||
func.count(LogEntry.id),
|
||||
).group_by(LogEntry.method).all()
|
||||
)
|
||||
# hierarchy distribution
|
||||
by_hier = dict(
|
||||
q.with_entities(
|
||||
LogEntry.hier_code,
|
||||
func.count(LogEntry.id),
|
||||
).group_by(LogEntry.hier_code).all()
|
||||
)
|
||||
|
||||
# top clients: most active + bytes
|
||||
top_clients = q.with_entities(
|
||||
LogEntry.client,
|
||||
func.count(LogEntry.id).label("c"),
|
||||
func.coalesce(func.sum(LogEntry.size_bytes), 0).label("b"),
|
||||
).group_by(LogEntry.client).order_by(func.count(LogEntry.id).desc()).limit(50).all()
|
||||
top_clients = [{"client": c or "", "count": int(c_), "bytes": int(b_)} for c, c_, b_ in top_clients]
|
||||
# top hosts
|
||||
top_hosts_rows = q.filter(LogEntry.host != "").with_entities(
|
||||
LogEntry.host,
|
||||
func.count(LogEntry.id).label("c"),
|
||||
func.coalesce(func.sum(LogEntry.size_bytes), 0).label("b"),
|
||||
).group_by(LogEntry.host).order_by(func.count(LogEntry.id).desc()).limit(50).all()
|
||||
top_hosts = [{"host": h, "count": int(c_), "bytes": int(b_)} for h, c_, b_ in top_hosts_rows]
|
||||
# top urls
|
||||
top_urls_rows = q.with_entities(
|
||||
LogEntry.url,
|
||||
func.count(LogEntry.id).label("c"),
|
||||
).group_by(LogEntry.url).order_by(func.count(LogEntry.id).desc()).limit(50).all()
|
||||
top_urls = [{"url": u, "count": int(c_)} for u, c_ in top_urls_rows]
|
||||
|
||||
# content-type: simpler approach - just count distinct top values
|
||||
by_ctype_rows = q.with_entities(
|
||||
LogEntry.content_type,
|
||||
func.count(LogEntry.id).label("c"),
|
||||
).group_by(LogEntry.content_type).order_by(func.count(LogEntry.id).desc()).limit(20).all()
|
||||
by_ctype = []
|
||||
for ct, c_ in by_ctype_rows:
|
||||
ct_main = (ct or "-").split(";")[0].strip() or "-"
|
||||
by_ctype.append((ct_main, int(c_)))
|
||||
# collapse same main types
|
||||
merged: dict[str, int] = defaultdict(int)
|
||||
for ct_main, c_ in by_ctype:
|
||||
merged[ct_main] += c_
|
||||
by_ctype = list(merged.items())[:20]
|
||||
|
||||
# hit/miss ratio using result_code categories
|
||||
hits = sum(c for code, c in by_result.items()
|
||||
if log_parser.RESULT_CATEGORIES.get(code or "") == "Hit")
|
||||
misses = sum(c for code, c in by_result.items()
|
||||
if log_parser.RESULT_CATEGORIES.get(code or "") == "Miss")
|
||||
denied = sum(c for code, c in by_result.items()
|
||||
if log_parser.RESULT_CATEGORIES.get(code or "") == "Denied")
|
||||
aborted = by_result.get("TCP_MISS_ABORTED", 0)
|
||||
denied_rate = denied / total if total else 0.0
|
||||
aborted_rate = aborted / total if total else 0.0
|
||||
hit_ratio = hits / (hits + misses) if (hits + misses) > 0 else 0.0
|
||||
errors_4xx = sum(c for code, c in by_http.items() if code.startswith("4"))
|
||||
errors_5xx = sum(c for code, c in by_http.items() if code.startswith("5"))
|
||||
errors_4xx_rate = errors_4xx / total if total else 0.0
|
||||
errors_5xx_rate = errors_5xx / total if total else 0.0
|
||||
|
||||
# unique clients / hosts
|
||||
unique_clients = q.with_entities(func.count(func.distinct(LogEntry.client))).scalar() or 0
|
||||
unique_hosts = q.filter(LogEntry.host != "").with_entities(
|
||||
func.count(func.distinct(LogEntry.host))
|
||||
).scalar() or 0
|
||||
|
||||
# hourly distribution (UTC hour of day)
|
||||
by_hour_counts: dict[int, int] = defaultdict(int)
|
||||
by_hour_bytes: dict[int, int] = defaultdict(int)
|
||||
# we already have min/max so size of an in-Python bucket is small
|
||||
# but to avoid pulling all rows we can do an UNION-ish query per
|
||||
# hour; the simpler approach: fetch (time, size) projection.
|
||||
rows = q.with_entities(LogEntry.log_time, LogEntry.size_bytes).all()
|
||||
for ts, size in rows:
|
||||
if not ts:
|
||||
continue
|
||||
hr = datetime.fromtimestamp(float(ts), tz=timezone.utc).hour
|
||||
by_hour_counts[hr] += 1
|
||||
by_hour_bytes[hr] += int(size or 0)
|
||||
|
||||
time_start = datetime.fromtimestamp(float(ts_min), tz=timezone.utc) if ts_min else None
|
||||
time_end = datetime.fromtimestamp(float(ts_max), tz=timezone.utc) if ts_max else None
|
||||
|
||||
stats = {
|
||||
"total_requests": total,
|
||||
"total_bytes": total_bytes,
|
||||
"total_elapsed_ms": total_elapsed,
|
||||
"unique_clients": int(unique_clients),
|
||||
"unique_hosts": int(unique_hosts),
|
||||
"hit_ratio": hit_ratio,
|
||||
"denied_rate": denied_rate,
|
||||
"aborted_rate": aborted_rate,
|
||||
"errors_4xx_rate": errors_4xx_rate,
|
||||
"errors_5xx_rate": errors_5xx_rate,
|
||||
"errors_4xx_count": errors_4xx,
|
||||
"errors_5xx_count": errors_5xx,
|
||||
"avg_elapsed_ms": avg_elapsed,
|
||||
"avg_size_bytes": avg_size,
|
||||
"hits": hits,
|
||||
"misses": misses,
|
||||
"denied": denied,
|
||||
"aborted": aborted,
|
||||
"time_start": time_start,
|
||||
"time_end": time_end,
|
||||
"by_result": log_parser._top_n(Counter(by_result), 50),
|
||||
"by_http": log_parser._top_n(Counter(by_http), 50),
|
||||
"by_method": log_parser._top_n(Counter(by_method), 20),
|
||||
"by_category": [], # we don't store category; compute inline if needed
|
||||
"by_hierarchy": log_parser._top_n(Counter(by_hier), 20),
|
||||
"by_ctype": by_ctype,
|
||||
"top_clients": top_clients,
|
||||
"top_hosts": top_hosts,
|
||||
"top_urls": top_urls,
|
||||
"hourly": [
|
||||
{"hour": h, "requests": int(by_hour_counts.get(h, 0)),
|
||||
"bytes": int(by_hour_bytes.get(h, 0))}
|
||||
for h in range(24)
|
||||
],
|
||||
"result_code_legend": log_parser.RESULT_CODE_DESCRIPTION,
|
||||
"hierarchy_legend": log_parser.HIERARCHY_LABELS,
|
||||
}
|
||||
|
||||
# augment with sync timestamp for the dashboard UI
|
||||
try:
|
||||
from app import AppConfig
|
||||
ts_str = AppConfig.query.filter_by(key="_log_last_sync_at").first()
|
||||
stats["last_sync_at"] = ts_str.value if ts_str else None
|
||||
except Exception:
|
||||
stats["last_sync_at"] = None
|
||||
return stats
|
||||
except SQLAlchemyError as e:
|
||||
try:
|
||||
db.session.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
s = _copy_empty_stats()
|
||||
s["last_sync_at"] = None
|
||||
s["error"] = f"{type(e).__name__}: {e}"
|
||||
return s
|
||||
|
||||
|
||||
def _copy_empty_stats() -> dict:
|
||||
"""Return a fresh empty-stats dict (avoid mutating log_parser._empty_stats)."""
|
||||
return log_parser._empty_stats()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Distinct facets
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_distinct_hosts(instance_id: int = 0, limit: int = 200) -> list[str]:
|
||||
try:
|
||||
rows = (
|
||||
db.session.query(LogEntry.host)
|
||||
.filter(LogEntry.host != "", LogEntry.instance_id == instance_id)
|
||||
.distinct()
|
||||
.order_by(LogEntry.host.asc())
|
||||
.limit(int(limit))
|
||||
.all()
|
||||
)
|
||||
return [r[0] for r in rows if r[0]]
|
||||
except SQLAlchemyError:
|
||||
try:
|
||||
db.session.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def get_distinct_clients(instance_id: int = 0, limit: int = 200) -> list[str]:
|
||||
try:
|
||||
rows = (
|
||||
db.session.query(LogEntry.client)
|
||||
.filter(LogEntry.client != "", LogEntry.instance_id == instance_id)
|
||||
.distinct()
|
||||
.order_by(LogEntry.client.asc())
|
||||
.limit(int(limit))
|
||||
.all()
|
||||
)
|
||||
return [r[0] for r in rows if r[0]]
|
||||
except SQLAlchemyError:
|
||||
try:
|
||||
db.session.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def get_distinct_methods(instance_id: int = 0) -> list[str]:
|
||||
try:
|
||||
rows = (
|
||||
db.session.query(LogEntry.method)
|
||||
.filter(LogEntry.method != "", LogEntry.instance_id == instance_id)
|
||||
.distinct()
|
||||
.order_by(LogEntry.method.asc())
|
||||
.all()
|
||||
)
|
||||
return [r[0] for r in rows if r[0]]
|
||||
except SQLAlchemyError:
|
||||
try:
|
||||
db.session.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sync bookkeeping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_LAST_SYNC_KEY = "_log_last_sync_at"
|
||||
|
||||
|
||||
def _record_last_sync(instance_id: int, path: str) -> None:
|
||||
"""Stamp ``AppConfig`` rows with the last successful sync timestamp.
|
||||
|
||||
Two values are stored: ``_log_last_sync_at`` (ISO timestamp) and
|
||||
``_log_last_sync_path`` (the file we synced from). We use ``AppConfig``
|
||||
rather than ``LogEntry`` so the timestamp survives even if log rows
|
||||
are later rotated out.
|
||||
"""
|
||||
try:
|
||||
from app import AppConfig
|
||||
ts_iso = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
for key, value in (
|
||||
(_LAST_SYNC_KEY, ts_iso),
|
||||
("_log_last_sync_path", path or ""),
|
||||
("_log_last_sync_instance", str(instance_id)),
|
||||
):
|
||||
row = AppConfig.query.filter_by(key=key).first()
|
||||
if row is None:
|
||||
row = AppConfig(key=key, value=value)
|
||||
db.session.add(row)
|
||||
else:
|
||||
row.value = value
|
||||
db.session.commit()
|
||||
except Exception:
|
||||
try:
|
||||
db.session.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def get_last_sync_at() -> str | None:
|
||||
"""Return the ISO timestamp of the last sync, or ``None``."""
|
||||
try:
|
||||
from app import AppConfig
|
||||
row = AppConfig.query.filter_by(key=_LAST_SYNC_KEY).first()
|
||||
return row.value if row else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def get_last_sync_path() -> str | None:
|
||||
try:
|
||||
from app import AppConfig
|
||||
row = AppConfig.query.filter_by(key="_log_last_sync_path").first()
|
||||
return row.value if row else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def import_full_history(
|
||||
path: str,
|
||||
instance_id: int = 0,
|
||||
batch_size: int = 500,
|
||||
progress_cb=None,
|
||||
) -> tuple[int, int, str | None]:
|
||||
"""Import the entire ``access.log`` from byte 0 (not just the tail).
|
||||
|
||||
Used by the "Import history" button on the logs page. Calls
|
||||
:func:`sync_log_to_db` with a very high ``max_lines`` (50M) so we
|
||||
stream every line. Progress is reported via ``progress_cb(processed)``
|
||||
so the caller can render a status indicator.
|
||||
"""
|
||||
try:
|
||||
# 50M lines ceiling - enough for multi-year logs; streaming means
|
||||
# we never actually load them all into memory.
|
||||
return sync_log_to_db(
|
||||
path=path,
|
||||
instance_id=instance_id,
|
||||
batch_size=batch_size,
|
||||
max_lines=50_000_000,
|
||||
)
|
||||
except Exception as e:
|
||||
return 0, 0, f"{type(e).__name__}: {e}"
|
||||
Reference in New Issue
Block a user