"""P2-2: Export utilities for Squid Web Manager. Provides CSV/JSON serialisation helpers and Flask response builders for downloading log entries, KPI summaries, anomaly lists and audit log entries. Conventions ----------- - All CSV output starts with a UTF-8 BOM (``\ufeff``) so Microsoft Excel opens it as UTF-8 rather than falling back to the local code page. - Timestamps are serialised as ISO-8601 strings (UTC when the source value carries tzinfo). ``None`` values become empty cells; ``inf`` becomes the literal string ``Infinity`` so downstream tools don't silently choke. - Every public function swallows exceptions and returns a safe empty value (``""`` / ``{}``). This makes it safe to call from request handlers - a broken export must never 500 the dashboard. - No third-party dependencies. The module only uses ``csv``, ``json``, ``io`` and Flask's ``Response``. """ from __future__ import annotations import csv import io import json from datetime import date, datetime from typing import Any, Iterable from flask import Response # --- Constants ------------------------------------------------------------- #: Default columns exported for a LogEntry. Order matters - this is the #: column order Excel/LibreOffice will display. DEFAULT_ENTRY_FIELDS: list[str] = [ "time", "client", "result_code", "http_code", "method", "url", "host", "size_bytes", "elapsed_ms", "hier_code", "content_type", ] #: UTF-8 BOM. Excel needs this prefix to recognise the file as UTF-8. _UTF8_BOM = "\ufeff" # --- Internal helpers ------------------------------------------------------ def _get_field(entry: Any, key: str) -> Any: """Dict-or-attribute getter that never raises. LogEntry subclasses dict, so ``entry.get(key)`` works, but we also support plain attribute access (e.g. SQLAlchemy model instances). """ try: if isinstance(entry, dict): return entry.get(key) return getattr(entry, key, None) except Exception: return None def _serialise_value(value: Any) -> str: """Convert a single cell value into a CSV-friendly string. - ``None`` -> empty cell (not the literal "None"). - ``datetime`` / ``date`` -> ISO-8601. - ``float('inf')`` / ``float('-inf')`` -> literal "Infinity" / "-Infinity". - everything else -> ``str(value)``. """ if value is None: return "" # datetime must come before date because datetime is a subclass of date if isinstance(value, datetime): try: return value.isoformat() except Exception: return str(value) if isinstance(value, date): try: return value.isoformat() except Exception: return str(value) if isinstance(value, float): # JSON-emitted inf / -inf / nan don't round-trip cleanly through # CSV consumers; spell them out instead. if value != value: # NaN return "NaN" if value == float("inf"): return "Infinity" if value == float("-inf"): return "-Infinity" # drop noisy trailing zeros for whole numbers if value.is_integer(): return str(int(value)) return repr(value) if isinstance(value, bool): return "true" if value else "false" if isinstance(value, (list, tuple)): try: return ",".join(str(v) for v in value) except Exception: return str(value) try: return str(value) except Exception: return "" def _timestamp_iso(entry: Any) -> str: """Best-effort ISO timestamp for ``entry`` - falls back to ``time`` field.""" try: ts = _get_field(entry, "timestamp") if isinstance(ts, datetime): return ts.isoformat() if isinstance(ts, (int, float)): return datetime.utcfromtimestamp(float(ts)).isoformat() + "Z" except Exception: pass try: t = _get_field(entry, "time") if isinstance(t, (int, float)): return datetime.utcfromtimestamp(float(t)).isoformat() + "Z" except Exception: pass return "" def _write_csv(headers: list[str], rows: Iterable[list[Any]]) -> str: """Render ``headers`` + ``rows`` into a CSV string with UTF-8 BOM. Uses ``csv.writer`` with QUOTE_MINIMAL so embedded newlines / quotes inside URLs are escaped correctly. Output is a plain string (no BytesIO shenanigans) so the caller can decide whether to wrap it in a Flask ``Response`` or write to disk. """ buf = io.StringIO() # Write the BOM first; CSV writer will then prepend the header row. buf.write(_UTF8_BOM) try: writer = csv.writer(buf, quoting=csv.QUOTE_MINIMAL, lineterminator="\n") writer.writerow(headers) for row in rows: try: writer.writerow([_serialise_value(v) for v in row]) except Exception: # one bad row shouldn't nuke the whole export continue except Exception: # return at least the BOM + header line so callers get *something* try: buf.write(",".join(headers)) except Exception: pass return buf.getvalue() # --- Entry export ---------------------------------------------------------- def entries_to_csv(entries: Any, fields: list[str] | None = None) -> str: """Serialise a list of log entries to a CSV string. ``entries`` may be any iterable of dict-like objects (LogEntry, SQLAlchemy model, plain dict). ``fields`` defaults to :data:`DEFAULT_ENTRY_FIELDS`. Returns an empty string on error. """ try: if not entries: # still emit the header so an empty download is a valid CSV cols = list(fields) if fields else list(DEFAULT_ENTRY_FIELDS) return _write_csv(cols, []) cols = list(fields) if fields else list(DEFAULT_ENTRY_FIELDS) def _row(entry: Any) -> list[Any]: out: list[Any] = [] for col in cols: if col == "time": # Prefer ISO timestamp; fall back to raw time field iso = _timestamp_iso(entry) if iso: out.append(iso) else: out.append(_get_field(entry, "time")) elif col == "size_bytes": # The DB column is ``size`` (LogEntry) or ``size`` (dict); # allow ``size_bytes`` as a fallback for either shape. val = _get_field(entry, "size_bytes") if val is None: val = _get_field(entry, "size") out.append(val) elif col == "elapsed_ms": val = _get_field(entry, "elapsed_ms") if val is None: val = _get_field(entry, "elapsed") out.append(val) elif col == "hier_code": val = _get_field(entry, "hier_code") if val is None: val = _get_field(entry, "hierarchy") out.append(val) else: out.append(_get_field(entry, col)) return out return _write_csv(cols, (_row(e) for e in entries)) except Exception: return "" def entries_to_json(entries: Any, fields: list[str] | None = None) -> str: """Serialise a list of log entries to a pretty-printed JSON string. Datetimes are converted to ISO-8601. ``fields`` constrains the exported keys (per-entry filtering); ``None`` exports every key the entry exposes. Returns an empty string on error. """ try: out: list[dict[str, Any]] = [] for entry in entries or []: try: if fields is not None: rec: dict[str, Any] = {} for col in fields: if col == "time": iso = _timestamp_iso(entry) rec["time"] = iso or _get_field(entry, "time") elif col == "size_bytes": v = _get_field(entry, "size_bytes") rec["size_bytes"] = v if v is not None else _get_field(entry, "size") elif col == "elapsed_ms": v = _get_field(entry, "elapsed_ms") rec["elapsed_ms"] = v if v is not None else _get_field(entry, "elapsed") elif col == "hier_code": v = _get_field(entry, "hier_code") rec["hier_code"] = v if v is not None else _get_field(entry, "hierarchy") else: rec[col] = _get_field(entry, col) else: rec = _entry_to_dict(entry) out.append(rec) except Exception: continue return json.dumps(out, indent=2, ensure_ascii=False, default=_json_default) except Exception: return "" def _entry_to_dict(entry: Any) -> dict[str, Any]: """Flatten a LogEntry / SQLAlchemy row into a JSON-friendly dict.""" try: if isinstance(entry, dict): return {str(k): _json_safe(v) for k, v in entry.items()} except Exception: pass # Fall back to per-attribute access result: dict[str, Any] = {} for col in DEFAULT_ENTRY_FIELDS: try: result[col] = _get_field(entry, col) except Exception: result[col] = None return result def _json_safe(value: Any) -> Any: """Make ``value`` JSON-serialisable. Datetimes -> ISO strings.""" if value is None or isinstance(value, (str, int, bool)): return value if isinstance(value, float): if value != value: return None # NaN -> null if value == float("inf"): return "Infinity" if value == float("-inf"): return "-Infinity" return value if isinstance(value, (datetime, date)): try: return value.isoformat() except Exception: return str(value) if isinstance(value, (list, tuple)): return [_json_safe(v) for v in value] if isinstance(value, dict): return {str(k): _json_safe(v) for k, v in value.items()} return str(value) def _json_default(value: Any) -> Any: """Fallback encoder for json.dumps - keeps the export robust.""" return _json_safe(value) # --- Stats export ---------------------------------------------------------- def stats_to_json(stats: Any) -> str: """Serialise a KPI summary dict to a pretty-printed JSON string. The dashboard's ``get_stats()`` returns a dict that may contain nested structures (top_clients list, etc.). Datetimes nested inside are normalised via :func:`_json_safe`. Returns ``""`` on error. """ try: if stats is None: stats = {} return json.dumps(_json_safe(stats), indent=2, ensure_ascii=False, default=_json_default) except Exception: return "" # --- Anomaly export -------------------------------------------------------- #: Field order for the long-form anomaly CSV. ``type`` distinguishes the #: detector; the rest of the columns are the union of useful fields. ANOMALY_CSV_FIELDS: list[str] = [ "type", "client", "severity", "time", "url", "host", "method", "result", "result_code", "http_code", "size_bytes", "size_mb", "request_count", "total_bytes", "avg_size", "baseline_rpm", "current_rpm", "baseline_count", "current_count", "ratio", "first_seen", "sample_url", "sample_host", ] def _anomaly_rows(summary: dict[str, Any]) -> list[dict[str, Any]]: """Flatten the four anomaly sub-lists into a single long table. Each row gets a ``type`` column so consumers can pivot / filter. Rows from different detectors carry different fields; missing fields are simply left empty. """ rows: list[dict[str, Any]] = [] if not isinstance(summary, dict): return rows # 1. anomalous_clients (spikes) for a in summary.get("anomalous_clients") or []: if not isinstance(a, dict): continue rec = {k: a.get(k) for k in ANOMALY_CSV_FIELDS if k in a} rec["type"] = "client_spike" rec.setdefault("client", a.get("client")) rec.setdefault("severity", a.get("severity")) rec.setdefault("baseline_rpm", a.get("baseline_rpm")) rec.setdefault("current_rpm", a.get("current_rpm")) rec.setdefault("baseline_count", a.get("baseline_count")) rec.setdefault("current_count", a.get("current_count")) rec.setdefault("ratio", a.get("ratio")) rows.append(rec) # 2. large_requests for a in summary.get("large_requests") or []: if not isinstance(a, dict): continue rec = {k: a.get(k) for k in ANOMALY_CSV_FIELDS if k in a} rec["type"] = "large_request" rec.setdefault("client", a.get("client")) rec.setdefault("severity", "warning") rec.setdefault("size_bytes", a.get("size_bytes")) rec.setdefault("size_mb", a.get("size_mb")) rows.append(rec) # 3. high_freq_small for a in summary.get("high_freq_small") or []: if not isinstance(a, dict): continue rec = {k: a.get(k) for k in ANOMALY_CSV_FIELDS if k in a} rec["type"] = "high_freq_small" rec.setdefault("client", a.get("client")) rec.setdefault("severity", "warning") rec.setdefault("sample_url", a.get("sample_url")) rec.setdefault("sample_host", a.get("sample_host")) rows.append(rec) # 4. new_clients for a in summary.get("new_clients") or []: if not isinstance(a, dict): continue rec = {k: a.get(k) for k in ANOMALY_CSV_FIELDS if k in a} rec["type"] = "new_client" rec.setdefault("client", a.get("client")) rec.setdefault("severity", "info") rec.setdefault("first_seen", a.get("first_seen")) rec.setdefault("request_count", a.get("request_count")) rec.setdefault("sample_url", a.get("sample_url")) rows.append(rec) return rows def anomalies_to_csv(summary: Any) -> str: """Serialise the anomaly summary to a long-form CSV. ``summary`` is the dict returned by ``_build_anomaly_summary`` (or anything with the same four sub-list keys). All four detector outputs are flattened into one table with a ``type`` column. Returns ``""`` on error. """ try: rows = _anomaly_rows(summary if isinstance(summary, dict) else {}) return _write_csv(ANOMALY_CSV_FIELDS, [ [r.get(col) for col in ANOMALY_CSV_FIELDS] for r in rows ]) except Exception: return "" def anomalies_to_json(summary: Any) -> str: """Serialise the anomaly summary to pretty JSON (raw structure).""" try: if summary is None: summary = {} return json.dumps(_json_safe(summary), indent=2, ensure_ascii=False, default=_json_default) except Exception: return "" # --- Audit export ---------------------------------------------------------- #: Field order for the audit log CSV. AUDIT_CSV_FIELDS: list[str] = [ "timestamp", "username", "action", "detail", ] def _audit_row(audit_obj: Any) -> list[Any]: """Convert an AuditLog row to a list aligned with AUDIT_CSV_FIELDS.""" try: ts = _get_field(audit_obj, "timestamp") if isinstance(ts, datetime): ts_str = ts.isoformat() elif isinstance(ts, (int, float)): try: ts_str = datetime.utcfromtimestamp(float(ts)).isoformat() + "Z" except Exception: ts_str = str(ts) else: ts_str = _serialise_value(ts) except Exception: ts_str = "" return [ ts_str, _get_field(audit_obj, "username"), _get_field(audit_obj, "action"), _get_field(audit_obj, "detail"), ] def audit_to_csv(entries: Any) -> str: """Serialise a list of AuditLog rows to a CSV string.""" try: items = list(entries) if entries else [] return _write_csv(AUDIT_CSV_FIELDS, (_audit_row(a) for a in items)) except Exception: return "" def audit_to_json(entries: Any) -> str: """Serialise a list of AuditLog rows to pretty JSON.""" try: out: list[dict[str, Any]] = [] for a in entries or []: try: out.append({ "timestamp": _json_safe(_get_field(a, "timestamp")), "username": _get_field(a, "username"), "action": _get_field(a, "action"), "detail": _get_field(a, "detail"), }) except Exception: continue return json.dumps(out, indent=2, ensure_ascii=False, default=_json_default) except Exception: return "" # --- Response builders ----------------------------------------------------- def csv_response(csv_text: str, filename: str) -> Response: """Wrap a CSV string in a Flask ``Response`` with download headers. Adds ``Content-Disposition: attachment; filename=...`` so browsers save the file rather than rendering it inline. ``Content-Type`` is ``text/csv; charset=utf-8`` (Flask appends the charset once). """ if not csv_text: csv_text = _UTF8_BOM # at least the BOM, so the file isn't empty # The CSV already has the UTF-8 BOM baked in; we hand Flask a plain # str response and let it encode via utf-8. safe_name = _safe_filename(filename, default="export.csv") resp = Response( csv_text, status=200, ) resp.headers["Content-Type"] = "text/csv; charset=utf-8" resp.headers["Content-Disposition"] = f'attachment; filename="{safe_name}"' resp.headers["Cache-Control"] = "no-store" return resp def json_response(json_text: str, filename: str) -> Response: """Wrap a JSON string in a Flask ``Response`` with download headers.""" if not json_text: json_text = "{}" safe_name = _safe_filename(filename, default="export.json") resp = Response( json_text, status=200, ) resp.headers["Content-Type"] = "application/json; charset=utf-8" resp.headers["Content-Disposition"] = f'attachment; filename="{safe_name}"' resp.headers["Cache-Control"] = "no-store" return resp def _safe_filename(name: str, default: str = "export") -> str: """Strip path separators / control chars from a download filename.""" try: if not name: return default # Drop any directory components - browsers only care about the basename. name = name.replace("\\", "/").split("/")[-1] # Reject anything that isn't a basic filename char. cleaned = "".join(c for c in name if c.isprintable() and c not in '"<>|:*?\n\r\t') return cleaned or default except Exception: return default # --- Filename helpers ------------------------------------------------------ def timestamped_filename(prefix: str, ext: str) -> str: """Build ``prefix_YYYYMMDD_HHMMSS.ext`` (local time).""" try: ts = datetime.now().strftime("%Y%m%d_%H%M%S") except Exception: ts = "unknown" # ext should be a plain suffix like "csv" or "json" (no leading dot) ext = (ext or "").lstrip(".").lower() or "bin" return f"{prefix}_{ts}.{ext}"