squid-manager/exporters.py

"""P2-2: Export utilities for Squid Web Manager.

Provides CSV/JSON serialisation helpers and Flask response builders for
downloading log entries, KPI summaries, anomaly lists and audit log entries.

Conventions
-----------
- All CSV output starts with a UTF-8 BOM (``\ufeff``) so Microsoft Excel
  opens it as UTF-8 rather than falling back to the local code page.
- Timestamps are serialised as ISO-8601 strings (UTC when the source
  value carries tzinfo). ``None`` values become empty cells; ``inf``
  becomes the literal string ``Infinity`` so downstream tools don't
  silently choke.
- Every public function swallows exceptions and returns a safe empty
  value (``""`` / ``{}``). This makes it safe to call from request
  handlers - a broken export must never 500 the dashboard.
- No third-party dependencies. The module only uses ``csv``, ``json``,
  ``io`` and Flask's ``Response``.
"""
from __future__ import annotations

import csv
import io
import json
from datetime import date, datetime
from typing import Any, Iterable

from flask import Response


# --- Constants -------------------------------------------------------------

#: Default columns exported for a LogEntry. Order matters - this is the
#: column order Excel/LibreOffice will display.
DEFAULT_ENTRY_FIELDS: list[str] = [
    "time",
    "client",
    "result_code",
    "http_code",
    "method",
    "url",
    "host",
    "size_bytes",
    "elapsed_ms",
    "hier_code",
    "content_type",
]

#: UTF-8 BOM. Excel needs this prefix to recognise the file as UTF-8.
_UTF8_BOM = "\ufeff"


# --- Internal helpers ------------------------------------------------------


def _get_field(entry: Any, key: str) -> Any:
    """Dict-or-attribute getter that never raises.

    LogEntry subclasses dict, so ``entry.get(key)`` works, but we also
    support plain attribute access (e.g. SQLAlchemy model instances).
    """
    try:
        if isinstance(entry, dict):
            return entry.get(key)
        return getattr(entry, key, None)
    except Exception:
        return None


def _serialise_value(value: Any) -> str:
    """Convert a single cell value into a CSV-friendly string.

    - ``None`` -> empty cell (not the literal "None").
    - ``datetime`` / ``date`` -> ISO-8601.
    - ``float('inf')`` / ``float('-inf')`` -> literal "Infinity" / "-Infinity".
    - everything else -> ``str(value)``.
    """
    if value is None:
        return ""
    # datetime must come before date because datetime is a subclass of date
    if isinstance(value, datetime):
        try:
            return value.isoformat()
        except Exception:
            return str(value)
    if isinstance(value, date):
        try:
            return value.isoformat()
        except Exception:
            return str(value)
    if isinstance(value, float):
        # JSON-emitted inf / -inf / nan don't round-trip cleanly through
        # CSV consumers; spell them out instead.
        if value != value:  # NaN
            return "NaN"
        if value == float("inf"):
            return "Infinity"
        if value == float("-inf"):
            return "-Infinity"
        # drop noisy trailing zeros for whole numbers
        if value.is_integer():
            return str(int(value))
        return repr(value)
    if isinstance(value, bool):
        return "true" if value else "false"
    if isinstance(value, (list, tuple)):
        try:
            return ",".join(str(v) for v in value)
        except Exception:
            return str(value)
    try:
        return str(value)
    except Exception:
        return ""


def _timestamp_iso(entry: Any) -> str:
    """Best-effort ISO timestamp for ``entry`` - falls back to ``time`` field."""
    try:
        ts = _get_field(entry, "timestamp")
        if isinstance(ts, datetime):
            return ts.isoformat()
        if isinstance(ts, (int, float)):
            return datetime.utcfromtimestamp(float(ts)).isoformat() + "Z"
    except Exception:
        pass
    try:
        t = _get_field(entry, "time")
        if isinstance(t, (int, float)):
            return datetime.utcfromtimestamp(float(t)).isoformat() + "Z"
    except Exception:
        pass
    return ""


def _write_csv(headers: list[str], rows: Iterable[list[Any]]) -> str:
    """Render ``headers`` + ``rows`` into a CSV string with UTF-8 BOM.

    Uses ``csv.writer`` with QUOTE_MINIMAL so embedded newlines / quotes
    inside URLs are escaped correctly. Output is a plain string (no
    BytesIO shenanigans) so the caller can decide whether to wrap it in
    a Flask ``Response`` or write to disk.
    """
    buf = io.StringIO()
    # Write the BOM first; CSV writer will then prepend the header row.
    buf.write(_UTF8_BOM)
    try:
        writer = csv.writer(buf, quoting=csv.QUOTE_MINIMAL, lineterminator="\n")
        writer.writerow(headers)
        for row in rows:
            try:
                writer.writerow([_serialise_value(v) for v in row])
            except Exception:
                # one bad row shouldn't nuke the whole export
                continue
    except Exception:
        # return at least the BOM + header line so callers get *something*
        try:
            buf.write(",".join(headers))
        except Exception:
            pass
    return buf.getvalue()


# --- Entry export ----------------------------------------------------------


def entries_to_csv(entries: Any, fields: list[str] | None = None) -> str:
    """Serialise a list of log entries to a CSV string.

    ``entries`` may be any iterable of dict-like objects (LogEntry,
    SQLAlchemy model, plain dict). ``fields`` defaults to
    :data:`DEFAULT_ENTRY_FIELDS`. Returns an empty string on error.
    """
    try:
        if not entries:
            # still emit the header so an empty download is a valid CSV
            cols = list(fields) if fields else list(DEFAULT_ENTRY_FIELDS)
            return _write_csv(cols, [])

        cols = list(fields) if fields else list(DEFAULT_ENTRY_FIELDS)

        def _row(entry: Any) -> list[Any]:
            out: list[Any] = []
            for col in cols:
                if col == "time":
                    # Prefer ISO timestamp; fall back to raw time field
                    iso = _timestamp_iso(entry)
                    if iso:
                        out.append(iso)
                    else:
                        out.append(_get_field(entry, "time"))
                elif col == "size_bytes":
                    # The DB column is ``size`` (LogEntry) or ``size`` (dict);
                    # allow ``size_bytes`` as a fallback for either shape.
                    val = _get_field(entry, "size_bytes")
                    if val is None:
                        val = _get_field(entry, "size")
                    out.append(val)
                elif col == "elapsed_ms":
                    val = _get_field(entry, "elapsed_ms")
                    if val is None:
                        val = _get_field(entry, "elapsed")
                    out.append(val)
                elif col == "hier_code":
                    val = _get_field(entry, "hier_code")
                    if val is None:
                        val = _get_field(entry, "hierarchy")
                    out.append(val)
                else:
                    out.append(_get_field(entry, col))
            return out

        return _write_csv(cols, (_row(e) for e in entries))
    except Exception:
        return ""


def entries_to_json(entries: Any, fields: list[str] | None = None) -> str:
    """Serialise a list of log entries to a pretty-printed JSON string.

    Datetimes are converted to ISO-8601. ``fields`` constrains the
    exported keys (per-entry filtering); ``None`` exports every key the
    entry exposes. Returns an empty string on error.
    """
    try:
        out: list[dict[str, Any]] = []
        for entry in entries or []:
            try:
                if fields is not None:
                    rec: dict[str, Any] = {}
                    for col in fields:
                        if col == "time":
                            iso = _timestamp_iso(entry)
                            rec["time"] = iso or _get_field(entry, "time")
                        elif col == "size_bytes":
                            v = _get_field(entry, "size_bytes")
                            rec["size_bytes"] = v if v is not None else _get_field(entry, "size")
                        elif col == "elapsed_ms":
                            v = _get_field(entry, "elapsed_ms")
                            rec["elapsed_ms"] = v if v is not None else _get_field(entry, "elapsed")
                        elif col == "hier_code":
                            v = _get_field(entry, "hier_code")
                            rec["hier_code"] = v if v is not None else _get_field(entry, "hierarchy")
                        else:
                            rec[col] = _get_field(entry, col)
                else:
                    rec = _entry_to_dict(entry)
                out.append(rec)
            except Exception:
                continue
        return json.dumps(out, indent=2, ensure_ascii=False, default=_json_default)
    except Exception:
        return ""


def _entry_to_dict(entry: Any) -> dict[str, Any]:
    """Flatten a LogEntry / SQLAlchemy row into a JSON-friendly dict."""
    try:
        if isinstance(entry, dict):
            return {str(k): _json_safe(v) for k, v in entry.items()}
    except Exception:
        pass
    # Fall back to per-attribute access
    result: dict[str, Any] = {}
    for col in DEFAULT_ENTRY_FIELDS:
        try:
            result[col] = _get_field(entry, col)
        except Exception:
            result[col] = None
    return result


def _json_safe(value: Any) -> Any:
    """Make ``value`` JSON-serialisable. Datetimes -> ISO strings."""
    if value is None or isinstance(value, (str, int, bool)):
        return value
    if isinstance(value, float):
        if value != value:
            return None  # NaN -> null
        if value == float("inf"):
            return "Infinity"
        if value == float("-inf"):
            return "-Infinity"
        return value
    if isinstance(value, (datetime, date)):
        try:
            return value.isoformat()
        except Exception:
            return str(value)
    if isinstance(value, (list, tuple)):
        return [_json_safe(v) for v in value]
    if isinstance(value, dict):
        return {str(k): _json_safe(v) for k, v in value.items()}
    return str(value)


def _json_default(value: Any) -> Any:
    """Fallback encoder for json.dumps - keeps the export robust."""
    return _json_safe(value)


# --- Stats export ----------------------------------------------------------


def stats_to_json(stats: Any) -> str:
    """Serialise a KPI summary dict to a pretty-printed JSON string.

    The dashboard's ``get_stats()`` returns a dict that may contain
    nested structures (top_clients list, etc.). Datetimes nested inside
    are normalised via :func:`_json_safe`. Returns ``""`` on error.
    """
    try:
        if stats is None:
            stats = {}
        return json.dumps(_json_safe(stats), indent=2,
                          ensure_ascii=False, default=_json_default)
    except Exception:
        return ""


# --- Anomaly export --------------------------------------------------------


#: Field order for the long-form anomaly CSV. ``type`` distinguishes the
#: detector; the rest of the columns are the union of useful fields.
ANOMALY_CSV_FIELDS: list[str] = [
    "type",
    "client",
    "severity",
    "time",
    "url",
    "host",
    "method",
    "result",
    "result_code",
    "http_code",
    "size_bytes",
    "size_mb",
    "request_count",
    "total_bytes",
    "avg_size",
    "baseline_rpm",
    "current_rpm",
    "baseline_count",
    "current_count",
    "ratio",
    "first_seen",
    "sample_url",
    "sample_host",
]


def _anomaly_rows(summary: dict[str, Any]) -> list[dict[str, Any]]:
    """Flatten the four anomaly sub-lists into a single long table.

    Each row gets a ``type`` column so consumers can pivot / filter.
    Rows from different detectors carry different fields; missing
    fields are simply left empty.
    """
    rows: list[dict[str, Any]] = []
    if not isinstance(summary, dict):
        return rows

    # 1. anomalous_clients (spikes)
    for a in summary.get("anomalous_clients") or []:
        if not isinstance(a, dict):
            continue
        rec = {k: a.get(k) for k in ANOMALY_CSV_FIELDS if k in a}
        rec["type"] = "client_spike"
        rec.setdefault("client", a.get("client"))
        rec.setdefault("severity", a.get("severity"))
        rec.setdefault("baseline_rpm", a.get("baseline_rpm"))
        rec.setdefault("current_rpm", a.get("current_rpm"))
        rec.setdefault("baseline_count", a.get("baseline_count"))
        rec.setdefault("current_count", a.get("current_count"))
        rec.setdefault("ratio", a.get("ratio"))
        rows.append(rec)

    # 2. large_requests
    for a in summary.get("large_requests") or []:
        if not isinstance(a, dict):
            continue
        rec = {k: a.get(k) for k in ANOMALY_CSV_FIELDS if k in a}
        rec["type"] = "large_request"
        rec.setdefault("client", a.get("client"))
        rec.setdefault("severity", "warning")
        rec.setdefault("size_bytes", a.get("size_bytes"))
        rec.setdefault("size_mb", a.get("size_mb"))
        rows.append(rec)

    # 3. high_freq_small
    for a in summary.get("high_freq_small") or []:
        if not isinstance(a, dict):
            continue
        rec = {k: a.get(k) for k in ANOMALY_CSV_FIELDS if k in a}
        rec["type"] = "high_freq_small"
        rec.setdefault("client", a.get("client"))
        rec.setdefault("severity", "warning")
        rec.setdefault("sample_url", a.get("sample_url"))
        rec.setdefault("sample_host", a.get("sample_host"))
        rows.append(rec)

    # 4. new_clients
    for a in summary.get("new_clients") or []:
        if not isinstance(a, dict):
            continue
        rec = {k: a.get(k) for k in ANOMALY_CSV_FIELDS if k in a}
        rec["type"] = "new_client"
        rec.setdefault("client", a.get("client"))
        rec.setdefault("severity", "info")
        rec.setdefault("first_seen", a.get("first_seen"))
        rec.setdefault("request_count", a.get("request_count"))
        rec.setdefault("sample_url", a.get("sample_url"))
        rows.append(rec)

    return rows


def anomalies_to_csv(summary: Any) -> str:
    """Serialise the anomaly summary to a long-form CSV.

    ``summary`` is the dict returned by ``_build_anomaly_summary`` (or
    anything with the same four sub-list keys). All four detector
    outputs are flattened into one table with a ``type`` column.
    Returns ``""`` on error.
    """
    try:
        rows = _anomaly_rows(summary if isinstance(summary, dict) else {})
        return _write_csv(ANOMALY_CSV_FIELDS, [
            [r.get(col) for col in ANOMALY_CSV_FIELDS] for r in rows
        ])
    except Exception:
        return ""


def anomalies_to_json(summary: Any) -> str:
    """Serialise the anomaly summary to pretty JSON (raw structure)."""
    try:
        if summary is None:
            summary = {}
        return json.dumps(_json_safe(summary), indent=2,
                          ensure_ascii=False, default=_json_default)
    except Exception:
        return ""


# --- Audit export ----------------------------------------------------------


#: Field order for the audit log CSV.
AUDIT_CSV_FIELDS: list[str] = [
    "timestamp",
    "username",
    "action",
    "detail",
]


def _audit_row(audit_obj: Any) -> list[Any]:
    """Convert an AuditLog row to a list aligned with AUDIT_CSV_FIELDS."""
    try:
        ts = _get_field(audit_obj, "timestamp")
        if isinstance(ts, datetime):
            ts_str = ts.isoformat()
        elif isinstance(ts, (int, float)):
            try:
                ts_str = datetime.utcfromtimestamp(float(ts)).isoformat() + "Z"
            except Exception:
                ts_str = str(ts)
        else:
            ts_str = _serialise_value(ts)
    except Exception:
        ts_str = ""
    return [
        ts_str,
        _get_field(audit_obj, "username"),
        _get_field(audit_obj, "action"),
        _get_field(audit_obj, "detail"),
    ]


def audit_to_csv(entries: Any) -> str:
    """Serialise a list of AuditLog rows to a CSV string."""
    try:
        items = list(entries) if entries else []
        return _write_csv(AUDIT_CSV_FIELDS, (_audit_row(a) for a in items))
    except Exception:
        return ""


def audit_to_json(entries: Any) -> str:
    """Serialise a list of AuditLog rows to pretty JSON."""
    try:
        out: list[dict[str, Any]] = []
        for a in entries or []:
            try:
                out.append({
                    "timestamp": _json_safe(_get_field(a, "timestamp")),
                    "username": _get_field(a, "username"),
                    "action": _get_field(a, "action"),
                    "detail": _get_field(a, "detail"),
                })
            except Exception:
                continue
        return json.dumps(out, indent=2, ensure_ascii=False,
                          default=_json_default)
    except Exception:
        return ""


# --- Response builders -----------------------------------------------------


def csv_response(csv_text: str, filename: str) -> Response:
    """Wrap a CSV string in a Flask ``Response`` with download headers.

    Adds ``Content-Disposition: attachment; filename=...`` so browsers
    save the file rather than rendering it inline. ``Content-Type`` is
    ``text/csv; charset=utf-8`` (Flask appends the charset once).
    """
    if not csv_text:
        csv_text = _UTF8_BOM  # at least the BOM, so the file isn't empty
    # The CSV already has the UTF-8 BOM baked in; we hand Flask a plain
    # str response and let it encode via utf-8.
    safe_name = _safe_filename(filename, default="export.csv")
    resp = Response(
        csv_text,
        status=200,
    )
    resp.headers["Content-Type"] = "text/csv; charset=utf-8"
    resp.headers["Content-Disposition"] = f'attachment; filename="{safe_name}"'
    resp.headers["Cache-Control"] = "no-store"
    return resp


def json_response(json_text: str, filename: str) -> Response:
    """Wrap a JSON string in a Flask ``Response`` with download headers."""
    if not json_text:
        json_text = "{}"
    safe_name = _safe_filename(filename, default="export.json")
    resp = Response(
        json_text,
        status=200,
    )
    resp.headers["Content-Type"] = "application/json; charset=utf-8"
    resp.headers["Content-Disposition"] = f'attachment; filename="{safe_name}"'
    resp.headers["Cache-Control"] = "no-store"
    return resp


def _safe_filename(name: str, default: str = "export") -> str:
    """Strip path separators / control chars from a download filename."""
    try:
        if not name:
            return default
        # Drop any directory components - browsers only care about the basename.
        name = name.replace("\\", "/").split("/")[-1]
        # Reject anything that isn't a basic filename char.
        cleaned = "".join(c for c in name if c.isprintable() and c not in '"<>|:*?\n\r\t')
        return cleaned or default
    except Exception:
        return default


# --- Filename helpers ------------------------------------------------------


def timestamped_filename(prefix: str, ext: str) -> str:
    """Build ``prefix_YYYYMMDD_HHMMSS.ext`` (local time)."""
    try:
        ts = datetime.now().strftime("%Y%m%d_%H%M%S")
    except Exception:
        ts = "unknown"
    # ext should be a plain suffix like "csv" or "json" (no leading dot)
    ext = (ext or "").lstrip(".").lower() or "bin"
    return f"{prefix}_{ts}.{ext}"