"""备份执行编排:解密配置 → 流式产出 → 流式上传 → 应用保留策略 → 写日志。""" import asyncio import hashlib import io import logging import os import tempfile import time from datetime import datetime from pathlib import Path from typing import Optional from sqlalchemy.orm import Session from app.core.backup.base import BaseBackup from app.core.backup.directory import DirectoryBackup from app.core.backup.mysql import MySQLBackup from app.core.retention import apply_retention from app.core.storage.factory import create_storage from app.database import SessionLocal from app.models.job import Job from app.models.run import Run, RunLog from app.utils.crypto import decrypt_dict from app.utils.logging import get_logger log = get_logger(__name__) def _make_backup(job: Job) -> BaseBackup: src = decrypt_dict(job.source_config_json) if job.type == "mysql": return MySQLBackup(src, job.name) if job.type == "directory": return DirectoryBackup(src, job.name) raise ValueError(f"Unsupported job type: {job.type}") async def run_backup(run_id: int, trigger: str = "manual") -> None: """执行一次备份。所有副作用都在自己 new 的 Session 上完成。""" started = datetime.utcnow() db: Session = SessionLocal() try: run: Run | None = db.get(Run, run_id) if not run: log.error("run_id=%s not found", run_id) return job: Job | None = db.get(Job, run.job_id) if not job: run.status = "failed" run.error_message = f"Job id={run.job_id} disappeared" run.finished_at = datetime.utcnow() db.commit() return log_buffer = io.StringIO() tee = _TeeLogger(log, log_buffer) run.status = "running" run.trigger = trigger run.started_at = started db.commit() tee.info("Backup started: run_id=%s job=%s trigger=%s", run_id, job.name, trigger) try: backup = _make_backup(job) meta = backup.metadata() tee.info("Metadata: filename=%s content_type=%s", meta.suggested_filename, meta.content_type) storage = create_storage(job.storage) ok, msg = await storage.test_connection() tee.info("Storage test: ok=%s msg=%s", ok, msg) if not ok: raise RuntimeError(f"Storage not reachable: {msg}") # 关键路径:备份文件名以 {job_name}/{YYYY-MM-DD}/{filename} 组织 date_prefix = datetime.utcnow().strftime("%Y-%m-%d") object_key = f"{job.name}/{date_prefix}/{meta.suggested_filename}" tee.info("Streaming backup to storage key=%s ...", object_key) async def _consume_with_progress(): async for chunk in backup.produce(): yield chunk t0 = time.monotonic() size, sha = await storage.upload_stream( _consume_with_progress(), object_key, content_type=meta.content_type, ) elapsed = int(time.monotonic() - t0) run.artifact_path = object_key run.artifact_size = size run.artifact_checksum = sha run.duration_seconds = elapsed run.status = "success" run.finished_at = datetime.utcnow() tee.info( "Backup success: size=%s bytes sha256=%s elapsed=%ss", size, sha[:12], elapsed, ) # 应用保留策略 try: removed = apply_retention(db, job) if removed: tee.info("Retention cleanup removed %s old runs", removed) except Exception as e: tee.warning("Retention cleanup failed: %s", e) except Exception as e: log.exception("Backup run_id=%s failed", run_id) tee.error("Backup failed: %s: %s", type(e).__name__, e) run.status = "failed" run.error_message = f"{type(e).__name__}: {e}"[:2000] run.finished_at = datetime.utcnow() finally: log_text = log_buffer.getvalue() run.log_excerpt = log_text[-4000:] if len(log_text) > 4000 else log_text run_log = RunLog(run_id=run.id, content=log_text) db.add(run_log) db.commit() log.info("Backup run_id=%s finished status=%s", run_id, run.status) finally: db.close() class _TeeLogger: """同时写标准 logging 和内存 StringIO(用于存数据库)。""" def __init__(self, real: logging.Logger, buf: io.StringIO): self._real = real self._buf = buf def _emit(self, level: int, msg: str) -> None: ts = datetime.utcnow().strftime("%H:%M:%S") line = f"[{ts}] {msg}" self._real.log(level, line) self._buf.write(line + "\n") def info(self, msg: str, *args) -> None: self._emit(logging.INFO, msg % args if args else msg) def warning(self, msg: str, *args) -> None: self._emit(logging.WARNING, msg % args if args else msg) def error(self, msg: str, *args) -> None: self._emit(logging.ERROR, msg % args if args else msg)