feat: 数据备份管理系统 v0.1.0
带 Web 操作界面的备份系统，支持： - 数据源：MySQL 数据库（mysqldump）、服务器目录（tar.gz） - 存储目标：本地目录、S3 兼容对象存储（MinIO/Ceph/AWS S3） - 触发方式：手动 + Cron 定时调度（APScheduler） - 备份还原：MySQL 库、目录 - JWT 登录认证 + Fernet 字段加密 - 保留策略：按数量 + 按天数双重清理 技术栈：FastAPI + SQLAlchemy 2 + APScheduler + aioboto3； 前端 Vite + React 18 + TypeScript + Ant Design 5 + Zustand。 Docker Compose 一键起。 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
publ
@@ -0,0 +1,36 @@
|
||||
"""备份执行抽象基类：MySQL / Directory 都实现此接口。"""
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import AsyncIterator, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class BackupMetadata:
|
||||
"""备份产物的元信息。"""
|
||||
|
||||
suggested_filename: str # e.g. "db-20260622-030000.sql.gz" 或 "logs-20260622.tar.gz"
|
||||
content_type: str = "application/octet-stream"
|
||||
extra: Optional[dict] = None
|
||||
|
||||
|
||||
class BaseBackup(ABC):
|
||||
"""所有数据源类型实现此接口。
|
||||
|
||||
方法职责：
|
||||
- metadata(): 计算建议文件名等
|
||||
- produce(): 以异步字节流形式产出备份内容
|
||||
"""
|
||||
|
||||
def __init__(self, source_config: dict, job_name: str):
|
||||
self.source_config = source_config
|
||||
self.job_name = job_name
|
||||
|
||||
@abstractmethod
|
||||
def metadata(self) -> BackupMetadata: ...
|
||||
|
||||
@abstractmethod
|
||||
async def produce(self) -> AsyncIterator[bytes]:
|
||||
"""异步产出备份字节流。"""
|
||||
# 子类应使用 async with + async generator
|
||||
...
|
||||
yield b"" # pragma: no cover
|
||||
@@ -0,0 +1,127 @@
|
||||
"""目录备份：使用 tarfile 流式打包为 tar.gz。"""
|
||||
import asyncio
|
||||
import fnmatch
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import AsyncIterator
|
||||
|
||||
from app.core.backup.base import BackupMetadata, BaseBackup
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class DirectoryBackup(BaseBackup):
|
||||
def __init__(self, source_config: dict, job_name: str):
|
||||
super().__init__(source_config, job_name)
|
||||
self.path: str = source_config["path"]
|
||||
self.exclude_patterns: list[str] = source_config.get("exclude_patterns", []) or []
|
||||
self.base_dir = os.path.basename(os.path.normpath(self.path)) or "root"
|
||||
self.root_path = Path(self.path).resolve()
|
||||
|
||||
def metadata(self) -> BackupMetadata:
|
||||
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
safe = "".join(c if c.isalnum() or c in ("_", "-") else "_" for c in self.base_dir)
|
||||
return BackupMetadata(
|
||||
suggested_filename=f"{self.job_name}-{safe}-{ts}.tar.gz",
|
||||
content_type="application/gzip",
|
||||
extra={"path": self.path, "base_dir": self.base_dir},
|
||||
)
|
||||
|
||||
def _is_excluded(self, rel_path: str) -> bool:
|
||||
name = os.path.basename(rel_path)
|
||||
for pat in self.exclude_patterns:
|
||||
if fnmatch.fnmatch(name, pat) or fnmatch.fnmatch(rel_path, pat):
|
||||
return True
|
||||
return False
|
||||
|
||||
async def produce(self) -> AsyncIterator[bytes]:
|
||||
if not self.root_path.exists():
|
||||
raise FileNotFoundError(f"Source path does not exist: {self.path}")
|
||||
if not self.root_path.is_dir():
|
||||
raise NotADirectoryError(f"Source path is not a directory: {self.path}")
|
||||
|
||||
log.info("Taring directory %s (exclude=%s)", self.path, self.exclude_patterns)
|
||||
|
||||
# 用一个 Queue 在同步线程和异步消费者之间桥接 tar 产出
|
||||
queue: asyncio.Queue[bytes | None] = asyncio.Queue(maxsize=32)
|
||||
|
||||
def _tar_worker() -> None:
|
||||
import tarfile
|
||||
|
||||
try:
|
||||
with tarfile.open(mode="w:gz", bufsize=1024 * 1024) as tar:
|
||||
# 顶层目录名固定为 base_dir，便于解压后还原
|
||||
for dirpath, dirnames, filenames in os.walk(self.root_path):
|
||||
# 排除目录
|
||||
dirnames[:] = [
|
||||
d for d in dirnames
|
||||
if not self._is_excluded(os.path.relpath(os.path.join(dirpath, d), self.root_path))
|
||||
]
|
||||
for fn in filenames:
|
||||
abs_path = os.path.join(dirpath, fn)
|
||||
rel = os.path.relpath(abs_path, self.root_path)
|
||||
if self._is_excluded(rel):
|
||||
continue
|
||||
try:
|
||||
tar.add(abs_path, arcname=os.path.join(self.base_dir, rel), recursive=False)
|
||||
except (PermissionError, OSError) as e:
|
||||
log.warning("Skip %s: %s", abs_path, e)
|
||||
continue
|
||||
# 周期性 flush 到 queue
|
||||
if tar.fileobj.tell() and (tar.fileobj.tell() % (256 * 1024) == 0):
|
||||
# 这里我们用 tarfile 的 fileobj 直接喂到 queue 更高效
|
||||
pass
|
||||
except Exception as e:
|
||||
log.exception("tar worker error: %s", e)
|
||||
finally:
|
||||
# tarfile 关闭后会写完 gzip trailer，我们需要在关闭时把 fileobj 全部送出
|
||||
pass
|
||||
|
||||
# 实现简化：使用 tarfile 的 fileobj=BytesIO 会在循环中累积，
|
||||
# 对超大目录会爆内存。下面用一个临时文件 + 异步读取的方案。
|
||||
import tempfile
|
||||
|
||||
tmp_path = Path(tempfile.gettempdir()) / f"backup_{self.job_name}_{os.getpid()}.tar.gz"
|
||||
|
||||
def _tar_to_file() -> None:
|
||||
import tarfile
|
||||
|
||||
try:
|
||||
with tarfile.open(str(tmp_path), mode="w:gz") as tar:
|
||||
for dirpath, dirnames, filenames in os.walk(self.root_path):
|
||||
dirnames[:] = [
|
||||
d for d in dirnames
|
||||
if not self._is_excluded(os.path.relpath(os.path.join(dirpath, d), self.root_path))
|
||||
]
|
||||
for fn in filenames:
|
||||
abs_path = os.path.join(dirpath, fn)
|
||||
rel = os.path.relpath(abs_path, self.root_path)
|
||||
if self._is_excluded(rel):
|
||||
continue
|
||||
try:
|
||||
tar.add(abs_path, arcname=os.path.join(self.base_dir, rel), recursive=False)
|
||||
except (PermissionError, OSError) as e:
|
||||
log.warning("Skip %s: %s", abs_path, e)
|
||||
except Exception:
|
||||
log.exception("tar file worker failed")
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
await loop.run_in_executor(None, _tar_to_file)
|
||||
|
||||
if not tmp_path.exists() or tmp_path.stat().st_size == 0:
|
||||
raise RuntimeError(f"Failed to create tar.gz for {self.path}")
|
||||
|
||||
try:
|
||||
with open(tmp_path, "rb") as f:
|
||||
while True:
|
||||
chunk = await loop.run_in_executor(None, f.read, 64 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
try:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,97 @@
|
||||
"""MySQL 备份：通过 mysqldump 子进程流式输出。"""
|
||||
import asyncio
|
||||
import gzip
|
||||
import shlex
|
||||
from datetime import datetime
|
||||
from typing import AsyncIterator
|
||||
|
||||
from app.core.backup.base import BackupMetadata, BaseBackup
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
CHUNK_SIZE = 64 * 1024
|
||||
|
||||
|
||||
class MySQLBackup(BaseBackup):
|
||||
def __init__(self, source_config: dict, job_name: str):
|
||||
super().__init__(source_config, job_name)
|
||||
self.host: str = source_config["host"]
|
||||
self.port: int = int(source_config.get("port", 3306))
|
||||
self.user: str = source_config["user"]
|
||||
self.password: str = source_config.get("password", "")
|
||||
self.database: str = source_config["database"]
|
||||
self.extra_args: str = source_config.get("extra_args", "")
|
||||
|
||||
def metadata(self) -> BackupMetadata:
|
||||
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
safe_db = "".join(c if c.isalnum() or c in ("_", "-") else "_" for c in self.database)
|
||||
return BackupMetadata(
|
||||
suggested_filename=f"{self.job_name}-{safe_db}-{ts}.sql.gz",
|
||||
content_type="application/gzip",
|
||||
extra={"database": self.database, "host": self.host, "port": self.port},
|
||||
)
|
||||
|
||||
async def produce(self) -> AsyncIterator[bytes]:
|
||||
args = [
|
||||
"mysqldump",
|
||||
"-h", self.host,
|
||||
"-P", str(self.port),
|
||||
"-u", self.user,
|
||||
"--single-transaction",
|
||||
"--quick",
|
||||
"--routines",
|
||||
"--triggers",
|
||||
"--events",
|
||||
"--default-character-set=utf8mb4",
|
||||
]
|
||||
if self.extra_args:
|
||||
args.extend(shlex.split(self.extra_args))
|
||||
args.append(self.database)
|
||||
|
||||
log.info("Executing mysqldump for db=%s host=%s", self.database, self.host)
|
||||
|
||||
# mysqldump 直接写 stdout，我们 pipe 给 gzip 再 yield
|
||||
# 用 -p 紧贴密码避免出现在 ps 里（空密码时 mysqldump 会要求交互输入）
|
||||
if self.password:
|
||||
args_with_pwd = args.copy()
|
||||
# 找到 -u 后插入 -pXXX
|
||||
u_idx = args_with_pwd.index("-u")
|
||||
args_with_pwd.insert(u_idx + 2, f"-p{self.password}")
|
||||
else:
|
||||
args_with_pwd = args
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*args_with_pwd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
|
||||
assert proc.stdout is not None
|
||||
|
||||
# 在线程里跑 gzip.compress，因为它是阻塞的
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
async def gzip_stream() -> AsyncIterator[bytes]:
|
||||
chunk_buf = bytearray()
|
||||
async for chunk in proc.stdout: # type: ignore[union-attr]
|
||||
# gzip 接受 bytes，返回 bytes
|
||||
compressed = await loop.run_in_executor(None, gzip.compress, chunk)
|
||||
if compressed:
|
||||
yield compressed
|
||||
|
||||
try:
|
||||
async for piece in gzip_stream():
|
||||
yield piece
|
||||
finally:
|
||||
try:
|
||||
await proc.wait()
|
||||
except Exception:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
|
||||
if proc.returncode != 0:
|
||||
stderr = (await proc.stderr.read()).decode(errors="replace") if proc.stderr else ""
|
||||
raise RuntimeError(
|
||||
f"mysqldump failed (exit={proc.returncode}): {stderr.strip()[:2000]}"
|
||||
)
|
||||
@@ -0,0 +1,151 @@
|
||||
"""备份执行编排：解密配置 → 流式产出 → 流式上传 → 应用保留策略 → 写日志。"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.backup.base import BaseBackup
|
||||
from app.core.backup.directory import DirectoryBackup
|
||||
from app.core.backup.mysql import MySQLBackup
|
||||
from app.core.retention import apply_retention
|
||||
from app.core.storage.factory import create_storage
|
||||
from app.database import SessionLocal
|
||||
from app.models.job import Job
|
||||
from app.models.run import Run, RunLog
|
||||
from app.utils.crypto import decrypt_dict
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
def _make_backup(job: Job) -> BaseBackup:
|
||||
src = decrypt_dict(job.source_config_json)
|
||||
if job.type == "mysql":
|
||||
return MySQLBackup(src, job.name)
|
||||
if job.type == "directory":
|
||||
return DirectoryBackup(src, job.name)
|
||||
raise ValueError(f"Unsupported job type: {job.type}")
|
||||
|
||||
|
||||
async def run_backup(run_id: int, trigger: str = "manual") -> None:
|
||||
"""执行一次备份。所有副作用都在自己 new 的 Session 上完成。"""
|
||||
started = datetime.utcnow()
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
run: Run | None = db.get(Run, run_id)
|
||||
if not run:
|
||||
log.error("run_id=%s not found", run_id)
|
||||
return
|
||||
|
||||
job: Job | None = db.get(Job, run.job_id)
|
||||
if not job:
|
||||
run.status = "failed"
|
||||
run.error_message = f"Job id={run.job_id} disappeared"
|
||||
run.finished_at = datetime.utcnow()
|
||||
db.commit()
|
||||
return
|
||||
|
||||
log_buffer = io.StringIO()
|
||||
tee = _TeeLogger(log, log_buffer)
|
||||
|
||||
run.status = "running"
|
||||
run.trigger = trigger
|
||||
run.started_at = started
|
||||
db.commit()
|
||||
|
||||
tee.info("Backup started: run_id=%s job=%s trigger=%s", run_id, job.name, trigger)
|
||||
|
||||
try:
|
||||
backup = _make_backup(job)
|
||||
meta = backup.metadata()
|
||||
tee.info("Metadata: filename=%s content_type=%s", meta.suggested_filename, meta.content_type)
|
||||
|
||||
storage = create_storage(job.storage)
|
||||
ok, msg = await storage.test_connection()
|
||||
tee.info("Storage test: ok=%s msg=%s", ok, msg)
|
||||
if not ok:
|
||||
raise RuntimeError(f"Storage not reachable: {msg}")
|
||||
|
||||
# 关键路径：备份文件名以 {job_name}/{YYYY-MM-DD}/{filename} 组织
|
||||
date_prefix = datetime.utcnow().strftime("%Y-%m-%d")
|
||||
object_key = f"{job.name}/{date_prefix}/{meta.suggested_filename}"
|
||||
|
||||
tee.info("Streaming backup to storage key=%s ...", object_key)
|
||||
|
||||
async def _consume_with_progress():
|
||||
async for chunk in backup.produce():
|
||||
yield chunk
|
||||
|
||||
t0 = time.monotonic()
|
||||
size, sha = await storage.upload_stream(
|
||||
_consume_with_progress(),
|
||||
object_key,
|
||||
content_type=meta.content_type,
|
||||
)
|
||||
elapsed = int(time.monotonic() - t0)
|
||||
|
||||
run.artifact_path = object_key
|
||||
run.artifact_size = size
|
||||
run.artifact_checksum = sha
|
||||
run.duration_seconds = elapsed
|
||||
run.status = "success"
|
||||
run.finished_at = datetime.utcnow()
|
||||
tee.info(
|
||||
"Backup success: size=%s bytes sha256=%s elapsed=%ss",
|
||||
size, sha[:12], elapsed,
|
||||
)
|
||||
|
||||
# 应用保留策略
|
||||
try:
|
||||
removed = apply_retention(db, job)
|
||||
if removed:
|
||||
tee.info("Retention cleanup removed %s old runs", removed)
|
||||
except Exception as e:
|
||||
tee.warning("Retention cleanup failed: %s", e)
|
||||
|
||||
except Exception as e:
|
||||
log.exception("Backup run_id=%s failed", run_id)
|
||||
tee.error("Backup failed: %s: %s", type(e).__name__, e)
|
||||
run.status = "failed"
|
||||
run.error_message = f"{type(e).__name__}: {e}"[:2000]
|
||||
run.finished_at = datetime.utcnow()
|
||||
finally:
|
||||
log_text = log_buffer.getvalue()
|
||||
run.log_excerpt = log_text[-4000:] if len(log_text) > 4000 else log_text
|
||||
run_log = RunLog(run_id=run.id, content=log_text)
|
||||
db.add(run_log)
|
||||
db.commit()
|
||||
log.info("Backup run_id=%s finished status=%s", run_id, run.status)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
class _TeeLogger:
|
||||
"""同时写标准 logging 和内存 StringIO（用于存数据库）。"""
|
||||
|
||||
def __init__(self, real: logging.Logger, buf: io.StringIO):
|
||||
self._real = real
|
||||
self._buf = buf
|
||||
|
||||
def _emit(self, level: int, msg: str) -> None:
|
||||
ts = datetime.utcnow().strftime("%H:%M:%S")
|
||||
line = f"[{ts}] {msg}"
|
||||
self._real.log(level, line)
|
||||
self._buf.write(line + "\n")
|
||||
|
||||
def info(self, msg: str, *args) -> None:
|
||||
self._emit(logging.INFO, msg % args if args else msg)
|
||||
|
||||
def warning(self, msg: str, *args) -> None:
|
||||
self._emit(logging.WARNING, msg % args if args else msg)
|
||||
|
||||
def error(self, msg: str, *args) -> None:
|
||||
self._emit(logging.ERROR, msg % args if args else msg)
|
||||
@@ -0,0 +1,165 @@
|
||||
"""恢复执行器：把 backup artifact 还原到目标（MySQL 库 / 目录路径）。"""
|
||||
import asyncio
|
||||
import gzip
|
||||
import io
|
||||
import os
|
||||
import shutil
|
||||
import tarfile
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from app.core.storage.factory import create_storage
|
||||
from app.database import SessionLocal
|
||||
from app.models.job import Job
|
||||
from app.models.restore import RestoreHistory
|
||||
from app.models.run import Run, RunLog
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
async def execute_restore(restore_id: int) -> None:
|
||||
"""执行一次恢复。"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
rh: RestoreHistory | None = db.get(RestoreHistory, restore_id)
|
||||
if not rh:
|
||||
return
|
||||
|
||||
rh.status = "running"
|
||||
rh.started_at = datetime.utcnow()
|
||||
db.commit()
|
||||
|
||||
run: Run | None = db.get(Run, rh.run_id)
|
||||
if not run or not run.artifact_path:
|
||||
raise RuntimeError("源 run 或 artifact 不存在")
|
||||
job: Job | None = db.get(Job, run.job_id)
|
||||
if not job:
|
||||
raise RuntimeError("源 job 不存在")
|
||||
|
||||
storage = create_storage(job.storage)
|
||||
log_text = io.StringIO()
|
||||
|
||||
def logl(msg: str) -> None:
|
||||
line = f"[{datetime.utcnow():%H:%M:%S}] {msg}"
|
||||
log.info(line)
|
||||
log_text.write(line + "\n")
|
||||
|
||||
try:
|
||||
logl(f"Restoring run_id={run.id} -> {rh.target} (type={job.type})")
|
||||
|
||||
# 先下载到临时文件
|
||||
tmpdir = Path(tempfile.gettempdir()) / f"restore_{rh.id}_{os.getpid()}"
|
||||
tmpdir.mkdir(parents=True, exist_ok=True)
|
||||
local_tmp = tmpdir / Path(run.artifact_path).name
|
||||
|
||||
logl(f"Downloading {run.artifact_path} -> {local_tmp}")
|
||||
size = 0
|
||||
with open(local_tmp, "wb") as f:
|
||||
async for chunk in storage.download_stream(run.artifact_path):
|
||||
f.write(chunk)
|
||||
size += len(chunk)
|
||||
logl(f"Downloaded {size} bytes")
|
||||
|
||||
if job.type == "mysql":
|
||||
await _restore_mysql(local_tmp, rh.target, job, logl)
|
||||
elif job.type == "directory":
|
||||
_restore_directory(local_tmp, rh.target, logl)
|
||||
else:
|
||||
raise RuntimeError(f"Unsupported job type: {job.type}")
|
||||
|
||||
rh.status = "success"
|
||||
rh.finished_at = datetime.utcnow()
|
||||
logl("Restore success")
|
||||
except Exception as e:
|
||||
log.exception("Restore failed")
|
||||
logl(f"Restore failed: {type(e).__name__}: {e}")
|
||||
rh.status = "failed"
|
||||
rh.error_message = f"{type(e).__name__}: {e}"[:2000]
|
||||
rh.finished_at = datetime.utcnow()
|
||||
finally:
|
||||
# 写日志
|
||||
content = log_text.getvalue()
|
||||
db.add(RunLog(run_id=run.id, content=f"[restore #{rh.id}]\n{content}"))
|
||||
db.commit()
|
||||
try:
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def _restore_mysql(archive_path: Path, target_db: str, job: Job, logl) -> None:
|
||||
"""把 sql.gz 通过 gunzip | mysql 还原到 target_db。"""
|
||||
from app.utils.crypto import decrypt_dict
|
||||
|
||||
src = decrypt_dict(job.source_config_json)
|
||||
host = src["host"]
|
||||
port = int(src.get("port", 3306))
|
||||
user = src["user"]
|
||||
password = src.get("password", "")
|
||||
|
||||
logl(f"Decompressing and piping to mysql db={target_db} host={host}:{port}")
|
||||
# 解压
|
||||
decompressed = archive_path.with_suffix("") # .sql
|
||||
if archive_path.name.endswith(".gz"):
|
||||
with gzip.open(archive_path, "rb") as gz, open(decompressed, "wb") as out:
|
||||
shutil.copyfileobj(gz, out)
|
||||
sql_file = decompressed
|
||||
else:
|
||||
sql_file = archive_path
|
||||
|
||||
# 调用 mysql client
|
||||
args = ["mysql", "-h", host, "-P", str(port), "-u", user, target_db]
|
||||
if password:
|
||||
# 在 -u 后插入 -pPWD
|
||||
idx = args.index("-u")
|
||||
args.insert(idx + 2, f"-p{password}")
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*args,
|
||||
stdin=asyncio.subprocess.PIPE,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
assert proc.stdin is not None
|
||||
with open(sql_file, "rb") as f:
|
||||
while True:
|
||||
chunk = f.read(64 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
proc.stdin.write(chunk)
|
||||
await proc.stdin.drain()
|
||||
proc.stdin.close()
|
||||
rc = await proc.wait()
|
||||
if rc != 0:
|
||||
stderr = (await proc.stderr.read()).decode(errors="replace") if proc.stderr else ""
|
||||
raise RuntimeError(f"mysql restore failed (exit={rc}): {stderr[:2000]}")
|
||||
try:
|
||||
decompressed.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _restore_directory(archive_path: Path, target_dir: str, logl) -> None:
|
||||
"""解压 tar.gz 到 target_dir（必须为空或不存在）。"""
|
||||
target = Path(target_dir).resolve()
|
||||
if target.exists() and any(target.iterdir()):
|
||||
raise RuntimeError(f"目标目录非空: {target}")
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logl(f"Extracting tar.gz to {target}")
|
||||
if archive_path.name.endswith(".gz"):
|
||||
mode = "r:gz"
|
||||
else:
|
||||
mode = "r:"
|
||||
with tarfile.open(str(archive_path), mode) as tar:
|
||||
# 安全检查：拒绝路径逃逸
|
||||
for member in tar.getmembers():
|
||||
member_path = (target / member.name).resolve()
|
||||
if not str(member_path).startswith(str(target)):
|
||||
raise RuntimeError(f"Refusing to extract unsafe path: {member.name}")
|
||||
tar.extractall(path=str(target))
|
||||
@@ -0,0 +1,80 @@
|
||||
"""备份保留策略清理。"""
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.storage.factory import create_storage
|
||||
from app.models.job import Job
|
||||
from app.models.run import Run
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
def apply_retention(db: Session, job: Job) -> int:
|
||||
"""根据 job 的 retention_count / retention_days 清理旧备份。
|
||||
|
||||
返回删除的 run 数。
|
||||
"""
|
||||
# 取该 job 所有成功 runs，按 finished_at 倒序
|
||||
runs = (
|
||||
db.query(Run)
|
||||
.filter(Run.job_id == job.id, Run.status == "success")
|
||||
.order_by(Run.finished_at.desc())
|
||||
.all()
|
||||
)
|
||||
if not runs:
|
||||
return 0
|
||||
|
||||
keep_count = job.retention_count or 7
|
||||
keep_days = job.retention_days
|
||||
|
||||
to_keep_ids: set[int] = set()
|
||||
for i, r in enumerate(runs):
|
||||
keep_by_count = i < keep_count
|
||||
keep_by_days = True
|
||||
if keep_days and r.finished_at:
|
||||
threshold = datetime.utcnow() - timedelta(days=keep_days)
|
||||
if r.finished_at < threshold:
|
||||
keep_by_days = False
|
||||
if keep_by_count or keep_by_days:
|
||||
to_keep_ids.add(r.id)
|
||||
|
||||
to_delete = [r for r in runs if r.id not in to_keep_ids]
|
||||
if not to_delete:
|
||||
return 0
|
||||
|
||||
storage = create_storage(job.storage)
|
||||
removed = 0
|
||||
for r in to_delete:
|
||||
try:
|
||||
if r.artifact_path:
|
||||
# 用同步方法简单调用即可（local 是同步包装 async，但这里 aioboto3/aiofiles 都是异步；
|
||||
# 实际上 base storage 都是 async 方法，所以这里需要在同步上下文跑 event loop）
|
||||
pass
|
||||
except Exception:
|
||||
log.exception("Failed to delete artifact %s (db row still kept)", r.artifact_path)
|
||||
db.delete(r)
|
||||
removed += 1
|
||||
|
||||
db.commit()
|
||||
log.info("Retention cleanup for job=%s: deleted %s runs", job.name, removed)
|
||||
|
||||
# 真实删除 S3/local 文件（需要在 async 上下文）
|
||||
import asyncio
|
||||
try:
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
for r in to_delete:
|
||||
if r.artifact_path:
|
||||
try:
|
||||
loop.run_until_complete(storage.delete(r.artifact_path))
|
||||
except Exception:
|
||||
log.exception("Artifact delete failed: %s", r.artifact_path)
|
||||
finally:
|
||||
loop.close()
|
||||
except Exception:
|
||||
log.exception("Artifact cleanup loop error")
|
||||
|
||||
return removed
|
||||
@@ -0,0 +1,45 @@
|
||||
"""存储后端抽象接口。"""
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import AsyncIterator, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class StorageObject:
|
||||
key: str
|
||||
size: int
|
||||
last_modified: Optional[str] = None # ISO 8601
|
||||
|
||||
|
||||
class BaseStorage(ABC):
|
||||
"""所有存储后端实现的统一接口。"""
|
||||
|
||||
type: str = "base"
|
||||
|
||||
def __init__(self, config: dict):
|
||||
self.config = config
|
||||
|
||||
@abstractmethod
|
||||
async def test_connection(self) -> tuple[bool, str]:
|
||||
"""测试连通性。返回 (ok, message)。"""
|
||||
|
||||
@abstractmethod
|
||||
async def upload_stream(
|
||||
self,
|
||||
source: AsyncIterator[bytes],
|
||||
key: str,
|
||||
content_type: str = "application/octet-stream",
|
||||
) -> tuple[int, str]:
|
||||
"""流式上传。返回 (size_bytes, sha256_hex)。"""
|
||||
|
||||
@abstractmethod
|
||||
async def download_stream(self, key: str) -> AsyncIterator[bytes]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def delete(self, key: str) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def exists(self, key: str) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list(self, prefix: str = "") -> list[StorageObject]: ...
|
||||
@@ -0,0 +1,15 @@
|
||||
"""存储工厂：根据 storage 行创建对应实现。"""
|
||||
from app.core.storage.base import BaseStorage
|
||||
from app.core.storage.local import LocalStorage
|
||||
from app.core.storage.s3 import S3Storage
|
||||
from app.models.storage import Storage
|
||||
from app.utils.crypto import decrypt_dict
|
||||
|
||||
|
||||
def create_storage(row: Storage) -> BaseStorage:
|
||||
config = decrypt_dict(row.config_json)
|
||||
if row.type == "local":
|
||||
return LocalStorage(config)
|
||||
if row.type == "s3":
|
||||
return S3Storage(config)
|
||||
raise ValueError(f"Unsupported storage type: {row.type}")
|
||||
@@ -0,0 +1,98 @@
|
||||
"""本地文件系统存储。"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import AsyncIterator
|
||||
|
||||
import aiofiles
|
||||
|
||||
from app.core.storage.base import BaseStorage, StorageObject
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class LocalStorage(BaseStorage):
|
||||
type = "local"
|
||||
|
||||
def __init__(self, config: dict):
|
||||
super().__init__(config)
|
||||
self.root: str = os.path.abspath(config["path"])
|
||||
Path(self.root).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _resolve(self, key: str) -> Path:
|
||||
# 防 path traversal：禁止 ..
|
||||
if ".." in key.split("/"):
|
||||
raise ValueError(f"Invalid key (path traversal): {key}")
|
||||
return Path(self.root) / key
|
||||
|
||||
async def test_connection(self) -> tuple[bool, str]:
|
||||
try:
|
||||
Path(self.root).mkdir(parents=True, exist_ok=True)
|
||||
test_file = Path(self.root) / ".backup_system_test"
|
||||
async with aiofiles.open(test_file, "w") as f:
|
||||
await f.write("ok")
|
||||
# aiofiles 不暴露 os 命名空间，用线程池跑同步删除
|
||||
await asyncio.to_thread(os.remove, test_file)
|
||||
return True, f"OK: writable {self.root}"
|
||||
except Exception as e:
|
||||
return False, f"FAIL: {e}"
|
||||
|
||||
async def upload_stream(
|
||||
self,
|
||||
source: AsyncIterator[bytes],
|
||||
key: str,
|
||||
content_type: str = "application/octet-stream",
|
||||
) -> tuple[int, str]:
|
||||
path = self._resolve(key)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
sha = hashlib.sha256()
|
||||
size = 0
|
||||
async with aiofiles.open(path, "wb") as f:
|
||||
async for chunk in source:
|
||||
await f.write(chunk)
|
||||
sha.update(chunk)
|
||||
size += len(chunk)
|
||||
return size, sha.hexdigest()
|
||||
|
||||
async def download_stream(self, key: str) -> AsyncIterator[bytes]:
|
||||
path = self._resolve(key)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Not found: {key}")
|
||||
async with aiofiles.open(path, "rb") as f:
|
||||
while True:
|
||||
chunk = await f.read(64 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
|
||||
async def delete(self, key: str) -> None:
|
||||
path = self._resolve(key)
|
||||
try:
|
||||
path.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
async def exists(self, key: str) -> bool:
|
||||
return self._resolve(key).exists()
|
||||
|
||||
async def list(self, prefix: str = "") -> list[StorageObject]:
|
||||
base = Path(self.root) / prefix if prefix else Path(self.root)
|
||||
if not base.exists():
|
||||
return []
|
||||
out: list[StorageObject] = []
|
||||
for p in base.rglob("*"):
|
||||
if p.is_file():
|
||||
rel = p.relative_to(Path(self.root)).as_posix()
|
||||
st = p.stat()
|
||||
out.append(
|
||||
StorageObject(
|
||||
key=rel,
|
||||
size=st.st_size,
|
||||
last_modified=datetime.fromtimestamp(st.st_mtime).isoformat(),
|
||||
)
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,176 @@
|
||||
"""S3 兼容对象存储（支持 MinIO / Ceph / AWS S3）。"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from typing import Any, AsyncIterator
|
||||
|
||||
import aioboto3
|
||||
|
||||
from app.core.storage.base import BaseStorage, StorageObject
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class S3Storage(BaseStorage):
|
||||
type = "s3"
|
||||
|
||||
def __init__(self, config: dict):
|
||||
super().__init__(config)
|
||||
self.endpoint_url: str | None = config.get("endpoint_url") or None
|
||||
self.region: str = config.get("region") or "us-east-1"
|
||||
self.bucket: str = config["bucket"]
|
||||
self.access_key: str = config["access_key"]
|
||||
self.secret_key: str = config["secret_key"]
|
||||
self.use_ssl: bool = bool(config.get("use_ssl", True))
|
||||
self.path_prefix: str = config.get("path_prefix", "").strip("/")
|
||||
self.addressing_style: str = config.get("addressing_style", "auto")
|
||||
|
||||
@asynccontextmanager
|
||||
async def _client(self):
|
||||
session = aioboto3.Session()
|
||||
async with session.client(
|
||||
"s3",
|
||||
endpoint_url=self.endpoint_url,
|
||||
region_name=self.region,
|
||||
aws_access_key_id=self.access_key,
|
||||
aws_secret_access_key=self.secret_key,
|
||||
use_ssl=self.use_ssl,
|
||||
config=__import__("botocore.config", fromlist=["Config"]).Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": self.addressing_style},
|
||||
),
|
||||
) as client:
|
||||
yield client
|
||||
|
||||
def _full_key(self, key: str) -> str:
|
||||
key = key.lstrip("/")
|
||||
if self.path_prefix:
|
||||
return f"{self.path_prefix}/{key}"
|
||||
return key
|
||||
|
||||
async def test_connection(self) -> tuple[bool, str]:
|
||||
try:
|
||||
async with self._client() as client:
|
||||
await client.head_bucket(Bucket=self.bucket)
|
||||
return True, f"OK: bucket={self.bucket}"
|
||||
except Exception as e:
|
||||
return False, f"FAIL: {type(e).__name__}: {e}"
|
||||
|
||||
async def upload_stream(
|
||||
self,
|
||||
source: AsyncIterator[bytes],
|
||||
key: str,
|
||||
content_type: str = "application/octet-stream",
|
||||
) -> tuple[int, str]:
|
||||
sha = hashlib.sha256()
|
||||
size = 0
|
||||
full_key = self._full_key(key)
|
||||
|
||||
async with self._client() as client:
|
||||
# 用 UploadPart + 内存缓冲整个对象再 put_object（对中小备份足够）
|
||||
# 大文件可改为 multipart；这里用 MultipartUploader 模式，兼顾大文件
|
||||
from io import BytesIO
|
||||
|
||||
buffer = BytesIO()
|
||||
async for chunk in source:
|
||||
buffer.write(chunk)
|
||||
sha.update(chunk)
|
||||
size += len(chunk)
|
||||
|
||||
# 5MB 阈值：超过用 multipart
|
||||
PART_SIZE = 5 * 1024 * 1024
|
||||
if size <= PART_SIZE:
|
||||
buffer.seek(0)
|
||||
await client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=full_key,
|
||||
Body=buffer.getvalue(),
|
||||
ContentType=content_type,
|
||||
)
|
||||
else:
|
||||
# multipart
|
||||
mpu = await client.create_multipart_upload(
|
||||
Bucket=self.bucket, Key=full_key, ContentType=content_type
|
||||
)
|
||||
parts: list[dict[str, Any]] = []
|
||||
try:
|
||||
buffer.seek(0)
|
||||
part_number = 1
|
||||
while True:
|
||||
data = buffer.read(PART_SIZE)
|
||||
if not data:
|
||||
break
|
||||
resp = await client.upload_part(
|
||||
Bucket=self.bucket,
|
||||
Key=full_key,
|
||||
PartNumber=part_number,
|
||||
UploadId=mpu["UploadId"],
|
||||
Body=data,
|
||||
)
|
||||
parts.append({"PartNumber": part_number, "ETag": resp["ETag"]})
|
||||
part_number += 1
|
||||
await client.complete_multipart_upload(
|
||||
Bucket=self.bucket,
|
||||
Key=full_key,
|
||||
UploadId=mpu["UploadId"],
|
||||
MultipartUpload={"Parts": parts},
|
||||
)
|
||||
except Exception:
|
||||
await client.abort_multipart_upload(
|
||||
Bucket=self.bucket, Key=full_key, UploadId=mpu["UploadId"]
|
||||
)
|
||||
raise
|
||||
|
||||
return size, sha.hexdigest()
|
||||
|
||||
async def download_stream(self, key: str) -> AsyncIterator[bytes]:
|
||||
full_key = self._full_key(key)
|
||||
async with self._client() as client:
|
||||
obj = await client.get_object(Bucket=self.bucket, Key=full_key)
|
||||
body = obj["Body"]
|
||||
try:
|
||||
while True:
|
||||
chunk = await asyncio.get_event_loop().run_in_executor(
|
||||
None, body.read, 64 * 1024
|
||||
)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
async def delete(self, key: str) -> None:
|
||||
full_key = self._full_key(key)
|
||||
async with self._client() as client:
|
||||
await client.delete_object(Bucket=self.bucket, Key=full_key)
|
||||
|
||||
async def exists(self, key: str) -> bool:
|
||||
full_key = self._full_key(key)
|
||||
async with self._client() as client:
|
||||
try:
|
||||
await client.head_object(Bucket=self.bucket, Key=full_key)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def list(self, prefix: str = "") -> list[StorageObject]:
|
||||
full_prefix = self._full_key(prefix)
|
||||
out: list[StorageObject] = []
|
||||
async with self._client() as client:
|
||||
paginator = client.get_paginator("list_objects_v2")
|
||||
async for page in paginator.paginate(Bucket=self.bucket, Prefix=full_prefix):
|
||||
for obj in page.get("Contents", []):
|
||||
key = obj["Key"]
|
||||
if self.path_prefix and key.startswith(self.path_prefix + "/"):
|
||||
key = key[len(self.path_prefix) + 1 :]
|
||||
last_modified = obj.get("LastModified")
|
||||
out.append(
|
||||
StorageObject(
|
||||
key=key,
|
||||
size=int(obj.get("Size", 0)),
|
||||
last_modified=last_modified.isoformat() if last_modified else None,
|
||||
)
|
||||
)
|
||||
return out
|
||||
Reference in New Issue
Block a user