feat: 数据备份管理系统 v0.1.0
带 Web 操作界面的备份系统,支持: - 数据源:MySQL 数据库(mysqldump)、服务器目录(tar.gz) - 存储目标:本地目录、S3 兼容对象存储(MinIO/Ceph/AWS S3) - 触发方式:手动 + Cron 定时调度(APScheduler) - 备份还原:MySQL 库、目录 - JWT 登录认证 + Fernet 字段加密 - 保留策略:按数量 + 按天数双重清理 技术栈:FastAPI + SQLAlchemy 2 + APScheduler + aioboto3; 前端 Vite + React 18 + TypeScript + Ant Design 5 + Zustand。 Docker Compose 一键起。 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,97 @@
|
||||
"""MySQL 备份:通过 mysqldump 子进程流式输出。"""
|
||||
import asyncio
|
||||
import gzip
|
||||
import shlex
|
||||
from datetime import datetime
|
||||
from typing import AsyncIterator
|
||||
|
||||
from app.core.backup.base import BackupMetadata, BaseBackup
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
CHUNK_SIZE = 64 * 1024
|
||||
|
||||
|
||||
class MySQLBackup(BaseBackup):
|
||||
def __init__(self, source_config: dict, job_name: str):
|
||||
super().__init__(source_config, job_name)
|
||||
self.host: str = source_config["host"]
|
||||
self.port: int = int(source_config.get("port", 3306))
|
||||
self.user: str = source_config["user"]
|
||||
self.password: str = source_config.get("password", "")
|
||||
self.database: str = source_config["database"]
|
||||
self.extra_args: str = source_config.get("extra_args", "")
|
||||
|
||||
def metadata(self) -> BackupMetadata:
|
||||
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
safe_db = "".join(c if c.isalnum() or c in ("_", "-") else "_" for c in self.database)
|
||||
return BackupMetadata(
|
||||
suggested_filename=f"{self.job_name}-{safe_db}-{ts}.sql.gz",
|
||||
content_type="application/gzip",
|
||||
extra={"database": self.database, "host": self.host, "port": self.port},
|
||||
)
|
||||
|
||||
async def produce(self) -> AsyncIterator[bytes]:
|
||||
args = [
|
||||
"mysqldump",
|
||||
"-h", self.host,
|
||||
"-P", str(self.port),
|
||||
"-u", self.user,
|
||||
"--single-transaction",
|
||||
"--quick",
|
||||
"--routines",
|
||||
"--triggers",
|
||||
"--events",
|
||||
"--default-character-set=utf8mb4",
|
||||
]
|
||||
if self.extra_args:
|
||||
args.extend(shlex.split(self.extra_args))
|
||||
args.append(self.database)
|
||||
|
||||
log.info("Executing mysqldump for db=%s host=%s", self.database, self.host)
|
||||
|
||||
# mysqldump 直接写 stdout,我们 pipe 给 gzip 再 yield
|
||||
# 用 -p 紧贴密码避免出现在 ps 里(空密码时 mysqldump 会要求交互输入)
|
||||
if self.password:
|
||||
args_with_pwd = args.copy()
|
||||
# 找到 -u 后插入 -pXXX
|
||||
u_idx = args_with_pwd.index("-u")
|
||||
args_with_pwd.insert(u_idx + 2, f"-p{self.password}")
|
||||
else:
|
||||
args_with_pwd = args
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*args_with_pwd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
|
||||
assert proc.stdout is not None
|
||||
|
||||
# 在线程里跑 gzip.compress,因为它是阻塞的
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
async def gzip_stream() -> AsyncIterator[bytes]:
|
||||
chunk_buf = bytearray()
|
||||
async for chunk in proc.stdout: # type: ignore[union-attr]
|
||||
# gzip 接受 bytes,返回 bytes
|
||||
compressed = await loop.run_in_executor(None, gzip.compress, chunk)
|
||||
if compressed:
|
||||
yield compressed
|
||||
|
||||
try:
|
||||
async for piece in gzip_stream():
|
||||
yield piece
|
||||
finally:
|
||||
try:
|
||||
await proc.wait()
|
||||
except Exception:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
|
||||
if proc.returncode != 0:
|
||||
stderr = (await proc.stderr.read()).decode(errors="replace") if proc.stderr else ""
|
||||
raise RuntimeError(
|
||||
f"mysqldump failed (exit={proc.returncode}): {stderr.strip()[:2000]}"
|
||||
)
|
||||
Reference in New Issue
Block a user