feat: 数据备份管理系统 v0.1.0

带 Web 操作界面的备份系统，支持： - 数据源：MySQL 数据库（mysqldump）、服务器目录（tar.gz） - 存储目标：本地目录、S3 兼容对象存储（MinIO/Ceph/AWS S3） - 触发方式：手动 + Cron 定时调度（APScheduler） - 备份还原：MySQL 库、目录 - JWT 登录认证 + Fernet 字段加密 - 保留策略：按数量 + 按天数双重清理技术栈：FastAPI + SQLAlchemy 2 + APScheduler + aioboto3；前端 Vite + React 18 + TypeScript + Ant Design 5 + Zustand。 Docker Compose 一键起。 Co-Authored-By: Claude <noreply@anthropic.com>
2026-06-22 18:27:41 +08:00
commit 4e3a4b4602
92 changed files with 5290 additions and 0 deletions
@@ -0,0 +1,45 @@
+"""存储后端抽象接口。"""
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import AsyncIterator, Optional
+
+
+@dataclass
+class StorageObject:
+    key: str
+    size: int
+    last_modified: Optional[str] = None  # ISO 8601
+
+
+class BaseStorage(ABC):
+    """所有存储后端实现的统一接口。"""
+
+    type: str = "base"
+
+    def __init__(self, config: dict):
+        self.config = config
+
+    @abstractmethod
+    async def test_connection(self) -> tuple[bool, str]:
+        """测试连通性。返回 (ok, message)。"""
+
+    @abstractmethod
+    async def upload_stream(
+        self,
+        source: AsyncIterator[bytes],
+        key: str,
+        content_type: str = "application/octet-stream",
+    ) -> tuple[int, str]:
+        """流式上传。返回 (size_bytes, sha256_hex)。"""
+
+    @abstractmethod
+    async def download_stream(self, key: str) -> AsyncIterator[bytes]: ...
+
+    @abstractmethod
+    async def delete(self, key: str) -> None: ...
+
+    @abstractmethod
+    async def exists(self, key: str) -> bool: ...
+
+    @abstractmethod
+    async def list(self, prefix: str = "") -> list[StorageObject]: ...
@@ -0,0 +1,15 @@
+"""存储工厂：根据 storage 行创建对应实现。"""
+from app.core.storage.base import BaseStorage
+from app.core.storage.local import LocalStorage
+from app.core.storage.s3 import S3Storage
+from app.models.storage import Storage
+from app.utils.crypto import decrypt_dict
+
+
+def create_storage(row: Storage) -> BaseStorage:
+    config = decrypt_dict(row.config_json)
+    if row.type == "local":
+        return LocalStorage(config)
+    if row.type == "s3":
+        return S3Storage(config)
+    raise ValueError(f"Unsupported storage type: {row.type}")
@@ -0,0 +1,98 @@
+"""本地文件系统存储。"""
+import asyncio
+import hashlib
+import os
+import shutil
+from datetime import datetime
+from pathlib import Path
+from typing import AsyncIterator
+
+import aiofiles
+
+from app.core.storage.base import BaseStorage, StorageObject
+from app.utils.logging import get_logger
+
+log = get_logger(__name__)
+
+
+class LocalStorage(BaseStorage):
+    type = "local"
+
+    def __init__(self, config: dict):
+        super().__init__(config)
+        self.root: str = os.path.abspath(config["path"])
+        Path(self.root).mkdir(parents=True, exist_ok=True)
+
+    def _resolve(self, key: str) -> Path:
+        # 防 path traversal：禁止 ..
+        if ".." in key.split("/"):
+            raise ValueError(f"Invalid key (path traversal): {key}")
+        return Path(self.root) / key
+
+    async def test_connection(self) -> tuple[bool, str]:
+        try:
+            Path(self.root).mkdir(parents=True, exist_ok=True)
+            test_file = Path(self.root) / ".backup_system_test"
+            async with aiofiles.open(test_file, "w") as f:
+                await f.write("ok")
+            # aiofiles 不暴露 os 命名空间，用线程池跑同步删除
+            await asyncio.to_thread(os.remove, test_file)
+            return True, f"OK: writable {self.root}"
+        except Exception as e:
+            return False, f"FAIL: {e}"
+
+    async def upload_stream(
+        self,
+        source: AsyncIterator[bytes],
+        key: str,
+        content_type: str = "application/octet-stream",
+    ) -> tuple[int, str]:
+        path = self._resolve(key)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        sha = hashlib.sha256()
+        size = 0
+        async with aiofiles.open(path, "wb") as f:
+            async for chunk in source:
+                await f.write(chunk)
+                sha.update(chunk)
+                size += len(chunk)
+        return size, sha.hexdigest()
+
+    async def download_stream(self, key: str) -> AsyncIterator[bytes]:
+        path = self._resolve(key)
+        if not path.exists():
+            raise FileNotFoundError(f"Not found: {key}")
+        async with aiofiles.open(path, "rb") as f:
+            while True:
+                chunk = await f.read(64 * 1024)
+                if not chunk:
+                    break
+                yield chunk
+
+    async def delete(self, key: str) -> None:
+        path = self._resolve(key)
+        try:
+            path.unlink()
+        except FileNotFoundError:
+            pass
+
+    async def exists(self, key: str) -> bool:
+        return self._resolve(key).exists()
+
+    async def list(self, prefix: str = "") -> list[StorageObject]:
+        base = Path(self.root) / prefix if prefix else Path(self.root)
+        if not base.exists():
+            return []
+        out: list[StorageObject] = []
+        for p in base.rglob("*"):
+            if p.is_file():
+                rel = p.relative_to(Path(self.root)).as_posix()
+                st = p.stat()
+                out.append(
+                    StorageObject(
+                        key=rel,
+                        size=st.st_size,
+                        last_modified=datetime.fromtimestamp(st.st_mtime).isoformat(),
+                    )
+                )
+        return out
@@ -0,0 +1,176 @@
+"""S3 兼容对象存储（支持 MinIO / Ceph / AWS S3）。"""
+import asyncio
+import hashlib
+from contextlib import asynccontextmanager
+from datetime import datetime
+from typing import Any, AsyncIterator
+
+import aioboto3
+
+from app.core.storage.base import BaseStorage, StorageObject
+from app.utils.logging import get_logger
+
+log = get_logger(__name__)
+
+
+class S3Storage(BaseStorage):
+    type = "s3"
+
+    def __init__(self, config: dict):
+        super().__init__(config)
+        self.endpoint_url: str | None = config.get("endpoint_url") or None
+        self.region: str = config.get("region") or "us-east-1"
+        self.bucket: str = config["bucket"]
+        self.access_key: str = config["access_key"]
+        self.secret_key: str = config["secret_key"]
+        self.use_ssl: bool = bool(config.get("use_ssl", True))
+        self.path_prefix: str = config.get("path_prefix", "").strip("/")
+        self.addressing_style: str = config.get("addressing_style", "auto")
+
+    @asynccontextmanager
+    async def _client(self):
+        session = aioboto3.Session()
+        async with session.client(
+            "s3",
+            endpoint_url=self.endpoint_url,
+            region_name=self.region,
+            aws_access_key_id=self.access_key,
+            aws_secret_access_key=self.secret_key,
+            use_ssl=self.use_ssl,
+            config=__import__("botocore.config", fromlist=["Config"]).Config(
+                signature_version="s3v4",
+                s3={"addressing_style": self.addressing_style},
+            ),
+        ) as client:
+            yield client
+
+    def _full_key(self, key: str) -> str:
+        key = key.lstrip("/")
+        if self.path_prefix:
+            return f"{self.path_prefix}/{key}"
+        return key
+
+    async def test_connection(self) -> tuple[bool, str]:
+        try:
+            async with self._client() as client:
+                await client.head_bucket(Bucket=self.bucket)
+                return True, f"OK: bucket={self.bucket}"
+        except Exception as e:
+            return False, f"FAIL: {type(e).__name__}: {e}"
+
+    async def upload_stream(
+        self,
+        source: AsyncIterator[bytes],
+        key: str,
+        content_type: str = "application/octet-stream",
+    ) -> tuple[int, str]:
+        sha = hashlib.sha256()
+        size = 0
+        full_key = self._full_key(key)
+
+        async with self._client() as client:
+            # 用 UploadPart + 内存缓冲整个对象再 put_object（对中小备份足够）
+            # 大文件可改为 multipart；这里用 MultipartUploader 模式，兼顾大文件
+            from io import BytesIO
+
+            buffer = BytesIO()
+            async for chunk in source:
+                buffer.write(chunk)
+                sha.update(chunk)
+                size += len(chunk)
+
+            # 5MB 阈值：超过用 multipart
+            PART_SIZE = 5 * 1024 * 1024
+            if size <= PART_SIZE:
+                buffer.seek(0)
+                await client.put_object(
+                    Bucket=self.bucket,
+                    Key=full_key,
+                    Body=buffer.getvalue(),
+                    ContentType=content_type,
+                )
+            else:
+                # multipart
+                mpu = await client.create_multipart_upload(
+                    Bucket=self.bucket, Key=full_key, ContentType=content_type
+                )
+                parts: list[dict[str, Any]] = []
+                try:
+                    buffer.seek(0)
+                    part_number = 1
+                    while True:
+                        data = buffer.read(PART_SIZE)
+                        if not data:
+                            break
+                        resp = await client.upload_part(
+                            Bucket=self.bucket,
+                            Key=full_key,
+                            PartNumber=part_number,
+                            UploadId=mpu["UploadId"],
+                            Body=data,
+                        )
+                        parts.append({"PartNumber": part_number, "ETag": resp["ETag"]})
+                        part_number += 1
+                    await client.complete_multipart_upload(
+                        Bucket=self.bucket,
+                        Key=full_key,
+                        UploadId=mpu["UploadId"],
+                        MultipartUpload={"Parts": parts},
+                    )
+                except Exception:
+                    await client.abort_multipart_upload(
+                        Bucket=self.bucket, Key=full_key, UploadId=mpu["UploadId"]
+                    )
+                    raise
+
+        return size, sha.hexdigest()
+
+    async def download_stream(self, key: str) -> AsyncIterator[bytes]:
+        full_key = self._full_key(key)
+        async with self._client() as client:
+            obj = await client.get_object(Bucket=self.bucket, Key=full_key)
+            body = obj["Body"]
+            try:
+                while True:
+                    chunk = await asyncio.get_event_loop().run_in_executor(
+                        None, body.read, 64 * 1024
+                    )
+                    if not chunk:
+                        break
+                    yield chunk
+            finally:
+                body.close()
+
+    async def delete(self, key: str) -> None:
+        full_key = self._full_key(key)
+        async with self._client() as client:
+            await client.delete_object(Bucket=self.bucket, Key=full_key)
+
+    async def exists(self, key: str) -> bool:
+        full_key = self._full_key(key)
+        async with self._client() as client:
+            try:
+                await client.head_object(Bucket=self.bucket, Key=full_key)
+                return True
+            except Exception:
+                return False
+
+    async def list(self, prefix: str = "") -> list[StorageObject]:
+        full_prefix = self._full_key(prefix)
+        out: list[StorageObject] = []
+        async with self._client() as client:
+            paginator = client.get_paginator("list_objects_v2")
+            async for page in paginator.paginate(Bucket=self.bucket, Prefix=full_prefix):
+                for obj in page.get("Contents", []):
+                    key = obj["Key"]
+                    if self.path_prefix and key.startswith(self.path_prefix + "/"):
+                        key = key[len(self.path_prefix) + 1 :]
+                    last_modified = obj.get("LastModified")
+                    out.append(
+                        StorageObject(
+                            key=key,
+                            size=int(obj.get("Size", 0)),
+                            last_modified=last_modified.isoformat() if last_modified else None,
+                        )
+                    )
+        return out