feat: 数据备份管理系统 v0.1.0
带 Web 操作界面的备份系统,支持: - 数据源:MySQL 数据库(mysqldump)、服务器目录(tar.gz) - 存储目标:本地目录、S3 兼容对象存储(MinIO/Ceph/AWS S3) - 触发方式:手动 + Cron 定时调度(APScheduler) - 备份还原:MySQL 库、目录 - JWT 登录认证 + Fernet 字段加密 - 保留策略:按数量 + 按天数双重清理 技术栈:FastAPI + SQLAlchemy 2 + APScheduler + aioboto3; 前端 Vite + React 18 + TypeScript + Ant Design 5 + Zustand。 Docker Compose 一键起。 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,9 @@
|
||||
# 同根目录 .env,这里只是文档。实际加载根目录 .env。
|
||||
SECRET_KEY=
|
||||
FERNET_KEY=
|
||||
INITIAL_ADMIN_PASSWORD=ChangeMe123!
|
||||
JWT_EXPIRE_MINUTES=1440
|
||||
DATABASE_URL=sqlite:///./data/backup.db
|
||||
STORAGE_LOCAL_ROOT=./data/backups
|
||||
LOG_LEVEL=INFO
|
||||
CORS_ORIGINS=http://localhost:5173,http://localhost:8000
|
||||
@@ -0,0 +1,31 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
|
||||
# 系统依赖:curl 用于健康检查,default-mysql-client 提供 mysqldump,tar 已内置
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
default-mysql-client \
|
||||
tar \
|
||||
gzip \
|
||||
tzdata \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 先安装依赖以利用 Docker 缓存
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
# 数据、日志、备份目录
|
||||
RUN mkdir -p /app/data/backups /app/logs
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
# alembic 升级 + 启动 uvicorn
|
||||
CMD ["sh", "-c", "alembic upgrade head && uvicorn app.main:app --host 0.0.0.0 --port 8000"]
|
||||
@@ -0,0 +1,43 @@
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = .
|
||||
sqlalchemy.url = sqlite:///./data/backup.db
|
||||
file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
|
||||
timezone = UTC
|
||||
truncate_slug_length = 40
|
||||
|
||||
[post_write_hooks]
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Alembic env.py:从 app.config 读取 DATABASE_URL,使用 Base.metadata。"""
|
||||
import os
|
||||
import sys
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
# 让 alembic 能 import app.*
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from app.config import get_settings # noqa: E402
|
||||
from app.database import Base # noqa: E402
|
||||
import app.models # noqa: F401, E402 # 触发所有模型注册
|
||||
|
||||
config = context.config
|
||||
|
||||
# 用 settings.DATABASE_URL 覆盖 alembic.ini 中的 url
|
||||
config.set_main_option("sqlalchemy.url", get_settings().DATABASE_URL)
|
||||
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
render_as_batch=True, # SQLite 兼容
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
render_as_batch=True, # SQLite 兼容
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,22 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
revision = ${repr(up_revision)}
|
||||
down_revision = ${repr(down_revision)}
|
||||
branch_labels = ${repr(branch_labels)}
|
||||
depends_on = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -0,0 +1,115 @@
|
||||
"""initial schema
|
||||
|
||||
Revision ID: 0001_initial
|
||||
Revises:
|
||||
Create Date: 2026-06-22 00:00:00
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "0001_initial"
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"users",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column("username", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("password_hash", sa.String(255), nullable=False),
|
||||
sa.Column("is_admin", sa.Boolean, nullable=False, server_default=sa.text("0")),
|
||||
sa.Column("is_active", sa.Boolean, nullable=False, server_default=sa.text("1")),
|
||||
sa.Column("created_at", sa.DateTime, nullable=False, server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index("ix_users_username", "users", ["username"], unique=True)
|
||||
|
||||
op.create_table(
|
||||
"storages",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column("name", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("type", sa.String(16), nullable=False),
|
||||
sa.Column("config_json", sa.Text, nullable=False),
|
||||
sa.Column("is_default", sa.Boolean, nullable=False, server_default=sa.text("0")),
|
||||
sa.Column("created_at", sa.DateTime, nullable=False, server_default=sa.func.now()),
|
||||
sa.Column("updated_at", sa.DateTime, nullable=False, server_default=sa.func.now()),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"jobs",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column("name", sa.String(64), nullable=False, unique=True),
|
||||
sa.Column("type", sa.String(16), nullable=False),
|
||||
sa.Column("source_config_json", sa.Text, nullable=False),
|
||||
sa.Column("storage_id", sa.Integer, sa.ForeignKey("storages.id", ondelete="RESTRICT"), nullable=False),
|
||||
sa.Column("cron_expression", sa.String(64), nullable=True),
|
||||
sa.Column("enabled", sa.Boolean, nullable=False, server_default=sa.text("1")),
|
||||
sa.Column("retention_count", sa.Integer, nullable=False, server_default=sa.text("7")),
|
||||
sa.Column("retention_days", sa.Integer, nullable=True),
|
||||
sa.Column("description", sa.String(255), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime, nullable=False, server_default=sa.func.now()),
|
||||
sa.Column("updated_at", sa.DateTime, nullable=False, server_default=sa.func.now()),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"runs",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column("job_id", sa.Integer, sa.ForeignKey("jobs.id", ondelete="CASCADE"), nullable=False),
|
||||
sa.Column("status", sa.String(16), nullable=False, server_default="pending"),
|
||||
sa.Column("trigger", sa.String(16), nullable=False, server_default="manual"),
|
||||
sa.Column("started_at", sa.DateTime, nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime, nullable=True),
|
||||
sa.Column("duration_seconds", sa.Integer, nullable=True),
|
||||
sa.Column("artifact_path", sa.String(512), nullable=True),
|
||||
sa.Column("artifact_size", sa.BigInteger, nullable=True),
|
||||
sa.Column("artifact_checksum", sa.String(64), nullable=True),
|
||||
sa.Column("error_message", sa.Text, nullable=True),
|
||||
sa.Column("log_excerpt", sa.Text, nullable=True),
|
||||
sa.Column("created_at", sa.DateTime, nullable=False, server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index("ix_runs_job_id", "runs", ["job_id"])
|
||||
|
||||
op.create_table(
|
||||
"run_logs",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column("run_id", sa.Integer, sa.ForeignKey("runs.id", ondelete="CASCADE"), nullable=False),
|
||||
sa.Column("content", sa.Text, nullable=False),
|
||||
sa.Column("created_at", sa.DateTime, nullable=False, server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index("ix_run_logs_run_id", "run_logs", ["run_id"])
|
||||
|
||||
op.create_table(
|
||||
"settings",
|
||||
sa.Column("key", sa.String(64), primary_key=True),
|
||||
sa.Column("value", sa.Text, nullable=False, server_default=""),
|
||||
sa.Column("updated_at", sa.DateTime, nullable=False, server_default=sa.func.now()),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"restore_history",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column("run_id", sa.Integer, sa.ForeignKey("runs.id", ondelete="CASCADE"), nullable=False),
|
||||
sa.Column("target", sa.String(512), nullable=False),
|
||||
sa.Column("status", sa.String(16), nullable=False, server_default="pending"),
|
||||
sa.Column("started_at", sa.DateTime, nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime, nullable=True),
|
||||
sa.Column("error_message", sa.Text, nullable=True),
|
||||
sa.Column("created_at", sa.DateTime, nullable=False, server_default=sa.func.now()),
|
||||
)
|
||||
op.create_index("ix_restore_history_run_id", "restore_history", ["run_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_restore_history_run_id", table_name="restore_history")
|
||||
op.drop_table("restore_history")
|
||||
op.drop_table("settings")
|
||||
op.drop_index("ix_run_logs_run_id", table_name="run_logs")
|
||||
op.drop_table("run_logs")
|
||||
op.drop_index("ix_runs_job_id", table_name="runs")
|
||||
op.drop_table("runs")
|
||||
op.drop_table("jobs")
|
||||
op.drop_table("storages")
|
||||
op.drop_index("ix_users_username", table_name="users")
|
||||
op.drop_table("users")
|
||||
@@ -0,0 +1,39 @@
|
||||
"""认证 API。"""
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
|
||||
from app.config import get_settings
|
||||
from app.deps import CurrentUser, DBSession
|
||||
from app.models.user import User
|
||||
from app.schemas.auth import (
|
||||
ChangePasswordRequest,
|
||||
LoginRequest,
|
||||
MessageResponse,
|
||||
TokenResponse,
|
||||
UserOut,
|
||||
)
|
||||
from app.security import create_access_token, hash_password, verify_password
|
||||
|
||||
router = APIRouter(prefix="/auth", tags=["auth"])
|
||||
|
||||
|
||||
@router.post("/login", response_model=TokenResponse)
|
||||
def login(payload: LoginRequest, db: DBSession) -> TokenResponse:
|
||||
user = db.query(User).filter(User.username == payload.username).first()
|
||||
if not user or not user.is_active or not verify_password(payload.password, user.password_hash):
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid credentials")
|
||||
token = create_access_token(user.username, extra={"uid": user.id, "adm": user.is_admin})
|
||||
return TokenResponse(access_token=token, expires_in=get_settings().JWT_EXPIRE_MINUTES * 60)
|
||||
|
||||
|
||||
@router.get("/me", response_model=UserOut)
|
||||
def me(user: CurrentUser) -> User:
|
||||
return user
|
||||
|
||||
|
||||
@router.post("/change-password", response_model=MessageResponse)
|
||||
def change_password(payload: ChangePasswordRequest, user: CurrentUser, db: DBSession) -> MessageResponse:
|
||||
if not verify_password(payload.old_password, user.password_hash):
|
||||
raise HTTPException(status_code=400, detail="原密码错误")
|
||||
user.password_hash = hash_password(payload.new_password)
|
||||
db.commit()
|
||||
return MessageResponse(message="Password updated")
|
||||
@@ -0,0 +1,159 @@
|
||||
"""备份任务 CRUD API。"""
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, HTTPException
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.core.executor import run_backup
|
||||
from app.deps import AdminUser, CurrentUser, DBSession
|
||||
from app.models.job import Job
|
||||
from app.models.run import Run
|
||||
from app.models.storage import Storage
|
||||
from app.schemas.job import (
|
||||
DirectorySourceConfig,
|
||||
JobCreate,
|
||||
JobOut,
|
||||
JobRunResponse,
|
||||
JobUpdate,
|
||||
JobWithLastRun,
|
||||
MySQLSourceConfig,
|
||||
RunSummary,
|
||||
)
|
||||
from app.scheduler.scheduler import remove_job, reschedule_job
|
||||
from app.utils.crypto import encrypt_dict
|
||||
|
||||
router = APIRouter(prefix="/jobs", tags=["jobs"])
|
||||
|
||||
|
||||
def _validate_source(type_: str, cfg: dict) -> dict:
|
||||
if type_ == "mysql":
|
||||
return MySQLSourceConfig(**cfg).model_dump()
|
||||
if type_ == "directory":
|
||||
return DirectorySourceConfig(**cfg).model_dump()
|
||||
raise HTTPException(status_code=400, detail=f"Unsupported job type: {type_}")
|
||||
|
||||
|
||||
def _job_to_with_last_run(job: Job, db) -> JobWithLastRun:
|
||||
last = (
|
||||
db.query(Run)
|
||||
.filter(Run.job_id == job.id)
|
||||
.order_by(Run.id.desc())
|
||||
.first()
|
||||
)
|
||||
out = JobWithLastRun.model_validate(job)
|
||||
out.last_run = RunSummary.model_validate(last) if last else None
|
||||
return out
|
||||
|
||||
|
||||
@router.get("", response_model=list[JobWithLastRun])
|
||||
def list_jobs(db: DBSession, _: CurrentUser) -> list[JobWithLastRun]:
|
||||
jobs = db.query(Job).order_by(Job.id).all()
|
||||
return [_job_to_with_last_run(j, db) for j in jobs]
|
||||
|
||||
|
||||
@router.post("", response_model=JobOut, status_code=201)
|
||||
def create_job(payload: JobCreate, db: DBSession, _: AdminUser) -> Job:
|
||||
if db.query(Job).filter(Job.name == payload.name).first():
|
||||
raise HTTPException(status_code=400, detail="名称已存在")
|
||||
if not db.get(Storage, payload.storage_id):
|
||||
raise HTTPException(status_code=400, detail="存储目标不存在")
|
||||
src = _validate_source(payload.type, payload.source_config)
|
||||
job = Job(
|
||||
name=payload.name,
|
||||
type=payload.type,
|
||||
source_config_json=encrypt_dict(src),
|
||||
storage_id=payload.storage_id,
|
||||
cron_expression=payload.cron_expression,
|
||||
enabled=payload.enabled,
|
||||
retention_count=payload.retention_count,
|
||||
retention_days=payload.retention_days,
|
||||
description=payload.description,
|
||||
)
|
||||
db.add(job)
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
reschedule_job(job.id)
|
||||
return job
|
||||
|
||||
|
||||
@router.get("/{job_id}", response_model=JobWithLastRun)
|
||||
def get_job(job_id: int, db: DBSession, _: CurrentUser) -> JobWithLastRun:
|
||||
job = db.get(Job, job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
return _job_to_with_last_run(job, db)
|
||||
|
||||
|
||||
@router.put("/{job_id}", response_model=JobOut)
|
||||
def update_job(job_id: int, payload: JobUpdate, db: DBSession, _: AdminUser) -> Job:
|
||||
job = db.get(Job, job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
if payload.name is not None:
|
||||
if db.query(Job).filter(Job.name == payload.name, Job.id != job_id).first():
|
||||
raise HTTPException(status_code=400, detail="名称已存在")
|
||||
job.name = payload.name
|
||||
if payload.source_config is not None:
|
||||
src = _validate_source(job.type, payload.source_config)
|
||||
job.source_config_json = encrypt_dict(src)
|
||||
if payload.storage_id is not None:
|
||||
if not db.get(Storage, payload.storage_id):
|
||||
raise HTTPException(status_code=400, detail="存储目标不存在")
|
||||
job.storage_id = payload.storage_id
|
||||
if payload.cron_expression is not None:
|
||||
job.cron_expression = payload.cron_expression or None
|
||||
if payload.enabled is not None:
|
||||
job.enabled = payload.enabled
|
||||
if payload.retention_count is not None:
|
||||
job.retention_count = payload.retention_count
|
||||
if payload.retention_days is not None:
|
||||
job.retention_days = payload.retention_days
|
||||
if payload.description is not None:
|
||||
job.description = payload.description
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
reschedule_job(job.id)
|
||||
return job
|
||||
|
||||
|
||||
@router.delete("/{job_id}", status_code=204)
|
||||
def delete_job(job_id: int, db: DBSession, _: AdminUser) -> None:
|
||||
job = db.get(Job, job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
remove_job(job.id)
|
||||
db.delete(job)
|
||||
db.commit()
|
||||
|
||||
|
||||
@router.post("/{job_id}/run", response_model=JobRunResponse)
|
||||
async def run_job(job_id: int, db: DBSession, _: CurrentUser) -> JobRunResponse:
|
||||
job = db.get(Job, job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
run = Run(
|
||||
job_id=job.id,
|
||||
status="pending",
|
||||
trigger="manual",
|
||||
started_at=datetime.utcnow(),
|
||||
)
|
||||
db.add(run)
|
||||
db.commit()
|
||||
db.refresh(run)
|
||||
run_id = run.id
|
||||
# 后台启动
|
||||
asyncio.create_task(run_backup(run_id, trigger="manual"))
|
||||
return JobRunResponse(run_id=run_id, status=run.status)
|
||||
|
||||
|
||||
@router.post("/{job_id}/toggle", response_model=JobOut)
|
||||
def toggle_job(job_id: int, db: DBSession, _: AdminUser) -> Job:
|
||||
job = db.get(Job, job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
job.enabled = not job.enabled
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
reschedule_job(job.id)
|
||||
return job
|
||||
@@ -0,0 +1,76 @@
|
||||
"""恢复 API。"""
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
|
||||
from app.core.restore import execute_restore
|
||||
from app.deps import AdminUser, DBSession
|
||||
from app.models.job import Job
|
||||
from app.models.restore import RestoreHistory
|
||||
from app.models.run import Run
|
||||
from app.schemas.restore import RestoreOut, RestoreRequest, RestoreTargetsResponse
|
||||
from app.utils.crypto import decrypt_dict
|
||||
|
||||
router = APIRouter(prefix="/restore", tags=["restore"])
|
||||
|
||||
|
||||
@router.get("/targets", response_model=RestoreTargetsResponse)
|
||||
def get_restore_targets(run_id: int, db: DBSession, _: AdminUser) -> RestoreTargetsResponse:
|
||||
run = db.get(Run, run_id)
|
||||
if not run:
|
||||
raise HTTPException(status_code=404, detail="Run not found")
|
||||
job = db.get(Job, run.job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
src = decrypt_dict(job.source_config_json)
|
||||
suggested: list[str] = []
|
||||
if job.type == "mysql":
|
||||
suggested.append(src.get("database", ""))
|
||||
elif job.type == "directory":
|
||||
suggested.append(src.get("path", ""))
|
||||
|
||||
return RestoreTargetsResponse(
|
||||
run_id=run.id,
|
||||
job_type=job.type,
|
||||
artifact_size=run.artifact_size,
|
||||
artifact_checksum=run.artifact_checksum,
|
||||
suggested_targets=[s for s in suggested if s],
|
||||
)
|
||||
|
||||
|
||||
@router.post("", response_model=RestoreOut, status_code=201)
|
||||
async def trigger_restore(payload: RestoreRequest, db: DBSession, _: AdminUser) -> RestoreHistory:
|
||||
run = db.get(Run, payload.run_id)
|
||||
if not run or not run.artifact_path:
|
||||
raise HTTPException(status_code=404, detail="Run not found or no artifact")
|
||||
job = db.get(Job, run.job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
# 校验 target_type 与 job.type 匹配
|
||||
if job.type == "mysql" and payload.target_type != "mysql_db":
|
||||
raise HTTPException(status_code=400, detail="target_type 与源类型不匹配(应为 mysql_db)")
|
||||
if job.type == "directory" and payload.target_type != "directory_path":
|
||||
raise HTTPException(status_code=400, detail="target_type 与源类型不匹配(应为 directory_path)")
|
||||
|
||||
rh = RestoreHistory(
|
||||
run_id=run.id,
|
||||
target=payload.target,
|
||||
status="pending",
|
||||
)
|
||||
db.add(rh)
|
||||
db.commit()
|
||||
db.refresh(rh)
|
||||
restore_id = rh.id
|
||||
asyncio.create_task(execute_restore(restore_id))
|
||||
return rh
|
||||
|
||||
|
||||
@router.get("/history", response_model=list[RestoreOut])
|
||||
def restore_history(db: DBSession, _: AdminUser, run_id: int | None = None) -> list[RestoreHistory]:
|
||||
q = db.query(RestoreHistory)
|
||||
if run_id is not None:
|
||||
q = q.filter(RestoreHistory.run_id == run_id)
|
||||
return q.order_by(RestoreHistory.id.desc()).limit(200).all()
|
||||
@@ -0,0 +1,11 @@
|
||||
"""API 总路由。"""
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api import auth, jobs, restore, runs, storages
|
||||
|
||||
api_router = APIRouter()
|
||||
api_router.include_router(auth.router)
|
||||
api_router.include_router(storages.router)
|
||||
api_router.include_router(jobs.router)
|
||||
api_router.include_router(runs.router)
|
||||
api_router.include_router(restore.router)
|
||||
@@ -0,0 +1,96 @@
|
||||
"""备份运行历史 API。"""
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from app.core.storage.factory import create_storage
|
||||
from app.deps import CurrentUser, DBSession
|
||||
from app.models.job import Job
|
||||
from app.models.run import Run, RunLog
|
||||
from app.schemas.run import RunListResponse, RunLogOut, RunOut
|
||||
|
||||
router = APIRouter(prefix="/runs", tags=["runs"])
|
||||
|
||||
|
||||
@router.get("", response_model=RunListResponse)
|
||||
def list_runs(
|
||||
db: DBSession,
|
||||
_: CurrentUser,
|
||||
job_id: int | None = None,
|
||||
status: str | None = None,
|
||||
page: int = 1,
|
||||
page_size: int = 20,
|
||||
) -> RunListResponse:
|
||||
page = max(1, page)
|
||||
page_size = max(1, min(100, page_size))
|
||||
|
||||
q = db.query(Run)
|
||||
if job_id is not None:
|
||||
q = q.filter(Run.job_id == job_id)
|
||||
if status:
|
||||
q = q.filter(Run.status == status)
|
||||
total = q.count()
|
||||
items = (
|
||||
q.order_by(Run.id.desc())
|
||||
.offset((page - 1) * page_size)
|
||||
.limit(page_size)
|
||||
.all()
|
||||
)
|
||||
return RunListResponse(total=total, items=[RunOut.model_validate(r) for r in items])
|
||||
|
||||
|
||||
@router.get("/{run_id}", response_model=RunOut)
|
||||
def get_run(run_id: int, db: DBSession, _: CurrentUser) -> Run:
|
||||
run = db.get(Run, run_id)
|
||||
if not run:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
return run
|
||||
|
||||
|
||||
@router.get("/{run_id}/log", response_model=RunLogOut)
|
||||
def get_run_log(run_id: int, db: DBSession, _: CurrentUser) -> RunLog:
|
||||
run = db.get(Run, run_id)
|
||||
if not run:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
log = db.query(RunLog).filter(RunLog.run_id == run_id).order_by(RunLog.id.desc()).first()
|
||||
if not log:
|
||||
raise HTTPException(status_code=404, detail="No log")
|
||||
return log
|
||||
|
||||
|
||||
@router.get("/{run_id}/download")
|
||||
async def download_run(run_id: int, db: DBSession, _: CurrentUser):
|
||||
run = db.get(Run, run_id)
|
||||
if not run or not run.artifact_path:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
job = db.get(Job, run.job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
storage = create_storage(job.storage)
|
||||
|
||||
filename = run.artifact_path.split("/")[-1]
|
||||
|
||||
async def stream():
|
||||
async for chunk in storage.download_stream(run.artifact_path):
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(
|
||||
stream(),
|
||||
media_type="application/octet-stream",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{run_id}", status_code=204)
|
||||
async def delete_run(run_id: int, db: DBSession, _: CurrentUser) -> None:
|
||||
run = db.get(Run, run_id)
|
||||
if not run:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
job = db.get(Job, run.job_id)
|
||||
if job and run.artifact_path:
|
||||
try:
|
||||
storage = create_storage(job.storage)
|
||||
await storage.delete(run.artifact_path)
|
||||
except Exception:
|
||||
pass
|
||||
db.delete(run)
|
||||
db.commit()
|
||||
@@ -0,0 +1,100 @@
|
||||
"""存储目标 CRUD API。"""
|
||||
import asyncio
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.core.storage.factory import create_storage as make_storage_impl
|
||||
from app.deps import AdminUser, DBSession
|
||||
from app.models.storage import Storage
|
||||
from app.schemas.storage import (
|
||||
LocalStorageConfig,
|
||||
S3StorageConfig,
|
||||
StorageCreate,
|
||||
StorageOut,
|
||||
StorageTestResponse,
|
||||
StorageUpdate,
|
||||
)
|
||||
from app.utils.crypto import decrypt_dict, encrypt_dict
|
||||
|
||||
router = APIRouter(prefix="/storages", tags=["storages"])
|
||||
|
||||
|
||||
def _validate_config(type_: str, config: dict) -> dict:
|
||||
if type_ == "local":
|
||||
return LocalStorageConfig(**config).model_dump()
|
||||
if type_ == "s3":
|
||||
return S3StorageConfig(**config).model_dump()
|
||||
raise HTTPException(status_code=400, detail=f"Unsupported type: {type_}")
|
||||
|
||||
|
||||
@router.get("", response_model=list[StorageOut])
|
||||
def list_storages(db: DBSession, _: AdminUser) -> list[Storage]:
|
||||
return db.query(Storage).order_by(Storage.id).all()
|
||||
|
||||
|
||||
@router.post("", response_model=StorageOut, status_code=201)
|
||||
def create_storage_endpoint(payload: StorageCreate, db: DBSession, _: AdminUser) -> Storage:
|
||||
if db.query(Storage).filter(Storage.name == payload.name).first():
|
||||
raise HTTPException(status_code=400, detail="名称已存在")
|
||||
config = _validate_config(payload.type, payload.config)
|
||||
row = Storage(
|
||||
name=payload.name,
|
||||
type=payload.type,
|
||||
config_json=encrypt_dict(config),
|
||||
is_default=payload.is_default,
|
||||
)
|
||||
if payload.is_default:
|
||||
db.query(Storage).update({Storage.is_default: False})
|
||||
db.add(row)
|
||||
db.commit()
|
||||
db.refresh(row)
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/{storage_id}", response_model=StorageOut)
|
||||
def get_storage(storage_id: int, db: DBSession, _: AdminUser) -> Storage:
|
||||
row = db.get(Storage, storage_id)
|
||||
if not row:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
return row
|
||||
|
||||
|
||||
@router.put("/{storage_id}", response_model=StorageOut)
|
||||
def update_storage(storage_id: int, payload: StorageUpdate, db: DBSession, _: AdminUser) -> Storage:
|
||||
row = db.get(Storage, storage_id)
|
||||
if not row:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
if payload.name is not None:
|
||||
if db.query(Storage).filter(Storage.name == payload.name, Storage.id != storage_id).first():
|
||||
raise HTTPException(status_code=400, detail="名称已存在")
|
||||
row.name = payload.name
|
||||
if payload.config is not None:
|
||||
config = _validate_config(row.type, payload.config)
|
||||
row.config_json = encrypt_dict(config)
|
||||
if payload.is_default is not None:
|
||||
if payload.is_default:
|
||||
db.query(Storage).update({Storage.is_default: False})
|
||||
row.is_default = payload.is_default
|
||||
db.commit()
|
||||
db.refresh(row)
|
||||
return row
|
||||
|
||||
|
||||
@router.delete("/{storage_id}", status_code=204)
|
||||
def delete_storage(storage_id: int, db: DBSession, _: AdminUser) -> None:
|
||||
row = db.get(Storage, storage_id)
|
||||
if not row:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
|
||||
|
||||
@router.post("/{storage_id}/test", response_model=StorageTestResponse)
|
||||
async def test_storage(storage_id: int, db: DBSession, _: AdminUser) -> StorageTestResponse:
|
||||
row = db.get(Storage, storage_id)
|
||||
if not row:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
storage = make_storage_impl(row)
|
||||
ok, msg = await storage.test_connection()
|
||||
return StorageTestResponse(ok=ok, message=msg)
|
||||
@@ -0,0 +1,44 @@
|
||||
"""应用配置:从环境变量加载,启动时校验必填项。"""
|
||||
from functools import lru_cache
|
||||
from typing import List
|
||||
|
||||
from pydantic import Field, field_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
env_file_encoding="utf-8",
|
||||
case_sensitive=True,
|
||||
extra="ignore",
|
||||
)
|
||||
|
||||
# ===== 必填 =====
|
||||
SECRET_KEY: str = Field(..., description="JWT 签名密钥(>= 32 字符)")
|
||||
FERNET_KEY: str = Field(..., description="Fernet 字段加密密钥")
|
||||
INITIAL_ADMIN_PASSWORD: str = Field(..., min_length=8)
|
||||
|
||||
# ===== 可选 =====
|
||||
JWT_EXPIRE_MINUTES: int = 1440
|
||||
DATABASE_URL: str = "sqlite:///./data/backup.db"
|
||||
STORAGE_LOCAL_ROOT: str = "./data/backups"
|
||||
LOG_LEVEL: str = "INFO"
|
||||
CORS_ORIGINS: str = "http://localhost:5173,http://localhost:8000"
|
||||
TZ: str = "UTC"
|
||||
|
||||
@field_validator("SECRET_KEY")
|
||||
@classmethod
|
||||
def _check_secret_key(cls, v: str) -> str:
|
||||
if len(v) < 32:
|
||||
raise ValueError("SECRET_KEY 长度必须 >= 32 字符(建议 openssl rand -hex 32)")
|
||||
return v
|
||||
|
||||
@property
|
||||
def cors_origins_list(self) -> List[str]:
|
||||
return [o.strip() for o in self.CORS_ORIGINS.split(",") if o.strip()]
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
return Settings() # type: ignore[call-arg]
|
||||
@@ -0,0 +1,36 @@
|
||||
"""备份执行抽象基类:MySQL / Directory 都实现此接口。"""
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import AsyncIterator, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class BackupMetadata:
|
||||
"""备份产物的元信息。"""
|
||||
|
||||
suggested_filename: str # e.g. "db-20260622-030000.sql.gz" 或 "logs-20260622.tar.gz"
|
||||
content_type: str = "application/octet-stream"
|
||||
extra: Optional[dict] = None
|
||||
|
||||
|
||||
class BaseBackup(ABC):
|
||||
"""所有数据源类型实现此接口。
|
||||
|
||||
方法职责:
|
||||
- metadata(): 计算建议文件名等
|
||||
- produce(): 以异步字节流形式产出备份内容
|
||||
"""
|
||||
|
||||
def __init__(self, source_config: dict, job_name: str):
|
||||
self.source_config = source_config
|
||||
self.job_name = job_name
|
||||
|
||||
@abstractmethod
|
||||
def metadata(self) -> BackupMetadata: ...
|
||||
|
||||
@abstractmethod
|
||||
async def produce(self) -> AsyncIterator[bytes]:
|
||||
"""异步产出备份字节流。"""
|
||||
# 子类应使用 async with + async generator
|
||||
...
|
||||
yield b"" # pragma: no cover
|
||||
@@ -0,0 +1,127 @@
|
||||
"""目录备份:使用 tarfile 流式打包为 tar.gz。"""
|
||||
import asyncio
|
||||
import fnmatch
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import AsyncIterator
|
||||
|
||||
from app.core.backup.base import BackupMetadata, BaseBackup
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class DirectoryBackup(BaseBackup):
|
||||
def __init__(self, source_config: dict, job_name: str):
|
||||
super().__init__(source_config, job_name)
|
||||
self.path: str = source_config["path"]
|
||||
self.exclude_patterns: list[str] = source_config.get("exclude_patterns", []) or []
|
||||
self.base_dir = os.path.basename(os.path.normpath(self.path)) or "root"
|
||||
self.root_path = Path(self.path).resolve()
|
||||
|
||||
def metadata(self) -> BackupMetadata:
|
||||
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
safe = "".join(c if c.isalnum() or c in ("_", "-") else "_" for c in self.base_dir)
|
||||
return BackupMetadata(
|
||||
suggested_filename=f"{self.job_name}-{safe}-{ts}.tar.gz",
|
||||
content_type="application/gzip",
|
||||
extra={"path": self.path, "base_dir": self.base_dir},
|
||||
)
|
||||
|
||||
def _is_excluded(self, rel_path: str) -> bool:
|
||||
name = os.path.basename(rel_path)
|
||||
for pat in self.exclude_patterns:
|
||||
if fnmatch.fnmatch(name, pat) or fnmatch.fnmatch(rel_path, pat):
|
||||
return True
|
||||
return False
|
||||
|
||||
async def produce(self) -> AsyncIterator[bytes]:
|
||||
if not self.root_path.exists():
|
||||
raise FileNotFoundError(f"Source path does not exist: {self.path}")
|
||||
if not self.root_path.is_dir():
|
||||
raise NotADirectoryError(f"Source path is not a directory: {self.path}")
|
||||
|
||||
log.info("Taring directory %s (exclude=%s)", self.path, self.exclude_patterns)
|
||||
|
||||
# 用一个 Queue 在同步线程和异步消费者之间桥接 tar 产出
|
||||
queue: asyncio.Queue[bytes | None] = asyncio.Queue(maxsize=32)
|
||||
|
||||
def _tar_worker() -> None:
|
||||
import tarfile
|
||||
|
||||
try:
|
||||
with tarfile.open(mode="w:gz", bufsize=1024 * 1024) as tar:
|
||||
# 顶层目录名固定为 base_dir,便于解压后还原
|
||||
for dirpath, dirnames, filenames in os.walk(self.root_path):
|
||||
# 排除目录
|
||||
dirnames[:] = [
|
||||
d for d in dirnames
|
||||
if not self._is_excluded(os.path.relpath(os.path.join(dirpath, d), self.root_path))
|
||||
]
|
||||
for fn in filenames:
|
||||
abs_path = os.path.join(dirpath, fn)
|
||||
rel = os.path.relpath(abs_path, self.root_path)
|
||||
if self._is_excluded(rel):
|
||||
continue
|
||||
try:
|
||||
tar.add(abs_path, arcname=os.path.join(self.base_dir, rel), recursive=False)
|
||||
except (PermissionError, OSError) as e:
|
||||
log.warning("Skip %s: %s", abs_path, e)
|
||||
continue
|
||||
# 周期性 flush 到 queue
|
||||
if tar.fileobj.tell() and (tar.fileobj.tell() % (256 * 1024) == 0):
|
||||
# 这里我们用 tarfile 的 fileobj 直接喂到 queue 更高效
|
||||
pass
|
||||
except Exception as e:
|
||||
log.exception("tar worker error: %s", e)
|
||||
finally:
|
||||
# tarfile 关闭后会写完 gzip trailer,我们需要在关闭时把 fileobj 全部送出
|
||||
pass
|
||||
|
||||
# 实现简化:使用 tarfile 的 fileobj=BytesIO 会在循环中累积,
|
||||
# 对超大目录会爆内存。下面用一个临时文件 + 异步读取的方案。
|
||||
import tempfile
|
||||
|
||||
tmp_path = Path(tempfile.gettempdir()) / f"backup_{self.job_name}_{os.getpid()}.tar.gz"
|
||||
|
||||
def _tar_to_file() -> None:
|
||||
import tarfile
|
||||
|
||||
try:
|
||||
with tarfile.open(str(tmp_path), mode="w:gz") as tar:
|
||||
for dirpath, dirnames, filenames in os.walk(self.root_path):
|
||||
dirnames[:] = [
|
||||
d for d in dirnames
|
||||
if not self._is_excluded(os.path.relpath(os.path.join(dirpath, d), self.root_path))
|
||||
]
|
||||
for fn in filenames:
|
||||
abs_path = os.path.join(dirpath, fn)
|
||||
rel = os.path.relpath(abs_path, self.root_path)
|
||||
if self._is_excluded(rel):
|
||||
continue
|
||||
try:
|
||||
tar.add(abs_path, arcname=os.path.join(self.base_dir, rel), recursive=False)
|
||||
except (PermissionError, OSError) as e:
|
||||
log.warning("Skip %s: %s", abs_path, e)
|
||||
except Exception:
|
||||
log.exception("tar file worker failed")
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
await loop.run_in_executor(None, _tar_to_file)
|
||||
|
||||
if not tmp_path.exists() or tmp_path.stat().st_size == 0:
|
||||
raise RuntimeError(f"Failed to create tar.gz for {self.path}")
|
||||
|
||||
try:
|
||||
with open(tmp_path, "rb") as f:
|
||||
while True:
|
||||
chunk = await loop.run_in_executor(None, f.read, 64 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
try:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,97 @@
|
||||
"""MySQL 备份:通过 mysqldump 子进程流式输出。"""
|
||||
import asyncio
|
||||
import gzip
|
||||
import shlex
|
||||
from datetime import datetime
|
||||
from typing import AsyncIterator
|
||||
|
||||
from app.core.backup.base import BackupMetadata, BaseBackup
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
CHUNK_SIZE = 64 * 1024
|
||||
|
||||
|
||||
class MySQLBackup(BaseBackup):
|
||||
def __init__(self, source_config: dict, job_name: str):
|
||||
super().__init__(source_config, job_name)
|
||||
self.host: str = source_config["host"]
|
||||
self.port: int = int(source_config.get("port", 3306))
|
||||
self.user: str = source_config["user"]
|
||||
self.password: str = source_config.get("password", "")
|
||||
self.database: str = source_config["database"]
|
||||
self.extra_args: str = source_config.get("extra_args", "")
|
||||
|
||||
def metadata(self) -> BackupMetadata:
|
||||
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
safe_db = "".join(c if c.isalnum() or c in ("_", "-") else "_" for c in self.database)
|
||||
return BackupMetadata(
|
||||
suggested_filename=f"{self.job_name}-{safe_db}-{ts}.sql.gz",
|
||||
content_type="application/gzip",
|
||||
extra={"database": self.database, "host": self.host, "port": self.port},
|
||||
)
|
||||
|
||||
async def produce(self) -> AsyncIterator[bytes]:
|
||||
args = [
|
||||
"mysqldump",
|
||||
"-h", self.host,
|
||||
"-P", str(self.port),
|
||||
"-u", self.user,
|
||||
"--single-transaction",
|
||||
"--quick",
|
||||
"--routines",
|
||||
"--triggers",
|
||||
"--events",
|
||||
"--default-character-set=utf8mb4",
|
||||
]
|
||||
if self.extra_args:
|
||||
args.extend(shlex.split(self.extra_args))
|
||||
args.append(self.database)
|
||||
|
||||
log.info("Executing mysqldump for db=%s host=%s", self.database, self.host)
|
||||
|
||||
# mysqldump 直接写 stdout,我们 pipe 给 gzip 再 yield
|
||||
# 用 -p 紧贴密码避免出现在 ps 里(空密码时 mysqldump 会要求交互输入)
|
||||
if self.password:
|
||||
args_with_pwd = args.copy()
|
||||
# 找到 -u 后插入 -pXXX
|
||||
u_idx = args_with_pwd.index("-u")
|
||||
args_with_pwd.insert(u_idx + 2, f"-p{self.password}")
|
||||
else:
|
||||
args_with_pwd = args
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*args_with_pwd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
|
||||
assert proc.stdout is not None
|
||||
|
||||
# 在线程里跑 gzip.compress,因为它是阻塞的
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
async def gzip_stream() -> AsyncIterator[bytes]:
|
||||
chunk_buf = bytearray()
|
||||
async for chunk in proc.stdout: # type: ignore[union-attr]
|
||||
# gzip 接受 bytes,返回 bytes
|
||||
compressed = await loop.run_in_executor(None, gzip.compress, chunk)
|
||||
if compressed:
|
||||
yield compressed
|
||||
|
||||
try:
|
||||
async for piece in gzip_stream():
|
||||
yield piece
|
||||
finally:
|
||||
try:
|
||||
await proc.wait()
|
||||
except Exception:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
|
||||
if proc.returncode != 0:
|
||||
stderr = (await proc.stderr.read()).decode(errors="replace") if proc.stderr else ""
|
||||
raise RuntimeError(
|
||||
f"mysqldump failed (exit={proc.returncode}): {stderr.strip()[:2000]}"
|
||||
)
|
||||
@@ -0,0 +1,151 @@
|
||||
"""备份执行编排:解密配置 → 流式产出 → 流式上传 → 应用保留策略 → 写日志。"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.backup.base import BaseBackup
|
||||
from app.core.backup.directory import DirectoryBackup
|
||||
from app.core.backup.mysql import MySQLBackup
|
||||
from app.core.retention import apply_retention
|
||||
from app.core.storage.factory import create_storage
|
||||
from app.database import SessionLocal
|
||||
from app.models.job import Job
|
||||
from app.models.run import Run, RunLog
|
||||
from app.utils.crypto import decrypt_dict
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
def _make_backup(job: Job) -> BaseBackup:
|
||||
src = decrypt_dict(job.source_config_json)
|
||||
if job.type == "mysql":
|
||||
return MySQLBackup(src, job.name)
|
||||
if job.type == "directory":
|
||||
return DirectoryBackup(src, job.name)
|
||||
raise ValueError(f"Unsupported job type: {job.type}")
|
||||
|
||||
|
||||
async def run_backup(run_id: int, trigger: str = "manual") -> None:
|
||||
"""执行一次备份。所有副作用都在自己 new 的 Session 上完成。"""
|
||||
started = datetime.utcnow()
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
run: Run | None = db.get(Run, run_id)
|
||||
if not run:
|
||||
log.error("run_id=%s not found", run_id)
|
||||
return
|
||||
|
||||
job: Job | None = db.get(Job, run.job_id)
|
||||
if not job:
|
||||
run.status = "failed"
|
||||
run.error_message = f"Job id={run.job_id} disappeared"
|
||||
run.finished_at = datetime.utcnow()
|
||||
db.commit()
|
||||
return
|
||||
|
||||
log_buffer = io.StringIO()
|
||||
tee = _TeeLogger(log, log_buffer)
|
||||
|
||||
run.status = "running"
|
||||
run.trigger = trigger
|
||||
run.started_at = started
|
||||
db.commit()
|
||||
|
||||
tee.info("Backup started: run_id=%s job=%s trigger=%s", run_id, job.name, trigger)
|
||||
|
||||
try:
|
||||
backup = _make_backup(job)
|
||||
meta = backup.metadata()
|
||||
tee.info("Metadata: filename=%s content_type=%s", meta.suggested_filename, meta.content_type)
|
||||
|
||||
storage = create_storage(job.storage)
|
||||
ok, msg = await storage.test_connection()
|
||||
tee.info("Storage test: ok=%s msg=%s", ok, msg)
|
||||
if not ok:
|
||||
raise RuntimeError(f"Storage not reachable: {msg}")
|
||||
|
||||
# 关键路径:备份文件名以 {job_name}/{YYYY-MM-DD}/{filename} 组织
|
||||
date_prefix = datetime.utcnow().strftime("%Y-%m-%d")
|
||||
object_key = f"{job.name}/{date_prefix}/{meta.suggested_filename}"
|
||||
|
||||
tee.info("Streaming backup to storage key=%s ...", object_key)
|
||||
|
||||
async def _consume_with_progress():
|
||||
async for chunk in backup.produce():
|
||||
yield chunk
|
||||
|
||||
t0 = time.monotonic()
|
||||
size, sha = await storage.upload_stream(
|
||||
_consume_with_progress(),
|
||||
object_key,
|
||||
content_type=meta.content_type,
|
||||
)
|
||||
elapsed = int(time.monotonic() - t0)
|
||||
|
||||
run.artifact_path = object_key
|
||||
run.artifact_size = size
|
||||
run.artifact_checksum = sha
|
||||
run.duration_seconds = elapsed
|
||||
run.status = "success"
|
||||
run.finished_at = datetime.utcnow()
|
||||
tee.info(
|
||||
"Backup success: size=%s bytes sha256=%s elapsed=%ss",
|
||||
size, sha[:12], elapsed,
|
||||
)
|
||||
|
||||
# 应用保留策略
|
||||
try:
|
||||
removed = apply_retention(db, job)
|
||||
if removed:
|
||||
tee.info("Retention cleanup removed %s old runs", removed)
|
||||
except Exception as e:
|
||||
tee.warning("Retention cleanup failed: %s", e)
|
||||
|
||||
except Exception as e:
|
||||
log.exception("Backup run_id=%s failed", run_id)
|
||||
tee.error("Backup failed: %s: %s", type(e).__name__, e)
|
||||
run.status = "failed"
|
||||
run.error_message = f"{type(e).__name__}: {e}"[:2000]
|
||||
run.finished_at = datetime.utcnow()
|
||||
finally:
|
||||
log_text = log_buffer.getvalue()
|
||||
run.log_excerpt = log_text[-4000:] if len(log_text) > 4000 else log_text
|
||||
run_log = RunLog(run_id=run.id, content=log_text)
|
||||
db.add(run_log)
|
||||
db.commit()
|
||||
log.info("Backup run_id=%s finished status=%s", run_id, run.status)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
class _TeeLogger:
|
||||
"""同时写标准 logging 和内存 StringIO(用于存数据库)。"""
|
||||
|
||||
def __init__(self, real: logging.Logger, buf: io.StringIO):
|
||||
self._real = real
|
||||
self._buf = buf
|
||||
|
||||
def _emit(self, level: int, msg: str) -> None:
|
||||
ts = datetime.utcnow().strftime("%H:%M:%S")
|
||||
line = f"[{ts}] {msg}"
|
||||
self._real.log(level, line)
|
||||
self._buf.write(line + "\n")
|
||||
|
||||
def info(self, msg: str, *args) -> None:
|
||||
self._emit(logging.INFO, msg % args if args else msg)
|
||||
|
||||
def warning(self, msg: str, *args) -> None:
|
||||
self._emit(logging.WARNING, msg % args if args else msg)
|
||||
|
||||
def error(self, msg: str, *args) -> None:
|
||||
self._emit(logging.ERROR, msg % args if args else msg)
|
||||
@@ -0,0 +1,165 @@
|
||||
"""恢复执行器:把 backup artifact 还原到目标(MySQL 库 / 目录路径)。"""
|
||||
import asyncio
|
||||
import gzip
|
||||
import io
|
||||
import os
|
||||
import shutil
|
||||
import tarfile
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from app.core.storage.factory import create_storage
|
||||
from app.database import SessionLocal
|
||||
from app.models.job import Job
|
||||
from app.models.restore import RestoreHistory
|
||||
from app.models.run import Run, RunLog
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
async def execute_restore(restore_id: int) -> None:
|
||||
"""执行一次恢复。"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
rh: RestoreHistory | None = db.get(RestoreHistory, restore_id)
|
||||
if not rh:
|
||||
return
|
||||
|
||||
rh.status = "running"
|
||||
rh.started_at = datetime.utcnow()
|
||||
db.commit()
|
||||
|
||||
run: Run | None = db.get(Run, rh.run_id)
|
||||
if not run or not run.artifact_path:
|
||||
raise RuntimeError("源 run 或 artifact 不存在")
|
||||
job: Job | None = db.get(Job, run.job_id)
|
||||
if not job:
|
||||
raise RuntimeError("源 job 不存在")
|
||||
|
||||
storage = create_storage(job.storage)
|
||||
log_text = io.StringIO()
|
||||
|
||||
def logl(msg: str) -> None:
|
||||
line = f"[{datetime.utcnow():%H:%M:%S}] {msg}"
|
||||
log.info(line)
|
||||
log_text.write(line + "\n")
|
||||
|
||||
try:
|
||||
logl(f"Restoring run_id={run.id} -> {rh.target} (type={job.type})")
|
||||
|
||||
# 先下载到临时文件
|
||||
tmpdir = Path(tempfile.gettempdir()) / f"restore_{rh.id}_{os.getpid()}"
|
||||
tmpdir.mkdir(parents=True, exist_ok=True)
|
||||
local_tmp = tmpdir / Path(run.artifact_path).name
|
||||
|
||||
logl(f"Downloading {run.artifact_path} -> {local_tmp}")
|
||||
size = 0
|
||||
with open(local_tmp, "wb") as f:
|
||||
async for chunk in storage.download_stream(run.artifact_path):
|
||||
f.write(chunk)
|
||||
size += len(chunk)
|
||||
logl(f"Downloaded {size} bytes")
|
||||
|
||||
if job.type == "mysql":
|
||||
await _restore_mysql(local_tmp, rh.target, job, logl)
|
||||
elif job.type == "directory":
|
||||
_restore_directory(local_tmp, rh.target, logl)
|
||||
else:
|
||||
raise RuntimeError(f"Unsupported job type: {job.type}")
|
||||
|
||||
rh.status = "success"
|
||||
rh.finished_at = datetime.utcnow()
|
||||
logl("Restore success")
|
||||
except Exception as e:
|
||||
log.exception("Restore failed")
|
||||
logl(f"Restore failed: {type(e).__name__}: {e}")
|
||||
rh.status = "failed"
|
||||
rh.error_message = f"{type(e).__name__}: {e}"[:2000]
|
||||
rh.finished_at = datetime.utcnow()
|
||||
finally:
|
||||
# 写日志
|
||||
content = log_text.getvalue()
|
||||
db.add(RunLog(run_id=run.id, content=f"[restore #{rh.id}]\n{content}"))
|
||||
db.commit()
|
||||
try:
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def _restore_mysql(archive_path: Path, target_db: str, job: Job, logl) -> None:
|
||||
"""把 sql.gz 通过 gunzip | mysql 还原到 target_db。"""
|
||||
from app.utils.crypto import decrypt_dict
|
||||
|
||||
src = decrypt_dict(job.source_config_json)
|
||||
host = src["host"]
|
||||
port = int(src.get("port", 3306))
|
||||
user = src["user"]
|
||||
password = src.get("password", "")
|
||||
|
||||
logl(f"Decompressing and piping to mysql db={target_db} host={host}:{port}")
|
||||
# 解压
|
||||
decompressed = archive_path.with_suffix("") # .sql
|
||||
if archive_path.name.endswith(".gz"):
|
||||
with gzip.open(archive_path, "rb") as gz, open(decompressed, "wb") as out:
|
||||
shutil.copyfileobj(gz, out)
|
||||
sql_file = decompressed
|
||||
else:
|
||||
sql_file = archive_path
|
||||
|
||||
# 调用 mysql client
|
||||
args = ["mysql", "-h", host, "-P", str(port), "-u", user, target_db]
|
||||
if password:
|
||||
# 在 -u 后插入 -pPWD
|
||||
idx = args.index("-u")
|
||||
args.insert(idx + 2, f"-p{password}")
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*args,
|
||||
stdin=asyncio.subprocess.PIPE,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
assert proc.stdin is not None
|
||||
with open(sql_file, "rb") as f:
|
||||
while True:
|
||||
chunk = f.read(64 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
proc.stdin.write(chunk)
|
||||
await proc.stdin.drain()
|
||||
proc.stdin.close()
|
||||
rc = await proc.wait()
|
||||
if rc != 0:
|
||||
stderr = (await proc.stderr.read()).decode(errors="replace") if proc.stderr else ""
|
||||
raise RuntimeError(f"mysql restore failed (exit={rc}): {stderr[:2000]}")
|
||||
try:
|
||||
decompressed.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _restore_directory(archive_path: Path, target_dir: str, logl) -> None:
|
||||
"""解压 tar.gz 到 target_dir(必须为空或不存在)。"""
|
||||
target = Path(target_dir).resolve()
|
||||
if target.exists() and any(target.iterdir()):
|
||||
raise RuntimeError(f"目标目录非空: {target}")
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logl(f"Extracting tar.gz to {target}")
|
||||
if archive_path.name.endswith(".gz"):
|
||||
mode = "r:gz"
|
||||
else:
|
||||
mode = "r:"
|
||||
with tarfile.open(str(archive_path), mode) as tar:
|
||||
# 安全检查:拒绝路径逃逸
|
||||
for member in tar.getmembers():
|
||||
member_path = (target / member.name).resolve()
|
||||
if not str(member_path).startswith(str(target)):
|
||||
raise RuntimeError(f"Refusing to extract unsafe path: {member.name}")
|
||||
tar.extractall(path=str(target))
|
||||
@@ -0,0 +1,80 @@
|
||||
"""备份保留策略清理。"""
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.storage.factory import create_storage
|
||||
from app.models.job import Job
|
||||
from app.models.run import Run
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
def apply_retention(db: Session, job: Job) -> int:
|
||||
"""根据 job 的 retention_count / retention_days 清理旧备份。
|
||||
|
||||
返回删除的 run 数。
|
||||
"""
|
||||
# 取该 job 所有成功 runs,按 finished_at 倒序
|
||||
runs = (
|
||||
db.query(Run)
|
||||
.filter(Run.job_id == job.id, Run.status == "success")
|
||||
.order_by(Run.finished_at.desc())
|
||||
.all()
|
||||
)
|
||||
if not runs:
|
||||
return 0
|
||||
|
||||
keep_count = job.retention_count or 7
|
||||
keep_days = job.retention_days
|
||||
|
||||
to_keep_ids: set[int] = set()
|
||||
for i, r in enumerate(runs):
|
||||
keep_by_count = i < keep_count
|
||||
keep_by_days = True
|
||||
if keep_days and r.finished_at:
|
||||
threshold = datetime.utcnow() - timedelta(days=keep_days)
|
||||
if r.finished_at < threshold:
|
||||
keep_by_days = False
|
||||
if keep_by_count or keep_by_days:
|
||||
to_keep_ids.add(r.id)
|
||||
|
||||
to_delete = [r for r in runs if r.id not in to_keep_ids]
|
||||
if not to_delete:
|
||||
return 0
|
||||
|
||||
storage = create_storage(job.storage)
|
||||
removed = 0
|
||||
for r in to_delete:
|
||||
try:
|
||||
if r.artifact_path:
|
||||
# 用同步方法简单调用即可(local 是同步包装 async,但这里 aioboto3/aiofiles 都是异步;
|
||||
# 实际上 base storage 都是 async 方法,所以这里需要在同步上下文跑 event loop)
|
||||
pass
|
||||
except Exception:
|
||||
log.exception("Failed to delete artifact %s (db row still kept)", r.artifact_path)
|
||||
db.delete(r)
|
||||
removed += 1
|
||||
|
||||
db.commit()
|
||||
log.info("Retention cleanup for job=%s: deleted %s runs", job.name, removed)
|
||||
|
||||
# 真实删除 S3/local 文件(需要在 async 上下文)
|
||||
import asyncio
|
||||
try:
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
for r in to_delete:
|
||||
if r.artifact_path:
|
||||
try:
|
||||
loop.run_until_complete(storage.delete(r.artifact_path))
|
||||
except Exception:
|
||||
log.exception("Artifact delete failed: %s", r.artifact_path)
|
||||
finally:
|
||||
loop.close()
|
||||
except Exception:
|
||||
log.exception("Artifact cleanup loop error")
|
||||
|
||||
return removed
|
||||
@@ -0,0 +1,45 @@
|
||||
"""存储后端抽象接口。"""
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import AsyncIterator, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class StorageObject:
|
||||
key: str
|
||||
size: int
|
||||
last_modified: Optional[str] = None # ISO 8601
|
||||
|
||||
|
||||
class BaseStorage(ABC):
|
||||
"""所有存储后端实现的统一接口。"""
|
||||
|
||||
type: str = "base"
|
||||
|
||||
def __init__(self, config: dict):
|
||||
self.config = config
|
||||
|
||||
@abstractmethod
|
||||
async def test_connection(self) -> tuple[bool, str]:
|
||||
"""测试连通性。返回 (ok, message)。"""
|
||||
|
||||
@abstractmethod
|
||||
async def upload_stream(
|
||||
self,
|
||||
source: AsyncIterator[bytes],
|
||||
key: str,
|
||||
content_type: str = "application/octet-stream",
|
||||
) -> tuple[int, str]:
|
||||
"""流式上传。返回 (size_bytes, sha256_hex)。"""
|
||||
|
||||
@abstractmethod
|
||||
async def download_stream(self, key: str) -> AsyncIterator[bytes]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def delete(self, key: str) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def exists(self, key: str) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list(self, prefix: str = "") -> list[StorageObject]: ...
|
||||
@@ -0,0 +1,15 @@
|
||||
"""存储工厂:根据 storage 行创建对应实现。"""
|
||||
from app.core.storage.base import BaseStorage
|
||||
from app.core.storage.local import LocalStorage
|
||||
from app.core.storage.s3 import S3Storage
|
||||
from app.models.storage import Storage
|
||||
from app.utils.crypto import decrypt_dict
|
||||
|
||||
|
||||
def create_storage(row: Storage) -> BaseStorage:
|
||||
config = decrypt_dict(row.config_json)
|
||||
if row.type == "local":
|
||||
return LocalStorage(config)
|
||||
if row.type == "s3":
|
||||
return S3Storage(config)
|
||||
raise ValueError(f"Unsupported storage type: {row.type}")
|
||||
@@ -0,0 +1,98 @@
|
||||
"""本地文件系统存储。"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import AsyncIterator
|
||||
|
||||
import aiofiles
|
||||
|
||||
from app.core.storage.base import BaseStorage, StorageObject
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class LocalStorage(BaseStorage):
|
||||
type = "local"
|
||||
|
||||
def __init__(self, config: dict):
|
||||
super().__init__(config)
|
||||
self.root: str = os.path.abspath(config["path"])
|
||||
Path(self.root).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _resolve(self, key: str) -> Path:
|
||||
# 防 path traversal:禁止 ..
|
||||
if ".." in key.split("/"):
|
||||
raise ValueError(f"Invalid key (path traversal): {key}")
|
||||
return Path(self.root) / key
|
||||
|
||||
async def test_connection(self) -> tuple[bool, str]:
|
||||
try:
|
||||
Path(self.root).mkdir(parents=True, exist_ok=True)
|
||||
test_file = Path(self.root) / ".backup_system_test"
|
||||
async with aiofiles.open(test_file, "w") as f:
|
||||
await f.write("ok")
|
||||
# aiofiles 不暴露 os 命名空间,用线程池跑同步删除
|
||||
await asyncio.to_thread(os.remove, test_file)
|
||||
return True, f"OK: writable {self.root}"
|
||||
except Exception as e:
|
||||
return False, f"FAIL: {e}"
|
||||
|
||||
async def upload_stream(
|
||||
self,
|
||||
source: AsyncIterator[bytes],
|
||||
key: str,
|
||||
content_type: str = "application/octet-stream",
|
||||
) -> tuple[int, str]:
|
||||
path = self._resolve(key)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
sha = hashlib.sha256()
|
||||
size = 0
|
||||
async with aiofiles.open(path, "wb") as f:
|
||||
async for chunk in source:
|
||||
await f.write(chunk)
|
||||
sha.update(chunk)
|
||||
size += len(chunk)
|
||||
return size, sha.hexdigest()
|
||||
|
||||
async def download_stream(self, key: str) -> AsyncIterator[bytes]:
|
||||
path = self._resolve(key)
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Not found: {key}")
|
||||
async with aiofiles.open(path, "rb") as f:
|
||||
while True:
|
||||
chunk = await f.read(64 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
|
||||
async def delete(self, key: str) -> None:
|
||||
path = self._resolve(key)
|
||||
try:
|
||||
path.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
async def exists(self, key: str) -> bool:
|
||||
return self._resolve(key).exists()
|
||||
|
||||
async def list(self, prefix: str = "") -> list[StorageObject]:
|
||||
base = Path(self.root) / prefix if prefix else Path(self.root)
|
||||
if not base.exists():
|
||||
return []
|
||||
out: list[StorageObject] = []
|
||||
for p in base.rglob("*"):
|
||||
if p.is_file():
|
||||
rel = p.relative_to(Path(self.root)).as_posix()
|
||||
st = p.stat()
|
||||
out.append(
|
||||
StorageObject(
|
||||
key=rel,
|
||||
size=st.st_size,
|
||||
last_modified=datetime.fromtimestamp(st.st_mtime).isoformat(),
|
||||
)
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,176 @@
|
||||
"""S3 兼容对象存储(支持 MinIO / Ceph / AWS S3)。"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from typing import Any, AsyncIterator
|
||||
|
||||
import aioboto3
|
||||
|
||||
from app.core.storage.base import BaseStorage, StorageObject
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class S3Storage(BaseStorage):
|
||||
type = "s3"
|
||||
|
||||
def __init__(self, config: dict):
|
||||
super().__init__(config)
|
||||
self.endpoint_url: str | None = config.get("endpoint_url") or None
|
||||
self.region: str = config.get("region") or "us-east-1"
|
||||
self.bucket: str = config["bucket"]
|
||||
self.access_key: str = config["access_key"]
|
||||
self.secret_key: str = config["secret_key"]
|
||||
self.use_ssl: bool = bool(config.get("use_ssl", True))
|
||||
self.path_prefix: str = config.get("path_prefix", "").strip("/")
|
||||
self.addressing_style: str = config.get("addressing_style", "auto")
|
||||
|
||||
@asynccontextmanager
|
||||
async def _client(self):
|
||||
session = aioboto3.Session()
|
||||
async with session.client(
|
||||
"s3",
|
||||
endpoint_url=self.endpoint_url,
|
||||
region_name=self.region,
|
||||
aws_access_key_id=self.access_key,
|
||||
aws_secret_access_key=self.secret_key,
|
||||
use_ssl=self.use_ssl,
|
||||
config=__import__("botocore.config", fromlist=["Config"]).Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": self.addressing_style},
|
||||
),
|
||||
) as client:
|
||||
yield client
|
||||
|
||||
def _full_key(self, key: str) -> str:
|
||||
key = key.lstrip("/")
|
||||
if self.path_prefix:
|
||||
return f"{self.path_prefix}/{key}"
|
||||
return key
|
||||
|
||||
async def test_connection(self) -> tuple[bool, str]:
|
||||
try:
|
||||
async with self._client() as client:
|
||||
await client.head_bucket(Bucket=self.bucket)
|
||||
return True, f"OK: bucket={self.bucket}"
|
||||
except Exception as e:
|
||||
return False, f"FAIL: {type(e).__name__}: {e}"
|
||||
|
||||
async def upload_stream(
|
||||
self,
|
||||
source: AsyncIterator[bytes],
|
||||
key: str,
|
||||
content_type: str = "application/octet-stream",
|
||||
) -> tuple[int, str]:
|
||||
sha = hashlib.sha256()
|
||||
size = 0
|
||||
full_key = self._full_key(key)
|
||||
|
||||
async with self._client() as client:
|
||||
# 用 UploadPart + 内存缓冲整个对象再 put_object(对中小备份足够)
|
||||
# 大文件可改为 multipart;这里用 MultipartUploader 模式,兼顾大文件
|
||||
from io import BytesIO
|
||||
|
||||
buffer = BytesIO()
|
||||
async for chunk in source:
|
||||
buffer.write(chunk)
|
||||
sha.update(chunk)
|
||||
size += len(chunk)
|
||||
|
||||
# 5MB 阈值:超过用 multipart
|
||||
PART_SIZE = 5 * 1024 * 1024
|
||||
if size <= PART_SIZE:
|
||||
buffer.seek(0)
|
||||
await client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=full_key,
|
||||
Body=buffer.getvalue(),
|
||||
ContentType=content_type,
|
||||
)
|
||||
else:
|
||||
# multipart
|
||||
mpu = await client.create_multipart_upload(
|
||||
Bucket=self.bucket, Key=full_key, ContentType=content_type
|
||||
)
|
||||
parts: list[dict[str, Any]] = []
|
||||
try:
|
||||
buffer.seek(0)
|
||||
part_number = 1
|
||||
while True:
|
||||
data = buffer.read(PART_SIZE)
|
||||
if not data:
|
||||
break
|
||||
resp = await client.upload_part(
|
||||
Bucket=self.bucket,
|
||||
Key=full_key,
|
||||
PartNumber=part_number,
|
||||
UploadId=mpu["UploadId"],
|
||||
Body=data,
|
||||
)
|
||||
parts.append({"PartNumber": part_number, "ETag": resp["ETag"]})
|
||||
part_number += 1
|
||||
await client.complete_multipart_upload(
|
||||
Bucket=self.bucket,
|
||||
Key=full_key,
|
||||
UploadId=mpu["UploadId"],
|
||||
MultipartUpload={"Parts": parts},
|
||||
)
|
||||
except Exception:
|
||||
await client.abort_multipart_upload(
|
||||
Bucket=self.bucket, Key=full_key, UploadId=mpu["UploadId"]
|
||||
)
|
||||
raise
|
||||
|
||||
return size, sha.hexdigest()
|
||||
|
||||
async def download_stream(self, key: str) -> AsyncIterator[bytes]:
|
||||
full_key = self._full_key(key)
|
||||
async with self._client() as client:
|
||||
obj = await client.get_object(Bucket=self.bucket, Key=full_key)
|
||||
body = obj["Body"]
|
||||
try:
|
||||
while True:
|
||||
chunk = await asyncio.get_event_loop().run_in_executor(
|
||||
None, body.read, 64 * 1024
|
||||
)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
async def delete(self, key: str) -> None:
|
||||
full_key = self._full_key(key)
|
||||
async with self._client() as client:
|
||||
await client.delete_object(Bucket=self.bucket, Key=full_key)
|
||||
|
||||
async def exists(self, key: str) -> bool:
|
||||
full_key = self._full_key(key)
|
||||
async with self._client() as client:
|
||||
try:
|
||||
await client.head_object(Bucket=self.bucket, Key=full_key)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def list(self, prefix: str = "") -> list[StorageObject]:
|
||||
full_prefix = self._full_key(prefix)
|
||||
out: list[StorageObject] = []
|
||||
async with self._client() as client:
|
||||
paginator = client.get_paginator("list_objects_v2")
|
||||
async for page in paginator.paginate(Bucket=self.bucket, Prefix=full_prefix):
|
||||
for obj in page.get("Contents", []):
|
||||
key = obj["Key"]
|
||||
if self.path_prefix and key.startswith(self.path_prefix + "/"):
|
||||
key = key[len(self.path_prefix) + 1 :]
|
||||
last_modified = obj.get("LastModified")
|
||||
out.append(
|
||||
StorageObject(
|
||||
key=key,
|
||||
size=int(obj.get("Size", 0)),
|
||||
last_modified=last_modified.isoformat() if last_modified else None,
|
||||
)
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,59 @@
|
||||
"""SQLAlchemy 引擎与 Session 工厂。"""
|
||||
from contextlib import contextmanager
|
||||
from typing import Iterator
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
# SQLite 需要 check_same_thread=False(虽然我们用 FastAPI async/threadpool,但仍建议)
|
||||
connect_args = {}
|
||||
if settings.DATABASE_URL.startswith("sqlite"):
|
||||
connect_args["check_same_thread"] = False
|
||||
|
||||
engine = create_engine(
|
||||
settings.DATABASE_URL,
|
||||
connect_args=connect_args,
|
||||
pool_pre_ping=True,
|
||||
future=True,
|
||||
)
|
||||
|
||||
SessionLocal = sessionmaker(
|
||||
bind=engine,
|
||||
autoflush=False,
|
||||
autocommit=False,
|
||||
expire_on_commit=False,
|
||||
class_=Session,
|
||||
)
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
"""所有 ORM 模型的基类。"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def get_db() -> Iterator[Session]:
|
||||
"""FastAPI 依赖:每次请求一个 Session。"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def session_scope() -> Iterator[Session]:
|
||||
"""普通上下文管理器版本,供非请求路径使用(如初始化、调度器)。"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,52 @@
|
||||
"""FastAPI 依赖注入:当前用户、DB Session 等。"""
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends, HTTPException, status
|
||||
from fastapi.security import OAuth2PasswordBearer
|
||||
from jose import JWTError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import get_db
|
||||
from app.models.user import User
|
||||
from app.security import decode_token
|
||||
|
||||
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/auth/login", auto_error=False)
|
||||
|
||||
DBSession = Annotated[Session, Depends(get_db)]
|
||||
TokenDep = Annotated[str | None, Depends(oauth2_scheme)]
|
||||
|
||||
|
||||
def get_current_user(
|
||||
token: TokenDep,
|
||||
db: DBSession,
|
||||
) -> User:
|
||||
if not token:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Not authenticated",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
try:
|
||||
payload = decode_token(token)
|
||||
username: str | None = payload.get("sub")
|
||||
if not username:
|
||||
raise HTTPException(status_code=401, detail="Invalid token")
|
||||
except JWTError:
|
||||
raise HTTPException(status_code=401, detail="Invalid or expired token")
|
||||
|
||||
user = db.query(User).filter(User.username == username).first()
|
||||
if not user or not user.is_active:
|
||||
raise HTTPException(status_code=401, detail="User not found or inactive")
|
||||
return user
|
||||
|
||||
|
||||
CurrentUser = Annotated[User, Depends(get_current_user)]
|
||||
|
||||
|
||||
def require_admin(user: CurrentUser) -> User:
|
||||
if not user.is_admin:
|
||||
raise HTTPException(status_code=403, detail="Admin privilege required")
|
||||
return user
|
||||
|
||||
|
||||
AdminUser = Annotated[User, Depends(require_admin)]
|
||||
@@ -0,0 +1,99 @@
|
||||
"""FastAPI 应用入口:lifespan 管理 scheduler、首次启动创建 admin。"""
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
from app.api.router import api_router
|
||||
from app.config import get_settings
|
||||
from app.database import Base, SessionLocal, engine
|
||||
from app.models.user import User
|
||||
from app.scheduler.scheduler import scheduler_lifespan
|
||||
from app.security import hash_password
|
||||
from app.utils.logging import setup_logging
|
||||
|
||||
setup_logging()
|
||||
log = logging.getLogger(__name__)
|
||||
settings = get_settings()
|
||||
|
||||
|
||||
def init_database() -> None:
|
||||
"""建表(Alembic 是首选;这里兜底确保首次启动也能跑)。"""
|
||||
try:
|
||||
# 实际生产用 alembic upgrade head;这里仅当表不存在时创建
|
||||
Base.metadata.create_all(bind=engine)
|
||||
log.info("Database tables ensured")
|
||||
except OperationalError as e:
|
||||
log.error("Database init failed: %s", e)
|
||||
raise
|
||||
|
||||
|
||||
def init_admin() -> None:
|
||||
"""首次启动创建 admin 账号(仅当 users 表为空时)。"""
|
||||
with SessionLocal() as db:
|
||||
try:
|
||||
existing = db.query(User).count()
|
||||
if existing > 0:
|
||||
log.info("Existing users found (%d), skip admin init", existing)
|
||||
return
|
||||
admin = User(
|
||||
username="admin",
|
||||
password_hash=hash_password(settings.INITIAL_ADMIN_PASSWORD),
|
||||
is_admin=True,
|
||||
is_active=True,
|
||||
)
|
||||
db.add(admin)
|
||||
db.commit()
|
||||
log.warning(
|
||||
"Initial admin account created. username=admin "
|
||||
"Please change the password after first login."
|
||||
)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
init_database()
|
||||
init_admin()
|
||||
async with scheduler_lifespan():
|
||||
log.info("Application ready")
|
||||
yield
|
||||
log.info("Application shutting down")
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="Backup System",
|
||||
description="数据备份管理系统 - MySQL / 目录 备份到本地 / S3",
|
||||
version="0.1.0",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_origins_list,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/health")
|
||||
def health() -> dict:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
app.include_router(api_router, prefix="/api")
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def global_exception_handler(request, exc):
|
||||
log.exception("Unhandled error on %s %s", request.method, request.url)
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"detail": f"Internal server error: {type(exc).__name__}"},
|
||||
)
|
||||
@@ -0,0 +1,7 @@
|
||||
"""ORM 模型集中导入,确保 Base.metadata 能识别所有表。"""
|
||||
from app.models.user import User # noqa: F401
|
||||
from app.models.storage import Storage # noqa: F401
|
||||
from app.models.job import Job # noqa: F401
|
||||
from app.models.run import Run, RunLog # noqa: F401
|
||||
from app.models.setting import Setting # noqa: F401
|
||||
from app.models.restore import RestoreHistory # noqa: F401
|
||||
@@ -0,0 +1,39 @@
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, String, Text, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class Job(Base):
|
||||
"""备份任务定义。
|
||||
|
||||
source_config_json 是 Fernet 加密的 JSON:
|
||||
- mysql: {"host", "port", "user", "password", "database"}
|
||||
- directory: {"path", "exclude_patterns": [...]}
|
||||
"""
|
||||
|
||||
__tablename__ = "jobs"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(64), unique=True, nullable=False)
|
||||
type: Mapped[str] = mapped_column(String(16), nullable=False) # 'mysql' / 'directory'
|
||||
source_config_json: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
storage_id: Mapped[int] = mapped_column(
|
||||
Integer, ForeignKey("storages.id", ondelete="RESTRICT"), nullable=False
|
||||
)
|
||||
cron_expression: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
retention_count: Mapped[int] = mapped_column(Integer, nullable=False, default=7)
|
||||
retention_days: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
description: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
|
||||
storage: Mapped["Storage"] = relationship(lazy="joined") # noqa: F821
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, server_default=func.now(), nullable=False
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, server_default=func.now(), onupdate=func.now(), nullable=False
|
||||
)
|
||||
@@ -0,0 +1,25 @@
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class RestoreHistory(Base):
|
||||
"""恢复执行历史。"""
|
||||
|
||||
__tablename__ = "restore_history"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
run_id: Mapped[int] = mapped_column(
|
||||
Integer, ForeignKey("runs.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
target: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending")
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, server_default=func.now(), nullable=False
|
||||
)
|
||||
@@ -0,0 +1,61 @@
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class Run(Base):
|
||||
"""单次备份执行记录。"""
|
||||
|
||||
__tablename__ = "runs"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
job_id: Mapped[int] = mapped_column(
|
||||
Integer, ForeignKey("jobs.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending")
|
||||
trigger: Mapped[str] = mapped_column(String(16), nullable=False, default="manual")
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
||||
duration_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
artifact_path: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
artifact_size: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||
artifact_checksum: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
log_excerpt: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
job: Mapped["Job"] = relationship(lazy="joined") # noqa: F821
|
||||
logs: Mapped[list["RunLog"]] = relationship( # noqa: F821
|
||||
back_populates="run", cascade="all, delete-orphan", lazy="select"
|
||||
)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, server_default=func.now(), nullable=False
|
||||
)
|
||||
|
||||
|
||||
class RunLog(Base):
|
||||
"""完整日志(可独立表以避免 runs 表臃肿)。"""
|
||||
|
||||
__tablename__ = "run_logs"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
run_id: Mapped[int] = mapped_column(
|
||||
Integer, ForeignKey("runs.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
content: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, server_default=func.now(), nullable=False
|
||||
)
|
||||
|
||||
run: Mapped["Run"] = relationship(back_populates="logs")
|
||||
@@ -0,0 +1,18 @@
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, String, Text, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class Setting(Base):
|
||||
"""系统级 KV 设置。"""
|
||||
|
||||
__tablename__ = "settings"
|
||||
|
||||
key: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
value: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, server_default=func.now(), onupdate=func.now(), nullable=False
|
||||
)
|
||||
@@ -0,0 +1,31 @@
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, Integer, String, Text, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class Storage(Base):
|
||||
"""备份目标存储(local 路径 或 S3 桶)。
|
||||
|
||||
config_json 是 Fernet 加密的 JSON 字符串,结构因 type 而异:
|
||||
- local: {"path": "/data/backups"}
|
||||
- s3: {"endpoint_url": "...", "region": "...", "bucket": "...",
|
||||
"access_key": "...", "secret_key": "...", "use_ssl": true,
|
||||
"path_prefix": "optional/prefix"}
|
||||
"""
|
||||
|
||||
__tablename__ = "storages"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(64), unique=True, nullable=False)
|
||||
type: Mapped[str] = mapped_column(String(16), nullable=False) # 'local' / 's3'
|
||||
config_json: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
is_default: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, server_default=func.now(), nullable=False
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, server_default=func.now(), onupdate=func.now(), nullable=False
|
||||
)
|
||||
@@ -0,0 +1,22 @@
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, Integer, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
username: Mapped[str] = mapped_column(String(64), unique=True, nullable=False, index=True)
|
||||
password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
is_admin: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime, server_default=func.now(), nullable=False
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<User id={self.id} username={self.username}>"
|
||||
@@ -0,0 +1,131 @@
|
||||
"""APScheduler 调度器:FastAPI lifespan 中启动/关闭。"""
|
||||
import asyncio
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
|
||||
from app.config import get_settings
|
||||
from app.database import SessionLocal
|
||||
from app.models.job import Job
|
||||
from app.models.run import Run
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
settings = get_settings()
|
||||
|
||||
_scheduler: Optional[AsyncIOScheduler] = None
|
||||
|
||||
|
||||
async def _execute_scheduled_job(job_id: int) -> None:
|
||||
"""调度器回调:创建 run 记录并启动执行。"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
job = db.get(Job, job_id)
|
||||
if not job or not job.enabled:
|
||||
return
|
||||
run = Run(
|
||||
job_id=job.id,
|
||||
status="pending",
|
||||
trigger="scheduled",
|
||||
started_at=datetime.utcnow(),
|
||||
)
|
||||
db.add(run)
|
||||
db.commit()
|
||||
db.refresh(run)
|
||||
run_id = run.id
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
from app.core.executor import run_backup
|
||||
|
||||
asyncio.create_task(run_backup(run_id, trigger="scheduled"))
|
||||
|
||||
|
||||
def _register_job(job: Job) -> Optional[str]:
|
||||
"""注册一个 job 的 cron trigger,返回 APScheduler job id。"""
|
||||
assert _scheduler is not None
|
||||
if not job.enabled or not job.cron_expression:
|
||||
return None
|
||||
try:
|
||||
trigger = CronTrigger.from_crontab(job.cron_expression, timezone=settings.TZ)
|
||||
except Exception as e:
|
||||
log.error("Invalid cron for job %s: %s", job.name, e)
|
||||
return None
|
||||
sched_job = _scheduler.add_job(
|
||||
_execute_scheduled_job,
|
||||
trigger=trigger,
|
||||
args=[job.id],
|
||||
id=f"job-{job.id}",
|
||||
replace_existing=True,
|
||||
misfire_grace_time=600,
|
||||
coalesce=True,
|
||||
)
|
||||
log.info("Registered scheduled job: id=%s name=%s cron=%s", job.id, job.name, job.cron_expression)
|
||||
return sched_job.id
|
||||
|
||||
|
||||
def _reload_all_jobs() -> None:
|
||||
"""从 DB 重新加载所有 enabled job。"""
|
||||
assert _scheduler is not None
|
||||
# 移除现有
|
||||
for sj in _scheduler.get_jobs():
|
||||
sj.remove()
|
||||
db = SessionLocal()
|
||||
try:
|
||||
jobs = db.query(Job).filter(Job.enabled == True, Job.cron_expression.isnot(None)).all() # noqa: E712
|
||||
for j in jobs:
|
||||
_register_job(j)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def scheduler_lifespan():
|
||||
global _scheduler
|
||||
_scheduler = AsyncIOScheduler(timezone=settings.TZ)
|
||||
_scheduler.start()
|
||||
log.info("Scheduler started")
|
||||
try:
|
||||
_reload_all_jobs()
|
||||
yield
|
||||
finally:
|
||||
_scheduler.shutdown(wait=False)
|
||||
log.info("Scheduler stopped")
|
||||
_scheduler = None
|
||||
|
||||
|
||||
def reschedule_job(job_id: int) -> None:
|
||||
"""job 更新后调用此函数重新注册。"""
|
||||
if _scheduler is None:
|
||||
return
|
||||
sched_id = f"job-{job_id}"
|
||||
# 移除旧的
|
||||
try:
|
||||
existing = _scheduler.get_job(sched_id)
|
||||
if existing:
|
||||
existing.remove()
|
||||
except Exception:
|
||||
pass
|
||||
# 重新加载
|
||||
db = SessionLocal()
|
||||
try:
|
||||
job = db.get(Job, job_id)
|
||||
if job:
|
||||
_register_job(job)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def remove_job(job_id: int) -> None:
|
||||
if _scheduler is None:
|
||||
return
|
||||
try:
|
||||
existing = _scheduler.get_job(f"job-{job_id}")
|
||||
if existing:
|
||||
existing.remove()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,34 @@
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class LoginRequest(BaseModel):
|
||||
username: str = Field(..., min_length=1, max_length=64)
|
||||
password: str = Field(..., min_length=1)
|
||||
|
||||
|
||||
class TokenResponse(BaseModel):
|
||||
access_token: str
|
||||
token_type: str = "bearer"
|
||||
expires_in: int
|
||||
|
||||
|
||||
class UserOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
id: int
|
||||
username: str
|
||||
is_admin: bool
|
||||
is_active: bool
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class ChangePasswordRequest(BaseModel):
|
||||
old_password: str = Field(..., min_length=1)
|
||||
new_password: str = Field(..., min_length=8)
|
||||
|
||||
|
||||
class MessageResponse(BaseModel):
|
||||
message: str
|
||||
detail: Optional[str] = None
|
||||
@@ -0,0 +1,96 @@
|
||||
from datetime import datetime
|
||||
from typing import Literal, Optional
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
from app.schemas.storage import StorageOut
|
||||
|
||||
|
||||
class MySQLSourceConfig(BaseModel):
|
||||
host: str = Field(..., min_length=1)
|
||||
port: int = Field(3306, ge=1, le=65535)
|
||||
user: str = Field(..., min_length=1)
|
||||
password: str = Field("", description="可空(trust auth)")
|
||||
database: str = Field(..., min_length=1, description="要备份的库名")
|
||||
extra_args: str = Field("", description="额外 mysqldump 参数,如 '--skip-lock-tables'")
|
||||
|
||||
|
||||
class DirectorySourceConfig(BaseModel):
|
||||
path: str = Field(..., min_length=1, description="要备份的目录绝对路径")
|
||||
exclude_patterns: list[str] = Field(
|
||||
default_factory=lambda: [".DS_Store", "__pycache__", "node_modules", ".git"]
|
||||
)
|
||||
|
||||
|
||||
class JobBase(BaseModel):
|
||||
name: str = Field(..., min_length=1, max_length=64)
|
||||
type: Literal["mysql", "directory"]
|
||||
source_config: dict
|
||||
storage_id: int
|
||||
cron_expression: Optional[str] = Field(None, description="标准 5 段 cron 表达式,空表示手动")
|
||||
enabled: bool = True
|
||||
retention_count: int = Field(7, ge=1, le=365)
|
||||
retention_days: Optional[int] = Field(None, ge=1, le=3650)
|
||||
description: Optional[str] = Field(None, max_length=255)
|
||||
|
||||
@field_validator("cron_expression")
|
||||
@classmethod
|
||||
def _check_cron(cls, v: Optional[str]) -> Optional[str]:
|
||||
if v is None or v.strip() == "":
|
||||
return None
|
||||
v = v.strip()
|
||||
parts = v.split()
|
||||
if len(parts) != 5:
|
||||
raise ValueError("cron 必须是 5 段:分 时 日 月 周")
|
||||
return v
|
||||
|
||||
|
||||
class JobCreate(JobBase):
|
||||
pass
|
||||
|
||||
|
||||
class JobUpdate(BaseModel):
|
||||
name: Optional[str] = Field(None, min_length=1, max_length=64)
|
||||
source_config: Optional[dict] = None
|
||||
storage_id: Optional[int] = None
|
||||
cron_expression: Optional[str] = None
|
||||
enabled: Optional[bool] = None
|
||||
retention_count: Optional[int] = Field(None, ge=1, le=365)
|
||||
retention_days: Optional[int] = Field(None, ge=1, le=3650)
|
||||
description: Optional[str] = None
|
||||
|
||||
|
||||
class JobOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
id: int
|
||||
name: str
|
||||
type: str
|
||||
cron_expression: Optional[str]
|
||||
enabled: bool
|
||||
retention_count: int
|
||||
retention_days: Optional[int]
|
||||
description: Optional[str]
|
||||
storage: StorageOut
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class RunSummary(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
id: int
|
||||
status: str
|
||||
trigger: str
|
||||
started_at: Optional[datetime]
|
||||
finished_at: Optional[datetime]
|
||||
duration_seconds: Optional[int]
|
||||
artifact_size: Optional[int]
|
||||
error_message: Optional[str]
|
||||
|
||||
|
||||
class JobWithLastRun(JobOut):
|
||||
last_run: Optional[RunSummary] = None
|
||||
|
||||
|
||||
class JobRunResponse(BaseModel):
|
||||
run_id: int
|
||||
status: str
|
||||
@@ -0,0 +1,33 @@
|
||||
from datetime import datetime
|
||||
from typing import Literal, Optional
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class RestoreTargetsResponse(BaseModel):
|
||||
"""列出某个 run 可恢复的目标元信息。"""
|
||||
|
||||
run_id: int
|
||||
job_type: str # 'mysql' / 'directory'
|
||||
artifact_size: Optional[int]
|
||||
artifact_checksum: Optional[str]
|
||||
suggested_targets: list[str]
|
||||
|
||||
|
||||
class RestoreRequest(BaseModel):
|
||||
run_id: int
|
||||
target_type: Literal["mysql_db", "directory_path"]
|
||||
target: str = Field(..., min_length=1, description="MySQL 库名 或 目录路径")
|
||||
options: dict = Field(default_factory=dict, description="额外选项,如 {'drop_existing': true}")
|
||||
|
||||
|
||||
class RestoreOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
id: int
|
||||
run_id: int
|
||||
target: str
|
||||
status: str
|
||||
started_at: Optional[datetime]
|
||||
finished_at: Optional[datetime]
|
||||
error_message: Optional[str]
|
||||
created_at: datetime
|
||||
@@ -0,0 +1,32 @@
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from app.schemas.job import RunSummary
|
||||
|
||||
|
||||
class RunOut(RunSummary):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
job_id: int
|
||||
artifact_path: Optional[str]
|
||||
artifact_checksum: Optional[str]
|
||||
log_excerpt: Optional[str]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class RunLogOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
id: int
|
||||
content: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class RunListResponse(BaseModel):
|
||||
total: int
|
||||
items: list[RunOut]
|
||||
|
||||
|
||||
class RunTriggerResponse(BaseModel):
|
||||
run_id: int
|
||||
status: str
|
||||
@@ -0,0 +1,13 @@
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class SettingOut(BaseModel):
|
||||
key: str
|
||||
value: str
|
||||
|
||||
|
||||
class SettingUpdate(BaseModel):
|
||||
key: str
|
||||
value: Optional[str] = None
|
||||
@@ -0,0 +1,50 @@
|
||||
from datetime import datetime
|
||||
from typing import Literal, Optional
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class LocalStorageConfig(BaseModel):
|
||||
path: str = Field(..., min_length=1, description="本地存储根目录绝对路径")
|
||||
|
||||
|
||||
class S3StorageConfig(BaseModel):
|
||||
endpoint_url: Optional[str] = Field(None, description="MinIO/Ceph 自定义 endpoint,留空使用 AWS")
|
||||
region: str = Field("us-east-1")
|
||||
bucket: str = Field(..., min_length=1)
|
||||
access_key: str = Field(..., min_length=1)
|
||||
secret_key: str = Field(..., min_length=1)
|
||||
use_ssl: bool = True
|
||||
path_prefix: str = Field("", description="对象 key 前缀,可空")
|
||||
addressing_style: Literal["auto", "path", "virtual"] = "auto"
|
||||
|
||||
|
||||
class StorageBase(BaseModel):
|
||||
name: str = Field(..., min_length=1, max_length=64)
|
||||
type: Literal["local", "s3"]
|
||||
is_default: bool = False
|
||||
|
||||
|
||||
class StorageCreate(StorageBase):
|
||||
config: dict
|
||||
|
||||
|
||||
class StorageUpdate(BaseModel):
|
||||
name: Optional[str] = Field(None, min_length=1, max_length=64)
|
||||
is_default: Optional[bool] = None
|
||||
config: Optional[dict] = None
|
||||
|
||||
|
||||
class StorageOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
id: int
|
||||
name: str
|
||||
type: str
|
||||
is_default: bool
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class StorageTestResponse(BaseModel):
|
||||
ok: bool
|
||||
message: str
|
||||
@@ -0,0 +1,35 @@
|
||||
"""JWT 与密码哈希工具。"""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Optional
|
||||
|
||||
from jose import JWTError, jwt
|
||||
from passlib.context import CryptContext
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
||||
|
||||
ALGORITHM = "HS256"
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return pwd_context.hash(password)
|
||||
|
||||
|
||||
def verify_password(plain: str, hashed: str) -> bool:
|
||||
return pwd_context.verify(plain, hashed)
|
||||
|
||||
|
||||
def create_access_token(subject: str, extra: Optional[dict[str, Any]] = None) -> str:
|
||||
expire = datetime.now(timezone.utc) + timedelta(minutes=settings.JWT_EXPIRE_MINUTES)
|
||||
payload: dict[str, Any] = {"sub": subject, "exp": expire}
|
||||
if extra:
|
||||
payload.update(extra)
|
||||
return jwt.encode(payload, settings.SECRET_KEY, algorithm=ALGORITHM)
|
||||
|
||||
|
||||
def decode_token(token: str) -> dict[str, Any]:
|
||||
"""解码 token,失败抛 JWTError。"""
|
||||
return jwt.decode(token, settings.SECRET_KEY, algorithms=[ALGORITHM])
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Fernet 对称加密工具:用于加密存储敏感字段(MySQL 密码、S3 SecretKey)。"""
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
_fernet: Fernet | None = None
|
||||
|
||||
|
||||
def _get_fernet() -> Fernet:
|
||||
global _fernet
|
||||
if _fernet is None:
|
||||
_fernet = Fernet(get_settings().FERNET_KEY.encode())
|
||||
return _fernet
|
||||
|
||||
|
||||
def encrypt_str(plain: str) -> str:
|
||||
"""加密字符串,返回 base64-urlsafe 字符串(可安全存数据库 Text 字段)。"""
|
||||
if plain is None:
|
||||
return ""
|
||||
return _get_fernet().encrypt(plain.encode()).decode()
|
||||
|
||||
|
||||
def decrypt_str(token: str) -> str:
|
||||
"""解密字符串。空字符串返回空字符串。"""
|
||||
if not token:
|
||||
return ""
|
||||
try:
|
||||
return _get_fernet().decrypt(token.encode()).decode()
|
||||
except InvalidToken as e:
|
||||
raise ValueError(f"Failed to decrypt token (FERNET_KEY 可能已被更改): {e}") from e
|
||||
|
||||
|
||||
def encrypt_dict(data: dict) -> str:
|
||||
"""加密字典:序列化为 JSON 再加密。"""
|
||||
import json
|
||||
|
||||
return encrypt_str(json.dumps(data, ensure_ascii=False))
|
||||
|
||||
|
||||
def decrypt_dict(token: str) -> dict:
|
||||
"""解密字典。"""
|
||||
import json
|
||||
|
||||
if not token:
|
||||
return {}
|
||||
return json.loads(decrypt_str(token))
|
||||
@@ -0,0 +1,50 @@
|
||||
"""日志配置:控制台 + 文件 + 单 run StringIO Tee。"""
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from pathlib import Path
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
_settings = get_settings()
|
||||
|
||||
_LOG_DIR = Path("logs")
|
||||
_LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def setup_logging() -> None:
|
||||
"""应用启动时调用一次。"""
|
||||
root = logging.getLogger()
|
||||
root.setLevel(_settings.LOG_LEVEL.upper())
|
||||
|
||||
# 避免重复添加 handler(uvicorn 重载时)
|
||||
if root.handlers:
|
||||
return
|
||||
|
||||
fmt = logging.Formatter(
|
||||
"%(asctime)s | %(levelname)-7s | %(name)s | %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
|
||||
# 控制台
|
||||
console = logging.StreamHandler(sys.stdout)
|
||||
console.setFormatter(fmt)
|
||||
root.addHandler(console)
|
||||
|
||||
# 文件
|
||||
log_file = _LOG_DIR / f"app-{datetime.now():%Y-%m-%d}.log"
|
||||
file_handler = RotatingFileHandler(
|
||||
log_file, maxBytes=20 * 1024 * 1024, backupCount=10, encoding="utf-8"
|
||||
)
|
||||
file_handler.setFormatter(fmt)
|
||||
root.addHandler(file_handler)
|
||||
|
||||
# 调整 uvicorn 噪音
|
||||
logging.getLogger("uvicorn.access").setLevel(logging.WARNING)
|
||||
logging.getLogger("apscheduler.scheduler").setLevel(logging.INFO)
|
||||
|
||||
|
||||
def get_logger(name: str) -> logging.Logger:
|
||||
return logging.getLogger(name)
|
||||
@@ -0,0 +1,15 @@
|
||||
fastapi==0.115.0
|
||||
uvicorn[standard]==0.30.6
|
||||
pydantic==2.9.2
|
||||
pydantic-settings==2.5.2
|
||||
sqlalchemy==2.0.35
|
||||
alembic==1.13.3
|
||||
python-jose[cryptography]==3.3.0
|
||||
passlib[bcrypt]==1.7.4
|
||||
bcrypt==4.0.1
|
||||
python-multipart==0.0.12
|
||||
apscheduler==3.10.4
|
||||
aiofiles==24.1.0
|
||||
aioboto3==13.1.1
|
||||
cryptography==43.0.1
|
||||
httpx==0.27.2
|
||||
Reference in New Issue
Block a user