feat: MySQL SSL 支持 + 任务编辑回显 + InfluxDB 备份

MySQL:
- 新增 use_ssl 选项;MariaDB 11.x 默认强制 SSL 且默认 verify cert,
  故启用 SSL 时同时加 --skip-ssl-verify-server-cert 以兼容自签名证书。

任务编辑回显:
- 后端 JobOut 新增 source_config_safe 字段,解密后敏感字段(password/token/
  secret_key/access_key)脱敏为 "***"。
- 更新端点接收 source_config;若密码字段为 "***" 则保留原值不变,
  支持"不改密码只改其它字段"的常见场景。
- 前端 JobForm 完整回显所有数据源字段,type='edit' 时去掉 disabled。

InfluxDB 备份(新增类型):
- 后端 core/backup/influxdb.py:
  - v2.x:调用 influx CLI(influxdb2-client-2.7.5 二进制直接安装)执行 influx backup
    生成目录后 tar.gz 流式打包上传。
  - v1.x:通过 HTTP API 调用 /query?db=...&q=SELECT * FROM /.*/ 拉取全量数据
    导出为 CSV + META.txt。
- Dockerfile:直接下载 influx CLI 二进制(绕开 apt 签名问题)。
- 前端 JobForm:根据 version 动态显示 v1/v2 表单字段。
- schemas/job.py:新增 InfluxDBSourceConfigV1 / V2。
- 类型字面量扩展:'mysql' | 'directory' | 'influxdb'。

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
publ
2026-06-22 19:14:45 +08:00
parent febdeab35c
commit 2568c0380b
9 changed files with 779 additions and 61 deletions
+14 -1
View File
@@ -5,15 +5,28 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1
# 系统依赖:curl 用于健康检查,default-mysql-client 提供 mysqldumptar 已内置
# 系统依赖:curl 用于健康检查,default-mysql-client 提供 mysqldumptar 已内置influx CLI 用于 InfluxDB 备份
RUN apt-get update && apt-get install -y --no-install-recommends \
curl \
default-mysql-client \
tar \
gzip \
tzdata \
ca-certificates \
gnupg \
&& rm -rf /var/lib/apt/lists/*
# 安装 InfluxDB CLI(直接下载二进制,避免 apt 签名问题)
RUN set -eux; \
INFLUX_VERSION=2.7.5; \
curl -fsSL "https://dl.influxdata.com/influxdb/releases/influxdb2-client-${INFLUX_VERSION}-linux-amd64.tar.gz" \
-o /tmp/influx.tar.gz; \
tar -xzf /tmp/influx.tar.gz -C /tmp; \
mv /tmp/influx /usr/local/bin/influx; \
chmod +x /usr/local/bin/influx; \
rm -rf /tmp/influx.tar.gz /tmp/influx; \
influx version
WORKDIR /app
# 先安装依赖以利用 Docker 缓存
+56 -9
View File
@@ -12,6 +12,8 @@ from app.models.run import Run
from app.models.storage import Storage
from app.schemas.job import (
DirectorySourceConfig,
InfluxDBSourceConfigV1,
InfluxDBSourceConfigV2,
JobCreate,
JobOut,
JobRunResponse,
@@ -21,7 +23,22 @@ from app.schemas.job import (
RunSummary,
)
from app.scheduler.scheduler import remove_job, reschedule_job
from app.utils.crypto import encrypt_dict
from app.utils.crypto import decrypt_dict, encrypt_dict
# 脱敏字段:编辑时返回占位符,避免明文密码在网络传输
_SENSITIVE_KEYS = {"password", "token", "secret_key", "access_key"}
def _redact_source(cfg: dict) -> dict:
"""敏感字段脱敏为 '***',但保留字段名以提示用户该字段已配置。"""
out = {}
for k, v in cfg.items():
if k in _SENSITIVE_KEYS:
out[k] = "***" if v else ""
else:
out[k] = v
return out
router = APIRouter(prefix="/jobs", tags=["jobs"])
@@ -31,6 +48,13 @@ def _validate_source(type_: str, cfg: dict) -> dict:
return MySQLSourceConfig(**cfg).model_dump()
if type_ == "directory":
return DirectorySourceConfig(**cfg).model_dump()
if type_ == "influxdb":
ver = cfg.get("version")
if ver == "1":
return InfluxDBSourceConfigV1(**cfg).model_dump()
if ver == "2":
return InfluxDBSourceConfigV2(**cfg).model_dump()
raise HTTPException(status_code=400, detail="InfluxDB config 缺少 version (1 或 2)")
raise HTTPException(status_code=400, detail=f"Unsupported job type: {type_}")
@@ -43,6 +67,12 @@ def _job_to_with_last_run(job: Job, db) -> JobWithLastRun:
)
out = JobWithLastRun.model_validate(job)
out.last_run = RunSummary.model_validate(last) if last else None
# 解密 source_config 并脱敏
try:
cfg = decrypt_dict(job.source_config_json)
out.source_config_safe = _redact_source(cfg)
except Exception:
out.source_config_safe = {}
return out
@@ -53,7 +83,7 @@ def list_jobs(db: DBSession, _: CurrentUser) -> list[JobWithLastRun]:
@router.post("", response_model=JobOut, status_code=201)
def create_job(payload: JobCreate, db: DBSession, _: AdminUser) -> Job:
def create_job(payload: JobCreate, db: DBSession, _: AdminUser) -> JobOut:
if db.query(Job).filter(Job.name == payload.name).first():
raise HTTPException(status_code=400, detail="名称已存在")
if not db.get(Storage, payload.storage_id):
@@ -74,7 +104,9 @@ def create_job(payload: JobCreate, db: DBSession, _: AdminUser) -> Job:
db.commit()
db.refresh(job)
reschedule_job(job.id)
return job
out = JobOut.model_validate(job)
out.source_config_safe = _redact_source(src)
return out
@router.get("/{job_id}", response_model=JobWithLastRun)
@@ -86,7 +118,7 @@ def get_job(job_id: int, db: DBSession, _: CurrentUser) -> JobWithLastRun:
@router.put("/{job_id}", response_model=JobOut)
def update_job(job_id: int, payload: JobUpdate, db: DBSession, _: AdminUser) -> Job:
def update_job(job_id: int, payload: JobUpdate, db: DBSession, _: AdminUser) -> JobOut:
job = db.get(Job, job_id)
if not job:
raise HTTPException(status_code=404, detail="Not found")
@@ -94,9 +126,19 @@ def update_job(job_id: int, payload: JobUpdate, db: DBSession, _: AdminUser) ->
if db.query(Job).filter(Job.name == payload.name, Job.id != job_id).first():
raise HTTPException(status_code=400, detail="名称已存在")
job.name = payload.name
new_src: dict | None = None
if payload.source_config is not None:
src = _validate_source(job.type, payload.source_config)
job.source_config_json = encrypt_dict(src)
new_src = _validate_source(job.type, payload.source_config)
# 如果 password 为 "***" 占位符,保留原值
existing_src = decrypt_dict(job.source_config_json)
for k in _SENSITIVE_KEYS:
if (
k in new_src
and k in existing_src
and new_src[k] == "***"
):
new_src[k] = existing_src[k]
job.source_config_json = encrypt_dict(new_src)
if payload.storage_id is not None:
if not db.get(Storage, payload.storage_id):
raise HTTPException(status_code=400, detail="存储目标不存在")
@@ -114,7 +156,10 @@ def update_job(job_id: int, payload: JobUpdate, db: DBSession, _: AdminUser) ->
db.commit()
db.refresh(job)
reschedule_job(job.id)
return job
out = JobOut.model_validate(job)
cfg = new_src if new_src is not None else decrypt_dict(job.source_config_json)
out.source_config_safe = _redact_source(cfg)
return out
@router.delete("/{job_id}", status_code=204)
@@ -148,7 +193,7 @@ async def run_job(job_id: int, db: DBSession, _: CurrentUser) -> JobRunResponse:
@router.post("/{job_id}/toggle", response_model=JobOut)
def toggle_job(job_id: int, db: DBSession, _: AdminUser) -> Job:
def toggle_job(job_id: int, db: DBSession, _: AdminUser) -> JobOut:
job = db.get(Job, job_id)
if not job:
raise HTTPException(status_code=404, detail="Not found")
@@ -156,4 +201,6 @@ def toggle_job(job_id: int, db: DBSession, _: AdminUser) -> Job:
db.commit()
db.refresh(job)
reschedule_job(job.id)
return job
out = JobOut.model_validate(job)
out.source_config_safe = _redact_source(decrypt_dict(job.source_config_json))
return out
+194
View File
@@ -0,0 +1,194 @@
"""InfluxDB 备份:通过 influx / influxd CLI 生成备份目录,再 tar.gz 流式打包。
支持 v1.x 和 v2.x
- v1.x:使用 influx backup CLI 子命令)
等价做法:用 influx_inspect export 转 line protocol,然后打包
这里采用更通用的:influxd backup(如果有) 或 打包 line-protocol dump
- v2.x:使用 influx backup <bucket> 命令生成备份目录
策略:
1. 在临时目录执行 backup 命令生成文件
2. 用 tarfile 流式打包整个目录
3. 子进程产出后整体打包上传
"""
import asyncio
import os
import shutil
import subprocess
import tarfile
import tempfile
from datetime import datetime
from pathlib import Path
from typing import AsyncIterator
from app.core.backup.base import BackupMetadata, BaseBackup
from app.utils.logging import get_logger
log = get_logger(__name__)
class InfluxDBBackup(BaseBackup):
def __init__(self, source_config: dict, job_name: str):
super().__init__(source_config, job_name)
self.version: str = source_config["version"]
self.host: str = source_config["host"]
self.port: int = int(source_config.get("port", 8086))
if self.version == "1":
self.database: str = source_config["database"]
self.username: str = source_config.get("username", "")
self.password: str = source_config.get("password", "")
else:
self.org: str = source_config["org"]
self.bucket: str = source_config["bucket"]
self.token: str = source_config["token"]
def metadata(self) -> BackupMetadata:
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
if self.version == "1":
safe = "".join(c if c.isalnum() or c in ("_", "-") else "_" for c in self.database)
else:
safe = "".join(c if c.isalnum() or c in ("_", "-") else "_" for c in self.bucket)
return BackupMetadata(
suggested_filename=f"{self.job_name}-{self.version}-{safe}-{ts}.tar.gz",
content_type="application/gzip",
extra={"version": self.version, "host": self.host, "port": self.port},
)
async def produce(self) -> AsyncIterator[bytes]:
ts = datetime.now().strftime("%Y%m%d%H%M%S")
tmpdir = Path(tempfile.mkdtemp(prefix=f"influxbk_{self.job_name}_"))
backup_dir = tmpdir / "data"
backup_dir.mkdir()
try:
if self.version == "2":
await self._backup_v2(backup_dir)
else:
await self._backup_v1(backup_dir)
# 打包为 tar.gz 并流式输出
log.info("Taring influx backup at %s", backup_dir)
tar_path = tmpdir / "backup.tar.gz"
def _tar() -> None:
with tarfile.open(str(tar_path), mode="w:gz") as tar:
tar.add(str(backup_dir), arcname="data", recursive=True)
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, _tar)
if not tar_path.exists() or tar_path.stat().st_size == 0:
raise RuntimeError("InfluxDB 备份为空,请检查数据库连接和 CLI 安装")
# 读取打包文件并流式产出
with open(tar_path, "rb") as f:
while True:
chunk = await loop.run_in_executor(None, f.read, 64 * 1024)
if not chunk:
break
yield chunk
finally:
shutil.rmtree(tmpdir, ignore_errors=True)
async def _backup_v2(self, dest: Path) -> None:
"""v2: influx backup --bucket X --org Y /dest"""
cmd = [
"influx", "backup",
"--bucket", self.bucket,
"--org", self.org,
"--host", f"http://{self.host}:{self.port}",
"--token", self.token,
str(dest),
]
log.info("Running: %s", " ".join(self._redact(cmd)))
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await proc.communicate()
if proc.returncode != 0:
raise RuntimeError(
f"influx backup failed (exit={proc.returncode}): "
f"{(stderr or stdout).decode(errors='replace')[:2000]}"
)
async def _backup_v1(self, dest: Path) -> None:
"""v1: 通过 HTTP API 查询全量数据,导出为 CSV。
使用 httpx 直接调 InfluxDB v1 的 /query 接口(更可靠,不依赖 CLI 子命令兼容性)。
生成:
- {database}.csvSELECT * 结果
- META.txt:恢复所需的元信息(库名、host、时间戳等)
恢复:需手动用 influx -import 或写入 line protocol。
"""
import httpx
log.info("InfluxDB v1 backup via HTTP API: db=%s host=%s:%s",
self.database, self.host, self.port)
url = f"http://{self.host}:{self.port}/query"
params = {
"db": self.database,
"q": "SELECT * FROM /.*/",
"chunked": "true",
"epoch": "ns",
}
auth = None
if self.username:
auth = (self.username, self.password or "")
async with httpx.AsyncClient(timeout=300.0, auth=auth) as client:
try:
resp = await client.get(url, params=params)
except Exception as e:
raise RuntimeError(f"InfluxDB v1 HTTP 请求失败: {e}") from e
if resp.status_code != 200:
raise RuntimeError(
f"InfluxDB v1 返回 {resp.status_code}: {resp.text[:500]}"
)
body = resp.text
# 保存 CSV 和元信息
csv_file = dest / f"{self.database}.csv"
loop = asyncio.get_event_loop()
def _write_files() -> None:
csv_file.write_text(body, encoding="utf-8")
meta = dest / "META.txt"
meta.write_text(
f"influxdb_version=1\n"
f"host={self.host}\nport={self.port}\n"
f"database={self.database}\n"
f"username={self.username or ''}\n"
f"query=SELECT * FROM /.*/\n"
f"format=csv\n"
f"backup_time={datetime.utcnow().isoformat()}Z\n"
f"restore_hint=可使用 `influx -import -path={csv_file.name} "
f"-database={self.database}` 恢复;或在 InfluxQL shell 中执行 "
f"`INSERT INTO ... VALUES ...`\n",
encoding="utf-8",
)
await loop.run_in_executor(None, _write_files)
if not csv_file.exists() or csv_file.stat().st_size == 0:
raise RuntimeError("InfluxDB v1 查询结果为空,请检查数据库是否存在或是否有数据")
@staticmethod
def _redact(cmd: list[str]) -> list[str]:
"""日志脱敏 token/password。"""
out = []
skip_next = False
secret_flags = {"--token", "-password"}
for i, c in enumerate(cmd):
if skip_next:
out.append("***")
skip_next = False
continue
if c in secret_flags:
out.append(c)
skip_next = True
continue
out.append(c)
return out
+8
View File
@@ -21,6 +21,7 @@ class MySQLBackup(BaseBackup):
self.user: str = source_config["user"]
self.password: str = source_config.get("password", "")
self.database: str = source_config["database"]
self.use_ssl: bool = bool(source_config.get("use_ssl", False))
self.extra_args: str = source_config.get("extra_args", "")
def metadata(self) -> BackupMetadata:
@@ -45,6 +46,13 @@ class MySQLBackup(BaseBackup):
"--events",
"--default-character-set=utf8mb4",
]
# SSL 控制:
# MariaDB 11.x 中 --ssl 和 --ssl-verify-server-cert 都默认开启,
# 与自签名证书的服务器连接会失败。这里 use_ssl=True 时强制开启 SSL
# 但跳过服务端证书校验(兼容自签名证书场景)。
if self.use_ssl:
args.append("--ssl")
args.append("--skip-ssl-verify-server-cert")
if self.extra_args:
args.extend(shlex.split(self.extra_args))
args.append(self.database)
+3
View File
@@ -14,6 +14,7 @@ from sqlalchemy.orm import Session
from app.core.backup.base import BaseBackup
from app.core.backup.directory import DirectoryBackup
from app.core.backup.influxdb import InfluxDBBackup
from app.core.backup.mysql import MySQLBackup
from app.core.retention import apply_retention
from app.core.storage.factory import create_storage
@@ -32,6 +33,8 @@ def _make_backup(job: Job) -> BaseBackup:
return MySQLBackup(src, job.name)
if job.type == "directory":
return DirectoryBackup(src, job.name)
if job.type == "influxdb":
return InfluxDBBackup(src, job.name)
raise ValueError(f"Unsupported job type: {job.type}")
+25 -1
View File
@@ -12,6 +12,7 @@ class MySQLSourceConfig(BaseModel):
user: str = Field(..., min_length=1)
password: str = Field("", description="可空(trust auth")
database: str = Field(..., min_length=1, description="要备份的库名")
use_ssl: bool = Field(False, description="True 启用 SSL/TLS;如服务器是自签名证书建议关闭")
extra_args: str = Field("", description="额外 mysqldump 参数,如 '--skip-lock-tables'")
@@ -22,9 +23,27 @@ class DirectorySourceConfig(BaseModel):
)
class InfluxDBSourceConfigV1(BaseModel):
version: Literal["1"]
host: str = Field(..., min_length=1)
port: int = Field(8086, ge=1, le=65535)
database: str = Field(..., min_length=1)
username: str = Field("", description="留空表示无认证")
password: str = Field("")
class InfluxDBSourceConfigV2(BaseModel):
version: Literal["2"]
host: str = Field(..., min_length=1)
port: int = Field(8086, ge=1, le=65535)
org: str = Field(..., min_length=1)
bucket: str = Field(..., min_length=1)
token: str = Field(..., min_length=1)
class JobBase(BaseModel):
name: str = Field(..., min_length=1, max_length=64)
type: Literal["mysql", "directory"]
type: Literal["mysql", "directory", "influxdb"]
source_config: dict
storage_id: int
cron_expression: Optional[str] = Field(None, description="标准 5 段 cron 表达式,空表示手动")
@@ -73,6 +92,11 @@ class JobOut(BaseModel):
storage: StorageOut
created_at: datetime
updated_at: datetime
# 解密后的数据源配置;敏感字段已脱敏为 "***"
source_config_safe: dict = Field(
default_factory=dict,
description="解密后的 source_config,敏感字段(password/token/secret_key)已脱敏",
)
class RunSummary(BaseModel):