feat: MySQL SSL 支持 + 任务编辑回显 + InfluxDB 备份
MySQL:
- 新增 use_ssl 选项;MariaDB 11.x 默认强制 SSL 且默认 verify cert,
故启用 SSL 时同时加 --skip-ssl-verify-server-cert 以兼容自签名证书。
任务编辑回显:
- 后端 JobOut 新增 source_config_safe 字段,解密后敏感字段(password/token/
secret_key/access_key)脱敏为 "***"。
- 更新端点接收 source_config;若密码字段为 "***" 则保留原值不变,
支持"不改密码只改其它字段"的常见场景。
- 前端 JobForm 完整回显所有数据源字段,type='edit' 时去掉 disabled。
InfluxDB 备份(新增类型):
- 后端 core/backup/influxdb.py:
- v2.x:调用 influx CLI(influxdb2-client-2.7.5 二进制直接安装)执行 influx backup
生成目录后 tar.gz 流式打包上传。
- v1.x:通过 HTTP API 调用 /query?db=...&q=SELECT * FROM /.*/ 拉取全量数据
导出为 CSV + META.txt。
- Dockerfile:直接下载 influx CLI 二进制(绕开 apt 签名问题)。
- 前端 JobForm:根据 version 动态显示 v1/v2 表单字段。
- schemas/job.py:新增 InfluxDBSourceConfigV1 / V2。
- 类型字面量扩展:'mysql' | 'directory' | 'influxdb'。
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
+14
-1
@@ -5,15 +5,28 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
|
||||
# 系统依赖:curl 用于健康检查,default-mysql-client 提供 mysqldump,tar 已内置
|
||||
# 系统依赖:curl 用于健康检查,default-mysql-client 提供 mysqldump,tar 已内置,influx CLI 用于 InfluxDB 备份
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
default-mysql-client \
|
||||
tar \
|
||||
gzip \
|
||||
tzdata \
|
||||
ca-certificates \
|
||||
gnupg \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 安装 InfluxDB CLI(直接下载二进制,避免 apt 签名问题)
|
||||
RUN set -eux; \
|
||||
INFLUX_VERSION=2.7.5; \
|
||||
curl -fsSL "https://dl.influxdata.com/influxdb/releases/influxdb2-client-${INFLUX_VERSION}-linux-amd64.tar.gz" \
|
||||
-o /tmp/influx.tar.gz; \
|
||||
tar -xzf /tmp/influx.tar.gz -C /tmp; \
|
||||
mv /tmp/influx /usr/local/bin/influx; \
|
||||
chmod +x /usr/local/bin/influx; \
|
||||
rm -rf /tmp/influx.tar.gz /tmp/influx; \
|
||||
influx version
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 先安装依赖以利用 Docker 缓存
|
||||
|
||||
+56
-9
@@ -12,6 +12,8 @@ from app.models.run import Run
|
||||
from app.models.storage import Storage
|
||||
from app.schemas.job import (
|
||||
DirectorySourceConfig,
|
||||
InfluxDBSourceConfigV1,
|
||||
InfluxDBSourceConfigV2,
|
||||
JobCreate,
|
||||
JobOut,
|
||||
JobRunResponse,
|
||||
@@ -21,7 +23,22 @@ from app.schemas.job import (
|
||||
RunSummary,
|
||||
)
|
||||
from app.scheduler.scheduler import remove_job, reschedule_job
|
||||
from app.utils.crypto import encrypt_dict
|
||||
from app.utils.crypto import decrypt_dict, encrypt_dict
|
||||
|
||||
|
||||
# 脱敏字段:编辑时返回占位符,避免明文密码在网络传输
|
||||
_SENSITIVE_KEYS = {"password", "token", "secret_key", "access_key"}
|
||||
|
||||
|
||||
def _redact_source(cfg: dict) -> dict:
|
||||
"""敏感字段脱敏为 '***',但保留字段名以提示用户该字段已配置。"""
|
||||
out = {}
|
||||
for k, v in cfg.items():
|
||||
if k in _SENSITIVE_KEYS:
|
||||
out[k] = "***" if v else ""
|
||||
else:
|
||||
out[k] = v
|
||||
return out
|
||||
|
||||
router = APIRouter(prefix="/jobs", tags=["jobs"])
|
||||
|
||||
@@ -31,6 +48,13 @@ def _validate_source(type_: str, cfg: dict) -> dict:
|
||||
return MySQLSourceConfig(**cfg).model_dump()
|
||||
if type_ == "directory":
|
||||
return DirectorySourceConfig(**cfg).model_dump()
|
||||
if type_ == "influxdb":
|
||||
ver = cfg.get("version")
|
||||
if ver == "1":
|
||||
return InfluxDBSourceConfigV1(**cfg).model_dump()
|
||||
if ver == "2":
|
||||
return InfluxDBSourceConfigV2(**cfg).model_dump()
|
||||
raise HTTPException(status_code=400, detail="InfluxDB config 缺少 version (1 或 2)")
|
||||
raise HTTPException(status_code=400, detail=f"Unsupported job type: {type_}")
|
||||
|
||||
|
||||
@@ -43,6 +67,12 @@ def _job_to_with_last_run(job: Job, db) -> JobWithLastRun:
|
||||
)
|
||||
out = JobWithLastRun.model_validate(job)
|
||||
out.last_run = RunSummary.model_validate(last) if last else None
|
||||
# 解密 source_config 并脱敏
|
||||
try:
|
||||
cfg = decrypt_dict(job.source_config_json)
|
||||
out.source_config_safe = _redact_source(cfg)
|
||||
except Exception:
|
||||
out.source_config_safe = {}
|
||||
return out
|
||||
|
||||
|
||||
@@ -53,7 +83,7 @@ def list_jobs(db: DBSession, _: CurrentUser) -> list[JobWithLastRun]:
|
||||
|
||||
|
||||
@router.post("", response_model=JobOut, status_code=201)
|
||||
def create_job(payload: JobCreate, db: DBSession, _: AdminUser) -> Job:
|
||||
def create_job(payload: JobCreate, db: DBSession, _: AdminUser) -> JobOut:
|
||||
if db.query(Job).filter(Job.name == payload.name).first():
|
||||
raise HTTPException(status_code=400, detail="名称已存在")
|
||||
if not db.get(Storage, payload.storage_id):
|
||||
@@ -74,7 +104,9 @@ def create_job(payload: JobCreate, db: DBSession, _: AdminUser) -> Job:
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
reschedule_job(job.id)
|
||||
return job
|
||||
out = JobOut.model_validate(job)
|
||||
out.source_config_safe = _redact_source(src)
|
||||
return out
|
||||
|
||||
|
||||
@router.get("/{job_id}", response_model=JobWithLastRun)
|
||||
@@ -86,7 +118,7 @@ def get_job(job_id: int, db: DBSession, _: CurrentUser) -> JobWithLastRun:
|
||||
|
||||
|
||||
@router.put("/{job_id}", response_model=JobOut)
|
||||
def update_job(job_id: int, payload: JobUpdate, db: DBSession, _: AdminUser) -> Job:
|
||||
def update_job(job_id: int, payload: JobUpdate, db: DBSession, _: AdminUser) -> JobOut:
|
||||
job = db.get(Job, job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
@@ -94,9 +126,19 @@ def update_job(job_id: int, payload: JobUpdate, db: DBSession, _: AdminUser) ->
|
||||
if db.query(Job).filter(Job.name == payload.name, Job.id != job_id).first():
|
||||
raise HTTPException(status_code=400, detail="名称已存在")
|
||||
job.name = payload.name
|
||||
new_src: dict | None = None
|
||||
if payload.source_config is not None:
|
||||
src = _validate_source(job.type, payload.source_config)
|
||||
job.source_config_json = encrypt_dict(src)
|
||||
new_src = _validate_source(job.type, payload.source_config)
|
||||
# 如果 password 为 "***" 占位符,保留原值
|
||||
existing_src = decrypt_dict(job.source_config_json)
|
||||
for k in _SENSITIVE_KEYS:
|
||||
if (
|
||||
k in new_src
|
||||
and k in existing_src
|
||||
and new_src[k] == "***"
|
||||
):
|
||||
new_src[k] = existing_src[k]
|
||||
job.source_config_json = encrypt_dict(new_src)
|
||||
if payload.storage_id is not None:
|
||||
if not db.get(Storage, payload.storage_id):
|
||||
raise HTTPException(status_code=400, detail="存储目标不存在")
|
||||
@@ -114,7 +156,10 @@ def update_job(job_id: int, payload: JobUpdate, db: DBSession, _: AdminUser) ->
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
reschedule_job(job.id)
|
||||
return job
|
||||
out = JobOut.model_validate(job)
|
||||
cfg = new_src if new_src is not None else decrypt_dict(job.source_config_json)
|
||||
out.source_config_safe = _redact_source(cfg)
|
||||
return out
|
||||
|
||||
|
||||
@router.delete("/{job_id}", status_code=204)
|
||||
@@ -148,7 +193,7 @@ async def run_job(job_id: int, db: DBSession, _: CurrentUser) -> JobRunResponse:
|
||||
|
||||
|
||||
@router.post("/{job_id}/toggle", response_model=JobOut)
|
||||
def toggle_job(job_id: int, db: DBSession, _: AdminUser) -> Job:
|
||||
def toggle_job(job_id: int, db: DBSession, _: AdminUser) -> JobOut:
|
||||
job = db.get(Job, job_id)
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
@@ -156,4 +201,6 @@ def toggle_job(job_id: int, db: DBSession, _: AdminUser) -> Job:
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
reschedule_job(job.id)
|
||||
return job
|
||||
out = JobOut.model_validate(job)
|
||||
out.source_config_safe = _redact_source(decrypt_dict(job.source_config_json))
|
||||
return out
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
"""InfluxDB 备份:通过 influx / influxd CLI 生成备份目录,再 tar.gz 流式打包。
|
||||
|
||||
支持 v1.x 和 v2.x:
|
||||
- v1.x:使用 influx backup (CLI 子命令)
|
||||
等价做法:用 influx_inspect export 转 line protocol,然后打包
|
||||
这里采用更通用的:influxd backup(如果有) 或 打包 line-protocol dump
|
||||
- v2.x:使用 influx backup <bucket> 命令生成备份目录
|
||||
|
||||
策略:
|
||||
1. 在临时目录执行 backup 命令生成文件
|
||||
2. 用 tarfile 流式打包整个目录
|
||||
3. 子进程产出后整体打包上传
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import AsyncIterator
|
||||
|
||||
from app.core.backup.base import BackupMetadata, BaseBackup
|
||||
from app.utils.logging import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class InfluxDBBackup(BaseBackup):
|
||||
def __init__(self, source_config: dict, job_name: str):
|
||||
super().__init__(source_config, job_name)
|
||||
self.version: str = source_config["version"]
|
||||
self.host: str = source_config["host"]
|
||||
self.port: int = int(source_config.get("port", 8086))
|
||||
if self.version == "1":
|
||||
self.database: str = source_config["database"]
|
||||
self.username: str = source_config.get("username", "")
|
||||
self.password: str = source_config.get("password", "")
|
||||
else:
|
||||
self.org: str = source_config["org"]
|
||||
self.bucket: str = source_config["bucket"]
|
||||
self.token: str = source_config["token"]
|
||||
|
||||
def metadata(self) -> BackupMetadata:
|
||||
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
if self.version == "1":
|
||||
safe = "".join(c if c.isalnum() or c in ("_", "-") else "_" for c in self.database)
|
||||
else:
|
||||
safe = "".join(c if c.isalnum() or c in ("_", "-") else "_" for c in self.bucket)
|
||||
return BackupMetadata(
|
||||
suggested_filename=f"{self.job_name}-{self.version}-{safe}-{ts}.tar.gz",
|
||||
content_type="application/gzip",
|
||||
extra={"version": self.version, "host": self.host, "port": self.port},
|
||||
)
|
||||
|
||||
async def produce(self) -> AsyncIterator[bytes]:
|
||||
ts = datetime.now().strftime("%Y%m%d%H%M%S")
|
||||
tmpdir = Path(tempfile.mkdtemp(prefix=f"influxbk_{self.job_name}_"))
|
||||
backup_dir = tmpdir / "data"
|
||||
backup_dir.mkdir()
|
||||
|
||||
try:
|
||||
if self.version == "2":
|
||||
await self._backup_v2(backup_dir)
|
||||
else:
|
||||
await self._backup_v1(backup_dir)
|
||||
|
||||
# 打包为 tar.gz 并流式输出
|
||||
log.info("Taring influx backup at %s", backup_dir)
|
||||
tar_path = tmpdir / "backup.tar.gz"
|
||||
|
||||
def _tar() -> None:
|
||||
with tarfile.open(str(tar_path), mode="w:gz") as tar:
|
||||
tar.add(str(backup_dir), arcname="data", recursive=True)
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
await loop.run_in_executor(None, _tar)
|
||||
|
||||
if not tar_path.exists() or tar_path.stat().st_size == 0:
|
||||
raise RuntimeError("InfluxDB 备份为空,请检查数据库连接和 CLI 安装")
|
||||
|
||||
# 读取打包文件并流式产出
|
||||
with open(tar_path, "rb") as f:
|
||||
while True:
|
||||
chunk = await loop.run_in_executor(None, f.read, 64 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
async def _backup_v2(self, dest: Path) -> None:
|
||||
"""v2: influx backup --bucket X --org Y /dest"""
|
||||
cmd = [
|
||||
"influx", "backup",
|
||||
"--bucket", self.bucket,
|
||||
"--org", self.org,
|
||||
"--host", f"http://{self.host}:{self.port}",
|
||||
"--token", self.token,
|
||||
str(dest),
|
||||
]
|
||||
log.info("Running: %s", " ".join(self._redact(cmd)))
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"influx backup failed (exit={proc.returncode}): "
|
||||
f"{(stderr or stdout).decode(errors='replace')[:2000]}"
|
||||
)
|
||||
|
||||
async def _backup_v1(self, dest: Path) -> None:
|
||||
"""v1: 通过 HTTP API 查询全量数据,导出为 CSV。
|
||||
|
||||
使用 httpx 直接调 InfluxDB v1 的 /query 接口(更可靠,不依赖 CLI 子命令兼容性)。
|
||||
生成:
|
||||
- {database}.csv:SELECT * 结果
|
||||
- META.txt:恢复所需的元信息(库名、host、时间戳等)
|
||||
恢复:需手动用 influx -import 或写入 line protocol。
|
||||
"""
|
||||
import httpx
|
||||
|
||||
log.info("InfluxDB v1 backup via HTTP API: db=%s host=%s:%s",
|
||||
self.database, self.host, self.port)
|
||||
|
||||
url = f"http://{self.host}:{self.port}/query"
|
||||
params = {
|
||||
"db": self.database,
|
||||
"q": "SELECT * FROM /.*/",
|
||||
"chunked": "true",
|
||||
"epoch": "ns",
|
||||
}
|
||||
auth = None
|
||||
if self.username:
|
||||
auth = (self.username, self.password or "")
|
||||
|
||||
async with httpx.AsyncClient(timeout=300.0, auth=auth) as client:
|
||||
try:
|
||||
resp = await client.get(url, params=params)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"InfluxDB v1 HTTP 请求失败: {e}") from e
|
||||
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(
|
||||
f"InfluxDB v1 返回 {resp.status_code}: {resp.text[:500]}"
|
||||
)
|
||||
body = resp.text
|
||||
|
||||
# 保存 CSV 和元信息
|
||||
csv_file = dest / f"{self.database}.csv"
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def _write_files() -> None:
|
||||
csv_file.write_text(body, encoding="utf-8")
|
||||
meta = dest / "META.txt"
|
||||
meta.write_text(
|
||||
f"influxdb_version=1\n"
|
||||
f"host={self.host}\nport={self.port}\n"
|
||||
f"database={self.database}\n"
|
||||
f"username={self.username or ''}\n"
|
||||
f"query=SELECT * FROM /.*/\n"
|
||||
f"format=csv\n"
|
||||
f"backup_time={datetime.utcnow().isoformat()}Z\n"
|
||||
f"restore_hint=可使用 `influx -import -path={csv_file.name} "
|
||||
f"-database={self.database}` 恢复;或在 InfluxQL shell 中执行 "
|
||||
f"`INSERT INTO ... VALUES ...`\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
await loop.run_in_executor(None, _write_files)
|
||||
if not csv_file.exists() or csv_file.stat().st_size == 0:
|
||||
raise RuntimeError("InfluxDB v1 查询结果为空,请检查数据库是否存在或是否有数据")
|
||||
|
||||
@staticmethod
|
||||
def _redact(cmd: list[str]) -> list[str]:
|
||||
"""日志脱敏 token/password。"""
|
||||
out = []
|
||||
skip_next = False
|
||||
secret_flags = {"--token", "-password"}
|
||||
for i, c in enumerate(cmd):
|
||||
if skip_next:
|
||||
out.append("***")
|
||||
skip_next = False
|
||||
continue
|
||||
if c in secret_flags:
|
||||
out.append(c)
|
||||
skip_next = True
|
||||
continue
|
||||
out.append(c)
|
||||
return out
|
||||
@@ -21,6 +21,7 @@ class MySQLBackup(BaseBackup):
|
||||
self.user: str = source_config["user"]
|
||||
self.password: str = source_config.get("password", "")
|
||||
self.database: str = source_config["database"]
|
||||
self.use_ssl: bool = bool(source_config.get("use_ssl", False))
|
||||
self.extra_args: str = source_config.get("extra_args", "")
|
||||
|
||||
def metadata(self) -> BackupMetadata:
|
||||
@@ -45,6 +46,13 @@ class MySQLBackup(BaseBackup):
|
||||
"--events",
|
||||
"--default-character-set=utf8mb4",
|
||||
]
|
||||
# SSL 控制:
|
||||
# MariaDB 11.x 中 --ssl 和 --ssl-verify-server-cert 都默认开启,
|
||||
# 与自签名证书的服务器连接会失败。这里 use_ssl=True 时强制开启 SSL
|
||||
# 但跳过服务端证书校验(兼容自签名证书场景)。
|
||||
if self.use_ssl:
|
||||
args.append("--ssl")
|
||||
args.append("--skip-ssl-verify-server-cert")
|
||||
if self.extra_args:
|
||||
args.extend(shlex.split(self.extra_args))
|
||||
args.append(self.database)
|
||||
|
||||
@@ -14,6 +14,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.backup.base import BaseBackup
|
||||
from app.core.backup.directory import DirectoryBackup
|
||||
from app.core.backup.influxdb import InfluxDBBackup
|
||||
from app.core.backup.mysql import MySQLBackup
|
||||
from app.core.retention import apply_retention
|
||||
from app.core.storage.factory import create_storage
|
||||
@@ -32,6 +33,8 @@ def _make_backup(job: Job) -> BaseBackup:
|
||||
return MySQLBackup(src, job.name)
|
||||
if job.type == "directory":
|
||||
return DirectoryBackup(src, job.name)
|
||||
if job.type == "influxdb":
|
||||
return InfluxDBBackup(src, job.name)
|
||||
raise ValueError(f"Unsupported job type: {job.type}")
|
||||
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ class MySQLSourceConfig(BaseModel):
|
||||
user: str = Field(..., min_length=1)
|
||||
password: str = Field("", description="可空(trust auth)")
|
||||
database: str = Field(..., min_length=1, description="要备份的库名")
|
||||
use_ssl: bool = Field(False, description="True 启用 SSL/TLS;如服务器是自签名证书建议关闭")
|
||||
extra_args: str = Field("", description="额外 mysqldump 参数,如 '--skip-lock-tables'")
|
||||
|
||||
|
||||
@@ -22,9 +23,27 @@ class DirectorySourceConfig(BaseModel):
|
||||
)
|
||||
|
||||
|
||||
class InfluxDBSourceConfigV1(BaseModel):
|
||||
version: Literal["1"]
|
||||
host: str = Field(..., min_length=1)
|
||||
port: int = Field(8086, ge=1, le=65535)
|
||||
database: str = Field(..., min_length=1)
|
||||
username: str = Field("", description="留空表示无认证")
|
||||
password: str = Field("")
|
||||
|
||||
|
||||
class InfluxDBSourceConfigV2(BaseModel):
|
||||
version: Literal["2"]
|
||||
host: str = Field(..., min_length=1)
|
||||
port: int = Field(8086, ge=1, le=65535)
|
||||
org: str = Field(..., min_length=1)
|
||||
bucket: str = Field(..., min_length=1)
|
||||
token: str = Field(..., min_length=1)
|
||||
|
||||
|
||||
class JobBase(BaseModel):
|
||||
name: str = Field(..., min_length=1, max_length=64)
|
||||
type: Literal["mysql", "directory"]
|
||||
type: Literal["mysql", "directory", "influxdb"]
|
||||
source_config: dict
|
||||
storage_id: int
|
||||
cron_expression: Optional[str] = Field(None, description="标准 5 段 cron 表达式,空表示手动")
|
||||
@@ -73,6 +92,11 @@ class JobOut(BaseModel):
|
||||
storage: StorageOut
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
# 解密后的数据源配置;敏感字段已脱敏为 "***"
|
||||
source_config_safe: dict = Field(
|
||||
default_factory=dict,
|
||||
description="解密后的 source_config,敏感字段(password/token/secret_key)已脱敏",
|
||||
)
|
||||
|
||||
|
||||
class RunSummary(BaseModel):
|
||||
|
||||
Reference in New Issue
Block a user