feat: 持久化诊断历史并修复 Agent 命令执行

This commit is contained in:
cnbugs
2026-07-25 12:40:53 +08:00
parent 4f3a7833a0
commit fdac1eaffc
10 changed files with 650 additions and 45 deletions
+41 -5
View File
@@ -1,5 +1,6 @@
"""AI Agent 分析模块 - 调用 OpenAI 兼容接口分析诊断结果 (支持流式输出)"""
import json
import re
import uuid
import httpx
from backend.config import AI_API_BASE, AI_API_KEY, AI_MODEL
@@ -22,7 +23,12 @@ _HEADERS = {
}
AGENT_SYSTEM_PROMPT = """你是 Kubernetes 自主诊断 Agent。你可以根据诊断报告决定执行 kubectl 命令获取更多证据,并在证据充分后给出结论
AGENT_SYSTEM_PROMPT = """你是 Kubernetes 自主诊断 Agent。你的首要职责是主动调用 kubectl 获取现场证据,而不是只给用户一段脚本或操作建议
行为要求:
1. 如果证据不足,必须返回 command 动作并逐条调查;至少检查与故障资源直接相关的状态、describe、events 或 logs。
2. 禁止在 final 中仅提供“请执行以下命令”的脚本。只有在已经依据真实命令输出形成结论时才能返回 final。
3. 每次只提出一条 kubectl 命令,服务端会执行并把真实输出返回给你,然后你继续下一轮判断。
安全规则:
1. 只允许提出单条 kubectl 命令,不得使用 shell、管道、重定向或命令连接符。
@@ -40,10 +46,25 @@ _PENDING_APPROVALS: dict[str, dict] = {}
def _parse_agent_action(content: str) -> dict:
try:
action = json.loads(content.strip())
except json.JSONDecodeError as exc:
raise ValueError("AI 未返回合法 JSON 动作") from exc
text = content.strip()
candidates = [text]
fenced = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.DOTALL | re.IGNORECASE)
if fenced:
candidates.insert(0, fenced.group(1))
first_brace = text.find("{")
last_brace = text.rfind("}")
if first_brace >= 0 and last_brace > first_brace:
candidates.append(text[first_brace:last_brace + 1])
action = None
for candidate in candidates:
try:
action = json.loads(candidate)
break
except json.JSONDecodeError:
continue
if action is None:
raise ValueError("AI 未返回合法 JSON 动作")
if not isinstance(action, dict) or action.get("action") not in {"command", "final"}:
raise ValueError("AI 返回了不支持的动作")
if action["action"] == "command" and not action.get("command"):
@@ -91,6 +112,20 @@ async def run_diagnostic_agent(
{"role": "user", "content": user_content},
]
if execution_mode == "ai":
initial_command = "kubectl get nodes -o wide"
yield {
"type": "command", "step": 0, "command": initial_command,
"reason": "建立集群节点状态基线,确保 AI 执行模式实际采集现场证据",
"mode": "read",
}
initial_result = await execute_kubectl(initial_command)
yield {"type": "result", **initial_result}
messages.append({
"role": "user",
"content": "AI 执行模式已自动采集第一份现场证据:\n" + json.dumps(initial_result, ensure_ascii=False),
})
for step in range(1, max_steps + 1):
yield {"type": "thinking", "step": step, "message": f"AI 正在规划第 {step} 轮诊断"}
try:
@@ -134,6 +169,7 @@ async def run_diagnostic_agent(
"command": command,
"reason": reason,
"created_by": AI_MODEL,
"run_id": "",
}
yield {
"type": "approval_required", "approval_id": approval_id,
+1
View File
@@ -9,3 +9,4 @@ AI_API_KEY = os.getenv("AI_API_KEY", "hermes")
AI_MODEL = os.getenv("AI_MODEL", "qwen3.8-max-preview")
PORT = int(os.getenv("PORT", "8900"))
KUBECONFIG_PATH = os.getenv("KUBECONFIG_PATH", "") or None
HISTORY_DB_PATH = os.getenv("HISTORY_DB_PATH", os.path.join(os.path.dirname(os.path.dirname(__file__)), "data", "history.db"))
+172
View File
@@ -0,0 +1,172 @@
"""SQLite 持久化诊断历史、AI 分析和 Agent 处理记录。"""
import json
import os
import sqlite3
import uuid
from datetime import datetime
from typing import Optional
def _now() -> str:
return datetime.now().isoformat(timespec="seconds")
class HistoryStore:
def __init__(self, path: str):
self.path = path
parent = os.path.dirname(os.path.abspath(path))
os.makedirs(parent, exist_ok=True)
self._init_db()
def _connect(self):
connection = sqlite3.connect(self.path, timeout=30)
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA journal_mode=WAL")
connection.execute("PRAGMA foreign_keys=ON")
return connection
def _init_db(self):
with self._connect() as db:
db.executescript("""
CREATE TABLE IF NOT EXISTS diagnoses (
id TEXT PRIMARY KEY,
created_at TEXT NOT NULL,
namespace TEXT NOT NULL,
score INTEGER NOT NULL,
status TEXT NOT NULL,
critical_count INTEGER NOT NULL DEFAULT 0,
warning_count INTEGER NOT NULL DEFAULT 0,
elapsed_seconds REAL NOT NULL DEFAULT 0,
diagnosis_json TEXT NOT NULL,
report TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_diagnoses_created_at ON diagnoses(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_diagnoses_status ON diagnoses(status);
CREATE TABLE IF NOT EXISTS processing_runs (
id TEXT PRIMARY KEY,
diagnosis_id TEXT NOT NULL,
kind TEXT NOT NULL,
mode TEXT NOT NULL DEFAULT '',
question TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL,
summary TEXT NOT NULL DEFAULT '',
started_at TEXT NOT NULL,
finished_at TEXT,
FOREIGN KEY(diagnosis_id) REFERENCES diagnoses(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_runs_diagnosis ON processing_runs(diagnosis_id, started_at);
CREATE TABLE IF NOT EXISTS processing_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
run_id TEXT NOT NULL,
sequence INTEGER NOT NULL,
type TEXT NOT NULL,
created_at TEXT NOT NULL,
payload_json TEXT NOT NULL,
FOREIGN KEY(run_id) REFERENCES processing_runs(id) ON DELETE CASCADE,
UNIQUE(run_id, sequence)
);
""")
def save_diagnosis(self, diagnosis: dict, report: str) -> str:
record_id = uuid.uuid4().hex
created_at = diagnosis.get("timestamp") or _now()
with self._connect() as db:
db.execute(
"""INSERT INTO diagnoses
(id, created_at, namespace, score, status, critical_count, warning_count,
elapsed_seconds, diagnosis_json, report)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(
record_id, created_at, diagnosis.get("namespace", "all"),
int(diagnosis.get("score", 0)), diagnosis.get("status", "unknown"),
int(diagnosis.get("critical_count", 0)), int(diagnosis.get("warning_count", 0)),
float(diagnosis.get("elapsed_seconds", 0)),
json.dumps(diagnosis, ensure_ascii=False), report,
),
)
return record_id
def list_diagnoses(self, status: str = "", namespace: str = "", limit: int = 100, offset: int = 0) -> list[dict]:
conditions = []
params: list = []
if status:
conditions.append("status = ?")
params.append(status)
if namespace:
conditions.append("namespace = ?")
params.append(namespace)
where = " WHERE " + " AND ".join(conditions) if conditions else ""
params.extend([max(1, min(limit, 500)), max(0, offset)])
with self._connect() as db:
rows = db.execute(
f"""SELECT id, created_at, namespace, score, status, critical_count,
warning_count, elapsed_seconds,
(SELECT COUNT(*) FROM processing_runs r WHERE r.diagnosis_id = diagnoses.id) AS run_count
FROM diagnoses{where}
ORDER BY created_at DESC, rowid DESC LIMIT ? OFFSET ?""",
params,
).fetchall()
return [dict(row) for row in rows]
def get_diagnosis(self, record_id: str) -> Optional[dict]:
with self._connect() as db:
row = db.execute("SELECT * FROM diagnoses WHERE id = ?", (record_id,)).fetchone()
if row is None:
return None
run_rows = db.execute(
"SELECT * FROM processing_runs WHERE diagnosis_id = ? ORDER BY started_at, rowid",
(record_id,),
).fetchall()
runs = []
for run_row in run_rows:
run = dict(run_row)
event_rows = db.execute(
"SELECT type, created_at, payload_json FROM processing_events WHERE run_id = ? ORDER BY sequence",
(run["id"],),
).fetchall()
run["events"] = [
{"type": event["type"], "created_at": event["created_at"], **json.loads(event["payload_json"])}
for event in event_rows
]
runs.append(run)
return {
"id": row["id"], "created_at": row["created_at"],
"diagnosis": json.loads(row["diagnosis_json"]),
"report": row["report"], "runs": runs,
}
def latest(self) -> Optional[dict]:
records = self.list_diagnoses(limit=1)
return self.get_diagnosis(records[0]["id"]) if records else None
def create_run(self, diagnosis_id: str, kind: str, mode: str = "", question: str = "") -> str:
run_id = uuid.uuid4().hex
with self._connect() as db:
db.execute(
"""INSERT INTO processing_runs
(id, diagnosis_id, kind, mode, question, status, started_at)
VALUES (?, ?, ?, ?, ?, 'running', ?)""",
(run_id, diagnosis_id, kind, mode, question, _now()),
)
return run_id
def append_event(self, run_id: str, event_type: str, payload: dict):
with self._connect() as db:
sequence = db.execute(
"SELECT COALESCE(MAX(sequence), 0) + 1 FROM processing_events WHERE run_id = ?",
(run_id,),
).fetchone()[0]
db.execute(
"""INSERT INTO processing_events (run_id, sequence, type, created_at, payload_json)
VALUES (?, ?, ?, ?, ?)""",
(run_id, sequence, event_type, _now(), json.dumps(payload, ensure_ascii=False)),
)
def finish_run(self, run_id: str, status: str, summary: str = ""):
with self._connect() as db:
db.execute(
"UPDATE processing_runs SET status = ?, summary = ?, finished_at = ? WHERE id = ?",
(status, summary, _now(), run_id),
)
+133 -32
View File
@@ -11,7 +11,7 @@ from fastapi.responses import StreamingResponse
from pydantic import BaseModel
from typing import Optional
from backend.config import PORT, AI_API_BASE, AI_MODEL
from backend.config import PORT, AI_API_BASE, AI_MODEL, HISTORY_DB_PATH
from backend.diagnosis import (
run_full_diagnosis, build_diagnosis_report, ALL_CHECKS,
check_connectivity,
@@ -25,8 +25,10 @@ from backend.ai_agent import (
run_diagnostic_agent, take_pending_approval,
)
from backend.agent_tools import execute_kubectl
from backend.history_store import HistoryStore
app = FastAPI(title="K8S 智能诊断平台", version="1.0.0")
history_store = HistoryStore(HISTORY_DB_PATH)
app.add_middleware(
CORSMiddleware,
@@ -38,6 +40,25 @@ app.add_middleware(
# 存储最近诊断结果 (内存)
_last_diagnosis: Optional[dict] = None
_last_report: str = ""
_last_diagnosis_id: Optional[str] = None
def _save_current_diagnosis(diagnosis: dict, report: str) -> str:
global _last_diagnosis, _last_report, _last_diagnosis_id
_last_diagnosis = diagnosis
_last_report = report
_last_diagnosis_id = history_store.save_diagnosis(diagnosis, report)
return _last_diagnosis_id
def _ensure_current_history() -> str:
global _last_diagnosis_id
if _last_diagnosis_id:
return _last_diagnosis_id
if _last_diagnosis is None:
raise RuntimeError("暂无诊断记录")
_last_diagnosis_id = history_store.save_diagnosis(_last_diagnosis, _last_report)
return _last_diagnosis_id
# 诊断项执行顺序 (带中文标题)
CHECK_ORDER = [
@@ -121,9 +142,9 @@ async def diagnose_stream(namespace: str = ""):
}],
"checks": {},
}
_last_diagnosis = diagnosis
_last_report = build_diagnosis_report(diagnosis)
yield f"data: {json.dumps({'type': 'complete', 'diagnosis': diagnosis}, ensure_ascii=False)}\n\n"
report = build_diagnosis_report(diagnosis)
_save_current_diagnosis(diagnosis, report)
yield f"data: {json.dumps({'type': 'complete', 'diagnosis': diagnosis, 'history_id': _last_diagnosis_id}, ensure_ascii=False)}\n\n"
return
yield f"data: {json.dumps({'type': 'progress', 'step': 0, 'total': len(CHECK_ORDER) + 1, 'name': 'connectivity', 'title': '集群连通性检查', 'status': 'done', 'issues': 0}, ensure_ascii=False)}\n\n"
@@ -176,10 +197,10 @@ async def diagnose_stream(namespace: str = ""):
"checks": results,
}
_last_diagnosis = diagnosis
_last_report = build_diagnosis_report(diagnosis)
report = build_diagnosis_report(diagnosis)
_save_current_diagnosis(diagnosis, report)
yield f"data: {json.dumps({'type': 'complete', 'diagnosis': diagnosis}, ensure_ascii=False)}\n\n"
yield f"data: {json.dumps({'type': 'complete', 'diagnosis': diagnosis, 'history_id': _last_diagnosis_id}, ensure_ascii=False)}\n\n"
return StreamingResponse(
event_generator(),
@@ -192,17 +213,24 @@ async def diagnose_stream(namespace: str = ""):
async def diagnose(req: DiagnoseRequest):
"""执行一键诊断 (非流式,兼容旧接口)"""
global _last_diagnosis, _last_report
_last_diagnosis = await run_full_diagnosis(namespace=req.namespace, checks=req.checks)
_last_report = build_diagnosis_report(_last_diagnosis)
return _last_diagnosis
diagnosis = await run_full_diagnosis(namespace=req.namespace, checks=req.checks)
report = build_diagnosis_report(diagnosis)
_save_current_diagnosis(diagnosis, report)
return {**diagnosis, "history_id": _last_diagnosis_id}
@app.get("/api/diagnosis/latest")
async def get_latest():
"""获取最近一次诊断结果"""
"""获取最近一次诊断结果,服务重启后从 SQLite 恢复。"""
global _last_diagnosis, _last_report, _last_diagnosis_id
if _last_diagnosis is None:
raise HTTPException(status_code=404, detail="暂无诊断记录,请先执行诊断")
return _last_diagnosis
latest = history_store.latest()
if latest is None:
raise HTTPException(status_code=404, detail="暂无诊断记录,请先执行诊断")
_last_diagnosis = latest["diagnosis"]
_last_report = latest["report"]
_last_diagnosis_id = latest["id"]
return {**_last_diagnosis, "history_id": _last_diagnosis_id}
@app.post("/api/analyze")
@@ -210,19 +238,25 @@ async def analyze(req: AnalyzeRequest):
"""AI 分析诊断结果 (非流式,兼容旧接口)"""
global _last_diagnosis, _last_report
if _last_diagnosis is None:
_last_diagnosis = await run_full_diagnosis()
_last_report = build_diagnosis_report(_last_diagnosis)
diagnosis = await run_full_diagnosis()
report = build_diagnosis_report(diagnosis)
_save_current_diagnosis(diagnosis, report)
run_id = history_store.create_run(_ensure_current_history(), "analysis", "standard", req.question)
result = await analyze_with_ai(_last_report, question=req.question)
return result
history_store.append_event(run_id, "analysis", result)
history_store.finish_run(run_id, "completed" if result.get("success") else "failed", result.get("analysis", ""))
return {**result, "run_id": run_id}
@app.get("/api/analyze/stream")
async def analyze_stream(question: str = ""):
"""SSE 流式 AI 分析 - 逐字推送分析内容"""
"""SSE 流式 AI 分析 - 逐字推送分析内容并持久化"""
global _last_diagnosis, _last_report
if _last_diagnosis is None:
_last_diagnosis = await run_full_diagnosis()
_last_report = build_diagnosis_report(_last_diagnosis)
diagnosis = await run_full_diagnosis()
report = build_diagnosis_report(diagnosis)
_save_current_diagnosis(diagnosis, report)
run_id = history_store.create_run(_ensure_current_history(), "analysis", "stream", question)
async def event_generator():
process_steps = [
@@ -232,23 +266,39 @@ async def analyze_stream(question: str = ""):
("summary", "整理分析结论", "正在组织根因、影响和预防建议"),
]
yield f"data: {json.dumps({'type': 'start', 'model': AI_MODEL}, ensure_ascii=False)}\n\n"
yield f"data: {json.dumps({'type': 'start', 'model': AI_MODEL, 'run_id': run_id}, ensure_ascii=False)}\n\n"
content = ""
try:
for step, title, detail in process_steps:
yield f"data: {json.dumps({'type': 'process', 'step': step, 'title': title, 'detail': detail, 'status': 'running'}, ensure_ascii=False)}\n\n"
process_event = {'step': step, 'title': title, 'detail': detail, 'status': 'running'}
history_store.append_event(run_id, "process", process_event)
yield f"data: {json.dumps({'type': 'process', **process_event}, ensure_ascii=False)}\n\n"
await asyncio.sleep(0)
yield f"data: {json.dumps({'type': 'process', 'step': step, 'title': title, 'detail': detail, 'status': 'done'}, ensure_ascii=False)}\n\n"
process_event = {'step': step, 'title': title, 'detail': detail, 'status': 'done'}
history_store.append_event(run_id, "process", process_event)
yield f"data: {json.dumps({'type': 'process', **process_event}, ensure_ascii=False)}\n\n"
async for chunk in analyze_with_ai_stream(_last_report, question=question):
content += chunk
yield f"data: {json.dumps({'type': 'chunk', 'content': chunk}, ensure_ascii=False)}\n\n"
yield f"data: {json.dumps({'type': 'done'}, ensure_ascii=False)}\n\n"
history_store.append_event(run_id, "final", {"content": content, "model": AI_MODEL})
history_store.finish_run(run_id, "completed", content)
yield f"data: {json.dumps({'type': 'done', 'run_id': run_id}, ensure_ascii=False)}\n\n"
except httpx.HTTPStatusError as e:
err = f"AI 接口返回错误: {e.response.status_code} - {str(e)[:300]}"
history_store.append_event(run_id, "error", {"message": err})
history_store.finish_run(run_id, "failed", err)
yield f"data: {json.dumps({'type': 'error', 'message': err}, ensure_ascii=False)}\n\n"
except httpx.ConnectError:
yield f"data: {json.dumps({'type': 'error', 'message': f'无法连接 AI 服务 ({AI_API_BASE}),请检查 .env 中的 AI_API_BASE 配置'}, ensure_ascii=False)}\n\n"
err = f"无法连接 AI 服务 ({AI_API_BASE}),请检查 .env 中的 AI_API_BASE 配置"
history_store.append_event(run_id, "error", {"message": err})
history_store.finish_run(run_id, "failed", err)
yield f"data: {json.dumps({'type': 'error', 'message': err}, ensure_ascii=False)}\n\n"
except Exception as e:
yield f"data: {json.dumps({'type': 'error', 'message': f'AI 分析失败: {str(e)}'}, ensure_ascii=False)}\n\n"
err = f"AI 分析失败: {str(e)}"
history_store.append_event(run_id, "error", {"message": err})
history_store.finish_run(run_id, "failed", err)
yield f"data: {json.dumps({'type': 'error', 'message': err}, ensure_ascii=False)}\n\n"
return StreamingResponse(
event_generator(),
@@ -264,21 +314,42 @@ async def agent_diagnose_stream(question: str = "", execution_mode: str = "ai"):
raise HTTPException(status_code=400, detail="execution_mode 仅支持 ai 或 manual")
global _last_diagnosis, _last_report
if _last_diagnosis is None:
_last_diagnosis = await run_full_diagnosis()
_last_report = build_diagnosis_report(_last_diagnosis)
diagnosis = await run_full_diagnosis()
report = build_diagnosis_report(diagnosis)
_save_current_diagnosis(diagnosis, report)
run_id = history_store.create_run(_ensure_current_history(), "agent", execution_mode, question)
async def event_generator():
yield f"data: {json.dumps({'type': 'start', 'model': AI_MODEL, 'mode': execution_mode}, ensure_ascii=False)}\n\n"
yield f"data: {json.dumps({'type': 'start', 'model': AI_MODEL, 'mode': execution_mode, 'run_id': run_id}, ensure_ascii=False)}\n\n"
final_status = "completed"
summary = ""
try:
async for event in run_diagnostic_agent(
_last_report,
question=question,
execution_mode=execution_mode,
):
history_store.append_event(run_id, event.get("type", "event"), {k: v for k, v in event.items() if k != "type"})
if event.get("type") == "final":
summary = event.get("content", "")
elif event.get("type") == "error":
final_status = "failed"
summary = event.get("message", "")
elif event.get("type") in {"manual_command", "approval_required"}:
final_status = "waiting"
if event.get("type") == "approval_required":
from backend.ai_agent import _PENDING_APPROVALS
pending = _PENDING_APPROVALS.get(event.get("approval_id"))
if pending is not None:
pending["run_id"] = run_id
yield f"data: {json.dumps(event, ensure_ascii=False)}\n\n"
yield f"data: {json.dumps({'type': 'done'}, ensure_ascii=False)}\n\n"
history_store.finish_run(run_id, final_status, summary)
yield f"data: {json.dumps({'type': 'done', 'run_id': run_id}, ensure_ascii=False)}\n\n"
except Exception as exc:
yield f"data: {json.dumps({'type': 'error', 'message': f'自主诊断失败: {exc}'}, ensure_ascii=False)}\n\n"
message = f"自主诊断失败: {exc}"
history_store.append_event(run_id, "error", {"message": message})
history_store.finish_run(run_id, "failed", message)
yield f"data: {json.dumps({'type': 'error', 'message': message}, ensure_ascii=False)}\n\n"
return StreamingResponse(
event_generator(),
@@ -294,9 +365,18 @@ async def execute_approved_agent_command(req: AgentApprovalRequest):
if pending is None:
raise HTTPException(status_code=404, detail="审批请求不存在或已经处理")
if not req.approved:
return {"approved": False, "command": pending["command"], "message": "用户已拒绝执行"}
result = {"approved": False, "command": pending["command"], "message": "用户已拒绝执行"}
if pending.get("run_id"):
history_store.append_event(pending["run_id"], "approval_rejected", result)
history_store.finish_run(pending["run_id"], "rejected", result["message"])
return result
result = await execute_kubectl(pending["command"])
return {"approved": True, "reason": pending["reason"], **result}
response = {"approved": True, "reason": pending["reason"], **result}
if pending.get("run_id"):
history_store.append_event(pending["run_id"], "approved_result", response)
status = "completed" if result.get("returncode") == 0 else "failed"
history_store.finish_run(pending["run_id"], status, result.get("stdout") or result.get("stderr", ""))
return response
@app.post("/api/chat")
@@ -325,6 +405,27 @@ async def chat_stream(req: ChatRequest):
)
@app.get("/api/history/diagnoses")
async def list_diagnosis_history(
status: str = "",
namespace: str = "",
limit: int = 100,
offset: int = 0,
):
"""查询全部历史诊断记录,可按状态和命名空间过滤。"""
items = history_store.list_diagnoses(status=status, namespace=namespace, limit=limit, offset=offset)
return {"total": len(items), "items": items, "limit": limit, "offset": offset}
@app.get("/api/history/diagnoses/{record_id}")
async def get_diagnosis_history(record_id: str):
"""查询单次诊断及其全部 AI 分析、Agent 命令和处理记录。"""
record = history_store.get_diagnosis(record_id)
if record is None:
raise HTTPException(status_code=404, detail="历史诊断记录不存在")
return record
@app.get("/api/report")
async def get_report():
"""获取诊断报告文本"""