AI-Agent/k8smanager-cli
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

fix: 修复集群不可达时误报100分的严重bug

Browse Source 
- 新增 check_connectivity() 连通性预检: kubectl存在性/kubeconfig/API Server连接
- 连不上集群直接返回0分+明确错误信息+排查建议(不再静默跳过)
- 每个检查项kubectl失败时必须报critical问题(不再忽略error)
- 前端: 连接失败时显示红色错误页+错误详情+排查建议+重新诊断按钮
- 智能识别常见错误: connection refused/证书过期/超时/DNS解析失败等
This commit is contained in:
cnbugs
2026-07-25 11:07:03 +08:00
parent 7a271f24d4
commit 8109204843
3 changed files with 263 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/diagnosis.py
+185 -57
View File
@@ -27,8 +27,8 @@ async def _run_kubectl(args: list[str], timeout: int = 30) -> dict:
return {"error": str(e), "data": None}
async def _run_kubectl_raw(args: list[str], timeout: int = 30) -> str:
"""执行 kubectl 命令，返回原始文本"""
async def _run_kubectl_raw(args: list[str], timeout: int = 30) -> tuple[str, str, int]:
"""执行 kubectl 命令，返回 (stdout, stderr, returncode)"""
cmd = [KUBECTL] + args
try:
proc = await asyncio.create_subprocess_exec(
@@ -37,9 +37,11 @@ async def _run_kubectl_raw(args: list[str], timeout: int = 30) -> str:
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
return stdout.decode().strip()
except Exception:
return ""
return stdout.decode().strip(), stderr.decode().strip(), proc.returncode
except asyncio.TimeoutError:
return "", f"命令超时 ({timeout}s)", -1
except Exception as e:
return "", str(e), -1
def _parse_resource_quantity(q: str) -> float:
@@ -61,6 +63,56 @@ def _parse_resource_quantity(q: str) -> float:
return 0
# ─── 连通性检查 (最优先) ──────────────────────────────────
async def check_connectivity() -> dict:
"""检查 kubectl 是否能连接到集群 API Server"""
# 1. 检查 kubectl 是否存在
import os
if not os.path.isfile(KUBECTL):
return {
"connected": False,
"error": f"kubectl 不存在: {KUBECTL}",
"suggestion": "请安装 kubectl 并确保在 PATH 中",
}
# 2. 检查 kubeconfig
kubeconfig = os.environ.get("KUBECONFIG", os.path.expanduser("~/.kube/config"))
if not os.path.isfile(kubeconfig):
return {
"connected": False,
"error": f"kubeconfig 不存在: {kubeconfig}",
"suggestion": "请将集群 kubeconfig 放到 ~/.kube/config 或设置 KUBECONFIG 环境变量",
}
# 3. 尝试连接 API Server
stdout, stderr, code = await _run_kubectl_raw(["cluster-info"], timeout=10)
if code != 0:
# 提取关键错误信息
err_msg = stderr or stdout or "未知错误"
# 常见错误模式
if "connection refused" in err_msg.lower():
suggestion = "API Server 连接被拒绝。请检查: 1) kubelet/apiserver 是否运行 (systemctl status kubelet) 2) 证书是否过期 3) 端口 6443 是否监听"
elif "no such host" in err_msg.lower() or "dial tcp" in err_msg.lower():
suggestion = "无法解析或连接 API Server 地址。请检查 kubeconfig 中的 server 地址是否正确"
elif "certificate" in err_msg.lower() or "x509" in err_msg.lower():
suggestion = "证书错误。请检查: 1) 集群证书是否过期 (kubeadm certs check-expiration) 2) 系统时间是否正确"
elif "timeout" in err_msg.lower() or "timed out" in err_msg.lower():
suggestion = "连接超时。请检查: 1) 网络是否可达 2) 防火墙是否放行 6443 端口 3) API Server 是否运行"
elif "unable to connect" in err_msg.lower() or "could not" in err_msg.lower():
suggestion = "无法连接到集群。请检查: 1) systemctl status kubelet 2) journalctl -u kubelet --no-pager -n 50 3) crictl ps 查看容器状态"
else:
suggestion = "请检查 kubelet 和 API Server 状态: systemctl status kubelet && journalctl -u kubelet -n 50"
return {
"connected": False,
"error": err_msg[:500],
"suggestion": suggestion,
}
return {"connected": True, "error": None, "info": stdout}
# ─── 诊断项 ───────────────────────────────────────────────
async def check_cluster_info() -> dict:
@@ -68,9 +120,15 @@ async def check_cluster_info() -> dict:
ver = await _run_kubectl(["version"])
nodes = await _run_kubectl(["get", "nodes"])
namespaces = await _run_kubectl(["get", "namespaces"])
issues = []
if ver.get("error"):
issues.append({"level": "critical", "resource": "cluster", "message": f"无法获取集群版本: {ver['error'][:200]}"})
return {
"name": "cluster_info",
"title": "集群信息",
"issues": issues,
"version": ver,
"nodes": nodes,
"namespaces": namespaces,
@@ -85,20 +143,23 @@ async def check_pods(namespace: str = "") -> dict:
else:
args += ["--all-namespaces"]
result = await _run_kubectl(args)
issues = []
pods_summary = {"total": 0, "running": 0, "pending": 0, "failed": 0, "unknown": 0, "crashloop": 0}
if result.get("error"):
issues.append({"level": "critical", "resource": "pods", "message": f"无法获取 Pod 列表: {result['error'][:200]}"})
return {"name": "pods", "title": "Pod 状态检查", "summary": pods_summary, "issues": issues, "raw": result}
if result["data"] and "items" in result["data"]:
for pod in result["data"]["items"]:
pods_summary["total"] += 1
phase = pod.get("status", {}).get("phase", "Unknown")
name = pod["metadata"]["name"]
ns = pod["metadata"].get("namespace", "default")
if phase == "Running":
pods_summary["running"] += 1
# 检查重启次数
for cs in pod.get("status", {}).get("containerStatuses", []):
restarts = cs.get("restartCount", 0)
if restarts > 5:
@@ -125,7 +186,7 @@ async def check_pods(namespace: str = "") -> dict:
issues.append({"level": "critical", "resource": f"{ns}/{name}", "message": "Pod 处于 Failed 状态"})
else:
pods_summary["unknown"] += 1
return {
"name": "pods",
"title": "Pod 状态检查",
@@ -140,7 +201,11 @@ async def check_nodes() -> dict:
result = await _run_kubectl(["get", "nodes"])
issues = []
nodes_summary = {"total": 0, "ready": 0, "not_ready": 0}
if result.get("error"):
issues.append({"level": "critical", "resource": "nodes", "message": f"无法获取 Node 列表: {result['error'][:200]}"})
return {"name": "nodes", "title": "Node 状态检查", "summary": nodes_summary, "issues": issues, "raw": result}
if result["data"] and "items" in result["data"]:
for node in result["data"]["items"]:
nodes_summary["total"] += 1
@@ -161,8 +226,7 @@ async def check_nodes() -> dict:
else:
nodes_summary["not_ready"] += 1
issues.append({"level": "critical", "resource": name, "message": "节点 NotReady"})
# 资源使用
alloc = node.get("status", {}).get("allocatable", {})
cap = node.get("status", {}).get("capacity", {})
if alloc and cap:
@@ -175,7 +239,7 @@ async def check_nodes() -> dict:
"resource": name,
"message": f"节点 {res} 可分配量不足 10%",
})
return {
"name": "nodes",
"title": "Node 状态检查",
@@ -193,11 +257,16 @@ async def check_events(namespace: str = "") -> dict:
else:
args += ["--all-namespaces"]
result = await _run_kubectl(args)
issues = []
events_list = []
if result.get("error"):
issues.append({"level": "critical", "resource": "events", "message": f"无法获取事件: {result['error'][:200]}"})
return {"name": "events", "title": "异常事件检查", "warning_count": 0, "events": [], "issues": issues, "raw": result}
if result["data"] and "items" in result["data"]:
for ev in result["data"]["items"][-50:]: # 最近50条
for ev in result["data"]["items"][-50:]:
events_list.append({
"namespace": ev["metadata"].get("namespace", ""),
"reason": ev.get("reason", ""),
@@ -213,11 +282,11 @@ async def check_events(namespace: str = "") -> dict:
"resource": f"{ev['metadata'].get('namespace', '')}/{ev.get('involvedObject', {}).get('name', '')}",
"message": f"[{ev.get('reason', '')}] {ev.get('message', '')[:200]}",
})
return {
"name": "events",
"title": "异常事件检查",
"warning_count": len(issues),
"warning_count": len([i for i in issues if i["level"] == "warning"]),
"events": events_list,
"issues": issues,
"raw": result,
@@ -232,9 +301,14 @@ async def check_deployments(namespace: str = "") -> dict:
else:
args += ["--all-namespaces"]
result = await _run_kubectl(args)
issues = []
deploys = []
if result.get("error"):
issues.append({"level": "critical", "resource": "deployments", "message": f"无法获取 Deployment: {result['error'][:200]}"})
return {"name": "deployments", "title": "Deployment 状态检查", "deployments": [], "issues": issues, "raw": result}
if result["data"] and "items" in result["data"]:
for dep in result["data"]["items"]:
name = dep["metadata"]["name"]
@@ -244,7 +318,7 @@ async def check_deployments(namespace: str = "") -> dict:
ready = status.get("readyReplicas", 0) or 0
available = status.get("availableReplicas", 0) or 0
updated = status.get("updatedReplicas", 0) or 0
deploys.append({
"namespace": ns, "name": name,
"desired": spec_replicas, "ready": ready,
@@ -262,7 +336,7 @@ async def check_deployments(namespace: str = "") -> dict:
"resource": f"{ns}/{name}",
"message": f"滚动更新未完成: 已更新 {updated}/{spec_replicas}",
})
return {
"name": "deployments",
"title": "Deployment 状态检查",
@@ -280,8 +354,13 @@ async def check_services(namespace: str = "") -> dict:
else:
args += ["--all-namespaces"]
result = await _run_kubectl(args)
issues = []
if result.get("error"):
issues.append({"level": "critical", "resource": "services", "message": f"无法获取 Endpoint: {result['error'][:200]}"})
return {"name": "services", "title": "Service/Endpoint 检查", "issues": issues, "raw": result}
if result["data"] and "items" in result["data"]:
for ep in result["data"]["items"]:
name = ep["metadata"]["name"]
@@ -294,7 +373,7 @@ async def check_services(namespace: str = "") -> dict:
"resource": f"{ns}/{name}",
"message": "Service 没有可用的后端 Endpoint",
})
return {
"name": "services",
"title": "Service/Endpoint 检查",
@@ -311,8 +390,13 @@ async def check_pvc(namespace: str = "") -> dict:
else:
args += ["--all-namespaces"]
result = await _run_kubectl(args)
issues = []
if result.get("error"):
issues.append({"level": "critical", "resource": "pvc", "message": f"无法获取 PVC: {result['error'][:200]}"})
return {"name": "pvc", "title": "PVC 存储检查", "issues": issues, "raw": result}
if result["data"] and "items" in result["data"]:
for pvc in result["data"]["items"]:
name = pvc["metadata"]["name"]
@@ -324,7 +408,7 @@ async def check_pvc(namespace: str = "") -> dict:
"resource": f"{ns}/{name}",
"message": f"PVC 状态异常: {phase}",
})
return {
"name": "pvc",
"title": "PVC 存储检查",
@@ -335,12 +419,23 @@ async def check_pvc(namespace: str = "") -> dict:
async def check_resource_usage() -> dict:
"""节点资源使用率 (需要 metrics-server)"""
raw = await _run_kubectl_raw(["top", "nodes", "--no-headers"])
stdout, stderr, code = await _run_kubectl_raw(["top", "nodes", "--no-headers"])
issues = []
nodes_usage = []
if raw and "error" not in raw.lower():
for line in raw.strip().split("\n"):
if code != 0:
# metrics-server 不可用不算 critical，只是跳过
return {
"name": "resource_usage",
"title": "资源使用率",
"issues": [],
"nodes": [],
"note": f"metrics-server 不可用: {(stderr or stdout)[:200]}",
"raw": {"error": stderr or stdout},
}
if stdout:
for line in stdout.strip().split("\n"):
parts = line.split()
if len(parts) >= 5:
name = parts[0]
@@ -359,22 +454,13 @@ async def check_resource_usage() -> dict:
issues.append({"level": "warning", "resource": name, "message": f"内存使用率 {mem_v}%"})
except ValueError:
pass
else:
return {
"name": "resource_usage",
"title": "资源使用率",
"issues": [],
"nodes": [],
"note": "metrics-server 不可用，跳过资源使用率检查",
"raw": {"error": raw},
}
return {
"name": "resource_usage",
"title": "资源使用率",
"issues": issues,
"nodes": nodes_usage,
"raw": {"text": raw},
"raw": {"text": stdout},
}
@@ -386,12 +472,18 @@ async def check_network_policies(namespace: str = "") -> dict:
else:
args += ["--all-namespaces"]
result = await _run_kubectl(args)
issues = []
if result.get("error"):
issues.append({"level": "warning", "resource": "network", "message": f"无法获取网络策略: {result['error'][:200]}"})
return {"name": "network", "title": "网络策略检查", "policy_count": 0, "issues": issues, "raw": result}
count = len(result["data"].get("items", [])) if result["data"] else 0
return {
"name": "network",
"title": "网络策略检查",
"policy_count": count,
"issues": [],
"issues": issues,
"raw": result,
}
@@ -414,7 +506,31 @@ ALL_CHECKS = {
async def run_full_diagnosis(namespace: str = "", checks: Optional[list[str]] = None) -> dict:
"""执行完整诊断"""
start = datetime.now()
# ★ 第一步: 连通性检查
conn = await check_connectivity()
if not conn["connected"]:
return {
"timestamp": start.isoformat(),
"elapsed_seconds": round((datetime.now() - start).total_seconds(), 2),
"namespace": namespace or "all",
"score": 0,
"status": "critical",
"connected": False,
"connectivity_error": conn["error"],
"connectivity_suggestion": conn.get("suggestion", ""),
"critical_count": 1,
"warning_count": 0,
"issues": [{
"level": "critical",
"check": "集群连通性",
"resource": "API Server",
"message": f"无法连接到集群: {conn['error'][:300]}",
}],
"checks": {},
}
# ★ 第二步: 执行各项检查
selected = checks or list(ALL_CHECKS.keys())
tasks = {}
for name in selected:
@@ -424,45 +540,49 @@ async def run_full_diagnosis(namespace: str = "", checks: Optional[list[str]] =
tasks[name] = fn(namespace)
else:
tasks[name] = fn()
results = {}
for name, coro in tasks.items():
try:
results[name] = await coro
except Exception as e:
results[name] = {"name": name, "title": name, "error": str(e), "issues": []}
results[name] = {
"name": name, "title": name, "error": str(e),
"issues": [{"level": "critical", "resource": name, "message": f"检查执行异常: {str(e)}"}],
}
# 汇总
all_issues = []
for r in results.values():
for issue in r.get("issues", []):
issue["check"] = r.get("title", r.get("name", ""))
all_issues.append(issue)
critical = [i for i in all_issues if i["level"] == "critical"]
warning = [i for i in all_issues if i["level"] == "warning"]
elapsed = (datetime.now() - start).total_seconds()
# 健康评分 (100 分制)
score = 100
score -= len(critical) * 15
score -= len(warning) * 5
score = max(0, score)
if score >= 90:
status = "healthy"
elif score >= 60:
status = "warning"
else:
status = "critical"
return {
"timestamp": start.isoformat(),
"elapsed_seconds": round(elapsed, 2),
"namespace": namespace or "all",
"score": score,
"status": status,
"connected": True,
"critical_count": len(critical),
"warning_count": len(warning),
"issues": all_issues,
@@ -479,45 +599,53 @@ def build_diagnosis_report(diagnosis: dict) -> str:
lines.append(f"健康评分: {diagnosis['score']}/100 ({diagnosis['status']})")
lines.append(f"严重问题: {diagnosis['critical_count']} 个, 警告: {diagnosis['warning_count']} 个")
lines.append("")
# 连通性失败
if not diagnosis.get("connected", True):
lines.append("## ⚠️ 集群连接失败")
lines.append(f"错误: {diagnosis.get('connectivity_error', '未知')}")
lines.append(f"建议: {diagnosis.get('connectivity_suggestion', '')}")
lines.append("")
return "\n".join(lines)
if diagnosis["issues"]:
lines.append("## 发现的问题")
for i, issue in enumerate(diagnosis["issues"], 1):
icon = "🔴" if issue["level"] == "critical" else "🟡"
lines.append(f"{i}. {icon} [{issue['check']}] {issue['resource']}: {issue['message']}")
lines.append("")
# Pod 摘要
pods = diagnosis["checks"].get("pods", {})
if pods.get("summary"):
s = pods["summary"]
lines.append(f"## Pod 状态: 总计 {s['total']}, Running {s['running']}, Pending {s['pending']}, Failed {s['failed']}, CrashLoop {s.get('crashloop', 0)}")
# Node 摘要
nodes = diagnosis["checks"].get("nodes", {})
if nodes.get("summary"):
s = nodes["summary"]
lines.append(f"## Node 状态: 总计 {s['total']}, Ready {s['ready']}, NotReady {s['not_ready']}")
# Deployment 摘要
deps = diagnosis["checks"].get("deployments", {})
if deps.get("deployments"):
lines.append("## Deployment 状态:")
for d in deps["deployments"]:
lines.append(f" - {d['namespace']}/{d['name']}: {d['ready']}/{d['desired']} ready")
# 异常事件
events = diagnosis["checks"].get("events", {})
if events.get("events"):
lines.append(f"\n## 最近异常事件 (共 {len(events['events'])} 条):")
for ev in events["events"][-20:]:
lines.append(f" - [{ev['type']}] {ev['object']}: {ev['reason']} - {ev['message'][:150]}")
# 资源使用
usage = diagnosis["checks"].get("resource_usage", {})
if usage.get("nodes"):
lines.append("\n## 节点资源使用:")
for n in usage["nodes"]:
lines.append(f" - {n['name']}: CPU {n['cpu_pct']}%, 内存 {n['mem_pct']}%")
return "\n".join(lines)