From 8109204843e530ccfbc20f302cbf7a183c57c34d Mon Sep 17 00:00:00 2001 From: cnbugs Date: Sat, 25 Jul 2026 11:07:03 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E9=9B=86=E7=BE=A4?= =?UTF-8?q?=E4=B8=8D=E5=8F=AF=E8=BE=BE=E6=97=B6=E8=AF=AF=E6=8A=A5100?= =?UTF-8?q?=E5=88=86=E7=9A=84=E4=B8=A5=E9=87=8Dbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 check_connectivity() 连通性预检: kubectl存在性/kubeconfig/API Server连接 - 连不上集群直接返回0分+明确错误信息+排查建议(不再静默跳过) - 每个检查项kubectl失败时必须报critical问题(不再忽略error) - 前端: 连接失败时显示红色错误页+错误详情+排查建议+重新诊断按钮 - 智能识别常见错误: connection refused/证书过期/超时/DNS解析失败等 --- backend/diagnosis.py | 242 +++++++++++++++++++++++++++++++++---------- backend/main.py | 50 +++++++-- frontend/src/App.vue | 36 ++++++- 3 files changed, 263 insertions(+), 65 deletions(-) diff --git a/backend/diagnosis.py b/backend/diagnosis.py index b5e9ed0..7822a1c 100644 --- a/backend/diagnosis.py +++ b/backend/diagnosis.py @@ -27,8 +27,8 @@ async def _run_kubectl(args: list[str], timeout: int = 30) -> dict: return {"error": str(e), "data": None} -async def _run_kubectl_raw(args: list[str], timeout: int = 30) -> str: - """执行 kubectl 命令，返回原始文本""" +async def _run_kubectl_raw(args: list[str], timeout: int = 30) -> tuple[str, str, int]: + """执行 kubectl 命令，返回 (stdout, stderr, returncode)""" cmd = [KUBECTL] + args try: proc = await asyncio.create_subprocess_exec( @@ -37,9 +37,11 @@ async def _run_kubectl_raw(args: list[str], timeout: int = 30) -> str: stderr=asyncio.subprocess.PIPE, ) stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout) - return stdout.decode().strip() - except Exception: - return "" + return stdout.decode().strip(), stderr.decode().strip(), proc.returncode + except asyncio.TimeoutError: + return "", f"命令超时 ({timeout}s)", -1 + except Exception as e: + return "", str(e), -1 def _parse_resource_quantity(q: str) -> float: @@ -61,6 +63,56 @@ def _parse_resource_quantity(q: str) -> float: return 0 +# ─── 连通性检查 (最优先) ────────────────────────────────── + +async def check_connectivity() -> dict: + """检查 kubectl 是否能连接到集群 API Server""" + # 1. 检查 kubectl 是否存在 + import os + if not os.path.isfile(KUBECTL): + return { + "connected": False, + "error": f"kubectl 不存在: {KUBECTL}", + "suggestion": "请安装 kubectl 并确保在 PATH 中", + } + + # 2. 检查 kubeconfig + kubeconfig = os.environ.get("KUBECONFIG", os.path.expanduser("~/.kube/config")) + if not os.path.isfile(kubeconfig): + return { + "connected": False, + "error": f"kubeconfig 不存在: {kubeconfig}", + "suggestion": "请将集群 kubeconfig 放到 ~/.kube/config 或设置 KUBECONFIG 环境变量", + } + + # 3. 尝试连接 API Server + stdout, stderr, code = await _run_kubectl_raw(["cluster-info"], timeout=10) + if code != 0: + # 提取关键错误信息 + err_msg = stderr or stdout or "未知错误" + # 常见错误模式 + if "connection refused" in err_msg.lower(): + suggestion = "API Server 连接被拒绝。请检查: 1) kubelet/apiserver 是否运行 (systemctl status kubelet) 2) 证书是否过期 3) 端口 6443 是否监听" + elif "no such host" in err_msg.lower() or "dial tcp" in err_msg.lower(): + suggestion = "无法解析或连接 API Server 地址。请检查 kubeconfig 中的 server 地址是否正确" + elif "certificate" in err_msg.lower() or "x509" in err_msg.lower(): + suggestion = "证书错误。请检查: 1) 集群证书是否过期 (kubeadm certs check-expiration) 2) 系统时间是否正确" + elif "timeout" in err_msg.lower() or "timed out" in err_msg.lower(): + suggestion = "连接超时。请检查: 1) 网络是否可达 2) 防火墙是否放行 6443 端口 3) API Server 是否运行" + elif "unable to connect" in err_msg.lower() or "could not" in err_msg.lower(): + suggestion = "无法连接到集群。请检查: 1) systemctl status kubelet 2) journalctl -u kubelet --no-pager -n 50 3) crictl ps 查看容器状态" + else: + suggestion = "请检查 kubelet 和 API Server 状态: systemctl status kubelet && journalctl -u kubelet -n 50" + + return { + "connected": False, + "error": err_msg[:500], + "suggestion": suggestion, + } + + return {"connected": True, "error": None, "info": stdout} + + # ─── 诊断项 ─────────────────────────────────────────────── async def check_cluster_info() -> dict: @@ -68,9 +120,15 @@ async def check_cluster_info() -> dict: ver = await _run_kubectl(["version"]) nodes = await _run_kubectl(["get", "nodes"]) namespaces = await _run_kubectl(["get", "namespaces"]) + + issues = [] + if ver.get("error"): + issues.append({"level": "critical", "resource": "cluster", "message": f"无法获取集群版本: {ver['error'][:200]}"}) + return { "name": "cluster_info", "title": "集群信息", + "issues": issues, "version": ver, "nodes": nodes, "namespaces": namespaces, @@ -85,20 +143,23 @@ async def check_pods(namespace: str = "") -> dict: else: args += ["--all-namespaces"] result = await _run_kubectl(args) - + issues = [] pods_summary = {"total": 0, "running": 0, "pending": 0, "failed": 0, "unknown": 0, "crashloop": 0} - + + if result.get("error"): + issues.append({"level": "critical", "resource": "pods", "message": f"无法获取 Pod 列表: {result['error'][:200]}"}) + return {"name": "pods", "title": "Pod 状态检查", "summary": pods_summary, "issues": issues, "raw": result} + if result["data"] and "items" in result["data"]: for pod in result["data"]["items"]: pods_summary["total"] += 1 phase = pod.get("status", {}).get("phase", "Unknown") name = pod["metadata"]["name"] ns = pod["metadata"].get("namespace", "default") - + if phase == "Running": pods_summary["running"] += 1 - # 检查重启次数 for cs in pod.get("status", {}).get("containerStatuses", []): restarts = cs.get("restartCount", 0) if restarts > 5: @@ -125,7 +186,7 @@ async def check_pods(namespace: str = "") -> dict: issues.append({"level": "critical", "resource": f"{ns}/{name}", "message": "Pod 处于 Failed 状态"}) else: pods_summary["unknown"] += 1 - + return { "name": "pods", "title": "Pod 状态检查", @@ -140,7 +201,11 @@ async def check_nodes() -> dict: result = await _run_kubectl(["get", "nodes"]) issues = [] nodes_summary = {"total": 0, "ready": 0, "not_ready": 0} - + + if result.get("error"): + issues.append({"level": "critical", "resource": "nodes", "message": f"无法获取 Node 列表: {result['error'][:200]}"}) + return {"name": "nodes", "title": "Node 状态检查", "summary": nodes_summary, "issues": issues, "raw": result} + if result["data"] and "items" in result["data"]: for node in result["data"]["items"]: nodes_summary["total"] += 1 @@ -161,8 +226,7 @@ async def check_nodes() -> dict: else: nodes_summary["not_ready"] += 1 issues.append({"level": "critical", "resource": name, "message": "节点 NotReady"}) - - # 资源使用 + alloc = node.get("status", {}).get("allocatable", {}) cap = node.get("status", {}).get("capacity", {}) if alloc and cap: @@ -175,7 +239,7 @@ async def check_nodes() -> dict: "resource": name, "message": f"节点 {res} 可分配量不足 10%", }) - + return { "name": "nodes", "title": "Node 状态检查", @@ -193,11 +257,16 @@ async def check_events(namespace: str = "") -> dict: else: args += ["--all-namespaces"] result = await _run_kubectl(args) - + issues = [] events_list = [] + + if result.get("error"): + issues.append({"level": "critical", "resource": "events", "message": f"无法获取事件: {result['error'][:200]}"}) + return {"name": "events", "title": "异常事件检查", "warning_count": 0, "events": [], "issues": issues, "raw": result} + if result["data"] and "items" in result["data"]: - for ev in result["data"]["items"][-50:]: # 最近50条 + for ev in result["data"]["items"][-50:]: events_list.append({ "namespace": ev["metadata"].get("namespace", ""), "reason": ev.get("reason", ""), @@ -213,11 +282,11 @@ async def check_events(namespace: str = "") -> dict: "resource": f"{ev['metadata'].get('namespace', '')}/{ev.get('involvedObject', {}).get('name', '')}", "message": f"[{ev.get('reason', '')}] {ev.get('message', '')[:200]}", }) - + return { "name": "events", "title": "异常事件检查", - "warning_count": len(issues), + "warning_count": len([i for i in issues if i["level"] == "warning"]), "events": events_list, "issues": issues, "raw": result, @@ -232,9 +301,14 @@ async def check_deployments(namespace: str = "") -> dict: else: args += ["--all-namespaces"] result = await _run_kubectl(args) - + issues = [] deploys = [] + + if result.get("error"): + issues.append({"level": "critical", "resource": "deployments", "message": f"无法获取 Deployment: {result['error'][:200]}"}) + return {"name": "deployments", "title": "Deployment 状态检查", "deployments": [], "issues": issues, "raw": result} + if result["data"] and "items" in result["data"]: for dep in result["data"]["items"]: name = dep["metadata"]["name"] @@ -244,7 +318,7 @@ async def check_deployments(namespace: str = "") -> dict: ready = status.get("readyReplicas", 0) or 0 available = status.get("availableReplicas", 0) or 0 updated = status.get("updatedReplicas", 0) or 0 - + deploys.append({ "namespace": ns, "name": name, "desired": spec_replicas, "ready": ready, @@ -262,7 +336,7 @@ async def check_deployments(namespace: str = "") -> dict: "resource": f"{ns}/{name}", "message": f"滚动更新未完成: 已更新 {updated}/{spec_replicas}", }) - + return { "name": "deployments", "title": "Deployment 状态检查", @@ -280,8 +354,13 @@ async def check_services(namespace: str = "") -> dict: else: args += ["--all-namespaces"] result = await _run_kubectl(args) - + issues = [] + + if result.get("error"): + issues.append({"level": "critical", "resource": "services", "message": f"无法获取 Endpoint: {result['error'][:200]}"}) + return {"name": "services", "title": "Service/Endpoint 检查", "issues": issues, "raw": result} + if result["data"] and "items" in result["data"]: for ep in result["data"]["items"]: name = ep["metadata"]["name"] @@ -294,7 +373,7 @@ async def check_services(namespace: str = "") -> dict: "resource": f"{ns}/{name}", "message": "Service 没有可用的后端 Endpoint", }) - + return { "name": "services", "title": "Service/Endpoint 检查", @@ -311,8 +390,13 @@ async def check_pvc(namespace: str = "") -> dict: else: args += ["--all-namespaces"] result = await _run_kubectl(args) - + issues = [] + + if result.get("error"): + issues.append({"level": "critical", "resource": "pvc", "message": f"无法获取 PVC: {result['error'][:200]}"}) + return {"name": "pvc", "title": "PVC 存储检查", "issues": issues, "raw": result} + if result["data"] and "items" in result["data"]: for pvc in result["data"]["items"]: name = pvc["metadata"]["name"] @@ -324,7 +408,7 @@ async def check_pvc(namespace: str = "") -> dict: "resource": f"{ns}/{name}", "message": f"PVC 状态异常: {phase}", }) - + return { "name": "pvc", "title": "PVC 存储检查", @@ -335,12 +419,23 @@ async def check_pvc(namespace: str = "") -> dict: async def check_resource_usage() -> dict: """节点资源使用率 (需要 metrics-server)""" - raw = await _run_kubectl_raw(["top", "nodes", "--no-headers"]) + stdout, stderr, code = await _run_kubectl_raw(["top", "nodes", "--no-headers"]) issues = [] nodes_usage = [] - - if raw and "error" not in raw.lower(): - for line in raw.strip().split("\n"): + + if code != 0: + # metrics-server 不可用不算 critical，只是跳过 + return { + "name": "resource_usage", + "title": "资源使用率", + "issues": [], + "nodes": [], + "note": f"metrics-server 不可用: {(stderr or stdout)[:200]}", + "raw": {"error": stderr or stdout}, + } + + if stdout: + for line in stdout.strip().split("\n"): parts = line.split() if len(parts) >= 5: name = parts[0] @@ -359,22 +454,13 @@ async def check_resource_usage() -> dict: issues.append({"level": "warning", "resource": name, "message": f"内存使用率 {mem_v}%"}) except ValueError: pass - else: - return { - "name": "resource_usage", - "title": "资源使用率", - "issues": [], - "nodes": [], - "note": "metrics-server 不可用，跳过资源使用率检查", - "raw": {"error": raw}, - } - + return { "name": "resource_usage", "title": "资源使用率", "issues": issues, "nodes": nodes_usage, - "raw": {"text": raw}, + "raw": {"text": stdout}, } @@ -386,12 +472,18 @@ async def check_network_policies(namespace: str = "") -> dict: else: args += ["--all-namespaces"] result = await _run_kubectl(args) + + issues = [] + if result.get("error"): + issues.append({"level": "warning", "resource": "network", "message": f"无法获取网络策略: {result['error'][:200]}"}) + return {"name": "network", "title": "网络策略检查", "policy_count": 0, "issues": issues, "raw": result} + count = len(result["data"].get("items", [])) if result["data"] else 0 return { "name": "network", "title": "网络策略检查", "policy_count": count, - "issues": [], + "issues": issues, "raw": result, } @@ -414,7 +506,31 @@ ALL_CHECKS = { async def run_full_diagnosis(namespace: str = "", checks: Optional[list[str]] = None) -> dict: """执行完整诊断""" start = datetime.now() - + + # ★ 第一步: 连通性检查 + conn = await check_connectivity() + if not conn["connected"]: + return { + "timestamp": start.isoformat(), + "elapsed_seconds": round((datetime.now() - start).total_seconds(), 2), + "namespace": namespace or "all", + "score": 0, + "status": "critical", + "connected": False, + "connectivity_error": conn["error"], + "connectivity_suggestion": conn.get("suggestion", ""), + "critical_count": 1, + "warning_count": 0, + "issues": [{ + "level": "critical", + "check": "集群连通性", + "resource": "API Server", + "message": f"无法连接到集群: {conn['error'][:300]}", + }], + "checks": {}, + } + + # ★ 第二步: 执行各项检查 selected = checks or list(ALL_CHECKS.keys()) tasks = {} for name in selected: @@ -424,45 +540,49 @@ async def run_full_diagnosis(namespace: str = "", checks: Optional[list[str]] = tasks[name] = fn(namespace) else: tasks[name] = fn() - + results = {} for name, coro in tasks.items(): try: results[name] = await coro except Exception as e: - results[name] = {"name": name, "title": name, "error": str(e), "issues": []} - + results[name] = { + "name": name, "title": name, "error": str(e), + "issues": [{"level": "critical", "resource": name, "message": f"检查执行异常: {str(e)}"}], + } + # 汇总 all_issues = [] for r in results.values(): for issue in r.get("issues", []): issue["check"] = r.get("title", r.get("name", "")) all_issues.append(issue) - + critical = [i for i in all_issues if i["level"] == "critical"] warning = [i for i in all_issues if i["level"] == "warning"] - + elapsed = (datetime.now() - start).total_seconds() - + # 健康评分 (100 分制) score = 100 score -= len(critical) * 15 score -= len(warning) * 5 score = max(0, score) - + if score >= 90: status = "healthy" elif score >= 60: status = "warning" else: status = "critical" - + return { "timestamp": start.isoformat(), "elapsed_seconds": round(elapsed, 2), "namespace": namespace or "all", "score": score, "status": status, + "connected": True, "critical_count": len(critical), "warning_count": len(warning), "issues": all_issues, @@ -479,45 +599,53 @@ def build_diagnosis_report(diagnosis: dict) -> str: lines.append(f"健康评分: {diagnosis['score']}/100 ({diagnosis['status']})") lines.append(f"严重问题: {diagnosis['critical_count']} 个, 警告: {diagnosis['warning_count']} 个") lines.append("") - + + # 连通性失败 + if not diagnosis.get("connected", True): + lines.append("## ⚠️ 集群连接失败") + lines.append(f"错误: {diagnosis.get('connectivity_error', '未知')}") + lines.append(f"建议: {diagnosis.get('connectivity_suggestion', '')}") + lines.append("") + return "\n".join(lines) + if diagnosis["issues"]: lines.append("## 发现的问题") for i, issue in enumerate(diagnosis["issues"], 1): icon = "🔴" if issue["level"] == "critical" else "🟡" lines.append(f"{i}. {icon} [{issue['check']}] {issue['resource']}: {issue['message']}") lines.append("") - + # Pod 摘要 pods = diagnosis["checks"].get("pods", {}) if pods.get("summary"): s = pods["summary"] lines.append(f"## Pod 状态: 总计 {s['total']}, Running {s['running']}, Pending {s['pending']}, Failed {s['failed']}, CrashLoop {s.get('crashloop', 0)}") - + # Node 摘要 nodes = diagnosis["checks"].get("nodes", {}) if nodes.get("summary"): s = nodes["summary"] lines.append(f"## Node 状态: 总计 {s['total']}, Ready {s['ready']}, NotReady {s['not_ready']}") - + # Deployment 摘要 deps = diagnosis["checks"].get("deployments", {}) if deps.get("deployments"): lines.append("## Deployment 状态:") for d in deps["deployments"]: lines.append(f" - {d['namespace']}/{d['name']}: {d['ready']}/{d['desired']} ready") - + # 异常事件 events = diagnosis["checks"].get("events", {}) if events.get("events"): lines.append(f"\n## 最近异常事件 (共 {len(events['events'])} 条):") for ev in events["events"][-20:]: lines.append(f" - [{ev['type']}] {ev['object']}: {ev['reason']} - {ev['message'][:150]}") - + # 资源使用 usage = diagnosis["checks"].get("resource_usage", {}) if usage.get("nodes"): lines.append("\n## 节点资源使用:") for n in usage["nodes"]: lines.append(f" - {n['name']}: CPU {n['cpu_pct']}%, 内存 {n['mem_pct']}%") - + return "\n".join(lines) diff --git a/backend/main.py b/backend/main.py index 8dd6363..6264c6b 100644 --- a/backend/main.py +++ b/backend/main.py @@ -13,6 +13,7 @@ from typing import Optional from backend.config import PORT from backend.diagnosis import ( run_full_diagnosis, build_diagnosis_report, ALL_CHECKS, + check_connectivity, check_cluster_info, check_pods, check_nodes, check_events, check_deployments, check_services, check_pvc, check_resource_usage, check_network_policies, @@ -82,12 +83,46 @@ async def diagnose_stream(namespace: str = ""): async def event_generator(): start = datetime.now() + + # ★ 第一步: 连通性检查 + yield f"data: {json.dumps({'type': 'progress', 'step': 0, 'total': len(CHECK_ORDER) + 1, 'name': 'connectivity', 'title': '集群连通性检查', 'status': 'running'}, ensure_ascii=False)}\n\n" + + conn = await check_connectivity() + if not conn["connected"]: + yield f"data: {json.dumps({'type': 'progress', 'step': 0, 'total': len(CHECK_ORDER) + 1, 'name': 'connectivity', 'title': '集群连通性检查', 'status': 'done', 'issues': 1}, ensure_ascii=False)}\n\n" + + diagnosis = { + "timestamp": start.isoformat(), + "elapsed_seconds": round((datetime.now() - start).total_seconds(), 2), + "namespace": namespace or "all", + "score": 0, + "status": "critical", + "connected": False, + "connectivity_error": conn["error"], + "connectivity_suggestion": conn.get("suggestion", ""), + "critical_count": 1, + "warning_count": 0, + "issues": [{ + "level": "critical", + "check": "集群连通性", + "resource": "API Server", + "message": f"无法连接到集群: {conn['error'][:300]}", + }], + "checks": {}, + } + _last_diagnosis = diagnosis + _last_report = build_diagnosis_report(diagnosis) + yield f"data: {json.dumps({'type': 'complete', 'diagnosis': diagnosis}, ensure_ascii=False)}\n\n" + return + + yield f"data: {json.dumps({'type': 'progress', 'step': 0, 'total': len(CHECK_ORDER) + 1, 'name': 'connectivity', 'title': '集群连通性检查', 'status': 'done', 'issues': 0}, ensure_ascii=False)}\n\n" + + # ★ 第二步: 执行各项检查 results = {} all_issues = [] - total = len(CHECK_ORDER) + total = len(CHECK_ORDER) + 1 for idx, (name, title, fn) in enumerate(CHECK_ORDER, 1): - # 推送: 开始检查 yield f"data: {json.dumps({'type': 'progress', 'step': idx, 'total': total, 'name': name, 'title': title, 'status': 'running'}, ensure_ascii=False)}\n\n" try: @@ -101,10 +136,13 @@ async def diagnose_stream(namespace: str = ""): issue["check"] = title all_issues.append(issue) except Exception as e: - results[name] = {"name": name, "title": title, "error": str(e), "issues": []} - issue_count = -1 + results[name] = { + "name": name, "title": title, "error": str(e), + "issues": [{"level": "critical", "resource": name, "message": f"检查执行异常: {str(e)}"}], + } + issue_count = 1 + all_issues.append({"level": "critical", "check": title, "resource": name, "message": f"检查执行异常: {str(e)}"}) - # 推送: 完成检查 yield f"data: {json.dumps({'type': 'progress', 'step': idx, 'total': total, 'name': name, 'title': title, 'status': 'done', 'issues': issue_count}, ensure_ascii=False)}\n\n" # 汇总 @@ -120,6 +158,7 @@ async def diagnose_stream(namespace: str = ""): "namespace": namespace or "all", "score": score, "status": status, + "connected": True, "critical_count": len(critical), "warning_count": len(warning), "issues": all_issues, @@ -129,7 +168,6 @@ async def diagnose_stream(namespace: str = ""): _last_diagnosis = diagnosis _last_report = build_diagnosis_report(diagnosis) - # 推送: 最终结果 yield f"data: {json.dumps({'type': 'complete', 'diagnosis': diagnosis}, ensure_ascii=False)}\n\n" return StreamingResponse( diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 739e506..7e4c5bb 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -58,8 +58,31 @@ + +
+ +
+ +
+

无法连接到 Kubernetes 集群

+
+ + + 
{{ diagnosis.connectivity_error }}
+ + + +
{{ diagnosis.connectivity_suggestion }}
+ +
+
+ 重新诊断 +
+ +
+ -