fix: 修复集群不可达时误报100分的严重bug
- 新增 check_connectivity() 连通性预检: kubectl存在性/kubeconfig/API Server连接 - 连不上集群直接返回0分+明确错误信息+排查建议(不再静默跳过) - 每个检查项kubectl失败时必须报critical问题(不再忽略error) - 前端: 连接失败时显示红色错误页+错误详情+排查建议+重新诊断按钮 - 智能识别常见错误: connection refused/证书过期/超时/DNS解析失败等
This commit is contained in:
+185
-57
@@ -27,8 +27,8 @@ async def _run_kubectl(args: list[str], timeout: int = 30) -> dict:
|
|||||||
return {"error": str(e), "data": None}
|
return {"error": str(e), "data": None}
|
||||||
|
|
||||||
|
|
||||||
async def _run_kubectl_raw(args: list[str], timeout: int = 30) -> str:
|
async def _run_kubectl_raw(args: list[str], timeout: int = 30) -> tuple[str, str, int]:
|
||||||
"""执行 kubectl 命令,返回原始文本"""
|
"""执行 kubectl 命令,返回 (stdout, stderr, returncode)"""
|
||||||
cmd = [KUBECTL] + args
|
cmd = [KUBECTL] + args
|
||||||
try:
|
try:
|
||||||
proc = await asyncio.create_subprocess_exec(
|
proc = await asyncio.create_subprocess_exec(
|
||||||
@@ -37,9 +37,11 @@ async def _run_kubectl_raw(args: list[str], timeout: int = 30) -> str:
|
|||||||
stderr=asyncio.subprocess.PIPE,
|
stderr=asyncio.subprocess.PIPE,
|
||||||
)
|
)
|
||||||
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
|
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
|
||||||
return stdout.decode().strip()
|
return stdout.decode().strip(), stderr.decode().strip(), proc.returncode
|
||||||
except Exception:
|
except asyncio.TimeoutError:
|
||||||
return ""
|
return "", f"命令超时 ({timeout}s)", -1
|
||||||
|
except Exception as e:
|
||||||
|
return "", str(e), -1
|
||||||
|
|
||||||
|
|
||||||
def _parse_resource_quantity(q: str) -> float:
|
def _parse_resource_quantity(q: str) -> float:
|
||||||
@@ -61,6 +63,56 @@ def _parse_resource_quantity(q: str) -> float:
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
# ─── 连通性检查 (最优先) ──────────────────────────────────
|
||||||
|
|
||||||
|
async def check_connectivity() -> dict:
|
||||||
|
"""检查 kubectl 是否能连接到集群 API Server"""
|
||||||
|
# 1. 检查 kubectl 是否存在
|
||||||
|
import os
|
||||||
|
if not os.path.isfile(KUBECTL):
|
||||||
|
return {
|
||||||
|
"connected": False,
|
||||||
|
"error": f"kubectl 不存在: {KUBECTL}",
|
||||||
|
"suggestion": "请安装 kubectl 并确保在 PATH 中",
|
||||||
|
}
|
||||||
|
|
||||||
|
# 2. 检查 kubeconfig
|
||||||
|
kubeconfig = os.environ.get("KUBECONFIG", os.path.expanduser("~/.kube/config"))
|
||||||
|
if not os.path.isfile(kubeconfig):
|
||||||
|
return {
|
||||||
|
"connected": False,
|
||||||
|
"error": f"kubeconfig 不存在: {kubeconfig}",
|
||||||
|
"suggestion": "请将集群 kubeconfig 放到 ~/.kube/config 或设置 KUBECONFIG 环境变量",
|
||||||
|
}
|
||||||
|
|
||||||
|
# 3. 尝试连接 API Server
|
||||||
|
stdout, stderr, code = await _run_kubectl_raw(["cluster-info"], timeout=10)
|
||||||
|
if code != 0:
|
||||||
|
# 提取关键错误信息
|
||||||
|
err_msg = stderr or stdout or "未知错误"
|
||||||
|
# 常见错误模式
|
||||||
|
if "connection refused" in err_msg.lower():
|
||||||
|
suggestion = "API Server 连接被拒绝。请检查: 1) kubelet/apiserver 是否运行 (systemctl status kubelet) 2) 证书是否过期 3) 端口 6443 是否监听"
|
||||||
|
elif "no such host" in err_msg.lower() or "dial tcp" in err_msg.lower():
|
||||||
|
suggestion = "无法解析或连接 API Server 地址。请检查 kubeconfig 中的 server 地址是否正确"
|
||||||
|
elif "certificate" in err_msg.lower() or "x509" in err_msg.lower():
|
||||||
|
suggestion = "证书错误。请检查: 1) 集群证书是否过期 (kubeadm certs check-expiration) 2) 系统时间是否正确"
|
||||||
|
elif "timeout" in err_msg.lower() or "timed out" in err_msg.lower():
|
||||||
|
suggestion = "连接超时。请检查: 1) 网络是否可达 2) 防火墙是否放行 6443 端口 3) API Server 是否运行"
|
||||||
|
elif "unable to connect" in err_msg.lower() or "could not" in err_msg.lower():
|
||||||
|
suggestion = "无法连接到集群。请检查: 1) systemctl status kubelet 2) journalctl -u kubelet --no-pager -n 50 3) crictl ps 查看容器状态"
|
||||||
|
else:
|
||||||
|
suggestion = "请检查 kubelet 和 API Server 状态: systemctl status kubelet && journalctl -u kubelet -n 50"
|
||||||
|
|
||||||
|
return {
|
||||||
|
"connected": False,
|
||||||
|
"error": err_msg[:500],
|
||||||
|
"suggestion": suggestion,
|
||||||
|
}
|
||||||
|
|
||||||
|
return {"connected": True, "error": None, "info": stdout}
|
||||||
|
|
||||||
|
|
||||||
# ─── 诊断项 ───────────────────────────────────────────────
|
# ─── 诊断项 ───────────────────────────────────────────────
|
||||||
|
|
||||||
async def check_cluster_info() -> dict:
|
async def check_cluster_info() -> dict:
|
||||||
@@ -68,9 +120,15 @@ async def check_cluster_info() -> dict:
|
|||||||
ver = await _run_kubectl(["version"])
|
ver = await _run_kubectl(["version"])
|
||||||
nodes = await _run_kubectl(["get", "nodes"])
|
nodes = await _run_kubectl(["get", "nodes"])
|
||||||
namespaces = await _run_kubectl(["get", "namespaces"])
|
namespaces = await _run_kubectl(["get", "namespaces"])
|
||||||
|
|
||||||
|
issues = []
|
||||||
|
if ver.get("error"):
|
||||||
|
issues.append({"level": "critical", "resource": "cluster", "message": f"无法获取集群版本: {ver['error'][:200]}"})
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"name": "cluster_info",
|
"name": "cluster_info",
|
||||||
"title": "集群信息",
|
"title": "集群信息",
|
||||||
|
"issues": issues,
|
||||||
"version": ver,
|
"version": ver,
|
||||||
"nodes": nodes,
|
"nodes": nodes,
|
||||||
"namespaces": namespaces,
|
"namespaces": namespaces,
|
||||||
@@ -85,20 +143,23 @@ async def check_pods(namespace: str = "") -> dict:
|
|||||||
else:
|
else:
|
||||||
args += ["--all-namespaces"]
|
args += ["--all-namespaces"]
|
||||||
result = await _run_kubectl(args)
|
result = await _run_kubectl(args)
|
||||||
|
|
||||||
issues = []
|
issues = []
|
||||||
pods_summary = {"total": 0, "running": 0, "pending": 0, "failed": 0, "unknown": 0, "crashloop": 0}
|
pods_summary = {"total": 0, "running": 0, "pending": 0, "failed": 0, "unknown": 0, "crashloop": 0}
|
||||||
|
|
||||||
|
if result.get("error"):
|
||||||
|
issues.append({"level": "critical", "resource": "pods", "message": f"无法获取 Pod 列表: {result['error'][:200]}"})
|
||||||
|
return {"name": "pods", "title": "Pod 状态检查", "summary": pods_summary, "issues": issues, "raw": result}
|
||||||
|
|
||||||
if result["data"] and "items" in result["data"]:
|
if result["data"] and "items" in result["data"]:
|
||||||
for pod in result["data"]["items"]:
|
for pod in result["data"]["items"]:
|
||||||
pods_summary["total"] += 1
|
pods_summary["total"] += 1
|
||||||
phase = pod.get("status", {}).get("phase", "Unknown")
|
phase = pod.get("status", {}).get("phase", "Unknown")
|
||||||
name = pod["metadata"]["name"]
|
name = pod["metadata"]["name"]
|
||||||
ns = pod["metadata"].get("namespace", "default")
|
ns = pod["metadata"].get("namespace", "default")
|
||||||
|
|
||||||
if phase == "Running":
|
if phase == "Running":
|
||||||
pods_summary["running"] += 1
|
pods_summary["running"] += 1
|
||||||
# 检查重启次数
|
|
||||||
for cs in pod.get("status", {}).get("containerStatuses", []):
|
for cs in pod.get("status", {}).get("containerStatuses", []):
|
||||||
restarts = cs.get("restartCount", 0)
|
restarts = cs.get("restartCount", 0)
|
||||||
if restarts > 5:
|
if restarts > 5:
|
||||||
@@ -125,7 +186,7 @@ async def check_pods(namespace: str = "") -> dict:
|
|||||||
issues.append({"level": "critical", "resource": f"{ns}/{name}", "message": "Pod 处于 Failed 状态"})
|
issues.append({"level": "critical", "resource": f"{ns}/{name}", "message": "Pod 处于 Failed 状态"})
|
||||||
else:
|
else:
|
||||||
pods_summary["unknown"] += 1
|
pods_summary["unknown"] += 1
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"name": "pods",
|
"name": "pods",
|
||||||
"title": "Pod 状态检查",
|
"title": "Pod 状态检查",
|
||||||
@@ -140,7 +201,11 @@ async def check_nodes() -> dict:
|
|||||||
result = await _run_kubectl(["get", "nodes"])
|
result = await _run_kubectl(["get", "nodes"])
|
||||||
issues = []
|
issues = []
|
||||||
nodes_summary = {"total": 0, "ready": 0, "not_ready": 0}
|
nodes_summary = {"total": 0, "ready": 0, "not_ready": 0}
|
||||||
|
|
||||||
|
if result.get("error"):
|
||||||
|
issues.append({"level": "critical", "resource": "nodes", "message": f"无法获取 Node 列表: {result['error'][:200]}"})
|
||||||
|
return {"name": "nodes", "title": "Node 状态检查", "summary": nodes_summary, "issues": issues, "raw": result}
|
||||||
|
|
||||||
if result["data"] and "items" in result["data"]:
|
if result["data"] and "items" in result["data"]:
|
||||||
for node in result["data"]["items"]:
|
for node in result["data"]["items"]:
|
||||||
nodes_summary["total"] += 1
|
nodes_summary["total"] += 1
|
||||||
@@ -161,8 +226,7 @@ async def check_nodes() -> dict:
|
|||||||
else:
|
else:
|
||||||
nodes_summary["not_ready"] += 1
|
nodes_summary["not_ready"] += 1
|
||||||
issues.append({"level": "critical", "resource": name, "message": "节点 NotReady"})
|
issues.append({"level": "critical", "resource": name, "message": "节点 NotReady"})
|
||||||
|
|
||||||
# 资源使用
|
|
||||||
alloc = node.get("status", {}).get("allocatable", {})
|
alloc = node.get("status", {}).get("allocatable", {})
|
||||||
cap = node.get("status", {}).get("capacity", {})
|
cap = node.get("status", {}).get("capacity", {})
|
||||||
if alloc and cap:
|
if alloc and cap:
|
||||||
@@ -175,7 +239,7 @@ async def check_nodes() -> dict:
|
|||||||
"resource": name,
|
"resource": name,
|
||||||
"message": f"节点 {res} 可分配量不足 10%",
|
"message": f"节点 {res} 可分配量不足 10%",
|
||||||
})
|
})
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"name": "nodes",
|
"name": "nodes",
|
||||||
"title": "Node 状态检查",
|
"title": "Node 状态检查",
|
||||||
@@ -193,11 +257,16 @@ async def check_events(namespace: str = "") -> dict:
|
|||||||
else:
|
else:
|
||||||
args += ["--all-namespaces"]
|
args += ["--all-namespaces"]
|
||||||
result = await _run_kubectl(args)
|
result = await _run_kubectl(args)
|
||||||
|
|
||||||
issues = []
|
issues = []
|
||||||
events_list = []
|
events_list = []
|
||||||
|
|
||||||
|
if result.get("error"):
|
||||||
|
issues.append({"level": "critical", "resource": "events", "message": f"无法获取事件: {result['error'][:200]}"})
|
||||||
|
return {"name": "events", "title": "异常事件检查", "warning_count": 0, "events": [], "issues": issues, "raw": result}
|
||||||
|
|
||||||
if result["data"] and "items" in result["data"]:
|
if result["data"] and "items" in result["data"]:
|
||||||
for ev in result["data"]["items"][-50:]: # 最近50条
|
for ev in result["data"]["items"][-50:]:
|
||||||
events_list.append({
|
events_list.append({
|
||||||
"namespace": ev["metadata"].get("namespace", ""),
|
"namespace": ev["metadata"].get("namespace", ""),
|
||||||
"reason": ev.get("reason", ""),
|
"reason": ev.get("reason", ""),
|
||||||
@@ -213,11 +282,11 @@ async def check_events(namespace: str = "") -> dict:
|
|||||||
"resource": f"{ev['metadata'].get('namespace', '')}/{ev.get('involvedObject', {}).get('name', '')}",
|
"resource": f"{ev['metadata'].get('namespace', '')}/{ev.get('involvedObject', {}).get('name', '')}",
|
||||||
"message": f"[{ev.get('reason', '')}] {ev.get('message', '')[:200]}",
|
"message": f"[{ev.get('reason', '')}] {ev.get('message', '')[:200]}",
|
||||||
})
|
})
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"name": "events",
|
"name": "events",
|
||||||
"title": "异常事件检查",
|
"title": "异常事件检查",
|
||||||
"warning_count": len(issues),
|
"warning_count": len([i for i in issues if i["level"] == "warning"]),
|
||||||
"events": events_list,
|
"events": events_list,
|
||||||
"issues": issues,
|
"issues": issues,
|
||||||
"raw": result,
|
"raw": result,
|
||||||
@@ -232,9 +301,14 @@ async def check_deployments(namespace: str = "") -> dict:
|
|||||||
else:
|
else:
|
||||||
args += ["--all-namespaces"]
|
args += ["--all-namespaces"]
|
||||||
result = await _run_kubectl(args)
|
result = await _run_kubectl(args)
|
||||||
|
|
||||||
issues = []
|
issues = []
|
||||||
deploys = []
|
deploys = []
|
||||||
|
|
||||||
|
if result.get("error"):
|
||||||
|
issues.append({"level": "critical", "resource": "deployments", "message": f"无法获取 Deployment: {result['error'][:200]}"})
|
||||||
|
return {"name": "deployments", "title": "Deployment 状态检查", "deployments": [], "issues": issues, "raw": result}
|
||||||
|
|
||||||
if result["data"] and "items" in result["data"]:
|
if result["data"] and "items" in result["data"]:
|
||||||
for dep in result["data"]["items"]:
|
for dep in result["data"]["items"]:
|
||||||
name = dep["metadata"]["name"]
|
name = dep["metadata"]["name"]
|
||||||
@@ -244,7 +318,7 @@ async def check_deployments(namespace: str = "") -> dict:
|
|||||||
ready = status.get("readyReplicas", 0) or 0
|
ready = status.get("readyReplicas", 0) or 0
|
||||||
available = status.get("availableReplicas", 0) or 0
|
available = status.get("availableReplicas", 0) or 0
|
||||||
updated = status.get("updatedReplicas", 0) or 0
|
updated = status.get("updatedReplicas", 0) or 0
|
||||||
|
|
||||||
deploys.append({
|
deploys.append({
|
||||||
"namespace": ns, "name": name,
|
"namespace": ns, "name": name,
|
||||||
"desired": spec_replicas, "ready": ready,
|
"desired": spec_replicas, "ready": ready,
|
||||||
@@ -262,7 +336,7 @@ async def check_deployments(namespace: str = "") -> dict:
|
|||||||
"resource": f"{ns}/{name}",
|
"resource": f"{ns}/{name}",
|
||||||
"message": f"滚动更新未完成: 已更新 {updated}/{spec_replicas}",
|
"message": f"滚动更新未完成: 已更新 {updated}/{spec_replicas}",
|
||||||
})
|
})
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"name": "deployments",
|
"name": "deployments",
|
||||||
"title": "Deployment 状态检查",
|
"title": "Deployment 状态检查",
|
||||||
@@ -280,8 +354,13 @@ async def check_services(namespace: str = "") -> dict:
|
|||||||
else:
|
else:
|
||||||
args += ["--all-namespaces"]
|
args += ["--all-namespaces"]
|
||||||
result = await _run_kubectl(args)
|
result = await _run_kubectl(args)
|
||||||
|
|
||||||
issues = []
|
issues = []
|
||||||
|
|
||||||
|
if result.get("error"):
|
||||||
|
issues.append({"level": "critical", "resource": "services", "message": f"无法获取 Endpoint: {result['error'][:200]}"})
|
||||||
|
return {"name": "services", "title": "Service/Endpoint 检查", "issues": issues, "raw": result}
|
||||||
|
|
||||||
if result["data"] and "items" in result["data"]:
|
if result["data"] and "items" in result["data"]:
|
||||||
for ep in result["data"]["items"]:
|
for ep in result["data"]["items"]:
|
||||||
name = ep["metadata"]["name"]
|
name = ep["metadata"]["name"]
|
||||||
@@ -294,7 +373,7 @@ async def check_services(namespace: str = "") -> dict:
|
|||||||
"resource": f"{ns}/{name}",
|
"resource": f"{ns}/{name}",
|
||||||
"message": "Service 没有可用的后端 Endpoint",
|
"message": "Service 没有可用的后端 Endpoint",
|
||||||
})
|
})
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"name": "services",
|
"name": "services",
|
||||||
"title": "Service/Endpoint 检查",
|
"title": "Service/Endpoint 检查",
|
||||||
@@ -311,8 +390,13 @@ async def check_pvc(namespace: str = "") -> dict:
|
|||||||
else:
|
else:
|
||||||
args += ["--all-namespaces"]
|
args += ["--all-namespaces"]
|
||||||
result = await _run_kubectl(args)
|
result = await _run_kubectl(args)
|
||||||
|
|
||||||
issues = []
|
issues = []
|
||||||
|
|
||||||
|
if result.get("error"):
|
||||||
|
issues.append({"level": "critical", "resource": "pvc", "message": f"无法获取 PVC: {result['error'][:200]}"})
|
||||||
|
return {"name": "pvc", "title": "PVC 存储检查", "issues": issues, "raw": result}
|
||||||
|
|
||||||
if result["data"] and "items" in result["data"]:
|
if result["data"] and "items" in result["data"]:
|
||||||
for pvc in result["data"]["items"]:
|
for pvc in result["data"]["items"]:
|
||||||
name = pvc["metadata"]["name"]
|
name = pvc["metadata"]["name"]
|
||||||
@@ -324,7 +408,7 @@ async def check_pvc(namespace: str = "") -> dict:
|
|||||||
"resource": f"{ns}/{name}",
|
"resource": f"{ns}/{name}",
|
||||||
"message": f"PVC 状态异常: {phase}",
|
"message": f"PVC 状态异常: {phase}",
|
||||||
})
|
})
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"name": "pvc",
|
"name": "pvc",
|
||||||
"title": "PVC 存储检查",
|
"title": "PVC 存储检查",
|
||||||
@@ -335,12 +419,23 @@ async def check_pvc(namespace: str = "") -> dict:
|
|||||||
|
|
||||||
async def check_resource_usage() -> dict:
|
async def check_resource_usage() -> dict:
|
||||||
"""节点资源使用率 (需要 metrics-server)"""
|
"""节点资源使用率 (需要 metrics-server)"""
|
||||||
raw = await _run_kubectl_raw(["top", "nodes", "--no-headers"])
|
stdout, stderr, code = await _run_kubectl_raw(["top", "nodes", "--no-headers"])
|
||||||
issues = []
|
issues = []
|
||||||
nodes_usage = []
|
nodes_usage = []
|
||||||
|
|
||||||
if raw and "error" not in raw.lower():
|
if code != 0:
|
||||||
for line in raw.strip().split("\n"):
|
# metrics-server 不可用不算 critical,只是跳过
|
||||||
|
return {
|
||||||
|
"name": "resource_usage",
|
||||||
|
"title": "资源使用率",
|
||||||
|
"issues": [],
|
||||||
|
"nodes": [],
|
||||||
|
"note": f"metrics-server 不可用: {(stderr or stdout)[:200]}",
|
||||||
|
"raw": {"error": stderr or stdout},
|
||||||
|
}
|
||||||
|
|
||||||
|
if stdout:
|
||||||
|
for line in stdout.strip().split("\n"):
|
||||||
parts = line.split()
|
parts = line.split()
|
||||||
if len(parts) >= 5:
|
if len(parts) >= 5:
|
||||||
name = parts[0]
|
name = parts[0]
|
||||||
@@ -359,22 +454,13 @@ async def check_resource_usage() -> dict:
|
|||||||
issues.append({"level": "warning", "resource": name, "message": f"内存使用率 {mem_v}%"})
|
issues.append({"level": "warning", "resource": name, "message": f"内存使用率 {mem_v}%"})
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
else:
|
|
||||||
return {
|
|
||||||
"name": "resource_usage",
|
|
||||||
"title": "资源使用率",
|
|
||||||
"issues": [],
|
|
||||||
"nodes": [],
|
|
||||||
"note": "metrics-server 不可用,跳过资源使用率检查",
|
|
||||||
"raw": {"error": raw},
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"name": "resource_usage",
|
"name": "resource_usage",
|
||||||
"title": "资源使用率",
|
"title": "资源使用率",
|
||||||
"issues": issues,
|
"issues": issues,
|
||||||
"nodes": nodes_usage,
|
"nodes": nodes_usage,
|
||||||
"raw": {"text": raw},
|
"raw": {"text": stdout},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -386,12 +472,18 @@ async def check_network_policies(namespace: str = "") -> dict:
|
|||||||
else:
|
else:
|
||||||
args += ["--all-namespaces"]
|
args += ["--all-namespaces"]
|
||||||
result = await _run_kubectl(args)
|
result = await _run_kubectl(args)
|
||||||
|
|
||||||
|
issues = []
|
||||||
|
if result.get("error"):
|
||||||
|
issues.append({"level": "warning", "resource": "network", "message": f"无法获取网络策略: {result['error'][:200]}"})
|
||||||
|
return {"name": "network", "title": "网络策略检查", "policy_count": 0, "issues": issues, "raw": result}
|
||||||
|
|
||||||
count = len(result["data"].get("items", [])) if result["data"] else 0
|
count = len(result["data"].get("items", [])) if result["data"] else 0
|
||||||
return {
|
return {
|
||||||
"name": "network",
|
"name": "network",
|
||||||
"title": "网络策略检查",
|
"title": "网络策略检查",
|
||||||
"policy_count": count,
|
"policy_count": count,
|
||||||
"issues": [],
|
"issues": issues,
|
||||||
"raw": result,
|
"raw": result,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -414,7 +506,31 @@ ALL_CHECKS = {
|
|||||||
async def run_full_diagnosis(namespace: str = "", checks: Optional[list[str]] = None) -> dict:
|
async def run_full_diagnosis(namespace: str = "", checks: Optional[list[str]] = None) -> dict:
|
||||||
"""执行完整诊断"""
|
"""执行完整诊断"""
|
||||||
start = datetime.now()
|
start = datetime.now()
|
||||||
|
|
||||||
|
# ★ 第一步: 连通性检查
|
||||||
|
conn = await check_connectivity()
|
||||||
|
if not conn["connected"]:
|
||||||
|
return {
|
||||||
|
"timestamp": start.isoformat(),
|
||||||
|
"elapsed_seconds": round((datetime.now() - start).total_seconds(), 2),
|
||||||
|
"namespace": namespace or "all",
|
||||||
|
"score": 0,
|
||||||
|
"status": "critical",
|
||||||
|
"connected": False,
|
||||||
|
"connectivity_error": conn["error"],
|
||||||
|
"connectivity_suggestion": conn.get("suggestion", ""),
|
||||||
|
"critical_count": 1,
|
||||||
|
"warning_count": 0,
|
||||||
|
"issues": [{
|
||||||
|
"level": "critical",
|
||||||
|
"check": "集群连通性",
|
||||||
|
"resource": "API Server",
|
||||||
|
"message": f"无法连接到集群: {conn['error'][:300]}",
|
||||||
|
}],
|
||||||
|
"checks": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
# ★ 第二步: 执行各项检查
|
||||||
selected = checks or list(ALL_CHECKS.keys())
|
selected = checks or list(ALL_CHECKS.keys())
|
||||||
tasks = {}
|
tasks = {}
|
||||||
for name in selected:
|
for name in selected:
|
||||||
@@ -424,45 +540,49 @@ async def run_full_diagnosis(namespace: str = "", checks: Optional[list[str]] =
|
|||||||
tasks[name] = fn(namespace)
|
tasks[name] = fn(namespace)
|
||||||
else:
|
else:
|
||||||
tasks[name] = fn()
|
tasks[name] = fn()
|
||||||
|
|
||||||
results = {}
|
results = {}
|
||||||
for name, coro in tasks.items():
|
for name, coro in tasks.items():
|
||||||
try:
|
try:
|
||||||
results[name] = await coro
|
results[name] = await coro
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
results[name] = {"name": name, "title": name, "error": str(e), "issues": []}
|
results[name] = {
|
||||||
|
"name": name, "title": name, "error": str(e),
|
||||||
|
"issues": [{"level": "critical", "resource": name, "message": f"检查执行异常: {str(e)}"}],
|
||||||
|
}
|
||||||
|
|
||||||
# 汇总
|
# 汇总
|
||||||
all_issues = []
|
all_issues = []
|
||||||
for r in results.values():
|
for r in results.values():
|
||||||
for issue in r.get("issues", []):
|
for issue in r.get("issues", []):
|
||||||
issue["check"] = r.get("title", r.get("name", ""))
|
issue["check"] = r.get("title", r.get("name", ""))
|
||||||
all_issues.append(issue)
|
all_issues.append(issue)
|
||||||
|
|
||||||
critical = [i for i in all_issues if i["level"] == "critical"]
|
critical = [i for i in all_issues if i["level"] == "critical"]
|
||||||
warning = [i for i in all_issues if i["level"] == "warning"]
|
warning = [i for i in all_issues if i["level"] == "warning"]
|
||||||
|
|
||||||
elapsed = (datetime.now() - start).total_seconds()
|
elapsed = (datetime.now() - start).total_seconds()
|
||||||
|
|
||||||
# 健康评分 (100 分制)
|
# 健康评分 (100 分制)
|
||||||
score = 100
|
score = 100
|
||||||
score -= len(critical) * 15
|
score -= len(critical) * 15
|
||||||
score -= len(warning) * 5
|
score -= len(warning) * 5
|
||||||
score = max(0, score)
|
score = max(0, score)
|
||||||
|
|
||||||
if score >= 90:
|
if score >= 90:
|
||||||
status = "healthy"
|
status = "healthy"
|
||||||
elif score >= 60:
|
elif score >= 60:
|
||||||
status = "warning"
|
status = "warning"
|
||||||
else:
|
else:
|
||||||
status = "critical"
|
status = "critical"
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"timestamp": start.isoformat(),
|
"timestamp": start.isoformat(),
|
||||||
"elapsed_seconds": round(elapsed, 2),
|
"elapsed_seconds": round(elapsed, 2),
|
||||||
"namespace": namespace or "all",
|
"namespace": namespace or "all",
|
||||||
"score": score,
|
"score": score,
|
||||||
"status": status,
|
"status": status,
|
||||||
|
"connected": True,
|
||||||
"critical_count": len(critical),
|
"critical_count": len(critical),
|
||||||
"warning_count": len(warning),
|
"warning_count": len(warning),
|
||||||
"issues": all_issues,
|
"issues": all_issues,
|
||||||
@@ -479,45 +599,53 @@ def build_diagnosis_report(diagnosis: dict) -> str:
|
|||||||
lines.append(f"健康评分: {diagnosis['score']}/100 ({diagnosis['status']})")
|
lines.append(f"健康评分: {diagnosis['score']}/100 ({diagnosis['status']})")
|
||||||
lines.append(f"严重问题: {diagnosis['critical_count']} 个, 警告: {diagnosis['warning_count']} 个")
|
lines.append(f"严重问题: {diagnosis['critical_count']} 个, 警告: {diagnosis['warning_count']} 个")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
|
# 连通性失败
|
||||||
|
if not diagnosis.get("connected", True):
|
||||||
|
lines.append("## ⚠️ 集群连接失败")
|
||||||
|
lines.append(f"错误: {diagnosis.get('connectivity_error', '未知')}")
|
||||||
|
lines.append(f"建议: {diagnosis.get('connectivity_suggestion', '')}")
|
||||||
|
lines.append("")
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
if diagnosis["issues"]:
|
if diagnosis["issues"]:
|
||||||
lines.append("## 发现的问题")
|
lines.append("## 发现的问题")
|
||||||
for i, issue in enumerate(diagnosis["issues"], 1):
|
for i, issue in enumerate(diagnosis["issues"], 1):
|
||||||
icon = "🔴" if issue["level"] == "critical" else "🟡"
|
icon = "🔴" if issue["level"] == "critical" else "🟡"
|
||||||
lines.append(f"{i}. {icon} [{issue['check']}] {issue['resource']}: {issue['message']}")
|
lines.append(f"{i}. {icon} [{issue['check']}] {issue['resource']}: {issue['message']}")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
# Pod 摘要
|
# Pod 摘要
|
||||||
pods = diagnosis["checks"].get("pods", {})
|
pods = diagnosis["checks"].get("pods", {})
|
||||||
if pods.get("summary"):
|
if pods.get("summary"):
|
||||||
s = pods["summary"]
|
s = pods["summary"]
|
||||||
lines.append(f"## Pod 状态: 总计 {s['total']}, Running {s['running']}, Pending {s['pending']}, Failed {s['failed']}, CrashLoop {s.get('crashloop', 0)}")
|
lines.append(f"## Pod 状态: 总计 {s['total']}, Running {s['running']}, Pending {s['pending']}, Failed {s['failed']}, CrashLoop {s.get('crashloop', 0)}")
|
||||||
|
|
||||||
# Node 摘要
|
# Node 摘要
|
||||||
nodes = diagnosis["checks"].get("nodes", {})
|
nodes = diagnosis["checks"].get("nodes", {})
|
||||||
if nodes.get("summary"):
|
if nodes.get("summary"):
|
||||||
s = nodes["summary"]
|
s = nodes["summary"]
|
||||||
lines.append(f"## Node 状态: 总计 {s['total']}, Ready {s['ready']}, NotReady {s['not_ready']}")
|
lines.append(f"## Node 状态: 总计 {s['total']}, Ready {s['ready']}, NotReady {s['not_ready']}")
|
||||||
|
|
||||||
# Deployment 摘要
|
# Deployment 摘要
|
||||||
deps = diagnosis["checks"].get("deployments", {})
|
deps = diagnosis["checks"].get("deployments", {})
|
||||||
if deps.get("deployments"):
|
if deps.get("deployments"):
|
||||||
lines.append("## Deployment 状态:")
|
lines.append("## Deployment 状态:")
|
||||||
for d in deps["deployments"]:
|
for d in deps["deployments"]:
|
||||||
lines.append(f" - {d['namespace']}/{d['name']}: {d['ready']}/{d['desired']} ready")
|
lines.append(f" - {d['namespace']}/{d['name']}: {d['ready']}/{d['desired']} ready")
|
||||||
|
|
||||||
# 异常事件
|
# 异常事件
|
||||||
events = diagnosis["checks"].get("events", {})
|
events = diagnosis["checks"].get("events", {})
|
||||||
if events.get("events"):
|
if events.get("events"):
|
||||||
lines.append(f"\n## 最近异常事件 (共 {len(events['events'])} 条):")
|
lines.append(f"\n## 最近异常事件 (共 {len(events['events'])} 条):")
|
||||||
for ev in events["events"][-20:]:
|
for ev in events["events"][-20:]:
|
||||||
lines.append(f" - [{ev['type']}] {ev['object']}: {ev['reason']} - {ev['message'][:150]}")
|
lines.append(f" - [{ev['type']}] {ev['object']}: {ev['reason']} - {ev['message'][:150]}")
|
||||||
|
|
||||||
# 资源使用
|
# 资源使用
|
||||||
usage = diagnosis["checks"].get("resource_usage", {})
|
usage = diagnosis["checks"].get("resource_usage", {})
|
||||||
if usage.get("nodes"):
|
if usage.get("nodes"):
|
||||||
lines.append("\n## 节点资源使用:")
|
lines.append("\n## 节点资源使用:")
|
||||||
for n in usage["nodes"]:
|
for n in usage["nodes"]:
|
||||||
lines.append(f" - {n['name']}: CPU {n['cpu_pct']}%, 内存 {n['mem_pct']}%")
|
lines.append(f" - {n['name']}: CPU {n['cpu_pct']}%, 内存 {n['mem_pct']}%")
|
||||||
|
|
||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|||||||
+44
-6
@@ -13,6 +13,7 @@ from typing import Optional
|
|||||||
from backend.config import PORT
|
from backend.config import PORT
|
||||||
from backend.diagnosis import (
|
from backend.diagnosis import (
|
||||||
run_full_diagnosis, build_diagnosis_report, ALL_CHECKS,
|
run_full_diagnosis, build_diagnosis_report, ALL_CHECKS,
|
||||||
|
check_connectivity,
|
||||||
check_cluster_info, check_pods, check_nodes, check_events,
|
check_cluster_info, check_pods, check_nodes, check_events,
|
||||||
check_deployments, check_services, check_pvc, check_resource_usage,
|
check_deployments, check_services, check_pvc, check_resource_usage,
|
||||||
check_network_policies,
|
check_network_policies,
|
||||||
@@ -82,12 +83,46 @@ async def diagnose_stream(namespace: str = ""):
|
|||||||
|
|
||||||
async def event_generator():
|
async def event_generator():
|
||||||
start = datetime.now()
|
start = datetime.now()
|
||||||
|
|
||||||
|
# ★ 第一步: 连通性检查
|
||||||
|
yield f"data: {json.dumps({'type': 'progress', 'step': 0, 'total': len(CHECK_ORDER) + 1, 'name': 'connectivity', 'title': '集群连通性检查', 'status': 'running'}, ensure_ascii=False)}\n\n"
|
||||||
|
|
||||||
|
conn = await check_connectivity()
|
||||||
|
if not conn["connected"]:
|
||||||
|
yield f"data: {json.dumps({'type': 'progress', 'step': 0, 'total': len(CHECK_ORDER) + 1, 'name': 'connectivity', 'title': '集群连通性检查', 'status': 'done', 'issues': 1}, ensure_ascii=False)}\n\n"
|
||||||
|
|
||||||
|
diagnosis = {
|
||||||
|
"timestamp": start.isoformat(),
|
||||||
|
"elapsed_seconds": round((datetime.now() - start).total_seconds(), 2),
|
||||||
|
"namespace": namespace or "all",
|
||||||
|
"score": 0,
|
||||||
|
"status": "critical",
|
||||||
|
"connected": False,
|
||||||
|
"connectivity_error": conn["error"],
|
||||||
|
"connectivity_suggestion": conn.get("suggestion", ""),
|
||||||
|
"critical_count": 1,
|
||||||
|
"warning_count": 0,
|
||||||
|
"issues": [{
|
||||||
|
"level": "critical",
|
||||||
|
"check": "集群连通性",
|
||||||
|
"resource": "API Server",
|
||||||
|
"message": f"无法连接到集群: {conn['error'][:300]}",
|
||||||
|
}],
|
||||||
|
"checks": {},
|
||||||
|
}
|
||||||
|
_last_diagnosis = diagnosis
|
||||||
|
_last_report = build_diagnosis_report(diagnosis)
|
||||||
|
yield f"data: {json.dumps({'type': 'complete', 'diagnosis': diagnosis}, ensure_ascii=False)}\n\n"
|
||||||
|
return
|
||||||
|
|
||||||
|
yield f"data: {json.dumps({'type': 'progress', 'step': 0, 'total': len(CHECK_ORDER) + 1, 'name': 'connectivity', 'title': '集群连通性检查', 'status': 'done', 'issues': 0}, ensure_ascii=False)}\n\n"
|
||||||
|
|
||||||
|
# ★ 第二步: 执行各项检查
|
||||||
results = {}
|
results = {}
|
||||||
all_issues = []
|
all_issues = []
|
||||||
total = len(CHECK_ORDER)
|
total = len(CHECK_ORDER) + 1
|
||||||
|
|
||||||
for idx, (name, title, fn) in enumerate(CHECK_ORDER, 1):
|
for idx, (name, title, fn) in enumerate(CHECK_ORDER, 1):
|
||||||
# 推送: 开始检查
|
|
||||||
yield f"data: {json.dumps({'type': 'progress', 'step': idx, 'total': total, 'name': name, 'title': title, 'status': 'running'}, ensure_ascii=False)}\n\n"
|
yield f"data: {json.dumps({'type': 'progress', 'step': idx, 'total': total, 'name': name, 'title': title, 'status': 'running'}, ensure_ascii=False)}\n\n"
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -101,10 +136,13 @@ async def diagnose_stream(namespace: str = ""):
|
|||||||
issue["check"] = title
|
issue["check"] = title
|
||||||
all_issues.append(issue)
|
all_issues.append(issue)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
results[name] = {"name": name, "title": title, "error": str(e), "issues": []}
|
results[name] = {
|
||||||
issue_count = -1
|
"name": name, "title": title, "error": str(e),
|
||||||
|
"issues": [{"level": "critical", "resource": name, "message": f"检查执行异常: {str(e)}"}],
|
||||||
|
}
|
||||||
|
issue_count = 1
|
||||||
|
all_issues.append({"level": "critical", "check": title, "resource": name, "message": f"检查执行异常: {str(e)}"})
|
||||||
|
|
||||||
# 推送: 完成检查
|
|
||||||
yield f"data: {json.dumps({'type': 'progress', 'step': idx, 'total': total, 'name': name, 'title': title, 'status': 'done', 'issues': issue_count}, ensure_ascii=False)}\n\n"
|
yield f"data: {json.dumps({'type': 'progress', 'step': idx, 'total': total, 'name': name, 'title': title, 'status': 'done', 'issues': issue_count}, ensure_ascii=False)}\n\n"
|
||||||
|
|
||||||
# 汇总
|
# 汇总
|
||||||
@@ -120,6 +158,7 @@ async def diagnose_stream(namespace: str = ""):
|
|||||||
"namespace": namespace or "all",
|
"namespace": namespace or "all",
|
||||||
"score": score,
|
"score": score,
|
||||||
"status": status,
|
"status": status,
|
||||||
|
"connected": True,
|
||||||
"critical_count": len(critical),
|
"critical_count": len(critical),
|
||||||
"warning_count": len(warning),
|
"warning_count": len(warning),
|
||||||
"issues": all_issues,
|
"issues": all_issues,
|
||||||
@@ -129,7 +168,6 @@ async def diagnose_stream(namespace: str = ""):
|
|||||||
_last_diagnosis = diagnosis
|
_last_diagnosis = diagnosis
|
||||||
_last_report = build_diagnosis_report(diagnosis)
|
_last_report = build_diagnosis_report(diagnosis)
|
||||||
|
|
||||||
# 推送: 最终结果
|
|
||||||
yield f"data: {json.dumps({'type': 'complete', 'diagnosis': diagnosis}, ensure_ascii=False)}\n\n"
|
yield f"data: {json.dumps({'type': 'complete', 'diagnosis': diagnosis}, ensure_ascii=False)}\n\n"
|
||||||
|
|
||||||
return StreamingResponse(
|
return StreamingResponse(
|
||||||
|
|||||||
+34
-2
@@ -58,8 +58,31 @@
|
|||||||
</el-card>
|
</el-card>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- 集群连接失败 -->
|
||||||
|
<div v-if="diagnosis && !diagnosing && diagnosis.connected === false" class="conn-error-box">
|
||||||
|
<el-card shadow="never" class="conn-error-card">
|
||||||
|
<div class="conn-error-icon">
|
||||||
|
<el-icon :size="64" color="#f56c6c"><CircleCloseFilled /></el-icon>
|
||||||
|
</div>
|
||||||
|
<h2>无法连接到 Kubernetes 集群</h2>
|
||||||
|
<div class="conn-error-detail">
|
||||||
|
<el-alert type="error" :closable="false" show-icon>
|
||||||
|
<template #title>错误信息</template>
|
||||||
|
<pre class="conn-error-pre">{{ diagnosis.connectivity_error }}</pre>
|
||||||
|
</el-alert>
|
||||||
|
<el-alert v-if="diagnosis.connectivity_suggestion" type="warning" :closable="false" show-icon style="margin-top: 12px">
|
||||||
|
<template #title>排查建议</template>
|
||||||
|
<div>{{ diagnosis.connectivity_suggestion }}</div>
|
||||||
|
</el-alert>
|
||||||
|
</div>
|
||||||
|
<div class="conn-error-actions">
|
||||||
|
<el-button type="primary" @click="runDiagnosis" :icon="Refresh">重新诊断</el-button>
|
||||||
|
</div>
|
||||||
|
</el-card>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- 诊断结果 -->
|
<!-- 诊断结果 -->
|
||||||
<template v-if="diagnosis && !diagnosing">
|
<template v-if="diagnosis && !diagnosing && diagnosis.connected !== false">
|
||||||
<!-- 概览卡片 -->
|
<!-- 概览卡片 -->
|
||||||
<el-row :gutter="16" class="summary-row">
|
<el-row :gutter="16" class="summary-row">
|
||||||
<el-col :span="4">
|
<el-col :span="4">
|
||||||
@@ -219,7 +242,7 @@
|
|||||||
<script setup>
|
<script setup>
|
||||||
import { ref, computed, nextTick } from 'vue'
|
import { ref, computed, nextTick } from 'vue'
|
||||||
import { marked } from 'marked'
|
import { marked } from 'marked'
|
||||||
import { Refresh, MagicStick, Loading, WarningFilled, Warning, Promotion, Monitor, CircleCheckFilled, CircleClose } from '@element-plus/icons-vue'
|
import { Refresh, MagicStick, Loading, WarningFilled, Warning, Promotion, Monitor, CircleCheckFilled, CircleClose, CircleCloseFilled } from '@element-plus/icons-vue'
|
||||||
import { ElMessage } from 'element-plus'
|
import { ElMessage } from 'element-plus'
|
||||||
|
|
||||||
const API = '/api'
|
const API = '/api'
|
||||||
@@ -405,6 +428,15 @@ body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'PingFang SC'
|
|||||||
/* 诊断进度 */
|
/* 诊断进度 */
|
||||||
.progress-box { display: flex; justify-content: center; padding-top: 40px; }
|
.progress-box { display: flex; justify-content: center; padding-top: 40px; }
|
||||||
.progress-card { width: 600px; }
|
.progress-card { width: 600px; }
|
||||||
|
|
||||||
|
/* 连接失败 */
|
||||||
|
.conn-error-box { display: flex; justify-content: center; padding-top: 40px; }
|
||||||
|
.conn-error-card { width: 700px; text-align: center; padding: 20px 0; }
|
||||||
|
.conn-error-icon { margin-bottom: 16px; }
|
||||||
|
.conn-error-card h2 { font-size: 22px; color: #303133; margin-bottom: 20px; }
|
||||||
|
.conn-error-detail { text-align: left; margin: 0 20px; }
|
||||||
|
.conn-error-pre { white-space: pre-wrap; word-break: break-all; font-size: 13px; margin: 8px 0 0; color: #f56c6c; }
|
||||||
|
.conn-error-actions { margin-top: 24px; }
|
||||||
.progress-steps { display: flex; flex-direction: column; gap: 4px; }
|
.progress-steps { display: flex; flex-direction: column; gap: 4px; }
|
||||||
.progress-item {
|
.progress-item {
|
||||||
display: flex; align-items: center; gap: 10px;
|
display: flex; align-items: center; gap: 10px;
|
||||||
|
|||||||
Reference in New Issue
Block a user