更新 README.md

完整 shell 能力支持：
- VAR=value command 语法（自动跳过赋值检测实际 binary）
- export VAR=value 支持（export 加入只读白名单）
- 多行命令（\n 分割后逐段检查）
- && || ; 等连接符的武器级命令检测
- _split_into_commands 重写：使用 while 索引遍历，正确处理 && / || / |
- _command_has_dangerous_cmds 跳过 VAR=value 和 export 前缀
- classify_agent_command 检测 VAR=value kubectl 语法路由到 kubectl 分类器
- execute_command 支持变量赋值后的 binary 定位
- 新增 shell 内置白名单：echo printf cd pwd export source . sleep timeout
  nohup unset shift return local readonly trap type command set shopt eval
  exec alias unalias declare typeset read mapfile
- 白名单只读命令新增：timeout、nohup、sleep 等

README.md

@@ -2,7 +2,7 @@
 
 一键诊断 Kubernetes 集群健康状态，接入 AI Agent 自动分析故障根因并给出修复建议。
 
-![界面预览](/home/cnbugs/.hermes/cache/screenshots/browser_screenshot_b258ca152c1e489fa186583946ecbd9e.png)
+![image-20260725221146269.png](https://wx-img.cnbugs.com/image-20260725221146269.png)
 
 ## 功能特性
 

backend/agent_tools.py

@@ -73,27 +73,52 @@ def _has_shell_operators(command: str) -> bool:
 
 
 def _split_into_commands(command: str) -> list[str]:
-    """按 shell 连接符（; && || | > >> < << 等）分割命令块。
+    """按 shell 连接符（; && || | \n 等）分割命令块。
 
-    返回每个可执行段，忽略引号内的操作符。用于白名单检查。
+    返回每个可执行段，忽略引号内的操作符。重定向符号 > < 不分割。
     """
     segments = []
     buf = []
     in_single = in_double = False
-    for ch in command:
+    i = 0
+    while i < len(command):
+        ch = command[i]
         if ch == "'" and not in_double:
             in_single = not in_single
         elif ch == '"' and not in_single:
             in_double = not in_double
-        if not in_single and not in_double and ch in ('|', ';', '&', '>', '<'):
+        if not in_single and not in_double:
-            # 包含重定向符号时，前面的部分也是可执行段
+            if ch in (';', '\n'):
-            if buf:
+                if buf:
-                segments.append(''.join(buf).strip())
+                    segments.append(''.join(buf).strip())
-                buf = []
+                    buf = []
+                i += 1
+                continue
+            if ch == '|':
+                if i + 1 < len(command) and command[i + 1] == '|':
+                    # || — 分割命令
+                    if buf:
+                        segments.append(''.join(buf).strip())
+                        buf = []
+                    i += 2
+                    continue
+                # 单 | — 分割命令
+                if buf:
+                    segments.append(''.join(buf).strip())
+                    buf = []
+                i += 1
+                continue
             if ch == '&':
-                continue  # & 符号自身不产生段
+                if i + 1 < len(command) and command[i + 1] == '&':
-            continue
+                    # && — 分割命令
+                    if buf:
+                        segments.append(''.join(buf).strip())
+                        buf = []
+                    i += 2
+                    continue
+                # 单独 & — 保留
         buf.append(ch)
+        i += 1
     if buf:
         segments.append(''.join(buf).strip())
     return [s for s in segments if s]
@@ -106,8 +131,21 @@ def _command_has_dangerous_cmds(command: str) -> bool:
             seg_args = shlex.split(segment)
         except ValueError:
             continue
-        if seg_args and seg_args[0] in _ALWAYS_DENY_CMDS:
+        if not seg_args:
+            continue
+        # 跳过 VAR=value 赋值
+        idx = 0
+        while idx < len(seg_args) and "=" in seg_args[idx] and not seg_args[idx].startswith("-"):
+            idx += 1
+        if idx < len(seg_args) and seg_args[idx] in _ALWAYS_DENY_CMDS:
             return True
+        # 也检查 export VAR=value 后的命令
+        if idx < len(seg_args) and seg_args[idx] == "export":
+            idx2 = idx + 1
+            while idx2 < len(seg_args) and "=" in seg_args[idx2]:
+                idx2 += 1
+            if idx2 < len(seg_args) and seg_args[idx2] in _ALWAYS_DENY_CMDS:
+                return True
     return False
 
 
@@ -209,8 +247,11 @@ SHELL_READ_ONLY_COMMANDS: dict[str, set[str] | None] = {
     "nc": None, "nmap": None, "telnet": None, "tcpdump": None, "ethtool": None,
     "lspci": None, "lsusb": None, "dmidecode": None, "sysctl": None, "udevadm": None,
     "echo": None, "printf": None, "cd": None, "pwd": None, "export": None,
-    "test": None, "[": None, "sleep": None,
+    "test": None, "[": None, "sleep": None, "source": None, ".": None,
-    "timeout": None, "nohup": None,
+    "timeout": None, "nohup": None, "unset": None, "shift": None, "return": None,
+    "local": None, "readonly": None, "trap": None, "type": None, "command": None,
+    "set": None, "shopt": None, "eval": None, "exec": None, "alias": None,
+    "unalias": None, "declare": None, "typeset": None, "read": None, "mapfile": None,
 }
 
 # etcdctl 由 classify_shell_command 中的专用块处理 (两段式命令)
@@ -288,48 +329,76 @@ def classify_shell_command(command: str) -> CommandDecision:
         if _command_has_dangerous_cmds(command):
             raise ValueError("命令包含不允许的危险命令（sh/bash/dd/mkfs 等），请手动执行")
 
-        # 多行命令：按行分割后逐行检查，取最高权限模式
+        # 多行或含操作符的命令：按行分割后，对每行逐段检查，取最高权限模式
         lines = command.split("\n") if "\n" in command else [command]
         overall_mode = "read"
         overall_approval = False
         first_binary = None
+        first_segment = ""
 
         for line in lines:
             line = line.strip()
             if not line or line.startswith("#"):
                 continue
-            try:
+            segments = _split_into_commands(line)
-                line_parts = shlex.split(line)
+            for segment in segments:
-            except ValueError:
+                # 检测 kubectl 命令
-                continue
+                if segment.startswith("kubectl ") or segment == "kubectl":
-            binary = line_parts[0]
+                    try:
-            if first_binary is None:
+                        kdec = classify_kubectl_command(segment)
-                first_binary = binary
+                    except (ValueError, IndexError) as exc:
-
+                        raise ValueError(str(exc))
-            if binary not in SHELL_READ_ONLY_COMMANDS and binary not in SHELL_WRITE_COMMANDS:
+                    if first_binary is None:
-                raise ValueError(f"不允许执行该命令: {binary}")
+                        first_binary = "kubectl"
-
+                        first_segment = segment
-            if binary == "cat" and len(line_parts) > 1:
+                    if kdec.mode == "write":
-                pass  # cat 不限制路径
-
-            if binary in SHELL_WRITE_COMMANDS:
-                allowed = SHELL_WRITE_COMMANDS[binary]
-                if allowed is None:
-                    overall_mode = "write"
-                    overall_approval = True
-                else:
-                    verb = _first_non_flag(line_parts[1:])
-                    if verb is not None and verb in allowed:
                         overall_mode = "write"
                         overall_approval = True
-                    elif binary in SHELL_READ_ONLY_COMMANDS:
+                    continue
-                        ro_allowed = SHELL_READ_ONLY_COMMANDS[binary]
+
-                        if ro_allowed is None or (verb is not None and verb in ro_allowed):
+                try:
-                            pass  # 该行是只读，不改变 overall
+                    seg_parts = shlex.split(segment)
+                except ValueError:
+                    continue
+                if not seg_parts:
+                    continue
+                # 跳过 VAR=value 赋值前缀（含 export）
+                idx = 0
+                while idx < len(seg_parts):
+                    if "=" in seg_parts[idx] and not seg_parts[idx].startswith("-"):
+                        idx += 1
+                    elif seg_parts[idx] == "export":
+                        idx += 1
+                    else:
+                        break
+                if idx >= len(seg_parts):
+                    continue  # 纯赋值段，跳过
+                binary = seg_parts[idx]
+                check_parts = seg_parts[idx:]
+                if first_binary is None:
+                    first_binary = binary
+
+                if binary not in SHELL_READ_ONLY_COMMANDS and binary not in SHELL_WRITE_COMMANDS:
+                    raise ValueError(f"不允许执行该命令: {binary}")
+
+                if binary in SHELL_WRITE_COMMANDS:
+                    allowed = SHELL_WRITE_COMMANDS[binary]
+                    if allowed is None:
+                        overall_mode = "write"
+                        overall_approval = True
+                    else:
+                        verb = _first_non_flag(check_parts[1:])
+                        if verb is not None and verb in allowed:
+                            overall_mode = "write"
+                            overall_approval = True
+                        elif binary in SHELL_READ_ONLY_COMMANDS:
+                            ro_allowed = SHELL_READ_ONLY_COMMANDS[binary]
+                            if ro_allowed is None or (verb is not None and verb in ro_allowed):
+                                pass  # 该段是只读，不改变 overall
+                            else:
+                                raise ValueError(f"命令 {binary} 不支持该子命令: {verb}")
                         else:
                             raise ValueError(f"命令 {binary} 不支持该子命令: {verb}")
-                    else:
-                        raise ValueError(f"命令 {binary} 不支持该子命令: {verb}")
 
         if first_binary is None:
             raise ValueError("命令不能为空")
@@ -338,30 +407,44 @@ def classify_shell_command(command: str) -> CommandDecision:
 
     parts = parse_shell_command(command)
     binary = parts[0]
-    verb = _first_non_flag(parts[1:])
+    # 支持 VAR=value command 语法
+    start_idx = 0
+    while start_idx < len(parts) and "=" in parts[start_idx] and not parts[start_idx].startswith("-"):
+        start_idx += 1
+    if start_idx > 0:
+        if start_idx < len(parts):
+            binary = parts[start_idx]
+            effective_parts = parts[start_idx:]
+        else:
+            # 纯赋值行如 LOG_DIR=/tmp/log — 放行
+            return CommandDecision("read", False, parts[0], parts[1:])
+        verb = _first_non_flag(effective_parts[1:])
+    else:
+        effective_parts = parts
+        verb = _first_non_flag(parts[1:])
 
     if binary in SHELL_WRITE_COMMANDS:
         allowed = SHELL_WRITE_COMMANDS[binary]
         if allowed is None or (verb is not None and verb in allowed):
-            return CommandDecision("write", True, binary, parts[1:])
+            return CommandDecision("write", True, binary, effective_parts[1:])
 
     # etcdctl: 单段式或两段式命令，verb/subverb 均跳过全局 flag (--endpoints 等)
     # member/alarm/auth 等同时有只读和写子命令，需按 (verb, subverb) 组合精确判断
     if binary == "etcdctl":
-        non_flags = [a for a in parts[1:] if not a.startswith("-")]
+        non_flags = [a for a in effective_parts[1:] if not a.startswith("-")]
         ev = non_flags[0].lower() if non_flags else None
         esub = non_flags[1].lower() if len(non_flags) > 1 else None
         # 单段式 verb
         if ev in _ETCDCTL_READ_VERBS:
-            return CommandDecision("read", False, ev, parts[1:])
+            return CommandDecision("read", False, ev, effective_parts[1:])
         if ev in _ETCDCTL_WRITE_VERBS:
-            return CommandDecision("write", True, ev, parts[1:])
+            return CommandDecision("write", True, ev, effective_parts[1:])
         # 两段式: 先查写子命令 (精确匹配 subverb)，再查只读子命令
         if ev in _ETCDCTL_WRITE_SUBVERBS and esub in _ETCDCTL_WRITE_SUBVERBS[ev]:
-            return CommandDecision("write", True, ev, parts[1:])
+            return CommandDecision("write", True, ev, effective_parts[1:])
         if ev in _ETCDCTL_READ_SUBVERBS:
             if esub is None or esub in _ETCDCTL_READ_SUBVERBS[ev]:
-                return CommandDecision("read", False, ev, parts[1:])
+                return CommandDecision("read", False, ev, effective_parts[1:])
             raise ValueError(f"etcdctl {ev} 不支持子命令: {esub}")
         raise ValueError(f"etcdctl 不支持该子命令: {ev}")
 
@@ -369,9 +452,9 @@ def classify_shell_command(command: str) -> CommandDecision:
         allowed = SHELL_READ_ONLY_COMMANDS[binary]
         if allowed is not None and (verb is None or verb not in allowed):
             raise ValueError(f"命令 {binary} 不支持该子命令: {verb}")
-        if binary == "ip" and any(arg in _IP_WRITE_ACTIONS for arg in parts[1:]):
+        if binary == "ip" and any(arg in _IP_WRITE_ACTIONS for arg in effective_parts[1:]):
             raise ValueError("ip 命令包含写操作，请使用只读形式如 'ip addr show'")
-        return CommandDecision("read", False, binary, parts[1:])
+        return CommandDecision("read", False, binary, effective_parts[1:])
 
     raise ValueError(f"不允许执行该命令: {binary}")
 
@@ -379,6 +462,29 @@ def classify_shell_command(command: str) -> CommandDecision:
 def classify_agent_command(command: str) -> CommandDecision:
     """统一分类 Agent 命令：kubectl 或白名单系统诊断命令。"""
     stripped = (command or "").strip()
+    # 检测 VAR=value kubectl 语法（有变量赋值时）
+    rest = stripped
+    while True:
+        try:
+            first = rest.split(None, 1)[0]
+        except IndexError:
+            break
+        if "=" in first and not first.startswith("-"):
+            idx = rest.index("=")
+            # 必须是合法变量赋值 VAR=value
+            parts_split = rest.split(None, 1)
+            if len(parts_split) < 2:
+                break
+            rest = parts_split[1]
+        elif first == "export":
+            parts_split = rest.split(None, 1)
+            if len(parts_split) < 2:
+                break
+            rest = parts_split[1]
+        else:
+            break
+    if rest.startswith("kubectl ") or rest == "kubectl":
+        return classify_kubectl_command(rest)
     if stripped.startswith("kubectl ") or stripped == "kubectl":
         return classify_kubectl_command(stripped)
     return classify_shell_command(stripped)
@@ -393,6 +499,12 @@ async def execute_command(command: str, timeout: int = 30) -> dict:
     """
     decision = classify_agent_command(command)
     parts = _parse_command(command)
+    # 支持 VAR=value command 语法，跳过赋值找实际 binary
+    cmd_idx = 0
+    while cmd_idx < len(parts) and "=" in parts[cmd_idx] and not parts[cmd_idx].startswith("-"):
+        cmd_idx += 1
+    if cmd_idx > 0 and cmd_idx < len(parts):
+        parts = parts[cmd_idx:]
     if parts[0] == "kubectl":
         executable = KUBECTL
         args = decision.args

故障报告/Kubernetes 集群故障分析报告.md

@@ -0,0 +1,765 @@
+# Kubernetes 集群故障分析报告
+
+**报告时间**: 2026-07-25T07:36:59
+**集群健康评分**: 0/100 (Critical)
+**问题严重程度**: 4个严重问题, 30个警告问题
+
+------
+
+## 一、故障概述
+
+本次集群故障表现为严重的控制平面组件异常，导致集群处于不可用状态。主要症状包括：
+
+- **etcd** (集群核心数据库) 完全不可用，两个节点均处于CrashLoopBackOff状态
+- **kube-apiserver** (API服务器) 三个节点全部异常，无法响应请求
+- **kube-scheduler** 和 **kube-controller-manager** 也出现不稳定状态
+- 整个集群基本功能瘫痪，无法处理任何工作负载部署请求
+
+------
+
+## 二、根本原因分析 (Root Cause Analysis)
+
+### 🔴 **问题1: etcd集群完全崩溃**
+
+**根本原因**: etcd集群的持久化存储或网络配置出现问题
+
+- 两个etcd节点 (ansible-client-01/02) 均出现503状态码错误
+- 高频次重启 (38次/30次) 表明容器启动后立即失败
+- 启动探针失败返回503，表明etcd服务无法正常初始化
+
+**可能的技术原因**:
+
+1. **存储问题**: etcd数据目录损坏或磁盘空间不足
+2. **网络分区**: etcd节点之间无法建立集群通信
+3. **证书过期**: etcd的TLS证书可能已过期
+4. **资源不足**: etcd容器内存或CPU资源不足
+
+### 🔴 **问题2: kube-apiserver无法启动**
+
+**根本原因**: kube-apiserver依赖的etcd集群不可用
+
+- 所有三个apiserver节点 (ansible-client-01/02, ubuntu2204) 都失败
+- 错误信息显示"connection reset by peer"和"connection refused"
+- 启动探针返回500错误，表明服务无法初始化
+
+**依赖链分析**:
+
+```
+etcd不可用 → apiserver无法连接etcd → apiserver启动失败 → 其他组件依赖apiserver失败
+```
+
+### 🟡 **问题3: kube-scheduler/kube-controller-manager不稳定**
+
+**根本原因**: 由于apiserver不可用，这些组件无法正常工作
+
+- 依赖apiserver获取集群状态和调度决策
+- 在等待apiserver恢复过程中进入BackOff状态
+
+### 🟡 **问题4: 节点存储异常**
+
+**根本原因**: ubuntu2204节点的Docker容器存储驱动配置错误
+
+- "invalid capacity 0 on image filesystem"错误
+- 容器运行时无法正确计算存储空间
+
+------
+
+## 三、影响评估
+
+### 🔴 **业务影响 (Critical)**
+
+1. **集群完全不可用**: 无法部署、更新或管理任何工作负载
+2. **现有应用可能受影响**: 依赖Kubernetes API的所有服务将失败
+3. **数据丢失风险**: etcd是集群状态存储，如果损坏会导致元数据丢失
+4. **恢复时间延长**: 控制平面完全崩溃需要完整恢复流程
+
+### 🟡 **运维影响 (High)**
+
+1. **无法进行任何kubectl操作**: 所有API请求失败
+2. **自动修复失效**: HPA、自愈等机制无法工作
+3. **监控告警风暴**: 持续产生大量错误事件
+
+------
+
+## 四、优先级排序与修复方案
+
+### **优先级1: 恢复etcd集群 (最紧急)**
+
+**问题**: etcd集群是Kubernetes的基础，必须首先恢复
+
+**修复步骤**:
+
+#### 步骤1: 检查etcd节点状态
+
+```bash
+# SSH到ansible-client-01节点
+ssh root@<ansible-client-01-ip>
+
+# 检查etcd容器日志
+docker logs etcd-ansible-client-01 --tail 100
+
+# 检查etcd端点状态
+ETCDCTL_API=3 etcdctl endpoint health \
+  --endpoints=https://192.168.40.11:2379,https://192.168.40.12:2379 \
+  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
+  --cert=/etc/kubernetes/pki/etcd/server.crt \
+  --key=/etc/kubernetes/pki/etcd/server.key
+```
+
+#### 步骤2: 检查存储和网络
+
+```bash
+# 检查磁盘空间
+df -h /var/lib/etcd
+
+# 检查网络连接
+telnet 192.168.40.11 2379
+telnet 192.168.40.12 2380  # etcd peer端口
+
+# 检查证书是否过期
+openssl x509 -in /etc/kubernetes/pki/etcd/server.crt -text -noout | grep "Not After"
+```
+
+#### 步骤3: 修复方案
+
+**方案A: 如果存储损坏**
+
+```bash
+# 备份现有数据目录
+mv /var/lib/etcd /var/lib/etcd.backup.$(date +%s)
+
+# 创建新的etcd数据目录
+mkdir -p /var/lib/etcd
+chown -R etcd:etcd /var/lib/etcd
+
+# 重启etcd服务
+systemctl restart etcd
+```
+
+**方案B: 如果证书过期**
+
+```bash
+# 重新生成etcd证书
+kubeadm init phase certs etcd-server
+kubeadm init phase certs etcd-peer
+kubeadm init phase certs etcd-healthcheck-client
+
+# 更新kube-apiserver的etcd客户端证书
+kubeadm init phase certs apiserver-etcd-client
+
+# 重启etcd和kube-apiserver
+systemctl restart etcd
+systemctl restart kubelet
+```
+
+**方案C: 完全重建etcd集群 (最后手段)**
+
+```bash
+# 在其中一个节点重建etcd集群
+# 1. 停止所有etcd实例
+systemctl stop etcd
+
+# 2. 备份数据
+mv /var/lib/etcd /var/lib/etcd.bak
+
+# 3. 重新初始化集群
+ETCDCTL_API=3 etcdctl member remove <member-id>
+ETCDCTL_API=3 etcdctl member add ansible-client-01 --peer-urls=https://192.168.40.11:2380
+
+# 4. 启动新的单节点etcd
+etcd --name ansible-client-01 \
+  --initial-advertise-peer-urls https://192.168.40.11:2380 \
+  --listen-peer-urls https://192.168.40.11:2380 \
+  --advertise-client-urls https://192.168.40.11:2379 \
+  --listen-client-urls https://192.168.40.11:2379,https://127.0.0.1:2379 \
+  --initial-cluster-token etcd-cluster-01 \
+  --initial-cluster ansible-client-01=https://192.168.40.11:2380 \
+  --initial-cluster-state new
+```
+
+### **优先级2: 恢复kube-apiserver**
+
+**前提**: etcd集群已恢复
+
+**修复步骤**:
+
+#### 步骤1: 检查apiserver配置
+
+```bash
+# 检查kube-apiserver Pod规格
+kubectl get pod kube-apiserver-ansible-client-01 -n kube-system -o yaml
+
+# 检查启动命令参数
+kubectl get pod kube-apiserver-ansible-client-01 -n kube-system -o jsonpath='{.spec.containers[0].command}'
+```
+
+#### 步骤2: 检查资源限制
+
+```bash
+# 查看apiserver的资源使用
+kubectl top pod kube-apiserver-ansible-client-01 -n kube-system
+
+# 检查节点资源
+kubectl describe node ansible-client-01
+```
+
+#### 步骤3: 修复方案
+
+**增加启动探针超时时间** (临时方案):
+
+```yaml
+# 编辑kube-apiserver Pod规范
+kubectl edit pod kube-apiserver-ansible-client-01 -n kube-system
+
+# 添加或修改以下字段:
+spec:
+  containers:
+  - name: kube-apiserver
+    startupProbe:
+      failureThreshold: 60      # 增加失败阈值
+      periodSeconds: 5           # 增加检查周期
+      timeoutSeconds: 15         # 增加超时时间
+```
+
+**检查并修复资源限制**:
+
+```yaml
+# 确保有足够的资源
+resources:
+  requests:
+    cpu: "500m"
+    memory: "2Gi"
+  limits:
+    cpu: "2000m"
+    memory: "4Gi"
+```
+
+### **优先级3: 恢复kube-controller-manager和kube-scheduler**
+
+**修复步骤**:
+
+```bash
+# 检查这些组件的依赖
+kubectl get event -n kube-system --field-selector reason=BackOff
+
+# 重启这些组件 (如果etcd和apiserver已恢复)
+systemctl restart kubelet
+```
+
+### **优先级4: 修复节点存储问题**
+
+**问题**: ubuntu2204节点的Docker存储驱动配置错误
+
+**修复步骤**:
+
+```bash
+# SSH到ubuntu2204节点
+ssh root@<ubuntu2204-ip>
+
+# 检查Docker存储驱动
+docker info | grep "Storage Driver"
+
+# 检查存储空间
+df -h /var/lib/docker
+
+# 修复方案1: 重启Docker服务
+systemctl restart docker
+
+# 修复方案2: 如果存储驱动配置错误
+# 编辑Docker配置文件
+vi /etc/docker/daemon.json
+# 确保包含正确的storage-driver配置，例如:
+{
+  "storage-driver": "overlay2"
+}
+
+# 重启Docker
+systemctl restart docker
+systemctl restart kubelet
+```
+
+------
+
+## 五、验证修复
+
+### 步骤1: 验证etcd健康状态
+
+```bash
+# 检查etcd集群状态
+ETCDCTL_API=3 etcdctl endpoint health \
+  --endpoints=https://192.168.40.11:2379,https://192.168.40.12:2379 \
+  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
+  --cert=/etc/kubernetes/pki/etcd/server.crt \
+  --key=/etc/kubernetes/pki/etcd/server.key
+
+# 检查etcd集群成员列表
+ETCDCTL_API=3 etcdctl member list \
+  --endpoints=https://192.168.40.11:2379 \
+  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
+  --cert=/etc/kubernetes/pki/etcd/server.crt \
+  --key=/etc/kubernetes/pki/etcd/server.key
+```
+
+### 步骤2: 验证kube-apiserver状态
+
+```bash
+# 检查apiserver Pod状态
+kubectl get pods -n kube-system | grep kube-apiserver
+
+# 检查apiserver健康状态
+kubectl get --raw='/readyz?verbose'
+
+# 检查集群状态
+kubectl cluster-info
+kubectl get nodes
+```
+
+### 步骤3: 运行完整集群诊断
+
+```bash
+# 使用kubectl检查组件状态
+kubectl get componentstatuses
+
+# 检查所有系统Pod
+kubectl get pods -n kube-system
+
+# 检查集群事件
+kubectl get events -n kube-system --sort-by='.lastTimestamp'
+```
+
+------
+
+## 六、预防措施
+
+### 1. **监控告警增强**
+
+```yaml
+# 添加以下Prometheus告警规则
+groups:
+- name: kubernetes-system
+  rules:
+  - alert: EtcdMembersDown
+    expr: etcd_server_has_leader{job="etcd"} == 0
+    for: 2m
+    labels:
+      severity: critical
+    annotations:
+      summary: "Etcd member has no leader"
+      
+  - alert: EtcdHighCommitDuration
+    expr: etcd_disk_wal_fsync_duration_seconds{job="etcd"} > 1
+    for: 10m
+    labels:
+      severity: warning
+```
+
+### 2. **定期维护任务**
+
+- **每周**: 检查etcd数据库大小，执行碎片整理
+
+  ```bash
+  etcdctl defrag --endpoints=<endpoint>
+  ```
+
+- **每月**: 检查证书过期时间
+
+  ```bash
+  kubeadm certs check-expiration
+  ```
+
+- **每季度**: 演练etcd备份恢复流程
+
+### 3. **资源规划和配置优化**
+
+- 为etcd分配专用存储和高性能磁盘
+- 设置合适的资源限制和请求
+- 启用etcd自动备份和远程存储
+
+### 4. **网络策略优化**
+
+- 确保etcd节点间网络延迟<10ms
+- 配置专用网络段给控制平面组件
+- 启用网络策略限制不必要的流量
+
+### 5. **升级和补丁管理**
+
+```bash
+# 定期升级Kubernetes版本
+kubeadm upgrade plan
+kubeadm upgrade apply
+
+# 及时应用安全补丁
+apt update && apt upgrade
+```
+
+### 6. **文档和演练**
+
+- 维护详细的集群架构文档
+- 定期进行灾难恢复演练
+- 建立标准化的操作手册
+
+------
+
+## 七、附录
+
+### 附录A: 关键命令参考
+
+```bash
+# etcd健康检查
+ETCDCTL_API=3 etcdctl endpoint health --endpoints=<endpoint> --cacert=<ca> --cert=<cert> --key=<key>
+
+# etcd备份
+ETCDCTL_API=3 etcdctl snapshot save /backup/etcd-snapshot.db
+
+# 集群证书检查
+kubeadm certs check-expiration
+
+# 节点资源检查
+kubectl describe node <node-name>
+```
+
+### 附录B: 相关文件位置
+
+- etcd数据目录: `/var/lib/etcd/`
+- Kubernetes证书: `/etc/kubernetes/pki/`
+- etcd证书: `/etc/kubernetes/pki/etcd/`
+- kubelet配置: `/var/lib/kubelet/config.yaml`
+
+### 附录C: 重要端口列表
+
+| 组件 | 端口 | 协议 | 用途 |
+|------|
+
+继续
+
+### 附录C: 重要端口列表
+
+| 组件                    | 端口  | 协议  | 用途                   |
+| ----------------------- | ----- | ----- | ---------------------- |
+| kube-apiserver          | 6443  | HTTPS | API服务器安全端口      |
+| etcd                    | 2379  | HTTPS | etcd客户端通信端口     |
+| etcd                    | 2380  | HTTPS | etcd集群节点间通信端口 |
+| kubelet                 | 10250 | HTTPS | kubelet API端口        |
+| kube-scheduler          | 10259 | HTTPS | 调度器端口             |
+| kube-controller-manager | 10257 | HTTPS | 控制器管理器端口       |
+
+------
+
+### 附录D: 本次故障事件时间线（还原）
+
+根据诊断报告中的重启次数和事件记录，推断出以下故障演进时间线：
+
+```
+时间线还原 (推测)
+
+阶段1: etcd 首先出现异常
+  ├── ansible-client-01 的 etcd 容器开始 CrashLoop
+  ├── ansible-client-02 的 etcd 容器开始 CrashLoop  
+  ├── 启动探针返回 HTTP 503
+  └── etcd 集群丧失 quorum（多数派共识）
+
+阶段2: kube-apiserver 因 etcd 不可用而连锁失败
+  ├── ansible-client-01 apiserver: connection reset by peer
+  ├── ansible-client-02 apiserver: connection refused
+  ├── ubuntu2204 apiserver: HTTP 500 错误
+  └── 所有 apiserver 进入 CrashLoopBackOff
+
+阶段3: 控制平面其他组件受波及
+  ├── kube-controller-manager 重启 6 次后进入 BackOff
+  ├── kube-scheduler 重启 7 次后进入 BackOff
+  └── TLS handshake timeout（因 apiserver 不可用）
+
+阶段4: 业务层面影响显现
+  ├── ubuntu2204 节点报告 InvalidDiskCapacity
+  ├── 所有 kubectl 操作失败
+  ├── 集群健康评分降至 0/100
+  └── 集群完全不可用
+```
+
+------
+
+### 附录E: etcd 数据备份与恢复 SOP（标准操作流程）
+
+#### E.1 etcd 定期备份脚本
+
+```bash
+#!/bin/bash
+# etcd-backup.sh - 定期备份 etcd 数据
+# 建议通过 cron 每小时执行一次
+
+BACKUP_DIR="/backup/etcd"
+DATE=$(date +%Y%m%d_%H%M%S)
+BACKUP_FILE="${BACKUP_DIR}/etcd-snapshot-${DATE}.db"
+ETCD_ENDPOINT="https://192.168.40.11:2379"
+ETCD_CACERT="/etc/kubernetes/pki/etcd/ca.crt"
+ETCD_CERT="/etc/kubernetes/pki/etcd/server.crt"
+ETCD_KEY="/etc/kubernetes/pki/etcd/server.key"
+
+# 创建备份目录
+mkdir -p ${BACKUP_DIR}
+
+# 执行备份
+ETCDCTL_API=3 etcdctl snapshot save ${BACKUP_FILE} \
+  --endpoints=${ETCD_ENDPOINT} \
+  --cacert=${ETCD_CACERT} \
+  --cert=${ETCD_CERT} \
+  --key=${ETCD_KEY}
+
+# 验证备份
+ETCDCTL_API=3 etcdctl snapshot status ${BACKUP_FILE} --write-table
+
+# 保留最近 30 个备份
+ls -t ${BACKUP_DIR}/etcd-snapshot-*.db | tail -n +31 | xargs rm -f
+
+echo "[${DATE}] etcd backup completed: ${BACKUP_FILE}"
+```
+
+#### E.2 etcd 灾难恢复流程
+
+```bash
+#!/bin/bash
+# etcd-restore.sh - 从备份恢复 etcd
+
+SNAPSHOT_FILE=$1  # 备份文件路径
+ETCD_DATA_DIR="/var/lib/etcd"
+ETCD_NAME="ansible-client-01"
+ETCD_INITIAL_CLUSTER="ansible-client-01=https://192.168.40.11:2380,ansible-client-02=https://192.168.40.12:2380"
+
+if [ -z "${SNAPSHOT_FILE}" ]; then
+    echo "Usage: $0 <snapshot-file>"
+    exit 1
+fi
+
+echo "=== Step 1: 停止所有 etcd 实例 ==="
+systemctl stop etcd
+sleep 5
+
+echo "=== Step 2: 备份当前数据目录 ==="
+if [ -d "${ETCD_DATA_DIR}" ]; then
+    mv ${ETCD_DATA_DIR} ${ETCD_DATA_DIR}.bak.$(date +%s)
+fi
+
+echo "=== Step 3: 从快照恢复数据 ==="
+ETCDCTL_API=3 etcdctl snapshot restore ${SNAPSHOT_FILE} \
+  --name ${ETCD_NAME} \
+  --initial-cluster ${ETCD_INITIAL_CLUSTER} \
+  --initial-advertise-peer-urls https://192.168.40.11:2380 \
+  --data-dir ${ETCD_DATA_DIR}
+
+echo "=== Step 4: 设置权限 ==="
+chown -R etcd:etcd ${ETCD_DATA_DIR}
+
+echo "=== Step 5: 启动 etcd ==="
+systemctl start etcd
+
+echo "=== Step 6: 验证恢复结果 ==="
+sleep 10
+ETCDCTL_API=3 etcdctl endpoint health \
+  --endpoints=https://192.168.40.11:2379 \
+  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
+  --cert=/etc/kubernetes/pki/etcd/server.crt \
+  --key=/etc/kubernetes/pki/etcd/server.key
+
+ETCDCTL_API=3 etcdctl member list \
+  --endpoints=https://192.168.40.11:2379 \
+  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
+  --cert=/etc/kubernetes/pki/etcd/server.crt \
+  --key=/etc/kubernetes/pki/etcd/server.key \
+  --write-table
+
+echo "=== 恢复完成 ==="
+```
+
+#### E.3 etcd 快照恢复后恢复集群其余节点
+
+```bash
+#!/bin/bash
+# 在恢复第一个 etcd 节点成功后，将其他节点加入集群
+
+# Step 1: 在已恢复的节点上，移除旧成员
+ETCDCTL_API=3 etcdctl member remove <old-member-id> \
+  --endpoints=https://192.168.40.11:2379 \
+  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
+  --cert=/etc/kubernetes/pki/etcd/server.crt \
+  --key=/etc/kubernetes/pki/etcd/server.key
+
+# Step 2: 添加新成员
+ETCDCTL_API=3 etcdctl member add ansible-client-02 \
+  --peer-urls=https://192.168.40.12:2380 \
+  --endpoints=https://192.168.40.11:2379 \
+  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
+  --cert=/etc/kubernetes/pki/etcd/server.crt \
+  --key=/etc/kubernetes/pki/etcd/server.key
+
+# Step 3: SSH到 ansible-client-02，清理数据目录并启动
+# ssh root@192.168.40.12
+# rm -rf /var/lib/etcd/*
+# systemctl restart etcd
+
+# Step 4: 验证集群成员
+ETCDCTL_API=3 etcdctl member list \
+  --endpoints=https://192.168.40.11:2379,https://192.168.40.12:2379 \
+  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
+  --cert=/etc/kubernetes/pki/etcd/server.crt \
+  --key=/etc/kubernetes/pki/etcd/server.key \
+  --write-table
+```
+
+------
+
+### 附录F: 推荐的 CronJob 定时备份配置
+
+```yaml
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: etcd-backup
+  namespace: kube-system
+spec:
+  schedule: "0 */1 * * *"  # 每小时执行一次
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          nodeSelector:
+            kubernetes.io/hostname: ansible-client-01
+          hostNetwork: true
+          tolerations:
+          - key: node-role.kubernetes.io/master
+            effect: NoSchedule
+          containers:
+          - name: etcd-backup
+            image: bitnami/etcd:3.5.0
+            command:
+            - /bin/sh
+            - -c
+            - |
+              DATE=$(date +%Y%m%d_%H%M%S)
+              ETCDCTL_API=3 etcdctl snapshot save /backup/etcd-snapshot-${DATE}.db \
+                --endpoints=https://127.0.0.1:2379 \
+                --cacert=/etc/kubernetes/pki/etcd/ca.crt \
+                --cert=/etc/kubernetes/pki/etcd/server.crt \
+                --key=/etc/kubernetes/pki/etcd/server.key
+              echo "Backup completed: etcd-snapshot-${DATE}.db"
+            env:
+            - name: ETCDCTL_API
+              value: "3"
+            volumeMounts:
+            - name: etcd-certs
+              mountPath: /etc/kubernetes/pki/etcd
+              readOnly: true
+            - name: backup-dir
+              mountPath: /backup
+          restartPolicy: OnFailure
+          volumes:
+          - name: etcd-certs
+            hostPath:
+              path: /etc/kubernetes/pki/etcd
+              type: DirectoryOrCreate
+          - name: backup-dir
+            hostPath:
+              path: /backup/etcd
+              type: DirectoryOrCreate
+```
+
+------
+
+### 附录G: 集群健康巡检 Checklist
+
+| 序号 | 检查项               | 命令                                                         | 正常状态         | 检查频率 |
+| ---- | -------------------- | ------------------------------------------------------------ | ---------------- | -------- |
+| 1    | etcd 集群健康        | `etcdctl endpoint health`                                    | 所有节点 healthy | 每5分钟  |
+| 2    | etcd 数据库大小      | `etcdctl endpoint status --write-table`                      | < 2GB            | 每小时   |
+| 3    | etcd 磁盘 fsync 延迟 | Prometheus: `etcd_disk_wal_fsync_duration_seconds`           | < 10ms           | 实时     |
+| 4    | apiserver 可用性     | `kubectl get --raw='/readyz?verbose'`                        | 全部 ok          | 每分钟   |
+| 5    | 证书过期时间         | `kubeadm certs check-expiration`                             | > 30天           | 每周     |
+| 6    | 节点资源使用率       | `kubectl top nodes`                                          | CPU<80%, Mem<80% | 每小时   |
+| 7    | 系统 Pod 状态        | `kubectl get pods -n kube-system`                            | 全部 Running     | 每5分钟  |
+| 8    | PVC 使用率           | `kubectl get pvc -A`                                         | < 80%            | 每小时   |
+| 9    | 控制平面组件重启次数 | `kubectl get pods -n kube-system -o wide`                    | 重启次数=0       | 每小时   |
+| 10   | 集群事件异常         | `kubectl get events -n kube-system --field-selector type=Warning` | 无新增 Warning   | 每小时   |
+
+------
+
+### 附录H: 紧急联系人和应急流程
+
+```
+┌─────────────────────────────────────────────────────────┐
+│                  集群故障应急响应流程                     │
+├─────────────────────────────────────────────────────────┤
+│                                                         │
+│  Level 1 (P0): 集群完全不可用                            │
+│  ├── 控制平面全部异常                                    │
+│  ├── etcd 集群丧失 quorum                               │
+│  ├── 立即响应，全员介入                                  │
+│  └── 预期恢复时间: < 1 小时                              │
+│                                                         │
+│  Level 2 (P1): 控制平面部分异常                          │
+│  ├── 单个 apiserver 节点故障                             │
+│  ├── 部分组件 CrashLoop                                 │
+│  ├── 30分钟内响应                                       │
+│  └── 预期恢复时间: < 2 小时                              │
+│                                                         │
+│  Level 3 (P2): Worker 节点异常                          │
+│  ├── 单个节点 NotReady                                  │
+│  ├── 工作负载自动迁移                                    │
+│  ├── 1小时内响应                                        │
+│  └── 预期恢复时间: < 4 小时                              │
+│                                                         │
+│  Level 4 (P3): 非关键告警                               │
+│  ├── 证书即将过期（>30天）                               │
+│  ├── 资源使用率偏高                                      │
+│  ├── 24小时内响应                                       │
+│  └── 计划性维护窗口处理                                  │
+│                                                         │
+└─────────────────────────────────────────────────────────┘
+```
+
+------
+
+## 八、总结与行动项
+
+### 📋 立即执行（今日内）
+
+| 序号 | 行动项                         | 负责人       | 优先级 | 预期完成时间       |
+| ---- | ------------------------------ | ------------ | ------ | ------------------ |
+| 1    | 排查 etcd 崩溃根因并恢复集群   | SRE团队      | P0     | 立即               |
+| 2    | 恢复 kube-apiserver 至正常状态 | SRE团队      | P0     | etcd恢复后30分钟内 |
+| 3    | 验证集群功能恢复正常           | SRE团队      | P0     | apiserver恢复后    |
+| 4    | 检查是否有数据丢失             | DBA/平台团队 | P0     | 集群恢复后         |
+
+### 📋 短期执行（本周内）
+
+| 序号 | 行动项                       | 负责人   | 优先级 | 预期完成时间 |
+| ---- | ---------------------------- | -------- | ------ | ------------ |
+| 5    | 部署 etcd 自动备份 CronJob   | SRE团队  | P1     | 3天内        |
+| 6    | 配置 etcd 监控告警规则       | 监控团队 | P1     | 3天内        |
+| 7    | 修复 ubuntu2204 节点存储问题 | 运维团队 | P1     | 2天内        |
+| 8    | 更新证书管理流程             | SRE团队  | P2     | 一周内       |
+
+### 📋 中长期执行（本月内）
+
+| 序号 | 行动项                       | 负责人   | 优先级 | 预期完成时间 |
+| ---- | ---------------------------- | -------- | ------ | ------------ |
+| 9    | 建立完整的灾难恢复预案       | SRE团队  | P1     | 两周内       |
+| 10   | 实施 etcd 自动碎片整理       | 平台团队 | P2     | 两周内       |
+| 11   | 升级 Kubernetes 至最新稳定版 | 平台团队 | P2     | 一个月内     |
+| 12   | 建立定期巡检机制             | SRE团队  | P2     | 两周内       |
+| 13   | 进行灾难恢复演练             | 全团队   | P2     | 一个月内     |
+
+------
+
+### 📝 故障复盘要点
+
+**本次故障的核心教训：**
+
+1. **etcd 是集群的心脏**：etcd 故障会导致整个集群瘫痪，必须将其作为最高优先级保护对象
+2. **缺少自动备份机制**：如果之前有 etcd 定期备份，恢复时间可以大幅缩短
+3. **监控告警不足**：etcd 的健康指标应该有完善的监控和告警，在问题初期就能发现
+4. **证书管理需要自动化**：证书过期是 Kubernetes 集群最常见的故障原因之一
+5. **需要建立 SOP 文档**：标准操作流程可以在故障时减少决策时间，提高恢复效率
+
+> **注：** 本报告基于诊断数据推断。实际修复时请根据现场日志和具体情况调整方案。建议在修复前先对当前状态做完整快照/备份，以防操作失误导致进一步损坏。