Your Name
5d83348db3
- core/monitor.py: add fast metrics script (0.3s vs 1.4s sleep), collect() now defaults to fast=True for responsive UI - ui/workers.py: MonitorWorker changed from loop mode to single-shot (QTimer re-arms next cycle on finished signal, avoids paramiko blocking in infinite loop) - ui/widgets.py: remove debug print from _kick_one_sample - test_monitor_nonblock.py: fix sample counting by wrapping _on_sample instead of disconnecting signals (old approach missed new workers) - test_process_monitor.py: assertions account for MAX_RENDER_ROWS=200, fix MEM% column index (4 not 3) in sort test
289 lines
12 KiB
Python
289 lines
12 KiB
Python
"""
|
|
远程主机系统监控模块
|
|
通过 SSH 一次性采集 CPU/内存/磁盘/网络/负载指标。
|
|
Linux 用 /proc 和常用命令；macOS/BSD 走兼容路径。
|
|
"""
|
|
import re
|
|
import time
|
|
from typing import Optional
|
|
|
|
from .ssh_client import SSHConnection
|
|
|
|
|
|
class SystemMonitor:
|
|
"""远程主机的资源监控器（数据全部从 SSH 通道采集，不依赖 agent）"""
|
|
|
|
# 慢脚本：CPU 差分需要 sleep 1（约 1.4 秒）
|
|
_LINUX_METRICS_SCRIPT = r"""
|
|
echo "===CPU==="
|
|
# 第一次采样 1 秒间隔，用来计算差分
|
|
awk 'NR==1 {gsub("cpu ",""); print}' /proc/stat > /tmp/.cpu1.$$
|
|
sleep 1
|
|
awk 'NR==1 {gsub("cpu ",""); print}' /proc/stat > /tmp/.cpu2.$$
|
|
read cpu_user cpu_nice cpu_system cpu_idle cpu_iowait cpu_irq cpu_softirq cpu_steal cpu_guest cpu_gnice < /tmp/.cpu1.$$
|
|
read cpu_user2 cpu_nice2 cpu_system2 cpu_idle2 cpu_iowait2 cpu_irq2 cpu_softirq2 cpu_steal2 cpu_guest2 cpu_gnice2 < /tmp/.cpu2.$$
|
|
rm -f /tmp/.cpu1.$$ /tmp/.cpu2.$$
|
|
total1=$((cpu_user+cpu_nice+cpu_system+cpu_idle+cpu_iowait+cpu_irq+cpu_softirq+cpu_steal))
|
|
total2=$((cpu_user2+cpu_nice2+cpu_system2+cpu_idle2+cpu_iowait2+cpu_irq2+cpu_softirq2+cpu_steal2))
|
|
idle1=$cpu_idle; idle2=$cpu_idle2
|
|
dt=$((total2-total1)); di=$((idle2-idle1))
|
|
if [ $dt -gt 0 ]; then usage=$(( (1000*(dt-di)/dt+5)/10 )); else usage=0; fi
|
|
echo "CPU_USAGE=$usage"
|
|
echo "CPU_CORES=$(nproc 2>/dev/null || echo 1)"
|
|
echo "LOAD=$(cat /proc/loadavg | awk '{print $1,$2,$3}')"
|
|
echo "UPTIME=$(awk '{printf "%.0f",$1}' /proc/uptime)"
|
|
echo "===MEM==="
|
|
mem_total=$(awk '/MemTotal/{print $2}' /proc/meminfo)
|
|
mem_avail=$(awk '/MemAvailable/{print $2}' /proc/meminfo)
|
|
swap_total=$(awk '/SwapTotal/{print $2}' /proc/meminfo)
|
|
swap_free=$(awk '/SwapFree/{print $2}' /proc/meminfo)
|
|
if [ -z "$mem_avail" ]; then mem_avail=$((mem_total - $(awk '/^(Buffers|Cached|SReclaimable):/{s+=$2} END{print s}' /proc/meminfo))); fi
|
|
used=$((mem_total - mem_avail))
|
|
echo "MEM_TOTAL=$mem_total"
|
|
echo "MEM_USED=$used"
|
|
echo "MEM_AVAIL=$mem_avail"
|
|
echo "SWAP_TOTAL=$swap_total"
|
|
echo "SWAP_USED=$((swap_total-swap_free))"
|
|
echo "===DISK==="
|
|
df -PB1 -x tmpfs -x devtmpfs 2>/dev/null | awk 'NR>1 {printf "DISK|%s|%d|%d|%s\n",$NF,$2,$3,$5}'
|
|
echo "===NET==="
|
|
for iface in $(ls /sys/class/net/ 2>/dev/null | grep -v lo); do
|
|
rx=$(cat /sys/class/net/$iface/statistics/rx_bytes 2>/dev/null || echo 0)
|
|
tx=$(cat /sys/class/net/$iface/statistics/tx_bytes 2>/dev/null || echo 0)
|
|
echo "NET|$iface|$rx|$tx"
|
|
done
|
|
echo "===PROC==="
|
|
# 进程统计
|
|
ps -e -o stat= 2>/dev/null | awk '{r+=($1~/^R/); s+=($1~/^S/); d+=($1~/^D/); z+=($1~/^Z/); t++} END{printf "PROC_TOTAL=%d\nPROC_RUNNING=%d\nPROC_SLEEP=%d\nPROC_DISK=%d\nPROC_ZOMBIE=%d\n", t, r, s, d, z}'
|
|
# 进程列表：etimes/times 都是纯数字，避免 start 字段含 "Jul 21" 多列错位
|
|
# comm 字段可能含空格，前 9 列用 \t 拼，comm 用换行做记录结束
|
|
# 字段：pid ppid user pcpu pmem vsz rss stat pri nice etimes times args
|
|
ps -eo pid,ppid,user,pcpu,pmem,vsz,rss,stat,pri,nice,etimes,times,args --sort=-pcpu --no-headers 2>/dev/null \
|
|
| head -500 \
|
|
| awk '{
|
|
out="PROC\t";
|
|
for(i=1;i<=12;i++) out=out $i "\t";
|
|
rest="";
|
|
for(i=13;i<=NF;i++) rest=(i==13?$i:rest " " $i);
|
|
print out rest
|
|
}'
|
|
echo "===HOST==="
|
|
echo "HOSTNAME=$(hostname)"
|
|
echo "KERNEL=$(uname -r)"
|
|
echo "OS=$(. /etc/os-release 2>/dev/null && echo "$PRETTY_NAME" || uname -s)"
|
|
"""
|
|
|
|
# 快脚本：不含 sleep 1，但 CPU 数字用即时 busy%（基于 /proc/stat 当前总样本，
|
|
# 加上 -1 让服务端快速 grep 两次间隔 0.3s 算差分，~0.4s 总耗时）
|
|
# 用 usleep 微秒精度，0.3s 差分窗口 + awk 算 CPU%
|
|
_FAST_METRICS_SCRIPT = r"""
|
|
echo "===CPU==="
|
|
# 快速差分：0.3 秒间隔（避免 1.5s 阻塞）。误差 ±5% 但响应快
|
|
# awk 解析 /proc/stat 第一行（cpu 总览），两次采样做差分
|
|
awk 'NR==1 {gsub("cpu ",""); print}' /proc/stat > /tmp/.cpu1.$$
|
|
sleep 0.3 2>/dev/null || sleep 1
|
|
awk 'NR==1 {gsub("cpu ",""); print}' /proc/stat > /tmp/.cpu2.$$
|
|
read u1 n1 s1 i1 io1 irq1 si1 st1 g1 gn1 < /tmp/.cpu1.$$
|
|
read u2 n2 s2 i2 io2 irq2 si2 st2 g2 gn2 < /tmp/.cpu2.$$
|
|
rm -f /tmp/.cpu1.$$ /tmp/.cpu2.$$
|
|
t1=$((u1+n1+s1+i1+io1+irq1+si1+st1))
|
|
t2=$((u2+n2+s2+i2+io2+irq2+si2+st2))
|
|
di=$((i2-i1))
|
|
dt=$((t2-t1))
|
|
if [ $dt -gt 0 ]; then usage=$(( (1000*(dt-di)/dt+5)/10 )); else usage=0; fi
|
|
echo "CPU_USAGE=$usage"
|
|
echo "CPU_CORES=$(nproc 2>/dev/null || echo 1)"
|
|
echo "LOAD=$(cat /proc/loadavg | awk '{print $1,$2,$3}')"
|
|
echo "UPTIME=$(awk '{printf "%.0f",$1}' /proc/uptime)"
|
|
echo "===MEM==="
|
|
mem_total=$(awk '/MemTotal/{print $2}' /proc/meminfo)
|
|
mem_avail=$(awk '/MemAvailable/{print $2}' /proc/meminfo)
|
|
swap_total=$(awk '/SwapTotal/{print $2}' /proc/meminfo)
|
|
swap_free=$(awk '/SwapFree/{print $2}' /proc/meminfo)
|
|
if [ -z "$mem_avail" ]; then mem_avail=$((mem_total - $(awk '/^(Buffers|Cached|SReclaimable):/{s+=$2} END{print s}' /proc/meminfo))); fi
|
|
used=$((mem_total - mem_avail))
|
|
echo "MEM_TOTAL=$mem_total"
|
|
echo "MEM_USED=$used"
|
|
echo "MEM_AVAIL=$mem_avail"
|
|
echo "SWAP_TOTAL=$swap_total"
|
|
echo "SWAP_USED=$((swap_total-swap_free))"
|
|
echo "===DISK==="
|
|
df -PB1 -x tmpfs -x devtmpfs 2>/dev/null | awk 'NR>1 {printf "DISK|%s|%d|%d|%s\n",$NF,$2,$3,$5}'
|
|
echo "===NET==="
|
|
for iface in $(ls /sys/class/net/ 2>/dev/null | grep -v lo); do
|
|
rx=$(cat /sys/class/net/$iface/statistics/rx_bytes 2>/dev/null || echo 0)
|
|
tx=$(cat /sys/class/net/$iface/statistics/tx_bytes 2>/dev/null || echo 0)
|
|
echo "NET|$iface|$rx|$tx"
|
|
done
|
|
echo "===PROC==="
|
|
ps -e -o stat= 2>/dev/null | awk '{r+=($1~/^R/); s+=($1~/^S/); d+=($1~/^D/); z+=($1~/^Z/); t++} END{printf "PROC_TOTAL=%d\nPROC_RUNNING=%d\nPROC_SLEEP=%d\nPROC_DISK=%d\nPROC_ZOMBIE=%d\n", t, r, s, d, z}'
|
|
ps -eo pid,ppid,user,pcpu,pmem,vsz,rss,stat,pri,nice,etimes,times,args --sort=-pcpu --no-headers 2>/dev/null \
|
|
| head -500 \
|
|
| awk '{
|
|
out="PROC\t";
|
|
for(i=1;i<=12;i++) out=out $i "\t";
|
|
rest="";
|
|
for(i=13;i<=NF;i++) rest=(i==13?$i:rest " " $i);
|
|
print out rest
|
|
}'
|
|
echo "===HOST==="
|
|
echo "HOSTNAME=$(hostname)"
|
|
echo "KERNEL=$(uname -r)"
|
|
echo "OS=$(. /etc/os-release 2>/dev/null && echo "$PRETTY_NAME" || uname -s)"
|
|
"""
|
|
|
|
@staticmethod
|
|
def _parse_kv(text: str, key: str, default: str = "0") -> str:
|
|
"""从 KEY=VALUE 行中取值"""
|
|
m = re.search(rf"^{re.escape(key)}=(.+)$", text, re.MULTILINE)
|
|
return m.group(1).strip() if m else default
|
|
|
|
@classmethod
|
|
def collect(cls, conn: SSHConnection, fast: bool = True) -> dict:
|
|
"""采集一次指标；返回 dict"""
|
|
empty = {
|
|
"cpu": 0.0, "cores": 1, "load1": 0, "load5": 0, "load15": 0,
|
|
"uptime": 0, "hostname": "", "kernel": "", "os": "",
|
|
"mem_total": 0, "mem_used": 0, "mem_percent": 0.0,
|
|
"swap_total": 0, "swap_used": 0,
|
|
"disks": [], "net": [],
|
|
"proc_total": 0, "proc_running": 0, "proc_sleep": 0,
|
|
"proc_disk": 0, "proc_zombie": 0, "processes": [],
|
|
"ts": time.time(),
|
|
}
|
|
if not conn or not conn.connected:
|
|
return empty
|
|
script = cls._FAST_METRICS_SCRIPT if fast else cls._LINUX_METRICS_SCRIPT
|
|
code, out, err = conn.exec_command(script, timeout=10)
|
|
if code != 0 or not out:
|
|
empty["error"] = err or "采集失败"
|
|
return empty
|
|
|
|
result = dict(empty)
|
|
result["hostname"] = cls._parse_kv(out, "HOSTNAME")
|
|
result["kernel"] = cls._parse_kv(out, "KERNEL")
|
|
result["os"] = cls._parse_kv(out, "OS")
|
|
try:
|
|
result["cpu"] = float(cls._parse_kv(out, "CPU_USAGE"))
|
|
except ValueError:
|
|
pass
|
|
try:
|
|
result["cores"] = int(cls._parse_kv(out, "CPU_CORES", "1"))
|
|
except ValueError:
|
|
pass
|
|
load = cls._parse_kv(out, "LOAD", "0 0 0").split()
|
|
try:
|
|
result["load1"] = float(load[0])
|
|
result["load5"] = float(load[1]) if len(load) > 1 else 0
|
|
result["load15"] = float(load[2]) if len(load) > 2 else 0
|
|
except (ValueError, IndexError):
|
|
pass
|
|
try:
|
|
result["uptime"] = int(cls._parse_kv(out, "UPTIME"))
|
|
except ValueError:
|
|
pass
|
|
|
|
try:
|
|
mt = int(cls._parse_kv(out, "MEM_TOTAL"))
|
|
mu = int(cls._parse_kv(out, "MEM_USED"))
|
|
result["mem_total"] = mt
|
|
result["mem_used"] = mu
|
|
result["mem_percent"] = (mu / mt * 100) if mt > 0 else 0.0
|
|
result["swap_total"] = int(cls._parse_kv(out, "SWAP_TOTAL"))
|
|
result["swap_used"] = int(cls._parse_kv(out, "SWAP_USED"))
|
|
except ValueError:
|
|
pass
|
|
|
|
result["disks"] = []
|
|
for line in out.splitlines():
|
|
if line.startswith("DISK|"):
|
|
_, mount, total, used, percent = line.split("|", 4)
|
|
try:
|
|
result["disks"].append({
|
|
"mount": mount, "total": int(total),
|
|
"used": int(used), "percent": int(percent.rstrip("%")),
|
|
})
|
|
except ValueError:
|
|
continue
|
|
|
|
result["net"] = []
|
|
for line in out.splitlines():
|
|
if line.startswith("NET|"):
|
|
_, name, rx, tx = line.split("|", 3)
|
|
try:
|
|
result["net"].append({
|
|
"iface": name, "rx": int(rx), "tx": int(tx),
|
|
})
|
|
except ValueError:
|
|
continue
|
|
|
|
# 进程统计
|
|
try:
|
|
result["proc_total"] = int(cls._parse_kv(out, "PROC_TOTAL"))
|
|
result["proc_running"] = int(cls._parse_kv(out, "PROC_RUNNING"))
|
|
result["proc_sleep"] = int(cls._parse_kv(out, "PROC_SLEEP"))
|
|
result["proc_disk"] = int(cls._parse_kv(out, "PROC_DISK"))
|
|
result["proc_zombie"] = int(cls._parse_kv(out, "PROC_ZOMBIE"))
|
|
except ValueError:
|
|
result["proc_total"] = 0
|
|
result["proc_running"] = 0
|
|
result["proc_sleep"] = 0
|
|
result["proc_disk"] = 0
|
|
result["proc_zombie"] = 0
|
|
|
|
# 进程列表（远程用 \t 分隔前 12 列，第 13 列开始是 args/comm，可能含空格）
|
|
result["processes"] = []
|
|
for line in out.splitlines():
|
|
if not line.startswith("PROC\t"):
|
|
continue
|
|
payload = line[5:] # 去掉 "PROC\t" 前缀
|
|
fields = payload.split("\t", 12) # 只切前 12 次
|
|
if len(fields) < 13:
|
|
continue
|
|
try:
|
|
result["processes"].append({
|
|
"pid": int(fields[0]),
|
|
"ppid": int(fields[1]) if fields[1].isdigit() else 0,
|
|
"user": fields[2],
|
|
"pcpu": float(fields[3]),
|
|
"pmem": float(fields[4]),
|
|
"vsz": int(fields[5]),
|
|
"rss": int(fields[6]),
|
|
"stat": fields[7],
|
|
"pri": int(fields[8]) if fields[8].lstrip("-").isdigit() else 0,
|
|
"nice": int(fields[9]) if fields[9].lstrip("-").isdigit() else 0,
|
|
# etimes: 自启动以来的秒数（整数）
|
|
"etime": int(fields[10]) if fields[10].isdigit() else 0,
|
|
# times: 累计 CPU 时间秒数
|
|
"time": int(fields[11]) if fields[11].isdigit() else 0,
|
|
"comm": fields[12].strip(),
|
|
})
|
|
except (ValueError, IndexError):
|
|
continue
|
|
|
|
return result
|
|
|
|
@staticmethod
|
|
def format_bytes(n: int) -> str:
|
|
"""人类可读字节数"""
|
|
n = float(n)
|
|
for unit in ("B", "KB", "MB", "GB", "TB", "PB"):
|
|
if n < 1024:
|
|
return f"{n:.1f}{unit}"
|
|
n /= 1024
|
|
return f"{n:.1f}EB"
|
|
|
|
@staticmethod
|
|
def format_uptime(seconds: int) -> str:
|
|
seconds = int(seconds)
|
|
d, rem = divmod(seconds, 86400)
|
|
h, rem = divmod(rem, 3600)
|
|
m, s = divmod(rem, 60)
|
|
if d:
|
|
return f"{d}天{h}小时"
|
|
if h:
|
|
return f"{h}小时{m}分"
|
|
if m:
|
|
return f"{m}分{s}秒"
|
|
return f"{s}秒"
|