Files
ipmi-monitor/backend/app/routers/monitor.py
T
cnbugs fa09603025 安全审计修复 + BUG 修复 + 新功能
审计基准 a3ae072。安全 14 项(严重 4、高危 10)全修:JWT 启动强制
校验(缺失/默认值/短于 32 位拒绝启动);默认 admin 改为
ADMIN_INITIAL_PASSWORD 必填(≥12 位);BMC/SMTP/飞书口令 Fernet
加密入库(新增 crypto.py,密钥走环境变量或 0600 文件,存量明文
启动时迁移),HostOut 与设置接口脱敏;InfluxDB 默认凭证清除、
Compose 改 ${VAR:?must be set}、start.sh 遇空/默认值退出;CORS
白名单;SPA 防遍历;生产关 /docs + 安全头;JWT 8h + jti + 登出/
改密(改密吊销全部旧 token);python-jose→PyJWT;requirements
全钉 == 版;SSRF 校验层(新增 ssrf.py);登录限流 5 次锁 15 分钟;
容器非 root、删源码挂载。

BUG 30 项全修:恢复误发、脏数据隔离、采集超时 30s→10s +
Semaphore(8) + 失败退避、SDR 解析、组件接口错文件、时区 8 小时、
InfluxDB 静默丢数据、密码明文返回、配置死变量、删 init_admin.py、
显式 commit、邮件 TLS/飞书 keyword、趋势功耗 fallback;剔除 -C 0
无认证与 IPMI 1.5 降级。

优化 11 项:按需查询、设置批量读、公共 resolve_total_power、
SQLite WAL/外键/索引、Dockerfile 加固、start.sh 轮询健康、
alert-trigger.sh 按真实接口重写、InfluxDB 超时、补测试 + README
重写(37 个端点)。

新功能 2 个:POST /api/auth/change-password(+ logout)、
GET /api/monitor/export CSV 导出;前端 Hosts/Settings 同步掩码展示。

验证:36/36 单元测试通过;py_compile 全过。未做真实 BMC/InfluxDB
联调与 Docker 构建验证。审计报告见 AUDIT.md 及三分报告。
2026-11-16 12:30:00 +08:00

611 lines
23 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Monitoring data REST API — dashboard, history, realtime."""
import csv
import io
import logging
from datetime import datetime, timedelta, timezone
from pathlib import Path as PathLib
from typing import Optional, List
from fastapi import APIRouter, Depends, HTTPException, Query, Path
from fastapi.responses import StreamingResponse
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app import influxdb as influxdb_module
from app.database import get_db
from app.models import AlertHistory, Host, Host as HostModel
from app.routers.auth import get_current_user
from app.models import User as UserModel
from app.services.component_parser import parse_component_presence, group_components_by_status
from app.utils import parse_dt, resolve_total_power, utc_now_naive
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/monitor", tags=["monitor"])
# Metric types accepted by the history/summary/trend endpoints (B12).
METRIC_TYPES = {"temperature", "fan", "voltage", "power", "component", "status"}
MAX_HOST_IDS = 50
def _parse_host_ids(host_ids: str) -> list[int]:
"""Parse a comma-separated host-id list (B12).
Raises 400 for non-integer ids or more than 50 ids.
"""
ids: list[int] = []
for raw in host_ids.split(","):
raw = raw.strip()
if not raw:
continue
try:
ids.append(int(raw))
except ValueError:
raise HTTPException(400, f"Invalid host id: {raw!r}")
if len(ids) > MAX_HOST_IDS:
raise HTTPException(400, f"Too many host ids (max {MAX_HOST_IDS})")
return ids
def _check_metric_type(metric_type: str) -> str:
"""Reject metric types outside the whitelist (B12)."""
if metric_type not in METRIC_TYPES:
raise HTTPException(400, f"Invalid metric_type: {metric_type!r}")
return metric_type
def _parse_time_range(
start: Optional[str], end: Optional[str]
) -> tuple[datetime, Optional[datetime]]:
"""Parse start/end query params (B11).
Bad values are a 400, never a 500. Defaults use UTC now (B11).
"""
try:
start_dt = parse_dt(start)
end_dt = parse_dt(end)
except ValueError as exc:
raise HTTPException(400, f"Invalid datetime: {exc}")
if start_dt is None:
start_dt = utc_now_naive() - timedelta(hours=1)
return start_dt, end_dt
# ---------------------------------------------------------------------------
# Dashboard
# ---------------------------------------------------------------------------
@router.get("/dashboard")
async def dashboard(
db: AsyncSession = Depends(get_db),
current_user: UserModel = Depends(get_current_user)
):
"""Return aggregate dashboard stats."""
# Total hosts / enabled hosts
total_result = await db.execute(select(func.count()).select_from(Host))
total_hosts = total_result.scalar() or 0
enabled_result = await db.execute(select(func.count()).select_from(Host).where(Host.enabled.is_(True)))
enabled_hosts = enabled_result.scalar() or 0
# Online: have been seen in the last 2× collect interval
from app.config import settings as app_settings
online_threshold = datetime.now() - timedelta(seconds=app_settings.COLLECT_INTERVAL * 2)
online_result = await db.execute(
select(func.count())
.select_from(Host)
.where(Host.enabled.is_(True), Host.last_seen >= online_threshold)
)
online_hosts = online_result.scalar() or 0
offline_hosts = enabled_hosts - online_hosts
# Unacknowledged alerts
alert_result = await db.execute(
select(func.count())
.select_from(AlertHistory)
.where(AlertHistory.acknowledged.is_(False))
)
active_alerts = alert_result.scalar() or 0
return {
"total_hosts": total_hosts,
"enabled_hosts": enabled_hosts,
"online_hosts": online_hosts,
"offline_hosts": offline_hosts,
"active_alerts": active_alerts,
}
# ---------------------------------------------------------------------------
# Multi-host & Summary APIs (MUST be before /history/{host_id})
# ---------------------------------------------------------------------------
@router.get("/history/multi")
async def multi_host_history(
host_ids: str = Query(..., description="Comma-separated host IDs, e.g. '1,2,3'"),
metric_type: str = Query("temperature", description="temperature/fan/voltage/power"),
metric_name: Optional[str] = Query(None, description="Specific sensor name, e.g. 'CPU Temp'"),
start: Optional[str] = Query(None, description="ISO datetime, default 1h ago"),
end: Optional[str] = Query(None, description="ISO datetime, default now"),
current_user: UserModel = Depends(get_current_user),
):
"""Query historical metrics for multiple hosts."""
host_id_list = _parse_host_ids(host_ids)
_check_metric_type(metric_type)
start_dt, end_dt = _parse_time_range(start, end)
all_data = {}
for hid in host_id_list:
data = await influxdb_module.aquery_metrics(hid, metric_type, start_dt, end_dt)
# Filter by metric_name if specified
if metric_name:
data = [d for d in data if d.get('metric_name') == metric_name]
all_data[str(hid)] = data
return {
"metric_type": metric_type,
"metric_name": metric_name,
"start": start_dt.isoformat(),
"end": end_dt.isoformat() if end_dt else None,
"data": all_data,
}
@router.get("/export")
async def export_metrics(
host_ids: str = Query(..., description="Comma-separated host IDs, e.g. '1,2,3'"),
metric_type: str = Query("temperature", description="temperature/fan/voltage/power/component/status"),
start: Optional[str] = Query(None, description="ISO datetime, default 1h ago"),
end: Optional[str] = Query(None, description="ISO datetime, default now"),
current_user: UserModel = Depends(get_current_user),
):
"""Export historical metrics for multiple hosts as a CSV file.
Columns: time,host_id,metric_type,metric_name,value,unit.
Reuses the same host-id / metric-type / time-range validation as the
history endpoints (400 on bad input) and ``influxdb.query_metrics``
(via its async wrapper) for the data.
"""
host_id_list = _parse_host_ids(host_ids)
if not host_id_list:
raise HTTPException(400, "At least one host id is required")
_check_metric_type(metric_type)
start_dt, end_dt = _parse_time_range(start, end)
unit = _get_unit(metric_type)
buf = io.StringIO()
writer = csv.writer(buf)
writer.writerow(["time", "host_id", "metric_type", "metric_name", "value", "unit"])
for hid in host_id_list:
data = await influxdb_module.aquery_metrics(hid, metric_type, start_dt, end_dt)
for d in data:
writer.writerow([
d.get("time", ""),
hid,
metric_type,
d.get("metric_name", ""),
d.get("value", ""),
unit,
])
filename = (
f"ipmi_metrics_{metric_type}_{start_dt.strftime('%Y%m%d%H%M%S')}.csv"
)
return StreamingResponse(
iter([buf.getvalue()]),
media_type="text/csv",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
@router.get("/summary/{metric_type}")
async def metrics_summary(
metric_type: str = Path(..., description="temperature/fan/voltage/power"),
host_ids: Optional[str] = Query(None, description="Comma-separated host IDs, e.g. '1,2,3'"),
db: AsyncSession = Depends(get_db),
current_user: UserModel = Depends(get_current_user),
):
"""Get all sensors for specified hosts (not aggregated).
Returns ALL sensors per host, not aggregated values.
For dashboard totals, use /dashboard/totals endpoint instead.
"""
_check_metric_type(metric_type)
# Parse host IDs
host_id_list: Optional[List[int]] = None
if host_ids:
host_id_list = _parse_host_ids(host_ids)
# Get enabled hosts
stmt = select(HostModel).where(HostModel.enabled.is_(True))
if host_id_list:
stmt = stmt.where(HostModel.id.in_(host_id_list))
result = await db.execute(stmt)
hosts = result.scalars().all()
# Return ALL sensors, not aggregated (only the requested type, O1)
all_sensors = []
for host in hosts:
grouped = await influxdb_module.aquery_latest_grouped(host.id, only={metric_type})
metrics = grouped.get(metric_type, [])
for m in metrics:
all_sensors.append({
"host_id": host.id,
"host_name": host.name,
"metric_name": m.get('metric_name'),
"value": m.get('value'),
"unit": _get_unit(metric_type),
"time": m.get('time'),
"timestamp": m.get('timestamp'),
})
return {"metric_type": metric_type, "sensors": all_sensors}
@router.get("/sensors/{host_id}")
async def host_sensors(
host_id: int,
current_user: UserModel = Depends(get_current_user)
):
"""Get all sensors for a specific host with latest values."""
grouped = await influxdb_module.aquery_latest_grouped(host_id)
sensors = []
for metric_type, metrics in grouped.items():
for m in metrics:
sensors.append({
"metric_type": metric_type,
"metric_name": m.get('metric_name'),
"value": m.get('value'),
"unit": _get_unit(metric_type),
"time": m.get('time'),
})
return {"host_id": host_id, "sensors": sensors}
def _get_unit(metric_type: str) -> str:
"""Get unit symbol for metric type."""
units = {
"temperature": "°C",
"fan": "RPM",
"voltage": "V",
"power": "W",
}
return units.get(metric_type, "")
@router.get("/dashboard/totals")
async def dashboard_totals(
group_id: Optional[int] = Query(None, description="Filter by group ID"),
db: AsyncSession = Depends(get_db),
current_user: UserModel = Depends(get_current_user),
):
"""Get aggregated totals for dashboard - sum of all power sensors per host."""
# Get hosts
stmt = select(HostModel).where(HostModel.enabled.is_(True))
if group_id:
stmt = stmt.where(HostModel.group_id == group_id)
result = await db.execute(stmt)
hosts = result.scalars().all()
# Get groups for reference
from app.routers.groups import get_all_groups
groups = await get_all_groups(db)
host_totals = []
for host in hosts:
grouped = await influxdb_module.aquery_latest_grouped(host.id)
# Total power via the shared resolver (O5): native system-wide
# reading first, otherwise the PSU{N}_PIN sum. A 0 reading from an
# existing sensor is valid and must not trigger the fallback.
total_power = resolve_total_power(grouped.get("power", []))
total_power = float(total_power) if total_power is not None else 0.0
# Get outlet temperature (exhaust air temperature)
temp_metrics = grouped.get("temperature", [])
outlet_temp = None
# Priority: Outlet_Temp -> Outlet Temp -> Sys_Outlet_Temp1 -> Exhaust Temp -> Rear Temp
for sensor_name in ['Outlet_Temp', 'Outlet Temp', 'Sys_Outlet_Temp1', 'Exhaust Temp',
'BB Rt Rear Temp', 'BB Lft Rear Temp', 'Rear Temp']:
for m in temp_metrics:
if m.get('metric_name') == sensor_name:
outlet_temp = m.get('value')
break
if outlet_temp is not None:
break
# Fallback to max temperature if no outlet sensor found (None-safe, B13)
if outlet_temp is not None:
try:
max_temp = float(outlet_temp)
except (TypeError, ValueError):
max_temp = 0.0
else:
temps = []
for m in temp_metrics:
v = m.get('value')
if v is None:
continue
try:
temps.append(float(v))
except (TypeError, ValueError):
continue
max_temp = max(temps) if temps else 0.0
host_totals.append({
"host_id": host.id,
"host_name": host.name,
"ipmi_address": host.ipmi_host,
"group_id": host.group_id,
"group_name": host.group.name if host.group else "Unknown",
"enabled": host.enabled,
"total_power": round(total_power, 1),
"max_temperature": round(max_temp, 1),
"last_seen": host.last_seen.isoformat() if host.last_seen else None,
})
# Group totals
group_totals = {}
for gt in host_totals:
gid = gt["group_id"]
if gid not in group_totals:
group_totals[gid] = {
"group_id": gid,
"group_name": next((g["name"] for g in groups if g["id"] == gid), "Unknown"),
"total_power": 0,
"avg_max_temperature": 0,
"host_count": 0,
}
group_totals[gid]["total_power"] += gt["total_power"] or 0
group_totals[gid]["avg_max_temperature"] += gt["max_temperature"] or 0
group_totals[gid]["host_count"] += 1
# Calculate averages. NOTE (B13): the field is named
# avg_max_temperature because it averages each host's max_temperature;
# the frontend does not read this field (Dashboard.vue computes its own
# average from max_temperature), so the rename is safe.
for gid in group_totals.values():
if gid["host_count"] > 0:
gid["avg_max_temperature"] = round(gid["avg_max_temperature"] / gid["host_count"], 1)
gid["total_power"] = round(gid["total_power"], 1)
return {
"hosts": host_totals,
"groups": [{"group_id": g["id"], "group_name": g["name"]} for g in groups],
"group_totals": list(group_totals.values()),
}
@router.get("/trend")
async def trend(
group_id: Optional[int] = Query(None, description="Filter by group ID"),
metric_type: str = Query("temperature", description="temperature or power"),
time_range: str = Query("default", description="Time range: default, 1m, 5m, 10m, 30m, 1h, 6h, 12h, 24h, 7d, 30d"),
db: AsyncSession = Depends(get_db),
current_user: UserModel = Depends(get_current_user),
):
"""Get trend data for all hosts in a group for the specified time range."""
if metric_type not in ("temperature", "fan", "voltage", "power"):
raise HTTPException(400, f"Invalid metric_type: {metric_type!r}")
# Parse time range to timedelta
range_map = {
"default": timedelta(hours=1),
"1m": timedelta(minutes=1),
"5m": timedelta(minutes=5),
"10m": timedelta(minutes=10),
"30m": timedelta(minutes=30),
"1h": timedelta(hours=1),
"6h": timedelta(hours=6),
"12h": timedelta(hours=12),
"24h": timedelta(hours=24),
"7d": timedelta(days=7),
"30d": timedelta(days=30),
}
delta = range_map.get(time_range, timedelta(hours=1))
start_time = datetime.now(timezone.utc).replace(tzinfo=None) - delta
# Get enabled hosts
stmt = select(HostModel).where(HostModel.enabled.is_(True))
if group_id is not None:
stmt = stmt.where(HostModel.group_id == group_id)
result = await db.execute(stmt)
hosts = result.scalars().all()
# Query trend data for each host
trend_data = {}
for host in hosts:
metrics = await influxdb_module.aquery_trend(host.id, metric_type, start_time)
points = []
if metric_type == "power":
# B30: same aggregation as the dashboard totals — per timestamp,
# native total-power sensor wins, otherwise sum PSU{N}_PIN.
by_ts: dict[str, list[dict]] = {}
for m in metrics:
ts = m.get("timestamp") or m.get("time")
if not ts or m.get("value") is None:
continue
by_ts.setdefault(str(ts), []).append(m)
for ts, ts_metrics in by_ts.items():
total = resolve_total_power(ts_metrics)
if total is None:
continue
points.append({
"timestamp": ts,
"metric_name": "Total_Power",
"value": total,
})
points.sort(key=lambda p: p["timestamp"])
else:
for m in metrics:
ts = m.get("timestamp") or m.get("time")
if not ts or m.get("value") is None:
continue
points.append({
"timestamp": ts,
"metric_name": m.get("metric_name", ""),
"value": m["value"],
})
trend_data[str(host.id)] = {
"host_id": host.id,
"host_name": host.name,
"points": points,
}
return {
"metric_type": metric_type,
"time_range": time_range,
"data": trend_data,
}
# ---------------------------------------------------------------------------
# History (single host) - MUST be after /history/multi
# ---------------------------------------------------------------------------
@router.get("/history/{host_id}")
async def history(
host_id: int,
metric_type: str = Query("temperature", description="temperature/fan/voltage/power"),
start: Optional[str] = Query(None, description="ISO datetime, default 1h ago"),
end: Optional[str] = Query(None, description="ISO datetime, default now"),
current_user: UserModel = Depends(get_current_user),
):
"""Query historical metrics for a host from InfluxDB."""
_check_metric_type(metric_type)
start_dt, end_dt = _parse_time_range(start, end)
data = await influxdb_module.aquery_metrics(host_id, metric_type, start_dt, end_dt)
return {"host_id": host_id, "metric_type": metric_type, "data": data}
# ---------------------------------------------------------------------------
# Realtime
# ---------------------------------------------------------------------------
@router.get("/realtime/{host_id}")
async def realtime(
host_id: int,
current_user: UserModel = Depends(get_current_user)
):
"""Return the latest metrics for a host, grouped by type."""
grouped = await influxdb_module.aquery_latest_grouped(host_id)
return {"host_id": host_id, "metrics": grouped}
# ---------------------------------------------------------------------------
# Component Presence Detection
# ---------------------------------------------------------------------------
@router.get("/components/{host_id}")
async def component_presence(
host_id: int,
current_user: UserModel = Depends(get_current_user)
):
"""Get component presence status for a host from raw IPMI data.
Reads /tmp/ipmi_raw_{host_id}.txt (B10) — the per-host file the
collector actually writes, same as the /grouped endpoint.
"""
raw_file = PathLib(f"/tmp/ipmi_raw_{host_id}.txt")
if not raw_file.exists():
raise HTTPException(
404, f"Raw IPMI data file not found for host {host_id} — no collection yet?"
)
try:
raw_data = raw_file.read_text(encoding="utf-8")
components = parse_component_presence(raw_data)
# Add summary statistics
summary = {
"total_components": sum(c["total"] for c in components.values() if isinstance(c, dict) and "total" in c),
"present_components": sum(c["present"] for c in components.values() if isinstance(c, dict) and "present" in c),
"total_power": components.get("total_power", 0),
}
return {
"host_id": host_id,
"components": components,
"summary": summary,
}
except Exception as exc:
logger.error("Failed to parse component presence for host %s: %s", host_id, exc)
raise HTTPException(500, f"Failed to parse component presence: {exc}")
@router.get("/components/{host_id}/grouped")
async def component_presence_grouped(
host_id: int,
current_user: UserModel = Depends(get_current_user)
):
"""Get component presence status grouped by present/absent for display."""
raw_file = PathLib(f"/tmp/ipmi_raw_{host_id}.txt")
if not raw_file.exists():
return {
"host_id": host_id,
"error": f"Raw IPMI data file not found for host {host_id}",
"grouped": None,
}
try:
raw_data = raw_file.read_text(encoding="utf-8")
logger.info(f"Raw IPMI data loaded, {len(raw_data)} bytes for host {host_id}")
# Debug: Check if DIMM_T sensors are present
if "_DIMM_T" in raw_data:
logger.info(f"Found _DIMM_T in raw data")
else:
logger.warning(f"_DIMM_T NOT found in raw data")
components = parse_component_presence(raw_data)
logger.info(f"Parsed components: CPU={components['cpu']['present']}/{components['cpu']['total']}, "
f"Memory={components['memory']['present']}/{components['memory']['total']}, "
f"PCIe={components['pcie']['present']}/{components['pcie']['total']}, "
f"Drive={components['drive']['present']}/{components['drive']['total']}, "
f"Fan={components['fan']['present']}/{components['fan']['total']}, "
f"PSU={components['power_supply']['present']}/{components['power_supply']['total']}, "
f"Power={components['total_power']}W")
# Debug: Log memory items
if components['memory']['items']:
logger.info(f"Memory items: {[item['name'] for item in components['memory']['items']]}")
else:
logger.warning(f"Memory items list is empty")
grouped = group_components_by_status(components)
# Build by_category with full component items and stats
by_category = {}
for category in ["cpu", "memory", "pcie", "drive", "fan", "power_supply"]:
cat_data = components.get(category, {})
by_category[category] = {
"total": cat_data.get("total", 0),
"present": cat_data.get("present", 0),
"items": cat_data.get("items", []),
"percentage": 0,
}
if cat_data.get("total", 0) > 0:
by_category[category]["percentage"] = round(
cat_data.get("present", 0) / cat_data.get("total", 1) * 100, 1
)
return {
"host_id": host_id,
"grouped": grouped,
"by_category": by_category,
"total_power": components.get("total_power", 0),
}
except Exception as exc:
logger.error("Failed to parse grouped component presence for host %s: %s", host_id, exc)
return {
"host_id": host_id,
"error": str(exc),
"grouped": None,
}