feat: host heartbeat status and dashboard metrics (2.2)

Track agent.heartbeat and report.daily.ssh per host; show online/stale
on Hosts UI; dashboard counters and SAC_HEARTBEAT_STALE_MINUTES config.
This commit is contained in:
PTah
2026-05-27 13:36:17 +10:00
parent 9ba3c4b63b
commit 04301ab359
14 changed files with 246 additions and 4 deletions
+20
View File
@@ -6,9 +6,11 @@ from sqlalchemy import func, select
from sqlalchemy.orm import Session, joinedload
from app.auth.jwt_auth import get_current_user
from app.config import get_settings
from app.database import get_db
from app.models import Event, Host, Problem
from app.schemas.list_models import EventSummary
from app.services.host_health import DAILY_REPORT_SSH, HEARTBEAT_TYPE, count_stale_hosts
router = APIRouter(prefix="/dashboards", tags=["dashboards"])
@@ -16,6 +18,9 @@ router = APIRouter(prefix="/dashboards", tags=["dashboards"])
class DashboardSummary(BaseModel):
events_last_24h: int
hosts_total: int
hosts_stale: int
heartbeats_24h: int
daily_reports_24h: int
problems_open: int
severity_24h: dict[str, int]
recent_events: list[EventSummary]
@@ -32,6 +37,18 @@ def dashboard_summary(
select(func.count()).select_from(Event).where(Event.received_at >= since)
) or 0
hosts_total = db.scalar(select(func.count()).select_from(Host)) or 0
settings = get_settings()
hosts_stale = count_stale_hosts(db, settings.sac_heartbeat_stale_minutes)
heartbeats_24h = db.scalar(
select(func.count())
.select_from(Event)
.where(Event.received_at >= since, Event.type == HEARTBEAT_TYPE)
) or 0
daily_reports_24h = db.scalar(
select(func.count())
.select_from(Event)
.where(Event.received_at >= since, Event.type == DAILY_REPORT_SSH)
) or 0
problems_open = (
db.scalar(select(func.count()).select_from(Problem).where(Problem.status == "open")) or 0
)
@@ -71,6 +88,9 @@ def dashboard_summary(
return DashboardSummary(
events_last_24h=events_24h,
hosts_total=hosts_total,
hosts_stale=hosts_stale,
heartbeats_24h=heartbeats_24h,
daily_reports_24h=daily_reports_24h,
problems_open=problems_open,
severity_24h=severity_24h,
recent_events=recent_events,
+17
View File
@@ -3,9 +3,16 @@ from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app.auth.jwt_auth import get_current_user
from app.config import get_settings
from app.database import get_db
from app.models import Event, Host
from app.schemas.list_models import HostListResponse, HostSummary
from app.services.host_health import (
DAILY_REPORT_SSH,
HEARTBEAT_TYPE,
agent_status,
max_event_time_by_host,
)
router = APIRouter(prefix="/hosts", tags=["hosts"])
@@ -37,11 +44,16 @@ def list_hosts(
.limit(page_size)
).all()
settings = get_settings()
hb_map = max_event_time_by_host(db, HEARTBEAT_TYPE)
report_map = max_event_time_by_host(db, DAILY_REPORT_SSH)
items: list[HostSummary] = []
for host in rows:
event_count = db.scalar(
select(func.count()).select_from(Event).where(Event.host_id == host.id)
)
last_hb = hb_map.get(host.id)
items.append(
HostSummary(
id=host.id,
@@ -53,6 +65,11 @@ def list_hosts(
ipv4=host.ipv4,
last_seen_at=host.last_seen_at,
event_count=int(event_count or 0),
last_heartbeat_at=last_hb,
last_daily_report_at=report_map.get(host.id),
agent_status=agent_status(
last_hb, stale_minutes=settings.sac_heartbeat_stale_minutes
),
)
)
+16
View File
@@ -10,6 +10,8 @@ from sqlalchemy.orm import Session
from app.auth.jwt_auth import verify_access_token
from app.database import SessionLocal
from app.models import Event, Problem
from app.config import get_settings
from app.services.host_health import DAILY_REPORT_SSH, HEARTBEAT_TYPE, count_stale_hosts
from app.version import APP_VERSION
router = APIRouter(prefix="/stream", tags=["stream"])
@@ -26,10 +28,24 @@ def _dashboard_snapshot(db: Session) -> dict:
db.scalar(select(func.count()).select_from(Problem).where(Problem.status == "open")) or 0
)
last_event = db.scalar(select(Event.id).order_by(Event.received_at.desc()).limit(1))
settings = get_settings()
return {
"type": "dashboard",
"version": APP_VERSION,
"events_last_24h": events_24h,
"hosts_stale": count_stale_hosts(db, settings.sac_heartbeat_stale_minutes),
"heartbeats_24h": db.scalar(
select(func.count())
.select_from(Event)
.where(Event.received_at >= since, Event.type == HEARTBEAT_TYPE)
)
or 0,
"daily_reports_24h": db.scalar(
select(func.count())
.select_from(Event)
.where(Event.received_at >= since, Event.type == DAILY_REPORT_SSH)
)
or 0,
"problems_open": problems_open,
"last_event_id": last_event,
"at": datetime.now(timezone.utc).isoformat(),
+3
View File
@@ -50,6 +50,9 @@ class Settings(BaseSettings):
telegram_chat_id: str = ""
telegram_min_severity: str = "high"
# Порог «живости» агента: agent.heartbeat раз в ~12 ч (ssh-monitor)
sac_heartbeat_stale_minutes: int = 780
@lru_cache
def get_settings() -> Settings:
+3
View File
@@ -13,6 +13,9 @@ class HostSummary(BaseModel):
ipv4: str | None
last_seen_at: datetime
event_count: int | None = None
last_heartbeat_at: datetime | None = None
last_daily_report_at: datetime | None = None
agent_status: str = "unknown"
model_config = {"from_attributes": True}
+53
View File
@@ -0,0 +1,53 @@
"""Статус агентов по событиям agent.heartbeat / report.daily.* в БД."""
from datetime import datetime, timezone
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from app.models import Event, Host
HEARTBEAT_TYPE = "agent.heartbeat"
DAILY_REPORT_SSH = "report.daily.ssh"
def max_event_time_by_host(db: Session, event_type: str) -> dict[int, datetime]:
rows = db.execute(
select(Event.host_id, func.max(Event.received_at))
.where(Event.type == event_type)
.group_by(Event.host_id)
).all()
return {int(host_id): ts for host_id, ts in rows if ts is not None}
def agent_status(
last_heartbeat_at: datetime | None,
*,
stale_minutes: int,
now: datetime | None = None,
) -> str:
"""
online — heartbeat не старше порога;
stale — был хост, но heartbeat устарел или отсутствует;
unknown — heartbeat ещё не приходил.
"""
if last_heartbeat_at is None:
return "unknown"
ref = now or datetime.now(timezone.utc)
hb = last_heartbeat_at
if hb.tzinfo is None:
hb = hb.replace(tzinfo=timezone.utc)
age_sec = (ref - hb).total_seconds()
if age_sec > stale_minutes * 60:
return "stale"
return "online"
def count_stale_hosts(db: Session, stale_minutes: int) -> int:
hb_map = max_event_time_by_host(db, HEARTBEAT_TYPE)
host_ids = db.scalars(select(Host.id)).all()
return sum(
1
for host_id in host_ids
if agent_status(hb_map.get(host_id), stale_minutes=stale_minutes) == "stale"
)