feat: live последние события на Dashboard; retention, health, runbook

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-05-28 11:29:31 +10:00
parent 9ed0670863
commit 108e1756c4
14 changed files with 625 additions and 249 deletions
@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta, timezone

-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, Query
 from pydantic import BaseModel
 from sqlalchemy import func, select
 from sqlalchemy.orm import Session, joinedload
@@ -15,6 +15,33 @@ from app.services.host_health import DAILY_REPORT_SSH, HEARTBEAT_TYPE, count_sta
 router = APIRouter(prefix="/dashboards", tags=["dashboards"])


+def fetch_recent_events(db: Session, *, limit: int = 8) -> list[EventSummary]:
+    """Последние события по времени приёма ingest (received_at)."""
+    rows = db.scalars(
+        select(Event)
+        .join(Host)
+        .options(joinedload(Event.host))
+        .order_by(Event.received_at.desc())
+        .limit(limit)
+    ).all()
+    return [
+        EventSummary(
+            id=e.id,
+            event_id=e.event_id,
+            host_id=e.host_id,
+            hostname=e.host.hostname,
+            occurred_at=e.occurred_at,
+            received_at=e.received_at,
+            category=e.category,
+            type=e.type,
+            severity=e.severity,
+            title=e.title,
+            summary=e.summary,
+        )
+        for e in rows
+    ]
+
+
 class TopHostItem(BaseModel):
    host_id: int
    hostname: str
@@ -109,30 +136,7 @@ def dashboard_summary(
    ).all()
    severity_24h = {row[0]: row[1] for row in severity_rows}

-    recent = db.scalars(
-        select(Event)
-        .join(Host)
-        .options(joinedload(Event.host))
-        .order_by(Event.received_at.desc())
-        .limit(8)
-    ).all()
-
-    recent_events = [
-        EventSummary(
-            id=e.id,
-            event_id=e.event_id,
-            host_id=e.host_id,
-            hostname=e.host.hostname,
-            occurred_at=e.occurred_at,
-            received_at=e.received_at,
-            category=e.category,
-            type=e.type,
-            severity=e.severity,
-            title=e.title,
-            summary=e.summary,
-        )
-        for e in recent
-    ]
+    recent_events = fetch_recent_events(db, limit=8)

    return DashboardSummary(
        events_last_24h=events_24h,
@@ -148,3 +152,12 @@ def dashboard_summary(
        top_event_types=top_event_types,
        recent_events=recent_events,
    )
+
+
+@router.get("/recent-events", response_model=list[EventSummary])
+def dashboard_recent_events(
+    limit: int = Query(8, ge=1, le=50),
+    db: Session = Depends(get_db),
+    _user: str = Depends(get_current_user),
+) -> list[EventSummary]:
+    return fetch_recent_events(db, limit=limit)
@@ -1,8 +1,11 @@
 from fastapi import APIRouter, Depends
-from sqlalchemy import text
+from sqlalchemy import func, select, text
 from sqlalchemy.orm import Session

+from app.config import get_settings
 from app.database import get_db
+from app.models import Event
+from app.services.host_health import count_stale_hosts
 from app.version import APP_NAME, APP_VERSION

 router = APIRouter(tags=["health"])
@@ -14,12 +17,28 @@ def build_health_payload(db: Session) -> dict:
        db.execute(text("SELECT 1"))
    except Exception:
        db_status = "error"
+
+    hosts_stale = 0
+    last_event_received_at: str | None = None
+    if db_status == "ok":
+        settings = get_settings()
+        hosts_stale = count_stale_hosts(db, settings.sac_heartbeat_stale_minutes)
+        last_received = db.scalar(select(func.max(Event.received_at)))
+        if last_received is not None:
+            last_event_received_at = last_received.isoformat()
+
+    overall = "ok" if db_status == "ok" else "degraded"
+    if db_status == "ok" and hosts_stale > 0:
+        overall = "degraded"
+
    return {
-        "status": "ok" if db_status == "ok" else "degraded",
+        "status": overall,
        "database": db_status,
        "service": "security-alert-center",
        "name": APP_NAME,
        "version": APP_VERSION,
+        "hosts_stale": hosts_stale,
+        "last_event_received_at": last_event_received_at,
    }


@@ -64,6 +64,10 @@ class Settings(BaseSettings):
    sac_privilege_spike_window_minutes: int = 10
    sac_privilege_spike_threshold: int = 10

+    # Retention (app.jobs.retention / systemd timer)
+    sac_events_retention_days: int = 90
+    sac_problems_retention_days: int = 180
+

@lru_cache
 def get_settings() -> Settings:
@@ -0,0 +1 @@
+"""Фоновые задачи SAC (retention и др.)."""
@@ -0,0 +1,36 @@
+"""CLI: python -m app.jobs.retention"""
+
+from __future__ import annotations
+
+import logging
+import sys
+
+from app.config import get_settings
+from app.database import SessionLocal
+from app.services.retention import purge_old_data
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+logger = logging.getLogger("sac.retention")
+
+
+def main() -> int:
+    settings = get_settings()
+    db = SessionLocal()
+    try:
+        stats = purge_old_data(db, settings)
+        logger.info(
+            "retention done events_deleted=%s problems_deleted=%s",
+            stats["events_deleted"],
+            stats["problems_deleted"],
+        )
+        return 0
+    except Exception:
+        db.rollback()
+        logger.exception("retention failed")
+        return 1
+    finally:
+        db.close()
+
+
+if __name__ == "__main__":
+    sys.exit(main())
@@ -0,0 +1,39 @@
+"""Удаление устаревших events и resolved problems."""
+
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import delete
+from sqlalchemy.orm import Session
+
+from app.config import Settings
+from app.models import Event, Problem
+
+
+def purge_old_data(db: Session, settings: Settings) -> dict:
+    """
+    Events: received_at старше sac_events_retention_days.
+    Problems: status=resolved и updated_at старше sac_problems_retention_days.
+    """
+    now = datetime.now(timezone.utc)
+    events_cutoff = now - timedelta(days=settings.sac_events_retention_days)
+    problems_cutoff = now - timedelta(days=settings.sac_problems_retention_days)
+
+    events_result = db.execute(delete(Event).where(Event.received_at < events_cutoff))
+    events_deleted = events_result.rowcount or 0
+
+    problems_result = db.execute(
+        delete(Problem).where(
+            Problem.status == "resolved",
+            Problem.updated_at < problems_cutoff,
+        )
+    )
+    problems_deleted = problems_result.rowcount or 0
+
+    db.commit()
+
+    return {
+        "events_deleted": events_deleted,
+        "problems_deleted": problems_deleted,
+        "events_cutoff": events_cutoff.isoformat(),
+        "problems_cutoff": problems_cutoff.isoformat(),
+    }
@@ -90,3 +90,19 @@ def test_dashboard_summary_top_and_problems_24h(client, db_session, jwt_headers)
    types = {row["type"]: row["count"] for row in body["top_event_types"]}
    assert types["ssh.login.failed"] == 3
    assert types["agent.heartbeat"] == 2
+
+
+def test_dashboard_recent_events_ordered_by_received_at(client, db_session, jwt_headers):
+    h = _host(db_session, "live-host")
+    now = datetime.now(timezone.utc)
+    older = _event(db_session, h, "agent.lifecycle", received_at=now - timedelta(minutes=5))
+    newer = _event(db_session, h, "agent.heartbeat", received_at=now)
+    db_session.commit()
+
+    r = client.get("/api/v1/dashboards/recent-events?limit=8", headers=jwt_headers)
+    assert r.status_code == 200
+    rows = r.json()
+    assert len(rows) >= 2
+    assert rows[0]["id"] == newer.id
+    assert rows[0]["type"] == "agent.heartbeat"
+    assert rows[1]["id"] == older.id
@@ -0,0 +1,78 @@
+"""Retention purge service."""
+
+import uuid
+from datetime import datetime, timedelta, timezone
+
+from app.config import Settings
+from app.models import Event, Host, Problem
+from app.services.retention import purge_old_data
+
+
+def _host(db, name: str) -> Host:
+    h = Host(
+        hostname=name,
+        os_family="linux",
+        product="ssh-monitor",
+        last_seen_at=datetime.now(timezone.utc),
+    )
+    db.add(h)
+    db.flush()
+    return h
+
+
+def test_purge_old_events_and_resolved_problems(db_session):
+    h = _host(db_session, "retention-host")
+    now = datetime.now(timezone.utc)
+    old_event = now - timedelta(days=120)
+    old_problem = now - timedelta(days=200)
+    _ev = Event(
+        event_id=str(uuid.uuid4()),
+        host_id=h.id,
+        occurred_at=old_event,
+        received_at=old_event,
+        category="agent",
+        type="agent.heartbeat",
+        severity="info",
+        title="old",
+        summary="s",
+        payload={},
+    )
+    db_session.add(_ev)
+    db_session.add(
+        Problem(
+            host_id=h.id,
+            title="resolved old",
+            summary="s",
+            severity="info",
+            status="resolved",
+            rule_id="rule:test",
+            fingerprint="fp-ret-old",
+            event_count=1,
+            last_seen_at=old_problem,
+            created_at=old_problem,
+            updated_at=old_problem,
+        )
+    )
+    db_session.add(
+        Problem(
+            host_id=h.id,
+            title="open stays",
+            summary="s",
+            severity="high",
+            status="open",
+            rule_id="rule:test",
+            fingerprint="fp-ret-open",
+            event_count=1,
+            last_seen_at=now,
+            created_at=now,
+            updated_at=now,
+        )
+    )
+    db_session.commit()
+
+    settings = Settings(sac_events_retention_days=90, sac_problems_retention_days=180)
+    stats = purge_old_data(db_session, settings)
+    assert stats["events_deleted"] == 1
+    assert stats["problems_deleted"] == 1
+    assert db_session.query(Event).count() == 0
+    assert db_session.query(Problem).filter(Problem.status == "open").count() == 1
@@ -37,4 +37,8 @@ SAC_BRUTE_FORCE_THRESHOLD=30
 SAC_PRIVILEGE_SPIKE_WINDOW_MINUTES=10
 SAC_PRIVILEGE_SPIKE_THRESHOLD=10

+# Retention (systemd sac-retention.timer)
+SAC_EVENTS_RETENTION_DAYS=90
+SAC_PROBLEMS_RETENTION_DAYS=180
+
 CORS_ORIGINS=*
@@ -0,0 +1,16 @@
+[Unit]
+Description=Security Alert Center data retention
+After=network-online.target postgresql.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+User=sac
+Group=sac
+WorkingDirectory=/opt/security-alert-center/backend
+EnvironmentFile=/opt/security-alert-center/config/sac-api.env
+ExecStart=/opt/security-alert-center/backend/.venv/bin/python -m app.jobs.retention
+Nice=10
+
+[Install]
+WantedBy=multi-user.target
@@ -0,0 +1,10 @@
+[Unit]
+Description=Daily SAC retention purge
+
+[Timer]
+OnCalendar=daily
+RandomizedDelaySec=900
+Persistent=true
+
+[Install]
+WantedBy=timers.target
@@ -0,0 +1,79 @@
+# Runbook: эксплуатация SAC
+
+Краткие процедуры для prod (`sac.kalinamall.ru`, native install в `/opt/security-alert-center`).
+
+## Деплой приложения
+
+```bash
+sudo /opt/sac-deploy.sh
+```
+
+Скрипт: `git pull` / `reset --hard`, `pip install`, `alembic upgrade head`, `npm run build`, `systemctl restart sac-api`.
+
+Проверка:
+
+```bash
+curl -sS https://sac.kalinamall.ru/health | jq .
+```
+
+Ожидается `status: ok`, `database: ok`. При устаревших heartbeat агентов — `status: degraded`, поле `hosts_stale` > 0.
+
+## Резервное копирование PostgreSQL
+
+```bash
+sudo -u postgres pg_dump -Fc sac > /var/backups/sac-$(date +%Y%m%d).dump
+```
+
+Восстановление (на тестовый инстанс):
+
+```bash
+sudo -u postgres pg_restore -d sac_test --clean /var/backups/sac-YYYYMMDD.dump
+```
+
+Перед restore на prod — остановить API: `sudo systemctl stop sac-api`.
+
+## Retention (очистка БД)
+
+Политика по умолчанию (`config/sac-api.env`):
+
+| Данные | Срок |
+|--------|------|
+| `events` | 90 дней (`SAC_EVENTS_RETENTION_DAYS`) |
+| `problems` со статусом `resolved` | 180 дней (`SAC_PROBLEMS_RETENTION_DAYS`) |
+
+Ручной прогон:
+
+```bash
+sudo -u sac bash -c '
+  set -a; source /opt/security-alert-center/config/sac-api.env; set +a
+  cd /opt/security-alert-center/backend
+  .venv/bin/python -m app.jobs.retention
+'
+```
+
+Установка ежедневного timer (один раз):
+
+```bash
+sudo cp /opt/security-alert-center/deploy/systemd/sac-retention.service /etc/systemd/system/
+sudo cp /opt/security-alert-center/deploy/systemd/sac-retention.timer /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl enable --now sac-retention.timer
+systemctl list-timers sac-retention.timer
+```
+
+## Health и мониторинг
+
+| Endpoint | Назначение |
+|----------|------------|
+| `GET /health` | БД, версия, `hosts_stale`, `last_event_received_at` |
+| `GET /api/v1/stream/events` | SSE для UI (счётчики + `last_event_id` каждые 5 с) |
+
+Агенты: `GET /health` без JWT. UI Dashboard: блок «Последние события» подгружается при смене `last_event_id` в SSE.
+
+## Типовые проблемы
+
+**События в Telegram есть, в UI нет** — проверить ingest (лог агента `SAC: accepted`), режим `UseSAC`, обновить Dashboard (live ~5 с).
+
+**422 на ingest** — см. `Logs/sac-last-post.json` на Windows-агенте; обновить `Sac-Client.ps1` (RDP) или `sac-client.sh` (Linux).
+
+**Устаревший heartbeat** — увеличить `SAC_HEARTBEAT_STALE_MINUTES` или проверить `agent.heartbeat` на хосте.
@@ -89,7 +89,7 @@

 - [x] `d2-1` UI Problems: список, фильтры, `Ack/Resolve`, карточка с таймлайном
 - [x] `d2-2` Dashboard: top hosts/types, `open vs resolved 24h`, drill-down
- [ ] `d2-3` Ops: retention, health checks, runbook backup/restore/deploy
+- [x] `d2-3` Ops: retention, health checks, runbook backup/restore/deploy
 - [ ] `dod` DoD: нет дублей `event_id`, Problems e2e, 3 правила, UI MVP, docs, push в kalinamall

 ### Отображение хостов (`display_name`, как `SERVER_DISPLAY_NAME` у ssh)
@@ -120,9 +120,9 @@
 - [x] `11:00–12:00` карточка проблемы + таймлайн событий
 - [x] `13:00–14:30` Dashboard: виджеты top hosts/types
 - [x] `14:30–16:00` Dashboard: `open/resolved 24h` + drill-down
- [ ] `16:00–17:00` retention job (`events 30–90d`, `problems 180d+`)
+- [x] `16:00–17:00` retention job (`events 30–90d`, `problems 180d+`)
 - [ ] `17:00–18:00` health checks (DB/worker/heartbeat stale)
- [ ] `18:00–19:00` runbook + финальный push в kalinamall + freeze dev
+- [x] `18:00–19:00` runbook + финальный push в kalinamall + freeze dev

 ### Неделя после freeze (только тестирование)

@@ -105,11 +105,14 @@
          <thead>

            <tr>
+
+              <th>Хост</th>
+
              <th>Событий</th>

            </tr>

-          <th>Время</th>
+          </thead>

          <tbody>

@@ -120,7 +123,7 @@
                <RouterLink :to="{ path: '/events', query: { hostname: h.hostname } }">

                  {{ h.hostname }}
-          <td>{{ formatDt(e.occurred_at) }}</td>
+
                </RouterLink>

              </td>
@@ -132,7 +135,7 @@
          </tbody>

        </table>
-      <button type="button" class="secondary" @click="load">Обновить</button>
+
        <p v-else class="muted">Нет событий за 24 ч</p>

      </section>
@@ -140,23 +143,47 @@


      <section class="dash-panel">
-import { apiFetch, getToken, type DashboardSummary } from "../api";
+
        <h2>Top типов (24 ч)</h2>

        <table v-if="data.top_event_types.length" class="dash-table">

          <thead>
+
            <tr>
+
              <th>Type</th>

              <th>Событий</th>

            </tr>
+
+          </thead>
+
+          <tbody>
+
+            <tr v-for="t in data.top_event_types" :key="t.type">
+
+              <td>
+
+                <RouterLink :to="{ path: '/events', query: { type: t.type } }">
+
+                  <code>{{ t.type }}</code>
+
+                </RouterLink>
+
+              </td>
+
+              <td>{{ t.count }}</td>
+
+            </tr>
+
          </tbody>

        </table>

        <p v-else class="muted">Нет событий за 24 ч</p>
+
      </section>

    </div>
@@ -164,6 +191,45 @@ async function load() {


    <h2>Severity за 24 ч</h2>
+
+    <ul class="severity-list">
+
+      <li v-for="(count, sev) in data.severity_24h" :key="sev">
+
+        <RouterLink :to="{ path: '/events', query: { severity: sev } }">
+
+          <span :class="'sev-' + sev">{{ sev }}</span>: {{ count }}
+
+        </RouterLink>
+
+      </li>
+
+      <li v-if="!Object.keys(data.severity_24h).length">Нет событий</li>
+
+    </ul>
+
+
+
+    <h2>Последние события</h2>
+
+    <p class="muted dash-recent-hint">
+
+      Обновляется при поступлении событий (live, ~5 с). Время — момент приёма SAC.
+
+    </p>
+
+    <table>
+
+      <thead>
+
+        <tr>
+
+          <th>ID</th>
+
+          <th>Получено</th>
+
+          <th>Хост</th>
+
          <th>Severity</th>

          <th>Title</th>
@@ -182,23 +248,10 @@ function connectLive() {

          </td>

+          <td>{{ formatDt(e.received_at) }}</td>

          <td>
-      if (typeof msg.events_last_24h === "number") {
-        data.value.events_last_24h = msg.events_last_24h;
-      }
-      if (typeof msg.problems_open === "number") {
-        data.value.problems_open = msg.problems_open;
-      }
-      if (typeof msg.hosts_stale === "number") {
-        data.value.hosts_stale = msg.hosts_stale;
-      }
-      if (typeof msg.heartbeats_24h === "number") {
-        data.value.heartbeats_24h = msg.heartbeats_24h;
-      }
-      if (typeof msg.daily_reports_24h === "number") {
-        data.value.daily_reports_24h = msg.daily_reports_24h;
-      }
+
            <RouterLink :to="{ path: '/events', query: { hostname: e.hostname } }">

              {{ e.hostname }}
@@ -217,3 +270,11 @@ onUnmounted(() => {

    </table>

+    <div class="filters" style="margin-top: 1rem">
+
+      <button type="button" class="secondary" @click="load">Обновить всё</button>
+
+      <span v-if="live" class="live-badge">live</span>
+
+    </div>
+
				`@@ -0,0 +1 @@`
				`"""Фоновые задачи SAC (retention и др.)."""`