From b148b558c32af63aadaad5b98811c7379ff1124d Mon Sep 17 00:00:00 2001 From: PTah Date: Wed, 8 Jul 2026 13:16:13 +1000 Subject: [PATCH] =?UTF-8?q?feat:=20ingest=20burst=20hardening=20=E2=80=94?= =?UTF-8?q?=20defer=20lifecycle/auth,=20workers,=20nginx=20(v0.5.5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Defer notify_lifecycle and notify_auth_login after commit; SAC_UVICORN_WORKERS via sac-api-start.sh; nginx /api/v1/events timeout; mass-update confirm in Hosts UI. --- backend/app/api/v1/events.py | 12 +++++-- backend/app/config.py | 1 + backend/app/services/notify_dispatch.py | 20 +++++++++-- backend/app/version.py | 2 +- backend/tests/test_ingest.py | 45 +++++++++++++++++++++++++ deploy/env.native.example | 2 ++ deploy/nginx/sac.conf.example | 13 +++++++ deploy/nginx/sac.conf.tls.example | 13 +++++++ deploy/sac-deploy.sh | 4 +++ deploy/systemd/sac-api-start.sh | 42 +++++++++++++++++++++++ deploy/systemd/sac-api.service | 2 +- docs/deployment.md | 2 +- docs/ingest-mass-update-backlog.ru.md | 8 ++--- frontend/src/views/HostsView.vue | 35 +++++++++++++++++++ 14 files changed, 189 insertions(+), 12 deletions(-) create mode 100644 deploy/systemd/sac-api-start.sh diff --git a/backend/app/api/v1/events.py b/backend/app/api/v1/events.py index 7afef5c..2114686 100644 --- a/backend/app/api/v1/events.py +++ b/backend/app/api/v1/events.py @@ -52,7 +52,9 @@ from app.services.notify_dispatch import ( notify_lifecycle, notify_problem, notify_rdg_connection, + schedule_notify_auth_login, schedule_notify_daily_report, + schedule_notify_lifecycle, ) router = APIRouter(prefix="/events", tags=["events"]) @@ -100,15 +102,17 @@ def post_event( problem = None problem_created = False deferred_daily_report_id: int | None = None + deferred_lifecycle_id: int | None = None + deferred_auth_login_id: int | None = None if created: problem, problem_created = maybe_create_problem(db, event) maybe_auto_disconnect_stuck_rdp_session(db, event) if event.type in DAILY_REPORT_EVENT_TYPES: deferred_daily_report_id = event.id elif event.type == LIFECYCLE_EVENT_TYPE: - notify_lifecycle(event, db=db) + deferred_lifecycle_id = event.id elif event.type in AUTH_LOGIN_SUCCESS_TYPES: - notify_auth_login(event, db=db) + deferred_auth_login_id = event.id elif event.type in RDG_CONNECTION_TYPES: notify_rdg_connection(event, db=db) else: @@ -122,6 +126,10 @@ def post_event( if deferred_daily_report_id is not None: background_tasks.add_task(schedule_notify_daily_report, deferred_daily_report_id) + if deferred_lifecycle_id is not None: + background_tasks.add_task(schedule_notify_lifecycle, deferred_lifecycle_id) + if deferred_auth_login_id is not None: + background_tasks.add_task(schedule_notify_auth_login, deferred_auth_login_id) body = _ingest_response(event, created=created) if problem is not None: diff --git a/backend/app/config.py b/backend/app/config.py index 3f657ef..272f515 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -36,6 +36,7 @@ class Settings(BaseSettings): database_url: str = "postgresql+psycopg2://sac:sac@localhost:5432/sac" sac_db_pool_size: int = 15 sac_db_max_overflow: int = 25 + sac_uvicorn_workers: int = 4 sac_public_url: str = "http://localhost:8000" # URL для скачивания RDP bundle с ПК (WinRM). По умолчанию = SAC_PUBLIC_URL. sac_agent_bundle_base_url: str = "" diff --git a/backend/app/services/notify_dispatch.py b/backend/app/services/notify_dispatch.py index b13cbe0..10ef463 100644 --- a/backend/app/services/notify_dispatch.py +++ b/backend/app/services/notify_dispatch.py @@ -216,16 +216,30 @@ def notify_rdg_connection(event: Event, *, db: Session | None = None) -> None: def schedule_notify_daily_report(event_db_id: int) -> None: """Отложенное оповещение по суточному отчёту (после commit ingest, вне горячего POST).""" + _schedule_deferred_event_notify(event_db_id, notify_daily_report, label="daily report") + + +def schedule_notify_lifecycle(event_db_id: int) -> None: + """Отложенное lifecycle-оповещение (после commit ingest).""" + _schedule_deferred_event_notify(event_db_id, notify_lifecycle, label="lifecycle") + + +def schedule_notify_auth_login(event_db_id: int) -> None: + """Отложенное оповещение об успешном RDP/SSH входе (после commit ingest).""" + _schedule_deferred_event_notify(event_db_id, notify_auth_login, label="auth login") + + +def _schedule_deferred_event_notify(event_db_id: int, handler, *, label: str) -> None: from app.database import SessionLocal db = SessionLocal() try: event = db.get(Event, event_db_id) if event is None: - logger.warning("deferred daily report notify: event id=%s not found", event_db_id) + logger.warning("deferred %s notify: event id=%s not found", label, event_db_id) return - notify_daily_report(event, db=db) + handler(event, db=db) except Exception: - logger.exception("deferred daily report notify failed event_db_id=%s", event_db_id) + logger.exception("deferred %s notify failed event_db_id=%s", label, event_db_id) finally: db.close() diff --git a/backend/app/version.py b/backend/app/version.py index ece43fc..2cf26fc 100644 --- a/backend/app/version.py +++ b/backend/app/version.py @@ -1,5 +1,5 @@ """Единый источник версии SAC (API, health, логи, OpenAPI).""" APP_NAME = "Security Alert Center" -APP_VERSION = "0.5.4" +APP_VERSION = "0.5.5" APP_VERSION_LABEL = f"{APP_NAME} v.{APP_VERSION}" diff --git a/backend/tests/test_ingest.py b/backend/tests/test_ingest.py index 99dfed2..9b5d5fc 100644 --- a/backend/tests/test_ingest.py +++ b/backend/tests/test_ingest.py @@ -133,3 +133,48 @@ def test_ingest_daily_report_calls_notify_daily_report(client, auth_headers): mock_daily.assert_called_once() mock_event.assert_not_called() + +def test_ingest_lifecycle_defers_notify(client, auth_headers): + from unittest.mock import patch + + from app.api.v1 import events as events_api + + event_id = str(uuid.uuid4()) + payload = { + **VALID_EVENT, + "event_id": event_id, + "type": "agent.lifecycle", + "title": "started", + "summary": "agent started", + "details": {"lifecycle": "started"}, + } + with patch.object(events_api, "schedule_notify_lifecycle") as mock_defer: + with patch.object(events_api, "notify_lifecycle") as mock_sync: + r = client.post("/api/v1/events", json=payload, headers=auth_headers) + assert r.status_code == 201 + mock_defer.assert_called_once() + mock_sync.assert_not_called() + + +def test_ingest_auth_login_defers_notify(client, auth_headers): + from unittest.mock import patch + + from app.api.v1 import events as events_api + + event_id = str(uuid.uuid4()) + payload = { + **VALID_EVENT, + "event_id": event_id, + "category": "auth", + "type": "ssh.login.success", + "severity": "info", + "title": "SSH login", + "summary": "user@host", + } + with patch.object(events_api, "schedule_notify_auth_login") as mock_defer: + with patch.object(events_api, "notify_auth_login") as mock_sync: + r = client.post("/api/v1/events", json=payload, headers=auth_headers) + assert r.status_code == 201 + mock_defer.assert_called_once() + mock_sync.assert_not_called() + diff --git a/deploy/env.native.example b/deploy/env.native.example index 51ca8e6..36079b2 100644 --- a/deploy/env.native.example +++ b/deploy/env.native.example @@ -7,6 +7,8 @@ DATABASE_URL=postgresql+psycopg2://sac:CHANGE_ME_POSTGRES_PASSWORD@127.0.0.1:543 # SQLAlchemy pool (uvicorn workers × concurrent ingest). Было по умолчанию 5+10 — мало для штурма 09:00. SAC_DB_POOL_SIZE=15 SAC_DB_MAX_OVERFLOW=25 +# Uvicorn workers (ingest burst). Читает deploy/systemd/sac-api-start.sh; при >1 отключите in-process host_silence scan ниже. +SAC_UVICORN_WORKERS=4 SAC_PUBLIC_URL=https://sac.kalinamall.ru # Опционально: другой базовый URL для WinRM-скачивания RDP bundle с ПК (LAN / split-DNS). diff --git a/deploy/nginx/sac.conf.example b/deploy/nginx/sac.conf.example index 7e60ce1..9a85086 100644 --- a/deploy/nginx/sac.conf.example +++ b/deploy/nginx/sac.conf.example @@ -44,6 +44,19 @@ server { proxy_read_timeout 960s; } + # Ingest burst: lifecycle/auth deferred в API, но POST всё равно может ждать notify_event/problem. + location = /api/v1/events { + proxy_pass http://sac_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 30s; + proxy_send_timeout 120s; + proxy_read_timeout 120s; + } + location / { proxy_pass http://sac_api; proxy_http_version 1.1; diff --git a/deploy/nginx/sac.conf.tls.example b/deploy/nginx/sac.conf.tls.example index 01906b8..6ff6e66 100644 --- a/deploy/nginx/sac.conf.tls.example +++ b/deploy/nginx/sac.conf.tls.example @@ -69,6 +69,19 @@ server { proxy_read_timeout 960s; } + # Ingest burst: lifecycle/auth deferred в API, но POST всё равно может ждать notify_event/problem. + location = /api/v1/events { + proxy_pass http://sac_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 30s; + proxy_send_timeout 120s; + proxy_read_timeout 120s; + } + location / { proxy_pass http://sac_api; proxy_http_version 1.1; diff --git a/deploy/sac-deploy.sh b/deploy/sac-deploy.sh index 9e1e0b9..56db75c 100644 --- a/deploy/sac-deploy.sh +++ b/deploy/sac-deploy.sh @@ -93,6 +93,10 @@ if [ -f "${APP_ROOT}/deploy/systemd/${SERVICE_NAME}.service" ]; then systemctl daemon-reload fi fi +START_SH="${APP_ROOT}/deploy/systemd/sac-api-start.sh" +if [ -f "${START_SH}" ]; then + chmod 755 "${START_SH}" +fi log "Проверка DATABASE_URL (как у uvicorn через SAC_CONFIG_FILE)" sudo -u "${APP_USER}" bash -c " diff --git a/deploy/systemd/sac-api-start.sh b/deploy/systemd/sac-api-start.sh new file mode 100644 index 0000000..6532fce --- /dev/null +++ b/deploy/systemd/sac-api-start.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Uvicorn launcher: SAC_UVICORN_WORKERS из sac-api.env (без systemd EnvironmentFile). +set -euo pipefail + +APP_ROOT="${SAC_APP_ROOT:-/opt/security-alert-center}" +CONFIG_FILE="${SAC_CONFIG_FILE:-${APP_ROOT}/config/sac-api.env}" +VENV="${APP_ROOT}/backend/.venv" +HOST="127.0.0.1" +PORT="8000" +WORKERS=4 + +_read_env_int() { + local key="$1" default="$2" line val + [ -f "$CONFIG_FILE" ] || { + printf '%s\n' "$default" + return 0 + } + line="$(grep -E "^[[:space:]]*${key}=" "$CONFIG_FILE" 2>/dev/null | tail -1)" || { + printf '%s\n' "$default" + return 0 + } + val="${line#*=}" + val="${val#"${val%%[![:space:]]*}"}" + val="${val%"${val##*[![:space:]]}"}" + val="${val#\"}" + val="${val%\"}" + val="${val#\'}" + val="${val%\'}" + if [[ "$val" =~ ^[0-9]+$ ]] && [ "$val" -ge 1 ]; then + printf '%s\n' "$val" + else + printf '%s\n' "$default" + fi +} + +WORKERS="$(_read_env_int SAC_UVICORN_WORKERS 4)" + +exec "${VENV}/bin/uvicorn" app.main:app \ + --host "$HOST" \ + --port "$PORT" \ + --workers "$WORKERS" \ + --timeout-graceful-shutdown 30 diff --git a/deploy/systemd/sac-api.service b/deploy/systemd/sac-api.service index 289594c..12bd60d 100644 --- a/deploy/systemd/sac-api.service +++ b/deploy/systemd/sac-api.service @@ -13,7 +13,7 @@ WorkingDirectory=/opt/security-alert-center/backend # Конфиг читает само приложение (pydantic), не systemd — иначе ломаются пароли с #, $ и т.д. Environment=SAC_CONFIG_FILE=/opt/security-alert-center/config/sac-api.env Environment=PYTHONPATH=/opt/security-alert-center/backend -ExecStart=/opt/security-alert-center/backend/.venv/bin/uvicorn app.main:app --host 127.0.0.1 --port 8000 --workers 2 --timeout-graceful-shutdown 30 +ExecStart=/opt/security-alert-center/deploy/systemd/sac-api-start.sh Restart=on-failure RestartSec=5 TimeoutStopSec=20 diff --git a/docs/deployment.md b/docs/deployment.md index 2a82022..24340ca 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -115,7 +115,7 @@ sudo /opt/sac-deploy.sh Краткий **502 Bad Gateway** в UI (~10 с) возможен при перезапуске `sac-api` — список хостов автоматически повторяет запрос; после деплоя обновите страницу. -**Важно в `sac-api.env`:** `SAC_PUBLIC_URL=https://sac.kalinamall.ru` — нужен для WinRM-обновления RDP (клиент скачивает zip с SAC). API: **2 worker** uvicorn (`deploy/systemd/sac-api.service`). +**Важно в `sac-api.env`:** `SAC_PUBLIC_URL=https://sac.kalinamall.ru` — нужен для WinRM-обновления RDP (клиент скачивает zip с SAC). API: **4 worker** uvicorn по умолчанию (`SAC_UVICORN_WORKERS`, `deploy/systemd/sac-api-start.sh`). Установка скрипта (один раз): `sudo cp /opt/security-alert-center/deploy/sac-deploy.sh /opt/sac-deploy.sh && sudo chmod 755 /opt/sac-deploy.sh` diff --git a/docs/ingest-mass-update-backlog.ru.md b/docs/ingest-mass-update-backlog.ru.md index 77d5030..74d30de 100644 --- a/docs/ingest-mass-update-backlog.ru.md +++ b/docs/ingest-mass-update-backlog.ru.md @@ -20,11 +20,11 @@ ## SAC (backend / deploy) -- [ ] **Defer** `notify_lifecycle` и `notify_auth_login` в background (как `report.daily.*` → `schedule_notify_daily_report`), чтобы ingest не ждал Telegram API. -- [ ] **Uvicorn workers:** 4 по умолчанию или `SAC_UVICORN_WORKERS` в `sac-api.env` / unit. -- [ ] **nginx:** отдельный `location` для `POST /api/v1/events` с увеличенным `proxy_read_timeout` (сейчас общий `location /` — 60s). +- [x] **Defer** `notify_lifecycle` и `notify_auth_login` в background (как `report.daily.*` → `schedule_notify_daily_report`), чтобы ingest не ждал Telegram API — **0.5.5** +- [x] **Uvicorn workers:** 4 по умолчанию или `SAC_UVICORN_WORKERS` в `sac-api.env` / `sac-api-start.sh` — **0.5.5** +- [x] **nginx:** отдельный `location` для `POST /api/v1/events` с увеличенным `proxy_read_timeout` — **0.5.5** - [ ] Опционально: метрики/лог длительности ingest и очереди при burst. -- [ ] UI: предупреждение при массовом update («N хостов — рекомендуется пачками»). +- [x] UI: предупреждение при массовом update («N хостов — рекомендуется пачками») — **0.5.5** --- diff --git a/frontend/src/views/HostsView.vue b/frontend/src/views/HostsView.vue index 8d28197..c8f9011 100644 --- a/frontend/src/views/HostsView.vue +++ b/frontend/src/views/HostsView.vue @@ -138,6 +138,7 @@ import { remoteActionTitleForHost, } from "../utils/hostAgentUpgrade"; import { + hostRemoteActionLogs, isHostRemoteActionActive, pollHostRemoteAction, runHostRemoteAction, @@ -244,6 +245,37 @@ function isVersionOutdated(h: HostSummary): boolean { return isAgentVersionOutdated(h.product_version, reference); } +function countOutdatedUpgradeableHosts(): number { + return (data.value?.items ?? []).filter( + (h) => isGitAgentUpgradeAvailable(h, data.value) && isVersionOutdated(h), + ).length; +} + +function countActiveRemoteUpgrades(): number { + return hostRemoteActionLogs.filter((e) => e.loading).length; +} + +function confirmMassUpgradeIfNeeded(hostname: string): boolean { + const active = countActiveRemoteUpgrades(); + const outdated = countOutdatedUpgradeableHosts(); + if (active >= 2) { + return window.confirm( + `Сейчас обновляется ${active} хост(ов). Рекомендуется пачками по 2–3. Продолжить обновление «${hostname}»?`, + ); + } + if (outdated > 3 && active >= 1) { + return window.confirm( + `Устарело агентов на странице: ${outdated}, уже идёт обновление. Продолжить «${hostname}»?`, + ); + } + if (outdated > 3) { + return window.confirm( + `Устарело агентов на странице: ${outdated}. Рекомендуется обновлять пачками по 2–3, не все сразу. Продолжить «${hostname}»?`, + ); + } + return true; +} + async function startAgentUpgrade(h: HostSummary) { if (!isGitAgentUpgradeAvailable(h, data.value)) { return; @@ -252,6 +284,9 @@ async function startAgentUpgrade(h: HostSummary) { showHostRemoteActionLog(h.id); return; } + if (!confirmMassUpgradeIfNeeded(h.hostname)) { + return; + } const kind = remoteActionKindForHost(h); if (!kind) return; void runHostRemoteAction(h.id, remoteActionTitleForHost(h), kind);