fix: 12h cooldown before reopening manually closed host_silence (0.9.9)
Track resolved_by on problems; suppress scan/ingest recreate after manual resolve; reopen same problem after cooldown; auto-close via heartbeat unchanged. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,26 @@
|
|||||||
|
"""problems.resolved_by — manual vs auto close for host_silence cooldown
|
||||||
|
|
||||||
|
Revision ID: 015
|
||||||
|
Revises: 014
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "015"
|
||||||
|
down_revision: Union[str, None] = "014"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"problems",
|
||||||
|
sa.Column("resolved_by", sa.String(length=16), nullable=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("problems", "resolved_by")
|
||||||
@@ -363,6 +363,7 @@ def resolve_problem(
|
|||||||
raise HTTPException(status_code=409, detail="Problem already resolved")
|
raise HTTPException(status_code=409, detail="Problem already resolved")
|
||||||
|
|
||||||
problem.status = "resolved"
|
problem.status = "resolved"
|
||||||
|
problem.resolved_by = "manual"
|
||||||
|
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
|
|||||||
@@ -88,6 +88,8 @@ class Settings(BaseSettings):
|
|||||||
# Периодическое сканирование stale heartbeat (proactive host_silence)
|
# Периодическое сканирование stale heartbeat (proactive host_silence)
|
||||||
sac_host_silence_scan_enabled: bool = True
|
sac_host_silence_scan_enabled: bool = True
|
||||||
sac_host_silence_scan_interval_minutes: int = 5
|
sac_host_silence_scan_interval_minutes: int = 5
|
||||||
|
# После ручного закрытия host_silence не открывать снова раньше N часов (auto-close по heartbeat — без паузы).
|
||||||
|
sac_host_silence_manual_resolve_cooldown_hours: int = 12
|
||||||
|
|
||||||
# Окно корреляции Problems: host + type + rule в одном open Problem
|
# Окно корреляции Problems: host + type + rule в одном open Problem
|
||||||
sac_problem_correlation_window_minutes: int = 60
|
sac_problem_correlation_window_minutes: int = 60
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ class Problem(Base):
|
|||||||
summary: Mapped[str] = mapped_column(Text)
|
summary: Mapped[str] = mapped_column(Text)
|
||||||
severity: Mapped[str] = mapped_column(String(16), index=True)
|
severity: Mapped[str] = mapped_column(String(16), index=True)
|
||||||
status: Mapped[str] = mapped_column(String(32), index=True, default="open")
|
status: Mapped[str] = mapped_column(String(32), index=True, default="open")
|
||||||
|
resolved_by: Mapped[str | None] = mapped_column(String(16))
|
||||||
rule_id: Mapped[str | None] = mapped_column(String(64))
|
rule_id: Mapped[str | None] = mapped_column(String(64))
|
||||||
fingerprint: Mapped[str] = mapped_column(String(128), index=True)
|
fingerprint: Mapped[str] = mapped_column(String(128), index=True)
|
||||||
event_count: Mapped[int] = mapped_column(Integer, default=1)
|
event_count: Mapped[int] = mapped_column(Integer, default=1)
|
||||||
|
|||||||
@@ -4,11 +4,12 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.config import get_settings
|
||||||
from app.models import Host, Problem
|
from app.models import Host, Problem
|
||||||
from app.services.problem_rules import RuleMatch, build_fingerprint, host_silence_match_for_host
|
from app.services.problem_rules import RuleMatch, build_fingerprint, host_silence_match_for_host
|
||||||
from app.services.problems import _correlation_cutoff
|
from app.services.problems import _correlation_cutoff
|
||||||
@@ -30,8 +31,12 @@ def open_or_refresh_host_silence_problem(
|
|||||||
match: RuleMatch,
|
match: RuleMatch,
|
||||||
*,
|
*,
|
||||||
now: datetime | None = None,
|
now: datetime | None = None,
|
||||||
) -> tuple[Problem, bool]:
|
) -> tuple[Problem | None, bool]:
|
||||||
"""Открыть или обновить open Problem host_silence без ingest-события."""
|
"""
|
||||||
|
Открыть или обновить open Problem host_silence без ingest-события.
|
||||||
|
|
||||||
|
Returns (problem, created). problem is None when suppressed by manual-resolve cooldown.
|
||||||
|
"""
|
||||||
ref = now or datetime.now(timezone.utc)
|
ref = now or datetime.now(timezone.utc)
|
||||||
fingerprint = build_fingerprint(
|
fingerprint = build_fingerprint(
|
||||||
host_id,
|
host_id,
|
||||||
@@ -60,6 +65,55 @@ def open_or_refresh_host_silence_problem(
|
|||||||
db.flush()
|
db.flush()
|
||||||
return open_problem, False
|
return open_problem, False
|
||||||
|
|
||||||
|
settings = get_settings()
|
||||||
|
cooldown_hours = settings.sac_host_silence_manual_resolve_cooldown_hours
|
||||||
|
if cooldown_hours > 0:
|
||||||
|
cooldown = timedelta(hours=cooldown_hours)
|
||||||
|
manual_recent = db.scalar(
|
||||||
|
select(Problem)
|
||||||
|
.where(
|
||||||
|
Problem.host_id == host_id,
|
||||||
|
Problem.rule_id == match.rule_id,
|
||||||
|
Problem.fingerprint == fingerprint,
|
||||||
|
Problem.status == "resolved",
|
||||||
|
Problem.resolved_by == "manual",
|
||||||
|
Problem.updated_at >= ref - cooldown,
|
||||||
|
)
|
||||||
|
.order_by(Problem.updated_at.desc())
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
if manual_recent is not None:
|
||||||
|
logger.debug(
|
||||||
|
"host_silence suppressed (manual cooldown %sh) host_id=%s problem_id=%s",
|
||||||
|
cooldown_hours,
|
||||||
|
host_id,
|
||||||
|
manual_recent.id,
|
||||||
|
)
|
||||||
|
return None, False
|
||||||
|
|
||||||
|
manual_expired = db.scalar(
|
||||||
|
select(Problem)
|
||||||
|
.where(
|
||||||
|
Problem.host_id == host_id,
|
||||||
|
Problem.rule_id == match.rule_id,
|
||||||
|
Problem.fingerprint == fingerprint,
|
||||||
|
Problem.status == "resolved",
|
||||||
|
Problem.resolved_by == "manual",
|
||||||
|
Problem.updated_at < ref - cooldown,
|
||||||
|
)
|
||||||
|
.order_by(Problem.updated_at.desc())
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
if manual_expired is not None:
|
||||||
|
manual_expired.status = "open"
|
||||||
|
manual_expired.resolved_by = None
|
||||||
|
manual_expired.summary = match.summary
|
||||||
|
manual_expired.title = match.title
|
||||||
|
manual_expired.last_seen_at = ref
|
||||||
|
manual_expired.updated_at = ref
|
||||||
|
db.flush()
|
||||||
|
return manual_expired, True
|
||||||
|
|
||||||
problem = Problem(
|
problem = Problem(
|
||||||
host_id=host_id,
|
host_id=host_id,
|
||||||
title=match.title,
|
title=match.title,
|
||||||
@@ -87,6 +141,8 @@ def run_host_silence_scan(db: Session, *, now: datetime | None = None) -> list[H
|
|||||||
if match is None:
|
if match is None:
|
||||||
continue
|
continue
|
||||||
problem, created = open_or_refresh_host_silence_problem(db, host.id, match, now=ref)
|
problem, created = open_or_refresh_host_silence_problem(db, host.id, match, now=ref)
|
||||||
|
if problem is None:
|
||||||
|
continue
|
||||||
results.append(
|
results.append(
|
||||||
HostSilenceScanResult(
|
HostSilenceScanResult(
|
||||||
host_id=host.id,
|
host_id=host.id,
|
||||||
|
|||||||
@@ -190,6 +190,7 @@ def resolve_host_silence_problems(db: Session, host_id: int) -> int:
|
|||||||
).all()
|
).all()
|
||||||
for problem in problems:
|
for problem in problems:
|
||||||
problem.status = "resolved"
|
problem.status = "resolved"
|
||||||
|
problem.resolved_by = "auto"
|
||||||
if problems:
|
if problems:
|
||||||
db.flush()
|
db.flush()
|
||||||
return len(problems)
|
return len(problems)
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ from app.models import Event, Problem, ProblemEvent
|
|||||||
from app.services.host_health import HEARTBEAT_TYPE
|
from app.services.host_health import HEARTBEAT_TYPE
|
||||||
from app.services.problem_rules import (
|
from app.services.problem_rules import (
|
||||||
RuleMatch,
|
RuleMatch,
|
||||||
|
RULE_HOST_SILENCE,
|
||||||
build_fingerprint,
|
build_fingerprint,
|
||||||
pick_rule_match,
|
pick_rule_match,
|
||||||
resolve_host_silence_problems,
|
resolve_host_silence_problems,
|
||||||
@@ -43,7 +44,21 @@ def _append_event(db: Session, problem: Problem, event: Event) -> None:
|
|||||||
|
|
||||||
def open_or_append_problem(
|
def open_or_append_problem(
|
||||||
db: Session, event: Event, match: RuleMatch
|
db: Session, event: Event, match: RuleMatch
|
||||||
) -> tuple[Problem, bool]:
|
) -> tuple[Problem | None, bool]:
|
||||||
|
if match.rule_id == RULE_HOST_SILENCE:
|
||||||
|
from app.services.host_silence_scan import open_or_refresh_host_silence_problem
|
||||||
|
|
||||||
|
ref = event.occurred_at
|
||||||
|
if ref.tzinfo is None:
|
||||||
|
ref = ref.replace(tzinfo=timezone.utc)
|
||||||
|
problem, created = open_or_refresh_host_silence_problem(
|
||||||
|
db, event.host_id, match, now=ref
|
||||||
|
)
|
||||||
|
if problem is None:
|
||||||
|
return None, False
|
||||||
|
_append_event(db, problem, event)
|
||||||
|
return problem, created
|
||||||
|
|
||||||
fingerprint = build_fingerprint(
|
fingerprint = build_fingerprint(
|
||||||
event.host_id,
|
event.host_id,
|
||||||
match.correlation_type,
|
match.correlation_type,
|
||||||
@@ -98,4 +113,5 @@ def maybe_create_problem(db: Session, event: Event) -> tuple[Problem | None, boo
|
|||||||
if match is None:
|
if match is None:
|
||||||
return None, False
|
return None, False
|
||||||
|
|
||||||
return open_or_append_problem(db, event, match)
|
problem, created = open_or_append_problem(db, event, match)
|
||||||
|
return problem, created
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
"""Единый источник версии SAC (API, health, логи, OpenAPI)."""
|
"""Единый источник версии SAC (API, health, логи, OpenAPI)."""
|
||||||
|
|
||||||
APP_NAME = "Security Alert Center"
|
APP_NAME = "Security Alert Center"
|
||||||
APP_VERSION = "0.9.8"
|
APP_VERSION = "0.9.9"
|
||||||
APP_VERSION_LABEL = f"{APP_NAME} v.{APP_VERSION}"
|
APP_VERSION_LABEL = f"{APP_NAME} v.{APP_VERSION}"
|
||||||
|
|||||||
@@ -95,6 +95,85 @@ def test_scan_skips_host_without_heartbeat(db_session, scan_settings):
|
|||||||
assert results == []
|
assert results == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_suppresses_after_manual_resolve_within_cooldown(db_session, scan_settings, monkeypatch):
|
||||||
|
monkeypatch.setenv("SAC_HOST_SILENCE_MANUAL_RESOLVE_COOLDOWN_HOURS", "12")
|
||||||
|
get_settings.cache_clear()
|
||||||
|
|
||||||
|
hb = _ingest(
|
||||||
|
db_session,
|
||||||
|
type="agent.heartbeat",
|
||||||
|
category="agent",
|
||||||
|
severity="info",
|
||||||
|
title="hb",
|
||||||
|
summary="heartbeat",
|
||||||
|
)
|
||||||
|
hb.received_at = datetime.now(timezone.utc) - timedelta(hours=2)
|
||||||
|
db_session.flush()
|
||||||
|
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
first = run_host_silence_scan(db_session, now=now)
|
||||||
|
assert first[0].created is True
|
||||||
|
problem = first[0].problem
|
||||||
|
problem.status = "resolved"
|
||||||
|
problem.resolved_by = "manual"
|
||||||
|
problem.updated_at = now
|
||||||
|
db_session.flush()
|
||||||
|
|
||||||
|
second = run_host_silence_scan(db_session, now=now + timedelta(minutes=10))
|
||||||
|
assert second == []
|
||||||
|
|
||||||
|
open_count = db_session.scalar(
|
||||||
|
select(Problem).where(
|
||||||
|
Problem.host_id == hb.host_id,
|
||||||
|
Problem.rule_id == RULE_HOST_SILENCE,
|
||||||
|
Problem.status == "open",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert open_count is None
|
||||||
|
|
||||||
|
resolved_count = len(
|
||||||
|
db_session.scalars(
|
||||||
|
select(Problem).where(
|
||||||
|
Problem.host_id == hb.host_id,
|
||||||
|
Problem.rule_id == RULE_HOST_SILENCE,
|
||||||
|
Problem.status == "resolved",
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
)
|
||||||
|
assert resolved_count == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_reopens_after_manual_cooldown_expires(db_session, scan_settings, monkeypatch):
|
||||||
|
monkeypatch.setenv("SAC_HOST_SILENCE_MANUAL_RESOLVE_COOLDOWN_HOURS", "12")
|
||||||
|
get_settings.cache_clear()
|
||||||
|
|
||||||
|
hb = _ingest(
|
||||||
|
db_session,
|
||||||
|
type="agent.heartbeat",
|
||||||
|
category="agent",
|
||||||
|
severity="info",
|
||||||
|
title="hb",
|
||||||
|
summary="heartbeat",
|
||||||
|
)
|
||||||
|
hb.received_at = datetime.now(timezone.utc) - timedelta(hours=2)
|
||||||
|
db_session.flush()
|
||||||
|
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
first = run_host_silence_scan(db_session, now=now)
|
||||||
|
problem = first[0].problem
|
||||||
|
problem.status = "resolved"
|
||||||
|
problem.resolved_by = "manual"
|
||||||
|
problem.updated_at = now - timedelta(hours=13)
|
||||||
|
db_session.flush()
|
||||||
|
|
||||||
|
later = run_host_silence_scan(db_session, now=now)
|
||||||
|
assert len(later) == 1
|
||||||
|
assert later[0].created is True
|
||||||
|
assert later[0].problem.id == problem.id
|
||||||
|
assert later[0].problem.status == "open"
|
||||||
|
assert later[0].problem.resolved_by is None
|
||||||
|
|
||||||
|
|
||||||
def test_scan_notifies_only_on_create(db_session, scan_settings):
|
def test_scan_notifies_only_on_create(db_session, scan_settings):
|
||||||
hb = _ingest(
|
hb = _ingest(
|
||||||
db_session,
|
db_session,
|
||||||
|
|||||||
@@ -182,3 +182,99 @@ def test_heartbeat_resolves_host_silence(db_session, rule_settings):
|
|||||||
maybe_create_problem(db_session, fresh_hb)
|
maybe_create_problem(db_session, fresh_hb)
|
||||||
db_session.refresh(open_silence)
|
db_session.refresh(open_silence)
|
||||||
assert open_silence.status == "resolved"
|
assert open_silence.status == "resolved"
|
||||||
|
assert open_silence.resolved_by == "auto"
|
||||||
|
|
||||||
|
|
||||||
|
def test_host_silence_suppressed_after_manual_resolve_on_ingest(db_session, rule_settings, monkeypatch):
|
||||||
|
monkeypatch.setenv("SAC_HOST_SILENCE_MANUAL_RESOLVE_COOLDOWN_HOURS", "12")
|
||||||
|
get_settings.cache_clear()
|
||||||
|
|
||||||
|
hb = _ingest(
|
||||||
|
db_session,
|
||||||
|
type="agent.heartbeat",
|
||||||
|
category="agent",
|
||||||
|
severity="info",
|
||||||
|
title="hb",
|
||||||
|
summary="heartbeat",
|
||||||
|
)
|
||||||
|
hb.received_at = datetime.now(timezone.utc) - timedelta(hours=2)
|
||||||
|
db_session.flush()
|
||||||
|
|
||||||
|
trigger = _ingest(
|
||||||
|
db_session,
|
||||||
|
type="ssh.login.success",
|
||||||
|
severity="info",
|
||||||
|
title="ok",
|
||||||
|
summary="login",
|
||||||
|
)
|
||||||
|
problem, created = maybe_create_problem(db_session, trigger)
|
||||||
|
assert created is True
|
||||||
|
problem.status = "resolved"
|
||||||
|
problem.resolved_by = "manual"
|
||||||
|
problem.updated_at = datetime.now(timezone.utc)
|
||||||
|
db_session.flush()
|
||||||
|
|
||||||
|
trigger2 = _ingest(
|
||||||
|
db_session,
|
||||||
|
type="ssh.login.success",
|
||||||
|
severity="info",
|
||||||
|
title="ok2",
|
||||||
|
summary="login2",
|
||||||
|
)
|
||||||
|
problem2, created2 = maybe_create_problem(db_session, trigger2)
|
||||||
|
assert problem2 is None
|
||||||
|
assert created2 is False
|
||||||
|
|
||||||
|
open_silence = db_session.scalar(
|
||||||
|
select(Problem).where(
|
||||||
|
Problem.rule_id == RULE_HOST_SILENCE,
|
||||||
|
Problem.status == "open",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
assert open_silence is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_host_silence_reopens_after_auto_resolve_when_still_stale(db_session, rule_settings, monkeypatch):
|
||||||
|
monkeypatch.setenv("SAC_HOST_SILENCE_MANUAL_RESOLVE_COOLDOWN_HOURS", "12")
|
||||||
|
get_settings.cache_clear()
|
||||||
|
|
||||||
|
hb = _ingest(
|
||||||
|
db_session,
|
||||||
|
type="agent.heartbeat",
|
||||||
|
category="agent",
|
||||||
|
severity="info",
|
||||||
|
title="hb",
|
||||||
|
summary="heartbeat",
|
||||||
|
)
|
||||||
|
hb.received_at = datetime.now(timezone.utc) - timedelta(hours=2)
|
||||||
|
db_session.flush()
|
||||||
|
|
||||||
|
trigger = _ingest(
|
||||||
|
db_session,
|
||||||
|
type="ssh.login.success",
|
||||||
|
severity="info",
|
||||||
|
title="ok",
|
||||||
|
summary="login",
|
||||||
|
)
|
||||||
|
problem, created = maybe_create_problem(db_session, trigger)
|
||||||
|
assert created is True
|
||||||
|
|
||||||
|
from app.services.problem_rules import resolve_host_silence_problems
|
||||||
|
|
||||||
|
resolve_host_silence_problems(db_session, hb.host_id)
|
||||||
|
db_session.refresh(problem)
|
||||||
|
assert problem.status == "resolved"
|
||||||
|
assert problem.resolved_by == "auto"
|
||||||
|
|
||||||
|
trigger2 = _ingest(
|
||||||
|
db_session,
|
||||||
|
type="ssh.login.success",
|
||||||
|
severity="info",
|
||||||
|
title="ok2",
|
||||||
|
summary="login2",
|
||||||
|
)
|
||||||
|
problem2, created2 = maybe_create_problem(db_session, trigger2)
|
||||||
|
assert created2 is True
|
||||||
|
assert problem2 is not None
|
||||||
|
assert problem2.status == "open"
|
||||||
|
assert problem2.id != problem.id
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
/** Fallback до загрузки /health; при релизе держите в sync с backend/app/version.py */
|
/** Fallback до загрузки /health; при релизе держите в sync с backend/app/version.py */
|
||||||
export const APP_NAME = "Security Alert Center";
|
export const APP_NAME = "Security Alert Center";
|
||||||
export const APP_VERSION = "0.9.8";
|
export const APP_VERSION = "0.9.9";
|
||||||
export const APP_VERSION_LABEL = `${APP_NAME} v.${APP_VERSION}`;
|
export const APP_VERSION_LABEL = `${APP_NAME} v.${APP_VERSION}`;
|
||||||
|
|||||||
Reference in New Issue
Block a user