7df718eed3
Track resolved_by on problems; suppress scan/ingest recreate after manual resolve; reopen same problem after cooldown; auto-close via heartbeat unchanged. Co-authored-by: Cursor <cursoragent@cursor.com>
183 lines
5.7 KiB
Python
183 lines
5.7 KiB
Python
"""Периодическое сканирование хостов с устаревшим agent.heartbeat (proactive host_silence)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.config import get_settings
|
|
from app.models import Host, Problem
|
|
from app.services.problem_rules import RuleMatch, build_fingerprint, host_silence_match_for_host
|
|
from app.services.problems import _correlation_cutoff
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class HostSilenceScanResult:
|
|
host_id: int
|
|
hostname: str
|
|
problem: Problem
|
|
created: bool
|
|
|
|
|
|
def open_or_refresh_host_silence_problem(
|
|
db: Session,
|
|
host_id: int,
|
|
match: RuleMatch,
|
|
*,
|
|
now: datetime | None = None,
|
|
) -> tuple[Problem | None, bool]:
|
|
"""
|
|
Открыть или обновить open Problem host_silence без ingest-события.
|
|
|
|
Returns (problem, created). problem is None when suppressed by manual-resolve cooldown.
|
|
"""
|
|
ref = now or datetime.now(timezone.utc)
|
|
fingerprint = build_fingerprint(
|
|
host_id,
|
|
match.correlation_type,
|
|
match.rule_id,
|
|
match.fingerprint_suffix,
|
|
)
|
|
cutoff = _correlation_cutoff(ref)
|
|
|
|
open_problem = db.scalar(
|
|
select(Problem)
|
|
.where(
|
|
Problem.status == "open",
|
|
Problem.fingerprint == fingerprint,
|
|
Problem.host_id == host_id,
|
|
Problem.last_seen_at >= cutoff,
|
|
)
|
|
.order_by(Problem.last_seen_at.desc())
|
|
.limit(1)
|
|
)
|
|
if open_problem:
|
|
open_problem.summary = match.summary
|
|
open_problem.title = match.title
|
|
open_problem.last_seen_at = ref
|
|
open_problem.updated_at = ref
|
|
db.flush()
|
|
return open_problem, False
|
|
|
|
settings = get_settings()
|
|
cooldown_hours = settings.sac_host_silence_manual_resolve_cooldown_hours
|
|
if cooldown_hours > 0:
|
|
cooldown = timedelta(hours=cooldown_hours)
|
|
manual_recent = db.scalar(
|
|
select(Problem)
|
|
.where(
|
|
Problem.host_id == host_id,
|
|
Problem.rule_id == match.rule_id,
|
|
Problem.fingerprint == fingerprint,
|
|
Problem.status == "resolved",
|
|
Problem.resolved_by == "manual",
|
|
Problem.updated_at >= ref - cooldown,
|
|
)
|
|
.order_by(Problem.updated_at.desc())
|
|
.limit(1)
|
|
)
|
|
if manual_recent is not None:
|
|
logger.debug(
|
|
"host_silence suppressed (manual cooldown %sh) host_id=%s problem_id=%s",
|
|
cooldown_hours,
|
|
host_id,
|
|
manual_recent.id,
|
|
)
|
|
return None, False
|
|
|
|
manual_expired = db.scalar(
|
|
select(Problem)
|
|
.where(
|
|
Problem.host_id == host_id,
|
|
Problem.rule_id == match.rule_id,
|
|
Problem.fingerprint == fingerprint,
|
|
Problem.status == "resolved",
|
|
Problem.resolved_by == "manual",
|
|
Problem.updated_at < ref - cooldown,
|
|
)
|
|
.order_by(Problem.updated_at.desc())
|
|
.limit(1)
|
|
)
|
|
if manual_expired is not None:
|
|
manual_expired.status = "open"
|
|
manual_expired.resolved_by = None
|
|
manual_expired.summary = match.summary
|
|
manual_expired.title = match.title
|
|
manual_expired.last_seen_at = ref
|
|
manual_expired.updated_at = ref
|
|
db.flush()
|
|
return manual_expired, True
|
|
|
|
problem = Problem(
|
|
host_id=host_id,
|
|
title=match.title,
|
|
summary=match.summary,
|
|
severity=match.severity,
|
|
status="open",
|
|
rule_id=match.rule_id,
|
|
fingerprint=fingerprint,
|
|
event_count=0,
|
|
last_seen_at=ref,
|
|
)
|
|
db.add(problem)
|
|
db.flush()
|
|
return problem, True
|
|
|
|
|
|
def run_host_silence_scan(db: Session, *, now: datetime | None = None) -> list[HostSilenceScanResult]:
|
|
"""Найти stale-хосты и создать/обновить Problems rule:host_silence."""
|
|
ref = now or datetime.now(timezone.utc)
|
|
hosts = db.scalars(select(Host).order_by(Host.id)).all()
|
|
results: list[HostSilenceScanResult] = []
|
|
|
|
for host in hosts:
|
|
match = host_silence_match_for_host(db, host.id, hostname=host.hostname, now=ref)
|
|
if match is None:
|
|
continue
|
|
problem, created = open_or_refresh_host_silence_problem(db, host.id, match, now=ref)
|
|
if problem is None:
|
|
continue
|
|
results.append(
|
|
HostSilenceScanResult(
|
|
host_id=host.id,
|
|
hostname=host.hostname,
|
|
problem=problem,
|
|
created=created,
|
|
)
|
|
)
|
|
if created:
|
|
logger.info(
|
|
"host_silence scan: new problem host=%s id=%s problem_id=%s",
|
|
host.hostname,
|
|
host.id,
|
|
problem.id,
|
|
)
|
|
|
|
return results
|
|
|
|
|
|
def dispatch_host_silence_notifications(db: Session, results: list[HostSilenceScanResult]) -> int:
|
|
"""Telegram/webhook/email только для вновь созданных Problems."""
|
|
from app.services.notify_dispatch import notify_problem
|
|
|
|
sent = 0
|
|
for item in results:
|
|
if not item.created:
|
|
continue
|
|
try:
|
|
notify_problem(item.problem, event=None, db=db)
|
|
sent += 1
|
|
except Exception:
|
|
logger.exception(
|
|
"host_silence notify failed host=%s problem_id=%s",
|
|
item.hostname,
|
|
item.problem.id,
|
|
)
|
|
return sent
|