fix: 12h cooldown before reopening manually closed host_silence (0.9.9)
Track resolved_by on problems; suppress scan/ingest recreate after manual resolve; reopen same problem after cooldown; auto-close via heartbeat unchanged. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -4,11 +4,12 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import get_settings
|
||||
from app.models import Host, Problem
|
||||
from app.services.problem_rules import RuleMatch, build_fingerprint, host_silence_match_for_host
|
||||
from app.services.problems import _correlation_cutoff
|
||||
@@ -30,8 +31,12 @@ def open_or_refresh_host_silence_problem(
|
||||
match: RuleMatch,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
) -> tuple[Problem, bool]:
|
||||
"""Открыть или обновить open Problem host_silence без ingest-события."""
|
||||
) -> tuple[Problem | None, bool]:
|
||||
"""
|
||||
Открыть или обновить open Problem host_silence без ingest-события.
|
||||
|
||||
Returns (problem, created). problem is None when suppressed by manual-resolve cooldown.
|
||||
"""
|
||||
ref = now or datetime.now(timezone.utc)
|
||||
fingerprint = build_fingerprint(
|
||||
host_id,
|
||||
@@ -60,6 +65,55 @@ def open_or_refresh_host_silence_problem(
|
||||
db.flush()
|
||||
return open_problem, False
|
||||
|
||||
settings = get_settings()
|
||||
cooldown_hours = settings.sac_host_silence_manual_resolve_cooldown_hours
|
||||
if cooldown_hours > 0:
|
||||
cooldown = timedelta(hours=cooldown_hours)
|
||||
manual_recent = db.scalar(
|
||||
select(Problem)
|
||||
.where(
|
||||
Problem.host_id == host_id,
|
||||
Problem.rule_id == match.rule_id,
|
||||
Problem.fingerprint == fingerprint,
|
||||
Problem.status == "resolved",
|
||||
Problem.resolved_by == "manual",
|
||||
Problem.updated_at >= ref - cooldown,
|
||||
)
|
||||
.order_by(Problem.updated_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
if manual_recent is not None:
|
||||
logger.debug(
|
||||
"host_silence suppressed (manual cooldown %sh) host_id=%s problem_id=%s",
|
||||
cooldown_hours,
|
||||
host_id,
|
||||
manual_recent.id,
|
||||
)
|
||||
return None, False
|
||||
|
||||
manual_expired = db.scalar(
|
||||
select(Problem)
|
||||
.where(
|
||||
Problem.host_id == host_id,
|
||||
Problem.rule_id == match.rule_id,
|
||||
Problem.fingerprint == fingerprint,
|
||||
Problem.status == "resolved",
|
||||
Problem.resolved_by == "manual",
|
||||
Problem.updated_at < ref - cooldown,
|
||||
)
|
||||
.order_by(Problem.updated_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
if manual_expired is not None:
|
||||
manual_expired.status = "open"
|
||||
manual_expired.resolved_by = None
|
||||
manual_expired.summary = match.summary
|
||||
manual_expired.title = match.title
|
||||
manual_expired.last_seen_at = ref
|
||||
manual_expired.updated_at = ref
|
||||
db.flush()
|
||||
return manual_expired, True
|
||||
|
||||
problem = Problem(
|
||||
host_id=host_id,
|
||||
title=match.title,
|
||||
@@ -87,6 +141,8 @@ def run_host_silence_scan(db: Session, *, now: datetime | None = None) -> list[H
|
||||
if match is None:
|
||||
continue
|
||||
problem, created = open_or_refresh_host_silence_problem(db, host.id, match, now=ref)
|
||||
if problem is None:
|
||||
continue
|
||||
results.append(
|
||||
HostSilenceScanResult(
|
||||
host_id=host.id,
|
||||
|
||||
@@ -190,6 +190,7 @@ def resolve_host_silence_problems(db: Session, host_id: int) -> int:
|
||||
).all()
|
||||
for problem in problems:
|
||||
problem.status = "resolved"
|
||||
problem.resolved_by = "auto"
|
||||
if problems:
|
||||
db.flush()
|
||||
return len(problems)
|
||||
|
||||
@@ -10,6 +10,7 @@ from app.models import Event, Problem, ProblemEvent
|
||||
from app.services.host_health import HEARTBEAT_TYPE
|
||||
from app.services.problem_rules import (
|
||||
RuleMatch,
|
||||
RULE_HOST_SILENCE,
|
||||
build_fingerprint,
|
||||
pick_rule_match,
|
||||
resolve_host_silence_problems,
|
||||
@@ -43,7 +44,21 @@ def _append_event(db: Session, problem: Problem, event: Event) -> None:
|
||||
|
||||
def open_or_append_problem(
|
||||
db: Session, event: Event, match: RuleMatch
|
||||
) -> tuple[Problem, bool]:
|
||||
) -> tuple[Problem | None, bool]:
|
||||
if match.rule_id == RULE_HOST_SILENCE:
|
||||
from app.services.host_silence_scan import open_or_refresh_host_silence_problem
|
||||
|
||||
ref = event.occurred_at
|
||||
if ref.tzinfo is None:
|
||||
ref = ref.replace(tzinfo=timezone.utc)
|
||||
problem, created = open_or_refresh_host_silence_problem(
|
||||
db, event.host_id, match, now=ref
|
||||
)
|
||||
if problem is None:
|
||||
return None, False
|
||||
_append_event(db, problem, event)
|
||||
return problem, created
|
||||
|
||||
fingerprint = build_fingerprint(
|
||||
event.host_id,
|
||||
match.correlation_type,
|
||||
@@ -98,4 +113,5 @@ def maybe_create_problem(db: Session, event: Event) -> tuple[Problem | None, boo
|
||||
if match is None:
|
||||
return None, False
|
||||
|
||||
return open_or_append_problem(db, event, match)
|
||||
problem, created = open_or_append_problem(db, event, match)
|
||||
return problem, created
|
||||
|
||||
Reference in New Issue
Block a user