feat: live update log in UI and suppress lifecycle Telegram during SAC update
Poll remote update_script.log while SSH update runs; skip lifecycle notify when host agent_update_state is running (SAC v0.5.1).
This commit is contained in:
@@ -567,6 +567,7 @@ def update_host_agent_via_ssh(
|
||||
host,
|
||||
title=title,
|
||||
runner=run_ssh_monitor_update_action,
|
||||
poll_remote_log=True,
|
||||
)
|
||||
except RemoteActionAlreadyRunningError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
|
||||
@@ -16,7 +16,7 @@ from app.services.agent_update import execute_agent_update_fallback
|
||||
from app.services.agent_update_types import AgentUpdateFallbackResult
|
||||
from app.services.linux_admin_settings import get_effective_linux_admin_config
|
||||
from app.services.agent_update_settings import get_effective_agent_update_config
|
||||
from app.services.ssh_connect import iter_ssh_targets, run_ssh_monitor_update, SshCommandResult
|
||||
from app.services.ssh_connect import iter_ssh_targets, run_ssh_monitor_update, SshCommandResult, tail_ssh_monitor_update_log
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -113,12 +113,61 @@ def _apply_ssh_update_result(db: Session, host: Host, result: SshCommandResult)
|
||||
host.agent_update_last_error = (result.message or "SSH update failed")[:2000]
|
||||
|
||||
|
||||
_REMOTE_LOG_POLL_SEC = 2.0
|
||||
|
||||
|
||||
def _poll_ssh_update_log_loop(
|
||||
host_id: int,
|
||||
*,
|
||||
stop: threading.Event,
|
||||
) -> None:
|
||||
"""Периодически подтягивает tail update_script.log в hosts.remote_action для UI."""
|
||||
while not stop.wait(_REMOTE_LOG_POLL_SEC):
|
||||
session = SessionLocal()
|
||||
try:
|
||||
row = session.get(Host, host_id)
|
||||
if row is None or row.agent_update_state != "running":
|
||||
continue
|
||||
cfg = get_effective_linux_admin_config(session)
|
||||
if not cfg.configured:
|
||||
continue
|
||||
try:
|
||||
targets = iter_ssh_targets(row)
|
||||
except Exception:
|
||||
continue
|
||||
tail = ""
|
||||
for target in targets:
|
||||
try:
|
||||
tail = tail_ssh_monitor_update_log(
|
||||
target=target,
|
||||
user=cfg.user,
|
||||
password=cfg.password,
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("update log tail failed host_id=%s target=%s", host_id, target, exc_info=True)
|
||||
continue
|
||||
if tail:
|
||||
break
|
||||
payload = dict(row.remote_action or {})
|
||||
if tail:
|
||||
payload["output"] = tail
|
||||
payload["message"] = "Выполняется обновление… (лог с хоста)"
|
||||
row.remote_action = payload
|
||||
session.commit()
|
||||
except Exception:
|
||||
logger.debug("remote log poll failed host_id=%s", host_id, exc_info=True)
|
||||
session.rollback()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
def start_host_remote_action(
|
||||
db: Session,
|
||||
host: Host,
|
||||
*,
|
||||
title: str,
|
||||
runner: ActionRunner,
|
||||
poll_remote_log: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
host_id = int(host.id)
|
||||
with _lock:
|
||||
@@ -133,6 +182,17 @@ def start_host_remote_action(
|
||||
|
||||
def _worker() -> None:
|
||||
session = SessionLocal()
|
||||
stop_poll = threading.Event()
|
||||
poll_thread: threading.Thread | None = None
|
||||
if poll_remote_log and runner is run_ssh_monitor_update_action:
|
||||
poll_thread = threading.Thread(
|
||||
target=_poll_ssh_update_log_loop,
|
||||
args=(host_id,),
|
||||
kwargs={"stop": stop_poll},
|
||||
name=f"sac-remote-log-{host_id}",
|
||||
daemon=True,
|
||||
)
|
||||
poll_thread.start()
|
||||
try:
|
||||
row = session.get(Host, host_id)
|
||||
if row is None:
|
||||
@@ -167,6 +227,9 @@ def start_host_remote_action(
|
||||
}
|
||||
session.commit()
|
||||
finally:
|
||||
stop_poll.set()
|
||||
if poll_thread is not None:
|
||||
poll_thread.join(timeout=5.0)
|
||||
session.close()
|
||||
with _lock:
|
||||
_running.discard(host_id)
|
||||
|
||||
@@ -4,7 +4,7 @@ import logging
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Event, Problem
|
||||
from app.models import Event, Host, Problem
|
||||
from app.services import email_notify, mobile_notify, telegram_notify, webhook_notify
|
||||
from app.services.notification_cooldown import should_notify_event, should_notify_problem
|
||||
from app.services.notification_policy import get_effective_notification_policy
|
||||
@@ -111,8 +111,43 @@ def notify_daily_report(event: Event, *, db: Session | None = None) -> None:
|
||||
_dispatch_lifecycle_channels(event, db=db, policy=policy)
|
||||
|
||||
|
||||
def _host_sac_update_running(event: Event, db: Session | None) -> bool:
|
||||
"""Пока SAC выполняет agent-update на хосте — не слать lifecycle в Telegram."""
|
||||
if db is None or not event.host_id:
|
||||
return False
|
||||
host = db.get(Host, event.host_id)
|
||||
if host is None:
|
||||
return False
|
||||
if (host.agent_update_state or "").strip().lower() == "running":
|
||||
logger.info(
|
||||
"notify lifecycle skipped host update running host_id=%s event_id=%s",
|
||||
host.id,
|
||||
event.event_id,
|
||||
)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _lifecycle_suppress_notifications(event: Event, *, db: Session | None = None) -> bool:
|
||||
"""Не слать TG при штатном SAC/cron update (lifecycle с trigger deploy_recycle)."""
|
||||
if _host_sac_update_running(event, db):
|
||||
return True
|
||||
details = event.details if isinstance(event.details, dict) else {}
|
||||
trigger = str(details.get("trigger") or "").strip().lower()
|
||||
if trigger in ("deploy_recycle", "sac_update", "agent_update"):
|
||||
logger.info(
|
||||
"notify lifecycle skipped trigger=%s event_id=%s",
|
||||
trigger,
|
||||
event.event_id,
|
||||
)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def notify_lifecycle(event: Event, *, db: Session | None = None) -> None:
|
||||
"""Старт/стоп/reload агента — всегда в каналы SAC (кроме TG, если telegram_via=agent)."""
|
||||
if _lifecycle_suppress_notifications(event, db=db):
|
||||
return
|
||||
if _skip_notifications_for_hidden_event(event, db):
|
||||
return
|
||||
policy = get_effective_notification_policy(db)
|
||||
|
||||
@@ -9,6 +9,7 @@ from dataclasses import dataclass
|
||||
from app.config import get_settings
|
||||
from app.models import Host
|
||||
|
||||
SSH_MONITOR_UPDATE_LOG_PATH = "/var/log/update_script.log"
|
||||
SSH_MONITOR_UPDATE_SCRIPT = "/opt/scripts/update_ssh_monitor.sh"
|
||||
SSH_MONITOR_UPDATE_DIR = "/opt/scripts/update"
|
||||
SSH_MONITOR_REPO_NAME = "ssh-monitor"
|
||||
@@ -549,3 +550,29 @@ def run_ssh_monitor_update(
|
||||
exit_code=updated.exit_code,
|
||||
)
|
||||
return updated
|
||||
|
||||
|
||||
def tail_ssh_monitor_update_log(
|
||||
*,
|
||||
target: str,
|
||||
user: str,
|
||||
password: str,
|
||||
lines: int = 120,
|
||||
) -> str:
|
||||
"""Хвост /var/log/update_script.log на удалённом хосте (для live-лога в SAC UI)."""
|
||||
safe_lines = max(20, min(int(lines), 400))
|
||||
remote_cmd = (
|
||||
f"test -r {SSH_MONITOR_UPDATE_LOG_PATH} && "
|
||||
f"tail -n {safe_lines} {SSH_MONITOR_UPDATE_LOG_PATH} 2>/dev/null || true"
|
||||
)
|
||||
result = run_ssh_command(
|
||||
target=target,
|
||||
user=user,
|
||||
password=password,
|
||||
remote_cmd=remote_cmd,
|
||||
command_timeout_sec=25,
|
||||
need_root=True,
|
||||
login_shell=False,
|
||||
)
|
||||
text = (result.stdout or "").strip()
|
||||
return _truncate_output(text)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""Единый источник версии SAC (API, health, логи, OpenAPI)."""
|
||||
|
||||
APP_NAME = "Security Alert Center"
|
||||
APP_VERSION = "0.5.0"
|
||||
APP_VERSION = "0.5.1"
|
||||
APP_VERSION_LABEL = f"{APP_NAME} v.{APP_VERSION}"
|
||||
|
||||
Reference in New Issue
Block a user