feat: live update log in UI and suppress lifecycle Telegram during SAC update

Poll remote update_script.log while SSH update runs; skip lifecycle notify
when host agent_update_state is running (SAC v0.5.1).
This commit is contained in:
PTah
2026-07-08 11:46:53 +10:00
parent 1639261cde
commit d47131cd9f
9 changed files with 157 additions and 5 deletions
+64 -1
View File
@@ -16,7 +16,7 @@ from app.services.agent_update import execute_agent_update_fallback
from app.services.agent_update_types import AgentUpdateFallbackResult
from app.services.linux_admin_settings import get_effective_linux_admin_config
from app.services.agent_update_settings import get_effective_agent_update_config
from app.services.ssh_connect import iter_ssh_targets, run_ssh_monitor_update, SshCommandResult
from app.services.ssh_connect import iter_ssh_targets, run_ssh_monitor_update, SshCommandResult, tail_ssh_monitor_update_log
logger = logging.getLogger(__name__)
@@ -113,12 +113,61 @@ def _apply_ssh_update_result(db: Session, host: Host, result: SshCommandResult)
host.agent_update_last_error = (result.message or "SSH update failed")[:2000]
_REMOTE_LOG_POLL_SEC = 2.0
def _poll_ssh_update_log_loop(
host_id: int,
*,
stop: threading.Event,
) -> None:
"""Периодически подтягивает tail update_script.log в hosts.remote_action для UI."""
while not stop.wait(_REMOTE_LOG_POLL_SEC):
session = SessionLocal()
try:
row = session.get(Host, host_id)
if row is None or row.agent_update_state != "running":
continue
cfg = get_effective_linux_admin_config(session)
if not cfg.configured:
continue
try:
targets = iter_ssh_targets(row)
except Exception:
continue
tail = ""
for target in targets:
try:
tail = tail_ssh_monitor_update_log(
target=target,
user=cfg.user,
password=cfg.password,
)
except Exception:
logger.debug("update log tail failed host_id=%s target=%s", host_id, target, exc_info=True)
continue
if tail:
break
payload = dict(row.remote_action or {})
if tail:
payload["output"] = tail
payload["message"] = "Выполняется обновление… (лог с хоста)"
row.remote_action = payload
session.commit()
except Exception:
logger.debug("remote log poll failed host_id=%s", host_id, exc_info=True)
session.rollback()
finally:
session.close()
def start_host_remote_action(
db: Session,
host: Host,
*,
title: str,
runner: ActionRunner,
poll_remote_log: bool = False,
) -> dict[str, Any]:
host_id = int(host.id)
with _lock:
@@ -133,6 +182,17 @@ def start_host_remote_action(
def _worker() -> None:
session = SessionLocal()
stop_poll = threading.Event()
poll_thread: threading.Thread | None = None
if poll_remote_log and runner is run_ssh_monitor_update_action:
poll_thread = threading.Thread(
target=_poll_ssh_update_log_loop,
args=(host_id,),
kwargs={"stop": stop_poll},
name=f"sac-remote-log-{host_id}",
daemon=True,
)
poll_thread.start()
try:
row = session.get(Host, host_id)
if row is None:
@@ -167,6 +227,9 @@ def start_host_remote_action(
}
session.commit()
finally:
stop_poll.set()
if poll_thread is not None:
poll_thread.join(timeout=5.0)
session.close()
with _lock:
_running.discard(host_id)
+36 -1
View File
@@ -4,7 +4,7 @@ import logging
from sqlalchemy.orm import Session
from app.models import Event, Problem
from app.models import Event, Host, Problem
from app.services import email_notify, mobile_notify, telegram_notify, webhook_notify
from app.services.notification_cooldown import should_notify_event, should_notify_problem
from app.services.notification_policy import get_effective_notification_policy
@@ -111,8 +111,43 @@ def notify_daily_report(event: Event, *, db: Session | None = None) -> None:
_dispatch_lifecycle_channels(event, db=db, policy=policy)
def _host_sac_update_running(event: Event, db: Session | None) -> bool:
"""Пока SAC выполняет agent-update на хосте — не слать lifecycle в Telegram."""
if db is None or not event.host_id:
return False
host = db.get(Host, event.host_id)
if host is None:
return False
if (host.agent_update_state or "").strip().lower() == "running":
logger.info(
"notify lifecycle skipped host update running host_id=%s event_id=%s",
host.id,
event.event_id,
)
return True
return False
def _lifecycle_suppress_notifications(event: Event, *, db: Session | None = None) -> bool:
"""Не слать TG при штатном SAC/cron update (lifecycle с trigger deploy_recycle)."""
if _host_sac_update_running(event, db):
return True
details = event.details if isinstance(event.details, dict) else {}
trigger = str(details.get("trigger") or "").strip().lower()
if trigger in ("deploy_recycle", "sac_update", "agent_update"):
logger.info(
"notify lifecycle skipped trigger=%s event_id=%s",
trigger,
event.event_id,
)
return True
return False
def notify_lifecycle(event: Event, *, db: Session | None = None) -> None:
"""Старт/стоп/reload агента — всегда в каналы SAC (кроме TG, если telegram_via=agent)."""
if _lifecycle_suppress_notifications(event, db=db):
return
if _skip_notifications_for_hidden_event(event, db):
return
policy = get_effective_notification_policy(db)
+27
View File
@@ -9,6 +9,7 @@ from dataclasses import dataclass
from app.config import get_settings
from app.models import Host
SSH_MONITOR_UPDATE_LOG_PATH = "/var/log/update_script.log"
SSH_MONITOR_UPDATE_SCRIPT = "/opt/scripts/update_ssh_monitor.sh"
SSH_MONITOR_UPDATE_DIR = "/opt/scripts/update"
SSH_MONITOR_REPO_NAME = "ssh-monitor"
@@ -549,3 +550,29 @@ def run_ssh_monitor_update(
exit_code=updated.exit_code,
)
return updated
def tail_ssh_monitor_update_log(
*,
target: str,
user: str,
password: str,
lines: int = 120,
) -> str:
"""Хвост /var/log/update_script.log на удалённом хосте (для live-лога в SAC UI)."""
safe_lines = max(20, min(int(lines), 400))
remote_cmd = (
f"test -r {SSH_MONITOR_UPDATE_LOG_PATH} && "
f"tail -n {safe_lines} {SSH_MONITOR_UPDATE_LOG_PATH} 2>/dev/null || true"
)
result = run_ssh_command(
target=target,
user=user,
password=password,
remote_cmd=remote_cmd,
command_timeout_sec=25,
need_root=True,
login_shell=False,
)
text = (result.stdout or "").strip()
return _truncate_output(text)