PTah
d47131cd9f
Poll remote update_script.log while SSH update runs; skip lifecycle notify when host agent_update_state is running (SAC v0.5.1).
579 lines
18 KiB
Python
579 lines
18 KiB
Python
"""SSH connectivity and remote commands for Linux hosts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import socket
|
|
from dataclasses import dataclass
|
|
|
|
from app.config import get_settings
|
|
from app.models import Host
|
|
|
|
SSH_MONITOR_UPDATE_LOG_PATH = "/var/log/update_script.log"
|
|
SSH_MONITOR_UPDATE_SCRIPT = "/opt/scripts/update_ssh_monitor.sh"
|
|
SSH_MONITOR_UPDATE_DIR = "/opt/scripts/update"
|
|
SSH_MONITOR_REPO_NAME = "ssh-monitor"
|
|
SSH_MONITOR_UPDATE_REPO_URL = "https://git.kalinamall.ru/PapaTramp/ssh-monitor.git"
|
|
SSH_MONITOR_BINARY = "/usr/local/bin/ssh-monitor"
|
|
SSH_MONITOR_VERSION_CMD = (
|
|
f"grep -m1 '^SSH_MONITOR_VERSION=' {SSH_MONITOR_BINARY} 2>/dev/null "
|
|
"| sed -E 's/^SSH_MONITOR_VERSION=[\"'\\'']*([^\"'\\'']+)[\"'\\'']*$/\\1/'"
|
|
)
|
|
SSH_OUTPUT_MAX_LEN = 12_000
|
|
_AGENT_VERSION_RE = re.compile(r"(\d+\.\d+\.\d+(?:-SAC)?)", re.IGNORECASE)
|
|
_IPV4_RE = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$")
|
|
_SSH_TRANSIENT_ERROR_MARKERS = (
|
|
"no existing session",
|
|
"error reading ssh protocol banner",
|
|
"connection reset",
|
|
"connection lost",
|
|
"session not active",
|
|
"eof",
|
|
"socket is closed",
|
|
"connection timed out",
|
|
)
|
|
_SSH_CONNECT_ATTEMPTS = 2
|
|
|
|
|
|
class LinuxAdminNotConfiguredError(Exception):
|
|
pass
|
|
|
|
|
|
class HostNotLinuxError(Exception):
|
|
pass
|
|
|
|
|
|
class HostTargetMissingError(Exception):
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SshCommandResult:
|
|
ok: bool
|
|
message: str
|
|
target: str
|
|
stdout: str = ""
|
|
stderr: str = ""
|
|
exit_code: int | None = None
|
|
agent_version: str | None = None
|
|
|
|
|
|
def is_linux_host(host: Host) -> bool:
|
|
if (host.os_family or "").strip().lower() == "linux":
|
|
return True
|
|
return (host.product or "").strip() == "ssh-monitor"
|
|
|
|
|
|
def _is_ipv4(value: str) -> bool:
|
|
return bool(_IPV4_RE.match(value.strip()))
|
|
|
|
|
|
def _looks_like_ssh_hostname(value: str) -> bool:
|
|
"""Короткое имя или FQDN; не человекочитаемый display_name вроде «Ubabuba Kalina (10.10.36.9)»."""
|
|
text = value.strip()
|
|
if not text or " " in text or "(" in text or ")" in text:
|
|
return False
|
|
if _is_ipv4(text):
|
|
return True
|
|
if len(text) > 253:
|
|
return False
|
|
return all(ch.isalnum() or ch in ".-_" for ch in text)
|
|
|
|
|
|
def _ssh_name_resolves(name: str) -> bool:
|
|
if _is_ipv4(name):
|
|
return True
|
|
try:
|
|
socket.getaddrinfo(name.strip(), 22, type=socket.SOCK_STREAM)
|
|
return True
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def iter_ssh_targets(host: Host) -> list[str]:
|
|
if not is_linux_host(host):
|
|
raise HostNotLinuxError("Host is not Linux")
|
|
|
|
seen: set[str] = set()
|
|
ordered: list[str] = []
|
|
|
|
def add(value: str | None) -> None:
|
|
text = (value or "").strip()
|
|
if not text or text.casefold() in seen:
|
|
return
|
|
if not _looks_like_ssh_hostname(text):
|
|
return
|
|
if not _ssh_name_resolves(text):
|
|
return
|
|
seen.add(text.casefold())
|
|
ordered.append(text)
|
|
|
|
add(host.hostname)
|
|
|
|
inventory = host.inventory if isinstance(host.inventory, dict) else {}
|
|
computer_name = inventory.get("computer_name")
|
|
if isinstance(computer_name, str):
|
|
add(computer_name)
|
|
|
|
if host.display_name and _looks_like_ssh_hostname(host.display_name):
|
|
add(host.display_name)
|
|
|
|
add(host.ipv4)
|
|
if not ordered:
|
|
raise HostTargetMissingError("Host has no resolvable hostname or IPv4 for SSH")
|
|
return ordered
|
|
|
|
|
|
def _truncate_output(text: str) -> str:
|
|
if len(text) <= SSH_OUTPUT_MAX_LEN:
|
|
return text
|
|
return text[:SSH_OUTPUT_MAX_LEN] + "\n… (truncated)"
|
|
|
|
|
|
def _is_transient_ssh_error(exc: BaseException) -> bool:
|
|
text = str(exc).casefold()
|
|
return any(marker in text for marker in _SSH_TRANSIENT_ERROR_MARKERS)
|
|
|
|
|
|
def _ssh_connect_hint(exc: BaseException) -> str:
|
|
text = str(exc).casefold()
|
|
if "no existing session" in text:
|
|
return (
|
|
" Сервер закрыл SSH-сессию во время подключения или auth "
|
|
"(проверьте пароль admin, лимиты sshd MaxSessions/MaxStartups, "
|
|
"доступ SAC→хост по 22/tcp; на SAC не должен мешать ssh-agent)."
|
|
)
|
|
return ""
|
|
|
|
|
|
def _connect_ssh_client(
|
|
client,
|
|
*,
|
|
target: str,
|
|
user: str,
|
|
password: str,
|
|
connect_timeout_sec: int,
|
|
) -> None:
|
|
client.connect(
|
|
hostname=target,
|
|
username=user,
|
|
password=password,
|
|
timeout=connect_timeout_sec,
|
|
banner_timeout=connect_timeout_sec,
|
|
auth_timeout=connect_timeout_sec,
|
|
allow_agent=False,
|
|
look_for_keys=False,
|
|
compress=False,
|
|
)
|
|
|
|
|
|
def _close_ssh_client(client) -> None:
|
|
if client is None:
|
|
return
|
|
try:
|
|
transport = client.get_transport()
|
|
if transport is not None and transport.is_active():
|
|
transport.close()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
client.close()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _shell_single_quote(value: str) -> str:
|
|
return "'" + value.replace("'", "'\"'\"'") + "'"
|
|
|
|
|
|
def _remote_shell_command(
|
|
user: str,
|
|
remote_cmd: str,
|
|
*,
|
|
need_root: bool = False,
|
|
login_shell: bool = True,
|
|
) -> tuple[str, bool]:
|
|
"""Build remote shell command. Returns (command, needs_sudo_password_on_stdin)."""
|
|
safe_cmd = _shell_single_quote(remote_cmd)
|
|
bash_flag = "lc" if login_shell else "c"
|
|
if user == "root" or not need_root:
|
|
return f"bash -{bash_flag} {safe_cmd}", False
|
|
return f"sudo -S -p '' bash -{bash_flag} {safe_cmd}", True
|
|
|
|
|
|
def _configure_ssh_client(client, target: str) -> None:
|
|
import paramiko
|
|
|
|
settings = get_settings()
|
|
if settings.sac_ssh_auto_add_host_key:
|
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
return
|
|
|
|
client.set_missing_host_key_policy(paramiko.RejectPolicy())
|
|
client.load_system_host_keys()
|
|
known_hosts = (settings.sac_ssh_known_hosts_file or "").strip()
|
|
if known_hosts:
|
|
try:
|
|
client.load_host_keys(known_hosts)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _exec_remote_command(
|
|
client,
|
|
*,
|
|
shell_cmd: str,
|
|
password: str,
|
|
needs_sudo_password: bool,
|
|
command_timeout_sec: int,
|
|
) -> tuple[int, str, str]:
|
|
if needs_sudo_password:
|
|
stdin, stdout, stderr = client.exec_command(shell_cmd, timeout=command_timeout_sec, get_pty=True)
|
|
stdin.write(f"{password}\n")
|
|
stdin.flush()
|
|
stdin.channel.shutdown_write()
|
|
else:
|
|
_stdin, stdout, stderr = client.exec_command(shell_cmd, timeout=command_timeout_sec)
|
|
exit_code = stdout.channel.recv_exit_status()
|
|
out_text = _truncate_output(stdout.read().decode("utf-8", errors="replace"))
|
|
err_text = _truncate_output(stderr.read().decode("utf-8", errors="replace"))
|
|
return exit_code, out_text, err_text
|
|
|
|
|
|
def run_ssh_command(
|
|
*,
|
|
target: str,
|
|
user: str,
|
|
password: str,
|
|
remote_cmd: str,
|
|
connect_timeout_sec: int = 15,
|
|
command_timeout_sec: int = 600,
|
|
need_root: bool = False,
|
|
login_shell: bool = True,
|
|
) -> SshCommandResult:
|
|
target = target.strip()
|
|
user = user.strip()
|
|
if not target or not user or not password:
|
|
raise LinuxAdminNotConfiguredError("Linux SSH admin credentials or target missing")
|
|
|
|
try:
|
|
import paramiko
|
|
except ImportError as exc:
|
|
raise RuntimeError("paramiko is not installed on SAC server") from exc
|
|
|
|
shell_cmd, needs_sudo_password = _remote_shell_command(
|
|
user,
|
|
remote_cmd,
|
|
need_root=need_root,
|
|
login_shell=login_shell,
|
|
)
|
|
exit_code: int | None = None
|
|
out_text = ""
|
|
err_text = ""
|
|
|
|
for attempt in range(1, _SSH_CONNECT_ATTEMPTS + 1):
|
|
client = paramiko.SSHClient()
|
|
try:
|
|
_configure_ssh_client(client, target)
|
|
_connect_ssh_client(
|
|
client,
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
connect_timeout_sec=connect_timeout_sec,
|
|
)
|
|
exit_code, out_text, err_text = _exec_remote_command(
|
|
client,
|
|
shell_cmd=shell_cmd,
|
|
password=password,
|
|
needs_sudo_password=needs_sudo_password,
|
|
command_timeout_sec=command_timeout_sec,
|
|
)
|
|
break
|
|
except paramiko.AuthenticationException:
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=f"SSH auth failed for {user}@{target}",
|
|
target=target,
|
|
)
|
|
except (socket.timeout, TimeoutError):
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=f"SSH timeout ({connect_timeout_sec}s connect / {command_timeout_sec}s cmd) to {target}",
|
|
target=target,
|
|
)
|
|
except Exception as exc:
|
|
if attempt < _SSH_CONNECT_ATTEMPTS and _is_transient_ssh_error(exc):
|
|
continue
|
|
hint = _ssh_connect_hint(exc)
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=f"SSH error ({target}): {exc}{hint}",
|
|
target=target,
|
|
)
|
|
finally:
|
|
_close_ssh_client(client)
|
|
else:
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=f"SSH error ({target}): repeated transient connection failures",
|
|
target=target,
|
|
)
|
|
|
|
ok = exit_code == 0
|
|
if ok:
|
|
message = f"SSH OK ({target}), exit 0"
|
|
if out_text.strip():
|
|
message = f"{message}\n{out_text.strip().splitlines()[0][:200]}"
|
|
else:
|
|
detail = err_text.strip() or out_text.strip() or f"exit code {exit_code}"
|
|
message = f"SSH command failed ({target}): {detail[:500]}"
|
|
|
|
return SshCommandResult(
|
|
ok=ok,
|
|
message=message,
|
|
target=target,
|
|
stdout=out_text,
|
|
stderr=err_text,
|
|
exit_code=exit_code,
|
|
)
|
|
|
|
|
|
def _ssh_output_hostname(stdout: str) -> str:
|
|
"""Last line that looks like a hostname (MOTD/login banners may precede command output)."""
|
|
candidates = [ln.strip() for ln in stdout.splitlines() if ln.strip()]
|
|
for line in reversed(candidates):
|
|
if re.match(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,253}$", line):
|
|
return line
|
|
return candidates[-1] if candidates else ""
|
|
|
|
|
|
def test_ssh_connection(
|
|
*,
|
|
target: str,
|
|
user: str,
|
|
password: str,
|
|
timeout_sec: int = 15,
|
|
) -> SshCommandResult:
|
|
result = run_ssh_command(
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
remote_cmd="hostname",
|
|
connect_timeout_sec=timeout_sec,
|
|
command_timeout_sec=timeout_sec,
|
|
login_shell=False,
|
|
)
|
|
if result.ok and result.stdout.strip():
|
|
host = _ssh_output_hostname(result.stdout)
|
|
if not host:
|
|
return result
|
|
return SshCommandResult(
|
|
ok=True,
|
|
message=f"SSH OK, hostname={host}",
|
|
target=target,
|
|
stdout=result.stdout,
|
|
stderr=result.stderr,
|
|
exit_code=result.exit_code,
|
|
)
|
|
return result
|
|
|
|
|
|
def parse_ssh_monitor_version_text(text: str) -> str | None:
|
|
if not text:
|
|
return None
|
|
match = _AGENT_VERSION_RE.search(text)
|
|
if not match:
|
|
return None
|
|
return match.group(1)
|
|
|
|
|
|
def probe_ssh_monitor_version(
|
|
*,
|
|
target: str,
|
|
user: str,
|
|
password: str,
|
|
) -> str | None:
|
|
result = run_ssh_command(
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
remote_cmd=SSH_MONITOR_VERSION_CMD,
|
|
command_timeout_sec=30,
|
|
login_shell=False,
|
|
)
|
|
if result.ok and result.stdout.strip():
|
|
version = parse_ssh_monitor_version_text(result.stdout)
|
|
if version:
|
|
return version
|
|
if result.stdout or result.stderr:
|
|
return parse_ssh_monitor_version_text(f"{result.stdout}\n{result.stderr}")
|
|
return None
|
|
|
|
|
|
def _ssh_monitor_preflight_git_remote(repo_url: str) -> str:
|
|
"""Align clone remotes with SAC-trusted URL (works even before updater script is refreshed)."""
|
|
safe_repo = _shell_single_quote(repo_url.strip())
|
|
repo_dir = _shell_single_quote(f"{SSH_MONITOR_UPDATE_DIR}/{SSH_MONITOR_REPO_NAME}")
|
|
return (
|
|
f"if [ -d {repo_dir}/.git ]; then "
|
|
f"git -C {repo_dir} remote add kalinamall {safe_repo} 2>/dev/null || "
|
|
f"git -C {repo_dir} remote set-url kalinamall {safe_repo}; "
|
|
f"if git -C {repo_dir} remote get-url origin >/dev/null 2>&1; then "
|
|
f"git -C {repo_dir} remote set-url origin {safe_repo}; "
|
|
f"fi; fi"
|
|
)
|
|
|
|
|
|
def _ssh_monitor_update_invoke_command(
|
|
repo_url: str,
|
|
*,
|
|
git_branch: str = "main",
|
|
) -> str:
|
|
"""Run installed updater with SAC-trusted git URL (overrides script default / stale origin)."""
|
|
safe_repo = _shell_single_quote(repo_url.strip())
|
|
safe_branch = _shell_single_quote((git_branch or "main").strip() or "main")
|
|
preflight = _ssh_monitor_preflight_git_remote(repo_url)
|
|
return f"{preflight}; UPDATE_VIA_SAC=1 REPO_URL={safe_repo} GIT_BRANCH={safe_branch} {SSH_MONITOR_UPDATE_SCRIPT}"
|
|
|
|
|
|
def _ssh_monitor_bootstrap_command(repo_url: str, *, git_branch: str = "main") -> str:
|
|
"""Clone ssh-monitor repo and install updater when /opt/scripts/update_ssh_monitor.sh is absent."""
|
|
safe_repo = repo_url.replace("\\", "\\\\").replace('"', '\\"')
|
|
safe_branch = (git_branch or "main").replace("\\", "\\\\").replace('"', '\\"')
|
|
return (
|
|
f'UPDATE_DIR="{SSH_MONITOR_UPDATE_DIR}";'
|
|
f'REPO_URL="{safe_repo}";'
|
|
f'GIT_BRANCH="{safe_branch}";'
|
|
f'SCRIPT_NAME="{SSH_MONITOR_REPO_NAME}";'
|
|
f'INSTALL="{SSH_MONITOR_UPDATE_SCRIPT}";'
|
|
f'mkdir -p "$UPDATE_DIR" && cd "$UPDATE_DIR" && '
|
|
f'([ -d "$SCRIPT_NAME" ] || git clone -b "$GIT_BRANCH" "$REPO_URL" "$SCRIPT_NAME") && '
|
|
f'cp "$UPDATE_DIR/$SCRIPT_NAME/update_ssh_monitor.sh" "$INSTALL" && '
|
|
f'chmod 750 "$INSTALL" && UPDATE_VIA_SAC=1 REPO_URL="$REPO_URL" GIT_BRANCH="$GIT_BRANCH" "$INSTALL" --deploy'
|
|
)
|
|
|
|
|
|
def run_ssh_monitor_update(
|
|
*,
|
|
target: str,
|
|
user: str,
|
|
password: str,
|
|
repo_url: str | None = None,
|
|
git_branch: str | None = None,
|
|
) -> SshCommandResult:
|
|
script = SSH_MONITOR_UPDATE_SCRIPT
|
|
effective_repo = (repo_url or SSH_MONITOR_UPDATE_REPO_URL).strip()
|
|
effective_branch = (git_branch or "main").strip() or "main"
|
|
update_cmd = _ssh_monitor_update_invoke_command(
|
|
effective_repo,
|
|
git_branch=effective_branch,
|
|
)
|
|
check_cmd = f"test -x {script} && echo ready || echo missing"
|
|
probe = run_ssh_command(
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
remote_cmd=check_cmd,
|
|
command_timeout_sec=30,
|
|
login_shell=False,
|
|
)
|
|
if not probe.ok:
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=f"SSH probe failed on {target}: {probe.message}",
|
|
target=target,
|
|
stdout=probe.stdout,
|
|
stderr=probe.stderr,
|
|
exit_code=probe.exit_code,
|
|
)
|
|
|
|
bootstrapped = False
|
|
if "missing" in probe.stdout:
|
|
bootstrap_cmd = _ssh_monitor_bootstrap_command(
|
|
effective_repo,
|
|
git_branch=effective_branch,
|
|
)
|
|
updated = run_ssh_command(
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
remote_cmd=bootstrap_cmd,
|
|
command_timeout_sec=900,
|
|
need_root=True,
|
|
login_shell=False,
|
|
)
|
|
bootstrapped = True
|
|
else:
|
|
updated = run_ssh_command(
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
remote_cmd=update_cmd,
|
|
command_timeout_sec=900,
|
|
need_root=True,
|
|
login_shell=False,
|
|
)
|
|
if not updated.ok:
|
|
prefix = "Updater bootstrap failed" if bootstrapped else "SSH update failed"
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=f"{prefix} ({target}): {updated.message}",
|
|
target=updated.target,
|
|
stdout=updated.stdout,
|
|
stderr=updated.stderr,
|
|
exit_code=updated.exit_code,
|
|
)
|
|
|
|
version = probe_ssh_monitor_version(target=target, user=user, password=password)
|
|
if not version:
|
|
version = parse_ssh_monitor_version_text(updated.stdout)
|
|
if version:
|
|
prefix = "Bootstrap + update OK" if bootstrapped else updated.message
|
|
message = f"{prefix}\nagent version: {version}"
|
|
return SshCommandResult(
|
|
ok=True,
|
|
message=message,
|
|
target=updated.target,
|
|
stdout=updated.stdout,
|
|
stderr=updated.stderr,
|
|
exit_code=updated.exit_code,
|
|
agent_version=version,
|
|
)
|
|
if bootstrapped:
|
|
return SshCommandResult(
|
|
ok=True,
|
|
message=f"Bootstrap + update OK ({target}), exit 0",
|
|
target=updated.target,
|
|
stdout=updated.stdout,
|
|
stderr=updated.stderr,
|
|
exit_code=updated.exit_code,
|
|
)
|
|
return updated
|
|
|
|
|
|
def tail_ssh_monitor_update_log(
|
|
*,
|
|
target: str,
|
|
user: str,
|
|
password: str,
|
|
lines: int = 120,
|
|
) -> str:
|
|
"""Хвост /var/log/update_script.log на удалённом хосте (для live-лога в SAC UI)."""
|
|
safe_lines = max(20, min(int(lines), 400))
|
|
remote_cmd = (
|
|
f"test -r {SSH_MONITOR_UPDATE_LOG_PATH} && "
|
|
f"tail -n {safe_lines} {SSH_MONITOR_UPDATE_LOG_PATH} 2>/dev/null || true"
|
|
)
|
|
result = run_ssh_command(
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
remote_cmd=remote_cmd,
|
|
command_timeout_sec=25,
|
|
need_root=True,
|
|
login_shell=False,
|
|
)
|
|
text = (result.stdout or "").strip()
|
|
return _truncate_output(text)
|