PTah
c21043d845
SAC now clones the ssh-monitor repo and installs update_ssh_monitor.sh before running the update when the remote updater script is absent.
376 lines
11 KiB
Python
376 lines
11 KiB
Python
"""SSH connectivity and remote commands for Linux hosts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import socket
|
|
from dataclasses import dataclass
|
|
|
|
from app.models import Host
|
|
|
|
SSH_MONITOR_UPDATE_SCRIPT = "/opt/scripts/update_ssh_monitor.sh"
|
|
SSH_MONITOR_UPDATE_DIR = "/opt/scripts/update"
|
|
SSH_MONITOR_REPO_NAME = "ssh-monitor"
|
|
SSH_MONITOR_UPDATE_REPO_URL = "https://git.kalinamall.ru/PapaTramp/ssh-monitor.git"
|
|
SSH_MONITOR_BINARY = "/usr/local/bin/ssh-monitor"
|
|
SSH_MONITOR_VERSION_CMD = (
|
|
f"grep -m1 '^SSH_MONITOR_VERSION=' {SSH_MONITOR_BINARY} 2>/dev/null "
|
|
"| sed -E 's/^SSH_MONITOR_VERSION=[\"'\\'']*([^\"'\\'']+)[\"'\\'']*$/\\1/'"
|
|
)
|
|
SSH_OUTPUT_MAX_LEN = 12_000
|
|
_AGENT_VERSION_RE = re.compile(r"(\d+\.\d+\.\d+(?:-SAC)?)", re.IGNORECASE)
|
|
_IPV4_RE = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$")
|
|
|
|
|
|
class LinuxAdminNotConfiguredError(Exception):
|
|
pass
|
|
|
|
|
|
class HostNotLinuxError(Exception):
|
|
pass
|
|
|
|
|
|
class HostTargetMissingError(Exception):
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SshCommandResult:
|
|
ok: bool
|
|
message: str
|
|
target: str
|
|
stdout: str = ""
|
|
stderr: str = ""
|
|
exit_code: int | None = None
|
|
agent_version: str | None = None
|
|
|
|
|
|
def is_linux_host(host: Host) -> bool:
|
|
if (host.os_family or "").strip().lower() == "linux":
|
|
return True
|
|
return (host.product or "").strip() == "ssh-monitor"
|
|
|
|
|
|
def _is_ipv4(value: str) -> bool:
|
|
return bool(_IPV4_RE.match(value.strip()))
|
|
|
|
|
|
def _looks_like_ssh_hostname(value: str) -> bool:
|
|
"""Короткое имя или FQDN; не человекочитаемый display_name вроде «Ubabuba Kalina (10.10.36.9)»."""
|
|
text = value.strip()
|
|
if not text or " " in text or "(" in text or ")" in text:
|
|
return False
|
|
if _is_ipv4(text):
|
|
return True
|
|
if len(text) > 253:
|
|
return False
|
|
return all(ch.isalnum() or ch in ".-_" for ch in text)
|
|
|
|
|
|
def _ssh_name_resolves(name: str) -> bool:
|
|
if _is_ipv4(name):
|
|
return True
|
|
try:
|
|
socket.getaddrinfo(name.strip(), 22, type=socket.SOCK_STREAM)
|
|
return True
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def iter_ssh_targets(host: Host) -> list[str]:
|
|
if not is_linux_host(host):
|
|
raise HostNotLinuxError("Host is not Linux")
|
|
|
|
seen: set[str] = set()
|
|
ordered: list[str] = []
|
|
|
|
def add(value: str | None) -> None:
|
|
text = (value or "").strip()
|
|
if not text or text.casefold() in seen:
|
|
return
|
|
if not _looks_like_ssh_hostname(text):
|
|
return
|
|
if not _ssh_name_resolves(text):
|
|
return
|
|
seen.add(text.casefold())
|
|
ordered.append(text)
|
|
|
|
add(host.hostname)
|
|
|
|
inventory = host.inventory if isinstance(host.inventory, dict) else {}
|
|
computer_name = inventory.get("computer_name")
|
|
if isinstance(computer_name, str):
|
|
add(computer_name)
|
|
|
|
if host.display_name and _looks_like_ssh_hostname(host.display_name):
|
|
add(host.display_name)
|
|
|
|
add(host.ipv4)
|
|
if not ordered:
|
|
raise HostTargetMissingError("Host has no resolvable hostname or IPv4 for SSH")
|
|
return ordered
|
|
|
|
|
|
def _truncate_output(text: str) -> str:
|
|
if len(text) <= SSH_OUTPUT_MAX_LEN:
|
|
return text
|
|
return text[:SSH_OUTPUT_MAX_LEN] + "\n… (truncated)"
|
|
|
|
|
|
def _shell_single_quote(value: str) -> str:
|
|
return "'" + value.replace("'", "'\"'\"'") + "'"
|
|
|
|
|
|
def _remote_shell_command(
|
|
user: str,
|
|
remote_cmd: str,
|
|
password: str,
|
|
*,
|
|
need_root: bool = False,
|
|
) -> str:
|
|
"""Build remote shell command. Sudo only when need_root and user is not root."""
|
|
safe_cmd = _shell_single_quote(remote_cmd)
|
|
if user == "root" or not need_root:
|
|
return f"bash -lc {safe_cmd}"
|
|
|
|
safe_pw = _shell_single_quote(password)
|
|
inner = f"printf '%s\\n' {safe_pw} | sudo -S -p '' bash -lc {safe_cmd}"
|
|
return f"bash -lc {_shell_single_quote(inner)}"
|
|
|
|
|
|
def run_ssh_command(
|
|
*,
|
|
target: str,
|
|
user: str,
|
|
password: str,
|
|
remote_cmd: str,
|
|
connect_timeout_sec: int = 15,
|
|
command_timeout_sec: int = 600,
|
|
need_root: bool = False,
|
|
) -> SshCommandResult:
|
|
target = target.strip()
|
|
user = user.strip()
|
|
if not target or not user or not password:
|
|
raise LinuxAdminNotConfiguredError("Linux SSH admin credentials or target missing")
|
|
|
|
try:
|
|
import paramiko
|
|
except ImportError as exc:
|
|
raise RuntimeError("paramiko is not installed on SAC server") from exc
|
|
|
|
client = paramiko.SSHClient()
|
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
shell_cmd = _remote_shell_command(user, remote_cmd, password, need_root=need_root)
|
|
|
|
try:
|
|
client.connect(
|
|
hostname=target,
|
|
username=user,
|
|
password=password,
|
|
timeout=connect_timeout_sec,
|
|
allow_agent=False,
|
|
look_for_keys=False,
|
|
)
|
|
_stdin, stdout, stderr = client.exec_command(shell_cmd, timeout=command_timeout_sec)
|
|
exit_code = stdout.channel.recv_exit_status()
|
|
out_text = _truncate_output(stdout.read().decode("utf-8", errors="replace"))
|
|
err_text = _truncate_output(stderr.read().decode("utf-8", errors="replace"))
|
|
except paramiko.AuthenticationException:
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=f"SSH auth failed for {user}@{target}",
|
|
target=target,
|
|
)
|
|
except (socket.timeout, TimeoutError):
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=f"SSH timeout ({connect_timeout_sec}s connect / {command_timeout_sec}s cmd) to {target}",
|
|
target=target,
|
|
)
|
|
except Exception as exc:
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=f"SSH error ({target}): {exc}",
|
|
target=target,
|
|
)
|
|
finally:
|
|
client.close()
|
|
|
|
ok = exit_code == 0
|
|
if ok:
|
|
message = f"SSH OK ({target}), exit 0"
|
|
if out_text.strip():
|
|
message = f"{message}\n{out_text.strip().splitlines()[0][:200]}"
|
|
else:
|
|
detail = err_text.strip() or out_text.strip() or f"exit code {exit_code}"
|
|
message = f"SSH command failed ({target}): {detail[:500]}"
|
|
|
|
return SshCommandResult(
|
|
ok=ok,
|
|
message=message,
|
|
target=target,
|
|
stdout=out_text,
|
|
stderr=err_text,
|
|
exit_code=exit_code,
|
|
)
|
|
|
|
|
|
def test_ssh_connection(
|
|
*,
|
|
target: str,
|
|
user: str,
|
|
password: str,
|
|
timeout_sec: int = 15,
|
|
) -> SshCommandResult:
|
|
result = run_ssh_command(
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
remote_cmd="hostname",
|
|
connect_timeout_sec=timeout_sec,
|
|
command_timeout_sec=timeout_sec,
|
|
)
|
|
if result.ok and result.stdout.strip():
|
|
host = result.stdout.strip().splitlines()[0]
|
|
return SshCommandResult(
|
|
ok=True,
|
|
message=f"SSH OK, hostname={host}",
|
|
target=target,
|
|
stdout=result.stdout,
|
|
stderr=result.stderr,
|
|
exit_code=result.exit_code,
|
|
)
|
|
return result
|
|
|
|
|
|
def parse_ssh_monitor_version_text(text: str) -> str | None:
|
|
if not text:
|
|
return None
|
|
match = _AGENT_VERSION_RE.search(text)
|
|
if not match:
|
|
return None
|
|
return match.group(1)
|
|
|
|
|
|
def probe_ssh_monitor_version(
|
|
*,
|
|
target: str,
|
|
user: str,
|
|
password: str,
|
|
) -> str | None:
|
|
result = run_ssh_command(
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
remote_cmd=SSH_MONITOR_VERSION_CMD,
|
|
command_timeout_sec=30,
|
|
)
|
|
if result.ok and result.stdout.strip():
|
|
version = parse_ssh_monitor_version_text(result.stdout)
|
|
if version:
|
|
return version
|
|
if result.stdout or result.stderr:
|
|
return parse_ssh_monitor_version_text(f"{result.stdout}\n{result.stderr}")
|
|
return None
|
|
|
|
|
|
def _ssh_monitor_bootstrap_command(repo_url: str) -> str:
|
|
"""Clone ssh-monitor repo and install updater when /opt/scripts/update_ssh_monitor.sh is absent."""
|
|
safe_repo = repo_url.replace("\\", "\\\\").replace('"', '\\"')
|
|
return (
|
|
f'UPDATE_DIR="{SSH_MONITOR_UPDATE_DIR}";'
|
|
f'REPO_URL="{safe_repo}";'
|
|
f'SCRIPT_NAME="{SSH_MONITOR_REPO_NAME}";'
|
|
f'INSTALL="{SSH_MONITOR_UPDATE_SCRIPT}";'
|
|
f'mkdir -p "$UPDATE_DIR" && cd "$UPDATE_DIR" && '
|
|
f'([ -d "$SCRIPT_NAME" ] || git clone "$REPO_URL" "$SCRIPT_NAME") && '
|
|
f'cp "$UPDATE_DIR/$SCRIPT_NAME/update_ssh_monitor.sh" "$INSTALL" && '
|
|
f'chmod 750 "$INSTALL" && "$INSTALL"'
|
|
)
|
|
|
|
|
|
def run_ssh_monitor_update(
|
|
*,
|
|
target: str,
|
|
user: str,
|
|
password: str,
|
|
repo_url: str | None = None,
|
|
) -> SshCommandResult:
|
|
script = SSH_MONITOR_UPDATE_SCRIPT
|
|
effective_repo = (repo_url or SSH_MONITOR_UPDATE_REPO_URL).strip()
|
|
check_cmd = f"test -x {script} && echo ready || echo missing"
|
|
probe = run_ssh_command(
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
remote_cmd=check_cmd,
|
|
command_timeout_sec=30,
|
|
)
|
|
if not probe.ok:
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=f"SSH probe failed on {target}: {probe.message}",
|
|
target=target,
|
|
stdout=probe.stdout,
|
|
stderr=probe.stderr,
|
|
exit_code=probe.exit_code,
|
|
)
|
|
|
|
bootstrapped = False
|
|
if "missing" in probe.stdout:
|
|
bootstrap_cmd = _ssh_monitor_bootstrap_command(effective_repo)
|
|
updated = run_ssh_command(
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
remote_cmd=bootstrap_cmd,
|
|
command_timeout_sec=900,
|
|
need_root=True,
|
|
)
|
|
bootstrapped = True
|
|
else:
|
|
updated = run_ssh_command(
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
remote_cmd=script,
|
|
command_timeout_sec=900,
|
|
need_root=True,
|
|
)
|
|
if not updated.ok:
|
|
prefix = "Updater bootstrap failed" if bootstrapped else "SSH update failed"
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=f"{prefix} ({target}): {updated.message}",
|
|
target=updated.target,
|
|
stdout=updated.stdout,
|
|
stderr=updated.stderr,
|
|
exit_code=updated.exit_code,
|
|
)
|
|
|
|
version = probe_ssh_monitor_version(target=target, user=user, password=password)
|
|
if not version:
|
|
version = parse_ssh_monitor_version_text(updated.stdout)
|
|
if version:
|
|
prefix = "Bootstrap + update OK" if bootstrapped else updated.message
|
|
message = f"{prefix}\nagent version: {version}"
|
|
return SshCommandResult(
|
|
ok=True,
|
|
message=message,
|
|
target=updated.target,
|
|
stdout=updated.stdout,
|
|
stderr=updated.stderr,
|
|
exit_code=updated.exit_code,
|
|
agent_version=version,
|
|
)
|
|
if bootstrapped:
|
|
return SshCommandResult(
|
|
ok=True,
|
|
message=f"Bootstrap + update OK ({target}), exit 0",
|
|
target=updated.target,
|
|
stdout=updated.stdout,
|
|
stderr=updated.stderr,
|
|
exit_code=updated.exit_code,
|
|
)
|
|
return updated
|