PTah
ec5ec62240
Pass REPO_URL/GIT_BRANCH from agent settings over SSH, preflight git remotes on stale hosts, and stop using GitHub origin on routers.
413 lines
13 KiB
Python
413 lines
13 KiB
Python
"""SSH connectivity and remote commands for Linux hosts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import socket
|
|
from dataclasses import dataclass
|
|
|
|
from app.models import Host
|
|
|
|
SSH_MONITOR_UPDATE_SCRIPT = "/opt/scripts/update_ssh_monitor.sh"
|
|
SSH_MONITOR_UPDATE_DIR = "/opt/scripts/update"
|
|
SSH_MONITOR_REPO_NAME = "ssh-monitor"
|
|
SSH_MONITOR_UPDATE_REPO_URL = "https://git.kalinamall.ru/PapaTramp/ssh-monitor.git"
|
|
SSH_MONITOR_BINARY = "/usr/local/bin/ssh-monitor"
|
|
SSH_MONITOR_VERSION_CMD = (
|
|
f"grep -m1 '^SSH_MONITOR_VERSION=' {SSH_MONITOR_BINARY} 2>/dev/null "
|
|
"| sed -E 's/^SSH_MONITOR_VERSION=[\"'\\'']*([^\"'\\'']+)[\"'\\'']*$/\\1/'"
|
|
)
|
|
SSH_OUTPUT_MAX_LEN = 12_000
|
|
_AGENT_VERSION_RE = re.compile(r"(\d+\.\d+\.\d+(?:-SAC)?)", re.IGNORECASE)
|
|
_IPV4_RE = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$")
|
|
|
|
|
|
class LinuxAdminNotConfiguredError(Exception):
|
|
pass
|
|
|
|
|
|
class HostNotLinuxError(Exception):
|
|
pass
|
|
|
|
|
|
class HostTargetMissingError(Exception):
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SshCommandResult:
|
|
ok: bool
|
|
message: str
|
|
target: str
|
|
stdout: str = ""
|
|
stderr: str = ""
|
|
exit_code: int | None = None
|
|
agent_version: str | None = None
|
|
|
|
|
|
def is_linux_host(host: Host) -> bool:
|
|
if (host.os_family or "").strip().lower() == "linux":
|
|
return True
|
|
return (host.product or "").strip() == "ssh-monitor"
|
|
|
|
|
|
def _is_ipv4(value: str) -> bool:
|
|
return bool(_IPV4_RE.match(value.strip()))
|
|
|
|
|
|
def _looks_like_ssh_hostname(value: str) -> bool:
|
|
"""Короткое имя или FQDN; не человекочитаемый display_name вроде «Ubabuba Kalina (10.10.36.9)»."""
|
|
text = value.strip()
|
|
if not text or " " in text or "(" in text or ")" in text:
|
|
return False
|
|
if _is_ipv4(text):
|
|
return True
|
|
if len(text) > 253:
|
|
return False
|
|
return all(ch.isalnum() or ch in ".-_" for ch in text)
|
|
|
|
|
|
def _ssh_name_resolves(name: str) -> bool:
|
|
if _is_ipv4(name):
|
|
return True
|
|
try:
|
|
socket.getaddrinfo(name.strip(), 22, type=socket.SOCK_STREAM)
|
|
return True
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def iter_ssh_targets(host: Host) -> list[str]:
|
|
if not is_linux_host(host):
|
|
raise HostNotLinuxError("Host is not Linux")
|
|
|
|
seen: set[str] = set()
|
|
ordered: list[str] = []
|
|
|
|
def add(value: str | None) -> None:
|
|
text = (value or "").strip()
|
|
if not text or text.casefold() in seen:
|
|
return
|
|
if not _looks_like_ssh_hostname(text):
|
|
return
|
|
if not _ssh_name_resolves(text):
|
|
return
|
|
seen.add(text.casefold())
|
|
ordered.append(text)
|
|
|
|
add(host.hostname)
|
|
|
|
inventory = host.inventory if isinstance(host.inventory, dict) else {}
|
|
computer_name = inventory.get("computer_name")
|
|
if isinstance(computer_name, str):
|
|
add(computer_name)
|
|
|
|
if host.display_name and _looks_like_ssh_hostname(host.display_name):
|
|
add(host.display_name)
|
|
|
|
add(host.ipv4)
|
|
if not ordered:
|
|
raise HostTargetMissingError("Host has no resolvable hostname or IPv4 for SSH")
|
|
return ordered
|
|
|
|
|
|
def _truncate_output(text: str) -> str:
|
|
if len(text) <= SSH_OUTPUT_MAX_LEN:
|
|
return text
|
|
return text[:SSH_OUTPUT_MAX_LEN] + "\n… (truncated)"
|
|
|
|
|
|
def _shell_single_quote(value: str) -> str:
|
|
return "'" + value.replace("'", "'\"'\"'") + "'"
|
|
|
|
|
|
def _remote_shell_command(
|
|
user: str,
|
|
remote_cmd: str,
|
|
password: str,
|
|
*,
|
|
need_root: bool = False,
|
|
) -> str:
|
|
"""Build remote shell command. Sudo only when need_root and user is not root."""
|
|
safe_cmd = _shell_single_quote(remote_cmd)
|
|
if user == "root" or not need_root:
|
|
return f"bash -lc {safe_cmd}"
|
|
|
|
safe_pw = _shell_single_quote(password)
|
|
inner = f"printf '%s\\n' {safe_pw} | sudo -S -p '' bash -lc {safe_cmd}"
|
|
return f"bash -lc {_shell_single_quote(inner)}"
|
|
|
|
|
|
def run_ssh_command(
|
|
*,
|
|
target: str,
|
|
user: str,
|
|
password: str,
|
|
remote_cmd: str,
|
|
connect_timeout_sec: int = 15,
|
|
command_timeout_sec: int = 600,
|
|
need_root: bool = False,
|
|
) -> SshCommandResult:
|
|
target = target.strip()
|
|
user = user.strip()
|
|
if not target or not user or not password:
|
|
raise LinuxAdminNotConfiguredError("Linux SSH admin credentials or target missing")
|
|
|
|
try:
|
|
import paramiko
|
|
except ImportError as exc:
|
|
raise RuntimeError("paramiko is not installed on SAC server") from exc
|
|
|
|
client = paramiko.SSHClient()
|
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
shell_cmd = _remote_shell_command(user, remote_cmd, password, need_root=need_root)
|
|
|
|
try:
|
|
client.connect(
|
|
hostname=target,
|
|
username=user,
|
|
password=password,
|
|
timeout=connect_timeout_sec,
|
|
allow_agent=False,
|
|
look_for_keys=False,
|
|
)
|
|
_stdin, stdout, stderr = client.exec_command(shell_cmd, timeout=command_timeout_sec)
|
|
exit_code = stdout.channel.recv_exit_status()
|
|
out_text = _truncate_output(stdout.read().decode("utf-8", errors="replace"))
|
|
err_text = _truncate_output(stderr.read().decode("utf-8", errors="replace"))
|
|
except paramiko.AuthenticationException:
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=f"SSH auth failed for {user}@{target}",
|
|
target=target,
|
|
)
|
|
except (socket.timeout, TimeoutError):
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=f"SSH timeout ({connect_timeout_sec}s connect / {command_timeout_sec}s cmd) to {target}",
|
|
target=target,
|
|
)
|
|
except Exception as exc:
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=f"SSH error ({target}): {exc}",
|
|
target=target,
|
|
)
|
|
finally:
|
|
client.close()
|
|
|
|
ok = exit_code == 0
|
|
if ok:
|
|
message = f"SSH OK ({target}), exit 0"
|
|
if out_text.strip():
|
|
message = f"{message}\n{out_text.strip().splitlines()[0][:200]}"
|
|
else:
|
|
detail = err_text.strip() or out_text.strip() or f"exit code {exit_code}"
|
|
message = f"SSH command failed ({target}): {detail[:500]}"
|
|
|
|
return SshCommandResult(
|
|
ok=ok,
|
|
message=message,
|
|
target=target,
|
|
stdout=out_text,
|
|
stderr=err_text,
|
|
exit_code=exit_code,
|
|
)
|
|
|
|
|
|
def test_ssh_connection(
|
|
*,
|
|
target: str,
|
|
user: str,
|
|
password: str,
|
|
timeout_sec: int = 15,
|
|
) -> SshCommandResult:
|
|
result = run_ssh_command(
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
remote_cmd="hostname",
|
|
connect_timeout_sec=timeout_sec,
|
|
command_timeout_sec=timeout_sec,
|
|
)
|
|
if result.ok and result.stdout.strip():
|
|
host = result.stdout.strip().splitlines()[0]
|
|
return SshCommandResult(
|
|
ok=True,
|
|
message=f"SSH OK, hostname={host}",
|
|
target=target,
|
|
stdout=result.stdout,
|
|
stderr=result.stderr,
|
|
exit_code=result.exit_code,
|
|
)
|
|
return result
|
|
|
|
|
|
def parse_ssh_monitor_version_text(text: str) -> str | None:
|
|
if not text:
|
|
return None
|
|
match = _AGENT_VERSION_RE.search(text)
|
|
if not match:
|
|
return None
|
|
return match.group(1)
|
|
|
|
|
|
def probe_ssh_monitor_version(
|
|
*,
|
|
target: str,
|
|
user: str,
|
|
password: str,
|
|
) -> str | None:
|
|
result = run_ssh_command(
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
remote_cmd=SSH_MONITOR_VERSION_CMD,
|
|
command_timeout_sec=30,
|
|
)
|
|
if result.ok and result.stdout.strip():
|
|
version = parse_ssh_monitor_version_text(result.stdout)
|
|
if version:
|
|
return version
|
|
if result.stdout or result.stderr:
|
|
return parse_ssh_monitor_version_text(f"{result.stdout}\n{result.stderr}")
|
|
return None
|
|
|
|
|
|
def _ssh_monitor_preflight_git_remote(repo_url: str) -> str:
|
|
"""Align clone remotes with SAC-trusted URL (works even before updater script is refreshed)."""
|
|
safe_repo = _shell_single_quote(repo_url.strip())
|
|
repo_dir = _shell_single_quote(f"{SSH_MONITOR_UPDATE_DIR}/{SSH_MONITOR_REPO_NAME}")
|
|
return (
|
|
f"if [ -d {repo_dir}/.git ]; then "
|
|
f"git -C {repo_dir} remote add kalinamall {safe_repo} 2>/dev/null || "
|
|
f"git -C {repo_dir} remote set-url kalinamall {safe_repo}; "
|
|
f"if git -C {repo_dir} remote get-url origin >/dev/null 2>&1; then "
|
|
f"git -C {repo_dir} remote set-url origin {safe_repo}; "
|
|
f"fi; fi"
|
|
)
|
|
|
|
|
|
def _ssh_monitor_update_invoke_command(
|
|
repo_url: str,
|
|
*,
|
|
git_branch: str = "main",
|
|
) -> str:
|
|
"""Run installed updater with SAC-trusted git URL (overrides script default / stale origin)."""
|
|
safe_repo = _shell_single_quote(repo_url.strip())
|
|
safe_branch = _shell_single_quote((git_branch or "main").strip() or "main")
|
|
preflight = _ssh_monitor_preflight_git_remote(repo_url)
|
|
return f"{preflight}; REPO_URL={safe_repo} GIT_BRANCH={safe_branch} {SSH_MONITOR_UPDATE_SCRIPT}"
|
|
|
|
|
|
def _ssh_monitor_bootstrap_command(repo_url: str, *, git_branch: str = "main") -> str:
|
|
"""Clone ssh-monitor repo and install updater when /opt/scripts/update_ssh_monitor.sh is absent."""
|
|
safe_repo = repo_url.replace("\\", "\\\\").replace('"', '\\"')
|
|
safe_branch = (git_branch or "main").replace("\\", "\\\\").replace('"', '\\"')
|
|
return (
|
|
f'UPDATE_DIR="{SSH_MONITOR_UPDATE_DIR}";'
|
|
f'REPO_URL="{safe_repo}";'
|
|
f'GIT_BRANCH="{safe_branch}";'
|
|
f'SCRIPT_NAME="{SSH_MONITOR_REPO_NAME}";'
|
|
f'INSTALL="{SSH_MONITOR_UPDATE_SCRIPT}";'
|
|
f'mkdir -p "$UPDATE_DIR" && cd "$UPDATE_DIR" && '
|
|
f'([ -d "$SCRIPT_NAME" ] || git clone -b "$GIT_BRANCH" "$REPO_URL" "$SCRIPT_NAME") && '
|
|
f'cp "$UPDATE_DIR/$SCRIPT_NAME/update_ssh_monitor.sh" "$INSTALL" && '
|
|
f'chmod 750 "$INSTALL" && REPO_URL="$REPO_URL" GIT_BRANCH="$GIT_BRANCH" "$INSTALL"'
|
|
)
|
|
|
|
|
|
def run_ssh_monitor_update(
|
|
*,
|
|
target: str,
|
|
user: str,
|
|
password: str,
|
|
repo_url: str | None = None,
|
|
git_branch: str | None = None,
|
|
) -> SshCommandResult:
|
|
script = SSH_MONITOR_UPDATE_SCRIPT
|
|
effective_repo = (repo_url or SSH_MONITOR_UPDATE_REPO_URL).strip()
|
|
effective_branch = (git_branch or "main").strip() or "main"
|
|
update_cmd = _ssh_monitor_update_invoke_command(
|
|
effective_repo,
|
|
git_branch=effective_branch,
|
|
)
|
|
check_cmd = f"test -x {script} && echo ready || echo missing"
|
|
probe = run_ssh_command(
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
remote_cmd=check_cmd,
|
|
command_timeout_sec=30,
|
|
)
|
|
if not probe.ok:
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=f"SSH probe failed on {target}: {probe.message}",
|
|
target=target,
|
|
stdout=probe.stdout,
|
|
stderr=probe.stderr,
|
|
exit_code=probe.exit_code,
|
|
)
|
|
|
|
bootstrapped = False
|
|
if "missing" in probe.stdout:
|
|
bootstrap_cmd = _ssh_monitor_bootstrap_command(
|
|
effective_repo,
|
|
git_branch=effective_branch,
|
|
)
|
|
updated = run_ssh_command(
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
remote_cmd=bootstrap_cmd,
|
|
command_timeout_sec=900,
|
|
need_root=True,
|
|
)
|
|
bootstrapped = True
|
|
else:
|
|
updated = run_ssh_command(
|
|
target=target,
|
|
user=user,
|
|
password=password,
|
|
remote_cmd=update_cmd,
|
|
command_timeout_sec=900,
|
|
need_root=True,
|
|
)
|
|
if not updated.ok:
|
|
prefix = "Updater bootstrap failed" if bootstrapped else "SSH update failed"
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=f"{prefix} ({target}): {updated.message}",
|
|
target=updated.target,
|
|
stdout=updated.stdout,
|
|
stderr=updated.stderr,
|
|
exit_code=updated.exit_code,
|
|
)
|
|
|
|
version = probe_ssh_monitor_version(target=target, user=user, password=password)
|
|
if not version:
|
|
version = parse_ssh_monitor_version_text(updated.stdout)
|
|
if version:
|
|
prefix = "Bootstrap + update OK" if bootstrapped else updated.message
|
|
message = f"{prefix}\nagent version: {version}"
|
|
return SshCommandResult(
|
|
ok=True,
|
|
message=message,
|
|
target=updated.target,
|
|
stdout=updated.stdout,
|
|
stderr=updated.stderr,
|
|
exit_code=updated.exit_code,
|
|
agent_version=version,
|
|
)
|
|
if bootstrapped:
|
|
return SshCommandResult(
|
|
ok=True,
|
|
message=f"Bootstrap + update OK ({target}), exit 0",
|
|
target=updated.target,
|
|
stdout=updated.stdout,
|
|
stderr=updated.stderr,
|
|
exit_code=updated.exit_code,
|
|
)
|
|
return updated
|