fix(backend): non-login SSH and strict loginctl parse, SSH retry (0.20.31)

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
PTah
2026-06-22 10:54:56 +10:00
parent a51a585b14
commit 54c1d96933
7 changed files with 246 additions and 43 deletions
+34 -3
View File
@@ -35,6 +35,21 @@ SESSION_EVENT_TYPES_LINUX = frozenset(
)
SESSION_EVENT_TYPES_WINDOWS = frozenset({"rdp.login.success"})
_LOGIND_SESSION_ID_RE = re.compile(r"^\d+$|^c\d+$", re.IGNORECASE)
_LOGIND_USER_RE = re.compile(r"^[a-z_][a-z0-9._-]*$", re.IGNORECASE)
_LOGIND_STATES = frozenset(
{
"active",
"online",
"closing",
"opening",
"degraded",
"lingering",
"preparing",
"unknown",
}
)
@dataclass(frozen=True)
class HostSessionRow:
@@ -78,14 +93,27 @@ def is_windows_host_event(event: Event) -> bool:
return host is not None and is_windows_host(host)
def _looks_like_loginctl_row(parts: list[str]) -> bool:
if len(parts) < 6:
return False
sid, uid, user, state = parts[0], parts[1], parts[2], parts[5].lower()
if not _LOGIND_SESSION_ID_RE.match(sid):
return False
if not uid.isdigit():
return False
if not _LOGIND_USER_RE.match(user):
return False
return state in _LOGIND_STATES
def parse_loginctl_sessions(stdout: str) -> list[HostSessionRow]:
rows: list[HostSessionRow] = []
for line in stdout.splitlines():
text = line.strip()
if not text:
if not text or text.startswith("SESSION"):
continue
parts = text.split()
if len(parts) < 3:
if not _looks_like_loginctl_row(parts):
continue
sid, user = parts[0], parts[2]
tty = parts[4] if len(parts) > 4 and parts[4] != "-" else None
@@ -143,7 +171,7 @@ def list_linux_sessions(host: Host, cfg: LinuxAdminConfig) -> tuple[list[HostSes
if not is_linux_host(host):
raise SshHostNotLinuxError("Host is not Linux")
targets = iter_ssh_targets(host)
cmd = "loginctl list-sessions --no-legend --no-pager 2>/dev/null || who -u"
cmd = "loginctl list-sessions --no-legend --no-pager"
last: SshCommandResult | None = None
for target in targets:
result = run_ssh_command(
@@ -154,6 +182,7 @@ def list_linux_sessions(host: Host, cfg: LinuxAdminConfig) -> tuple[list[HostSes
connect_timeout_sec=15,
command_timeout_sec=45,
need_root=True,
login_shell=False,
)
last = result
if result.ok and result.stdout.strip():
@@ -184,6 +213,7 @@ def terminate_linux_session(
connect_timeout_sec=15,
command_timeout_sec=45,
need_root=True,
login_shell=False,
)
last = result
if result.ok:
@@ -286,6 +316,7 @@ def _terminate_linux_user_sessions(host: Host, cfg: LinuxAdminConfig, user: str)
connect_timeout_sec=15,
command_timeout_sec=45,
need_root=True,
login_shell=False,
)
last = result
if result.ok:
+137 -36
View File
@@ -20,6 +20,17 @@ SSH_MONITOR_VERSION_CMD = (
SSH_OUTPUT_MAX_LEN = 12_000
_AGENT_VERSION_RE = re.compile(r"(\d+\.\d+\.\d+(?:-SAC)?)", re.IGNORECASE)
_IPV4_RE = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$")
_SSH_TRANSIENT_ERROR_MARKERS = (
"no existing session",
"error reading ssh protocol banner",
"connection reset",
"connection lost",
"session not active",
"eof",
"socket is closed",
"connection timed out",
)
_SSH_CONNECT_ATTEMPTS = 2
class LinuxAdminNotConfiguredError(Exception):
@@ -117,6 +128,58 @@ def _truncate_output(text: str) -> str:
return text[:SSH_OUTPUT_MAX_LEN] + "\n… (truncated)"
def _is_transient_ssh_error(exc: BaseException) -> bool:
text = str(exc).casefold()
return any(marker in text for marker in _SSH_TRANSIENT_ERROR_MARKERS)
def _ssh_connect_hint(exc: BaseException) -> str:
text = str(exc).casefold()
if "no existing session" in text:
return (
" Сервер закрыл SSH-сессию во время подключения или auth "
"(проверьте пароль admin, лимиты sshd MaxSessions/MaxStartups, "
"доступ SAC→хост по 22/tcp; на SAC не должен мешать ssh-agent)."
)
return ""
def _connect_ssh_client(
client,
*,
target: str,
user: str,
password: str,
connect_timeout_sec: int,
) -> None:
client.connect(
hostname=target,
username=user,
password=password,
timeout=connect_timeout_sec,
banner_timeout=connect_timeout_sec,
auth_timeout=connect_timeout_sec,
allow_agent=False,
look_for_keys=False,
compress=False,
)
def _close_ssh_client(client) -> None:
if client is None:
return
try:
transport = client.get_transport()
if transport is not None and transport.is_active():
transport.close()
except Exception:
pass
try:
client.close()
except Exception:
pass
def _shell_single_quote(value: str) -> str:
return "'" + value.replace("'", "'\"'\"'") + "'"
@@ -127,15 +190,17 @@ def _remote_shell_command(
password: str,
*,
need_root: bool = False,
login_shell: bool = True,
) -> str:
"""Build remote shell command. Sudo only when need_root and user is not root."""
safe_cmd = _shell_single_quote(remote_cmd)
bash_flag = "lc" if login_shell else "c"
if user == "root" or not need_root:
return f"bash -lc {safe_cmd}"
return f"bash -{bash_flag} {safe_cmd}"
safe_pw = _shell_single_quote(password)
inner = f"printf '%s\\n' {safe_pw} | sudo -S -p '' bash -lc {safe_cmd}"
return f"bash -lc {_shell_single_quote(inner)}"
inner = f"printf '%s\\n' {safe_pw} | sudo -S -p '' bash -{bash_flag} {safe_cmd}"
return f"bash -{bash_flag} {_shell_single_quote(inner)}"
def run_ssh_command(
@@ -147,6 +212,7 @@ def run_ssh_command(
connect_timeout_sec: int = 15,
command_timeout_sec: int = 600,
need_root: bool = False,
login_shell: bool = True,
) -> SshCommandResult:
target = target.strip()
user = user.strip()
@@ -158,43 +224,62 @@ def run_ssh_command(
except ImportError as exc:
raise RuntimeError("paramiko is not installed on SAC server") from exc
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
shell_cmd = _remote_shell_command(user, remote_cmd, password, need_root=need_root)
shell_cmd = _remote_shell_command(
user,
remote_cmd,
password,
need_root=need_root,
login_shell=login_shell,
)
exit_code: int | None = None
out_text = ""
err_text = ""
try:
client.connect(
hostname=target,
username=user,
password=password,
timeout=connect_timeout_sec,
allow_agent=False,
look_for_keys=False,
)
_stdin, stdout, stderr = client.exec_command(shell_cmd, timeout=command_timeout_sec)
exit_code = stdout.channel.recv_exit_status()
out_text = _truncate_output(stdout.read().decode("utf-8", errors="replace"))
err_text = _truncate_output(stderr.read().decode("utf-8", errors="replace"))
except paramiko.AuthenticationException:
for attempt in range(1, _SSH_CONNECT_ATTEMPTS + 1):
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
_connect_ssh_client(
client,
target=target,
user=user,
password=password,
connect_timeout_sec=connect_timeout_sec,
)
_stdin, stdout, stderr = client.exec_command(shell_cmd, timeout=command_timeout_sec)
exit_code = stdout.channel.recv_exit_status()
out_text = _truncate_output(stdout.read().decode("utf-8", errors="replace"))
err_text = _truncate_output(stderr.read().decode("utf-8", errors="replace"))
break
except paramiko.AuthenticationException:
return SshCommandResult(
ok=False,
message=f"SSH auth failed for {user}@{target}",
target=target,
)
except (socket.timeout, TimeoutError):
return SshCommandResult(
ok=False,
message=f"SSH timeout ({connect_timeout_sec}s connect / {command_timeout_sec}s cmd) to {target}",
target=target,
)
except Exception as exc:
if attempt < _SSH_CONNECT_ATTEMPTS and _is_transient_ssh_error(exc):
continue
hint = _ssh_connect_hint(exc)
return SshCommandResult(
ok=False,
message=f"SSH error ({target}): {exc}{hint}",
target=target,
)
finally:
_close_ssh_client(client)
else:
return SshCommandResult(
ok=False,
message=f"SSH auth failed for {user}@{target}",
message=f"SSH error ({target}): repeated transient connection failures",
target=target,
)
except (socket.timeout, TimeoutError):
return SshCommandResult(
ok=False,
message=f"SSH timeout ({connect_timeout_sec}s connect / {command_timeout_sec}s cmd) to {target}",
target=target,
)
except Exception as exc:
return SshCommandResult(
ok=False,
message=f"SSH error ({target}): {exc}",
target=target,
)
finally:
client.close()
ok = exit_code == 0
if ok:
@@ -215,6 +300,15 @@ def run_ssh_command(
)
def _ssh_output_hostname(stdout: str) -> str:
"""Last line that looks like a hostname (MOTD/login banners may precede command output)."""
candidates = [ln.strip() for ln in stdout.splitlines() if ln.strip()]
for line in reversed(candidates):
if re.match(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,253}$", line):
return line
return candidates[-1] if candidates else ""
def test_ssh_connection(
*,
target: str,
@@ -229,9 +323,12 @@ def test_ssh_connection(
remote_cmd="hostname",
connect_timeout_sec=timeout_sec,
command_timeout_sec=timeout_sec,
login_shell=False,
)
if result.ok and result.stdout.strip():
host = result.stdout.strip().splitlines()[0]
host = _ssh_output_hostname(result.stdout)
if not host:
return result
return SshCommandResult(
ok=True,
message=f"SSH OK, hostname={host}",
@@ -264,6 +361,7 @@ def probe_ssh_monitor_version(
password=password,
remote_cmd=SSH_MONITOR_VERSION_CMD,
command_timeout_sec=30,
login_shell=False,
)
if result.ok and result.stdout.strip():
version = parse_ssh_monitor_version_text(result.stdout)
@@ -339,6 +437,7 @@ def run_ssh_monitor_update(
password=password,
remote_cmd=check_cmd,
command_timeout_sec=30,
login_shell=False,
)
if not probe.ok:
return SshCommandResult(
@@ -363,6 +462,7 @@ def run_ssh_monitor_update(
remote_cmd=bootstrap_cmd,
command_timeout_sec=900,
need_root=True,
login_shell=False,
)
bootstrapped = True
else:
@@ -373,6 +473,7 @@ def run_ssh_monitor_update(
remote_cmd=update_cmd,
command_timeout_sec=900,
need_root=True,
login_shell=False,
)
if not updated.ok:
prefix = "Updater bootstrap failed" if bootstrapped else "SSH update failed"