fix(backend): non-login SSH and strict loginctl parse, SSH retry (0.20.31)
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -35,6 +35,21 @@ SESSION_EVENT_TYPES_LINUX = frozenset(
|
||||
)
|
||||
SESSION_EVENT_TYPES_WINDOWS = frozenset({"rdp.login.success"})
|
||||
|
||||
_LOGIND_SESSION_ID_RE = re.compile(r"^\d+$|^c\d+$", re.IGNORECASE)
|
||||
_LOGIND_USER_RE = re.compile(r"^[a-z_][a-z0-9._-]*$", re.IGNORECASE)
|
||||
_LOGIND_STATES = frozenset(
|
||||
{
|
||||
"active",
|
||||
"online",
|
||||
"closing",
|
||||
"opening",
|
||||
"degraded",
|
||||
"lingering",
|
||||
"preparing",
|
||||
"unknown",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HostSessionRow:
|
||||
@@ -78,14 +93,27 @@ def is_windows_host_event(event: Event) -> bool:
|
||||
return host is not None and is_windows_host(host)
|
||||
|
||||
|
||||
def _looks_like_loginctl_row(parts: list[str]) -> bool:
|
||||
if len(parts) < 6:
|
||||
return False
|
||||
sid, uid, user, state = parts[0], parts[1], parts[2], parts[5].lower()
|
||||
if not _LOGIND_SESSION_ID_RE.match(sid):
|
||||
return False
|
||||
if not uid.isdigit():
|
||||
return False
|
||||
if not _LOGIND_USER_RE.match(user):
|
||||
return False
|
||||
return state in _LOGIND_STATES
|
||||
|
||||
|
||||
def parse_loginctl_sessions(stdout: str) -> list[HostSessionRow]:
|
||||
rows: list[HostSessionRow] = []
|
||||
for line in stdout.splitlines():
|
||||
text = line.strip()
|
||||
if not text:
|
||||
if not text or text.startswith("SESSION"):
|
||||
continue
|
||||
parts = text.split()
|
||||
if len(parts) < 3:
|
||||
if not _looks_like_loginctl_row(parts):
|
||||
continue
|
||||
sid, user = parts[0], parts[2]
|
||||
tty = parts[4] if len(parts) > 4 and parts[4] != "-" else None
|
||||
@@ -143,7 +171,7 @@ def list_linux_sessions(host: Host, cfg: LinuxAdminConfig) -> tuple[list[HostSes
|
||||
if not is_linux_host(host):
|
||||
raise SshHostNotLinuxError("Host is not Linux")
|
||||
targets = iter_ssh_targets(host)
|
||||
cmd = "loginctl list-sessions --no-legend --no-pager 2>/dev/null || who -u"
|
||||
cmd = "loginctl list-sessions --no-legend --no-pager"
|
||||
last: SshCommandResult | None = None
|
||||
for target in targets:
|
||||
result = run_ssh_command(
|
||||
@@ -154,6 +182,7 @@ def list_linux_sessions(host: Host, cfg: LinuxAdminConfig) -> tuple[list[HostSes
|
||||
connect_timeout_sec=15,
|
||||
command_timeout_sec=45,
|
||||
need_root=True,
|
||||
login_shell=False,
|
||||
)
|
||||
last = result
|
||||
if result.ok and result.stdout.strip():
|
||||
@@ -184,6 +213,7 @@ def terminate_linux_session(
|
||||
connect_timeout_sec=15,
|
||||
command_timeout_sec=45,
|
||||
need_root=True,
|
||||
login_shell=False,
|
||||
)
|
||||
last = result
|
||||
if result.ok:
|
||||
@@ -286,6 +316,7 @@ def _terminate_linux_user_sessions(host: Host, cfg: LinuxAdminConfig, user: str)
|
||||
connect_timeout_sec=15,
|
||||
command_timeout_sec=45,
|
||||
need_root=True,
|
||||
login_shell=False,
|
||||
)
|
||||
last = result
|
||||
if result.ok:
|
||||
|
||||
@@ -20,6 +20,17 @@ SSH_MONITOR_VERSION_CMD = (
|
||||
SSH_OUTPUT_MAX_LEN = 12_000
|
||||
_AGENT_VERSION_RE = re.compile(r"(\d+\.\d+\.\d+(?:-SAC)?)", re.IGNORECASE)
|
||||
_IPV4_RE = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$")
|
||||
_SSH_TRANSIENT_ERROR_MARKERS = (
|
||||
"no existing session",
|
||||
"error reading ssh protocol banner",
|
||||
"connection reset",
|
||||
"connection lost",
|
||||
"session not active",
|
||||
"eof",
|
||||
"socket is closed",
|
||||
"connection timed out",
|
||||
)
|
||||
_SSH_CONNECT_ATTEMPTS = 2
|
||||
|
||||
|
||||
class LinuxAdminNotConfiguredError(Exception):
|
||||
@@ -117,6 +128,58 @@ def _truncate_output(text: str) -> str:
|
||||
return text[:SSH_OUTPUT_MAX_LEN] + "\n… (truncated)"
|
||||
|
||||
|
||||
def _is_transient_ssh_error(exc: BaseException) -> bool:
|
||||
text = str(exc).casefold()
|
||||
return any(marker in text for marker in _SSH_TRANSIENT_ERROR_MARKERS)
|
||||
|
||||
|
||||
def _ssh_connect_hint(exc: BaseException) -> str:
|
||||
text = str(exc).casefold()
|
||||
if "no existing session" in text:
|
||||
return (
|
||||
" Сервер закрыл SSH-сессию во время подключения или auth "
|
||||
"(проверьте пароль admin, лимиты sshd MaxSessions/MaxStartups, "
|
||||
"доступ SAC→хост по 22/tcp; на SAC не должен мешать ssh-agent)."
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
def _connect_ssh_client(
|
||||
client,
|
||||
*,
|
||||
target: str,
|
||||
user: str,
|
||||
password: str,
|
||||
connect_timeout_sec: int,
|
||||
) -> None:
|
||||
client.connect(
|
||||
hostname=target,
|
||||
username=user,
|
||||
password=password,
|
||||
timeout=connect_timeout_sec,
|
||||
banner_timeout=connect_timeout_sec,
|
||||
auth_timeout=connect_timeout_sec,
|
||||
allow_agent=False,
|
||||
look_for_keys=False,
|
||||
compress=False,
|
||||
)
|
||||
|
||||
|
||||
def _close_ssh_client(client) -> None:
|
||||
if client is None:
|
||||
return
|
||||
try:
|
||||
transport = client.get_transport()
|
||||
if transport is not None and transport.is_active():
|
||||
transport.close()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
client.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _shell_single_quote(value: str) -> str:
|
||||
return "'" + value.replace("'", "'\"'\"'") + "'"
|
||||
|
||||
@@ -127,15 +190,17 @@ def _remote_shell_command(
|
||||
password: str,
|
||||
*,
|
||||
need_root: bool = False,
|
||||
login_shell: bool = True,
|
||||
) -> str:
|
||||
"""Build remote shell command. Sudo only when need_root and user is not root."""
|
||||
safe_cmd = _shell_single_quote(remote_cmd)
|
||||
bash_flag = "lc" if login_shell else "c"
|
||||
if user == "root" or not need_root:
|
||||
return f"bash -lc {safe_cmd}"
|
||||
return f"bash -{bash_flag} {safe_cmd}"
|
||||
|
||||
safe_pw = _shell_single_quote(password)
|
||||
inner = f"printf '%s\\n' {safe_pw} | sudo -S -p '' bash -lc {safe_cmd}"
|
||||
return f"bash -lc {_shell_single_quote(inner)}"
|
||||
inner = f"printf '%s\\n' {safe_pw} | sudo -S -p '' bash -{bash_flag} {safe_cmd}"
|
||||
return f"bash -{bash_flag} {_shell_single_quote(inner)}"
|
||||
|
||||
|
||||
def run_ssh_command(
|
||||
@@ -147,6 +212,7 @@ def run_ssh_command(
|
||||
connect_timeout_sec: int = 15,
|
||||
command_timeout_sec: int = 600,
|
||||
need_root: bool = False,
|
||||
login_shell: bool = True,
|
||||
) -> SshCommandResult:
|
||||
target = target.strip()
|
||||
user = user.strip()
|
||||
@@ -158,43 +224,62 @@ def run_ssh_command(
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("paramiko is not installed on SAC server") from exc
|
||||
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
shell_cmd = _remote_shell_command(user, remote_cmd, password, need_root=need_root)
|
||||
shell_cmd = _remote_shell_command(
|
||||
user,
|
||||
remote_cmd,
|
||||
password,
|
||||
need_root=need_root,
|
||||
login_shell=login_shell,
|
||||
)
|
||||
exit_code: int | None = None
|
||||
out_text = ""
|
||||
err_text = ""
|
||||
|
||||
try:
|
||||
client.connect(
|
||||
hostname=target,
|
||||
username=user,
|
||||
password=password,
|
||||
timeout=connect_timeout_sec,
|
||||
allow_agent=False,
|
||||
look_for_keys=False,
|
||||
)
|
||||
_stdin, stdout, stderr = client.exec_command(shell_cmd, timeout=command_timeout_sec)
|
||||
exit_code = stdout.channel.recv_exit_status()
|
||||
out_text = _truncate_output(stdout.read().decode("utf-8", errors="replace"))
|
||||
err_text = _truncate_output(stderr.read().decode("utf-8", errors="replace"))
|
||||
except paramiko.AuthenticationException:
|
||||
for attempt in range(1, _SSH_CONNECT_ATTEMPTS + 1):
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
try:
|
||||
_connect_ssh_client(
|
||||
client,
|
||||
target=target,
|
||||
user=user,
|
||||
password=password,
|
||||
connect_timeout_sec=connect_timeout_sec,
|
||||
)
|
||||
_stdin, stdout, stderr = client.exec_command(shell_cmd, timeout=command_timeout_sec)
|
||||
exit_code = stdout.channel.recv_exit_status()
|
||||
out_text = _truncate_output(stdout.read().decode("utf-8", errors="replace"))
|
||||
err_text = _truncate_output(stderr.read().decode("utf-8", errors="replace"))
|
||||
break
|
||||
except paramiko.AuthenticationException:
|
||||
return SshCommandResult(
|
||||
ok=False,
|
||||
message=f"SSH auth failed for {user}@{target}",
|
||||
target=target,
|
||||
)
|
||||
except (socket.timeout, TimeoutError):
|
||||
return SshCommandResult(
|
||||
ok=False,
|
||||
message=f"SSH timeout ({connect_timeout_sec}s connect / {command_timeout_sec}s cmd) to {target}",
|
||||
target=target,
|
||||
)
|
||||
except Exception as exc:
|
||||
if attempt < _SSH_CONNECT_ATTEMPTS and _is_transient_ssh_error(exc):
|
||||
continue
|
||||
hint = _ssh_connect_hint(exc)
|
||||
return SshCommandResult(
|
||||
ok=False,
|
||||
message=f"SSH error ({target}): {exc}{hint}",
|
||||
target=target,
|
||||
)
|
||||
finally:
|
||||
_close_ssh_client(client)
|
||||
else:
|
||||
return SshCommandResult(
|
||||
ok=False,
|
||||
message=f"SSH auth failed for {user}@{target}",
|
||||
message=f"SSH error ({target}): repeated transient connection failures",
|
||||
target=target,
|
||||
)
|
||||
except (socket.timeout, TimeoutError):
|
||||
return SshCommandResult(
|
||||
ok=False,
|
||||
message=f"SSH timeout ({connect_timeout_sec}s connect / {command_timeout_sec}s cmd) to {target}",
|
||||
target=target,
|
||||
)
|
||||
except Exception as exc:
|
||||
return SshCommandResult(
|
||||
ok=False,
|
||||
message=f"SSH error ({target}): {exc}",
|
||||
target=target,
|
||||
)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
ok = exit_code == 0
|
||||
if ok:
|
||||
@@ -215,6 +300,15 @@ def run_ssh_command(
|
||||
)
|
||||
|
||||
|
||||
def _ssh_output_hostname(stdout: str) -> str:
|
||||
"""Last line that looks like a hostname (MOTD/login banners may precede command output)."""
|
||||
candidates = [ln.strip() for ln in stdout.splitlines() if ln.strip()]
|
||||
for line in reversed(candidates):
|
||||
if re.match(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,253}$", line):
|
||||
return line
|
||||
return candidates[-1] if candidates else ""
|
||||
|
||||
|
||||
def test_ssh_connection(
|
||||
*,
|
||||
target: str,
|
||||
@@ -229,9 +323,12 @@ def test_ssh_connection(
|
||||
remote_cmd="hostname",
|
||||
connect_timeout_sec=timeout_sec,
|
||||
command_timeout_sec=timeout_sec,
|
||||
login_shell=False,
|
||||
)
|
||||
if result.ok and result.stdout.strip():
|
||||
host = result.stdout.strip().splitlines()[0]
|
||||
host = _ssh_output_hostname(result.stdout)
|
||||
if not host:
|
||||
return result
|
||||
return SshCommandResult(
|
||||
ok=True,
|
||||
message=f"SSH OK, hostname={host}",
|
||||
@@ -264,6 +361,7 @@ def probe_ssh_monitor_version(
|
||||
password=password,
|
||||
remote_cmd=SSH_MONITOR_VERSION_CMD,
|
||||
command_timeout_sec=30,
|
||||
login_shell=False,
|
||||
)
|
||||
if result.ok and result.stdout.strip():
|
||||
version = parse_ssh_monitor_version_text(result.stdout)
|
||||
@@ -339,6 +437,7 @@ def run_ssh_monitor_update(
|
||||
password=password,
|
||||
remote_cmd=check_cmd,
|
||||
command_timeout_sec=30,
|
||||
login_shell=False,
|
||||
)
|
||||
if not probe.ok:
|
||||
return SshCommandResult(
|
||||
@@ -363,6 +462,7 @@ def run_ssh_monitor_update(
|
||||
remote_cmd=bootstrap_cmd,
|
||||
command_timeout_sec=900,
|
||||
need_root=True,
|
||||
login_shell=False,
|
||||
)
|
||||
bootstrapped = True
|
||||
else:
|
||||
@@ -373,6 +473,7 @@ def run_ssh_monitor_update(
|
||||
remote_cmd=update_cmd,
|
||||
command_timeout_sec=900,
|
||||
need_root=True,
|
||||
login_shell=False,
|
||||
)
|
||||
if not updated.ok:
|
||||
prefix = "Updater bootstrap failed" if bootstrapped else "SSH update failed"
|
||||
|
||||
Reference in New Issue
Block a user