fix(backend): non-login SSH and strict loginctl parse, SSH retry (0.20.31)

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-06-22 10:54:56 +10:00
parent a51a585b14
commit 54c1d96933
7 changed files with 246 additions and 43 deletions
+34 -3
View File
@@ -35,6 +35,21 @@ SESSION_EVENT_TYPES_LINUX = frozenset(
) )
SESSION_EVENT_TYPES_WINDOWS = frozenset({"rdp.login.success"}) SESSION_EVENT_TYPES_WINDOWS = frozenset({"rdp.login.success"})
_LOGIND_SESSION_ID_RE = re.compile(r"^\d+$|^c\d+$", re.IGNORECASE)
_LOGIND_USER_RE = re.compile(r"^[a-z_][a-z0-9._-]*$", re.IGNORECASE)
_LOGIND_STATES = frozenset(
{
"active",
"online",
"closing",
"opening",
"degraded",
"lingering",
"preparing",
"unknown",
}
)
@dataclass(frozen=True) @dataclass(frozen=True)
class HostSessionRow: class HostSessionRow:
@@ -78,14 +93,27 @@ def is_windows_host_event(event: Event) -> bool:
return host is not None and is_windows_host(host) return host is not None and is_windows_host(host)
def _looks_like_loginctl_row(parts: list[str]) -> bool:
if len(parts) < 6:
return False
sid, uid, user, state = parts[0], parts[1], parts[2], parts[5].lower()
if not _LOGIND_SESSION_ID_RE.match(sid):
return False
if not uid.isdigit():
return False
if not _LOGIND_USER_RE.match(user):
return False
return state in _LOGIND_STATES
def parse_loginctl_sessions(stdout: str) -> list[HostSessionRow]: def parse_loginctl_sessions(stdout: str) -> list[HostSessionRow]:
rows: list[HostSessionRow] = [] rows: list[HostSessionRow] = []
for line in stdout.splitlines(): for line in stdout.splitlines():
text = line.strip() text = line.strip()
if not text: if not text or text.startswith("SESSION"):
continue continue
parts = text.split() parts = text.split()
if len(parts) < 3: if not _looks_like_loginctl_row(parts):
continue continue
sid, user = parts[0], parts[2] sid, user = parts[0], parts[2]
tty = parts[4] if len(parts) > 4 and parts[4] != "-" else None tty = parts[4] if len(parts) > 4 and parts[4] != "-" else None
@@ -143,7 +171,7 @@ def list_linux_sessions(host: Host, cfg: LinuxAdminConfig) -> tuple[list[HostSes
if not is_linux_host(host): if not is_linux_host(host):
raise SshHostNotLinuxError("Host is not Linux") raise SshHostNotLinuxError("Host is not Linux")
targets = iter_ssh_targets(host) targets = iter_ssh_targets(host)
cmd = "loginctl list-sessions --no-legend --no-pager 2>/dev/null || who -u" cmd = "loginctl list-sessions --no-legend --no-pager"
last: SshCommandResult | None = None last: SshCommandResult | None = None
for target in targets: for target in targets:
result = run_ssh_command( result = run_ssh_command(
@@ -154,6 +182,7 @@ def list_linux_sessions(host: Host, cfg: LinuxAdminConfig) -> tuple[list[HostSes
connect_timeout_sec=15, connect_timeout_sec=15,
command_timeout_sec=45, command_timeout_sec=45,
need_root=True, need_root=True,
login_shell=False,
) )
last = result last = result
if result.ok and result.stdout.strip(): if result.ok and result.stdout.strip():
@@ -184,6 +213,7 @@ def terminate_linux_session(
connect_timeout_sec=15, connect_timeout_sec=15,
command_timeout_sec=45, command_timeout_sec=45,
need_root=True, need_root=True,
login_shell=False,
) )
last = result last = result
if result.ok: if result.ok:
@@ -286,6 +316,7 @@ def _terminate_linux_user_sessions(host: Host, cfg: LinuxAdminConfig, user: str)
connect_timeout_sec=15, connect_timeout_sec=15,
command_timeout_sec=45, command_timeout_sec=45,
need_root=True, need_root=True,
login_shell=False,
) )
last = result last = result
if result.ok: if result.ok:
+115 -14
View File
@@ -20,6 +20,17 @@ SSH_MONITOR_VERSION_CMD = (
SSH_OUTPUT_MAX_LEN = 12_000 SSH_OUTPUT_MAX_LEN = 12_000
_AGENT_VERSION_RE = re.compile(r"(\d+\.\d+\.\d+(?:-SAC)?)", re.IGNORECASE) _AGENT_VERSION_RE = re.compile(r"(\d+\.\d+\.\d+(?:-SAC)?)", re.IGNORECASE)
_IPV4_RE = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$") _IPV4_RE = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$")
_SSH_TRANSIENT_ERROR_MARKERS = (
"no existing session",
"error reading ssh protocol banner",
"connection reset",
"connection lost",
"session not active",
"eof",
"socket is closed",
"connection timed out",
)
_SSH_CONNECT_ATTEMPTS = 2
class LinuxAdminNotConfiguredError(Exception): class LinuxAdminNotConfiguredError(Exception):
@@ -117,6 +128,58 @@ def _truncate_output(text: str) -> str:
return text[:SSH_OUTPUT_MAX_LEN] + "\n… (truncated)" return text[:SSH_OUTPUT_MAX_LEN] + "\n… (truncated)"
def _is_transient_ssh_error(exc: BaseException) -> bool:
text = str(exc).casefold()
return any(marker in text for marker in _SSH_TRANSIENT_ERROR_MARKERS)
def _ssh_connect_hint(exc: BaseException) -> str:
text = str(exc).casefold()
if "no existing session" in text:
return (
" Сервер закрыл SSH-сессию во время подключения или auth "
"(проверьте пароль admin, лимиты sshd MaxSessions/MaxStartups, "
"доступ SAC→хост по 22/tcp; на SAC не должен мешать ssh-agent)."
)
return ""
def _connect_ssh_client(
client,
*,
target: str,
user: str,
password: str,
connect_timeout_sec: int,
) -> None:
client.connect(
hostname=target,
username=user,
password=password,
timeout=connect_timeout_sec,
banner_timeout=connect_timeout_sec,
auth_timeout=connect_timeout_sec,
allow_agent=False,
look_for_keys=False,
compress=False,
)
def _close_ssh_client(client) -> None:
if client is None:
return
try:
transport = client.get_transport()
if transport is not None and transport.is_active():
transport.close()
except Exception:
pass
try:
client.close()
except Exception:
pass
def _shell_single_quote(value: str) -> str: def _shell_single_quote(value: str) -> str:
return "'" + value.replace("'", "'\"'\"'") + "'" return "'" + value.replace("'", "'\"'\"'") + "'"
@@ -127,15 +190,17 @@ def _remote_shell_command(
password: str, password: str,
*, *,
need_root: bool = False, need_root: bool = False,
login_shell: bool = True,
) -> str: ) -> str:
"""Build remote shell command. Sudo only when need_root and user is not root.""" """Build remote shell command. Sudo only when need_root and user is not root."""
safe_cmd = _shell_single_quote(remote_cmd) safe_cmd = _shell_single_quote(remote_cmd)
bash_flag = "lc" if login_shell else "c"
if user == "root" or not need_root: if user == "root" or not need_root:
return f"bash -lc {safe_cmd}" return f"bash -{bash_flag} {safe_cmd}"
safe_pw = _shell_single_quote(password) safe_pw = _shell_single_quote(password)
inner = f"printf '%s\\n' {safe_pw} | sudo -S -p '' bash -lc {safe_cmd}" inner = f"printf '%s\\n' {safe_pw} | sudo -S -p '' bash -{bash_flag} {safe_cmd}"
return f"bash -lc {_shell_single_quote(inner)}" return f"bash -{bash_flag} {_shell_single_quote(inner)}"
def run_ssh_command( def run_ssh_command(
@@ -147,6 +212,7 @@ def run_ssh_command(
connect_timeout_sec: int = 15, connect_timeout_sec: int = 15,
command_timeout_sec: int = 600, command_timeout_sec: int = 600,
need_root: bool = False, need_root: bool = False,
login_shell: bool = True,
) -> SshCommandResult: ) -> SshCommandResult:
target = target.strip() target = target.strip()
user = user.strip() user = user.strip()
@@ -158,23 +224,33 @@ def run_ssh_command(
except ImportError as exc: except ImportError as exc:
raise RuntimeError("paramiko is not installed on SAC server") from exc raise RuntimeError("paramiko is not installed on SAC server") from exc
shell_cmd = _remote_shell_command(
user,
remote_cmd,
password,
need_root=need_root,
login_shell=login_shell,
)
exit_code: int | None = None
out_text = ""
err_text = ""
for attempt in range(1, _SSH_CONNECT_ATTEMPTS + 1):
client = paramiko.SSHClient() client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
shell_cmd = _remote_shell_command(user, remote_cmd, password, need_root=need_root)
try: try:
client.connect( _connect_ssh_client(
hostname=target, client,
username=user, target=target,
user=user,
password=password, password=password,
timeout=connect_timeout_sec, connect_timeout_sec=connect_timeout_sec,
allow_agent=False,
look_for_keys=False,
) )
_stdin, stdout, stderr = client.exec_command(shell_cmd, timeout=command_timeout_sec) _stdin, stdout, stderr = client.exec_command(shell_cmd, timeout=command_timeout_sec)
exit_code = stdout.channel.recv_exit_status() exit_code = stdout.channel.recv_exit_status()
out_text = _truncate_output(stdout.read().decode("utf-8", errors="replace")) out_text = _truncate_output(stdout.read().decode("utf-8", errors="replace"))
err_text = _truncate_output(stderr.read().decode("utf-8", errors="replace")) err_text = _truncate_output(stderr.read().decode("utf-8", errors="replace"))
break
except paramiko.AuthenticationException: except paramiko.AuthenticationException:
return SshCommandResult( return SshCommandResult(
ok=False, ok=False,
@@ -188,13 +264,22 @@ def run_ssh_command(
target=target, target=target,
) )
except Exception as exc: except Exception as exc:
if attempt < _SSH_CONNECT_ATTEMPTS and _is_transient_ssh_error(exc):
continue
hint = _ssh_connect_hint(exc)
return SshCommandResult( return SshCommandResult(
ok=False, ok=False,
message=f"SSH error ({target}): {exc}", message=f"SSH error ({target}): {exc}{hint}",
target=target, target=target,
) )
finally: finally:
client.close() _close_ssh_client(client)
else:
return SshCommandResult(
ok=False,
message=f"SSH error ({target}): repeated transient connection failures",
target=target,
)
ok = exit_code == 0 ok = exit_code == 0
if ok: if ok:
@@ -215,6 +300,15 @@ def run_ssh_command(
) )
def _ssh_output_hostname(stdout: str) -> str:
"""Last line that looks like a hostname (MOTD/login banners may precede command output)."""
candidates = [ln.strip() for ln in stdout.splitlines() if ln.strip()]
for line in reversed(candidates):
if re.match(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,253}$", line):
return line
return candidates[-1] if candidates else ""
def test_ssh_connection( def test_ssh_connection(
*, *,
target: str, target: str,
@@ -229,9 +323,12 @@ def test_ssh_connection(
remote_cmd="hostname", remote_cmd="hostname",
connect_timeout_sec=timeout_sec, connect_timeout_sec=timeout_sec,
command_timeout_sec=timeout_sec, command_timeout_sec=timeout_sec,
login_shell=False,
) )
if result.ok and result.stdout.strip(): if result.ok and result.stdout.strip():
host = result.stdout.strip().splitlines()[0] host = _ssh_output_hostname(result.stdout)
if not host:
return result
return SshCommandResult( return SshCommandResult(
ok=True, ok=True,
message=f"SSH OK, hostname={host}", message=f"SSH OK, hostname={host}",
@@ -264,6 +361,7 @@ def probe_ssh_monitor_version(
password=password, password=password,
remote_cmd=SSH_MONITOR_VERSION_CMD, remote_cmd=SSH_MONITOR_VERSION_CMD,
command_timeout_sec=30, command_timeout_sec=30,
login_shell=False,
) )
if result.ok and result.stdout.strip(): if result.ok and result.stdout.strip():
version = parse_ssh_monitor_version_text(result.stdout) version = parse_ssh_monitor_version_text(result.stdout)
@@ -339,6 +437,7 @@ def run_ssh_monitor_update(
password=password, password=password,
remote_cmd=check_cmd, remote_cmd=check_cmd,
command_timeout_sec=30, command_timeout_sec=30,
login_shell=False,
) )
if not probe.ok: if not probe.ok:
return SshCommandResult( return SshCommandResult(
@@ -363,6 +462,7 @@ def run_ssh_monitor_update(
remote_cmd=bootstrap_cmd, remote_cmd=bootstrap_cmd,
command_timeout_sec=900, command_timeout_sec=900,
need_root=True, need_root=True,
login_shell=False,
) )
bootstrapped = True bootstrapped = True
else: else:
@@ -373,6 +473,7 @@ def run_ssh_monitor_update(
remote_cmd=update_cmd, remote_cmd=update_cmd,
command_timeout_sec=900, command_timeout_sec=900,
need_root=True, need_root=True,
login_shell=False,
) )
if not updated.ok: if not updated.ok:
prefix = "Updater bootstrap failed" if bootstrapped else "SSH update failed" prefix = "Updater bootstrap failed" if bootstrapped else "SSH update failed"
+1 -1
View File
@@ -1,5 +1,5 @@
"""Единый источник версии SAC (API, health, логи, OpenAPI).""" """Единый источник версии SAC (API, health, логи, OpenAPI)."""
APP_NAME = "Security Alert Center" APP_NAME = "Security Alert Center"
APP_VERSION = "0.20.30" APP_VERSION = "0.20.31"
APP_VERSION_LABEL = f"{APP_NAME} v.{APP_VERSION}" APP_VERSION_LABEL = f"{APP_NAME} v.{APP_VERSION}"
+2 -2
View File
@@ -4,6 +4,6 @@ from app.version import APP_NAME, APP_VERSION, APP_VERSION_LABEL
def test_version_constants(): def test_version_constants():
assert APP_VERSION == "0.20.30" assert APP_VERSION == "0.20.31"
assert APP_NAME == "Security Alert Center" assert APP_NAME == "Security Alert Center"
assert APP_VERSION_LABEL == "Security Alert Center v.0.20.30" assert APP_VERSION_LABEL == "Security Alert Center v.0.20.31"
+14
View File
@@ -16,6 +16,20 @@ def test_parse_loginctl_sessions():
assert rows[0].tty == "pts/0" assert rows[0].tty == "pts/0"
def test_parse_loginctl_sessions_ignores_motd_noise():
stdout = """This server is powered by FASTPANEL
Ubuntu Operating LTS
configuration By can be not
11090 1000 papatramp - - active -
11102 1000 papatramp - - active -
"""
rows = parse_loginctl_sessions(stdout)
assert len(rows) == 2
assert rows[0].session_id == "11090"
assert rows[1].session_id == "11102"
assert all(row.user == "papatramp" for row in rows)
def test_parse_qwinsta_sessions_filters_user(): def test_parse_qwinsta_sessions_filters_user():
stdout = """SESSIONNAME USERNAME ID STATE stdout = """SESSIONNAME USERNAME ID STATE
console Administrator 1 Active console Administrator 1 Active
+57
View File
@@ -36,6 +36,16 @@ def _install_fake_paramiko(monkeypatch, *, connect_side_effect=None, exec_setup=
return client return client
def test_ssh_output_hostname_ignores_motd():
stdout = "Welcome!\nFASTPANEL\n\ncz-server\n"
assert ssh_connect._ssh_output_hostname(stdout) == "cz-server"
def test_remote_shell_command_non_login_for_sessions():
cmd = _remote_shell_command("root", "loginctl list-sessions", "secret", login_shell=False)
assert cmd == "bash -c 'loginctl list-sessions'"
def test_remote_shell_command_non_root_probe_has_no_sudo(): def test_remote_shell_command_non_root_probe_has_no_sudo():
cmd = _remote_shell_command("deploy", "hostname", "secret", need_root=False) cmd = _remote_shell_command("deploy", "hostname", "secret", need_root=False)
assert "sudo" not in cmd assert "sudo" not in cmd
@@ -125,6 +135,53 @@ def test_iter_ssh_targets_requires_address():
pass pass
def test_run_ssh_command_retries_transient_no_existing_session(monkeypatch):
attempts = {"count": 0}
class _NoSessionError(Exception):
pass
def connect_side_effect(*args, **kwargs):
attempts["count"] += 1
if attempts["count"] == 1:
raise _NoSessionError("No existing session")
def setup(client):
stdout = MagicMock()
stdout.read.return_value = b"ready\n"
stdout.channel.recv_exit_status.return_value = 0
stderr = MagicMock()
stderr.read.return_value = b""
def exec_ok(cmd, **kwargs):
return (None, stdout, stderr)
client.exec_command.side_effect = exec_ok
fake = MagicMock()
fake.SSHClient = MagicMock()
fake.AutoAddPolicy = MagicMock()
fake.AuthenticationException = _AuthError
def make_client():
client = MagicMock()
client.connect.side_effect = connect_side_effect
setup(client)
return client
fake.SSHClient.side_effect = make_client
monkeypatch.setitem(sys.modules, "paramiko", fake)
result = run_ssh_command(
target="185.87.149.9",
user="root",
password="pw",
remote_cmd="hostname",
)
assert result.ok is True
assert attempts["count"] == 2
def test_probe_ssh_connection_success(monkeypatch): def test_probe_ssh_connection_success(monkeypatch):
def setup(client): def setup(client):
stdout = MagicMock() stdout = MagicMock()
+1 -1
View File
@@ -1,4 +1,4 @@
/** Fallback до загрузки /health; при релизе держите в sync с backend/app/version.py */ /** Fallback до загрузки /health; при релизе держите в sync с backend/app/version.py */
export const APP_NAME = "Security Alert Center"; export const APP_NAME = "Security Alert Center";
export const APP_VERSION = "0.20.30"; export const APP_VERSION = "0.20.31";
export const APP_VERSION_LABEL = `${APP_NAME} v.${APP_VERSION}`; export const APP_VERSION_LABEL = `${APP_NAME} v.${APP_VERSION}`;