"""SSH connectivity and remote commands for Linux hosts.""" from __future__ import annotations import re import socket from dataclasses import dataclass from app.config import get_settings from app.models import Host SSH_MONITOR_UPDATE_SCRIPT = "/opt/scripts/update_ssh_monitor.sh" SSH_MONITOR_UPDATE_DIR = "/opt/scripts/update" SSH_MONITOR_REPO_NAME = "ssh-monitor" SSH_MONITOR_UPDATE_REPO_URL = "https://git.kalinamall.ru/PapaTramp/ssh-monitor.git" SSH_MONITOR_BINARY = "/usr/local/bin/ssh-monitor" SSH_MONITOR_VERSION_CMD = ( f"grep -m1 '^SSH_MONITOR_VERSION=' {SSH_MONITOR_BINARY} 2>/dev/null " "| sed -E 's/^SSH_MONITOR_VERSION=[\"'\\'']*([^\"'\\'']+)[\"'\\'']*$/\\1/'" ) SSH_OUTPUT_MAX_LEN = 12_000 _AGENT_VERSION_RE = re.compile(r"(\d+\.\d+\.\d+(?:-SAC)?)", re.IGNORECASE) _IPV4_RE = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$") _SSH_TRANSIENT_ERROR_MARKERS = ( "no existing session", "error reading ssh protocol banner", "connection reset", "connection lost", "session not active", "eof", "socket is closed", "connection timed out", ) _SSH_CONNECT_ATTEMPTS = 2 class LinuxAdminNotConfiguredError(Exception): pass class HostNotLinuxError(Exception): pass class HostTargetMissingError(Exception): pass @dataclass(frozen=True) class SshCommandResult: ok: bool message: str target: str stdout: str = "" stderr: str = "" exit_code: int | None = None agent_version: str | None = None def is_linux_host(host: Host) -> bool: if (host.os_family or "").strip().lower() == "linux": return True return (host.product or "").strip() == "ssh-monitor" def _is_ipv4(value: str) -> bool: return bool(_IPV4_RE.match(value.strip())) def _looks_like_ssh_hostname(value: str) -> bool: """Короткое имя или FQDN; не человекочитаемый display_name вроде «Ubabuba Kalina (10.10.36.9)».""" text = value.strip() if not text or " " in text or "(" in text or ")" in text: return False if _is_ipv4(text): return True if len(text) > 253: return False return all(ch.isalnum() or ch in ".-_" for ch in text) def _ssh_name_resolves(name: str) -> bool: if _is_ipv4(name): return True try: socket.getaddrinfo(name.strip(), 22, type=socket.SOCK_STREAM) return True except OSError: return False def iter_ssh_targets(host: Host) -> list[str]: if not is_linux_host(host): raise HostNotLinuxError("Host is not Linux") seen: set[str] = set() ordered: list[str] = [] def add(value: str | None) -> None: text = (value or "").strip() if not text or text.casefold() in seen: return if not _looks_like_ssh_hostname(text): return if not _ssh_name_resolves(text): return seen.add(text.casefold()) ordered.append(text) add(host.hostname) inventory = host.inventory if isinstance(host.inventory, dict) else {} computer_name = inventory.get("computer_name") if isinstance(computer_name, str): add(computer_name) if host.display_name and _looks_like_ssh_hostname(host.display_name): add(host.display_name) add(host.ipv4) if not ordered: raise HostTargetMissingError("Host has no resolvable hostname or IPv4 for SSH") return ordered def _truncate_output(text: str) -> str: if len(text) <= SSH_OUTPUT_MAX_LEN: return text return text[:SSH_OUTPUT_MAX_LEN] + "\n… (truncated)" def _is_transient_ssh_error(exc: BaseException) -> bool: text = str(exc).casefold() return any(marker in text for marker in _SSH_TRANSIENT_ERROR_MARKERS) def _ssh_connect_hint(exc: BaseException) -> str: text = str(exc).casefold() if "no existing session" in text: return ( " Сервер закрыл SSH-сессию во время подключения или auth " "(проверьте пароль admin, лимиты sshd MaxSessions/MaxStartups, " "доступ SAC→хост по 22/tcp; на SAC не должен мешать ssh-agent)." ) return "" def _connect_ssh_client( client, *, target: str, user: str, password: str, connect_timeout_sec: int, ) -> None: client.connect( hostname=target, username=user, password=password, timeout=connect_timeout_sec, banner_timeout=connect_timeout_sec, auth_timeout=connect_timeout_sec, allow_agent=False, look_for_keys=False, compress=False, ) def _close_ssh_client(client) -> None: if client is None: return try: transport = client.get_transport() if transport is not None and transport.is_active(): transport.close() except Exception: pass try: client.close() except Exception: pass def _shell_single_quote(value: str) -> str: return "'" + value.replace("'", "'\"'\"'") + "'" def _remote_shell_command( user: str, remote_cmd: str, *, need_root: bool = False, login_shell: bool = True, ) -> tuple[str, bool]: """Build remote shell command. Returns (command, needs_sudo_password_on_stdin).""" safe_cmd = _shell_single_quote(remote_cmd) bash_flag = "lc" if login_shell else "c" if user == "root" or not need_root: return f"bash -{bash_flag} {safe_cmd}", False return f"sudo -S -p '' bash -{bash_flag} {safe_cmd}", True def _configure_ssh_client(client, target: str) -> None: import paramiko settings = get_settings() if settings.sac_ssh_auto_add_host_key: client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) return client.set_missing_host_key_policy(paramiko.RejectPolicy()) client.load_system_host_keys() known_hosts = (settings.sac_ssh_known_hosts_file or "").strip() if known_hosts: try: client.load_host_keys(known_hosts) except OSError: pass def _exec_remote_command( client, *, shell_cmd: str, password: str, needs_sudo_password: bool, command_timeout_sec: int, ) -> tuple[int, str, str]: if needs_sudo_password: stdin, stdout, stderr = client.exec_command(shell_cmd, timeout=command_timeout_sec, get_pty=True) stdin.write(f"{password}\n") stdin.flush() stdin.channel.shutdown_write() else: _stdin, stdout, stderr = client.exec_command(shell_cmd, timeout=command_timeout_sec) exit_code = stdout.channel.recv_exit_status() out_text = _truncate_output(stdout.read().decode("utf-8", errors="replace")) err_text = _truncate_output(stderr.read().decode("utf-8", errors="replace")) return exit_code, out_text, err_text def run_ssh_command( *, target: str, user: str, password: str, remote_cmd: str, connect_timeout_sec: int = 15, command_timeout_sec: int = 600, need_root: bool = False, login_shell: bool = True, ) -> SshCommandResult: target = target.strip() user = user.strip() if not target or not user or not password: raise LinuxAdminNotConfiguredError("Linux SSH admin credentials or target missing") try: import paramiko except ImportError as exc: raise RuntimeError("paramiko is not installed on SAC server") from exc shell_cmd, needs_sudo_password = _remote_shell_command( user, remote_cmd, need_root=need_root, login_shell=login_shell, ) exit_code: int | None = None out_text = "" err_text = "" for attempt in range(1, _SSH_CONNECT_ATTEMPTS + 1): client = paramiko.SSHClient() try: _configure_ssh_client(client, target) _connect_ssh_client( client, target=target, user=user, password=password, connect_timeout_sec=connect_timeout_sec, ) exit_code, out_text, err_text = _exec_remote_command( client, shell_cmd=shell_cmd, password=password, needs_sudo_password=needs_sudo_password, command_timeout_sec=command_timeout_sec, ) break except paramiko.AuthenticationException: return SshCommandResult( ok=False, message=f"SSH auth failed for {user}@{target}", target=target, ) except (socket.timeout, TimeoutError): return SshCommandResult( ok=False, message=f"SSH timeout ({connect_timeout_sec}s connect / {command_timeout_sec}s cmd) to {target}", target=target, ) except Exception as exc: if attempt < _SSH_CONNECT_ATTEMPTS and _is_transient_ssh_error(exc): continue hint = _ssh_connect_hint(exc) return SshCommandResult( ok=False, message=f"SSH error ({target}): {exc}{hint}", target=target, ) finally: _close_ssh_client(client) else: return SshCommandResult( ok=False, message=f"SSH error ({target}): repeated transient connection failures", target=target, ) ok = exit_code == 0 if ok: message = f"SSH OK ({target}), exit 0" if out_text.strip(): message = f"{message}\n{out_text.strip().splitlines()[0][:200]}" else: detail = err_text.strip() or out_text.strip() or f"exit code {exit_code}" message = f"SSH command failed ({target}): {detail[:500]}" return SshCommandResult( ok=ok, message=message, target=target, stdout=out_text, stderr=err_text, exit_code=exit_code, ) def _ssh_output_hostname(stdout: str) -> str: """Last line that looks like a hostname (MOTD/login banners may precede command output).""" candidates = [ln.strip() for ln in stdout.splitlines() if ln.strip()] for line in reversed(candidates): if re.match(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,253}$", line): return line return candidates[-1] if candidates else "" def test_ssh_connection( *, target: str, user: str, password: str, timeout_sec: int = 15, ) -> SshCommandResult: result = run_ssh_command( target=target, user=user, password=password, remote_cmd="hostname", connect_timeout_sec=timeout_sec, command_timeout_sec=timeout_sec, login_shell=False, ) if result.ok and result.stdout.strip(): host = _ssh_output_hostname(result.stdout) if not host: return result return SshCommandResult( ok=True, message=f"SSH OK, hostname={host}", target=target, stdout=result.stdout, stderr=result.stderr, exit_code=result.exit_code, ) return result def parse_ssh_monitor_version_text(text: str) -> str | None: if not text: return None match = _AGENT_VERSION_RE.search(text) if not match: return None return match.group(1) def probe_ssh_monitor_version( *, target: str, user: str, password: str, ) -> str | None: result = run_ssh_command( target=target, user=user, password=password, remote_cmd=SSH_MONITOR_VERSION_CMD, command_timeout_sec=30, login_shell=False, ) if result.ok and result.stdout.strip(): version = parse_ssh_monitor_version_text(result.stdout) if version: return version if result.stdout or result.stderr: return parse_ssh_monitor_version_text(f"{result.stdout}\n{result.stderr}") return None def _ssh_monitor_preflight_git_remote(repo_url: str) -> str: """Align clone remotes with SAC-trusted URL (works even before updater script is refreshed).""" safe_repo = _shell_single_quote(repo_url.strip()) repo_dir = _shell_single_quote(f"{SSH_MONITOR_UPDATE_DIR}/{SSH_MONITOR_REPO_NAME}") return ( f"if [ -d {repo_dir}/.git ]; then " f"git -C {repo_dir} remote add kalinamall {safe_repo} 2>/dev/null || " f"git -C {repo_dir} remote set-url kalinamall {safe_repo}; " f"if git -C {repo_dir} remote get-url origin >/dev/null 2>&1; then " f"git -C {repo_dir} remote set-url origin {safe_repo}; " f"fi; fi" ) def _ssh_monitor_update_invoke_command( repo_url: str, *, git_branch: str = "main", ) -> str: """Run installed updater with SAC-trusted git URL (overrides script default / stale origin).""" safe_repo = _shell_single_quote(repo_url.strip()) safe_branch = _shell_single_quote((git_branch or "main").strip() or "main") preflight = _ssh_monitor_preflight_git_remote(repo_url) return f"{preflight}; REPO_URL={safe_repo} GIT_BRANCH={safe_branch} {SSH_MONITOR_UPDATE_SCRIPT}" def _ssh_monitor_bootstrap_command(repo_url: str, *, git_branch: str = "main") -> str: """Clone ssh-monitor repo and install updater when /opt/scripts/update_ssh_monitor.sh is absent.""" safe_repo = repo_url.replace("\\", "\\\\").replace('"', '\\"') safe_branch = (git_branch or "main").replace("\\", "\\\\").replace('"', '\\"') return ( f'UPDATE_DIR="{SSH_MONITOR_UPDATE_DIR}";' f'REPO_URL="{safe_repo}";' f'GIT_BRANCH="{safe_branch}";' f'SCRIPT_NAME="{SSH_MONITOR_REPO_NAME}";' f'INSTALL="{SSH_MONITOR_UPDATE_SCRIPT}";' f'mkdir -p "$UPDATE_DIR" && cd "$UPDATE_DIR" && ' f'([ -d "$SCRIPT_NAME" ] || git clone -b "$GIT_BRANCH" "$REPO_URL" "$SCRIPT_NAME") && ' f'cp "$UPDATE_DIR/$SCRIPT_NAME/update_ssh_monitor.sh" "$INSTALL" && ' f'chmod 750 "$INSTALL" && REPO_URL="$REPO_URL" GIT_BRANCH="$GIT_BRANCH" "$INSTALL"' ) def run_ssh_monitor_update( *, target: str, user: str, password: str, repo_url: str | None = None, git_branch: str | None = None, ) -> SshCommandResult: script = SSH_MONITOR_UPDATE_SCRIPT effective_repo = (repo_url or SSH_MONITOR_UPDATE_REPO_URL).strip() effective_branch = (git_branch or "main").strip() or "main" update_cmd = _ssh_monitor_update_invoke_command( effective_repo, git_branch=effective_branch, ) check_cmd = f"test -x {script} && echo ready || echo missing" probe = run_ssh_command( target=target, user=user, password=password, remote_cmd=check_cmd, command_timeout_sec=30, login_shell=False, ) if not probe.ok: return SshCommandResult( ok=False, message=f"SSH probe failed on {target}: {probe.message}", target=target, stdout=probe.stdout, stderr=probe.stderr, exit_code=probe.exit_code, ) bootstrapped = False if "missing" in probe.stdout: bootstrap_cmd = _ssh_monitor_bootstrap_command( effective_repo, git_branch=effective_branch, ) updated = run_ssh_command( target=target, user=user, password=password, remote_cmd=bootstrap_cmd, command_timeout_sec=900, need_root=True, login_shell=False, ) bootstrapped = True else: updated = run_ssh_command( target=target, user=user, password=password, remote_cmd=update_cmd, command_timeout_sec=900, need_root=True, login_shell=False, ) if not updated.ok: prefix = "Updater bootstrap failed" if bootstrapped else "SSH update failed" return SshCommandResult( ok=False, message=f"{prefix} ({target}): {updated.message}", target=updated.target, stdout=updated.stdout, stderr=updated.stderr, exit_code=updated.exit_code, ) version = probe_ssh_monitor_version(target=target, user=user, password=password) if not version: version = parse_ssh_monitor_version_text(updated.stdout) if version: prefix = "Bootstrap + update OK" if bootstrapped else updated.message message = f"{prefix}\nagent version: {version}" return SshCommandResult( ok=True, message=message, target=updated.target, stdout=updated.stdout, stderr=updated.stderr, exit_code=updated.exit_code, agent_version=version, ) if bootstrapped: return SshCommandResult( ok=True, message=f"Bootstrap + update OK ({target}), exit 0", target=updated.target, stdout=updated.stdout, stderr=updated.stderr, exit_code=updated.exit_code, ) return updated