"""Background SSH/WinRM host actions (survives UI navigation and multi-worker API).""" from __future__ import annotations import logging import os import threading from datetime import datetime, timezone from typing import Any, Callable from sqlalchemy.orm import Session from app.database import SessionLocal from app.models import Host from app.services.agent_update import execute_agent_update_fallback from app.services.agent_update_types import AgentUpdateFallbackResult from app.services.linux_admin_settings import get_effective_linux_admin_config from app.services.agent_update_settings import get_effective_agent_update_config from app.services.ssh_connect import iter_ssh_targets, run_ssh_monitor_update, SshCommandResult, tail_ssh_monitor_update_log logger = logging.getLogger(__name__) ActionRunner = Callable[[Session, Host], AgentUpdateFallbackResult | SshCommandResult] _lock = threading.Lock() _running: set[int] = set() class RemoteActionAlreadyRunningError(Exception): pass def _run_inline() -> bool: return os.environ.get("SAC_REMOTE_ACTION_INLINE", "").strip().lower() in ("1", "true", "yes") def _utcnow() -> datetime: return datetime.now(timezone.utc) def _format_output(result: AgentUpdateFallbackResult | SshCommandResult) -> str: parts: list[str] = [] stdout = (getattr(result, "stdout", None) or "").strip() stderr = (getattr(result, "stderr", None) or "").strip() if stdout: parts.append(stdout) if stderr: parts.append(stderr) exit_code = getattr(result, "exit_code", None) if exit_code is not None: parts.append(f"exit code: {exit_code}") return "\n\n".join(parts) def _result_payload( result: AgentUpdateFallbackResult | SshCommandResult, *, title: str, started_at: str, ) -> dict[str, Any]: finished = _utcnow().isoformat() product_version = getattr(result, "product_version", None) or getattr(result, "agent_version", None) return { "title": title, "status": "success" if result.ok else "failed", "message": result.message, "stdout": getattr(result, "stdout", None) or "", "stderr": getattr(result, "stderr", None) or "", "output": _format_output(result), "ok": result.ok, "exit_code": getattr(result, "exit_code", None), "product_version": product_version, "target": getattr(result, "target", None) or "", "started_at": started_at, "finished_at": finished, } def _mark_running(db: Session, host: Host, *, title: str) -> str: started_at = _utcnow().isoformat() host.agent_update_state = "running" host.agent_update_last_error = None host.remote_action = { "title": title, "status": "running", "message": "Подключение к хосту и выполнение команды…", "stdout": "", "stderr": "", "output": "", "ok": None, "exit_code": None, "product_version": None, "target": host.hostname, "started_at": started_at, "finished_at": None, } db.flush() return started_at def _apply_ssh_update_result(db: Session, host: Host, result: SshCommandResult) -> None: now = _utcnow() host.agent_update_last_at = now if result.ok: host.agent_update_state = "success" host.agent_update_last_error = None if result.agent_version: host.product_version = result.agent_version host.ssh_admin_ok = True host.ssh_admin_checked_at = now else: host.agent_update_state = "failed" host.agent_update_last_error = (result.message or "SSH update failed")[:2000] _REMOTE_LOG_POLL_SEC = 2.0 def _poll_ssh_update_log_loop( host_id: int, *, stop: threading.Event, ) -> None: """Периодически подтягивает tail update_script.log в hosts.remote_action для UI.""" while not stop.wait(_REMOTE_LOG_POLL_SEC): session = SessionLocal() try: row = session.get(Host, host_id) if row is None or (row.agent_update_state or "").strip().lower() != "running": continue payload = dict(row.remote_action or {}) if (payload.get("status") or "").strip().lower() != "running": continue cfg = get_effective_linux_admin_config(session) if not cfg.configured: continue try: targets = iter_ssh_targets(row) except Exception: continue tail = "" for target in targets: try: tail = tail_ssh_monitor_update_log( target=target, user=cfg.user, password=cfg.password, ) except Exception: logger.debug("update log tail failed host_id=%s target=%s", host_id, target, exc_info=True) continue if tail: break session.refresh(row) if (row.agent_update_state or "").strip().lower() != "running": continue payload = dict(row.remote_action or {}) if (payload.get("status") or "").strip().lower() != "running": continue if tail: payload["output"] = tail payload["message"] = "Выполняется обновление… (лог с хоста)" row.remote_action = payload session.commit() except Exception: logger.debug("remote log poll failed host_id=%s", host_id, exc_info=True) session.rollback() finally: session.close() def start_host_remote_action( db: Session, host: Host, *, title: str, runner: ActionRunner, poll_remote_log: bool = False, ) -> dict[str, Any]: host_id = int(host.id) with _lock: if host_id in _running or host.agent_update_state == "running": raise RemoteActionAlreadyRunningError( f"Remote action already running for host {host.hostname}" ) _running.add(host_id) started_at = _mark_running(db, host, title=title) db.commit() def _worker() -> None: session = SessionLocal() stop_poll = threading.Event() poll_thread: threading.Thread | None = None if poll_remote_log and runner is run_ssh_monitor_update_action: poll_thread = threading.Thread( target=_poll_ssh_update_log_loop, args=(host_id,), kwargs={"stop": stop_poll}, name=f"sac-remote-log-{host_id}", daemon=True, ) poll_thread.start() try: row = session.get(Host, host_id) if row is None: return result = runner(session, row) started = (row.remote_action or {}).get("started_at") or started_at row.remote_action = _result_payload(result, title=title, started_at=started) if isinstance(result, SshCommandResult): _apply_ssh_update_result(session, row, result) session.commit() except Exception: logger.exception("Background remote action failed for host_id=%s", host_id) session.rollback() row = session.get(Host, host_id) if row is not None: started = (row.remote_action or {}).get("started_at") or started_at row.agent_update_state = "failed" row.agent_update_last_error = "Internal error during remote action" row.remote_action = { "title": title, "status": "failed", "message": "Внутренняя ошибка SAC при выполнении действия", "stdout": "", "stderr": "", "output": "", "ok": False, "exit_code": None, "product_version": None, "target": row.hostname, "started_at": started, "finished_at": _utcnow().isoformat(), } session.commit() finally: stop_poll.set() if poll_thread is not None: poll_thread.join(timeout=5.0) session.close() with _lock: _running.discard(host_id) if _run_inline(): _worker() else: threading.Thread(target=_worker, name=f"sac-remote-action-{host_id}", daemon=True).start() return dict(host.remote_action or {}) def run_ssh_monitor_update_action(db: Session, host: Host) -> SshCommandResult: cfg = get_effective_linux_admin_config(db) if not cfg.configured: return SshCommandResult( ok=False, message="Linux SSH admin is not configured", target=host.hostname or "", ) agent_cfg = get_effective_agent_update_config(db) repo_url = (agent_cfg.ssh_git_repo_url or "").strip() or None branch = (agent_cfg.git_branch or "main").strip() or "main" try: targets = iter_ssh_targets(host) except Exception as exc: return SshCommandResult( ok=False, message=str(exc), target=host.hostname or "", ) last: SshCommandResult | None = None for target in targets: last = run_ssh_monitor_update( target=target, user=cfg.user, password=cfg.password, repo_url=repo_url, git_branch=branch, ) if last.ok: return last return last or SshCommandResult( ok=False, message="SSH update failed", target=host.hostname or "", ) def run_agent_fallback_action(db: Session, host: Host) -> AgentUpdateFallbackResult: return execute_agent_update_fallback(db, host) def clear_stale_running_remote_actions( db: Session, *, reason: str = "Прервано перезапуском SAC (фоновый worker не завершил job)", host_ids: list[int] | None = None, ) -> list[str]: """Сбросить зависшие running после restart sac-api (daemon thread умер, запись в БД осталась).""" from sqlalchemy import select stmt = select(Host).where(Host.agent_update_state == "running") if host_ids is not None: stmt = stmt.where(Host.id.in_(host_ids)) rows = list(db.scalars(stmt).all()) if not rows: return [] finished = _utcnow().isoformat() hostnames: list[str] = [] for host in rows: started = (host.remote_action or {}).get("started_at") payload = dict(host.remote_action or {}) payload.update( { "title": payload.get("title") or "Remote action", "status": "failed", "message": reason, "ok": False, "finished_at": finished, "started_at": started, "target": payload.get("target") or host.hostname, } ) host.remote_action = payload host.agent_update_state = "failed" host.agent_update_last_error = reason[:2000] hostnames.append(host.hostname or str(host.id)) db.commit() with _lock: for host in rows: _running.discard(int(host.id)) return hostnames def cancel_host_remote_action(db: Session, host: Host, *, reason: str | None = None) -> bool: """Отменить зависший running job вручную (admin).""" if host.agent_update_state != "running": return False msg = reason or "Отменено администратором SAC" cleared = clear_stale_running_remote_actions(db, reason=msg, host_ids=[int(host.id)]) return bool(cleared) def get_remote_action_status(host: Host) -> dict[str, Any]: payload = dict(host.remote_action or {}) if not payload: return {"active": False, "host_id": host.id} agent_state = (host.agent_update_state or "").strip().lower() if agent_state == "running": active = True status = payload.get("status") or "running" elif agent_state in ("success", "failed"): active = False status = payload.get("status") or agent_state else: status = payload.get("status") or host.agent_update_state or "unknown" active = status == "running" return { "active": active, "host_id": host.id, "hostname": host.hostname, "status": status, "title": payload.get("title") or "", "message": payload.get("message") or "", "stdout": payload.get("stdout") or "", "stderr": payload.get("stderr") or "", "output": payload.get("output") or "", "ok": payload.get("ok"), "exit_code": payload.get("exit_code"), "product_version": payload.get("product_version") or host.product_version, "target": payload.get("target") or host.hostname, "agent_update_state": host.agent_update_state, "started_at": payload.get("started_at"), "finished_at": payload.get("finished_at"), }