PapaTramp
86fd400f0b
Clear zombie running jobs on deploy and add admin API to cancel a stuck job. Co-authored-by: Cursor <cursoragent@cursor.com>
299 lines
10 KiB
Python
299 lines
10 KiB
Python
"""Background SSH/WinRM host actions (survives UI navigation and multi-worker API)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import threading
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Callable
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.database import SessionLocal
|
|
from app.models import Host
|
|
from app.services.agent_update import execute_agent_update_fallback
|
|
from app.services.agent_update_types import AgentUpdateFallbackResult
|
|
from app.services.linux_admin_settings import get_effective_linux_admin_config
|
|
from app.services.agent_update_settings import get_effective_agent_update_config
|
|
from app.services.ssh_connect import iter_ssh_targets, run_ssh_monitor_update, SshCommandResult
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
ActionRunner = Callable[[Session, Host], AgentUpdateFallbackResult | SshCommandResult]
|
|
|
|
_lock = threading.Lock()
|
|
_running: set[int] = set()
|
|
|
|
|
|
class RemoteActionAlreadyRunningError(Exception):
|
|
pass
|
|
|
|
|
|
def _run_inline() -> bool:
|
|
return os.environ.get("SAC_REMOTE_ACTION_INLINE", "").strip().lower() in ("1", "true", "yes")
|
|
|
|
|
|
def _utcnow() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def _format_output(result: AgentUpdateFallbackResult | SshCommandResult) -> str:
|
|
parts: list[str] = []
|
|
stdout = (getattr(result, "stdout", None) or "").strip()
|
|
stderr = (getattr(result, "stderr", None) or "").strip()
|
|
if stdout:
|
|
parts.append(stdout)
|
|
if stderr:
|
|
parts.append(stderr)
|
|
exit_code = getattr(result, "exit_code", None)
|
|
if exit_code is not None:
|
|
parts.append(f"exit code: {exit_code}")
|
|
return "\n\n".join(parts)
|
|
|
|
|
|
def _result_payload(
|
|
result: AgentUpdateFallbackResult | SshCommandResult,
|
|
*,
|
|
title: str,
|
|
started_at: str,
|
|
) -> dict[str, Any]:
|
|
finished = _utcnow().isoformat()
|
|
product_version = getattr(result, "product_version", None) or getattr(result, "agent_version", None)
|
|
return {
|
|
"title": title,
|
|
"status": "success" if result.ok else "failed",
|
|
"message": result.message,
|
|
"stdout": getattr(result, "stdout", None) or "",
|
|
"stderr": getattr(result, "stderr", None) or "",
|
|
"output": _format_output(result),
|
|
"ok": result.ok,
|
|
"exit_code": getattr(result, "exit_code", None),
|
|
"product_version": product_version,
|
|
"target": getattr(result, "target", None) or "",
|
|
"started_at": started_at,
|
|
"finished_at": finished,
|
|
}
|
|
|
|
|
|
def _mark_running(db: Session, host: Host, *, title: str) -> str:
|
|
started_at = _utcnow().isoformat()
|
|
host.agent_update_state = "running"
|
|
host.agent_update_last_error = None
|
|
host.remote_action = {
|
|
"title": title,
|
|
"status": "running",
|
|
"message": "Подключение к хосту и выполнение команды…",
|
|
"stdout": "",
|
|
"stderr": "",
|
|
"output": "",
|
|
"ok": None,
|
|
"exit_code": None,
|
|
"product_version": None,
|
|
"target": host.hostname,
|
|
"started_at": started_at,
|
|
"finished_at": None,
|
|
}
|
|
db.flush()
|
|
return started_at
|
|
|
|
|
|
def _apply_ssh_update_result(db: Session, host: Host, result: SshCommandResult) -> None:
|
|
now = _utcnow()
|
|
host.agent_update_last_at = now
|
|
if result.ok:
|
|
host.agent_update_state = "success"
|
|
host.agent_update_last_error = None
|
|
if result.agent_version:
|
|
host.product_version = result.agent_version
|
|
host.ssh_admin_ok = True
|
|
host.ssh_admin_checked_at = now
|
|
else:
|
|
host.agent_update_state = "failed"
|
|
host.agent_update_last_error = (result.message or "SSH update failed")[:2000]
|
|
|
|
|
|
def start_host_remote_action(
|
|
db: Session,
|
|
host: Host,
|
|
*,
|
|
title: str,
|
|
runner: ActionRunner,
|
|
) -> dict[str, Any]:
|
|
host_id = int(host.id)
|
|
with _lock:
|
|
if host_id in _running or host.agent_update_state == "running":
|
|
raise RemoteActionAlreadyRunningError(
|
|
f"Remote action already running for host {host.hostname}"
|
|
)
|
|
_running.add(host_id)
|
|
|
|
started_at = _mark_running(db, host, title=title)
|
|
db.commit()
|
|
|
|
def _worker() -> None:
|
|
session = SessionLocal()
|
|
try:
|
|
row = session.get(Host, host_id)
|
|
if row is None:
|
|
return
|
|
result = runner(session, row)
|
|
started = (row.remote_action or {}).get("started_at") or started_at
|
|
row.remote_action = _result_payload(result, title=title, started_at=started)
|
|
if isinstance(result, SshCommandResult):
|
|
_apply_ssh_update_result(session, row, result)
|
|
session.commit()
|
|
except Exception:
|
|
logger.exception("Background remote action failed for host_id=%s", host_id)
|
|
session.rollback()
|
|
row = session.get(Host, host_id)
|
|
if row is not None:
|
|
started = (row.remote_action or {}).get("started_at") or started_at
|
|
row.agent_update_state = "failed"
|
|
row.agent_update_last_error = "Internal error during remote action"
|
|
row.remote_action = {
|
|
"title": title,
|
|
"status": "failed",
|
|
"message": "Внутренняя ошибка SAC при выполнении действия",
|
|
"stdout": "",
|
|
"stderr": "",
|
|
"output": "",
|
|
"ok": False,
|
|
"exit_code": None,
|
|
"product_version": None,
|
|
"target": row.hostname,
|
|
"started_at": started,
|
|
"finished_at": _utcnow().isoformat(),
|
|
}
|
|
session.commit()
|
|
finally:
|
|
session.close()
|
|
with _lock:
|
|
_running.discard(host_id)
|
|
|
|
if _run_inline():
|
|
_worker()
|
|
else:
|
|
threading.Thread(target=_worker, name=f"sac-remote-action-{host_id}", daemon=True).start()
|
|
return dict(host.remote_action or {})
|
|
|
|
|
|
def run_ssh_monitor_update_action(db: Session, host: Host) -> SshCommandResult:
|
|
cfg = get_effective_linux_admin_config(db)
|
|
if not cfg.configured:
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message="Linux SSH admin is not configured",
|
|
target=host.hostname or "",
|
|
)
|
|
agent_cfg = get_effective_agent_update_config(db)
|
|
repo_url = (agent_cfg.ssh_git_repo_url or "").strip() or None
|
|
branch = (agent_cfg.git_branch or "main").strip() or "main"
|
|
try:
|
|
targets = iter_ssh_targets(host)
|
|
except Exception as exc:
|
|
return SshCommandResult(
|
|
ok=False,
|
|
message=str(exc),
|
|
target=host.hostname or "",
|
|
)
|
|
last: SshCommandResult | None = None
|
|
for target in targets:
|
|
last = run_ssh_monitor_update(
|
|
target=target,
|
|
user=cfg.user,
|
|
password=cfg.password,
|
|
repo_url=repo_url,
|
|
git_branch=branch,
|
|
)
|
|
if last.ok:
|
|
return last
|
|
return last or SshCommandResult(
|
|
ok=False,
|
|
message="SSH update failed",
|
|
target=host.hostname or "",
|
|
)
|
|
|
|
|
|
def run_agent_fallback_action(db: Session, host: Host) -> AgentUpdateFallbackResult:
|
|
return execute_agent_update_fallback(db, host)
|
|
|
|
|
|
def clear_stale_running_remote_actions(
|
|
db: Session,
|
|
*,
|
|
reason: str = "Прервано перезапуском SAC (фоновый worker не завершил job)",
|
|
host_ids: list[int] | None = None,
|
|
) -> list[str]:
|
|
"""Сбросить зависшие running после restart sac-api (daemon thread умер, запись в БД осталась)."""
|
|
from sqlalchemy import select
|
|
|
|
stmt = select(Host).where(Host.agent_update_state == "running")
|
|
if host_ids is not None:
|
|
stmt = stmt.where(Host.id.in_(host_ids))
|
|
rows = list(db.scalars(stmt).all())
|
|
if not rows:
|
|
return []
|
|
|
|
finished = _utcnow().isoformat()
|
|
hostnames: list[str] = []
|
|
for host in rows:
|
|
started = (host.remote_action or {}).get("started_at")
|
|
payload = dict(host.remote_action or {})
|
|
payload.update(
|
|
{
|
|
"title": payload.get("title") or "Remote action",
|
|
"status": "failed",
|
|
"message": reason,
|
|
"ok": False,
|
|
"finished_at": finished,
|
|
"started_at": started,
|
|
"target": payload.get("target") or host.hostname,
|
|
}
|
|
)
|
|
host.remote_action = payload
|
|
host.agent_update_state = "failed"
|
|
host.agent_update_last_error = reason[:2000]
|
|
hostnames.append(host.hostname or str(host.id))
|
|
|
|
db.commit()
|
|
with _lock:
|
|
for host in rows:
|
|
_running.discard(int(host.id))
|
|
return hostnames
|
|
|
|
|
|
def cancel_host_remote_action(db: Session, host: Host, *, reason: str | None = None) -> bool:
|
|
"""Отменить зависший running job вручную (admin)."""
|
|
if host.agent_update_state != "running":
|
|
return False
|
|
msg = reason or "Отменено администратором SAC"
|
|
cleared = clear_stale_running_remote_actions(db, reason=msg, host_ids=[int(host.id)])
|
|
return bool(cleared)
|
|
|
|
|
|
def get_remote_action_status(host: Host) -> dict[str, Any]:
|
|
payload = dict(host.remote_action or {})
|
|
if not payload:
|
|
return {"active": False, "host_id": host.id}
|
|
status = payload.get("status") or host.agent_update_state or "unknown"
|
|
active = status == "running" or host.agent_update_state == "running"
|
|
return {
|
|
"active": active,
|
|
"host_id": host.id,
|
|
"hostname": host.hostname,
|
|
"status": status,
|
|
"title": payload.get("title") or "",
|
|
"message": payload.get("message") or "",
|
|
"stdout": payload.get("stdout") or "",
|
|
"stderr": payload.get("stderr") or "",
|
|
"output": payload.get("output") or "",
|
|
"ok": payload.get("ok"),
|
|
"exit_code": payload.get("exit_code"),
|
|
"product_version": payload.get("product_version") or host.product_version,
|
|
"target": payload.get("target") or host.hostname,
|
|
"agent_update_state": host.agent_update_state,
|
|
"started_at": payload.get("started_at"),
|
|
"finished_at": payload.get("finished_at"),
|
|
}
|