feat: background remote agent updates survive SAC navigation (0.20.19)
This commit is contained in:
+99
-35
@@ -1,10 +1,9 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from functools import partial
|
||||
|
||||
from app.auth.jwt_auth import get_current_user, require_admin
|
||||
from app.config import get_settings
|
||||
@@ -18,10 +17,16 @@ from app.services.agent_version import (
|
||||
from app.services.agent_git_release import get_git_release_versions
|
||||
from app.services.agent_update import (
|
||||
AgentUpdateNotAllowedError,
|
||||
execute_agent_update_fallback,
|
||||
host_version_status,
|
||||
request_agent_update,
|
||||
)
|
||||
from app.services.host_remote_actions import (
|
||||
RemoteActionAlreadyRunningError,
|
||||
get_remote_action_status,
|
||||
run_agent_fallback_action,
|
||||
run_ssh_monitor_update_action,
|
||||
start_host_remote_action,
|
||||
)
|
||||
from app.services.agent_update_settings import get_effective_agent_update_config
|
||||
from app.services.agent_host_config import get_host_agent_settings, update_host_agent_config
|
||||
from app.services.host_delete import delete_host_and_related
|
||||
@@ -264,6 +269,32 @@ class HostAgentConfigResponse(BaseModel):
|
||||
settings: dict[str, object]
|
||||
|
||||
|
||||
class HostRemoteActionStartResponse(BaseModel):
|
||||
status: str
|
||||
host_id: int
|
||||
title: str
|
||||
message: str
|
||||
|
||||
|
||||
class HostRemoteActionJobResponse(BaseModel):
|
||||
active: bool
|
||||
host_id: int
|
||||
hostname: str | None = None
|
||||
status: str | None = None
|
||||
title: str = ""
|
||||
message: str = ""
|
||||
stdout: str | None = None
|
||||
stderr: str | None = None
|
||||
output: str | None = None
|
||||
ok: bool | None = None
|
||||
exit_code: int | None = None
|
||||
product_version: str | None = None
|
||||
target: str | None = None
|
||||
agent_update_state: str | None = None
|
||||
started_at: str | None = None
|
||||
finished_at: str | None = None
|
||||
|
||||
|
||||
def _ssh_action_response(
|
||||
result: SshCommandResult,
|
||||
*,
|
||||
@@ -399,12 +430,16 @@ def test_host_ssh(
|
||||
return response.model_copy(update={"ssh_admin_ok": host.ssh_admin_ok})
|
||||
|
||||
|
||||
@router.post("/{host_id}/actions/agent-update", response_model=HostSshActionResponse)
|
||||
@router.post(
|
||||
"/{host_id}/actions/agent-update",
|
||||
response_model=HostRemoteActionStartResponse,
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
)
|
||||
def update_host_agent_via_ssh(
|
||||
host_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
_user=Depends(require_admin),
|
||||
) -> HostSshActionResponse:
|
||||
) -> HostRemoteActionStartResponse:
|
||||
host = db.get(Host, host_id)
|
||||
if host is None:
|
||||
raise HTTPException(status_code=404, detail="Host not found")
|
||||
@@ -413,23 +448,21 @@ def update_host_agent_via_ssh(
|
||||
if not cfg.configured:
|
||||
raise HTTPException(status_code=400, detail="Linux SSH admin is not configured")
|
||||
|
||||
agent_cfg = get_effective_agent_update_config(db)
|
||||
repo_url = (agent_cfg.ssh_git_repo_url or "").strip() or None
|
||||
branch = (agent_cfg.git_branch or "main").strip() or "main"
|
||||
response = _run_ssh_action_on_host(
|
||||
host,
|
||||
cfg,
|
||||
action=partial(run_ssh_monitor_update, repo_url=repo_url, git_branch=branch),
|
||||
)
|
||||
_set_ssh_admin_status(host, response.ok)
|
||||
if response.ok and response.product_version:
|
||||
host.product_version = response.product_version
|
||||
db.commit()
|
||||
return response.model_copy(
|
||||
update={
|
||||
"product_version": host.product_version if response.ok else response.product_version,
|
||||
"ssh_admin_ok": host.ssh_admin_ok,
|
||||
}
|
||||
title = "Обновление ssh-monitor (SSH)"
|
||||
try:
|
||||
start_host_remote_action(
|
||||
db,
|
||||
host,
|
||||
title=title,
|
||||
runner=run_ssh_monitor_update_action,
|
||||
)
|
||||
except RemoteActionAlreadyRunningError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
return HostRemoteActionStartResponse(
|
||||
status="running",
|
||||
host_id=host.id,
|
||||
title=title,
|
||||
message="Обновление запущено на сервере SAC и продолжится в фоне",
|
||||
)
|
||||
|
||||
|
||||
@@ -455,30 +488,61 @@ def post_request_agent_update(
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{host_id}/actions/agent-update-fallback", response_model=HostSshActionResponse)
|
||||
@router.post(
|
||||
"/{host_id}/actions/agent-update-fallback",
|
||||
response_model=HostRemoteActionStartResponse,
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
)
|
||||
def post_agent_update_fallback(
|
||||
host_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
_user=Depends(require_admin),
|
||||
) -> HostSshActionResponse:
|
||||
) -> HostRemoteActionStartResponse:
|
||||
host = db.get(Host, host_id)
|
||||
if host is None:
|
||||
raise HTTPException(status_code=404, detail="Host not found")
|
||||
|
||||
result = execute_agent_update_fallback(db, host)
|
||||
db.commit()
|
||||
return HostSshActionResponse(
|
||||
ok=result.ok,
|
||||
message=result.message,
|
||||
target=result.target or host.hostname,
|
||||
stdout=result.stdout or None,
|
||||
stderr=result.stderr or None,
|
||||
exit_code=result.exit_code,
|
||||
product_version=result.product_version or (host.product_version if result.ok else None),
|
||||
ssh_admin_ok=host.ssh_admin_ok,
|
||||
from app.services.agent_update_settings import PRODUCT_RDP, PRODUCT_SSH
|
||||
from app.services.ssh_connect import is_linux_host
|
||||
from app.services.winrm_connect import is_windows_host
|
||||
|
||||
product = (host.product or "").strip()
|
||||
if product == PRODUCT_RDP or is_windows_host(host):
|
||||
title = "Обновление через WinRM"
|
||||
elif product == PRODUCT_SSH or is_linux_host(host):
|
||||
title = "Fallback SSH (ssh-monitor)"
|
||||
else:
|
||||
title = "Обновление агента (fallback)"
|
||||
|
||||
try:
|
||||
start_host_remote_action(
|
||||
db,
|
||||
host,
|
||||
title=title,
|
||||
runner=run_agent_fallback_action,
|
||||
)
|
||||
except RemoteActionAlreadyRunningError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
return HostRemoteActionStartResponse(
|
||||
status="running",
|
||||
host_id=host.id,
|
||||
title=title,
|
||||
message="Обновление запущено на сервере SAC и продолжится в фоне",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{host_id}/actions/remote-job", response_model=HostRemoteActionJobResponse)
|
||||
def get_host_remote_job(
|
||||
host_id: int,
|
||||
db: Session = Depends(get_db),
|
||||
_user=Depends(require_admin),
|
||||
) -> HostRemoteActionJobResponse:
|
||||
host = db.get(Host, host_id)
|
||||
if host is None:
|
||||
raise HTTPException(status_code=404, detail="Host not found")
|
||||
return HostRemoteActionJobResponse(**get_remote_action_status(host))
|
||||
|
||||
|
||||
@router.patch("/{host_id}/agent-config", response_model=HostAgentConfigResponse)
|
||||
def patch_host_agent_config(
|
||||
host_id: int,
|
||||
|
||||
@@ -34,6 +34,7 @@ class Host(Base):
|
||||
agent_update_state: Mapped[str | None] = mapped_column(String(32))
|
||||
agent_update_last_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
agent_update_last_error: Mapped[str | None] = mapped_column(Text)
|
||||
remote_action: Mapped[dict | None] = mapped_column(JSONB)
|
||||
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Background SSH/WinRM host actions (survives UI navigation and multi-worker API)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Callable
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.database import SessionLocal
|
||||
from app.models import Host
|
||||
from app.services.agent_update import execute_agent_update_fallback
|
||||
from app.services.agent_update_types import AgentUpdateFallbackResult
|
||||
from app.services.linux_admin_settings import get_effective_linux_admin_config
|
||||
from app.services.agent_update_settings import get_effective_agent_update_config
|
||||
from app.services.ssh_connect import iter_ssh_targets, run_ssh_monitor_update, SshCommandResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ActionRunner = Callable[[Session, Host], AgentUpdateFallbackResult | SshCommandResult]
|
||||
|
||||
_lock = threading.Lock()
|
||||
_running: set[int] = set()
|
||||
|
||||
|
||||
class RemoteActionAlreadyRunningError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _run_inline() -> bool:
|
||||
return os.environ.get("SAC_REMOTE_ACTION_INLINE", "").strip().lower() in ("1", "true", "yes")
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _format_output(result: AgentUpdateFallbackResult | SshCommandResult) -> str:
|
||||
parts: list[str] = []
|
||||
stdout = (getattr(result, "stdout", None) or "").strip()
|
||||
stderr = (getattr(result, "stderr", None) or "").strip()
|
||||
if stdout:
|
||||
parts.append(stdout)
|
||||
if stderr:
|
||||
parts.append(stderr)
|
||||
exit_code = getattr(result, "exit_code", None)
|
||||
if exit_code is not None:
|
||||
parts.append(f"exit code: {exit_code}")
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
def _result_payload(
|
||||
result: AgentUpdateFallbackResult | SshCommandResult,
|
||||
*,
|
||||
title: str,
|
||||
started_at: str,
|
||||
) -> dict[str, Any]:
|
||||
finished = _utcnow().isoformat()
|
||||
product_version = getattr(result, "product_version", None) or getattr(result, "agent_version", None)
|
||||
return {
|
||||
"title": title,
|
||||
"status": "success" if result.ok else "failed",
|
||||
"message": result.message,
|
||||
"stdout": getattr(result, "stdout", None) or "",
|
||||
"stderr": getattr(result, "stderr", None) or "",
|
||||
"output": _format_output(result),
|
||||
"ok": result.ok,
|
||||
"exit_code": getattr(result, "exit_code", None),
|
||||
"product_version": product_version,
|
||||
"target": getattr(result, "target", None) or "",
|
||||
"started_at": started_at,
|
||||
"finished_at": finished,
|
||||
}
|
||||
|
||||
|
||||
def _mark_running(db: Session, host: Host, *, title: str) -> str:
|
||||
started_at = _utcnow().isoformat()
|
||||
host.agent_update_state = "running"
|
||||
host.agent_update_last_error = None
|
||||
host.remote_action = {
|
||||
"title": title,
|
||||
"status": "running",
|
||||
"message": "Подключение к хосту и выполнение команды…",
|
||||
"stdout": "",
|
||||
"stderr": "",
|
||||
"output": "",
|
||||
"ok": None,
|
||||
"exit_code": None,
|
||||
"product_version": None,
|
||||
"target": host.hostname,
|
||||
"started_at": started_at,
|
||||
"finished_at": None,
|
||||
}
|
||||
db.flush()
|
||||
return started_at
|
||||
|
||||
|
||||
def _apply_ssh_update_result(db: Session, host: Host, result: SshCommandResult) -> None:
|
||||
now = _utcnow()
|
||||
host.agent_update_last_at = now
|
||||
if result.ok:
|
||||
host.agent_update_state = "success"
|
||||
host.agent_update_last_error = None
|
||||
if result.agent_version:
|
||||
host.product_version = result.agent_version
|
||||
host.ssh_admin_ok = True
|
||||
host.ssh_admin_checked_at = now
|
||||
else:
|
||||
host.agent_update_state = "failed"
|
||||
host.agent_update_last_error = (result.message or "SSH update failed")[:2000]
|
||||
|
||||
|
||||
def start_host_remote_action(
|
||||
db: Session,
|
||||
host: Host,
|
||||
*,
|
||||
title: str,
|
||||
runner: ActionRunner,
|
||||
) -> dict[str, Any]:
|
||||
host_id = int(host.id)
|
||||
with _lock:
|
||||
if host_id in _running or host.agent_update_state == "running":
|
||||
raise RemoteActionAlreadyRunningError(
|
||||
f"Remote action already running for host {host.hostname}"
|
||||
)
|
||||
_running.add(host_id)
|
||||
|
||||
started_at = _mark_running(db, host, title=title)
|
||||
db.commit()
|
||||
|
||||
def _worker() -> None:
|
||||
session = SessionLocal()
|
||||
try:
|
||||
row = session.get(Host, host_id)
|
||||
if row is None:
|
||||
return
|
||||
result = runner(session, row)
|
||||
started = (row.remote_action or {}).get("started_at") or started_at
|
||||
row.remote_action = _result_payload(result, title=title, started_at=started)
|
||||
if isinstance(result, SshCommandResult):
|
||||
_apply_ssh_update_result(session, row, result)
|
||||
session.commit()
|
||||
except Exception:
|
||||
logger.exception("Background remote action failed for host_id=%s", host_id)
|
||||
session.rollback()
|
||||
row = session.get(Host, host_id)
|
||||
if row is not None:
|
||||
started = (row.remote_action or {}).get("started_at") or started_at
|
||||
row.agent_update_state = "failed"
|
||||
row.agent_update_last_error = "Internal error during remote action"
|
||||
row.remote_action = {
|
||||
"title": title,
|
||||
"status": "failed",
|
||||
"message": "Внутренняя ошибка SAC при выполнении действия",
|
||||
"stdout": "",
|
||||
"stderr": "",
|
||||
"output": "",
|
||||
"ok": False,
|
||||
"exit_code": None,
|
||||
"product_version": None,
|
||||
"target": row.hostname,
|
||||
"started_at": started,
|
||||
"finished_at": _utcnow().isoformat(),
|
||||
}
|
||||
session.commit()
|
||||
finally:
|
||||
session.close()
|
||||
with _lock:
|
||||
_running.discard(host_id)
|
||||
|
||||
if _run_inline():
|
||||
_worker()
|
||||
else:
|
||||
threading.Thread(target=_worker, name=f"sac-remote-action-{host_id}", daemon=True).start()
|
||||
return dict(host.remote_action or {})
|
||||
|
||||
|
||||
def run_ssh_monitor_update_action(db: Session, host: Host) -> SshCommandResult:
|
||||
cfg = get_effective_linux_admin_config(db)
|
||||
if not cfg.configured:
|
||||
return SshCommandResult(
|
||||
ok=False,
|
||||
message="Linux SSH admin is not configured",
|
||||
target=host.hostname or "",
|
||||
)
|
||||
agent_cfg = get_effective_agent_update_config(db)
|
||||
repo_url = (agent_cfg.ssh_git_repo_url or "").strip() or None
|
||||
branch = (agent_cfg.git_branch or "main").strip() or "main"
|
||||
try:
|
||||
targets = iter_ssh_targets(host)
|
||||
except Exception as exc:
|
||||
return SshCommandResult(
|
||||
ok=False,
|
||||
message=str(exc),
|
||||
target=host.hostname or "",
|
||||
)
|
||||
last: SshCommandResult | None = None
|
||||
for target in targets:
|
||||
last = run_ssh_monitor_update(
|
||||
target=target,
|
||||
user=cfg.user,
|
||||
password=cfg.password,
|
||||
repo_url=repo_url,
|
||||
git_branch=branch,
|
||||
)
|
||||
if last.ok:
|
||||
return last
|
||||
return last or SshCommandResult(
|
||||
ok=False,
|
||||
message="SSH update failed",
|
||||
target=host.hostname or "",
|
||||
)
|
||||
|
||||
|
||||
def run_agent_fallback_action(db: Session, host: Host) -> AgentUpdateFallbackResult:
|
||||
return execute_agent_update_fallback(db, host)
|
||||
|
||||
|
||||
def get_remote_action_status(host: Host) -> dict[str, Any]:
|
||||
payload = dict(host.remote_action or {})
|
||||
if not payload:
|
||||
return {"active": False, "host_id": host.id}
|
||||
status = payload.get("status") or host.agent_update_state or "unknown"
|
||||
active = status == "running" or host.agent_update_state == "running"
|
||||
return {
|
||||
"active": active,
|
||||
"host_id": host.id,
|
||||
"hostname": host.hostname,
|
||||
"status": status,
|
||||
"title": payload.get("title") or "",
|
||||
"message": payload.get("message") or "",
|
||||
"stdout": payload.get("stdout") or "",
|
||||
"stderr": payload.get("stderr") or "",
|
||||
"output": payload.get("output") or "",
|
||||
"ok": payload.get("ok"),
|
||||
"exit_code": payload.get("exit_code"),
|
||||
"product_version": payload.get("product_version") or host.product_version,
|
||||
"target": payload.get("target") or host.hostname,
|
||||
"agent_update_state": host.agent_update_state,
|
||||
"started_at": payload.get("started_at"),
|
||||
"finished_at": payload.get("finished_at"),
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
"""Единый источник версии SAC (API, health, логи, OpenAPI)."""
|
||||
|
||||
APP_NAME = "Security Alert Center"
|
||||
APP_VERSION = "0.20.18"
|
||||
APP_VERSION = "0.20.19"
|
||||
APP_VERSION_LABEL = f"{APP_NAME} v.{APP_VERSION}"
|
||||
|
||||
Reference in New Issue
Block a user