fix: reset stale remote agent updates after sac-api restart (0.4.3)

Clear zombie running jobs on deploy and add admin API to cancel a stuck job.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-06-25 10:43:31 +10:00
parent dd2ae51b43
commit 86fd400f0b
6 changed files with 133 additions and 2 deletions
@@ -219,6 +219,59 @@ def run_agent_fallback_action(db: Session, host: Host) -> AgentUpdateFallbackRes
return execute_agent_update_fallback(db, host)
def clear_stale_running_remote_actions(
db: Session,
*,
reason: str = "Прервано перезапуском SAC (фоновый worker не завершил job)",
host_ids: list[int] | None = None,
) -> list[str]:
"""Сбросить зависшие running после restart sac-api (daemon thread умер, запись в БД осталась)."""
from sqlalchemy import select
stmt = select(Host).where(Host.agent_update_state == "running")
if host_ids is not None:
stmt = stmt.where(Host.id.in_(host_ids))
rows = list(db.scalars(stmt).all())
if not rows:
return []
finished = _utcnow().isoformat()
hostnames: list[str] = []
for host in rows:
started = (host.remote_action or {}).get("started_at")
payload = dict(host.remote_action or {})
payload.update(
{
"title": payload.get("title") or "Remote action",
"status": "failed",
"message": reason,
"ok": False,
"finished_at": finished,
"started_at": started,
"target": payload.get("target") or host.hostname,
}
)
host.remote_action = payload
host.agent_update_state = "failed"
host.agent_update_last_error = reason[:2000]
hostnames.append(host.hostname or str(host.id))
db.commit()
with _lock:
for host in rows:
_running.discard(int(host.id))
return hostnames
def cancel_host_remote_action(db: Session, host: Host, *, reason: str | None = None) -> bool:
"""Отменить зависший running job вручную (admin)."""
if host.agent_update_state != "running":
return False
msg = reason or "Отменено администратором SAC"
cleared = clear_stale_running_remote_actions(db, reason=msg, host_ids=[int(host.id)])
return bool(cleared)
def get_remote_action_status(host: Host) -> dict[str, Any]:
payload = dict(host.remote_action or {})
if not payload: