feat: host heartbeat status and dashboard metrics (2.2)

Track agent.heartbeat and report.daily.ssh per host; show online/stale on Hosts UI; dashboard counters and SAC_HEARTBEAT_STALE_MINUTES config.
2026-05-27 13:36:17 +10:00
parent 9ba3c4b63b
commit 04301ab359
14 changed files with 246 additions and 4 deletions
@@ -6,9 +6,11 @@ from sqlalchemy import func, select
 from sqlalchemy.orm import Session, joinedload

 from app.auth.jwt_auth import get_current_user
+from app.config import get_settings
 from app.database import get_db
 from app.models import Event, Host, Problem
 from app.schemas.list_models import EventSummary
+from app.services.host_health import DAILY_REPORT_SSH, HEARTBEAT_TYPE, count_stale_hosts

 router = APIRouter(prefix="/dashboards", tags=["dashboards"])

@@ -16,6 +18,9 @@ router = APIRouter(prefix="/dashboards", tags=["dashboards"])
 class DashboardSummary(BaseModel):
    events_last_24h: int
    hosts_total: int
+    hosts_stale: int
+    heartbeats_24h: int
+    daily_reports_24h: int
    problems_open: int
    severity_24h: dict[str, int]
    recent_events: list[EventSummary]
@@ -32,6 +37,18 @@ def dashboard_summary(
        select(func.count()).select_from(Event).where(Event.received_at >= since)
    ) or 0
    hosts_total = db.scalar(select(func.count()).select_from(Host)) or 0
+    settings = get_settings()
+    hosts_stale = count_stale_hosts(db, settings.sac_heartbeat_stale_minutes)
+    heartbeats_24h = db.scalar(
+        select(func.count())
+        .select_from(Event)
+        .where(Event.received_at >= since, Event.type == HEARTBEAT_TYPE)
+    ) or 0
+    daily_reports_24h = db.scalar(
+        select(func.count())
+        .select_from(Event)
+        .where(Event.received_at >= since, Event.type == DAILY_REPORT_SSH)
+    ) or 0
    problems_open = (
        db.scalar(select(func.count()).select_from(Problem).where(Problem.status == "open")) or 0
    )
@@ -71,6 +88,9 @@ def dashboard_summary(
    return DashboardSummary(
        events_last_24h=events_24h,
        hosts_total=hosts_total,
+        hosts_stale=hosts_stale,
+        heartbeats_24h=heartbeats_24h,
+        daily_reports_24h=daily_reports_24h,
        problems_open=problems_open,
        severity_24h=severity_24h,
        recent_events=recent_events,
@@ -3,9 +3,16 @@ from sqlalchemy import func, select
 from sqlalchemy.orm import Session

 from app.auth.jwt_auth import get_current_user
+from app.config import get_settings
 from app.database import get_db
 from app.models import Event, Host
 from app.schemas.list_models import HostListResponse, HostSummary
+from app.services.host_health import (
+    DAILY_REPORT_SSH,
+    HEARTBEAT_TYPE,
+    agent_status,
+    max_event_time_by_host,
+)

 router = APIRouter(prefix="/hosts", tags=["hosts"])

@@ -37,11 +44,16 @@ def list_hosts(
        .limit(page_size)
    ).all()

+    settings = get_settings()
+    hb_map = max_event_time_by_host(db, HEARTBEAT_TYPE)
+    report_map = max_event_time_by_host(db, DAILY_REPORT_SSH)
+
    items: list[HostSummary] = []
    for host in rows:
        event_count = db.scalar(
            select(func.count()).select_from(Event).where(Event.host_id == host.id)
        )
+        last_hb = hb_map.get(host.id)
        items.append(
            HostSummary(
                id=host.id,
@@ -53,6 +65,11 @@ def list_hosts(
                ipv4=host.ipv4,
                last_seen_at=host.last_seen_at,
                event_count=int(event_count or 0),
+                last_heartbeat_at=last_hb,
+                last_daily_report_at=report_map.get(host.id),
+                agent_status=agent_status(
+                    last_hb, stale_minutes=settings.sac_heartbeat_stale_minutes
+                ),
            )
        )

@@ -10,6 +10,8 @@ from sqlalchemy.orm import Session
 from app.auth.jwt_auth import verify_access_token
 from app.database import SessionLocal
 from app.models import Event, Problem
+from app.config import get_settings
+from app.services.host_health import DAILY_REPORT_SSH, HEARTBEAT_TYPE, count_stale_hosts
 from app.version import APP_VERSION

 router = APIRouter(prefix="/stream", tags=["stream"])
@@ -26,10 +28,24 @@ def _dashboard_snapshot(db: Session) -> dict:
        db.scalar(select(func.count()).select_from(Problem).where(Problem.status == "open")) or 0
    )
    last_event = db.scalar(select(Event.id).order_by(Event.received_at.desc()).limit(1))
+    settings = get_settings()
    return {
        "type": "dashboard",
        "version": APP_VERSION,
        "events_last_24h": events_24h,
+        "hosts_stale": count_stale_hosts(db, settings.sac_heartbeat_stale_minutes),
+        "heartbeats_24h": db.scalar(
+            select(func.count())
+            .select_from(Event)
+            .where(Event.received_at >= since, Event.type == HEARTBEAT_TYPE)
+        )
+        or 0,
+        "daily_reports_24h": db.scalar(
+            select(func.count())
+            .select_from(Event)
+            .where(Event.received_at >= since, Event.type == DAILY_REPORT_SSH)
+        )
+        or 0,
        "problems_open": problems_open,
        "last_event_id": last_event,
        "at": datetime.now(timezone.utc).isoformat(),
@@ -50,6 +50,9 @@ class Settings(BaseSettings):
    telegram_chat_id: str = ""
    telegram_min_severity: str = "high"

+    # Порог «живости» агента: agent.heartbeat раз в ~12 ч (ssh-monitor)
+    sac_heartbeat_stale_minutes: int = 780
+

@lru_cache
 def get_settings() -> Settings:
@@ -13,6 +13,9 @@ class HostSummary(BaseModel):
    ipv4: str | None
    last_seen_at: datetime
    event_count: int | None = None
+    last_heartbeat_at: datetime | None = None
+    last_daily_report_at: datetime | None = None
+    agent_status: str = "unknown"

    model_config = {"from_attributes": True}

@@ -0,0 +1,53 @@
+"""Статус агентов по событиям agent.heartbeat / report.daily.* в БД."""
+
+from datetime import datetime, timezone
+
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from app.models import Event, Host
+
+HEARTBEAT_TYPE = "agent.heartbeat"
+DAILY_REPORT_SSH = "report.daily.ssh"
+
+
+def max_event_time_by_host(db: Session, event_type: str) -> dict[int, datetime]:
+    rows = db.execute(
+        select(Event.host_id, func.max(Event.received_at))
+        .where(Event.type == event_type)
+        .group_by(Event.host_id)
+    ).all()
+    return {int(host_id): ts for host_id, ts in rows if ts is not None}
+
+
+def agent_status(
+    last_heartbeat_at: datetime | None,
+    *,
+    stale_minutes: int,
+    now: datetime | None = None,
+) -> str:
+    """
+    online — heartbeat не старше порога;
+    stale — был хост, но heartbeat устарел или отсутствует;
+    unknown — heartbeat ещё не приходил.
+    """
+    if last_heartbeat_at is None:
+        return "unknown"
+    ref = now or datetime.now(timezone.utc)
+    hb = last_heartbeat_at
+    if hb.tzinfo is None:
+        hb = hb.replace(tzinfo=timezone.utc)
+    age_sec = (ref - hb).total_seconds()
+    if age_sec > stale_minutes * 60:
+        return "stale"
+    return "online"
+
+
+def count_stale_hosts(db: Session, stale_minutes: int) -> int:
+    hb_map = max_event_time_by_host(db, HEARTBEAT_TYPE)
+    host_ids = db.scalars(select(Host.id)).all()
+    return sum(
+        1
+        for host_id in host_ids
+        if agent_status(hb_map.get(host_id), stale_minutes=stale_minutes) == "stale"
+    )
@@ -23,4 +23,7 @@ TELEGRAM_BOT_TOKEN=
 TELEGRAM_CHAT_ID=
 TELEGRAM_MIN_SEVERITY=high

+# Статус хоста по agent.heartbeat (минуты; ssh-monitor шлёт ~раз в 12 ч)
+SAC_HEARTBEAT_STALE_MINUTES=780
+
 CORS_ORIGINS=*
@@ -0,0 +1,61 @@
+# Фаза 2.2 — Heartbeat и daily report через SAC
+
+Агент **уже** шлёт в SAC (в т.ч. в `exclusive`):
+
+| Тип | Интервал (ssh-monitor) |
+|-----|-------------------------|
+| `agent.heartbeat` | каждые **12 ч** |
+| `report.daily.ssh` | раз в сутки после `DAILY_REPORT_HOUR` |
+
+SAC отображает их в **События** и на **Dashboard / Хосты**.
+
+---
+
+## UI SAC
+
+- **Dashboard:** счётчики heartbeat/отчётов за 24 ч, хосты с устаревшим heartbeat.
+- **Хосты:** колонки *Агент* (`online` / `stale` / `unknown`), время последнего heartbeat и отчёта.
+
+---
+
+## Настройка порога «живости» (сервер SAC)
+
+В `config/sac-api.env`:
+
+```bash
+# По умолчанию 780 мин (13 ч) — чуть больше интервала heartbeat агента (12 ч)
+SAC_HEARTBEAT_STALE_MINUTES=780
+```
+
+После изменения: `sudo systemctl restart sac-api`.
+
+---
+
+## Проверка на пилотном хосте (exclusive)
+
+1. Дождаться heartbeat или ускорить тест: на агенте временно уменьшить интервал нельзя без правки кода — проще подождать или вручную:
+   ```bash
+   sudo /usr/local/bin/ssh-monitor --dry-run
+   ```
+   (не шлёт реально; для реального теста — дождаться цикла 12 ч или смотреть уже пришедшие события).
+
+2. В SAC UI → **События**, фильтр `type=agent.heartbeat`.
+
+3. **Хосты** → статус `online` после свежего heartbeat.
+
+4. Ежедневный отчёт: после `DAILY_REPORT_HOUR` — событие `report.daily.ssh`.
+
+---
+
+## Деплой
+
+```bash
+sudo /opt/sac-deploy.sh
+```
+
+---
+
+## См. также
+
+- [pilot-2.1-exclusive.md](pilot-2.1-exclusive.md)
+- [agent-integration.md](agent-integration.md)
@@ -63,9 +63,9 @@

 | # | Задача |
 |---|--------|
-| 2.1 | E2E: 1 Linux + 1 Windows, `UseSAC=exclusive` | 🔄 Linux: [pilot-2.1-exclusive.md](pilot-2.1-exclusive.md); Windows ⏳ RDP |
-| 2.2 | Daily report / heartbeat через SAC |
-| 2.3 | fallback на агентах |
+| 2.1 | E2E: 1 Linux + 1 Windows, `UseSAC=exclusive` | ✅ Linux [pilot-2.1-exclusive.md](pilot-2.1-exclusive.md) |
+| 2.2 | Daily report / heartbeat через SAC | ✅ [pilot-2.2-heartbeat-reports.md](pilot-2.2-heartbeat-reports.md) |
+| 2.3 | fallback на агентах | ✅ счётчик `SAC_FALLBACK_FAILURES` в sac-client |

 **Выход:** `v0.2.0`.

@@ -76,6 +76,9 @@ export interface HostSummary {
  ipv4: string | null;
  last_seen_at: string;
  event_count: number | null;
+  last_heartbeat_at: string | null;
+  last_daily_report_at: string | null;
+  agent_status: "online" | "stale" | "unknown";
 }

 export interface HostListResponse {
@@ -116,6 +119,9 @@ export async function resolveProblem(problemId: number): Promise<{ id: number; s
 export interface DashboardSummary {
  events_last_24h: number;
  hosts_total: number;
+  hosts_stale: number;
+  heartbeats_24h: number;
+  daily_reports_24h: number;
  problems_open: number;
  severity_24h: Record<string, number>;
  recent_events: EventSummary[];
@@ -160,6 +160,10 @@ pre {
  margin: 0.25rem 0 0.5rem;
 }

+.dash-card.dash-warn {
+  border-color: #ff6b6b;
+}
+
 .severity-list {
  list-style: none;
  padding: 0;
@@ -168,6 +172,16 @@ pre {
  gap: 1rem;
 }

+.agent-online {
+  color: #6bcb77;
+}
+.agent-stale {
+  color: #ff6b6b;
+}
+.agent-unknown {
+  color: #9aa4b2;
+}
+
 .live-badge {
  color: #6bcb77;
  font-size: 0.85rem;
@@ -19,6 +19,21 @@
        <div class="dash-label">Хостов</div>
        <RouterLink to="/hosts">Хосты →</RouterLink>
      </div>
+      <div class="dash-card" :class="{ 'dash-warn': data.hosts_stale > 0 }">
+        <div class="dash-value">{{ data.hosts_stale }}</div>
+        <div class="dash-label">Устаревший heartbeat</div>
+        <RouterLink to="/hosts">Хосты →</RouterLink>
+      </div>
+      <div class="dash-card">
+        <div class="dash-value">{{ data.heartbeats_24h }}</div>
+        <div class="dash-label">Heartbeat за 24 ч</div>
+        <RouterLink to="/events?type=agent.heartbeat">События →</RouterLink>
+      </div>
+      <div class="dash-card">
+        <div class="dash-value">{{ data.daily_reports_24h }}</div>
+        <div class="dash-label">Отчёты за 24 ч</div>
+        <RouterLink to="/events?type=report.daily.ssh">События →</RouterLink>
+      </div>
    </div>

    <h2>Severity за 24 ч</h2>
@@ -100,6 +115,9 @@ function connectLive() {
        type?: string;
        events_last_24h?: number;
        problems_open?: number;
+        hosts_stale?: number;
+        heartbeats_24h?: number;
+        daily_reports_24h?: number;
      };
      if (msg.type !== "dashboard" || !data.value) return;
      if (typeof msg.events_last_24h === "number") {
@@ -108,6 +126,15 @@ function connectLive() {
      if (typeof msg.problems_open === "number") {
        data.value.problems_open = msg.problems_open;
      }
+      if (typeof msg.hosts_stale === "number") {
+        data.value.hosts_stale = msg.hosts_stale;
+      }
+      if (typeof msg.heartbeats_24h === "number") {
+        data.value.heartbeats_24h = msg.heartbeats_24h;
+      }
+      if (typeof msg.daily_reports_24h === "number") {
+        data.value.daily_reports_24h = msg.daily_reports_24h;
+      }
    } catch {
      /* ignore malformed SSE */
    }
@@ -57,8 +57,11 @@

 <script setup lang="ts">
 import { onMounted, ref } from "vue";
+import { useRoute } from "vue-router";
 import { apiFetch, type EventListResponse } from "../api";

+const route = useRoute();
+
 const data = ref<EventListResponse | null>(null);
 const loading = ref(false);
 const error = ref("");
@@ -92,5 +95,9 @@ async function load(p: number) {
  }
 }

-onMounted(() => load(1));
+onMounted(() => {
+  const t = route.query.type;
+  if (typeof t === "string" && t) typeFilter.value = t;
+  load(1);
+});
 </script>
@@ -12,6 +12,9 @@
          <th>Product</th>
          <th>OS</th>
          <th>IPv4</th>
+          <th>Агент</th>
+          <th>Heartbeat</th>
+          <th>Отчёт</th>
          <th>Last seen</th>
          <th>Events</th>
        </tr>
@@ -23,6 +26,9 @@
          <td>{{ h.product }} {{ h.product_version || "" }}</td>
          <td>{{ h.os_family }}</td>
          <td>{{ h.ipv4 || "—" }}</td>
+          <td :class="'agent-' + h.agent_status">{{ agentLabel(h.agent_status) }}</td>
+          <td>{{ h.last_heartbeat_at ? formatDt(h.last_heartbeat_at) : "—" }}</td>
+          <td>{{ h.last_daily_report_at ? formatDt(h.last_daily_report_at) : "—" }}</td>
          <td>{{ formatDt(h.last_seen_at) }}</td>
          <td>{{ h.event_count ?? 0 }}</td>
        </tr>
@@ -43,6 +49,12 @@ function formatDt(iso: string) {
  return new Date(iso).toLocaleString("ru-RU");
 }

+function agentLabel(status: string) {
+  if (status === "online") return "online";
+  if (status === "stale") return "stale";
+  return "unknown";
+}
+
 onMounted(async () => {
  loading.value = true;
  try {