feat: host heartbeat status and dashboard metrics (2.2)
Track agent.heartbeat and report.daily.ssh per host; show online/stale on Hosts UI; dashboard counters and SAC_HEARTBEAT_STALE_MINUTES config.
This commit is contained in:
@@ -6,9 +6,11 @@ from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session, joinedload
|
||||
|
||||
from app.auth.jwt_auth import get_current_user
|
||||
from app.config import get_settings
|
||||
from app.database import get_db
|
||||
from app.models import Event, Host, Problem
|
||||
from app.schemas.list_models import EventSummary
|
||||
from app.services.host_health import DAILY_REPORT_SSH, HEARTBEAT_TYPE, count_stale_hosts
|
||||
|
||||
router = APIRouter(prefix="/dashboards", tags=["dashboards"])
|
||||
|
||||
@@ -16,6 +18,9 @@ router = APIRouter(prefix="/dashboards", tags=["dashboards"])
|
||||
class DashboardSummary(BaseModel):
|
||||
events_last_24h: int
|
||||
hosts_total: int
|
||||
hosts_stale: int
|
||||
heartbeats_24h: int
|
||||
daily_reports_24h: int
|
||||
problems_open: int
|
||||
severity_24h: dict[str, int]
|
||||
recent_events: list[EventSummary]
|
||||
@@ -32,6 +37,18 @@ def dashboard_summary(
|
||||
select(func.count()).select_from(Event).where(Event.received_at >= since)
|
||||
) or 0
|
||||
hosts_total = db.scalar(select(func.count()).select_from(Host)) or 0
|
||||
settings = get_settings()
|
||||
hosts_stale = count_stale_hosts(db, settings.sac_heartbeat_stale_minutes)
|
||||
heartbeats_24h = db.scalar(
|
||||
select(func.count())
|
||||
.select_from(Event)
|
||||
.where(Event.received_at >= since, Event.type == HEARTBEAT_TYPE)
|
||||
) or 0
|
||||
daily_reports_24h = db.scalar(
|
||||
select(func.count())
|
||||
.select_from(Event)
|
||||
.where(Event.received_at >= since, Event.type == DAILY_REPORT_SSH)
|
||||
) or 0
|
||||
problems_open = (
|
||||
db.scalar(select(func.count()).select_from(Problem).where(Problem.status == "open")) or 0
|
||||
)
|
||||
@@ -71,6 +88,9 @@ def dashboard_summary(
|
||||
return DashboardSummary(
|
||||
events_last_24h=events_24h,
|
||||
hosts_total=hosts_total,
|
||||
hosts_stale=hosts_stale,
|
||||
heartbeats_24h=heartbeats_24h,
|
||||
daily_reports_24h=daily_reports_24h,
|
||||
problems_open=problems_open,
|
||||
severity_24h=severity_24h,
|
||||
recent_events=recent_events,
|
||||
|
||||
@@ -3,9 +3,16 @@ from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth.jwt_auth import get_current_user
|
||||
from app.config import get_settings
|
||||
from app.database import get_db
|
||||
from app.models import Event, Host
|
||||
from app.schemas.list_models import HostListResponse, HostSummary
|
||||
from app.services.host_health import (
|
||||
DAILY_REPORT_SSH,
|
||||
HEARTBEAT_TYPE,
|
||||
agent_status,
|
||||
max_event_time_by_host,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/hosts", tags=["hosts"])
|
||||
|
||||
@@ -37,11 +44,16 @@ def list_hosts(
|
||||
.limit(page_size)
|
||||
).all()
|
||||
|
||||
settings = get_settings()
|
||||
hb_map = max_event_time_by_host(db, HEARTBEAT_TYPE)
|
||||
report_map = max_event_time_by_host(db, DAILY_REPORT_SSH)
|
||||
|
||||
items: list[HostSummary] = []
|
||||
for host in rows:
|
||||
event_count = db.scalar(
|
||||
select(func.count()).select_from(Event).where(Event.host_id == host.id)
|
||||
)
|
||||
last_hb = hb_map.get(host.id)
|
||||
items.append(
|
||||
HostSummary(
|
||||
id=host.id,
|
||||
@@ -53,6 +65,11 @@ def list_hosts(
|
||||
ipv4=host.ipv4,
|
||||
last_seen_at=host.last_seen_at,
|
||||
event_count=int(event_count or 0),
|
||||
last_heartbeat_at=last_hb,
|
||||
last_daily_report_at=report_map.get(host.id),
|
||||
agent_status=agent_status(
|
||||
last_hb, stale_minutes=settings.sac_heartbeat_stale_minutes
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@ from sqlalchemy.orm import Session
|
||||
from app.auth.jwt_auth import verify_access_token
|
||||
from app.database import SessionLocal
|
||||
from app.models import Event, Problem
|
||||
from app.config import get_settings
|
||||
from app.services.host_health import DAILY_REPORT_SSH, HEARTBEAT_TYPE, count_stale_hosts
|
||||
from app.version import APP_VERSION
|
||||
|
||||
router = APIRouter(prefix="/stream", tags=["stream"])
|
||||
@@ -26,10 +28,24 @@ def _dashboard_snapshot(db: Session) -> dict:
|
||||
db.scalar(select(func.count()).select_from(Problem).where(Problem.status == "open")) or 0
|
||||
)
|
||||
last_event = db.scalar(select(Event.id).order_by(Event.received_at.desc()).limit(1))
|
||||
settings = get_settings()
|
||||
return {
|
||||
"type": "dashboard",
|
||||
"version": APP_VERSION,
|
||||
"events_last_24h": events_24h,
|
||||
"hosts_stale": count_stale_hosts(db, settings.sac_heartbeat_stale_minutes),
|
||||
"heartbeats_24h": db.scalar(
|
||||
select(func.count())
|
||||
.select_from(Event)
|
||||
.where(Event.received_at >= since, Event.type == HEARTBEAT_TYPE)
|
||||
)
|
||||
or 0,
|
||||
"daily_reports_24h": db.scalar(
|
||||
select(func.count())
|
||||
.select_from(Event)
|
||||
.where(Event.received_at >= since, Event.type == DAILY_REPORT_SSH)
|
||||
)
|
||||
or 0,
|
||||
"problems_open": problems_open,
|
||||
"last_event_id": last_event,
|
||||
"at": datetime.now(timezone.utc).isoformat(),
|
||||
|
||||
@@ -50,6 +50,9 @@ class Settings(BaseSettings):
|
||||
telegram_chat_id: str = ""
|
||||
telegram_min_severity: str = "high"
|
||||
|
||||
# Порог «живости» агента: agent.heartbeat раз в ~12 ч (ssh-monitor)
|
||||
sac_heartbeat_stale_minutes: int = 780
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
|
||||
@@ -13,6 +13,9 @@ class HostSummary(BaseModel):
|
||||
ipv4: str | None
|
||||
last_seen_at: datetime
|
||||
event_count: int | None = None
|
||||
last_heartbeat_at: datetime | None = None
|
||||
last_daily_report_at: datetime | None = None
|
||||
agent_status: str = "unknown"
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Статус агентов по событиям agent.heartbeat / report.daily.* в БД."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Event, Host
|
||||
|
||||
HEARTBEAT_TYPE = "agent.heartbeat"
|
||||
DAILY_REPORT_SSH = "report.daily.ssh"
|
||||
|
||||
|
||||
def max_event_time_by_host(db: Session, event_type: str) -> dict[int, datetime]:
|
||||
rows = db.execute(
|
||||
select(Event.host_id, func.max(Event.received_at))
|
||||
.where(Event.type == event_type)
|
||||
.group_by(Event.host_id)
|
||||
).all()
|
||||
return {int(host_id): ts for host_id, ts in rows if ts is not None}
|
||||
|
||||
|
||||
def agent_status(
|
||||
last_heartbeat_at: datetime | None,
|
||||
*,
|
||||
stale_minutes: int,
|
||||
now: datetime | None = None,
|
||||
) -> str:
|
||||
"""
|
||||
online — heartbeat не старше порога;
|
||||
stale — был хост, но heartbeat устарел или отсутствует;
|
||||
unknown — heartbeat ещё не приходил.
|
||||
"""
|
||||
if last_heartbeat_at is None:
|
||||
return "unknown"
|
||||
ref = now or datetime.now(timezone.utc)
|
||||
hb = last_heartbeat_at
|
||||
if hb.tzinfo is None:
|
||||
hb = hb.replace(tzinfo=timezone.utc)
|
||||
age_sec = (ref - hb).total_seconds()
|
||||
if age_sec > stale_minutes * 60:
|
||||
return "stale"
|
||||
return "online"
|
||||
|
||||
|
||||
def count_stale_hosts(db: Session, stale_minutes: int) -> int:
|
||||
hb_map = max_event_time_by_host(db, HEARTBEAT_TYPE)
|
||||
host_ids = db.scalars(select(Host.id)).all()
|
||||
return sum(
|
||||
1
|
||||
for host_id in host_ids
|
||||
if agent_status(hb_map.get(host_id), stale_minutes=stale_minutes) == "stale"
|
||||
)
|
||||
Reference in New Issue
Block a user