feat: proactive host_silence scan for stale agent.heartbeat (0.8.3)
Background scan every 5 min opens rule:host_silence and notifies without waiting for the next ingest event. CLI job + optional systemd timer. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,119 @@
|
||||
"""Периодическое сканирование хостов с устаревшим agent.heartbeat (proactive host_silence)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Host, Problem
|
||||
from app.services.problem_rules import RuleMatch, build_fingerprint, host_silence_match_for_host
|
||||
from app.services.problems import _correlation_cutoff
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HostSilenceScanResult:
|
||||
host_id: int
|
||||
hostname: str
|
||||
problem: Problem
|
||||
created: bool
|
||||
|
||||
|
||||
def open_or_refresh_host_silence_problem(
|
||||
db: Session,
|
||||
host_id: int,
|
||||
match: RuleMatch,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
) -> tuple[Problem, bool]:
|
||||
"""Открыть или обновить open Problem host_silence без ingest-события."""
|
||||
ref = now or datetime.now(timezone.utc)
|
||||
fingerprint = build_fingerprint(
|
||||
host_id,
|
||||
match.correlation_type,
|
||||
match.rule_id,
|
||||
match.fingerprint_suffix,
|
||||
)
|
||||
cutoff = _correlation_cutoff(ref)
|
||||
|
||||
open_problem = db.scalar(
|
||||
select(Problem)
|
||||
.where(
|
||||
Problem.status == "open",
|
||||
Problem.fingerprint == fingerprint,
|
||||
Problem.host_id == host_id,
|
||||
Problem.last_seen_at >= cutoff,
|
||||
)
|
||||
.order_by(Problem.last_seen_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
if open_problem:
|
||||
open_problem.summary = match.summary
|
||||
open_problem.title = match.title
|
||||
open_problem.last_seen_at = ref
|
||||
open_problem.updated_at = ref
|
||||
db.flush()
|
||||
return open_problem, False
|
||||
|
||||
problem = Problem(
|
||||
host_id=host_id,
|
||||
title=match.title,
|
||||
summary=match.summary,
|
||||
severity=match.severity,
|
||||
status="open",
|
||||
rule_id=match.rule_id,
|
||||
fingerprint=fingerprint,
|
||||
event_count=0,
|
||||
last_seen_at=ref,
|
||||
)
|
||||
db.add(problem)
|
||||
db.flush()
|
||||
return problem, True
|
||||
|
||||
|
||||
def run_host_silence_scan(db: Session, *, now: datetime | None = None) -> list[HostSilenceScanResult]:
|
||||
"""Найти stale-хосты и создать/обновить Problems rule:host_silence."""
|
||||
ref = now or datetime.now(timezone.utc)
|
||||
hosts = db.scalars(select(Host).order_by(Host.id)).all()
|
||||
results: list[HostSilenceScanResult] = []
|
||||
|
||||
for host in hosts:
|
||||
match = host_silence_match_for_host(db, host.id, hostname=host.hostname, now=ref)
|
||||
if match is None:
|
||||
continue
|
||||
problem, created = open_or_refresh_host_silence_problem(db, host.id, match, now=ref)
|
||||
results.append(
|
||||
HostSilenceScanResult(
|
||||
host_id=host.id,
|
||||
hostname=host.hostname,
|
||||
problem=problem,
|
||||
created=created,
|
||||
)
|
||||
)
|
||||
if created:
|
||||
logger.info(
|
||||
"host_silence scan: new problem host=%s id=%s problem_id=%s",
|
||||
host.hostname,
|
||||
host.id,
|
||||
problem.id,
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def dispatch_host_silence_notifications(db: Session, results: list[HostSilenceScanResult]) -> int:
|
||||
"""Telegram/webhook/email только для вновь созданных Problems."""
|
||||
from app.services.notify_dispatch import notify_problem
|
||||
|
||||
sent = 0
|
||||
for item in results:
|
||||
if not item.created:
|
||||
continue
|
||||
notify_problem(item.problem, event=None, db=db)
|
||||
sent += 1
|
||||
return sent
|
||||
Reference in New Issue
Block a user