PapaTramp
7df718eed3
Track resolved_by on problems; suppress scan/ingest recreate after manual resolve; reopen same problem after cooldown; auto-close via heartbeat unchanged. Co-authored-by: Cursor <cursoragent@cursor.com>
118 lines
3.6 KiB
Python
118 lines
3.6 KiB
Python
"""Auto-create Problems from ingested events (rules + correlation)."""
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.config import get_settings
|
|
from app.models import Event, Problem, ProblemEvent
|
|
from app.services.host_health import HEARTBEAT_TYPE
|
|
from app.services.problem_rules import (
|
|
RuleMatch,
|
|
RULE_HOST_SILENCE,
|
|
build_fingerprint,
|
|
pick_rule_match,
|
|
resolve_host_silence_problems,
|
|
)
|
|
|
|
|
|
def _correlation_cutoff(now: datetime) -> datetime:
|
|
from datetime import timedelta
|
|
|
|
minutes = get_settings().sac_problem_correlation_window_minutes
|
|
return now - timedelta(minutes=minutes)
|
|
|
|
|
|
def _append_event(db: Session, problem: Problem, event: Event) -> None:
|
|
linked = db.scalar(
|
|
select(ProblemEvent).where(
|
|
ProblemEvent.problem_id == problem.id,
|
|
ProblemEvent.event_id == event.id,
|
|
)
|
|
)
|
|
if linked is not None:
|
|
return
|
|
db.add(ProblemEvent(problem_id=problem.id, event_id=event.id))
|
|
problem.event_count = (problem.event_count or 0) + 1
|
|
problem.last_seen_at = event.occurred_at
|
|
problem.updated_at = datetime.now(timezone.utc)
|
|
if event.severity in {"high", "critical"} and problem.severity not in {"high", "critical"}:
|
|
problem.severity = event.severity
|
|
db.flush()
|
|
|
|
|
|
def open_or_append_problem(
|
|
db: Session, event: Event, match: RuleMatch
|
|
) -> tuple[Problem | None, bool]:
|
|
if match.rule_id == RULE_HOST_SILENCE:
|
|
from app.services.host_silence_scan import open_or_refresh_host_silence_problem
|
|
|
|
ref = event.occurred_at
|
|
if ref.tzinfo is None:
|
|
ref = ref.replace(tzinfo=timezone.utc)
|
|
problem, created = open_or_refresh_host_silence_problem(
|
|
db, event.host_id, match, now=ref
|
|
)
|
|
if problem is None:
|
|
return None, False
|
|
_append_event(db, problem, event)
|
|
return problem, created
|
|
|
|
fingerprint = build_fingerprint(
|
|
event.host_id,
|
|
match.correlation_type,
|
|
match.rule_id,
|
|
match.fingerprint_suffix,
|
|
)
|
|
now = datetime.now(timezone.utc)
|
|
cutoff = _correlation_cutoff(now)
|
|
|
|
open_problem = db.scalar(
|
|
select(Problem)
|
|
.where(
|
|
Problem.status == "open",
|
|
Problem.fingerprint == fingerprint,
|
|
Problem.host_id == event.host_id,
|
|
Problem.last_seen_at >= cutoff,
|
|
)
|
|
.order_by(Problem.last_seen_at.desc())
|
|
.limit(1)
|
|
)
|
|
if open_problem:
|
|
_append_event(db, open_problem, event)
|
|
if match.severity in {"high", "critical"} and open_problem.severity not in {"high", "critical"}:
|
|
open_problem.severity = match.severity
|
|
open_problem.summary = match.summary
|
|
return open_problem, False
|
|
|
|
problem = Problem(
|
|
host_id=event.host_id,
|
|
title=match.title,
|
|
summary=match.summary,
|
|
severity=match.severity,
|
|
status="open",
|
|
rule_id=match.rule_id,
|
|
fingerprint=fingerprint,
|
|
event_count=1,
|
|
last_seen_at=event.occurred_at,
|
|
)
|
|
db.add(problem)
|
|
db.flush()
|
|
db.add(ProblemEvent(problem_id=problem.id, event_id=event.id))
|
|
db.flush()
|
|
return problem, True
|
|
|
|
|
|
def maybe_create_problem(db: Session, event: Event) -> tuple[Problem | None, bool]:
|
|
if event.type == HEARTBEAT_TYPE:
|
|
resolve_host_silence_problems(db, event.host_id)
|
|
return None, False
|
|
|
|
match = pick_rule_match(db, event)
|
|
if match is None:
|
|
return None, False
|
|
|
|
problem, created = open_or_append_problem(db, event, match)
|
|
return problem, created
|