fix: 12h cooldown before reopening manually closed host_silence (0.9.9)

Track resolved_by on problems; suppress scan/ingest recreate after manual
resolve; reopen same problem after cooldown; auto-close via heartbeat unchanged.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-06-11 10:33:55 +10:00
parent befaf86bf4
commit 7df718eed3
11 changed files with 285 additions and 7 deletions
+59 -3
View File
@@ -4,11 +4,12 @@ from __future__ import annotations
import logging
from dataclasses import dataclass
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.config import get_settings
from app.models import Host, Problem
from app.services.problem_rules import RuleMatch, build_fingerprint, host_silence_match_for_host
from app.services.problems import _correlation_cutoff
@@ -30,8 +31,12 @@ def open_or_refresh_host_silence_problem(
match: RuleMatch,
*,
now: datetime | None = None,
) -> tuple[Problem, bool]:
"""Открыть или обновить open Problem host_silence без ingest-события."""
) -> tuple[Problem | None, bool]:
"""
Открыть или обновить open Problem host_silence без ingest-события.
Returns (problem, created). problem is None when suppressed by manual-resolve cooldown.
"""
ref = now or datetime.now(timezone.utc)
fingerprint = build_fingerprint(
host_id,
@@ -60,6 +65,55 @@ def open_or_refresh_host_silence_problem(
db.flush()
return open_problem, False
settings = get_settings()
cooldown_hours = settings.sac_host_silence_manual_resolve_cooldown_hours
if cooldown_hours > 0:
cooldown = timedelta(hours=cooldown_hours)
manual_recent = db.scalar(
select(Problem)
.where(
Problem.host_id == host_id,
Problem.rule_id == match.rule_id,
Problem.fingerprint == fingerprint,
Problem.status == "resolved",
Problem.resolved_by == "manual",
Problem.updated_at >= ref - cooldown,
)
.order_by(Problem.updated_at.desc())
.limit(1)
)
if manual_recent is not None:
logger.debug(
"host_silence suppressed (manual cooldown %sh) host_id=%s problem_id=%s",
cooldown_hours,
host_id,
manual_recent.id,
)
return None, False
manual_expired = db.scalar(
select(Problem)
.where(
Problem.host_id == host_id,
Problem.rule_id == match.rule_id,
Problem.fingerprint == fingerprint,
Problem.status == "resolved",
Problem.resolved_by == "manual",
Problem.updated_at < ref - cooldown,
)
.order_by(Problem.updated_at.desc())
.limit(1)
)
if manual_expired is not None:
manual_expired.status = "open"
manual_expired.resolved_by = None
manual_expired.summary = match.summary
manual_expired.title = match.title
manual_expired.last_seen_at = ref
manual_expired.updated_at = ref
db.flush()
return manual_expired, True
problem = Problem(
host_id=host_id,
title=match.title,
@@ -87,6 +141,8 @@ def run_host_silence_scan(db: Session, *, now: datetime | None = None) -> list[H
if match is None:
continue
problem, created = open_or_refresh_host_silence_problem(db, host.id, match, now=ref)
if problem is None:
continue
results.append(
HostSilenceScanResult(
host_id=host.id,
+1
View File
@@ -190,6 +190,7 @@ def resolve_host_silence_problems(db: Session, host_id: int) -> int:
).all()
for problem in problems:
problem.status = "resolved"
problem.resolved_by = "auto"
if problems:
db.flush()
return len(problems)
+18 -2
View File
@@ -10,6 +10,7 @@ from app.models import Event, Problem, ProblemEvent
from app.services.host_health import HEARTBEAT_TYPE
from app.services.problem_rules import (
RuleMatch,
RULE_HOST_SILENCE,
build_fingerprint,
pick_rule_match,
resolve_host_silence_problems,
@@ -43,7 +44,21 @@ def _append_event(db: Session, problem: Problem, event: Event) -> None:
def open_or_append_problem(
db: Session, event: Event, match: RuleMatch
) -> tuple[Problem, bool]:
) -> tuple[Problem | None, bool]:
if match.rule_id == RULE_HOST_SILENCE:
from app.services.host_silence_scan import open_or_refresh_host_silence_problem
ref = event.occurred_at
if ref.tzinfo is None:
ref = ref.replace(tzinfo=timezone.utc)
problem, created = open_or_refresh_host_silence_problem(
db, event.host_id, match, now=ref
)
if problem is None:
return None, False
_append_event(db, problem, event)
return problem, created
fingerprint = build_fingerprint(
event.host_id,
match.correlation_type,
@@ -98,4 +113,5 @@ def maybe_create_problem(db: Session, event: Event) -> tuple[Problem | None, boo
if match is None:
return None, False
return open_or_append_problem(db, event, match)
problem, created = open_or_append_problem(db, event, match)
return problem, created