fix: dedupe host_silence problems under concurrent scans (0.4.11)

Treat acknowledged as active, dedupe by hostname, add PostgreSQL advisory lock and a unique index. Migration closes existing duplicate open problems. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-03 09:05:28 +10:00
parent 3765d4d476
commit 6c43b8637e
9 changed files with 334 additions and 31 deletions
@@ -6,15 +6,19 @@ import logging
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone

-from sqlalchemy import select
+from sqlalchemy import func, select, text
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session

 from app.config import get_settings
 from app.models import Host, Problem
-from app.services.problem_rules import RuleMatch, build_fingerprint, host_silence_match_for_host
+from app.services.problem_rules import RULE_HOST_SILENCE, RuleMatch, build_fingerprint, host_silence_match_for_host

 logger = logging.getLogger(__name__)

+ACTIVE_HOST_SILENCE_STATUSES = ("open", "acknowledged")
+_HOST_SILENCE_SCAN_LOCK_KEY = 915_001_001
+

@dataclass(frozen=True)
 class HostSilenceScanResult:
@@ -24,19 +28,92 @@ class HostSilenceScanResult:
    created: bool


+def _host_silence_scan_lock(db: Session) -> bool:
+    """Один scan за раз (uvicorn workers + systemd timer)."""
+    bind = db.get_bind()
+    if bind.dialect.name != "postgresql":
+        return True
+    acquired = db.execute(
+        text("SELECT pg_try_advisory_xact_lock(:key)"),
+        {"key": _HOST_SILENCE_SCAN_LOCK_KEY},
+    ).scalar()
+    return bool(acquired)
+
+
+def _find_active_host_silence_problem(
+    db: Session,
+    *,
+    host_id: int,
+    hostname: str | None,
+) -> Problem | None:
+    active = db.scalar(
+        select(Problem)
+        .where(
+            Problem.host_id == host_id,
+            Problem.rule_id == RULE_HOST_SILENCE,
+            Problem.status.in_(ACTIVE_HOST_SILENCE_STATUSES),
+        )
+        .order_by(Problem.last_seen_at.desc())
+        .limit(1)
+    )
+    if active is not None:
+        return active
+
+    name = (hostname or "").strip()
+    if not name:
+        return None
+
+    sibling = db.scalar(
+        select(Problem)
+        .join(Host, Problem.host_id == Host.id)
+        .where(
+            Problem.rule_id == RULE_HOST_SILENCE,
+            Problem.status.in_(ACTIVE_HOST_SILENCE_STATUSES),
+            func.lower(Host.hostname) == name.lower(),
+        )
+        .order_by(Problem.last_seen_at.desc())
+        .limit(1)
+    )
+    if sibling is not None and sibling.host_id != host_id:
+        sibling.host_id = host_id
+        db.flush()
+    return sibling
+
+
+def _apply_host_silence_refresh(
+    problem: Problem,
+    *,
+    host_id: int,
+    match: RuleMatch,
+    fingerprint: str,
+    ref: datetime,
+) -> None:
+    problem.host_id = host_id
+    problem.summary = match.summary
+    problem.title = match.title
+    problem.fingerprint = fingerprint
+    problem.last_seen_at = ref
+    problem.updated_at = ref
+
+
 def open_or_refresh_host_silence_problem(
    db: Session,
    host_id: int,
    match: RuleMatch,
    *,
    now: datetime | None = None,
+    hostname: str | None = None,
 ) -> tuple[Problem | None, bool]:
    """
-    Открыть или обновить open Problem host_silence без ingest-события.
+    Открыть или обновить open/ack Problem host_silence без ingest-события.

    Returns (problem, created). problem is None when suppressed by manual-resolve cooldown.
    """
    ref = now or datetime.now(timezone.utc)
+    if hostname is None:
+        host = db.get(Host, host_id)
+        hostname = host.hostname if host is not None else None
+
    fingerprint = build_fingerprint(
        host_id,
        match.correlation_type,
@@ -44,24 +121,17 @@ def open_or_refresh_host_silence_problem(
        match.fingerprint_suffix,
    )

-    open_problem = db.scalar(
-        select(Problem)
-        .where(
-            Problem.status == "open",
-            Problem.rule_id == match.rule_id,
-            Problem.host_id == host_id,
+    active_problem = _find_active_host_silence_problem(db, host_id=host_id, hostname=hostname)
+    if active_problem is not None:
+        _apply_host_silence_refresh(
+            active_problem,
+            host_id=host_id,
+            match=match,
+            fingerprint=fingerprint,
+            ref=ref,
        )
-        .order_by(Problem.last_seen_at.desc())
-        .limit(1)
-    )
-    if open_problem:
-        open_problem.summary = match.summary
-        open_problem.title = match.title
-        open_problem.fingerprint = fingerprint
-        open_problem.last_seen_at = ref
-        open_problem.updated_at = ref
        db.flush()
-        return open_problem, False
+        return active_problem, False

    settings = get_settings()
    cooldown_hours = settings.sac_host_silence_manual_resolve_cooldown_hours
@@ -103,12 +173,27 @@ def open_or_refresh_host_silence_problem(
            .limit(1)
        )
        if manual_expired is not None:
+            active_problem = _find_active_host_silence_problem(db, host_id=host_id, hostname=hostname)
+            if active_problem is not None:
+                _apply_host_silence_refresh(
+                    active_problem,
+                    host_id=host_id,
+                    match=match,
+                    fingerprint=fingerprint,
+                    ref=ref,
+                )
+                db.flush()
+                return active_problem, False
+
            manual_expired.status = "open"
            manual_expired.resolved_by = None
-            manual_expired.summary = match.summary
-            manual_expired.title = match.title
-            manual_expired.last_seen_at = ref
-            manual_expired.updated_at = ref
+            _apply_host_silence_refresh(
+                manual_expired,
+                host_id=host_id,
+                match=match,
+                fingerprint=fingerprint,
+                ref=ref,
+            )
            db.flush()
            return manual_expired, True

@@ -123,13 +208,32 @@ def open_or_refresh_host_silence_problem(
        event_count=0,
        last_seen_at=ref,
    )
-    db.add(problem)
-    db.flush()
+    try:
+        with db.begin_nested():
+            db.add(problem)
+            db.flush()
+    except IntegrityError:
+        active_problem = _find_active_host_silence_problem(db, host_id=host_id, hostname=hostname)
+        if active_problem is None:
+            raise
+        _apply_host_silence_refresh(
+            active_problem,
+            host_id=host_id,
+            match=match,
+            fingerprint=fingerprint,
+            ref=ref,
+        )
+        db.flush()
+        return active_problem, False
    return problem, True


 def run_host_silence_scan(db: Session, *, now: datetime | None = None) -> list[HostSilenceScanResult]:
    """Найти stale-хосты и создать/обновить Problems rule:host_silence."""
+    if not _host_silence_scan_lock(db):
+        logger.debug("host_silence scan skipped (another runner holds advisory lock)")
+        return []
+
    ref = now or datetime.now(timezone.utc)
    hosts = db.scalars(select(Host).order_by(Host.id)).all()
    results: list[HostSilenceScanResult] = []
@@ -138,7 +242,13 @@ def run_host_silence_scan(db: Session, *, now: datetime | None = None) -> list[H
        match = host_silence_match_for_host(db, host.id, hostname=host.hostname, now=ref)
        if match is None:
            continue
-        problem, created = open_or_refresh_host_silence_problem(db, host.id, match, now=ref)
+        problem, created = open_or_refresh_host_silence_problem(
+            db,
+            host.id,
+            match,
+            now=ref,
+            hostname=host.hostname,
+        )
        if problem is None:
            continue
        results.append(