feat: SAC fallback failure counter (2.3)

After SAC_FALLBACK_FAILURES consecutive POST errors, skip SAC until
/health recovers; persist count in SAC_FAIL_COUNT_FILE.
This commit is contained in:
2026-05-27 13:36:19 +10:00
parent 3a1c9655ca
commit a4ea24c3f1
2 changed files with 65 additions and 0 deletions
+63
View File
@@ -70,6 +70,59 @@ sac_spool_remove() {
rm -f "${dir}/${event_id}.json"
}
sac_fail_count_file() {
printf '%s' "${SAC_FAIL_COUNT_FILE:-/var/lib/ssh-monitor/sac-fail.count}"
}
sac_read_fail_count() {
local f
f="$(sac_fail_count_file)"
if [ -f "$f" ]; then
cat "$f" 2>/dev/null | tr -dc '0-9'
else
printf '0'
fi
}
sac_write_fail_count() {
local n="$1"
local f
f="$(sac_fail_count_file)"
mkdir -p "$(dirname "$f")" 2>/dev/null || true
printf '%s' "$n" >"$f"
}
sac_reset_fail_count() {
sac_write_fail_count 0
}
sac_increment_fail_count() {
local n max
max="${SAC_FALLBACK_FAILURES:-5}"
n="$(sac_read_fail_count)"
n=$((n + 1))
sac_write_fail_count "$n"
[ "$n" -ge "$max" ]
}
# В fallback: после N подряд ошибок POST не дергать SAC, пока /health снова не OK.
sac_should_attempt_send() {
local mode
mode="$(sac_normalize_mode "${UseSAC:-off}")"
[ "$mode" = "fallback" ] || return 0
local max n
max="${SAC_FALLBACK_FAILURES:-5}"
n="$(sac_read_fail_count)"
[ "$n" -lt "$max" ] && return 0
if sac_check_health; then
sac_reset_fail_count
write_log "SAC: /health восстановлен, снова отправляем в SAC (fallback)"
return 0
fi
write_log "WARN: SAC fallback: пропуск POST ($n>=$max сбоев подряд), только локальные каналы"
return 1
}
# POST готового JSON (ingest / повтор из spool). Возврат 0 при HTTP 202.
sac_post_payload() {
local payload="$1"
@@ -83,6 +136,10 @@ sac_post_payload() {
return 0
fi
if ! sac_should_attempt_send; then
return 1
fi
event_id="$(printf '%s' "$payload" | python3 -c 'import json,sys; print(json.load(sys.stdin)["event_id"])')" || return 1
event_type="$(printf '%s' "$payload" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("type","?"))')" || event_type="?"
@@ -101,12 +158,14 @@ sac_post_payload() {
if [ "$http_code" = "202" ]; then
sac_spool_remove "$event_id" 2>/dev/null || true
sac_reset_fail_count
write_log "SAC: принято event_id=$event_id type=$event_type"
return 0
fi
write_log "WARN: SAC POST HTTP $http_code: $(tr '\n' ' ' <"$resp" | head -c 200)"
sac_spool_write "$event_id" "$payload"
sac_increment_fail_count || write_log "WARN: SAC fallback: достигнут порог SAC_FALLBACK_FAILURES"
return 1
}
@@ -156,6 +215,10 @@ sac_send_event() {
return 0
fi
if ! sac_should_attempt_send; then
return 1
fi
local payload http_code event_id
payload="$(SAC_EVENT_TYPE="$event_type" \
SAC_SEVERITY="$severity" \
+2
View File
@@ -32,6 +32,8 @@ UseSAC="off"
SAC_URL="https://sac.kalinamall.ru"
SAC_API_KEY=""
SAC_SPOOL_DIR="/var/lib/ssh-monitor/sac-spool"
SAC_FAIL_COUNT_FILE="/var/lib/ssh-monitor/sac-fail.count"
SAC_FALLBACK_FAILURES="5"
SAC_TIMEOUT_SEC="12"
# SMTP (python3). Для авто-режима почты нужны MAIL_SMTP_HOST, MAIL_FROM, MAIL_TO.