533 lines
15 KiB
Bash
533 lines
15 KiB
Bash
# SAC client for ssh-monitor (source from ssh-monitor)
|
||
# Версия для SAC = SSH_MONITOR_VERSION в ssh-monitor (отдельный номер здесь не задаётся).
|
||
# shellcheck shell=bash
|
||
|
||
sac_tls_insecure_enabled() {
|
||
case "${SAC_TLS_INSECURE:-0}" in
|
||
1 | yes | true | on) return 0 ;;
|
||
esac
|
||
return 1
|
||
}
|
||
|
||
sac_warn_tls_insecure() {
|
||
if sac_tls_insecure_enabled; then
|
||
write_log 'CRITICAL: SAC_TLS_INSECURE=1 — проверка TLS для SAC отключена (риск MITM). Только для краткой отладки в lab.'
|
||
fi
|
||
}
|
||
|
||
sac_curl() {
|
||
if sac_tls_insecure_enabled; then
|
||
curl -k -sS "$@"
|
||
else
|
||
curl -sS "$@"
|
||
fi
|
||
}
|
||
sac_normalize_mode() {
|
||
printf '%s' "${1:-off}" | tr '[:upper:]' '[:lower:]'
|
||
}
|
||
|
||
sac_is_configured() {
|
||
[ -n "${SAC_URL:-}" ] && [ -n "${SAC_API_KEY:-}" ]
|
||
}
|
||
|
||
sac_base_url() {
|
||
local url="${SAC_URL%/}"
|
||
case "$url" in
|
||
*/api/v1/events) url="${url%/api/v1/events}" ;;
|
||
esac
|
||
printf '%s' "${url%/}"
|
||
}
|
||
|
||
# URL ingest (POST): всегда .../api/v1/events, даже если в конфиге указан только хост
|
||
sac_ingest_url() {
|
||
local base
|
||
base="$(sac_base_url)"
|
||
[ -n "$base" ] || return 1
|
||
printf '%s/api/v1/events' "$base"
|
||
}
|
||
|
||
sac_agent_instance_id() {
|
||
local id_file="${SAC_AGENT_ID_FILE:-/var/lib/ssh-monitor/agent_instance_id}"
|
||
local dir
|
||
dir="$(dirname "$id_file")"
|
||
if [ ! -d "$dir" ]; then
|
||
ssh_monitor_secure_dir "$dir" 2>/dev/null || mkdir -p "$dir" 2>/dev/null || id_file="/tmp/ssh-monitor-agent_instance_id"
|
||
fi
|
||
if [ -f "$id_file" ]; then
|
||
cat "$id_file"
|
||
return 0
|
||
fi
|
||
local new_id
|
||
new_id="$(python3 -c 'import uuid; print(uuid.uuid4())')"
|
||
ssh_monitor_secure_file "$id_file" 2>/dev/null || true
|
||
printf '%s\n' "$new_id" >"$id_file" 2>/dev/null || true
|
||
printf '%s' "$new_id"
|
||
}
|
||
|
||
sac_check_health() {
|
||
local base code
|
||
base="$(sac_base_url)"
|
||
[ -n "$base" ] || return 1
|
||
if [ "${DRY_RUN:-0}" = "1" ]; then
|
||
return 0
|
||
fi
|
||
code="$(sac_curl -o /dev/null -w '%{http_code}' \
|
||
--connect-timeout "${SAC_TIMEOUT_SEC:-45}" \
|
||
--max-time "${SAC_TIMEOUT_SEC:-45}" \
|
||
"${base}/health" || echo 000)"
|
||
[ "$code" = "200" ]
|
||
}
|
||
|
||
sac_spool_write() {
|
||
local event_id="$1"
|
||
local body="$2"
|
||
local dir="${SAC_SPOOL_DIR:-/var/lib/ssh-monitor/sac-spool}"
|
||
ssh_monitor_secure_dir "$dir" 2>/dev/null || mkdir -p "$dir" 2>/dev/null || return 1
|
||
ssh_monitor_secure_file "${dir}/${event_id}.json" 2>/dev/null || true
|
||
printf '%s' "$body" >"${dir}/${event_id}.json"
|
||
}
|
||
|
||
sac_spool_remove() {
|
||
local event_id="$1"
|
||
local dir="${SAC_SPOOL_DIR:-/var/lib/ssh-monitor/sac-spool}"
|
||
rm -f "${dir}/${event_id}.json"
|
||
}
|
||
|
||
sac_fail_count_file() {
|
||
printf '%s' "${SAC_FAIL_COUNT_FILE:-/var/lib/ssh-monitor/sac-fail.count}"
|
||
}
|
||
|
||
sac_read_fail_count() {
|
||
local f
|
||
f="$(sac_fail_count_file)"
|
||
if [ -f "$f" ]; then
|
||
cat "$f" 2>/dev/null | tr -dc '0-9'
|
||
else
|
||
printf '0'
|
||
fi
|
||
}
|
||
|
||
sac_write_fail_count() {
|
||
local n="$1"
|
||
local f
|
||
f="$(sac_fail_count_file)"
|
||
ssh_monitor_secure_file "$f" 2>/dev/null || mkdir -p "$(dirname "$f")" 2>/dev/null || true
|
||
printf '%s' "$n" >"$f"
|
||
}
|
||
|
||
sac_reset_fail_count() {
|
||
sac_write_fail_count 0
|
||
}
|
||
|
||
# Внутренний флаг: при shutdown/SIGTERM не крутить счётчик (restart во время SAC-deploy).
|
||
sac_fail_count_increment_suppressed() {
|
||
case "${SAC_SUPPRESS_FAIL_COUNT:-0}" in
|
||
1 | yes | true | on) return 0 ;;
|
||
esac
|
||
return 1
|
||
}
|
||
|
||
sac_increment_fail_count() {
|
||
local n max
|
||
if sac_fail_count_increment_suppressed; then
|
||
write_log "SAC: пропуск sac-fail.count (shutdown/maintenance)"
|
||
return 1
|
||
fi
|
||
max="${SAC_FALLBACK_FAILURES:-5}"
|
||
n="$(sac_read_fail_count)"
|
||
n=$((n + 1))
|
||
sac_write_fail_count "$n"
|
||
[ "$n" -ge "$max" ]
|
||
}
|
||
|
||
# В fallback: после N подряд ошибок POST не дергать SAC, пока /health снова не OK.
|
||
sac_should_attempt_send() {
|
||
local mode
|
||
mode="$(sac_normalize_mode "${UseSAC:-off}")"
|
||
[ "$mode" = "fallback" ] || return 0
|
||
local max n
|
||
max="${SAC_FALLBACK_FAILURES:-5}"
|
||
n="$(sac_read_fail_count)"
|
||
[ "$n" -lt "$max" ] && return 0
|
||
if sac_check_health; then
|
||
sac_reset_fail_count
|
||
write_log "SAC: /health восстановлен, снова отправляем в SAC (fallback)"
|
||
return 0
|
||
fi
|
||
write_log "WARN: SAC fallback: пропуск POST ($n>=$max сбоев подряд), только локальные каналы"
|
||
return 1
|
||
}
|
||
|
||
# POST готового JSON (ingest / повтор из spool). Возврат 0 при HTTP 201, 409 или 202 (legacy).
|
||
sac_post_payload() {
|
||
local payload="$1"
|
||
local event_id http_code tmp resp event_type
|
||
|
||
if ! sac_is_configured; then
|
||
return 1
|
||
fi
|
||
if [ "${DRY_RUN:-0}" = "1" ]; then
|
||
printf '[DRY-RUN SAC POST] %s\n' "$(printf '%s' "$payload" | head -c 120)" >&2
|
||
return 0
|
||
fi
|
||
|
||
if ! sac_should_attempt_send; then
|
||
return 1
|
||
fi
|
||
|
||
event_id="$(printf '%s' "$payload" | python3 -c 'import json,sys; print(json.load(sys.stdin)["event_id"])')" || return 1
|
||
event_type="$(printf '%s' "$payload" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("type","?"))')" || event_type="?"
|
||
|
||
tmp="$(mktemp)"
|
||
resp="$(mktemp)"
|
||
printf '%s' "$payload" >"$tmp"
|
||
http_code="$(sac_curl -o "$resp" -w '%{http_code}' \
|
||
-X POST "$(sac_ingest_url)" \
|
||
-H "Authorization: Bearer ${SAC_API_KEY}" \
|
||
-H "Content-Type: application/json" \
|
||
-H "Idempotency-Key: ${event_id}" \
|
||
--connect-timeout "${SAC_TIMEOUT_SEC:-45}" \
|
||
--max-time "${SAC_TIMEOUT_SEC:-45}" \
|
||
-d @"$tmp" 2>/dev/null || echo 000)"
|
||
rm -f "$tmp"
|
||
|
||
if [ "$http_code" = "201" ] || [ "$http_code" = "409" ] || [ "$http_code" = "202" ]; then
|
||
sac_spool_remove "$event_id" 2>/dev/null || true
|
||
sac_reset_fail_count
|
||
write_log "SAC: принято event_id=$event_id type=$event_type"
|
||
return 0
|
||
fi
|
||
|
||
write_log "WARN: SAC POST HTTP $http_code: $(tr '\n' ' ' <"$resp" | head -c 200)"
|
||
sac_spool_write "$event_id" "$payload"
|
||
sac_increment_fail_count || write_log "WARN: SAC fallback: достигнут порог SAC_FALLBACK_FAILURES"
|
||
return 1
|
||
}
|
||
|
||
# Список spool: report.daily.* первыми (старые раньше), затем остальное.
|
||
sac_flush_spool_list_files() {
|
||
local dir="${1:-}"
|
||
[ -n "$dir" ] && [ -d "$dir" ] || return 0
|
||
SPOOL_DIR="$dir" python3 <<'PY'
|
||
import glob
|
||
import json
|
||
import os
|
||
|
||
dir_path = os.environ.get("SPOOL_DIR", "")
|
||
daily, other = [], []
|
||
for path in glob.glob(os.path.join(dir_path, "*.json")):
|
||
event_type = ""
|
||
try:
|
||
with open(path, encoding="utf-8") as handle:
|
||
event_type = json.load(handle).get("type", "") or ""
|
||
except Exception:
|
||
pass
|
||
mtime = os.path.getmtime(path)
|
||
bucket = daily if event_type in ("report.daily.ssh", "report.daily.rdp") else other
|
||
bucket.append((mtime, path))
|
||
for bucket in (daily, other):
|
||
for _, path in sorted(bucket):
|
||
print(path)
|
||
PY
|
||
}
|
||
|
||
# Повторная отправка файлов из SAC_SPOOL_DIR (до max_files за вызов).
|
||
sac_flush_spool() {
|
||
local mode max_files="${1:-${SAC_SPOOL_FLUSH_MAX_FILES:-50}}"
|
||
mode="$(sac_normalize_mode "${UseSAC:-off}")"
|
||
case "$mode" in
|
||
off) return 0 ;;
|
||
esac
|
||
if ! sac_is_configured; then
|
||
return 0
|
||
fi
|
||
|
||
local dir="${SAC_SPOOL_DIR:-/var/lib/ssh-monitor/sac-spool}"
|
||
[ -d "$dir" ] || return 0
|
||
|
||
local f count=0 payload
|
||
while IFS= read -r f; do
|
||
[ -n "$f" ] && [ -f "$f" ] || continue
|
||
count=$((count + 1))
|
||
[ "$count" -gt "$max_files" ] && break
|
||
payload="$(cat "$f")"
|
||
sac_post_payload "$payload" || true
|
||
done < <(sac_flush_spool_list_files "$dir")
|
||
return 0
|
||
}
|
||
|
||
# send_sac_event type severity title summary [details_json]
|
||
sac_send_event() {
|
||
local event_type="$1"
|
||
local severity="$2"
|
||
local title="$3"
|
||
local summary="$4"
|
||
local details_json="${5:-}"
|
||
|
||
if ! sac_is_configured; then
|
||
write_log "WARN: SAC не настроен (SAC_URL / SAC_API_KEY)"
|
||
return 1
|
||
fi
|
||
|
||
if [ "${DRY_RUN:-0}" = "1" ]; then
|
||
printf '[DRY-RUN SAC] %s | %s | %s\n' "$event_type" "$severity" "$title" >&2
|
||
return 0
|
||
fi
|
||
|
||
if ! sac_should_attempt_send; then
|
||
return 1
|
||
fi
|
||
|
||
local payload http_code event_id
|
||
payload="$(SAC_EVENT_TYPE="$event_type" \
|
||
SAC_SEVERITY="$severity" \
|
||
SAC_TITLE="$title" \
|
||
SAC_SUMMARY="$summary" \
|
||
SAC_DETAILS_JSON="$details_json" \
|
||
SAC_AGENT_ID="$(sac_agent_instance_id)" \
|
||
SAC_PRODUCT_VERSION="${SSH_MONITOR_VERSION:-unknown}" \
|
||
SAC_SERVER_DISPLAY_NAME="$(server_display_name_with_ip)" \
|
||
SAC_HOST_IPV4="${SERVER_IPV4:-}" \
|
||
python3 <<'PY'
|
||
import json, os, socket, uuid
|
||
from datetime import datetime, timezone
|
||
|
||
def details():
|
||
raw = os.environ.get("SAC_DETAILS_JSON", "").strip()
|
||
if not raw:
|
||
return None
|
||
return json.loads(raw)
|
||
|
||
def valid_ipv4(value: str) -> bool:
|
||
try:
|
||
parts = value.split(".")
|
||
return len(parts) == 4 and all(0 <= int(p) <= 255 for p in parts)
|
||
except Exception:
|
||
return False
|
||
|
||
def detect_host_ipv4() -> str:
|
||
manual = os.environ.get("SAC_HOST_IPV4", "").strip()
|
||
if manual and valid_ipv4(manual):
|
||
return manual
|
||
try:
|
||
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||
s.connect(("1.1.1.1", 53))
|
||
ip = s.getsockname()[0]
|
||
s.close()
|
||
if valid_ipv4(ip) and not ip.startswith(("127.", "169.254.")):
|
||
return ip
|
||
except Exception:
|
||
pass
|
||
try:
|
||
infos = socket.getaddrinfo(socket.gethostname(), None, family=socket.AF_INET)
|
||
for info in infos:
|
||
ip = info[4][0]
|
||
if valid_ipv4(ip) and not ip.startswith(("127.", "169.254.")):
|
||
return ip
|
||
except Exception:
|
||
pass
|
||
return ""
|
||
|
||
host = socket.gethostname()
|
||
display = os.environ.get("SAC_SERVER_DISPLAY_NAME", "").strip()
|
||
host_ipv4 = detect_host_ipv4()
|
||
etype = os.environ["SAC_EVENT_TYPE"]
|
||
if etype.startswith(("ssh.", "auth.", "rdp.")):
|
||
category = "auth"
|
||
elif etype.startswith("privilege."):
|
||
category = "privilege"
|
||
elif etype.startswith("session."):
|
||
category = "session"
|
||
elif etype.startswith("report."):
|
||
category = "report"
|
||
elif etype.startswith("rdg."):
|
||
category = "network"
|
||
else:
|
||
category = "agent"
|
||
payload = {
|
||
"schema_version": "1.0",
|
||
"event_id": str(uuid.uuid4()),
|
||
"occurred_at": datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds"),
|
||
"source": {
|
||
"product": "ssh-monitor",
|
||
"product_version": os.environ.get("SAC_PRODUCT_VERSION", "unknown"),
|
||
"agent_instance_id": os.environ["SAC_AGENT_ID"],
|
||
},
|
||
"host": {
|
||
"hostname": host,
|
||
"os_family": "linux",
|
||
**({"display_name": display} if display else {}),
|
||
**({"ipv4": host_ipv4} if host_ipv4 else {}),
|
||
},
|
||
"category": category,
|
||
"type": os.environ["SAC_EVENT_TYPE"],
|
||
"severity": os.environ["SAC_SEVERITY"],
|
||
"title": os.environ["SAC_TITLE"],
|
||
"summary": os.environ["SAC_SUMMARY"],
|
||
}
|
||
d = details()
|
||
if d is not None:
|
||
payload["details"] = d
|
||
print(json.dumps(payload, ensure_ascii=False))
|
||
PY
|
||
)" || return 1
|
||
|
||
sac_post_payload "$payload"
|
||
}
|
||
|
||
# JSON details из переменных SAC_D_* (опционально)
|
||
sac_build_details_json() {
|
||
python3 <<'PY'
|
||
import json, os
|
||
out = {}
|
||
for k, v in os.environ.items():
|
||
if k.startswith("SAC_D_") and v:
|
||
out[k[6:].lower()] = v
|
||
print(json.dumps(out) if out else "")
|
||
PY
|
||
}
|
||
|
||
sac_is_heartbeat_only_event() {
|
||
[ "${1:-}" = "agent.heartbeat" ]
|
||
}
|
||
|
||
sac_is_daily_report_event() {
|
||
case "${1:-}" in
|
||
report.daily.ssh|report.daily.rdp) return 0 ;;
|
||
*) return 1 ;;
|
||
esac
|
||
}
|
||
|
||
# Добавляет generated_by / telegram_via в JSON details для ingest.
|
||
sac_merge_notify_details() {
|
||
local details_json="${1:-}"
|
||
local telegram_via="${2:-}"
|
||
DETAILS_IN="$details_json" TG_VIA="$telegram_via" python3 <<'PY'
|
||
import json, os
|
||
d = {}
|
||
raw = os.environ.get("DETAILS_IN", "").strip()
|
||
if raw:
|
||
try:
|
||
d = json.loads(raw)
|
||
except Exception:
|
||
d = {}
|
||
d.setdefault("generated_by", "agent")
|
||
via = os.environ.get("TG_VIA", "").strip()
|
||
if via:
|
||
d.setdefault("telegram_via", via)
|
||
print(json.dumps(d, ensure_ascii=False))
|
||
PY
|
||
}
|
||
|
||
# notify_or_sac type severity title summary telegram_message [details_json]
|
||
notify_or_sac() {
|
||
local event_type="$1"
|
||
local severity="$2"
|
||
local title="$3"
|
||
local summary="$4"
|
||
local telegram_message="${5:-$summary}"
|
||
local details_json="${6:-}"
|
||
if [ -z "$details_json" ] && [ -n "${SAC_D_USER:-}${SAC_D_IP:-}${SAC_D_COMMAND:-}" ]; then
|
||
details_json="$(sac_build_details_json)"
|
||
fi
|
||
unset SAC_D_USER SAC_D_IP SAC_D_SOURCE_IP SAC_D_PORT SAC_D_COMMAND SAC_D_ATTEMPT SAC_D_MAX SAC_D_RUN_AS SAC_D_PWD SAC_D_SESSION_ID SAC_D_TTY SAC_D_PTS SAC_D_SINCE 2>/dev/null || true
|
||
local mode
|
||
mode="$(sac_normalize_mode "${UseSAC:-off}")"
|
||
local tg_via=""
|
||
case "$mode" in
|
||
exclusive|fallback) tg_via="sac" ;;
|
||
dual) tg_via="agent" ;;
|
||
esac
|
||
if [ -n "$tg_via" ]; then
|
||
details_json="$(sac_merge_notify_details "$details_json" "$tg_via")"
|
||
fi
|
||
|
||
# Периодический heartbeat — только SAC (браузер), без Telegram/email в любом режиме.
|
||
if sac_is_heartbeat_only_event "$event_type"; then
|
||
case "$mode" in
|
||
off)
|
||
return 0
|
||
;;
|
||
exclusive|dual|fallback)
|
||
sac_send_event "$event_type" "$severity" "$title" "$summary" "$details_json"
|
||
return $?
|
||
;;
|
||
*)
|
||
return 0
|
||
;;
|
||
esac
|
||
fi
|
||
|
||
# Суточный отчёт — только SAC/spool; при сбое ingest не дублировать в локальный Telegram.
|
||
if sac_is_daily_report_event "$event_type"; then
|
||
case "$mode" in
|
||
off)
|
||
return 0
|
||
;;
|
||
exclusive|dual|fallback)
|
||
if sac_send_event "$event_type" "$severity" "$title" "$summary" "$details_json"; then
|
||
return 0
|
||
fi
|
||
write_log "WARN: daily report не принят SAC — остаётся в spool (локальный Telegram пропущен)"
|
||
return 1
|
||
;;
|
||
*)
|
||
return 0
|
||
;;
|
||
esac
|
||
fi
|
||
|
||
case "$mode" in
|
||
off)
|
||
notify_send "$telegram_message"
|
||
;;
|
||
exclusive)
|
||
sac_send_event "$event_type" "$severity" "$title" "$summary" "$details_json"
|
||
;;
|
||
dual)
|
||
sac_send_event "$event_type" "$severity" "$title" "$summary" "$details_json" || true
|
||
NOTIFY_SKIP_SAC_MIRROR=1 notify_send "$telegram_message"
|
||
;;
|
||
fallback)
|
||
if sac_send_event "$event_type" "$severity" "$title" "$summary" "$details_json"; then
|
||
return 0
|
||
fi
|
||
NOTIFY_SKIP_SAC_MIRROR=1 notify_send "$telegram_message"
|
||
;;
|
||
*)
|
||
write_log "WARN: неизвестный UseSAC=$mode, только Telegram"
|
||
notify_send "$telegram_message"
|
||
;;
|
||
esac
|
||
}
|
||
|
||
run_check_sac() {
|
||
printf 'Проверка SAC (ssh-monitor)\n'
|
||
printf 'UseSAC=%s\n' "${UseSAC:-off}"
|
||
case "$(sac_normalize_mode "${UseSAC:-off}")" in
|
||
exclusive) printf 'Режим exclusive: уведомления только в SAC (Telegram/email с агента не отправляются)\n' ;;
|
||
dual) printf 'Режим dual: SAC + локальные каналы NOTIFY_CHAIN\n' ;;
|
||
fallback) printf 'Режим fallback: SAC, при сбое — NOTIFY_CHAIN\n' ;;
|
||
esac
|
||
printf 'SAC_URL=%s\n' "${SAC_URL:-}"
|
||
if sac_is_configured; then
|
||
printf 'SAC ingest URL=%s\n' "$(sac_ingest_url)"
|
||
fi
|
||
if ! sac_is_configured; then
|
||
printf 'SAC: SAC_URL или SAC_API_KEY не заданы\n' >&2
|
||
return 1
|
||
fi
|
||
if sac_check_health; then
|
||
printf 'SAC health: OK\n'
|
||
else
|
||
printf 'SAC health: FAIL\n' >&2
|
||
return 1
|
||
fi
|
||
if sac_send_event "agent.test" "info" "SAC test" "ssh-monitor --check-sac"; then
|
||
printf 'SAC ingest agent.test: OK (ожидается HTTP 201)\n'
|
||
return 0
|
||
fi
|
||
printf 'SAC ingest agent.test: FAIL\n' >&2
|
||
return 1
|
||
}
|