feat: live update log in UI and suppress lifecycle Telegram during SAC update
Poll remote update_script.log while SSH update runs; skip lifecycle notify when host agent_update_state is running (SAC v0.5.1).
This commit is contained in:
@@ -567,6 +567,7 @@ def update_host_agent_via_ssh(
|
|||||||
host,
|
host,
|
||||||
title=title,
|
title=title,
|
||||||
runner=run_ssh_monitor_update_action,
|
runner=run_ssh_monitor_update_action,
|
||||||
|
poll_remote_log=True,
|
||||||
)
|
)
|
||||||
except RemoteActionAlreadyRunningError as exc:
|
except RemoteActionAlreadyRunningError as exc:
|
||||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ from app.services.agent_update import execute_agent_update_fallback
|
|||||||
from app.services.agent_update_types import AgentUpdateFallbackResult
|
from app.services.agent_update_types import AgentUpdateFallbackResult
|
||||||
from app.services.linux_admin_settings import get_effective_linux_admin_config
|
from app.services.linux_admin_settings import get_effective_linux_admin_config
|
||||||
from app.services.agent_update_settings import get_effective_agent_update_config
|
from app.services.agent_update_settings import get_effective_agent_update_config
|
||||||
from app.services.ssh_connect import iter_ssh_targets, run_ssh_monitor_update, SshCommandResult
|
from app.services.ssh_connect import iter_ssh_targets, run_ssh_monitor_update, SshCommandResult, tail_ssh_monitor_update_log
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -113,12 +113,61 @@ def _apply_ssh_update_result(db: Session, host: Host, result: SshCommandResult)
|
|||||||
host.agent_update_last_error = (result.message or "SSH update failed")[:2000]
|
host.agent_update_last_error = (result.message or "SSH update failed")[:2000]
|
||||||
|
|
||||||
|
|
||||||
|
_REMOTE_LOG_POLL_SEC = 2.0
|
||||||
|
|
||||||
|
|
||||||
|
def _poll_ssh_update_log_loop(
|
||||||
|
host_id: int,
|
||||||
|
*,
|
||||||
|
stop: threading.Event,
|
||||||
|
) -> None:
|
||||||
|
"""Периодически подтягивает tail update_script.log в hosts.remote_action для UI."""
|
||||||
|
while not stop.wait(_REMOTE_LOG_POLL_SEC):
|
||||||
|
session = SessionLocal()
|
||||||
|
try:
|
||||||
|
row = session.get(Host, host_id)
|
||||||
|
if row is None or row.agent_update_state != "running":
|
||||||
|
continue
|
||||||
|
cfg = get_effective_linux_admin_config(session)
|
||||||
|
if not cfg.configured:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
targets = iter_ssh_targets(row)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
tail = ""
|
||||||
|
for target in targets:
|
||||||
|
try:
|
||||||
|
tail = tail_ssh_monitor_update_log(
|
||||||
|
target=target,
|
||||||
|
user=cfg.user,
|
||||||
|
password=cfg.password,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.debug("update log tail failed host_id=%s target=%s", host_id, target, exc_info=True)
|
||||||
|
continue
|
||||||
|
if tail:
|
||||||
|
break
|
||||||
|
payload = dict(row.remote_action or {})
|
||||||
|
if tail:
|
||||||
|
payload["output"] = tail
|
||||||
|
payload["message"] = "Выполняется обновление… (лог с хоста)"
|
||||||
|
row.remote_action = payload
|
||||||
|
session.commit()
|
||||||
|
except Exception:
|
||||||
|
logger.debug("remote log poll failed host_id=%s", host_id, exc_info=True)
|
||||||
|
session.rollback()
|
||||||
|
finally:
|
||||||
|
session.close()
|
||||||
|
|
||||||
|
|
||||||
def start_host_remote_action(
|
def start_host_remote_action(
|
||||||
db: Session,
|
db: Session,
|
||||||
host: Host,
|
host: Host,
|
||||||
*,
|
*,
|
||||||
title: str,
|
title: str,
|
||||||
runner: ActionRunner,
|
runner: ActionRunner,
|
||||||
|
poll_remote_log: bool = False,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
host_id = int(host.id)
|
host_id = int(host.id)
|
||||||
with _lock:
|
with _lock:
|
||||||
@@ -133,6 +182,17 @@ def start_host_remote_action(
|
|||||||
|
|
||||||
def _worker() -> None:
|
def _worker() -> None:
|
||||||
session = SessionLocal()
|
session = SessionLocal()
|
||||||
|
stop_poll = threading.Event()
|
||||||
|
poll_thread: threading.Thread | None = None
|
||||||
|
if poll_remote_log and runner is run_ssh_monitor_update_action:
|
||||||
|
poll_thread = threading.Thread(
|
||||||
|
target=_poll_ssh_update_log_loop,
|
||||||
|
args=(host_id,),
|
||||||
|
kwargs={"stop": stop_poll},
|
||||||
|
name=f"sac-remote-log-{host_id}",
|
||||||
|
daemon=True,
|
||||||
|
)
|
||||||
|
poll_thread.start()
|
||||||
try:
|
try:
|
||||||
row = session.get(Host, host_id)
|
row = session.get(Host, host_id)
|
||||||
if row is None:
|
if row is None:
|
||||||
@@ -167,6 +227,9 @@ def start_host_remote_action(
|
|||||||
}
|
}
|
||||||
session.commit()
|
session.commit()
|
||||||
finally:
|
finally:
|
||||||
|
stop_poll.set()
|
||||||
|
if poll_thread is not None:
|
||||||
|
poll_thread.join(timeout=5.0)
|
||||||
session.close()
|
session.close()
|
||||||
with _lock:
|
with _lock:
|
||||||
_running.discard(host_id)
|
_running.discard(host_id)
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import logging
|
|||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.models import Event, Problem
|
from app.models import Event, Host, Problem
|
||||||
from app.services import email_notify, mobile_notify, telegram_notify, webhook_notify
|
from app.services import email_notify, mobile_notify, telegram_notify, webhook_notify
|
||||||
from app.services.notification_cooldown import should_notify_event, should_notify_problem
|
from app.services.notification_cooldown import should_notify_event, should_notify_problem
|
||||||
from app.services.notification_policy import get_effective_notification_policy
|
from app.services.notification_policy import get_effective_notification_policy
|
||||||
@@ -111,8 +111,43 @@ def notify_daily_report(event: Event, *, db: Session | None = None) -> None:
|
|||||||
_dispatch_lifecycle_channels(event, db=db, policy=policy)
|
_dispatch_lifecycle_channels(event, db=db, policy=policy)
|
||||||
|
|
||||||
|
|
||||||
|
def _host_sac_update_running(event: Event, db: Session | None) -> bool:
|
||||||
|
"""Пока SAC выполняет agent-update на хосте — не слать lifecycle в Telegram."""
|
||||||
|
if db is None or not event.host_id:
|
||||||
|
return False
|
||||||
|
host = db.get(Host, event.host_id)
|
||||||
|
if host is None:
|
||||||
|
return False
|
||||||
|
if (host.agent_update_state or "").strip().lower() == "running":
|
||||||
|
logger.info(
|
||||||
|
"notify lifecycle skipped host update running host_id=%s event_id=%s",
|
||||||
|
host.id,
|
||||||
|
event.event_id,
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _lifecycle_suppress_notifications(event: Event, *, db: Session | None = None) -> bool:
|
||||||
|
"""Не слать TG при штатном SAC/cron update (lifecycle с trigger deploy_recycle)."""
|
||||||
|
if _host_sac_update_running(event, db):
|
||||||
|
return True
|
||||||
|
details = event.details if isinstance(event.details, dict) else {}
|
||||||
|
trigger = str(details.get("trigger") or "").strip().lower()
|
||||||
|
if trigger in ("deploy_recycle", "sac_update", "agent_update"):
|
||||||
|
logger.info(
|
||||||
|
"notify lifecycle skipped trigger=%s event_id=%s",
|
||||||
|
trigger,
|
||||||
|
event.event_id,
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def notify_lifecycle(event: Event, *, db: Session | None = None) -> None:
|
def notify_lifecycle(event: Event, *, db: Session | None = None) -> None:
|
||||||
"""Старт/стоп/reload агента — всегда в каналы SAC (кроме TG, если telegram_via=agent)."""
|
"""Старт/стоп/reload агента — всегда в каналы SAC (кроме TG, если telegram_via=agent)."""
|
||||||
|
if _lifecycle_suppress_notifications(event, db=db):
|
||||||
|
return
|
||||||
if _skip_notifications_for_hidden_event(event, db):
|
if _skip_notifications_for_hidden_event(event, db):
|
||||||
return
|
return
|
||||||
policy = get_effective_notification_policy(db)
|
policy = get_effective_notification_policy(db)
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ from dataclasses import dataclass
|
|||||||
from app.config import get_settings
|
from app.config import get_settings
|
||||||
from app.models import Host
|
from app.models import Host
|
||||||
|
|
||||||
|
SSH_MONITOR_UPDATE_LOG_PATH = "/var/log/update_script.log"
|
||||||
SSH_MONITOR_UPDATE_SCRIPT = "/opt/scripts/update_ssh_monitor.sh"
|
SSH_MONITOR_UPDATE_SCRIPT = "/opt/scripts/update_ssh_monitor.sh"
|
||||||
SSH_MONITOR_UPDATE_DIR = "/opt/scripts/update"
|
SSH_MONITOR_UPDATE_DIR = "/opt/scripts/update"
|
||||||
SSH_MONITOR_REPO_NAME = "ssh-monitor"
|
SSH_MONITOR_REPO_NAME = "ssh-monitor"
|
||||||
@@ -549,3 +550,29 @@ def run_ssh_monitor_update(
|
|||||||
exit_code=updated.exit_code,
|
exit_code=updated.exit_code,
|
||||||
)
|
)
|
||||||
return updated
|
return updated
|
||||||
|
|
||||||
|
|
||||||
|
def tail_ssh_monitor_update_log(
|
||||||
|
*,
|
||||||
|
target: str,
|
||||||
|
user: str,
|
||||||
|
password: str,
|
||||||
|
lines: int = 120,
|
||||||
|
) -> str:
|
||||||
|
"""Хвост /var/log/update_script.log на удалённом хосте (для live-лога в SAC UI)."""
|
||||||
|
safe_lines = max(20, min(int(lines), 400))
|
||||||
|
remote_cmd = (
|
||||||
|
f"test -r {SSH_MONITOR_UPDATE_LOG_PATH} && "
|
||||||
|
f"tail -n {safe_lines} {SSH_MONITOR_UPDATE_LOG_PATH} 2>/dev/null || true"
|
||||||
|
)
|
||||||
|
result = run_ssh_command(
|
||||||
|
target=target,
|
||||||
|
user=user,
|
||||||
|
password=password,
|
||||||
|
remote_cmd=remote_cmd,
|
||||||
|
command_timeout_sec=25,
|
||||||
|
need_root=True,
|
||||||
|
login_shell=False,
|
||||||
|
)
|
||||||
|
text = (result.stdout or "").strip()
|
||||||
|
return _truncate_output(text)
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
"""Единый источник версии SAC (API, health, логи, OpenAPI)."""
|
"""Единый источник версии SAC (API, health, логи, OpenAPI)."""
|
||||||
|
|
||||||
APP_NAME = "Security Alert Center"
|
APP_NAME = "Security Alert Center"
|
||||||
APP_VERSION = "0.5.0"
|
APP_VERSION = "0.5.1"
|
||||||
APP_VERSION_LABEL = f"{APP_NAME} v.{APP_VERSION}"
|
APP_VERSION_LABEL = f"{APP_NAME} v.{APP_VERSION}"
|
||||||
|
|||||||
@@ -32,7 +32,8 @@
|
|||||||
|
|
||||||
- [ ] При **shutdown** / `SIGTERM` не увеличивать `sac-fail.count` (или отдельный флаг «update in progress»).
|
- [ ] При **shutdown** / `SIGTERM` не увеличивать `sac-fail.count` (или отдельный флаг «update in progress»).
|
||||||
- [ ] После успешного heartbeat сбрасывать fail counter агрессивнее (уже частично есть).
|
- [ ] После успешного heartbeat сбрасывать fail counter агрессивнее (уже частично есть).
|
||||||
- [ ] Watchdog: не слать Telegram при штатном restart во время SAC-update (флаг/state file от updater) — **отдельно от** подписи сервера (2.1.9).
|
- [x] Watchdog: не слать Telegram при штатном restart во время SAC-update (state file `/var/lib/ssh-monitor/agent-update-in-progress` от updater) — **2.2.1-SAC**
|
||||||
|
- [x] Lifecycle stop/start при SAC-update: подавление на агенте + state file — **2.2.1-SAC**
|
||||||
- [ ] Документировать рекомендуемый `SAC_TIMEOUT_SEC` при тяжёлом ingest (сейчас 45s).
|
- [ ] Документировать рекомендуемый `SAC_TIMEOUT_SEC` при тяжёлом ingest (сейчас 45s).
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
@@ -19,8 +19,18 @@
|
|||||||
</p>
|
</p>
|
||||||
<p v-if="message && !loading" class="host-action-log-message" :class="messageClass">{{ message }}</p>
|
<p v-if="message && !loading" class="host-action-log-message" :class="messageClass">{{ message }}</p>
|
||||||
<p v-else-if="message && loading" class="muted host-action-log-sub host-action-log-message">{{ message }}</p>
|
<p v-else-if="message && loading" class="muted host-action-log-sub host-action-log-message">{{ message }}</p>
|
||||||
<pre v-if="output" class="host-action-log-output">{{ output }}</pre>
|
<pre v-if="output || (loading && logVisible)" class="host-action-log-output">{{
|
||||||
|
output || "Ожидание лога с хоста…\n(обновление /var/log/update_script.log)"
|
||||||
|
}}</pre>
|
||||||
<div class="host-action-log-actions">
|
<div class="host-action-log-actions">
|
||||||
|
<button
|
||||||
|
v-if="loading"
|
||||||
|
type="button"
|
||||||
|
class="secondary"
|
||||||
|
@click="emit('toggle-log')"
|
||||||
|
>
|
||||||
|
{{ logVisible ? "Скрыть лог" : "Показать лог" }}
|
||||||
|
</button>
|
||||||
<button v-if="loading" type="button" class="secondary" @click="emit('close')">
|
<button v-if="loading" type="button" class="secondary" @click="emit('close')">
|
||||||
Свернуть
|
Свернуть
|
||||||
</button>
|
</button>
|
||||||
@@ -42,10 +52,12 @@ const props = defineProps<{
|
|||||||
ok: boolean | null;
|
ok: boolean | null;
|
||||||
message: string;
|
message: string;
|
||||||
output: string;
|
output: string;
|
||||||
|
logVisible: boolean;
|
||||||
}>();
|
}>();
|
||||||
|
|
||||||
const emit = defineEmits<{
|
const emit = defineEmits<{
|
||||||
close: [];
|
close: [];
|
||||||
|
"toggle-log": [];
|
||||||
}>();
|
}>();
|
||||||
|
|
||||||
const messageClass = computed(() => {
|
const messageClass = computed(() => {
|
||||||
|
|||||||
@@ -9,7 +9,9 @@
|
|||||||
:ok="entry.ok"
|
:ok="entry.ok"
|
||||||
:message="entry.message"
|
:message="entry.message"
|
||||||
:output="entry.output"
|
:output="entry.output"
|
||||||
|
:log-visible="entry.logVisible"
|
||||||
@close="closeHostRemoteActionLog(entry.id)"
|
@close="closeHostRemoteActionLog(entry.id)"
|
||||||
|
@toggle-log="toggleHostRemoteActionLog(entry.id)"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
@@ -20,6 +22,7 @@ import HostActionLogModal from "./HostActionLogModal.vue";
|
|||||||
import {
|
import {
|
||||||
closeHostRemoteActionLog,
|
closeHostRemoteActionLog,
|
||||||
hostRemoteActionLogs,
|
hostRemoteActionLogs,
|
||||||
|
toggleHostRemoteActionLog,
|
||||||
} from "../composables/useHostRemoteAction";
|
} from "../composables/useHostRemoteAction";
|
||||||
|
|
||||||
const openLogs = computed(() => hostRemoteActionLogs.filter((e) => e.open));
|
const openLogs = computed(() => hostRemoteActionLogs.filter((e) => e.open));
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ export type HostRemoteActionLogEntry = {
|
|||||||
id: string;
|
id: string;
|
||||||
hostId: number;
|
hostId: number;
|
||||||
open: boolean;
|
open: boolean;
|
||||||
|
logVisible: boolean;
|
||||||
loading: boolean;
|
loading: boolean;
|
||||||
ok: boolean | null;
|
ok: boolean | null;
|
||||||
title: string;
|
title: string;
|
||||||
@@ -180,6 +181,7 @@ export async function pollHostRemoteAction(
|
|||||||
id: newLogId(),
|
id: newLogId(),
|
||||||
hostId,
|
hostId,
|
||||||
open: true,
|
open: true,
|
||||||
|
logVisible: false,
|
||||||
loading: true,
|
loading: true,
|
||||||
ok: null,
|
ok: null,
|
||||||
title,
|
title,
|
||||||
@@ -229,6 +231,7 @@ export async function runHostRemoteAction(
|
|||||||
id: newLogId(),
|
id: newLogId(),
|
||||||
hostId,
|
hostId,
|
||||||
open: true,
|
open: true,
|
||||||
|
logVisible: false,
|
||||||
loading: true,
|
loading: true,
|
||||||
ok: null,
|
ok: null,
|
||||||
title,
|
title,
|
||||||
@@ -246,6 +249,13 @@ export async function runHostRemoteAction(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function toggleHostRemoteActionLog(entryId: string) {
|
||||||
|
const entry = findLogById(entryId);
|
||||||
|
if (!entry) return;
|
||||||
|
entry.logVisible = !entry.logVisible;
|
||||||
|
entry.open = true;
|
||||||
|
}
|
||||||
|
|
||||||
export function closeHostRemoteActionLog(entryId: string) {
|
export function closeHostRemoteActionLog(entryId: string) {
|
||||||
clearAutoCloseTimer(entryId);
|
clearAutoCloseTimer(entryId);
|
||||||
const entry = findLogById(entryId);
|
const entry = findLogById(entryId);
|
||||||
|
|||||||
Reference in New Issue
Block a user