feat: backend ingest API, Docker Compose, Ubuntu install guide

- FastAPI: POST /api/v1/events, GET /health, JSON Schema validation

- PostgreSQL models, Alembic migration, bootstrap API key

- deploy/docker-compose.yml, .env.example

- docs/install-ubuntu-24.04.md, updated work-plan (agents first)
This commit is contained in:
2026-05-26 20:21:31 +10:00
parent bf1c1c9fe9
commit 69a232d08a
34 changed files with 1139 additions and 134 deletions
+21
View File
@@ -0,0 +1,21 @@
FROM python:3.12-slim-bookworm
WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends libpq5 \
&& rm -rf /var/lib/apt/lists/*
COPY backend/requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY backend/app ./app
COPY backend/alembic.ini .
COPY backend/alembic ./alembic
COPY schemas ./schemas
ENV PYTHONPATH=/app
ENV PYTHONUNBUFFERED=1
ENV EVENT_SCHEMA_PATH=/app/schemas/event-schema-v1.json
EXPOSE 8000
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
+33 -8
View File
@@ -1,11 +1,36 @@
# Backend (FastAPI)
# Backend — Security Alert Center
**Статус:** не реализован. См. [docs/work-plan.md](../docs/work-plan.md) фаза 1.
FastAPI + PostgreSQL + Alembic.
Планируется:
## Локальная разработка (без Docker)
- `app/main.py` — точка входа
- `app/api/v1/events.py` — ingest
- `app/models/` — SQLAlchemy
- `app/services/` — правила, уведомления
- `alembic/` — миграции
```bash
cd backend
python3.12 -m venv .venv
source .venv/bin/activate # Windows: .venv\Scripts\activate
pip install -r requirements.txt
export DATABASE_URL=postgresql+psycopg2://sac:sac@localhost:5432/sac
export SAC_BOOTSTRAP_API_KEY=sac_dev_test_key_change_me
alembic upgrade head
uvicorn app.main:app --reload --port 8000
```
## Docker (из корня репозитория)
```bash
cd deploy
cp .env.example .env
# отредактировать .env
docker compose up -d
docker compose run --rm migrate
curl http://127.0.0.1:8000/health
```
## API
| Метод | Путь | Auth |
|-------|------|------|
| GET | `/health` | нет |
| POST | `/api/v1/events` | Bearer API key |
См. [docs/agent-integration.md](../docs/agent-integration.md) и [docs/event-schema-v1.json](../docs/event-schema-v1.json).
+42
View File
@@ -0,0 +1,42 @@
[alembic]
script_location = alembic
prepend_sys_path = .
version_path_separator = os
sqlalchemy.url = driver://user:pass@localhost/dbname
[post_write_hooks]
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
+46
View File
@@ -0,0 +1,46 @@
from logging.config import fileConfig
from alembic import context
from sqlalchemy import engine_from_config, pool
from app.config import get_settings
from app.database import Base
from app.models import ApiKey, Event, Host # noqa: F401
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
settings = get_settings()
config.set_main_option("sqlalchemy.url", settings.database_url)
def run_migrations_offline() -> None:
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online() -> None:
connectable = engine_from_config(
config.get_section(config.config_ini_section, {}),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
+25
View File
@@ -0,0 +1,25 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
+90
View File
@@ -0,0 +1,90 @@
"""initial hosts events api_keys
Revision ID: 001
Revises:
Create Date: 2026-05-26
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision: str = "001"
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"hosts",
sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
sa.Column("agent_instance_id", sa.String(length=128), nullable=True),
sa.Column("hostname", sa.String(length=255), nullable=False),
sa.Column("display_name", sa.String(length=255), nullable=True),
sa.Column("os_family", sa.String(length=32), nullable=False),
sa.Column("os_version", sa.String(length=128), nullable=True),
sa.Column("product", sa.String(length=64), nullable=False),
sa.Column("product_version", sa.String(length=64), nullable=True),
sa.Column("ipv4", sa.String(length=45), nullable=True),
sa.Column("ipv6", sa.String(length=45), nullable=True),
sa.Column("tags", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column("use_sac_mode", sa.String(length=32), nullable=True),
sa.Column("last_seen_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("agent_instance_id"),
)
op.create_index("ix_hosts_hostname", "hosts", ["hostname"])
op.create_index("ix_hosts_product", "hosts", ["product"])
op.create_table(
"api_keys",
sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
sa.Column("name", sa.String(length=128), nullable=False),
sa.Column("key_prefix", sa.String(length=16), nullable=False),
sa.Column("key_hash", sa.String(length=128), nullable=False),
sa.Column("is_active", sa.Boolean(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("key_hash"),
)
op.create_index("ix_api_keys_key_prefix", "api_keys", ["key_prefix"])
op.create_table(
"events",
sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
sa.Column("event_id", sa.String(length=36), nullable=False),
sa.Column("host_id", sa.Integer(), nullable=False),
sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("received_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("category", sa.String(length=64), nullable=False),
sa.Column("type", sa.String(length=128), nullable=False),
sa.Column("severity", sa.String(length=16), nullable=False),
sa.Column("title", sa.String(length=256), nullable=False),
sa.Column("summary", sa.Text(), nullable=False),
sa.Column("details", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column("raw", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column("dedup_key", sa.String(length=512), nullable=True),
sa.Column("correlation_id", sa.String(length=36), nullable=True),
sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
sa.ForeignKeyConstraint(["host_id"], ["hosts.id"]),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("event_id", name="uq_events_event_id"),
)
op.create_index("ix_events_event_id", "events", ["event_id"])
op.create_index("ix_events_host_id", "events", ["host_id"])
op.create_index("ix_events_occurred_at", "events", ["occurred_at"])
op.create_index("ix_events_category", "events", ["category"])
op.create_index("ix_events_type", "events", ["type"])
op.create_index("ix_events_severity", "events", ["severity"])
op.create_index("ix_events_dedup_key", "events", ["dedup_key"])
def downgrade() -> None:
op.drop_table("events")
op.drop_table("api_keys")
op.drop_table("hosts")
View File
View File
View File
+43
View File
@@ -0,0 +1,43 @@
from typing import Any
from fastapi import APIRouter, Body, Depends, HTTPException
from pydantic import BaseModel
from sqlalchemy.orm import Session
from app.auth.api_key import get_api_key_auth
from app.config import get_settings
from app.database import get_db
from app.services.ingest import ingest_event
from app.services.schema_validate import validate_event_payload
router = APIRouter(prefix="/events", tags=["events"])
class IngestResponse(BaseModel):
status: str
event_id: str
created: bool
sac_event_url: str
@router.post("", status_code=202, response_model=IngestResponse)
def post_event(
payload: dict[str, Any] = Body(...),
db: Session = Depends(get_db),
_api_key: str = Depends(get_api_key_auth),
) -> IngestResponse:
errors = validate_event_payload(payload)
if errors:
raise HTTPException(status_code=422, detail={"schema_errors": errors[:20]})
event, created = ingest_event(db, payload)
db.commit()
settings = get_settings()
base = settings.sac_public_url.rstrip("/")
return IngestResponse(
status="accepted",
event_id=event.event_id,
created=created,
sac_event_url=f"{base}/api/v1/events/{event.id}",
)
+21
View File
@@ -0,0 +1,21 @@
from fastapi import APIRouter, Depends
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.database import get_db
router = APIRouter(tags=["health"])
@router.get("/health")
def health(db: Session = Depends(get_db)) -> dict:
db_status = "ok"
try:
db.execute(text("SELECT 1"))
except Exception:
db_status = "error"
return {
"status": "ok" if db_status == "ok" else "degraded",
"database": db_status,
"service": "security-alert-center",
}
+7
View File
@@ -0,0 +1,7 @@
from fastapi import APIRouter
from app.api.v1 import events, health
api_router = APIRouter()
api_router.include_router(health.router)
api_router.include_router(events.router)
+40
View File
@@ -0,0 +1,40 @@
import hashlib
import secrets
from fastapi import Depends, HTTPException, Security
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.database import get_db
from app.models import ApiKey
security_scheme = HTTPBearer(auto_error=False)
def hash_api_key(raw_key: str) -> str:
return hashlib.sha256(raw_key.encode("utf-8")).hexdigest()
def generate_api_key() -> tuple[str, str, str]:
"""Returns (full_key, prefix, hash)."""
raw = f"sac_{secrets.token_urlsafe(32)}"
prefix = raw[:12]
return raw, prefix, hash_api_key(raw)
def verify_api_key(db: Session, raw_key: str) -> bool:
key_hash = hash_api_key(raw_key)
row = db.scalar(select(ApiKey).where(ApiKey.key_hash == key_hash, ApiKey.is_active.is_(True)))
return row is not None
def get_api_key_auth(
credentials: HTTPAuthorizationCredentials | None = Security(security_scheme),
db: Session = Depends(get_db),
) -> str:
if credentials is None or credentials.scheme.lower() != "bearer":
raise HTTPException(status_code=401, detail="Missing or invalid Authorization header")
if not verify_api_key(db, credentials.credentials):
raise HTTPException(status_code=401, detail="Invalid API key")
return credentials.credentials
+33
View File
@@ -0,0 +1,33 @@
from functools import lru_cache
from pathlib import Path
from pydantic_settings import BaseSettings, SettingsConfigDict
ROOT = Path(__file__).resolve().parents[2]
SCHEMA_PATH = ROOT / "schemas" / "event-schema-v1.json"
class Settings(BaseSettings):
model_config = SettingsConfigDict(
env_file=".env",
env_file_encoding="utf-8",
extra="ignore",
)
database_url: str = "postgresql+psycopg2://sac:sac@localhost:5432/sac"
sac_public_url: str = "http://localhost:8000"
jwt_secret: str = "change-me-in-production"
jwt_algorithm: str = "HS256"
jwt_expire_minutes: int = 60 * 24
sac_bootstrap_api_key: str = ""
sac_admin_username: str = "admin"
sac_admin_password: str = ""
event_schema_path: str = str(SCHEMA_PATH) # override: EVENT_SCHEMA_PATH
cors_origins: str = "*"
@lru_cache
def get_settings() -> Settings:
return Settings()
+22
View File
@@ -0,0 +1,22 @@
from collections.abc import Generator
from sqlalchemy import create_engine
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
from app.config import get_settings
settings = get_settings()
engine = create_engine(settings.database_url, pool_pre_ping=True)
SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False)
class Base(DeclarativeBase):
pass
def get_db() -> Generator[Session, None, None]:
db = SessionLocal()
try:
yield db
finally:
db.close()
+66
View File
@@ -0,0 +1,66 @@
from contextlib import asynccontextmanager
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from sqlalchemy import select
from app.api.v1.router import api_router
from app.auth.api_key import hash_api_key
from app.config import get_settings
from app.database import SessionLocal
from app.models import ApiKey
def bootstrap_api_key() -> None:
settings = get_settings()
if not settings.sac_bootstrap_api_key:
return
db = SessionLocal()
try:
key_hash = hash_api_key(settings.sac_bootstrap_api_key)
exists = db.scalar(select(ApiKey).where(ApiKey.key_hash == key_hash))
if exists:
return
prefix = settings.sac_bootstrap_api_key[:12]
db.add(
ApiKey(
name="bootstrap",
key_prefix=prefix,
key_hash=key_hash,
is_active=True,
)
)
db.commit()
finally:
db.close()
@asynccontextmanager
async def lifespan(_app: FastAPI):
bootstrap_api_key()
yield
def create_app() -> FastAPI:
settings = get_settings()
app = FastAPI(
title="Security Alert Center",
version="0.1.0",
lifespan=lifespan,
)
origins = [o.strip() for o in settings.cors_origins.split(",") if o.strip()]
app.add_middleware(
CORSMiddleware,
allow_origins=origins if origins != ["*"] else ["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
from app.api.v1.health import router as health_router
app.include_router(api_router, prefix="/api/v1")
app.include_router(health_router)
return app
app = create_app()
+5
View File
@@ -0,0 +1,5 @@
from app.models.api_key import ApiKey
from app.models.event import Event
from app.models.host import Host
__all__ = ["ApiKey", "Event", "Host"]
+17
View File
@@ -0,0 +1,17 @@
from datetime import datetime
from sqlalchemy import Boolean, DateTime, String, func
from sqlalchemy.orm import Mapped, mapped_column
from app.database import Base
class ApiKey(Base):
__tablename__ = "api_keys"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
name: Mapped[str] = mapped_column(String(128))
key_prefix: Mapped[str] = mapped_column(String(16), index=True)
key_hash: Mapped[str] = mapped_column(String(128), unique=True)
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+30
View File
@@ -0,0 +1,30 @@
from datetime import datetime
from sqlalchemy import DateTime, ForeignKey, String, Text, UniqueConstraint, func
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.database import Base
class Event(Base):
__tablename__ = "events"
__table_args__ = (UniqueConstraint("event_id", name="uq_events_event_id"),)
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
event_id: Mapped[str] = mapped_column(String(36), index=True)
host_id: Mapped[int] = mapped_column(ForeignKey("hosts.id"), index=True)
occurred_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
received_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
category: Mapped[str] = mapped_column(String(64), index=True)
type: Mapped[str] = mapped_column(String(128), index=True)
severity: Mapped[str] = mapped_column(String(16), index=True)
title: Mapped[str] = mapped_column(String(256))
summary: Mapped[str] = mapped_column(Text)
details: Mapped[dict | None] = mapped_column(JSONB)
raw: Mapped[dict | None] = mapped_column(JSONB)
dedup_key: Mapped[str | None] = mapped_column(String(512), index=True)
correlation_id: Mapped[str | None] = mapped_column(String(36))
payload: Mapped[dict] = mapped_column(JSONB)
host: Mapped["Host"] = relationship(back_populates="events")
+28
View File
@@ -0,0 +1,28 @@
from datetime import datetime
from sqlalchemy import DateTime, String, Text, func
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.database import Base
class Host(Base):
__tablename__ = "hosts"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
agent_instance_id: Mapped[str | None] = mapped_column(String(128), unique=True, index=True)
hostname: Mapped[str] = mapped_column(String(255), index=True)
display_name: Mapped[str | None] = mapped_column(String(255))
os_family: Mapped[str] = mapped_column(String(32))
os_version: Mapped[str | None] = mapped_column(String(128))
product: Mapped[str] = mapped_column(String(64), index=True)
product_version: Mapped[str | None] = mapped_column(String(64))
ipv4: Mapped[str | None] = mapped_column(String(45))
ipv6: Mapped[str | None] = mapped_column(String(45))
tags: Mapped[list | None] = mapped_column(JSONB, default=list)
use_sac_mode: Mapped[str | None] = mapped_column(String(32))
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
events: Mapped[list["Event"]] = relationship(back_populates="host")
+89
View File
@@ -0,0 +1,89 @@
from datetime import datetime, timezone
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.models import Event, Host
def _parse_dt(value: str) -> datetime:
dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt
def upsert_host(db: Session, payload: dict) -> Host:
source = payload.get("source") or {}
host_data = payload.get("host") or {}
agent_id = source.get("agent_instance_id")
hostname = host_data.get("hostname") or "unknown"
product = source.get("product") or "unknown"
host: Host | None = None
if agent_id:
host = db.scalar(select(Host).where(Host.agent_instance_id == agent_id))
if host is None:
host = db.scalar(
select(Host).where(Host.hostname == hostname, Host.product == product).limit(1)
)
if host is None:
host = Host(
agent_instance_id=agent_id,
hostname=hostname,
display_name=host_data.get("display_name"),
os_family=host_data.get("os_family", "linux"),
os_version=host_data.get("os_version"),
product=product,
product_version=source.get("product_version"),
ipv4=host_data.get("ipv4"),
ipv6=host_data.get("ipv6"),
tags=payload.get("tags") or [],
)
db.add(host)
else:
host.hostname = hostname
host.display_name = host_data.get("display_name") or host.display_name
host.os_version = host_data.get("os_version") or host.os_version
host.product_version = source.get("product_version") or host.product_version
host.ipv4 = host_data.get("ipv4") or host.ipv4
host.ipv6 = host_data.get("ipv6") or host.ipv6
if agent_id and not host.agent_instance_id:
host.agent_instance_id = agent_id
host.last_seen_at = datetime.now(timezone.utc)
db.flush()
return host
def ingest_event(db: Session, payload: dict) -> tuple[Event, bool]:
"""
Returns (event, created).
created=False if event_id already exists (idempotent).
"""
event_id = payload["event_id"]
existing = db.scalar(select(Event).where(Event.event_id == event_id))
if existing:
return existing, False
host = upsert_host(db, payload)
event = Event(
event_id=event_id,
host_id=host.id,
occurred_at=_parse_dt(payload["occurred_at"]),
category=payload["category"],
type=payload["type"],
severity=payload["severity"],
title=payload["title"],
summary=payload["summary"],
details=payload.get("details"),
raw=payload.get("raw"),
dedup_key=payload.get("dedup_key"),
correlation_id=payload.get("correlation_id"),
payload=payload,
)
db.add(event)
db.flush()
return event, True
+24
View File
@@ -0,0 +1,24 @@
import json
from functools import lru_cache
from pathlib import Path
import jsonschema
from jsonschema import Draft202012Validator
from app.config import get_settings
@lru_cache
def get_event_validator() -> Draft202012Validator:
path = Path(get_settings().event_schema_path)
if not path.is_file():
path = Path(__file__).resolve().parents[3] / "schemas" / "event-schema-v1.json"
schema = json.loads(path.read_text(encoding="utf-8"))
Draft202012Validator.check_schema(schema)
return Draft202012Validator(schema)
def validate_event_payload(payload: dict) -> list[str]:
validator = get_event_validator()
errors = sorted(validator.iter_errors(payload), key=lambda e: e.path)
return [e.message for e in errors]
+16
View File
@@ -0,0 +1,16 @@
fastapi>=0.115.0,<1
uvicorn[standard]>=0.32.0,<1
sqlalchemy>=2.0.36,<3
alembic>=1.14.0,<2
psycopg2-binary>=2.9.10,<3
pydantic>=2.10.0,<3
pydantic-settings>=2.6.0,<3
python-multipart>=0.0.17,<1
jsonschema>=4.23.0,<5
passlib[bcrypt]>=1.7.4,<2
python-jose[cryptography]>=3.3.0,<4
httpx>=0.28.0,<1
# dev
pytest>=8.3.0,<9
pytest-asyncio>=0.24.0,<1
+7
View File
@@ -0,0 +1,7 @@
"""Smoke tests — run with PostgreSQL or skip integration."""
import pytest
def test_placeholder():
assert True