feat: backend ingest API, Docker Compose, Ubuntu install guide

- FastAPI: POST /api/v1/events, GET /health, JSON Schema validation - PostgreSQL models, Alembic migration, bootstrap API key - deploy/docker-compose.yml, .env.example - docs/install-ubuntu-24.04.md, updated work-plan (agents first)
2026-05-26 20:21:31 +10:00
parent bf1c1c9fe9
commit 69a232d08a
34 changed files with 1139 additions and 134 deletions
@@ -0,0 +1,46 @@
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+
+from app.config import get_settings
+from app.database import Base
+from app.models import ApiKey, Event, Host  # noqa: F401
+
+config = context.config
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+target_metadata = Base.metadata
+settings = get_settings()
+config.set_main_option("sqlalchemy.url", settings.database_url)
+
+
+def run_migrations_offline() -> None:
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+    with connectable.connect() as connection:
+        context.configure(connection=connection, target_metadata=target_metadata)
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()
@@ -0,0 +1,25 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}
@@ -0,0 +1,90 @@
+"""initial hosts events api_keys
+
+Revision ID: 001
+Revises:
+Create Date: 2026-05-26
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+revision: str = "001"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "hosts",
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("agent_instance_id", sa.String(length=128), nullable=True),
+        sa.Column("hostname", sa.String(length=255), nullable=False),
+        sa.Column("display_name", sa.String(length=255), nullable=True),
+        sa.Column("os_family", sa.String(length=32), nullable=False),
+        sa.Column("os_version", sa.String(length=128), nullable=True),
+        sa.Column("product", sa.String(length=64), nullable=False),
+        sa.Column("product_version", sa.String(length=64), nullable=True),
+        sa.Column("ipv4", sa.String(length=45), nullable=True),
+        sa.Column("ipv6", sa.String(length=45), nullable=True),
+        sa.Column("tags", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+        sa.Column("use_sac_mode", sa.String(length=32), nullable=True),
+        sa.Column("last_seen_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("agent_instance_id"),
+    )
+    op.create_index("ix_hosts_hostname", "hosts", ["hostname"])
+    op.create_index("ix_hosts_product", "hosts", ["product"])
+
+    op.create_table(
+        "api_keys",
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("name", sa.String(length=128), nullable=False),
+        sa.Column("key_prefix", sa.String(length=16), nullable=False),
+        sa.Column("key_hash", sa.String(length=128), nullable=False),
+        sa.Column("is_active", sa.Boolean(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("key_hash"),
+    )
+    op.create_index("ix_api_keys_key_prefix", "api_keys", ["key_prefix"])
+
+    op.create_table(
+        "events",
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("event_id", sa.String(length=36), nullable=False),
+        sa.Column("host_id", sa.Integer(), nullable=False),
+        sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("received_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("category", sa.String(length=64), nullable=False),
+        sa.Column("type", sa.String(length=128), nullable=False),
+        sa.Column("severity", sa.String(length=16), nullable=False),
+        sa.Column("title", sa.String(length=256), nullable=False),
+        sa.Column("summary", sa.Text(), nullable=False),
+        sa.Column("details", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+        sa.Column("raw", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+        sa.Column("dedup_key", sa.String(length=512), nullable=True),
+        sa.Column("correlation_id", sa.String(length=36), nullable=True),
+        sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+        sa.ForeignKeyConstraint(["host_id"], ["hosts.id"]),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("event_id", name="uq_events_event_id"),
+    )
+    op.create_index("ix_events_event_id", "events", ["event_id"])
+    op.create_index("ix_events_host_id", "events", ["host_id"])
+    op.create_index("ix_events_occurred_at", "events", ["occurred_at"])
+    op.create_index("ix_events_category", "events", ["category"])
+    op.create_index("ix_events_type", "events", ["type"])
+    op.create_index("ix_events_severity", "events", ["severity"])
+    op.create_index("ix_events_dedup_key", "events", ["dedup_key"])
+
+
+def downgrade() -> None:
+    op.drop_table("events")
+    op.drop_table("api_keys")
+    op.drop_table("hosts")