From 1b3084ddfa204dfde7f28d81e20ea51eb6eadca0 Mon Sep 17 00:00:00 2001 From: curo1305 Date: Sat, 20 Jun 2026 10:45:22 +0200 Subject: [PATCH 1/4] feat(12-05): add migration-gated Compose startup path - Add one-shot migrate service using DATABASE_MIGRATE_URL and alembic upgrade head - Add migrate dependency (service_completed_successfully) to backend, celery-worker, celery-beat - migrate service has same read_only/cap_drop/no-new-privileges hardening as other services - Add test_compose_migrations.py: 7 static assertions on Compose configuration --- backend/tests/test_compose_migrations.py | 166 +++++++++++++++++++++++ docker-compose.yml | 29 ++++ 2 files changed, 195 insertions(+) create mode 100644 backend/tests/test_compose_migrations.py diff --git a/backend/tests/test_compose_migrations.py b/backend/tests/test_compose_migrations.py new file mode 100644 index 0000000..1b7cabb --- /dev/null +++ b/backend/tests/test_compose_migrations.py @@ -0,0 +1,166 @@ +""" +Static regression tests for the Compose migration gate. + +These tests parse docker-compose.yml (via PyYAML) and assert the structural +invariants that ensure Alembic runs before any application process starts. +No Docker daemon is required — they are pure-Python configuration checks. + +Requirements covered: + CONN-04 — Cloud connections require an up-to-date schema before use + CLOUD-01 — cloud_items/cloud_folder_states tables must exist at runtime +""" +from __future__ import annotations + +import os +import pathlib + +import pytest +import yaml + + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] +COMPOSE_PATH = REPO_ROOT / "docker-compose.yml" + +APPLICATION_SERVICES = {"backend", "celery-worker", "celery-beat"} + + +@pytest.fixture(scope="module") +def compose() -> dict: + """Load docker-compose.yml once for all tests in this module.""" + with COMPOSE_PATH.open() as fh: + return yaml.safe_load(fh) + + +# --------------------------------------------------------------------------- +# 1. Exactly one service runs "alembic upgrade head" +# --------------------------------------------------------------------------- + +def test_exactly_one_alembic_upgrade_service(compose): + """Exactly one service must run 'alembic upgrade head' — the migrate service.""" + alembic_services = [ + name + for name, svc in compose["services"].items() + if "alembic upgrade head" in " ".join( + svc.get("command", []) + if isinstance(svc.get("command"), list) + else [str(svc.get("command", ""))] + ) + ] + assert len(alembic_services) == 1, ( + f"Expected exactly 1 service running 'alembic upgrade head', " + f"found: {alembic_services}" + ) + assert alembic_services[0] == "migrate", ( + f"Migration service must be named 'migrate', got: {alembic_services[0]}" + ) + + +# --------------------------------------------------------------------------- +# 2. The migrate service uses DATABASE_MIGRATE_URL, not DATABASE_URL +# --------------------------------------------------------------------------- + +def test_migrate_service_uses_migrate_url(compose): + """The migrate service must reference DATABASE_MIGRATE_URL.""" + svc = compose["services"]["migrate"] + env = svc.get("environment", []) + # environment may be a list of "KEY=VALUE" or "KEY" strings, or a dict + if isinstance(env, dict): + keys = set(env.keys()) + else: + keys = {e.split("=")[0] for e in env} + + assert "DATABASE_MIGRATE_URL" in keys, ( + "migrate service must declare DATABASE_MIGRATE_URL in its environment" + ) + assert "DATABASE_URL" not in keys, ( + "migrate service must NOT have DATABASE_URL — it uses the DDL-capable migrate role only" + ) + + +# --------------------------------------------------------------------------- +# 3. Application services must NOT run alembic upgrade head in their command +# --------------------------------------------------------------------------- + +def test_application_services_do_not_run_alembic(compose): + """backend/celery-worker/celery-beat must not run 'alembic upgrade' themselves.""" + for name in APPLICATION_SERVICES: + svc = compose["services"][name] + cmd = svc.get("command", "") + if isinstance(cmd, list): + cmd = " ".join(cmd) + assert "alembic upgrade" not in cmd, ( + f"Service '{name}' must not run 'alembic upgrade' — only the migrate service should" + ) + + +# --------------------------------------------------------------------------- +# 4. All application services depend on migrate with service_completed_successfully +# --------------------------------------------------------------------------- + +def test_application_services_depend_on_migrate(compose): + """backend, celery-worker, and celery-beat must all wait for migrate to complete.""" + for name in APPLICATION_SERVICES: + svc = compose["services"][name] + depends = svc.get("depends_on", {}) + assert "migrate" in depends, ( + f"Service '{name}' is missing 'migrate' in depends_on" + ) + condition = depends["migrate"].get("condition") + assert condition == "service_completed_successfully", ( + f"Service '{name}' depends_on.migrate.condition must be " + f"'service_completed_successfully', got: {condition!r}" + ) + + +# --------------------------------------------------------------------------- +# 5. The migrate service preserves container hardening +# --------------------------------------------------------------------------- + +def test_migrate_service_has_container_hardening(compose): + """migrate service must have read_only, cap_drop, and no-new-privileges.""" + svc = compose["services"]["migrate"] + + assert svc.get("read_only") is True, ( + "migrate service must set read_only: true" + ) + + cap_drop = svc.get("cap_drop", []) + assert "ALL" in cap_drop, ( + "migrate service must cap_drop ALL" + ) + + sec_opts = svc.get("security_opt", []) + assert "no-new-privileges:true" in sec_opts, ( + "migrate service must set no-new-privileges:true" + ) + + +# --------------------------------------------------------------------------- +# 6. The migrate service has restart: "no" (it is one-shot, not a daemon) +# --------------------------------------------------------------------------- + +def test_migrate_service_is_one_shot(compose): + """migrate service must have restart: 'no' — it exits after running migrations.""" + svc = compose["services"]["migrate"] + restart = svc.get("restart", "") + assert restart == "no", ( + f"migrate service restart policy must be 'no' (one-shot), got: {restart!r}" + ) + + +# --------------------------------------------------------------------------- +# 7. The migrate service only depends on postgres (not minio/redis) +# --------------------------------------------------------------------------- + +def test_migrate_service_minimal_dependencies(compose): + """migrate only needs postgres to be healthy — minio/redis are runtime concerns.""" + svc = compose["services"]["migrate"] + depends = svc.get("depends_on", {}) + assert "postgres" in depends, "migrate must depend on postgres" + assert depends["postgres"]["condition"] == "service_healthy", ( + "migrate must wait for postgres to be healthy" + ) + for extra in ("minio", "redis", "minio-init"): + assert extra not in depends, ( + f"migrate should NOT depend on '{extra}' — only postgres is needed for schema migrations" + ) diff --git a/docker-compose.yml b/docker-compose.yml index f8dcc43..5316ca8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -95,6 +95,29 @@ services: timeout: 3s retries: 5 + migrate: + build: ./backend + # One-shot service: runs "alembic upgrade head" then exits 0. + # All application services depend on this with condition: service_completed_successfully. + # Uses DATABASE_MIGRATE_URL (docuvault_migrate role, DDL-capable) — not DATABASE_URL. + command: alembic upgrade head + environment: + - DATABASE_MIGRATE_URL=${DATABASE_MIGRATE_URL} + - PYTHONDONTWRITEBYTECODE=1 + volumes: + - ./backend:/app + depends_on: + postgres: + condition: service_healthy + read_only: true + tmpfs: + - "/tmp:mode=1777" + cap_drop: + - ALL + security_opt: + - "no-new-privileges:true" + restart: "no" + backend: build: ./backend ports: @@ -135,6 +158,8 @@ services: condition: service_completed_successfully redis: condition: service_healthy + migrate: + condition: service_completed_successfully read_only: true tmpfs: - "/tmp:mode=1777" @@ -174,6 +199,8 @@ services: condition: service_completed_successfully redis: condition: service_healthy + migrate: + condition: service_completed_successfully read_only: true tmpfs: - "/tmp:mode=1777" @@ -205,6 +232,8 @@ services: condition: service_completed_successfully redis: condition: service_healthy + migrate: + condition: service_completed_successfully read_only: true tmpfs: - "/tmp:mode=1777" From de2efd166462a559729e8b0424b5e5a0f76c553e Mon Sep 17 00:00:00 2001 From: curo1305 Date: Sat, 20 Jun 2026 10:47:11 +0200 Subject: [PATCH 2/4] =?UTF-8?q?test(12-05):=20add=20migration=200005?= =?UTF-8?q?=E2=86=920006=20integration=20regression=20and=20RUNBOOK=20migr?= =?UTF-8?q?ation=20ops?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - test_migration_0006.py: proves display_name_override, cloud_items, cloud_item_topics, cloud_folder_states, unique constraints, indexes, and DDL privilege boundary - Tests skip cleanly without INTEGRATION=1/INTEGRATION_DATABASE_URL (no dev DB mutation) - RUNBOOK: add Database Migrations section with revision check, upgrade, recovery, and emergency schema-drift remediation commands --- RUNBOOK.md | 105 +++++++++- backend/tests/test_migration_0006.py | 288 +++++++++++++++++++++++++++ 2 files changed, 385 insertions(+), 8 deletions(-) create mode 100644 backend/tests/test_migration_0006.py diff --git a/RUNBOOK.md b/RUNBOOK.md index 3a4cbf0..04293e2 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -13,12 +13,13 @@ for phase history and roadmap see `.planning/ROADMAP.md`. 1. [Environment Variables](#1-environment-variables) 2. [Startup / Shutdown](#2-startup--shutdown) -3. [Backup Strategy](#3-backup-strategy) -4. [Health Checks](#4-health-checks) -5. [Security Gate — docker scout CVE Scan (D-10)](#5-security-gate--docker-scout-cve-scan-d-10) -6. [On-Call Escalation](#6-on-call-escalation) -7. [Common Failure Modes](#7-common-failure-modes) -8. [Phase 6 Deferred Items](#8-phase-6-deferred-items) +3. [Database Migrations](#3-database-migrations) +4. [Backup Strategy](#4-backup-strategy) +5. [Health Checks](#5-health-checks) +6. [Security Gate — docker scout CVE Scan (D-10)](#6-security-gate--docker-scout-cve-scan-d-10) +7. [On-Call Escalation](#7-on-call-escalation) +8. [Common Failure Modes](#8-common-failure-modes) +9. [Phase 6 Deferred Items](#9-phase-6-deferred-items) --- @@ -146,11 +147,18 @@ docker compose logs -f --tail 50 Services start in dependency order: ``` -postgres / minio / redis → backend / celery-worker / celery-beat +postgres / minio / redis → migrate (alembic upgrade head) + → backend / celery-worker / celery-beat → loki → promtail / grafana → frontend ``` +The `migrate` service is a one-shot container that runs `alembic upgrade head` using +`DATABASE_MIGRATE_URL` (DDL-capable `docuvault_migrate` role) and exits 0 on success. +All application services (`backend`, `celery-worker`, `celery-beat`) depend on it with +`condition: service_completed_successfully`. If `migrate` exits non-zero, no application +process starts and the deployment is blocked until the migration issue is resolved. + Access points after a healthy startup: | Service | URL | @@ -194,7 +202,88 @@ Do not add `read_only:` to the celery-beat block — it will exit immediately wi --- -## 3. Backup Strategy +## 3. Database Migrations + +### Checking the current revision + +```bash +# Inside the running backend container +docker compose exec backend alembic current + +# Via a one-off migrate container (preferred — uses DDL role) +docker compose run --rm migrate alembic current +``` + +A healthy deployment at Phase 12 head reports: + +``` +0006 (head) +``` + +### Upgrading to the latest migration + +```bash +# Compose will run migrate automatically on docker compose up. +# To run it manually (e.g., after a git pull): +docker compose run --rm migrate +``` + +`migrate` runs `alembic upgrade head` using `DATABASE_MIGRATE_URL`. On success it exits 0. +Application services will not start until this exits 0 — this is the deployment gate. + +### Viewing migration history + +```bash +docker compose run --rm migrate alembic history --verbose +``` + +### Recovering from a failed migration + +1. **Read the logs** — the migrate container logs contain the SQL and the error: + ```bash + docker compose logs migrate + ``` + +2. **Fix the migration** — edit the failing migration revision file in `backend/migrations/versions/`. + +3. **Re-run** — the migrate container is idempotent; Alembic skips already-applied revisions: + ```bash + docker compose run --rm migrate + ``` + +4. **Check the revision** — confirm head is applied before restarting backend: + ```bash + docker compose run --rm migrate alembic current + ``` + +5. **Restart application services** after a successful migration: + ```bash + docker compose up -d backend celery-worker celery-beat + ``` + +### Emergency: schema drift on live database (Phase 12 gap-closure scenario) + +If the backend is already running against a database that is behind head (e.g., it started +before the migrate service was introduced and reports `UndefinedColumn: display_name_override`): + +```bash +# 1. Stop application processes (not postgres) +docker compose stop backend celery-worker celery-beat + +# 2. Apply missing migrations via the dedicated migrate service +docker compose run --rm migrate + +# 3. Verify the revision +docker compose run --rm migrate alembic current +# Expected: 0006 (head) + +# 4. Restart application processes through the gated path +docker compose up -d backend celery-worker celery-beat +``` + +--- + +## 4. Backup Strategy ### PostgreSQL diff --git a/backend/tests/test_migration_0006.py b/backend/tests/test_migration_0006.py new file mode 100644 index 0000000..dc654a5 --- /dev/null +++ b/backend/tests/test_migration_0006.py @@ -0,0 +1,288 @@ +""" +Integration test: PostgreSQL upgrade from revision 0005 to Phase 12 head (0006). + +Prerequisite: a disposable PostgreSQL database accessible via + INTEGRATION_DATABASE_URL (psycopg sync DSN, e.g. postgresql://user:pw@host/db) + or + INTEGRATION=1 with DATABASE_MIGRATE_URL pointing at the test database. + +If neither is set the tests are skipped with an explicit message. + +The test uses an isolated Alembic migration run against this disposable DB — it +NEVER touches the normal development database and NEVER runs downgrade on it. + +Requirements covered: + CONN-04 — Connections endpoint requires display_name_override column + CLOUD-01 — cloud_items/cloud_folder_states tables must exist +""" +from __future__ import annotations + +import os +import subprocess +import sys + +import pytest + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +REQUIRED_ENV = ( + "INTEGRATION_DATABASE_URL", + "DATABASE_MIGRATE_URL", +) + +PHASE_12_TABLES = { + "cloud_items", + "cloud_item_topics", + "cloud_folder_states", +} + +PHASE_12_UNIQUE_CONSTRAINTS = { + "uq_cloud_items_connection_provider_item", + "uq_cloud_folder_states_connection_parent", +} + +PHASE_12_INDEXES = { + "ix_cloud_items_user_id", + "ix_cloud_items_connection_id", + "ix_cloud_items_connection_parent", + "ix_cloud_folder_states_connection", +} + + +def _get_migrate_url() -> str | None: + """Return a disposable database URL for migration tests, or None to skip.""" + # Highest priority: dedicated integration URL + url = os.environ.get("INTEGRATION_DATABASE_URL") + if url: + return url + # Second: INTEGRATION=1 activates the normal DATABASE_MIGRATE_URL + if os.environ.get("INTEGRATION") == "1": + return os.environ.get("DATABASE_MIGRATE_URL") + return None + + +def _psycopg_conn(dsn: str): + """Return a psycopg3 (sync) connection or raise ImportError / OperationalError.""" + try: + import psycopg # psycopg v3 sync + except ImportError as exc: + pytest.skip(f"psycopg (v3) not installed — cannot run migration integration test: {exc}") + return psycopg.connect(dsn, autocommit=True) + + +def _run_alembic(target: str, *, dsn: str) -> None: + """Run alembic upgrade/downgrade to target using the given DSN.""" + env = {**os.environ, "DATABASE_MIGRATE_URL": dsn} + result = subprocess.run( + [sys.executable, "-m", "alembic", "upgrade", target], + capture_output=True, + text=True, + env=env, + ) + if result.returncode != 0: + raise RuntimeError( + f"alembic upgrade {target} failed (rc={result.returncode}):\n" + f"STDOUT: {result.stdout}\nSTDERR: {result.stderr}" + ) + + +def _run_alembic_downgrade(target: str, *, dsn: str) -> None: + """Run alembic downgrade to target using the given DSN.""" + env = {**os.environ, "DATABASE_MIGRATE_URL": dsn} + result = subprocess.run( + [sys.executable, "-m", "alembic", "downgrade", target], + capture_output=True, + text=True, + env=env, + ) + if result.returncode != 0: + raise RuntimeError( + f"alembic downgrade {target} failed (rc={result.returncode}):\n" + f"STDOUT: {result.stdout}\nSTDERR: {result.stderr}" + ) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture(scope="module") +def migrate_dsn() -> str: + """Provide the integration database DSN, or skip the whole module.""" + dsn = _get_migrate_url() + if not dsn: + pytest.skip( + "Migration integration tests skipped — set INTEGRATION_DATABASE_URL or " + "INTEGRATION=1 (with DATABASE_MIGRATE_URL) to activate. " + "These tests require a disposable PostgreSQL database and must never " + "run against the normal development database." + ) + return dsn + + +@pytest.fixture(scope="module") +def pg_conn(migrate_dsn): + """Open a psycopg3 connection to the integration database.""" + conn = _psycopg_conn(migrate_dsn) + yield conn + conn.close() + + +# --------------------------------------------------------------------------- +# Test: upgrade 0005 → 0006/head +# --------------------------------------------------------------------------- + +class TestMigration0005To0006: + """Upgrade integration tests that prove the 0005->0006 transition.""" + + def test_upgrade_to_0005(self, migrate_dsn): + """Apply all migrations through 0005 (the pre-phase-12 baseline).""" + _run_alembic("0005", dsn=migrate_dsn) + + def test_0005_lacks_display_name_override(self, pg_conn): + """After 0005 only, display_name_override must NOT exist yet.""" + cur = pg_conn.execute( + """ + SELECT column_name + FROM information_schema.columns + WHERE table_name = 'cloud_connections' + AND column_name = 'display_name_override' + """ + ) + row = cur.fetchone() + assert row is None, ( + "display_name_override should not exist after revision 0005; " + "migration 0006 has not run yet" + ) + + def test_0005_lacks_phase12_tables(self, pg_conn): + """After 0005 only, cloud_items/cloud_item_topics/cloud_folder_states must be absent.""" + for table in PHASE_12_TABLES: + cur = pg_conn.execute( + "SELECT table_name FROM information_schema.tables " + "WHERE table_schema='public' AND table_name=%s", + (table,), + ) + assert cur.fetchone() is None, ( + f"Table '{table}' must not exist after revision 0005" + ) + + def test_upgrade_to_head(self, migrate_dsn): + """Upgrade from 0005 to head (0006).""" + _run_alembic("head", dsn=migrate_dsn) + + def test_alembic_version_is_0006(self, pg_conn): + """alembic_version must report '0006' after upgrade to head.""" + cur = pg_conn.execute("SELECT version_num FROM alembic_version") + row = cur.fetchone() + assert row is not None, "alembic_version table has no rows after migration" + assert row[0] == "0006", ( + f"Expected alembic_version='0006', got {row[0]!r}" + ) + + def test_display_name_override_column_exists(self, pg_conn): + """cloud_connections.display_name_override must exist after 0006.""" + cur = pg_conn.execute( + """ + SELECT column_name, is_nullable, data_type + FROM information_schema.columns + WHERE table_name = 'cloud_connections' + AND column_name = 'display_name_override' + """ + ) + row = cur.fetchone() + assert row is not None, ( + "cloud_connections.display_name_override column is missing after migration 0006" + ) + col_name, is_nullable, data_type = row + assert is_nullable == "YES", ( + "display_name_override must be nullable (optional user-defined override)" + ) + + def test_cloud_items_table_exists_with_key_columns(self, pg_conn): + """cloud_items table must exist with essential columns.""" + expected_columns = { + "id", "user_id", "connection_id", "provider_item_id", "name", "kind", + "analysis_status", "semantic_index_status", "created_at", "updated_at", + } + cur = pg_conn.execute( + "SELECT column_name FROM information_schema.columns " + "WHERE table_name='cloud_items'" + ) + actual = {row[0] for row in cur.fetchall()} + missing = expected_columns - actual + assert not missing, f"cloud_items is missing columns: {sorted(missing)}" + + def test_cloud_item_topics_table_exists(self, pg_conn): + """cloud_item_topics association table must exist.""" + cur = pg_conn.execute( + "SELECT table_name FROM information_schema.tables " + "WHERE table_schema='public' AND table_name='cloud_item_topics'" + ) + assert cur.fetchone() is not None, "cloud_item_topics table missing after migration 0006" + + def test_cloud_folder_states_table_exists_with_key_columns(self, pg_conn): + """cloud_folder_states table must exist with essential columns.""" + expected_columns = { + "id", "user_id", "connection_id", "parent_ref", "refresh_state", + "last_refreshed_at", "error_code", "created_at", "updated_at", + } + cur = pg_conn.execute( + "SELECT column_name FROM information_schema.columns " + "WHERE table_name='cloud_folder_states'" + ) + actual = {row[0] for row in cur.fetchall()} + missing = expected_columns - actual + assert not missing, f"cloud_folder_states is missing columns: {sorted(missing)}" + + def test_unique_constraints_exist(self, pg_conn): + """Named unique constraints from 0006 must exist in the catalog.""" + for constraint in PHASE_12_UNIQUE_CONSTRAINTS: + cur = pg_conn.execute( + """ + SELECT constraint_name + FROM information_schema.table_constraints + WHERE constraint_type = 'UNIQUE' + AND constraint_name = %s + """, + (constraint,), + ) + row = cur.fetchone() + assert row is not None, ( + f"Unique constraint '{constraint}' is missing after migration 0006" + ) + + def test_indexes_exist(self, pg_conn): + """Named indexes from 0006 must exist in pg_indexes.""" + for idx in PHASE_12_INDEXES: + cur = pg_conn.execute( + "SELECT indexname FROM pg_indexes WHERE indexname=%s", + (idx,), + ) + row = cur.fetchone() + assert row is not None, ( + f"Index '{idx}' is missing after migration 0006" + ) + + def test_docuvault_app_cannot_create_table(self, migrate_dsn, pg_conn): + """docuvault_app (DML-only role) must not be able to create tables (DDL forbidden).""" + # Derive app DSN from migrate DSN by substituting the user + # The test DB is shared; we test privilege by executing as the app role. + # We check the privilege via information_schema — if the role has CREATE + # privilege on the public schema, that is a misconfiguration. + # + # NOTE: In the Docker initdb.d setup, docuvault_app is granted USAGE but + # not CREATE on the public schema. We assert this via has_schema_privilege. + cur = pg_conn.execute( + "SELECT has_schema_privilege('docuvault_app', 'public', 'CREATE')" + ) + row = cur.fetchone() + # If the user doesn't exist (integration DB doesn't have it), skip gracefully + if row is None: + pytest.skip("docuvault_app role does not exist in the integration database") + assert row[0] is False, ( + "docuvault_app must NOT have CREATE privilege on the public schema — " + "only docuvault_migrate should be able to run DDL" + ) From 3ca57dcd0c8af6760f6a38eaeb4c4fed64d4b892 Mon Sep 17 00:00:00 2001 From: curo1305 Date: Sat, 20 Jun 2026 10:49:21 +0200 Subject: [PATCH 3/4] fix(12-05): bump version to 0.2.1 and update docs for migration-gated startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - backend/main.py, frontend/package.json: version 0.2.0 → 0.2.1 - CLAUDE.md, AGENTS.md: current state updated to reflect gap-closure - README.md: startup instructions note migrate runs automatically; update migration commands - RUNBOOK.md: startup diagram includes migrate service; migration gate explanation - SECURITY.md: Phase 12 gap-closure threat register and security gate evidence --- AGENTS.md | 6 +++--- CLAUDE.md | 4 ++-- README.md | 17 ++++++++++------- SECURITY.md | 32 ++++++++++++++++++++++++++++++++ backend/main.py | 2 +- frontend/package.json | 2 +- 6 files changed, 49 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ece9e29..7c28a1c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ DocuVault is a multi-user SaaS document management platform built on FastAPI (Python) + Vue 3. It handles document upload, text extraction (PDF/DOCX/image/text), AI-based topic classification, per-user isolated storage, folder organization, document sharing, and pluggable cloud storage backends (OneDrive, Google Drive, Nextcloud, WebDAV). -**Current state:** v0.2.0 — Phase 12 complete (2026-06-19). Cloud resource foundation: connection-ID browse API, CloudResourceAdapter contract, durable cloud item metadata, capability-aware action rendering, breadcrumb freshness indicators. Cloud and local files share one StorageBrowser implementation. Security-negative suite in test_cloud_security.py. Not cleared for public internet deployment. +**Current state:** v0.2.1 — Phase 12 gap-closure complete (2026-06-20). Migration-gated startup: one-shot migrate service runs alembic upgrade head before backend/worker/beat start (service_completed_successfully gate). 0005→0006 upgrade regression in test_migration_0006.py. RUNBOOK migration ops section. Cloud resource foundation: connection-ID browse API, CloudResourceAdapter contract, durable cloud item metadata, capability-aware action rendering. Not cleared for public internet deployment. ## Stack @@ -123,9 +123,9 @@ This project uses the GSD (Get Shit Done) planning workflow. Planning artifacts /gsd:progress — check status and advance workflow ``` -### Current state: v0.2.0 — Phase 12 complete (2026-06-19) +### Current state: v0.2.1 — Phase 12 gap-closure complete (2026-06-20) -Phase 12 (cloud-resource-foundation): full cloud browse foundation — CloudResourceAdapter contract, connection-ID browse API (`GET /api/cloud/connections/{id}/items`), durable cloud item metadata (CloudItem ORM), connection-ID routing (`/cloud/:connectionId`), capability-aware action rendering (CapabilityButton / aria-disabled), breadcrumb freshness indicators (refreshing/fresh/stale), session-only folder memory, display-name rename for cloud connections, and a dedicated security-negative integration suite (`test_cloud_security.py`). Cloud and local files share one StorageBrowser row/action implementation. The browse foundation is complete; Phase 13 adds cloud mutations (upload/move/delete) and Phase 14 adds byte caching. The app is functional but alpha-quality — not cleared for public internet deployment. +Phase 12 gap-closure: migration-gated Compose startup (migrate service, service_completed_successfully dependency gate on backend/worker/beat), 0005→0006 upgrade regression tests (test_migration_0006.py), RUNBOOK migration ops section. Full Phase 12 cloud browse foundation — CloudResourceAdapter contract, connection-ID browse API (`GET /api/cloud/connections/{id}/items`), durable cloud item metadata (CloudItem ORM), connection-ID routing (`/cloud/:connectionId`), capability-aware action rendering (CapabilityButton / aria-disabled), breadcrumb freshness indicators (refreshing/fresh/stale), session-only folder memory, display-name rename for cloud connections, and a dedicated security-negative integration suite (`test_cloud_security.py`). Cloud and local files share one StorageBrowser row/action implementation. The browse foundation is complete; Phase 13 adds cloud mutations (upload/move/delete) and Phase 14 adds byte caching. The app is functional but alpha-quality — not cleared for public internet deployment. ## Development Setup diff --git a/CLAUDE.md b/CLAUDE.md index 13f9bcd..779ba2c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ DocuVault is a multi-user SaaS document management platform built on FastAPI (Python) + Vue 3. It handles document upload, text extraction (PDF/DOCX/image/text), AI-based topic classification, per-user isolated storage, folder organization, document sharing, and pluggable cloud storage backends (OneDrive, Google Drive, Nextcloud, WebDAV). -**Current state:** v0.2.0 — Phase 12 complete 2026-06-19. Provider-neutral cloud resource contract (CloudResourceAdapter, CloudResource, CloudCapability), durable CloudItem/CloudFolderState metadata layer (Alembic migration 0006), connection-ID browse API with IDOR protection and stale-while-revalidate, refresh_cloud_folder Celery task, and capability-aware StorageBrowser frontend. 12 of 12 phases complete. Not cleared for public internet deployment. +**Current state:** v0.2.1 — Phase 12 gap-closure complete 2026-06-20. Migration-gated Compose startup (migrate service, service_completed_successfully dependency gate), 0005→0006 upgrade regression tests, RUNBOOK migration ops section. Provider-neutral cloud resource contract (CloudResourceAdapter, CloudResource, CloudCapability), durable CloudItem/CloudFolderState metadata layer (Alembic migration 0006), connection-ID browse API with IDOR protection and stale-while-revalidate, refresh_cloud_folder Celery task, and capability-aware StorageBrowser frontend. 12 of 12 phases complete. Not cleared for public internet deployment. ## Stack @@ -119,7 +119,7 @@ This project uses the GSD (Get Shit Done) planning workflow. Planning artifacts /gsd:progress — check status and advance workflow ``` -### Current state: v0.2.0 — Phase 12 (cloud-resource-foundation) complete (2026-06-19) +### Current state: v0.2.1 — Phase 12 gap-closure complete (2026-06-20) All phases are marked complete in `.planning/ROADMAP.md`. The app is functional but alpha-quality — not cleared for public internet deployment. The next milestone has not been started. diff --git a/README.md b/README.md index f818d0c..d09da19 100644 --- a/README.md +++ b/README.md @@ -117,16 +117,15 @@ print('JWT_PUBLIC_KEY=' + base64.b64encode(k.public_key().public_bytes(serializ # (everything else has usable defaults for local dev) nano .env -# 5. Start all services +# 5. Start all services (migrations run automatically before backend starts) docker compose up -d --build -# 6. Run database migrations -docker compose exec backend alembic upgrade head - -# 7. Open the app +# 6. Open the app open http://localhost:5173 ``` +The `migrate` service runs `alembic upgrade head` automatically before the backend, Celery worker, and Celery beat start. No manual migration step is needed on a normal `docker compose up`. + The bootstrap admin account (email + password from `.env`) is created automatically on first startup if no users exist. --- @@ -222,8 +221,12 @@ npm run dev # http://localhost:5173 ### Database migrations ```bash -# Apply all pending migrations -docker compose exec backend alembic upgrade head +# Migrations run automatically on docker compose up via the migrate service. +# To run them manually (e.g., after a git pull without restarting): +docker compose run --rm migrate + +# Check current revision +docker compose run --rm migrate alembic current # Create a new migration after changing db/models.py docker compose exec backend alembic revision --autogenerate -m "describe change" diff --git a/SECURITY.md b/SECURITY.md index f9cd6d2..0a8c3db 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -385,3 +385,35 @@ None. Phase 12 adds a read-only cloud browse surface (no MinIO writes, no quota |---------|-----------|---------------|-----------| | T-12-SC | pip-audit tooling | pip-audit not installed in local Python 3.9 environment | No new Python packages added in Phase 12; existing packages audited in Phase 8. Operator should run pip-audit in CI or with Python 3.12 target environment. | | T-12-disabled | DISABLED connection returns 200 with unavailable capabilities | Browse endpoint is accessible but all operations are unavailable | This is correct behavior: ownership is still asserted (user owns the connection), and capabilities convey the unavailability. No cloud bytes or credentials are served. | + +--- + +## Phase 12 Gap Closure — Schema Drift Fix (2026-06-20) + +### Gap Closure Threat Register + +| Threat ID | Threat | Disposition | Status | Evidence | +|-----------|--------|-------------|--------|----------| +| T-12-05-01 | App starts against stale schema (UndefinedColumn) | mitigate | CLOSED | `docker-compose.yml` migrate service + `service_completed_successfully` dependency gate; `backend/tests/test_compose_migrations.py` — 7 static assertions | +| T-12-05-02 | DDL privilege escalation (app uses migrate URL) | mitigate | CLOSED | migrate service sets only `DATABASE_MIGRATE_URL`; backend/worker/beat receive only `DATABASE_URL`; test assertions in `test_compose_migrations.py:test_migrate_service_uses_migrate_url` | +| T-12-05-03 | Concurrent migrations (multiple replicas upgrading) | mitigate | CLOSED | Exactly one migrate service; replicas only wait on `service_completed_successfully` — never run Alembic themselves | +| T-12-05-04 | Secrets in Compose config or test output | mitigate | CLOSED | Environment references via `${VAR}` only; no rendered values in source or committed output | +| T-12-05-05 | Integration test destroys developer database | mitigate | CLOSED | `test_migration_0006.py` skips without `INTEGRATION=1`/`INTEGRATION_DATABASE_URL`; explicit skip message; never touches `DATABASE_URL` | + +### Gap Closure Security Gate Evidence + +**Gate 1 — Compose configuration invariants:** +``` +pytest -q backend/tests/test_compose_migrations.py +7 passed +``` + +**Gate 2 — Migration regression tests:** +``` +pytest -q backend/tests/test_migration_0006.py +12 skipped (INTEGRATION not set — correct behavior without disposable DB) +``` +Integration tests activate with `INTEGRATION=1` against a disposable PostgreSQL database. + +**Gate 3 — No new Python or frontend packages introduced.** +**Gate 4 — No hardcoded credentials in Compose or test files.** diff --git a/backend/main.py b/backend/main.py index 8208120..b8afa88 100644 --- a/backend/main.py +++ b/backend/main.py @@ -244,7 +244,7 @@ async def lifespan(app: FastAPI): # ── Application factory ─────────────────────────────────────────────────────── -app = FastAPI(title="Document Scanner API", version="0.2.0", lifespan=lifespan) +app = FastAPI(title="Document Scanner API", version="0.2.1", lifespan=lifespan) # Rate limiter state (slowapi) app.state.limiter = auth_limiter diff --git a/frontend/package.json b/frontend/package.json index 63c2f44..0287603 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "document-scanner-frontend", - "version": "0.2.0", + "version": "0.2.1", "type": "module", "scripts": { "dev": "vite", From 67d3e4bcac2e8ba953e23a8124af2ce7de315b08 Mon Sep 17 00:00:00 2001 From: curo1305 Date: Sat, 20 Jun 2026 10:49:59 +0200 Subject: [PATCH 4/4] docs(12-05): complete gap closure plan SUMMARY --- .../12-05-SUMMARY.md | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 .planning/phases/12-cloud-resource-foundation/12-05-SUMMARY.md diff --git a/.planning/phases/12-cloud-resource-foundation/12-05-SUMMARY.md b/.planning/phases/12-cloud-resource-foundation/12-05-SUMMARY.md new file mode 100644 index 0000000..e680bb0 --- /dev/null +++ b/.planning/phases/12-cloud-resource-foundation/12-05-SUMMARY.md @@ -0,0 +1,119 @@ +--- +phase: "12" +plan: "05" +type: gap_closure +subsystem: infrastructure +tags: [migrations, compose, docker, testing, devops] +dependency_graph: + requires: ["12-04"] + provides: [migration-gated-startup, upgrade-regression] + affects: [docker-compose, alembic, backend, celery-worker, celery-beat] +tech_stack: + added: [] + patterns: [service_completed_successfully, one-shot-container, disposable-pg-integration-test] +key_files: + created: + - backend/tests/test_compose_migrations.py + - backend/tests/test_migration_0006.py + modified: + - docker-compose.yml + - backend/main.py + - frontend/package.json + - CLAUDE.md + - README.md + - RUNBOOK.md + - SECURITY.md + - AGENTS.md +decisions: + - "migrate service uses only DATABASE_MIGRATE_URL (DDL role); application services retain DATABASE_URL (DML role only)" + - "migrate service has restart: no — one-shot, not a daemon; failure blocks all application services" + - "integration tests skip cleanly without INTEGRATION=1 — no developer DB mutation risk" +metrics: + duration: "~25 minutes" + completed: "2026-06-20" + tasks: 3 + files: 9 +--- + +# Phase 12 Plan 05: Schema Drift Gap Closure Summary + +**One-liner:** Migration-gated Compose startup gate (one-shot migrate service + service_completed_successfully) closes the Phase 12 UAT schema drift blocker that left the live database at revision 0005. + +## What Was Built + +### Task 1: Migration-gated Compose startup path + +Added a one-shot `migrate` service to `docker-compose.yml` that: +- Runs `alembic upgrade head` as its only command +- Receives only `DATABASE_MIGRATE_URL` (DDL-capable `docuvault_migrate` role) +- Has the same hardening as other containers: `read_only: true`, `cap_drop: ALL`, `no-new-privileges:true`, tmpfs `/tmp` +- Uses `restart: no` — it exits 0 on success and does not restart + +All three application services (`backend`, `celery-worker`, `celery-beat`) now depend on `migrate` with `condition: service_completed_successfully`. A failed migration exit blocks application startup. + +Added `backend/tests/test_compose_migrations.py` with 7 static assertions: +- Exactly one service runs `alembic upgrade head` and it is named `migrate` +- `migrate` uses `DATABASE_MIGRATE_URL`, not `DATABASE_URL` +- Application services do not run `alembic upgrade` themselves +- All three application services depend on `migrate: service_completed_successfully` +- Container hardening invariants (read_only, cap_drop ALL, no-new-privileges) +- One-shot restart policy (`restart: no`) +- Minimal dependencies — only postgres, not minio/redis + +### Task 2: PostgreSQL upgrade regression from 0005 to head + +Added `backend/tests/test_migration_0006.py` with 12 integration test cases: +- Skips cleanly without `INTEGRATION=1` or `INTEGRATION_DATABASE_URL` (no developer DB mutation) +- Applies migrations to 0005, asserts `display_name_override` absent and Phase 12 tables absent +- Upgrades to head, asserts `alembic_version = '0006'` +- Proves `display_name_override` column exists and is nullable +- Proves `cloud_items`, `cloud_item_topics`, `cloud_folder_states` tables exist +- Asserts key columns on each table +- Asserts named unique constraints: `uq_cloud_items_connection_provider_item`, `uq_cloud_folder_states_connection_parent` +- Asserts named indexes: `ix_cloud_items_user_id`, `ix_cloud_items_connection_id`, `ix_cloud_items_connection_parent`, `ix_cloud_folder_states_connection` +- Asserts `docuvault_app` lacks CREATE privilege on the public schema + +Updated `RUNBOOK.md` with a new "Database Migrations" section covering: +- Checking current revision (`alembic current`) +- Normal upgrade via `docker compose run --rm migrate` +- Viewing migration history +- Failed-migration recovery steps +- Emergency schema-drift remediation commands (the exact Phase 12 UAT blocker scenario) + +### Task 3: Version bump and documentation + +- `backend/main.py`: `0.2.0` → `0.2.1` +- `frontend/package.json`: `0.2.0` → `0.2.1` +- `CLAUDE.md`: current state updated, startup startup pattern noted +- `README.md`: startup instructions updated — no manual `alembic upgrade head` needed; migration commands updated +- `SECURITY.md`: Phase 12 gap-closure threat register and gate evidence appended +- `AGENTS.md`: current state updated to v0.2.1 gap-closure + +## Deviations from Plan + +### Auto-added + +**[Rule 2 - Missing critical tests] RUNBOOK startup diagram** +- Added description of the migrate→application-services dependency chain to the startup section diagram in RUNBOOK.md (the plan specified migration lifecycle documentation; the startup flow description was an obvious omission) + +### Descoped + +The plan mentioned "Apply `alembic upgrade head` through the new migration service in the current Compose environment" and "restart/recreate backend plus Celery processes through the gated path" as Task 3 actions. These are live-environment operations that require a running Docker stack. The Compose changes and tests fully satisfy the architectural fix; the live-environment verification step was documented in RUNBOOK.md for operators to execute rather than run as part of this test-only execution environment. + +## Threat Surface Scan + +No new network endpoints, auth paths, or file access patterns introduced. The `migrate` service has no ports exposed and no external network access. + +## Self-Check + +### Created files exist: +- backend/tests/test_compose_migrations.py: present +- backend/tests/test_migration_0006.py: present +- .planning/phases/12-cloud-resource-foundation/12-05-SUMMARY.md: present (this file) + +### Commits: +- 1b3084d: feat(12-05): add migration-gated Compose startup path +- de2efd1: test(12-05): add migration 0005→0006 integration regression and RUNBOOK migration ops +- 3ca57dc: fix(12-05): bump version to 0.2.1 and update docs for migration-gated startup + +## Self-Check: PASSED