From 4febe2f7048fc63dfa03bc5b4fa90005152abfc9 Mon Sep 17 00:00:00 2001 From: curo1305 Date: Thu, 4 Jun 2026 18:44:53 +0200 Subject: [PATCH 1/4] test(07-01): Wave 0 xfail stubs for D-01..D-16 (13 new stubs) - Create backend/tests/test_ai_providers.py with 7 stubs: test_generic_openai_json_mode, test_anthropic_structured_output, test_get_provider_typed, test_client_singleton, test_context_chars_truncation, test_smart_truncation, test_gemini_fallback_to_parse_classification - Create backend/tests/test_ai_config.py with 2 stubs: test_load_provider_config, test_api_key_encrypt_decrypt - Create backend/tests/test_admin_ai_config.py with 2 stubs: test_get_never_returns_key, test_put_writes_active_provider - Create backend/tests/test_document_tasks.py with 2 stubs: test_retry_backoff, test_exhaustion_sets_failed_status - Append test_reclassify_requeues_celery xfail stub to test_documents.py - D-14 regression guard: grep confirms 3 host-gateway entries (>=2 required) --- backend/tests/test_admin_ai_config.py | 25 +++++++++++++++ backend/tests/test_ai_config.py | 25 +++++++++++++++ backend/tests/test_ai_providers.py | 46 +++++++++++++++++++++++++++ backend/tests/test_document_tasks.py | 25 +++++++++++++++ backend/tests/test_documents.py | 10 ++++++ 5 files changed, 131 insertions(+) create mode 100644 backend/tests/test_admin_ai_config.py create mode 100644 backend/tests/test_ai_config.py create mode 100644 backend/tests/test_ai_providers.py create mode 100644 backend/tests/test_document_tasks.py diff --git a/backend/tests/test_admin_ai_config.py b/backend/tests/test_admin_ai_config.py new file mode 100644 index 0000000..2a99647 --- /dev/null +++ b/backend/tests/test_admin_ai_config.py @@ -0,0 +1,25 @@ +""" +Wave 0 xfail stubs for Phase 7 admin AI config endpoint tests. + +Covers: + - T-07-01: GET /api/admin/ai-config never returns api_key_enc (D-05/D-08) + - D-08: PUT /api/admin/ai-config writes active provider (admin AI Providers panel) + +Each function is a placeholder to be promoted in Plan 07-05 once the admin +AI config endpoints exist. + +Stub policy (STATE.md decision: xfail(strict=False) for Wave 0): + - Body is a single pytest.xfail() call — no assertion code. + - strict=False so unexpected passes (xpass) never break CI. +""" +import pytest + + +@pytest.mark.xfail(strict=False, reason="Wave 0 stub — promoted in Plan 07-05") +async def test_get_never_returns_key(): + pytest.xfail("not implemented yet — Plan 07-05") + + +@pytest.mark.xfail(strict=False, reason="Wave 0 stub — promoted in Plan 07-05") +async def test_put_writes_active_provider(): + pytest.xfail("not implemented yet — Plan 07-05") diff --git a/backend/tests/test_ai_config.py b/backend/tests/test_ai_config.py new file mode 100644 index 0000000..819a64e --- /dev/null +++ b/backend/tests/test_ai_config.py @@ -0,0 +1,25 @@ +""" +Wave 0 xfail stubs for Phase 7 AI config service tests. + +Covers: + - D-04: load_provider_config() reads from system_settings table + - D-05: API key HKDF encryption round-trip + +Each function is a placeholder to be promoted in Plan 07-01 (green gate) once +the services/ai_config.py module and system_settings table exist. + +Stub policy (STATE.md decision: xfail(strict=False) for Wave 0): + - Body is a single pytest.xfail() call — no assertion code. + - strict=False so unexpected passes (xpass) never break CI. +""" +import pytest + + +@pytest.mark.xfail(strict=False, reason="Wave 0 stub — promoted in Plan 07-01") +async def test_load_provider_config(): + pytest.xfail("not implemented yet — Plan 07-01") + + +@pytest.mark.xfail(strict=False, reason="Wave 0 stub — promoted in Plan 07-01") +async def test_api_key_encrypt_decrypt(): + pytest.xfail("not implemented yet — Plan 07-01") diff --git a/backend/tests/test_ai_providers.py b/backend/tests/test_ai_providers.py new file mode 100644 index 0000000..9775c2a --- /dev/null +++ b/backend/tests/test_ai_providers.py @@ -0,0 +1,46 @@ +""" +Wave 0 xfail stubs for Phase 7 AI provider tests. + +Each function is a placeholder for a test that will be promoted to green +in a later plan wave (per 07-VALIDATION.md per-task-verification map). + +Stub policy (STATE.md decision: xfail(strict=False) for Wave 0): + - Body is a single pytest.xfail() call — no assertion code. + - strict=False so unexpected passes (xpass) never break CI. +""" +import pytest + + +@pytest.mark.xfail(strict=False, reason="Wave 0 stub — promoted in Plan 07-02") +async def test_generic_openai_json_mode(): + pytest.xfail("not implemented yet — Plan 07-02") + + +@pytest.mark.xfail(strict=False, reason="Wave 0 stub — promoted in Plan 07-03") +async def test_anthropic_structured_output(): + pytest.xfail("not implemented yet — Plan 07-03") + + +@pytest.mark.xfail(strict=False, reason="Wave 0 stub — promoted in Plan 07-02") +async def test_get_provider_typed(): + pytest.xfail("not implemented yet — Plan 07-02") + + +@pytest.mark.xfail(strict=False, reason="Wave 0 stub — promoted in Plan 07-02") +async def test_client_singleton(): + pytest.xfail("not implemented yet — Plan 07-02") + + +@pytest.mark.xfail(strict=False, reason="Wave 0 stub — promoted in Plan 07-03") +async def test_context_chars_truncation(): + pytest.xfail("not implemented yet — Plan 07-03") + + +@pytest.mark.xfail(strict=False, reason="Wave 0 stub — promoted in Plan 07-03") +async def test_smart_truncation(): + pytest.xfail("not implemented yet — Plan 07-03") + + +@pytest.mark.xfail(strict=False, reason="Wave 0 stub — promoted in Plan 07-02 Task 4 (D-02 Gemini fallback path)") +async def test_gemini_fallback_to_parse_classification(): + pytest.xfail("not implemented yet — Plan 07-02") diff --git a/backend/tests/test_document_tasks.py b/backend/tests/test_document_tasks.py new file mode 100644 index 0000000..3b5d632 --- /dev/null +++ b/backend/tests/test_document_tasks.py @@ -0,0 +1,25 @@ +""" +Wave 0 xfail stubs for Phase 7 Celery document task tests. + +Covers: + - D-09: Celery retry with 30s/90s/270s exponential backoff + - D-10: After 3 retries doc.status = "classification_failed" + +Each function is a placeholder to be promoted in Plan 07-04 once the +bind=True retry harness is implemented in document_tasks.py. + +Stub policy (STATE.md decision: xfail(strict=False) for Wave 0): + - Body is a single pytest.xfail() call — no assertion code. + - strict=False so unexpected passes (xpass) never break CI. +""" +import pytest + + +@pytest.mark.xfail(strict=False, reason="Wave 0 stub — promoted in Plan 07-04") +async def test_retry_backoff(): + pytest.xfail("not implemented yet — Plan 07-04") + + +@pytest.mark.xfail(strict=False, reason="Wave 0 stub — promoted in Plan 07-04") +async def test_exhaustion_sets_failed_status(): + pytest.xfail("not implemented yet — Plan 07-04") diff --git a/backend/tests/test_documents.py b/backend/tests/test_documents.py index 57c891a..6a76fee 100644 --- a/backend/tests/test_documents.py +++ b/backend/tests/test_documents.py @@ -923,3 +923,13 @@ async def test_delete_cloud_remove_only(async_client, auth_user, db_session): # DB row removed deleted = await db_session.get(Document, doc_id) assert deleted is None + + +# --------------------------------------------------------------------------- +# Phase 7 Wave 0 xfail stub — D-11 re-classify endpoint +# --------------------------------------------------------------------------- + + +@pytest.mark.xfail(strict=False, reason="Wave 0 stub — promoted in Plan 07-04") +async def test_reclassify_requeues_celery(): + pytest.xfail("not implemented yet — Plan 07-04") From 4eb317749f512e99585fbf31dfc3f77b0969690c Mon Sep 17 00:00:00 2001 From: curo1305 Date: Thu, 4 Jun 2026 18:46:11 +0200 Subject: [PATCH 2/4] feat(07-01): Alembic migration 0005 + SystemSettings ORM model - Add backend/migrations/versions/0005_system_settings.py (revision 0005, down_revision 0004) - Creates system_settings table with 9 columns: id (UUID PK gen_random_uuid()), provider_id (String NOT NULL UNIQUE), api_key_enc (Text NULL), base_url (Text NULL), model_name (Text NOT NULL default ''), context_chars (Integer NOT NULL default 8000), is_active (Boolean NOT NULL default false), created_at/updated_at (TIMESTAMPTZ default now()) - UniqueConstraint uq_system_settings_provider_id on provider_id - Add SystemSettings ORM model to backend/db/models.py following CloudConnection style - downgrade() drops system_settings table cleanly --- backend/db/models.py | 40 +++++++++ .../versions/0005_system_settings.py | 82 +++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 backend/migrations/versions/0005_system_settings.py diff --git a/backend/db/models.py b/backend/db/models.py index 5e9db49..ca058bf 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -335,3 +335,43 @@ class Group(Base): created_at: Mapped[datetime] = mapped_column( TIMESTAMP(timezone=True), nullable=False, server_default=func.now() ) + + +class SystemSettings(Base): + """AI provider configuration table — one row per provider. + + Stores the active AI provider's API key (Fernet-encrypted), base URL, + model name, and context window size. Only one row has is_active=TRUE + at any given time; the admin panel flips all rows atomically. + + Encryption (D-05): + api_key_enc is encrypted with HKDF/Fernet using info=b"ai-provider-settings" + for domain separation from cloud credentials (info=b"cloud-credentials"). + Master key is settings.cloud_creds_key (env var CLOUD_CREDS_KEY). + See services/ai_config.py for the encryption helpers. + + Design reference: 07-RESEARCH.md Pattern 1. + Migration: 0005_system_settings.py. + """ + + __tablename__ = "system_settings" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + provider_id: Mapped[str] = mapped_column(String, nullable=False, unique=True) + api_key_enc: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + base_url: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + model_name: Mapped[str] = mapped_column(Text, nullable=False, default="") + context_chars: Mapped[int] = mapped_column(Integer, nullable=False, default=8000) + is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + created_at: Mapped[datetime] = mapped_column( + TIMESTAMP(timezone=True), nullable=False, server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + TIMESTAMP(timezone=True), nullable=False, server_default=func.now() + ) + + __table_args__ = ( + UniqueConstraint("provider_id", name="uq_system_settings_provider_id"), + ) diff --git a/backend/migrations/versions/0005_system_settings.py b/backend/migrations/versions/0005_system_settings.py new file mode 100644 index 0000000..1d263fb --- /dev/null +++ b/backend/migrations/versions/0005_system_settings.py @@ -0,0 +1,82 @@ +"""Add system_settings table for AI provider configuration. + +Revision ID: 0005 +Revises: 0004 +Create Date: 2026-06-04 + +Changes: + 1. Create system_settings table with one row per AI provider. + Stores: api_key_enc (Fernet-encrypted), base_url, model_name, + context_chars, is_active flag, and timestamps. + 2. Add UNIQUE constraint on provider_id (uq_system_settings_provider_id) + to enforce one row per provider name. + +Design notes (07-RESEARCH.md Pattern 1): + - One row per provider (not key-value store) — typed columns, enforced uniqueness. + - id column uses gen_random_uuid() server default. + - api_key_enc is Text nullable (NULL for local providers like Ollama/LMStudio). + - is_active = TRUE for the single active provider; all others FALSE. + - Encryption uses HKDF with info=b"ai-provider-settings" (domain separation from + cloud credentials which use info=b"cloud-credentials" — same master key, different + derived keys; see services/ai_config.py). +""" +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID as PG_UUID +from alembic import op + +# revision identifiers, used by Alembic. +revision = "0005" +down_revision = "0004" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table("system_settings", + sa.Column( + "id", + PG_UUID(as_uuid=True), + primary_key=True, + server_default=sa.text("gen_random_uuid()"), + ), + sa.Column("provider_id", sa.String, nullable=False), + sa.Column("api_key_enc", sa.Text, nullable=True), + sa.Column("base_url", sa.Text, nullable=True), + sa.Column( + "model_name", + sa.Text, + nullable=False, + server_default="", + ), + sa.Column( + "context_chars", + sa.Integer, + nullable=False, + server_default="8000", + ), + sa.Column( + "is_active", + sa.Boolean, + nullable=False, + server_default="false", + ), + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.Column( + "updated_at", + sa.TIMESTAMP(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.UniqueConstraint("provider_id", name="uq_system_settings_provider_id"), + ) + + +def downgrade() -> None: + op.drop_table("system_settings") From 0fd6930b41f47a31e1c0b78bdbadda316935b43a Mon Sep 17 00:00:00 2001 From: curo1305 Date: Thu, 4 Jun 2026 18:50:01 +0200 Subject: [PATCH 3/4] feat(07-01): services/ai_config.py HKDF helpers, ProviderConfig loader, env seed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Create backend/services/ai_config.py with: - _derive_ai_settings_key(master_key, provider_id): fresh HKDF per call, salt=provider_id.encode("utf-8"), info=b"ai-provider-settings" (domain-separated from b"cloud-credentials") - encrypt_api_key / decrypt_api_key: Fernet round-trip without JSON wrapping - load_provider_config(session): reads is_active=True row from system_settings, decrypts api_key_enc; returns stub ProviderConfig (real class lands in Plan 02) - seed_system_settings_from_env(session): idempotent insert of default provider on first boot - _ProviderConfigStub: minimal Pydantic model stub until ai/provider_config.py exists (Plan 02) - Update backend/main.py: import seed_system_settings_from_env, call in lifespan with try/except so missing table (pre-migration fresh boot) doesn't crash startup - Round-trip smoke test: encrypt_api_key → decrypt_api_key == original plaintext --- backend/main.py | 14 +++ backend/services/ai_config.py | 225 ++++++++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+) create mode 100644 backend/services/ai_config.py diff --git a/backend/main.py b/backend/main.py index 9ab1ea5..0c10759 100644 --- a/backend/main.py +++ b/backend/main.py @@ -22,6 +22,7 @@ from api.documents import router as documents_router from api.topics import router as topics_router from config import settings from db.session import AsyncSessionLocal, engine +from services.ai_config import seed_system_settings_from_env from services.logging import setup_logging from services.rate_limiting import account_limiter @@ -158,6 +159,19 @@ async def lifespan(app: FastAPI): async with AsyncSessionLocal() as session: await bootstrap_admin(session) + # AI provider seed (D-04 / Phase 7): populate system_settings from env vars + # on first boot. Wrapped in try/except so that a missing table (fresh container + # before migrations run) does not crash startup — logs a warning and skips. + try: + async with AsyncSessionLocal() as session: + await seed_system_settings_from_env(session) + await session.commit() + except Exception as _seed_exc: + import logging as _logging + _logging.getLogger(__name__).warning( + "AI provider seed skipped (table may not exist yet): %s", _seed_exc + ) + yield # Shutdown: close pooled connections and Redis diff --git a/backend/services/ai_config.py b/backend/services/ai_config.py new file mode 100644 index 0000000..6135222 --- /dev/null +++ b/backend/services/ai_config.py @@ -0,0 +1,225 @@ +""" +AI provider configuration service for DocuVault. + +Provides HKDF/Fernet encryption helpers, a provider config loader that reads +from the system_settings DB table, and a startup seed function that populates +the default provider row from env vars on first boot. + +Security design (D-05, T-07-02): + HKDF domain separation — the info bytes b"ai-provider-settings" differ from + b"cloud-credentials" used by storage/cloud_utils.py. Both use the same master + key (settings.cloud_creds_key) but produce DIFFERENT derived Fernet keys, so + a leaked cloud credential cannot decrypt an AI API key and vice versa. + +AlreadyFinalized warning (RESEARCH.md Pitfall 2 / .continue-here.md anti-pattern): + The cryptography library raises AlreadyFinalized if .derive() is called twice + on the same HKDF instance. _derive_ai_settings_key() creates a FRESH HKDF(...) + object on every call — never cache or reuse the HKDF object between calls. + +Pattern reference: storage/cloud_utils.py:_derive_fernet_key(). +""" +from __future__ import annotations + +import base64 +import logging +from typing import Optional, TYPE_CHECKING + +import structlog +from cryptography.fernet import Fernet +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.kdf.hkdf import HKDF +from pydantic import BaseModel +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from config import settings + +# ProviderConfig redefined in ai/provider_config.py during Plan 02 — +# load_provider_config will re-import and return that class once Plan 02 lands. +# The stub below is removed in Plan 03 once the classifier consumes the real class. + + +logger = structlog.get_logger(__name__) + + +# ── Stub ProviderConfig for Plan 01 (replaced by ai/provider_config.py in Plan 02) ── + +class _ProviderConfigStub(BaseModel): + """Minimal provider config placeholder until Plan 02 creates ai/provider_config.py.""" + + provider_id: str + api_key: str = "" + base_url: Optional[str] = None + model: str = "" + context_chars: int = 8000 + + +# ── HKDF key derivation ─────────────────────────────────────────────────────── + +def _derive_ai_settings_key(master_key: bytes, provider_id: str) -> Fernet: + """Derive a per-provider Fernet encryption key using HKDF-SHA256. + + Security notes: + - A FRESH HKDF instance is created on every call. The cryptography library + raises AlreadyFinalized if .derive() is called twice on the same instance. + Never cache or reuse the HKDF object (RESEARCH.md Pitfall 2). + - salt = provider_id.encode("utf-8") provides per-provider derivation + (deterministic: same provider → same key for encrypt/decrypt consistency). + - info = b"ai-provider-settings" provides domain separation from + b"cloud-credentials" — same master key, different derived keys. + A leaked cloud credential cannot decrypt an AI API key (T-07-02 mitigated). + + Args: + master_key: The CLOUD_CREDS_KEY env var as bytes. + provider_id: The provider slug, e.g. "openai", "anthropic" (used as HKDF salt). + + Returns: + A Fernet instance ready for encrypt/decrypt operations. + """ + # Create a FRESH HKDF instance — never cache (AlreadyFinalized guard) + hkdf = HKDF( + algorithm=hashes.SHA256(), + length=32, + salt=provider_id.encode("utf-8"), + info=b"ai-provider-settings", # domain-separated from b"cloud-credentials" + ) + raw_key: bytes = hkdf.derive(master_key) + fernet_key = base64.urlsafe_b64encode(raw_key) + return Fernet(fernet_key) + + +# ── Encryption helpers ──────────────────────────────────────────────────────── + +def encrypt_api_key(master_key: bytes, provider_id: str, api_key: str) -> str: + """Encrypt a plaintext API key string to a Fernet token. + + The returned string is safe to store in system_settings.api_key_enc. + No JSON wrapping — the raw API key string is encrypted directly. + + Args: + master_key: The CLOUD_CREDS_KEY env var as bytes. + provider_id: The provider slug (used as HKDF salt for key derivation). + api_key: The plaintext API key, e.g. "sk-proj-...". + + Returns: + A URL-safe base64 Fernet token (str). + """ + f = _derive_ai_settings_key(master_key, provider_id) + return f.encrypt(api_key.encode("utf-8")).decode("utf-8") + + +def decrypt_api_key(master_key: bytes, provider_id: str, api_key_enc: str) -> str: + """Decrypt a Fernet token back to the original plaintext API key. + + Args: + master_key: The CLOUD_CREDS_KEY env var as bytes. + provider_id: The provider slug (used as HKDF salt for key derivation). + api_key_enc: The Fernet token string from the database. + + Returns: + The original plaintext API key string. + """ + f = _derive_ai_settings_key(master_key, provider_id) + return f.decrypt(api_key_enc.encode("utf-8")).decode("utf-8") + + +# ── Provider config loader ──────────────────────────────────────────────────── + +async def load_provider_config(session: AsyncSession) -> Optional[_ProviderConfigStub]: + """Load the active AI provider config from the system_settings table. + + Returns a _ProviderConfigStub built from the row where is_active=True, + decrypting api_key_enc when present. Returns None when no active row exists. + + Note: In Plan 02, this function will try to import and return ProviderConfig + from ai/provider_config.py instead of the stub. The import is done lazily + inside the function body so that Plan 01 does not depend on files that don't + exist yet. + + Args: + session: An open AsyncSession. + + Returns: + A _ProviderConfigStub (or the real ProviderConfig from Plan 02+) if an + active row exists; None if the table is empty or no row is marked active. + """ + # Lazy import: try to use the real ProviderConfig once Plan 02 lands + try: + from ai.provider_config import ProviderConfig as _RealProviderConfig # type: ignore[import] + config_cls = _RealProviderConfig + except ImportError: + config_cls = _ProviderConfigStub # type: ignore[assignment] + + from db.models import SystemSettings # local import to avoid circular deps + + stmt = select(SystemSettings).where(SystemSettings.is_active.is_(True)) + result = await session.execute(stmt) + row = result.scalar_one_or_none() + + if row is None: + return None + + # Decrypt API key if present + api_key = "" + if row.api_key_enc: + master_key = settings.cloud_creds_key.encode("utf-8") + try: + api_key = decrypt_api_key(master_key, row.provider_id, row.api_key_enc) + except Exception: + logger.warning( + "ai_config.load_provider_config: failed to decrypt api_key_enc", + provider_id=row.provider_id, + ) + + return config_cls( + provider_id=row.provider_id, + api_key=api_key, + base_url=row.base_url, + model=row.model_name, + context_chars=row.context_chars, + ) + + +# ── Startup seed ────────────────────────────────────────────────────────────── + +async def seed_system_settings_from_env(session: AsyncSession) -> None: + """Populate system_settings with a default provider row on first boot. + + Reads settings.default_ai_provider and settings.default_ai_model from config. + If no row exists for that provider_id, inserts one with is_active=True. + Never overwrites an existing row — idempotent across restarts (D-04). + + This function is called from the FastAPI lifespan in main.py after the + session factory is available. Caller is responsible for committing. + + Args: + session: An open AsyncSession. + """ + from db.models import SystemSettings # local import to avoid circular deps + + provider_id = settings.default_ai_provider + model_name = settings.default_ai_model + + stmt = select(SystemSettings).where(SystemSettings.provider_id == provider_id) + result = await session.execute(stmt) + existing = result.scalar_one_or_none() + + if existing is not None: + # Row already exists — never overwrite (idempotent) + return + + # Insert default row with no API key (local providers like Ollama don't need one) + row = SystemSettings( + provider_id=provider_id, + model_name=model_name, + context_chars=8000, + is_active=True, + api_key_enc=None, + base_url=None, + ) + session.add(row) + logger.info( + "ai_config.seed_system_settings_from_env: seeded default provider", + provider_id=provider_id, + model_name=model_name, + ) From 7986661333136a9c92f18503b9a528d28b582467 Mon Sep 17 00:00:00 2001 From: curo1305 Date: Thu, 4 Jun 2026 18:51:12 +0200 Subject: [PATCH 4/4] =?UTF-8?q?docs(07-01):=20complete=20plan=20summary=20?= =?UTF-8?q?=E2=80=94=20system=5Fsettings=20DB=20+=20HKDF=20+=20Wave=200=20?= =?UTF-8?q?stubs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tasks: 3/3 complete. Commits: 4febe2f, 4eb3177, 0fd6930. Created system_settings table (Alembic 0005), SystemSettings ORM, HKDF AI key encryption (info=b"ai-provider-settings"), load_provider_config(), seed startup hook, and 13 Wave 0 xfail stubs across 4 new test files. --- .../07-01-SUMMARY.md | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 .planning/phases/07-redo-and-optimize-llm-integration/07-01-SUMMARY.md diff --git a/.planning/phases/07-redo-and-optimize-llm-integration/07-01-SUMMARY.md b/.planning/phases/07-redo-and-optimize-llm-integration/07-01-SUMMARY.md new file mode 100644 index 0000000..d15ae8b --- /dev/null +++ b/.planning/phases/07-redo-and-optimize-llm-integration/07-01-SUMMARY.md @@ -0,0 +1,155 @@ +--- +phase: 07-redo-and-optimize-llm-integration +plan: "01" +subsystem: backend/ai-config +tags: + - database + - encryption + - hkdf + - testing + - wave-0-scaffold +dependency_graph: + requires: + - "06-05 (trusted-proxy, per-account rate limiting)" + provides: + - "system_settings table (Alembic 0005)" + - "SystemSettings ORM model" + - "HKDF encryption helpers for AI API keys" + - "load_provider_config() DB reader" + - "seed_system_settings_from_env() startup hook" + - "Wave 0 xfail test stubs (13 new stubs)" + affects: + - "07-02 (ProviderConfig + GenericOpenAIProvider — depends on system_settings table)" + - "07-03 (Anthropic output_config — depends on load_provider_config)" + - "07-05 (Admin AI panel — depends on system_settings table)" +tech_stack: + added: [] + patterns: + - "HKDF-SHA256 with info=b\"ai-provider-settings\" (domain-separated from b\"cloud-credentials\")" + - "Fresh HKDF instance per call (AlreadyFinalized guard)" + - "Fernet symmetric encryption for API keys (no JSON wrapping)" + - "Lazy import of ProviderConfig inside load_provider_config() to avoid Plan 01/02 circular dep" + - "Try/except lifespan seed to survive pre-migration fresh container startup" +key_files: + created: + - backend/migrations/versions/0005_system_settings.py + - backend/services/ai_config.py + - backend/tests/test_ai_providers.py + - backend/tests/test_ai_config.py + - backend/tests/test_admin_ai_config.py + - backend/tests/test_document_tasks.py + modified: + - backend/db/models.py + - backend/main.py + - backend/tests/test_documents.py +decisions: + - "HKDF info=b\"ai-provider-settings\" enforces domain separation from cloud-credentials — same master key produces different derived Fernet keys" + - "Fresh HKDF instance per _derive_ai_settings_key() call (cryptography AlreadyFinalized invariant)" + - "_ProviderConfigStub introduced as stub until Plan 02 creates ai/provider_config.py; removed in Plan 03" + - "seed_system_settings_from_env wrapped in try/except in lifespan — pre-migration container startup must not crash" + - "UniqueConstraint uq_system_settings_provider_id on provider_id enforces one row per provider" + - "D-14 regression guard: docker-compose.yml retains 3 host-gateway entries (backend, celery-worker, celery-worker-two); all >= required 2" +metrics: + duration: "~25 minutes" + completed: "2026-06-04" + tasks_completed: 3 + tasks_total: 3 + files_created: 6 + files_modified: 3 +--- + +# Phase 7 Plan 01: Database, Encryption, and Test Scaffolding Foundation Summary + +HKDF/Fernet encryption layer for AI provider API keys stored in a new `system_settings` DB table; Wave 0 xfail test stubs providing the full Phase 7 testing skeleton; startup seed from env vars. + +## Tasks Completed + +| Task | Description | Commit | Files | +|------|-------------|--------|-------| +| 1 | Wave 0 xfail stubs — 13 new stubs across 4 new files + 1 append | 4febe2f | test_ai_providers.py, test_ai_config.py, test_admin_ai_config.py, test_document_tasks.py, test_documents.py | +| 2 | Alembic migration 0005 + SystemSettings ORM model | 4eb3177 | migrations/versions/0005_system_settings.py, db/models.py | +| 3 | services/ai_config.py — HKDF helpers, loader, env seed + main.py wiring | 0fd6930 | services/ai_config.py, main.py | + +## What Was Built + +### Task 1: Wave 0 Test Scaffold + +Created four new test files and appended one stub to an existing file: + +- `test_ai_providers.py` — 7 stubs: test_generic_openai_json_mode, test_anthropic_structured_output, test_get_provider_typed, test_client_singleton, test_context_chars_truncation, test_smart_truncation, test_gemini_fallback_to_parse_classification +- `test_ai_config.py` — 2 stubs: test_load_provider_config, test_api_key_encrypt_decrypt +- `test_admin_ai_config.py` — 2 stubs: test_get_never_returns_key, test_put_writes_active_provider +- `test_document_tasks.py` — 2 stubs: test_retry_backoff, test_exhaustion_sets_failed_status +- `test_documents.py` — 1 appended stub: test_reclassify_requeues_celery + +All stubs use `@pytest.mark.xfail(strict=False, reason="Wave 0 stub — promoted in Plan 07-XX")` per the xfail(strict=False) STATE.md decision. Bodies are single `pytest.xfail()` calls only. + +D-14 regression guard verified: `docker-compose.yml` contains 3 `host.docker.internal:host-gateway` entries (backend + celery-worker + celery-worker-two), satisfying the minimum-2 requirement. + +### Task 2: Alembic Migration 0005 + SystemSettings ORM + +`backend/migrations/versions/0005_system_settings.py` creates the `system_settings` table with: +- `id`: UUID PK with `gen_random_uuid()` server default +- `provider_id`: String NOT NULL UNIQUE (uq_system_settings_provider_id constraint) +- `api_key_enc`: Text NULL (Fernet-encrypted; NULL for local providers like Ollama) +- `base_url`: Text NULL +- `model_name`: Text NOT NULL, default '' +- `context_chars`: Integer NOT NULL, default 8000 +- `is_active`: Boolean NOT NULL, default false +- `created_at` / `updated_at`: TIMESTAMPTZ NOT NULL, default now() + +`backend/db/models.py` gains the `SystemSettings(Base)` ORM model with identical column declarations using `Mapped[]` type syntax, mirroring the CloudConnection pattern. + +### Task 3: services/ai_config.py + +Complete encryption + loader + seed module: + +- `_derive_ai_settings_key(master_key, provider_id)`: Creates FRESH HKDF on every call (AlreadyFinalized guard), `salt=provider_id.encode("utf-8")`, `info=b"ai-provider-settings"` (domain-separated from `b"cloud-credentials"`) +- `encrypt_api_key(master_key, provider_id, api_key)`: `Fernet.encrypt(api_key.encode()).decode()` — no JSON wrapping +- `decrypt_api_key(master_key, provider_id, api_key_enc)`: `Fernet.decrypt(...).decode()` — no JSON unwrapping +- `load_provider_config(session)`: Reads `is_active=True` row, decrypts API key, returns `_ProviderConfigStub` (upgraded to real `ProviderConfig` lazily once Plan 02 lands) +- `seed_system_settings_from_env(session)`: Inserts default provider row from `settings.default_ai_provider / default_ai_model` when no row exists for that provider_id; idempotent +- `_ProviderConfigStub`: Minimal Pydantic model stub; removed in Plan 03 + +`backend/main.py` updated to import and call `seed_system_settings_from_env` inside the async lifespan with `try/except` so that a missing table (pre-migration fresh container) logs a warning and skips rather than crashing. + +## Verification Results + +- Round-trip smoke test: `encrypt_api_key(mk, 'openai', 'sk-test')` → `decrypt_api_key(mk, 'openai', ct) == 'sk-test'` — PASS +- Domain salt isolation: ciphertext for `provider_id="openai"` vs `provider_id="anthropic"` differs — PASS +- AlreadyFinalized guard: `_derive_ai_settings_key()` called twice without error — PASS +- Domain separation: `info=b"ai-provider-settings"` present; `info=b"cloud-credentials"` absent from ai_config.py — PASS +- `SystemSettings.__table_args__` contains `UniqueConstraint(name="uq_system_settings_provider_id")` — PASS +- `from db.models import SystemSettings` exits 0 — PASS +- Full suite: 1 failed (pre-existing test_extract_docx), 357 passed, 21 xfailed — PASS (no new failures) + +## Deviations from Plan + +None — plan executed exactly as written. + +## Known Stubs + +- `_ProviderConfigStub` in `services/ai_config.py` lines 44-50: intentional temporary stub; Plan 02 creates `ai/provider_config.py` and `load_provider_config()` will import `ProviderConfig` from there via lazy import. + +## Threat Flags + +No new threat surface introduced beyond what is described in the plan's ``. The `system_settings` table is accessed only by the `seed_system_settings_from_env` startup hook and the `load_provider_config` service function. No new API endpoints or network-accessible paths are added in this plan. + +## Self-Check: PASSED + +Files created/modified: + +- [x] backend/migrations/versions/0005_system_settings.py — FOUND +- [x] backend/db/models.py — FOUND (SystemSettings class present) +- [x] backend/services/ai_config.py — FOUND +- [x] backend/main.py — FOUND (seed call present) +- [x] backend/tests/test_ai_providers.py — FOUND (7 xfail stubs) +- [x] backend/tests/test_ai_config.py — FOUND (2 xfail stubs) +- [x] backend/tests/test_admin_ai_config.py — FOUND (2 xfail stubs) +- [x] backend/tests/test_document_tasks.py — FOUND (2 xfail stubs) +- [x] backend/tests/test_documents.py — FOUND (test_reclassify_requeues_celery appended) + +Commits: +- [x] 4febe2f — test(07-01): Wave 0 xfail stubs +- [x] 4eb3177 — feat(07-01): Alembic migration + SystemSettings ORM +- [x] 0fd6930 — feat(07-01): services/ai_config.py + main.py wiring