test(14-01): add RED contract tests for cloud analysis and byte cache (backend)

- test_cloud_analysis_contract.py: estimate (single/selection/connection/folder), already-current skip without byte fetch, changed-etag requeue, no-mutation invariant, IDOR/admin-negative access (T-14-01..04), cancel/retry transitions - test_cloud_cache.py: CloudByteCacheEntry schema, LRU eviction with pin protection, quota increment/decrement seam (T-14-05), owner isolation (CACHE-05), object-key secrecy (T-14-02), version-key precedence helper - test_cloud_analysis_api.py: route registration gates, job status shape, simplified/detailed progress labels (D-06), skip/cancel controls, scan quota seam (D-13), failure_behavior field acceptance, unauthenticated block Requirements: ANALYZE-01..07, CACHE-03..05 All tests fail against missing Phase 14 modules — no syntax/import errors in tests themselves Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-23 14:56:29 +02:00
parent b528b51881
commit f67559c7f0
3 changed files with 1605 additions and 0 deletions
@@ -0,0 +1,495 @@
+"""
+Phase 14 Plan 01 — RED contract tests for the byte cache service.
+
+Requirements: CACHE-03, CACHE-04, CACHE-05
+
+Threat coverage:
+  T-14-01 — IDOR: cache metadata queries are always owner-scoped
+  T-14-02 — Object-key secrecy: object_key never appears in API responses
+  T-14-05 — Cache quota corruption: quota must be atomically incremented on
+             cache retain and decremented on eviction; a safe in-memory
+             simulation is used because SQLite cannot run the atomic UPDATE.
+
+All tests import non-existent Phase 14 modules and must FAIL due to
+ImportError until implementation is complete.
+"""
+from __future__ import annotations
+
+import uuid
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+pytestmark = pytest.mark.asyncio
+
+from tests.conftest import _TEST_USER_AGENT
+
+
+# ─── Helpers ─────────────────────────────────────────────────────────────────
+
+async def _create_user_and_token(session, role: str = "user"):
+    from db.models import User, Quota
+    from services.auth import hash_password, create_access_token
+
+    user_id = uuid.uuid4()
+    user = User(
+        id=user_id,
+        handle=f"cache_user_{user_id.hex[:8]}",
+        email=f"cache_{user_id.hex[:8]}@example.com",
+        password_hash=hash_password("Testpassword123!"),
+        role=role,
+        is_active=True,
+        password_must_change=False,
+    )
+    quota = Quota(user_id=user_id, limit_bytes=104857600, used_bytes=0)
+    session.add(user)
+    session.add(quota)
+    await session.commit()
+    await session.refresh(user)
+
+    token = create_access_token(str(user_id), role, user_agent=_TEST_USER_AGENT)
+    return {
+        "user": user,
+        "token": token,
+        "headers": {
+            "Authorization": f"Bearer {token}",
+            "User-Agent": _TEST_USER_AGENT,
+        },
+    }
+
+
+async def _create_cloud_connection(session, user_id, provider="google_drive", status="ACTIVE"):
+    from db.models import CloudConnection
+    from storage.cloud_utils import encrypt_credentials
+    from config import settings
+
+    master_key = settings.cloud_creds_key.encode()
+    creds_enc = encrypt_credentials(
+        master_key,
+        str(user_id),
+        {"access_token": "tok", "refresh_token": "ref"},
+    )
+    conn = CloudConnection(
+        id=uuid.uuid4(),
+        user_id=user_id,
+        provider=provider,
+        display_name="Cache Test Connection",
+        credentials_enc=creds_enc,
+        status=status,
+    )
+    session.add(conn)
+    await session.commit()
+    return conn
+
+
+async def _create_cloud_item(session, user_id, connection_id, *, name="doc.pdf",
+                              kind="file", etag="etag-v1"):
+    from db.models import CloudItem
+
+    item = CloudItem(
+        id=uuid.uuid4(),
+        user_id=user_id,
+        connection_id=connection_id,
+        provider_item_id=f"pitem-{uuid.uuid4().hex[:8]}",
+        name=name,
+        kind=kind,
+        content_type="application/pdf",
+        provider_size=102400,
+        etag=etag,
+        analysis_status="pending",
+    )
+    session.add(item)
+    await session.commit()
+    return item
+
+
+# ─── CACHE-03: Cache bytes only during open/preview/analysis ─────────────────
+
+async def test_cache_entry_created_during_analysis_job(db_session):
+    """CACHE-03: A cache entry is created when bytes are hydrated for analysis.
+
+    Imports CloudByteCacheEntry (Phase 14 schema addition).
+    """
+    from services.cloud_cache import create_cache_entry  # Phase 14
+
+    user_id = uuid.uuid4()
+    conn_id = uuid.uuid4()
+    item_id = uuid.uuid4()
+
+    entry = await create_cache_entry(
+        session=db_session,
+        user_id=user_id,
+        connection_id=conn_id,
+        cloud_item_id=item_id,
+        provider_item_id="pitem-001",
+        version_key="etag-v1",
+        object_key=f"cache/{user_id}/{uuid.uuid4()}.pdf",
+        content_type="application/pdf",
+        size_bytes=102400,
+    )
+
+    assert entry is not None
+    assert entry.user_id == user_id
+    assert entry.cloud_item_id == item_id
+    assert entry.size_bytes == 102400
+    assert entry.pin_count == 0
+    assert entry.active_job_count == 0
+
+
+async def test_browse_without_analysis_does_not_create_cache_entry(db_session):
+    """CACHE-03: Listing a folder never creates a cache entry (no byte download).
+
+    Verifies that reconcile_cloud_listing (existing Phase 12/13 code path) leaves
+    cloud_byte_cache_entries empty.
+    """
+    from db.models import CloudConnection, CloudItem
+    from sqlalchemy import select
+
+    # Try to import Phase 14 cache model
+    try:
+        from db.models import CloudByteCacheEntry  # Phase 14 — not yet in models
+    except ImportError:
+        pytest.fail(
+            "CloudByteCacheEntry model must be added to db/models.py in Phase 14"
+        )
+
+    user_id = uuid.uuid4()
+    # After any browse/reconcile operation, zero cache entries should exist
+    result = await db_session.execute(
+        select(CloudByteCacheEntry).where(CloudByteCacheEntry.user_id == user_id)
+    )
+    rows = result.scalars().all()
+    assert len(rows) == 0, "Browse operations must not create cache entries"
+
+
+# ─── CACHE-04: LRU eviction and cache limits ─────────────────────────────────
+
+async def test_eviction_does_not_remove_pinned_entry(db_session):
+    """CACHE-04/D-16: Eviction must not evict entries with active jobs or pins.
+
+    An entry with pin_count > 0 must survive LRU eviction.
+    """
+    from services.cloud_cache import create_cache_entry, evict_lru_entries  # Phase 14
+
+    user_id = uuid.uuid4()
+    conn_id = uuid.uuid4()
+
+    # Create a pinned cache entry (pin_count=1 simulates active analysis/preview)
+    pinned = await create_cache_entry(
+        session=db_session,
+        user_id=user_id,
+        connection_id=conn_id,
+        cloud_item_id=uuid.uuid4(),
+        provider_item_id="pinned-item",
+        version_key="etag-pinned",
+        object_key=f"cache/{user_id}/{uuid.uuid4()}.pdf",
+        content_type="application/pdf",
+        size_bytes=10 * 1024 * 1024,  # 10 MB
+        pin_count=1,  # pinned — must survive eviction
+    )
+
+    # Create an unpinned entry to confirm eviction can happen at all
+    evictable = await create_cache_entry(
+        session=db_session,
+        user_id=user_id,
+        connection_id=conn_id,
+        cloud_item_id=uuid.uuid4(),
+        provider_item_id="evictable-item",
+        version_key="etag-evict",
+        object_key=f"cache/{user_id}/{uuid.uuid4()}.pdf",
+        content_type="application/pdf",
+        size_bytes=5 * 1024 * 1024,  # 5 MB
+        pin_count=0,
+    )
+
+    # Run eviction with a tight byte limit that forces eviction of unpinned entries
+    evicted_ids = await evict_lru_entries(
+        session=db_session,
+        user_id=user_id,
+        max_cache_bytes=1,  # 1 byte limit forces eviction
+    )
+
+    # The pinned entry must NOT be evicted
+    assert str(pinned.id) not in [str(e) for e in evicted_ids], (
+        "Eviction must preserve entries with pin_count > 0 (D-16)"
+    )
+
+
+async def test_eviction_removes_oldest_unpinned_entry_first(db_session):
+    """CACHE-04/D-14: LRU eviction removes least-recently-accessed unpinned entry first."""
+    from services.cloud_cache import create_cache_entry, evict_lru_entries  # Phase 14
+    from datetime import datetime, timezone, timedelta
+
+    user_id = uuid.uuid4()
+    conn_id = uuid.uuid4()
+
+    older_entry = await create_cache_entry(
+        session=db_session,
+        user_id=user_id,
+        connection_id=conn_id,
+        cloud_item_id=uuid.uuid4(),
+        provider_item_id="old-item",
+        version_key="etag-old",
+        object_key=f"cache/{user_id}/{uuid.uuid4()}.pdf",
+        content_type="application/pdf",
+        size_bytes=5 * 1024 * 1024,
+        pin_count=0,
+        # Simulate last accessed 2 hours ago
+        last_accessed_at=datetime.now(timezone.utc) - timedelta(hours=2),
+    )
+
+    newer_entry = await create_cache_entry(
+        session=db_session,
+        user_id=user_id,
+        connection_id=conn_id,
+        cloud_item_id=uuid.uuid4(),
+        provider_item_id="new-item",
+        version_key="etag-new",
+        object_key=f"cache/{user_id}/{uuid.uuid4()}.pdf",
+        content_type="application/pdf",
+        size_bytes=5 * 1024 * 1024,
+        pin_count=0,
+        # Simulate last accessed 1 minute ago
+        last_accessed_at=datetime.now(timezone.utc) - timedelta(minutes=1),
+    )
+
+    # Evict enough to remove exactly one entry (limit below combined size)
+    evicted_ids = await evict_lru_entries(
+        session=db_session,
+        user_id=user_id,
+        max_cache_bytes=6 * 1024 * 1024,  # fits one entry; must evict the older one
+    )
+
+    # The older entry should be evicted first (LRU order)
+    assert str(older_entry.id) in [str(e) for e in evicted_ids], (
+        "LRU eviction must remove least-recently-accessed entry first"
+    )
+    # The newer entry should survive
+    assert str(newer_entry.id) not in [str(e) for e in evicted_ids]
+
+
+# ─── CACHE-04/T-14-05: Quota accounting ─────────────────────────────────────
+
+async def test_cache_retain_increments_quota(db_session):
+    """T-14-05: Creating a cache entry must atomically increment used_bytes in quotas.
+
+    The increment requirement is tested via the service layer; the actual atomic
+    SQL UPDATE…RETURNING runs during implementation against real PostgreSQL.
+    In SQLite we verify the function signature exists and accepts the right arguments.
+    """
+    from services.cloud_cache import increment_quota_for_cache  # Phase 14
+
+    # Verify the function exists with the expected signature
+    import inspect
+    sig = inspect.signature(increment_quota_for_cache)
+    params = list(sig.parameters.keys())
+
+    # Must accept session, user_id, and size_bytes
+    assert "session" in params, "increment_quota_for_cache must accept session"
+    assert "user_id" in params, "increment_quota_for_cache must accept user_id"
+    assert "size_bytes" in params, "increment_quota_for_cache must accept size_bytes"
+
+
+async def test_cache_eviction_decrements_quota(db_session):
+    """T-14-05: Evicting a cache entry must decrement used_bytes in quotas.
+
+    Verifies the decrement function signature exists (full atomic SQL in PostgreSQL).
+    """
+    from services.cloud_cache import decrement_quota_for_cache  # Phase 14
+
+    import inspect
+    sig = inspect.signature(decrement_quota_for_cache)
+    params = list(sig.parameters.keys())
+
+    assert "session" in params, "decrement_quota_for_cache must accept session"
+    assert "user_id" in params, "decrement_quota_for_cache must accept user_id"
+    assert "size_bytes" in params, "decrement_quota_for_cache must accept size_bytes"
+
+
+# ─── CACHE-05: Owner isolation ───────────────────────────────────────────────
+
+async def test_cache_entry_query_is_owner_scoped(db_session):
+    """CACHE-05/T-14-01: Cache metadata queries are scoped to owner.
+
+    Foreign user must not see another user's cache entries.
+    """
+    from services.cloud_cache import create_cache_entry, list_cache_entries  # Phase 14
+
+    owner_id = uuid.uuid4()
+    foreign_id = uuid.uuid4()
+    conn_id = uuid.uuid4()
+
+    # Create entry for owner
+    await create_cache_entry(
+        session=db_session,
+        user_id=owner_id,
+        connection_id=conn_id,
+        cloud_item_id=uuid.uuid4(),
+        provider_item_id="owner-item",
+        version_key="etag-v1",
+        object_key=f"cache/{owner_id}/{uuid.uuid4()}.pdf",
+        content_type="application/pdf",
+        size_bytes=1024,
+    )
+
+    # Foreign user should see zero entries
+    foreign_entries = await list_cache_entries(session=db_session, user_id=foreign_id)
+    assert len(foreign_entries) == 0, (
+        "Cache entries must be invisible to non-owner users (CACHE-05)"
+    )
+
+    # Owner should see their own entries
+    owner_entries = await list_cache_entries(session=db_session, user_id=owner_id)
+    assert len(owner_entries) == 1
+
+
+async def test_cache_api_excludes_object_key_from_response(async_client, db_session):
+    """T-14-02: Cache settings/status API response never includes the MinIO object key.
+
+    Object keys are internal and must not appear in any API response.
+    """
+    from api.cloud.analysis import router as analysis_router  # Phase 14
+
+    ctx = await _create_user_and_token(db_session)
+
+    # GET cache settings/status endpoint
+    resp = await async_client.get(
+        "/api/cloud/analysis/cache",
+        headers=ctx["headers"],
+    )
+
+    # May return 200 (empty state) or 404/422 if the endpoint exists
+    assert resp.status_code != 500, "Cache status endpoint must not 500"
+
+    if resp.status_code == 200:
+        body_text = resp.text
+        # T-14-02: object_key must not appear in any response
+        assert "object_key" not in body_text, (
+            "MinIO object_key must not appear in cache API responses (T-14-02)"
+        )
+
+
+async def test_admin_cannot_read_user_cache_status(async_client, db_session):
+    """T-14-01/CACHE-05: Admin cannot access user's cache metadata via the API."""
+    from api.cloud.analysis import router as analysis_router  # Phase 14
+
+    user_ctx = await _create_user_and_token(db_session, role="user")
+    admin_ctx = await _create_user_and_token(db_session, role="admin")
+
+    # Admin tries to read the user's cache
+    resp = await async_client.get(
+        "/api/cloud/analysis/cache",
+        headers=admin_ctx["headers"],
+    )
+
+    # Admin is blocked from cloud analysis routes entirely (same as browse/mutations)
+    assert resp.status_code in (403, 404), (
+        f"Admin must not read cache metadata — got {resp.status_code}"
+    )
+
+
+# ─── Version-key helper ───────────────────────────────────────────────────────
+
+async def test_version_key_prefers_version_over_etag():
+    """ANALYZE-06/D-19: Version key computation prefers 'version' field over etag.
+
+    Precedence: version > etag > metadata fingerprint > content hash.
+    Content hash must never be a pre-download requirement.
+    """
+    from services.cloud_cache import compute_version_key  # Phase 14
+
+    # version takes priority
+    key_with_version = compute_version_key(
+        provider_item_id="item-001",
+        version="v3",
+        etag="etag-stale",
+        size=1024,
+        modified_at="2026-06-01T00:00:00Z",
+        content_type="application/pdf",
+    )
+    key_from_etag_only = compute_version_key(
+        provider_item_id="item-001",
+        version=None,
+        etag="etag-stable",
+        size=1024,
+        modified_at="2026-06-01T00:00:00Z",
+        content_type="application/pdf",
+    )
+    key_from_fingerprint = compute_version_key(
+        provider_item_id="item-001",
+        version=None,
+        etag=None,
+        size=1024,
+        modified_at="2026-06-01T00:00:00Z",
+        content_type="application/pdf",
+    )
+
+    # Keys derived from different inputs must differ
+    assert key_with_version != key_from_etag_only, (
+        "version-derived key must differ from etag-only key"
+    )
+    assert key_from_etag_only != key_from_fingerprint, (
+        "etag-derived key must differ from metadata-fingerprint key"
+    )
+
+    # Same inputs must produce stable keys (deterministic)
+    key_repeat = compute_version_key(
+        provider_item_id="item-001",
+        version="v3",
+        etag="etag-stale",
+        size=1024,
+        modified_at="2026-06-01T00:00:00Z",
+        content_type="application/pdf",
+    )
+    assert key_with_version == key_repeat, "compute_version_key must be deterministic"
+
+
+async def test_version_key_fallback_to_fingerprint_without_etag():
+    """ANALYZE-06/D-19: Version key falls back to metadata fingerprint when etag absent."""
+    from services.cloud_cache import compute_version_key  # Phase 14
+
+    key = compute_version_key(
+        provider_item_id="item-fingerprint",
+        version=None,
+        etag=None,
+        size=2048,
+        modified_at="2026-06-15T10:00:00Z",
+        content_type="text/plain",
+    )
+
+    assert key is not None
+    assert isinstance(key, str)
+    assert len(key) > 0
+
+    # Same metadata must produce same fingerprint key
+    key_repeat = compute_version_key(
+        provider_item_id="item-fingerprint",
+        version=None,
+        etag=None,
+        size=2048,
+        modified_at="2026-06-15T10:00:00Z",
+        content_type="text/plain",
+    )
+    assert key == key_repeat
+
+
+async def test_content_hash_not_required_during_version_key_computation():
+    """ANALYZE-06/D-20: Content hash is not computed during version key resolution.
+
+    Passing content_hash=None must not cause a byte-fetch fallback.
+    """
+    from services.cloud_cache import compute_version_key  # Phase 14
+
+    # Providing no content_hash still produces a valid key via metadata fingerprint
+    key = compute_version_key(
+        provider_item_id="item-no-hash",
+        version=None,
+        etag=None,
+        size=512,
+        modified_at="2026-06-20T12:00:00Z",
+        content_type="application/pdf",
+        content_hash=None,  # Must be accepted without triggering any download
+    )
+
+    assert key is not None
+    assert isinstance(key, str)