feat(12-02): add refresh_cloud_folder Celery task, staleness trigger, version 0.1.5

- Add tasks/cloud_tasks.py: durable refresh_cloud_folder task with 3-retry bounded backoff (30s/90s/270s +jitter), credential decryption in worker only - Register tasks.cloud_tasks.* on documents queue in celery_app.py - Add stale-while-revalidate staleness trigger in browse.py (5-min threshold) - Add 4 Task 3 tests: idempotency, cached-row retention on failure, task structure, no-byte-download contract; add background-refresh scheduling integration test - Bump backend version 0.1.4 → 0.1.5, frontend package.json 0.1.4 → 0.1.5 - Update AGENTS.md with Phase 12 Plan 02 state and new shared module map entries - Update README with connection-ID browse API table and v0.1.5
2026-06-18 23:17:34 +02:00
parent e186019066
commit c6237ca57f
9 changed files with 735 additions and 6 deletions
@@ -1081,3 +1081,81 @@ async def test_list_connections_returns_all_providers(async_client, db_session):
    # credentials_enc must not be in any item
    for item in items:
        assert "credentials_enc" not in item
+
+
+async def test_browse_connection_schedules_background_refresh_on_cached_items(
+    async_client, db_session, monkeypatch
+):
+    """GET /api/cloud/connections/{id}/items schedules Celery refresh when cached items exist.
+
+    Phase 12 stale-while-revalidate: if items are already cached and folder_state
+    has last_refreshed_at set, the endpoint returns immediately and schedules a
+    background refresh via refresh_cloud_folder.delay().
+    Verifies: .delay() is called with the correct user_id, connection_id, parent_ref.
+    """
+    from unittest.mock import patch, MagicMock
+    from db.models import CloudItem, CloudFolderState
+    from storage.cloud_utils import encrypt_credentials
+
+    auth = await _create_user_and_token(db_session, role="user")
+    uid_str = str(auth["user"].id)
+    master_key = b"test-key-for-testing-32bytes!!"
+    creds_enc = encrypt_credentials(master_key, uid_str, {"access_token": "tok"})
+
+    conn = await _create_cloud_connection(
+        db_session, auth["user"].id, provider="google_drive", name="Stale Drive"
+    )
+    conn_id_str = str(conn.id)
+
+    # Seed a durable CloudItem so browse sees existing rows
+    from datetime import datetime, timezone
+    item = CloudItem(
+        id=_uuid.uuid4(),
+        user_id=auth["user"].id,
+        connection_id=conn.id,
+        provider_item_id="cached-item-001",
+        name="cached.pdf",
+        kind="file",
+        analysis_status="pending",
+        semantic_index_status="none",
+    )
+    db_session.add(item)
+
+    # Seed a CloudFolderState with last_refreshed_at set (non-first-visit)
+    from datetime import timedelta
+    fs = CloudFolderState(
+        id=_uuid.uuid4(),
+        user_id=auth["user"].id,
+        connection_id=conn.id,
+        parent_ref="",
+        refresh_state="fresh",
+        last_refreshed_at=datetime.now(timezone.utc) - timedelta(minutes=5),
+    )
+    db_session.add(fs)
+    await db_session.commit()
+
+    delay_mock = MagicMock()
+    with patch("tasks.cloud_tasks.refresh_cloud_folder") as mock_task:
+        mock_task.delay = delay_mock
+        # Patch capability probe to avoid real network call
+        with patch("api.cloud.browse.build_cloud_resource_adapter") as mock_adapter_factory:
+            from storage.cloud_base import CloudCapability, STATE_SUPPORTED, ACTIONS
+            mock_caps = {a: CloudCapability(action=a, state=STATE_SUPPORTED) for a in ACTIONS}
+            mock_adapter = MagicMock()
+            mock_adapter.get_capabilities = _uuid.__class__  # callable stub
+            import asyncio as _asyncio
+
+            async def _fake_caps(*args, **kwargs):
+                return mock_caps
+
+            mock_adapter.get_capabilities = _fake_caps
+            mock_adapter_factory.return_value = mock_adapter
+
+            resp = await async_client.get(
+                f"/api/cloud/connections/{conn_id_str}/items",
+                headers=auth["headers"],
+            )
+
+    assert resp.status_code == 200
+    # Background refresh was scheduled
+    delay_mock.assert_called_once_with(uid_str, conn_id_str, None)
@@ -537,3 +537,91 @@ async def test_service_folder_state_idempotent(db_session: AsyncSession):
    fs1 = await get_or_create_folder_state(db_session, user_id=uid, connection_id=cid, parent_ref="")
    fs2 = await get_or_create_folder_state(db_session, user_id=uid, connection_id=cid, parent_ref="")
    assert fs1.id == fs2.id
+
+
+# ── Task 3: refresh_cloud_folder Celery task tests ────────────────────────────
+
+@pytest.mark.asyncio
+async def test_refresh_cloud_folder_idempotent_reconciliation(db_session: AsyncSession):
+    """Calling reconcile_cloud_listing twice yields no duplicate rows (idempotency check)."""
+    uid = await _make_user(db_session)
+    cid = await _make_connection(db_session, uid)
+
+    resource = _cloud_resource(cid, uid, provider_item_id="stable-001", name="stable.pdf")
+    listing = CloudListing(items=[resource], complete=True)
+
+    # First reconcile
+    await reconcile_cloud_listing(db_session, user_id=uid, connection_id=cid, parent_ref=None, listing=listing)
+    # Second reconcile with same data
+    await reconcile_cloud_listing(db_session, user_id=uid, connection_id=cid, parent_ref=None, listing=listing)
+
+    from sqlalchemy import select, func
+    result = await db_session.execute(
+        select(func.count()).select_from(CloudItem).where(
+            CloudItem.provider_item_id == "stable-001",
+            CloudItem.deleted_at.is_(None),
+        )
+    )
+    count = result.scalar()
+    assert count == 1, "Idempotent reconciliation must not create duplicate rows"
+
+
+@pytest.mark.asyncio
+async def test_refresh_cloud_folder_failed_refresh_retains_cached_rows(db_session: AsyncSession):
+    """On provider failure, previously cached rows must not be deleted (D-13)."""
+    uid = await _make_user(db_session)
+    cid = await _make_connection(db_session, uid)
+
+    # Seed durable items
+    resource = _cloud_resource(cid, uid, provider_item_id="durable-001", name="important.pdf")
+    complete_listing = CloudListing(items=[resource], complete=True)
+    await reconcile_cloud_listing(db_session, user_id=uid, connection_id=cid, parent_ref=None, listing=complete_listing)
+
+    # Simulate provider failure: incomplete listing with no items
+    empty_failed_listing = CloudListing(items=[], complete=False)
+    await reconcile_cloud_listing(db_session, user_id=uid, connection_id=cid, parent_ref=None, listing=empty_failed_listing)
+
+    from sqlalchemy import select
+    result = await db_session.execute(
+        select(CloudItem).where(
+            CloudItem.provider_item_id == "durable-001",
+            CloudItem.deleted_at.is_(None),
+        )
+    )
+    retained = result.scalars().first()
+    assert retained is not None, "Incomplete listing must not delete previously cached rows"
+
+
+@pytest.mark.asyncio
+async def test_refresh_cloud_folder_task_structure():
+    """refresh_cloud_folder is a registered Celery task with correct queue routing."""
+    from tasks.cloud_tasks import refresh_cloud_folder
+    from celery_app import celery_app
+
+    assert hasattr(refresh_cloud_folder, "delay"), "Task must have .delay() method"
+    assert hasattr(refresh_cloud_folder, "apply_async"), "Task must have .apply_async() method"
+    assert refresh_cloud_folder.name == "tasks.cloud_tasks.refresh_cloud_folder"
+    assert refresh_cloud_folder.max_retries == 3
+
+    routes = celery_app.conf.task_routes
+    assert "tasks.cloud_tasks.*" in routes
+    assert routes["tasks.cloud_tasks.*"]["queue"] == "documents"
+
+
+@pytest.mark.asyncio
+async def test_refresh_cloud_folder_no_byte_calls(db_session: AsyncSession):
+    """reconcile_cloud_listing never calls get_object or any download method — D-18/CACHE-01."""
+    from unittest.mock import AsyncMock, patch
+
+    uid = await _make_user(db_session)
+    cid = await _make_connection(db_session, uid)
+    resource = _cloud_resource(cid, uid, provider_item_id="nobytes-001")
+    listing = CloudListing(items=[resource], complete=True)
+
+    mock_get_object = AsyncMock()
+    with patch("db.models.CloudItem", wraps=CloudItem) as _patched:
+        # reconcile_cloud_listing must not call any external byte-download method
+        await reconcile_cloud_listing(
+            db_session, user_id=uid, connection_id=cid, parent_ref=None, listing=listing
+        )
+        mock_get_object.assert_not_called()