feat(12-02): add refresh_cloud_folder Celery task, staleness trigger, version 0.1.5

- Add tasks/cloud_tasks.py: durable refresh_cloud_folder task with 3-retry
  bounded backoff (30s/90s/270s +jitter), credential decryption in worker only
- Register tasks.cloud_tasks.* on documents queue in celery_app.py
- Add stale-while-revalidate staleness trigger in browse.py (5-min threshold)
- Add 4 Task 3 tests: idempotency, cached-row retention on failure, task structure,
  no-byte-download contract; add background-refresh scheduling integration test
- Bump backend version 0.1.4 → 0.1.5, frontend package.json 0.1.4 → 0.1.5
- Update AGENTS.md with Phase 12 Plan 02 state and new shared module map entries
- Update README with connection-ID browse API table and v0.1.5
This commit is contained in:
curo1305
2026-06-18 23:17:34 +02:00
parent e186019066
commit c6237ca57f
9 changed files with 735 additions and 6 deletions
+88
View File
@@ -537,3 +537,91 @@ async def test_service_folder_state_idempotent(db_session: AsyncSession):
fs1 = await get_or_create_folder_state(db_session, user_id=uid, connection_id=cid, parent_ref="")
fs2 = await get_or_create_folder_state(db_session, user_id=uid, connection_id=cid, parent_ref="")
assert fs1.id == fs2.id
# ── Task 3: refresh_cloud_folder Celery task tests ────────────────────────────
@pytest.mark.asyncio
async def test_refresh_cloud_folder_idempotent_reconciliation(db_session: AsyncSession):
"""Calling reconcile_cloud_listing twice yields no duplicate rows (idempotency check)."""
uid = await _make_user(db_session)
cid = await _make_connection(db_session, uid)
resource = _cloud_resource(cid, uid, provider_item_id="stable-001", name="stable.pdf")
listing = CloudListing(items=[resource], complete=True)
# First reconcile
await reconcile_cloud_listing(db_session, user_id=uid, connection_id=cid, parent_ref=None, listing=listing)
# Second reconcile with same data
await reconcile_cloud_listing(db_session, user_id=uid, connection_id=cid, parent_ref=None, listing=listing)
from sqlalchemy import select, func
result = await db_session.execute(
select(func.count()).select_from(CloudItem).where(
CloudItem.provider_item_id == "stable-001",
CloudItem.deleted_at.is_(None),
)
)
count = result.scalar()
assert count == 1, "Idempotent reconciliation must not create duplicate rows"
@pytest.mark.asyncio
async def test_refresh_cloud_folder_failed_refresh_retains_cached_rows(db_session: AsyncSession):
"""On provider failure, previously cached rows must not be deleted (D-13)."""
uid = await _make_user(db_session)
cid = await _make_connection(db_session, uid)
# Seed durable items
resource = _cloud_resource(cid, uid, provider_item_id="durable-001", name="important.pdf")
complete_listing = CloudListing(items=[resource], complete=True)
await reconcile_cloud_listing(db_session, user_id=uid, connection_id=cid, parent_ref=None, listing=complete_listing)
# Simulate provider failure: incomplete listing with no items
empty_failed_listing = CloudListing(items=[], complete=False)
await reconcile_cloud_listing(db_session, user_id=uid, connection_id=cid, parent_ref=None, listing=empty_failed_listing)
from sqlalchemy import select
result = await db_session.execute(
select(CloudItem).where(
CloudItem.provider_item_id == "durable-001",
CloudItem.deleted_at.is_(None),
)
)
retained = result.scalars().first()
assert retained is not None, "Incomplete listing must not delete previously cached rows"
@pytest.mark.asyncio
async def test_refresh_cloud_folder_task_structure():
"""refresh_cloud_folder is a registered Celery task with correct queue routing."""
from tasks.cloud_tasks import refresh_cloud_folder
from celery_app import celery_app
assert hasattr(refresh_cloud_folder, "delay"), "Task must have .delay() method"
assert hasattr(refresh_cloud_folder, "apply_async"), "Task must have .apply_async() method"
assert refresh_cloud_folder.name == "tasks.cloud_tasks.refresh_cloud_folder"
assert refresh_cloud_folder.max_retries == 3
routes = celery_app.conf.task_routes
assert "tasks.cloud_tasks.*" in routes
assert routes["tasks.cloud_tasks.*"]["queue"] == "documents"
@pytest.mark.asyncio
async def test_refresh_cloud_folder_no_byte_calls(db_session: AsyncSession):
"""reconcile_cloud_listing never calls get_object or any download method — D-18/CACHE-01."""
from unittest.mock import AsyncMock, patch
uid = await _make_user(db_session)
cid = await _make_connection(db_session, uid)
resource = _cloud_resource(cid, uid, provider_item_id="nobytes-001")
listing = CloudListing(items=[resource], complete=True)
mock_get_object = AsyncMock()
with patch("db.models.CloudItem", wraps=CloudItem) as _patched:
# reconcile_cloud_listing must not call any external byte-download method
await reconcile_cloud_listing(
db_session, user_id=uid, connection_id=cid, parent_ref=None, listing=listing
)
mock_get_object.assert_not_called()