From e5d6b9ea5377a9a10a31879edec624291121593e Mon Sep 17 00:00:00 2001 From: curo1305 Date: Thu, 18 Jun 2026 23:18:30 +0200 Subject: [PATCH] =?UTF-8?q?docs(12-02):=20execution=20summary=20=E2=80=94?= =?UTF-8?q?=20CloudResourceAdapter,=20browse=20endpoint,=20Celery=20refres?= =?UTF-8?q?h=20task?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../12-02-SUMMARY.md | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 .planning/phases/12-cloud-resource-foundation/12-02-SUMMARY.md diff --git a/.planning/phases/12-cloud-resource-foundation/12-02-SUMMARY.md b/.planning/phases/12-cloud-resource-foundation/12-02-SUMMARY.md new file mode 100644 index 0000000..1be03ae --- /dev/null +++ b/.planning/phases/12-cloud-resource-foundation/12-02-SUMMARY.md @@ -0,0 +1,162 @@ +--- +phase: "12" +plan: "02" +subsystem: cloud-browse +tags: [cloud, browse, celery, idor, stale-while-revalidate, provider-normalization] +dependency_graph: + requires: + - "12-01" # CloudResourceAdapter contract, cloud_base.py, db migrations + provides: + - GET /api/cloud/connections/{connection_id}/items + - PATCH /api/cloud/connections/{id} + - tasks.cloud_tasks.refresh_cloud_folder + - CloudResourceAdapter mixin on all 4 providers + affects: + - backend/api/cloud/ (new package) + - backend/services/cloud_items.py + - backend/tasks/cloud_tasks.py +tech_stack: + added: + - CloudResourceAdapter abstract mixin (storage/cloud_base.py) + - refresh_cloud_folder Celery task (tasks/cloud_tasks.py) + - api/cloud/ package with connections.py, browse.py, schemas.py + patterns: + - stale-while-revalidate (5-min threshold, background Celery refresh) + - sentinel exception pattern for Celery retry in sync layer + - whitelisted Pydantic response schemas (credential-free) + - UUID coerce-to-uuid.UUID pattern for SQLAlchemy UUID(as_uuid=True) columns +key_files: + created: + - backend/api/cloud/__init__.py + - backend/api/cloud/browse.py + - backend/api/cloud/connections.py + - backend/api/cloud/schemas.py + - backend/tasks/cloud_tasks.py + - AGENTS.md + modified: + - backend/storage/google_drive_backend.py + - backend/storage/onedrive_backend.py + - backend/storage/webdav_backend.py + - backend/storage/nextcloud_backend.py + - backend/storage/cloud_backend_factory.py + - backend/services/cloud_items.py + - backend/db/models.py + - backend/celery_app.py + - backend/main.py + - frontend/package.json + - backend/tests/test_cloud.py + - backend/tests/test_cloud_items.py + - backend/tests/conftest.py + - README.md +decisions: + - "UUID coercion in cloud_items.py service layer: always converts to uuid.UUID objects for WHERE clause binding with UUID(as_uuid=True) columns; test_cloud_items.py updated to use UUID(as_uuid=True) instead of String(36) patch to match production ORM" + - "Stale-while-revalidate threshold set to 5 minutes: returns cached rows immediately on first browse if stale, schedules background refresh" + - "refresh_cloud_folder: decrypts credentials inside worker, never in broker payload; 3-retry bounded backoff 30s/90s/270s with ±10s jitter" + - "Celery retry sentinel pattern: _TransientProviderError and _TerminalProviderError escape asyncio.run() to Celery sync layer" + - "display_name_override ORM field added to CloudConnection model (migration 0006 already had the column)" +metrics: + duration: "~4 hours (continued from prior session)" + completed: "2026-06-18T21:17:38Z" + tasks_completed: 3 + files_changed: 19 +--- + +# Phase 12 Plan 02: Cloud Resource Foundation Browse API Summary + +Connection-ID browse endpoint with all four providers implementing `CloudResourceAdapter`, stale-while-revalidate caching, and durable `refresh_cloud_folder` Celery task. + +## What Was Built + +**Task 1: Provider normalization (committed ff33439)** +All four cloud providers (Google Drive, OneDrive, WebDAV, Nextcloud) now implement `CloudResourceAdapter` as a mixin. Each provider adds `list_folder()` and `get_capabilities()` returning normalized `CloudListing` and capability dicts. Factory adds `build_cloud_resource_adapter()`. 22 new tests in `test_cloud_backends.py` verify pagination, no-byte-download, and capability evidence contracts. + +**Task 2: api/cloud/ package decomposition (committed e186019)** +Replaced flat `api/cloud.py` with a package: +- `connections.py`: all connection management routes (OAuth, WebDAV, list, delete, rename) +- `browse.py`: canonical `GET /api/cloud/connections/{connection_id}/items` +- `schemas.py`: credential-free `CloudBrowseResponse`, `CloudItemOut`, `FolderFreshnessOut` +- `__init__.py`: aggregates sub-routers under `/api/cloud` prefix + +Fixed UUID type incompatibility: `test_cloud_items.py` now uses `UUID(as_uuid=True)` matching the production ORM instead of `String(36)` patch. Service layer coerces all inputs to `uuid.UUID` objects for WHERE clauses. + +**Task 3: Celery task + version bump (committed c6237ca)** +- `refresh_cloud_folder` Celery task: idempotent, bounded 3-retry (30s/90s/270s ± 10s jitter), credential decryption inside worker +- Stale-while-revalidate: 5-minute threshold in browse endpoint +- Version bumped 0.1.4 → 0.1.5 in backend/main.py and frontend/package.json +- AGENTS.md created/updated with Phase 12 state and new shared module map entries +- README.md updated with connection-ID browse API table + +## Commits + +| Hash | Message | +|------|---------| +| ff33439 | feat(12-02): normalize all four providers into CloudResourceAdapter contract | +| e186019 | feat(12-02): decompose api/cloud.py into package with connection-ID browse endpoint | +| c6237ca | feat(12-02): add refresh_cloud_folder Celery task, staleness trigger, version 0.1.5 | + +## Test Results + +493 passing, 1 pre-existing failure (`test_extract_docx` — missing `python-docx` module, unrelated to this plan), 6 skipped, 7 xfailed. + +New tests added: +- 22 in `test_cloud_backends.py` (Task 1) +- 11 in `test_cloud.py` (Task 2: IDOR, admin block, credential exclusion, rename, duplicate providers, malformed UUID) +- 5 in `test_cloud_items.py` (Task 3: idempotency, cached-row retention, task structure, no-byte-download, scheduling) + +## Deviations from Plan + +### Auto-fixed Issues + +**1. [Rule 1 - Bug] UUID type incompatibility between test fixtures** +- **Found during:** Task 2 debugging +- **Issue:** `test_cloud_items.py` patched UUID columns to `String(36)` while conftest used `UUID(as_uuid=True)`. Service functions could not work with both simultaneously — UUID objects caused InterfaceError on String(36) columns; strings caused AttributeError (.hex) on UUID(as_uuid=True) columns. +- **Fix:** Updated `test_cloud_items.py` fixture to use `UUID(as_uuid=True)` approach (matching production ORM). Updated all helpers to use `uuid.UUID` objects. Updated `cloud_items.py` service to always coerce to `uuid.UUID` objects in WHERE clauses. +- **Files modified:** `backend/tests/test_cloud_items.py`, `backend/services/cloud_items.py`, `backend/tests/conftest.py` +- **Commits:** e186019, c6237ca + +**2. [Rule 2 - Missing functionality] display_name_override missing from ORM model** +- **Found during:** Task 2 +- **Issue:** Migration 0006 added `display_name_override` column to `cloud_connections` table but ORM model didn't declare it, causing AttributeError when PATCH endpoint tried to set it. +- **Fix:** Added `display_name_override: Mapped[Optional[str]] = mapped_column(Text, nullable=True)` to `CloudConnection` model. +- **Files modified:** `backend/db/models.py` +- **Commit:** e186019 + +**3. [Rule 1 - Bug] Celery patch target wrong in refresh scheduling test** +- **Found during:** Task 3 test authoring +- **Issue:** Test patched `api.cloud.browse.refresh_cloud_folder` but browse.py uses a local import inside the function body, so the module attribute doesn't exist at test time. +- **Fix:** Changed patch target to `tasks.cloud_tasks.refresh_cloud_folder`. +- **Files modified:** `backend/tests/test_cloud.py` +- **Commit:** c6237ca + +**4. [Rule 1 - Bug] should_refresh condition never triggered for stale cached items** +- **Found during:** Task 3 scheduling test +- **Issue:** Browse endpoint's `should_refresh` only checked for no items, "refreshing" state, or `last_refreshed_at is None`. Cached items with `refresh_state="fresh"` and a recent `last_refreshed_at` would never schedule background refresh. +- **Fix:** Added 5-minute staleness check — if `last_refreshed_at` is older than 5 minutes, refresh is scheduled. +- **Files modified:** `backend/api/cloud/browse.py` +- **Commit:** c6237ca + +## Known Stubs + +None — all data sources wired. Browse endpoint returns real durable cached rows from `cloud_items` table and schedules real Celery task. + +## Threat Flags + +| Flag | File | Description | +|------|------|-------------| +| threat_flag: credential_in_worker | backend/tasks/cloud_tasks.py | Credentials decrypted inside Celery worker — master key must be in CLOUD_CREDS_KEY env var on all worker nodes. Never log credentials. Already mitigated by existing key-from-env pattern. | + +## Self-Check: PASSED + +Files confirmed created: +- backend/api/cloud/__init__.py ✓ +- backend/api/cloud/browse.py ✓ +- backend/api/cloud/connections.py ✓ +- backend/api/cloud/schemas.py ✓ +- backend/tasks/cloud_tasks.py ✓ + +Commits confirmed: +- ff33439 ✓ +- e186019 ✓ +- c6237ca ✓ + +Test result: 493 passed, 1 pre-existing failure ✓