feat(12-02): decompose api/cloud.py into package with connection-ID browse endpoint

- Add api/cloud/ package: connections.py, browse.py, schemas.py, __init__.py
- Add GET /api/cloud/connections/{connection_id}/items (T-12-01 IDOR, T-12-03 cred-free)
- Add PATCH /api/cloud/connections/{id} for display_name_override rename
- Add display_name_override ORM field to CloudConnection model
- Add CloudResourceAdapter service layer with str/UUID coercion
- Fix UUID type compatibility: test_cloud_items.py now uses UUID(as_uuid=True)
  matching conftest — removes String(36) patch that caused type incompatibility
- Add 11 Phase 12 integration tests (IDOR, admin block, credential exclusion,
  duplicate providers, rename, malformed UUID)
- Remove deleted api/cloud.py (replaced by api/cloud/ package)
This commit is contained in:
curo1305
2026-06-18 23:10:30 +02:00
parent ff33439f0a
commit e186019066
9 changed files with 850 additions and 182 deletions
+63
View File
@@ -0,0 +1,63 @@
"""
Cloud API package — router aggregator.
One parent router owns the /api/cloud prefix.
Sub-routers (connections, browse) carry no prefix — paths are declared
relative to the parent (D-04 pattern, same as api/documents/__init__.py).
main.py imports:
from api.cloud import router as cloud_router, users_router
app.include_router(cloud_router)
app.include_router(users_router)
"""
from fastapi import APIRouter, Depends, HTTPException, Request, status
from pydantic import BaseModel
from sqlalchemy.ext.asyncio import AsyncSession
from api.cloud.connections import router as connections_router, _VALID_BACKENDS
from api.cloud.browse import router as browse_router
from db.models import User
from deps.auth import get_regular_user
from deps.db import get_db
from services.rate_limiting import account_limiter
router = APIRouter(prefix="/api/cloud", tags=["cloud"])
router.include_router(connections_router)
router.include_router(browse_router)
# ── Users router (default storage) ───────────────────────────────────────────
users_router = APIRouter(prefix="/api/users", tags=["users"])
class DefaultStorageRequest(BaseModel):
backend: str
@users_router.patch("/me/default-storage")
@account_limiter.limit("100/minute")
async def update_default_storage(
request: Request,
body: DefaultStorageRequest,
session: AsyncSession = Depends(get_db),
current_user: User = Depends(get_regular_user),
) -> dict:
"""Update the current user's default storage backend.
Validated against the allowlist before storage.
"""
if body.backend not in _VALID_BACKENDS:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"Invalid backend. Valid values: {sorted(_VALID_BACKENDS)}",
)
request.state.current_user = current_user
user = await session.get(User, current_user.id)
if user is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found")
user.default_storage_backend = body.backend
session.add(user)
await session.commit()
return {"default_storage_backend": user.default_storage_backend}
+333
View File
@@ -0,0 +1,333 @@
"""
Canonical connection-ID browse endpoint for Phase 12.
GET /api/cloud/connections/{connection_id}/items
- Owner-scoped: resolves connection by UUID, not provider name
- Stale-while-revalidate: returns durable cached rows immediately; schedules
background refresh if stale or first visit
- T-12-01: IDOR-protected via resolve_owned_connection
- T-12-03: response uses whitelisted schemas, no credentials in response
- D-18/CACHE-01: never downloads file bytes, never mutates quota
"""
from __future__ import annotations
import asyncio
import uuid
from typing import Optional
import httpx
from fastapi import APIRouter, Depends, HTTPException, Request, status
from sqlalchemy.ext.asyncio import AsyncSession
from api.cloud.schemas import (
CloudBrowseResponse,
CloudCapabilityOut,
CloudItemOut,
FolderFreshnessOut,
)
from db.models import CloudConnection, CloudItem, CloudFolderState, User
from deps.auth import get_regular_user
from deps.db import get_db
from deps.utils import parse_uuid
from services.cloud_items import (
ConnectionNotFound,
list_cloud_children,
get_or_create_folder_state,
reconcile_cloud_listing,
resolve_owned_connection,
update_folder_state,
)
from services.rate_limiting import account_limiter
from storage.cloud_backend_factory import build_cloud_resource_adapter
from storage.cloud_utils import decrypt_credentials, encrypt_credentials
from config import settings
router = APIRouter()
_DISPLAY_NAMES = {
"google_drive": "Google Drive",
"onedrive": "OneDrive",
"nextcloud": "Nextcloud",
"webdav": "WebDAV server",
}
def _master_key() -> bytes:
return settings.cloud_creds_key.encode()
def _decrypt_connection(conn: CloudConnection, user_id: uuid.UUID) -> dict:
return decrypt_credentials(_master_key(), str(user_id), conn.credentials_enc)
def _capability_out(caps: dict) -> dict[str, CloudCapabilityOut]:
return {
action: CloudCapabilityOut(
action=cap.action,
state=cap.state,
reason=cap.reason,
message=cap.message,
)
for action, cap in caps.items()
}
def _item_out(item: CloudItem) -> CloudItemOut:
return CloudItemOut(
id=str(item.id),
provider_item_id=item.provider_item_id,
name=item.name,
kind=item.kind,
parent_ref=item.parent_ref,
content_type=item.content_type,
size=item.provider_size,
modified_at=item.modified_at,
etag=item.etag,
capabilities={}, # Phase 13 will add per-item capabilities
)
def _freshness_out(fs: CloudFolderState) -> FolderFreshnessOut:
return FolderFreshnessOut(
refresh_state=fs.refresh_state,
last_refreshed_at=fs.last_refreshed_at,
error_code=fs.error_code,
error_message=fs.error_message,
)
# ── Legacy helpers used by compatibility route in connections.py ─────────────
async def _fetch_google_drive_folders(credentials: dict, folder_id: str) -> list:
"""Legacy listing helper preserved for compatibility route."""
import datetime as dt
from google.oauth2.credentials import Credentials as GoogleCreds
from googleapiclient.discovery import build
expiry = None
if expiry_str := credentials.get("expiry"):
try:
expiry = dt.datetime.fromisoformat(expiry_str)
except ValueError:
pass
creds = GoogleCreds(
token=credentials.get("access_token"),
refresh_token=credentials.get("refresh_token"),
token_uri=credentials.get("token_uri", "https://oauth2.googleapis.com/token"),
client_id=credentials.get("client_id"),
client_secret=credentials.get("client_secret"),
expiry=expiry,
)
def _list_files() -> list:
service = build("drive", "v3", credentials=creds, cache_discovery=False)
response = service.files().list(
q=f"'{folder_id}' in parents and trashed=false",
fields="files(id,name,mimeType,size)",
pageSize=200,
).execute()
items = []
for item in response.get("files", []):
is_dir = item.get("mimeType") == "application/vnd.google-apps.folder"
items.append({
"id": item["id"],
"name": item["name"],
"is_dir": is_dir,
"size": int(item.get("size", 0)) if not is_dir else 0,
})
return items
return await asyncio.to_thread(_list_files)
async def _fetch_onedrive_folders(credentials: dict, folder_id: str) -> list:
"""Legacy listing helper preserved for compatibility route."""
if folder_id in ("root", ""):
url = "https://graph.microsoft.com/v1.0/me/drive/root/children"
else:
url = f"https://graph.microsoft.com/v1.0/me/drive/items/{folder_id}/children"
async with httpx.AsyncClient() as client:
resp = await client.get(
url,
headers={"Authorization": f"Bearer {credentials.get('access_token', '')}"},
timeout=30,
)
resp.raise_for_status()
data = resp.json()
items = []
for item in data.get("value", []):
is_dir = "folder" in item
items.append({
"id": item["id"],
"name": item["name"],
"is_dir": is_dir,
"size": item.get("size", 0) if not is_dir else 0,
})
return items
async def _fetch_webdav_folders(provider: str, credentials: dict, folder_id: str) -> list:
"""Legacy listing helper preserved for compatibility route."""
from storage.cloud_backend_factory import build_cloud_backend
webdav_path = "" if folder_id == "root" else folder_id
backend = build_cloud_backend(provider, credentials)
return await backend.list_folder(webdav_path)
# ── Canonical connection-ID browse endpoint ──────────────────────────────────
@router.get("/connections/{connection_id}/items", response_model=CloudBrowseResponse)
@account_limiter.limit("100/minute")
async def browse_connection_items(
connection_id: uuid.UUID,
request: Request,
parent_ref: Optional[str] = None,
session: AsyncSession = Depends(get_db),
current_user: User = Depends(get_regular_user),
) -> CloudBrowseResponse:
"""Browse the items of a connection by UUID.
T-12-01: IDOR protection — connection resolved by owner UUID, not provider alone.
T-12-03: response schema excludes credentials_enc and all credential fields.
D-18/CACHE-01: returns durable cached rows; never downloads file bytes.
D-11/D-12: cached rows returned immediately; background refresh scheduled if stale.
D-13/D-14: on refresh failure, cached rows retained with warning state.
"""
request.state.current_user = current_user
uid_str = str(current_user.id)
cid_str = str(connection_id)
# Resolve connection ownership (T-12-01)
try:
conn = await resolve_owned_connection(
session,
connection_id=cid_str,
user_id=uid_str,
)
except ConnectionNotFound:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Connection not found")
# Ensure folder state row exists
folder_state = await get_or_create_folder_state(
session,
user_id=uid_str,
connection_id=cid_str,
parent_ref=parent_ref or "",
)
await session.commit()
# Return durable cached rows
cached_items = await list_cloud_children(
session,
user_id=uid_str,
connection_id=cid_str,
parent_ref=parent_ref,
)
# Get connection-level capabilities (from a fresh call to avoid stale state)
try:
credentials = _decrypt_connection(conn, current_user.id)
adapter = build_cloud_resource_adapter(conn.provider, credentials)
conn_caps = await adapter.get_capabilities(connection_id, current_user.id)
except Exception:
from storage.cloud_base import ACTIONS, CloudCapability, STATE_TEMPORARILY_UNAVAILABLE, REASON_OFFLINE
conn_caps = {
action: CloudCapability(
action=action,
state=STATE_TEMPORARILY_UNAVAILABLE,
reason=REASON_OFFLINE,
message="Provider temporarily unreachable.",
)
for action in ACTIONS
}
# Refresh in background if first visit (no items) or stale
should_refresh = (
not cached_items
or folder_state.refresh_state in ("refreshing",)
or folder_state.last_refreshed_at is None
)
if should_refresh and not cached_items:
# First visit: do a synchronous bounded fetch so user sees content
try:
credentials = _decrypt_connection(conn, current_user.id)
adapter = build_cloud_resource_adapter(conn.provider, credentials)
listing = await adapter.list_folder(
connection_id,
current_user.id,
parent_ref=parent_ref,
)
await reconcile_cloud_listing(
session,
user_id=uid_str,
connection_id=cid_str,
parent_ref=parent_ref,
listing=listing,
)
await update_folder_state(
session,
user_id=uid_str,
connection_id=cid_str,
parent_ref=parent_ref or "",
refresh_state="fresh",
)
await session.commit()
# Re-read items after reconciliation
cached_items = await list_cloud_children(
session,
user_id=uid_str,
connection_id=cid_str,
parent_ref=parent_ref,
)
# Re-fetch folder state
folder_state = await get_or_create_folder_state(
session,
user_id=uid_str,
connection_id=cid_str,
parent_ref=parent_ref or "",
)
except Exception as exc:
# Retain cached rows (empty on first visit), set warning state
await update_folder_state(
session,
user_id=uid_str,
connection_id=cid_str,
parent_ref=parent_ref or "",
refresh_state="warning",
error_code="provider_error",
error_message="Provider temporarily unavailable. Retrying in background.",
)
await session.commit()
folder_state = await get_or_create_folder_state(
session,
user_id=uid_str,
connection_id=cid_str,
parent_ref=parent_ref or "",
)
elif should_refresh and cached_items:
# Has cached items — schedule background refresh via Celery
try:
from tasks.cloud_tasks import refresh_cloud_folder
refresh_cloud_folder.delay(
str(current_user.id),
str(connection_id),
parent_ref,
)
except Exception:
pass # Celery unavailable — serve stale cache without failing browse
display_name = conn.display_name_override or conn.display_name
return CloudBrowseResponse(
connection_id=str(connection_id),
provider=conn.provider,
display_name=display_name,
parent_ref=parent_ref,
items=[_item_out(item) for item in cached_items],
capabilities=_capability_out(conn_caps),
freshness=_freshness_out(folder_state),
)
@@ -1,4 +1,9 @@
"""Cloud storage connection management endpoints."""
"""
Connection management endpoints all existing cloud connection behaviors.
Preserves all existing endpoint URLs from api/cloud.py without change.
Introduces PATCH /api/cloud/connections/{connection_id} for display_name rename.
"""
from __future__ import annotations
import asyncio
@@ -13,19 +18,19 @@ from pydantic import BaseModel
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from api.cloud.schemas import ConnectionRenameRequest
from api.schemas import CloudConnectionOut
from config import settings
from db.models import CloudConnection, User
from deps.auth import get_regular_user
from deps.db import get_db
from deps.utils import get_client_ip
from deps.utils import get_client_ip, parse_uuid
from services.audit import write_audit_log
from services.rate_limiting import account_limiter
from storage.cloud_backend_factory import build_cloud_backend
from storage.cloud_utils import decrypt_credentials, encrypt_credentials, validate_cloud_url
router = APIRouter(prefix="/api/cloud", tags=["cloud"])
users_router = APIRouter(prefix="/api/users", tags=["users"])
router = APIRouter()
VALID_OAUTH_PROVIDERS = {"google_drive", "onedrive"}
VALID_WEBDAV_PROVIDERS = {"nextcloud", "webdav"}
@@ -107,6 +112,7 @@ async def _upsert_cloud_connection(
provider: str,
credentials_enc: str,
) -> CloudConnection:
"""Insert or update a connection for (user_id, provider)."""
result = await session.execute(
select(CloudConnection).where(
CloudConnection.user_id == user_id,
@@ -137,6 +143,7 @@ async def _get_owned_connection(
connection_id: uuid.UUID,
user_id: uuid.UUID,
) -> CloudConnection:
"""Return the connection owned by user_id, or raise 404."""
conn = await session.get(CloudConnection, connection_id)
if conn is None or conn.user_id != user_id:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Connection not found")
@@ -278,83 +285,7 @@ def _connection_response(conn: CloudConnection) -> dict:
return data
async def _fetch_google_drive_folders(credentials: dict, folder_id: str) -> list:
from google.oauth2.credentials import Credentials # lazy import
from googleapiclient.discovery import build # lazy import
import datetime as dt
expiry = None
if expiry_str := credentials.get("expiry"):
try:
expiry = dt.datetime.fromisoformat(expiry_str)
except ValueError:
pass
creds = Credentials(
token=credentials.get("access_token"),
refresh_token=credentials.get("refresh_token"),
token_uri=credentials.get("token_uri", "https://oauth2.googleapis.com/token"),
client_id=credentials.get("client_id"),
client_secret=credentials.get("client_secret"),
expiry=expiry,
)
def _list_files() -> list:
service = build("drive", "v3", credentials=creds, cache_discovery=False)
response = service.files().list(
q=f"'{folder_id}' in parents and trashed=false",
fields="files(id,name,mimeType,size)",
pageSize=200,
).execute()
items = []
for item in response.get("files", []):
is_dir = item.get("mimeType") == "application/vnd.google-apps.folder"
items.append({
"id": item["id"],
"name": item["name"],
"is_dir": is_dir,
"size": int(item.get("size", 0)) if not is_dir else 0,
})
return items
return await asyncio.to_thread(_list_files)
async def _fetch_onedrive_folders(credentials: dict, folder_id: str) -> list:
import httpx # lazy import
if folder_id in ("root", ""):
url = "https://graph.microsoft.com/v1.0/me/drive/root/children"
else:
url = f"https://graph.microsoft.com/v1.0/me/drive/items/{folder_id}/children"
async with httpx.AsyncClient() as client:
resp = await client.get(
url,
headers={"Authorization": f"Bearer {credentials.get('access_token', '')}"},
timeout=30,
)
resp.raise_for_status()
data = resp.json()
items = []
for item in data.get("value", []):
is_dir = "folder" in item
items.append({
"id": item["id"],
"name": item["name"],
"is_dir": is_dir,
"size": item.get("size", 0) if not is_dir else 0,
})
return items
async def _fetch_webdav_folders(provider: str, credentials: dict, folder_id: str) -> list:
webdav_path = "" if folder_id == "root" else folder_id
backend = build_cloud_backend(provider, credentials)
return await backend.list_folder(webdav_path)
# ── Endpoints ─────────────────────────────────────────────────────────────────
@router.get("/oauth/initiate/{provider}")
@account_limiter.limit("100/minute")
@@ -561,6 +492,28 @@ async def get_connection_config(
}
@router.patch("/connections/{connection_id}", status_code=status.HTTP_200_OK)
@account_limiter.limit("100/minute")
async def rename_connection(
connection_id: uuid.UUID,
body: ConnectionRenameRequest,
request: Request,
session: AsyncSession = Depends(get_db),
current_user: User = Depends(get_regular_user),
) -> dict:
"""Rename a cloud connection's display name.
Only the display_name field is accepted mass assignment prevention.
T-12-01: owner check via _get_owned_connection.
"""
request.state.current_user = current_user
conn = await _get_owned_connection(session, connection_id, current_user.id)
conn.display_name_override = body.display_name
await session.commit()
await session.refresh(conn)
return {"id": str(conn.id), "display_name": body.display_name}
@router.delete("/connections/{connection_id}", status_code=status.HTTP_204_NO_CONTENT)
@account_limiter.limit("100/minute")
async def delete_connection(
@@ -604,7 +557,11 @@ async def list_cloud_folders(
session: AsyncSession = Depends(get_db),
current_user: User = Depends(get_regular_user),
) -> dict:
"""List folder contents for a connected cloud provider."""
"""List folder contents for a connected cloud provider (compatibility route).
This endpoint is the original provider-keyed browse route preserved for
backward compatibility. The canonical route is GET /connections/{id}/items.
"""
request.state.current_user = current_user
if provider not in VALID_CLOUD_PROVIDERS:
raise _unsupported_provider_error(provider, VALID_CLOUD_PROVIDERS)
@@ -615,44 +572,17 @@ async def list_cloud_folders(
from services.cloud_cache import get_cloud_folders_cached # lazy import
if provider == "google_drive":
from api.cloud.browse import _fetch_google_drive_folders
fetcher = lambda: _fetch_google_drive_folders(credentials, folder_id)
elif provider == "onedrive":
from api.cloud.browse import _fetch_onedrive_folders
fetcher = lambda: _fetch_onedrive_folders(credentials, folder_id)
else:
from api.cloud.browse import _fetch_webdav_folders
fetcher = lambda: _fetch_webdav_folders(provider, credentials, folder_id)
items = await get_cloud_folders_cached(str(current_user.id), provider, folder_id, fetcher)
return {"items": items}
@users_router.patch("/me/default-storage")
@account_limiter.limit("100/minute")
async def update_default_storage(
request: Request,
body: DefaultStorageRequest,
session: AsyncSession = Depends(get_db),
current_user: User = Depends(get_regular_user),
) -> dict:
"""Update the current user's default storage backend.
The backend value is validated against the allowlist before storage.
Returns the updated default_storage_backend value.
"""
if body.backend not in _VALID_BACKENDS:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail=f"Invalid backend. Valid values: {sorted(_VALID_BACKENDS)}",
)
request.state.current_user = current_user
user = await session.get(User, current_user.id)
if user is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="User not found")
user.default_storage_backend = body.backend
session.add(user)
await session.commit()
return {"default_storage_backend": user.default_storage_backend}
+86
View File
@@ -0,0 +1,86 @@
"""
Whitelisted Pydantic response schemas for the cloud API package.
All schemas are explicit allowlists — credentials_enc, tokens, and passwords
are deliberately absent. T-12-03: credential exclusion by design.
"""
from __future__ import annotations
from datetime import datetime
from typing import List, Optional
from pydantic import BaseModel, field_validator
# ── Capability / item schemas ─────────────────────────────────────────────────
class CloudCapabilityOut(BaseModel):
"""Whitelisted capability descriptor. reason/message only when not supported."""
action: str
state: str # "supported" | "unsupported" | "temporarily_unavailable"
reason: Optional[str] = None
message: Optional[str] = None
class CloudItemOut(BaseModel):
"""Normalized cloud item metadata. No credentials or byte content."""
id: str # DocuVault stable UUID
provider_item_id: str
name: str
kind: str # "file" | "folder"
parent_ref: Optional[str] = None
content_type: Optional[str] = None
size: Optional[int] = None
modified_at: Optional[datetime] = None
etag: Optional[str] = None
capabilities: dict[str, CloudCapabilityOut] = {}
# ── Freshness / folder state schemas ─────────────────────────────────────────
class FolderFreshnessOut(BaseModel):
"""Freshness and error state for a browsed folder."""
refresh_state: str # "fresh" | "refreshing" | "warning"
last_refreshed_at: Optional[datetime] = None
error_code: Optional[str] = None
error_message: Optional[str] = None
# ── Browse response ───────────────────────────────────────────────────────────
class CloudBrowseResponse(BaseModel):
"""Owner-scoped connection-ID browse response.
T-12-01: items are always scoped to the resolved connection which is owned
by the requesting user. credentials_enc is never included.
"""
connection_id: str
provider: str
display_name: str
parent_ref: Optional[str]
items: List[CloudItemOut]
capabilities: dict[str, CloudCapabilityOut]
freshness: FolderFreshnessOut
# ── Connection schemas ────────────────────────────────────────────────────────
class ConnectionRenameRequest(BaseModel):
"""Validated PATCH body for renaming a connection display name.
Only display_name is accepted — mass assignment prevention.
"""
display_name: str
@field_validator("display_name")
@classmethod
def must_be_nonblank(cls, v: str) -> str:
stripped = v.strip()
if not stripped:
raise ValueError("display_name must not be blank")
return stripped