""" Celery tasks for cloud analysis processing — Phase 14 Plan 05. Provides the sync Celery bridge to the async processing service. Task: process_cloud_analysis_item - Broker payload: job_id (str), item_id (str), user_id (str), connection_id (str), cloud_item_id (str) — IDs only, no credentials, bytes, or filenames (T-14-10). - Credentials are decrypted inside the worker after ownership revalidation. - Supports bounded transient provider/classifier retry (max_retries=3). - Terminal auth failures do NOT retry — they write a controlled failure status. - MaxRetriesExceededError writes final "failed" status to DB. - Cooperative cancellation is checked between all major processing stages by the processing service. Retry harness (same pattern as cloud_tasks.py / document_tasks.py): CRITICAL: self.retry() MUST be raised from the outer sync task, NOT inside asyncio.run(). _TransientError and _ClassificationRetry are sentinels raised by _run_processing() to signal retryable failures that escape asyncio.run(). Aggregate job counter updates: - Handled by services.cloud_analysis_processing._set_item_status and _update_job_completion (the processing service owns all counter transitions). - The Celery task only records the final outcome returned by process_job_item. Queue routing: - Added to celery_app.py task_routes under "tasks.cloud_analysis_tasks.*" → "documents". Security: - T-14-10: Broker payload contains no credentials, provider URLs, filenames, or bytes. - Owner revalidation inside the worker — no trust in the broker message alone. """ from __future__ import annotations import asyncio import random from celery.exceptions import MaxRetriesExceededError from celery_app import celery_app # ── Retry sentinels ─────────────────────────────────────────────────────────── class _TransientError(Exception): """Retryable transient provider or network failure. Raised by _run_processing() to escape asyncio.run() so self.retry() can be called in the outer sync Celery task layer. """ class _ClassificationRetry(Exception): """Retryable AI classification failure. Separate sentinel from _TransientError so the caller can apply appropriate backoff and log the classification failure path distinctly. """ # ── Async processing worker ─────────────────────────────────────────────────── async def _run_processing( job_id: str, item_id: str, user_id: str, connection_id: str, cloud_item_id: str, ) -> dict: """Async body of process_cloud_analysis_item. Opens its own AsyncSession (never shared with other tasks/requests). Decrypts credentials inside the worker — never put them in the broker message. Returns a result dict with at least: {"status": "indexed"|"failed"|"cancelled"|"stale"}. Raises: _TransientError: Provider/network failure that should be retried. _ClassificationRetry: AI classification failure that should be retried. """ import uuid as _uuid from db.session import AsyncSessionLocal from services.cloud_items import resolve_owned_connection, ConnectionNotFound from storage.cloud_backend_factory import build_cloud_resource_adapter from storage.cloud_utils import decrypt_credentials from storage import get_storage_backend from config import settings master_key = settings.cloud_creds_key.encode() async with AsyncSessionLocal() as session: # Revalidate ownership — worker never trusts broker payload alone (T-14-10) try: conn = await resolve_owned_connection( session, connection_id=connection_id, user_id=user_id, ) except ConnectionNotFound: # Connection deleted or user changed — silently drop, do not retry return {"status": "cancelled", "reason": "connection_not_found"} # Decrypt credentials inside the worker (T-14-10) try: credentials = decrypt_credentials(master_key, user_id, conn.credentials_enc) except Exception as exc: # Decryption failure is terminal — write failure status await _write_item_failure( session, job_id=job_id, item_id=item_id, user_id=user_id, error_code="credential_error", error_message="Credential decryption failed. Re-connect the account.", ) return {"status": "failed", "reason": "credential_error"} # Build read-only provider adapter try: adapter = build_cloud_resource_adapter(conn.provider, credentials) except Exception as exc: await _write_item_failure( session, job_id=job_id, item_id=item_id, user_id=user_id, error_code="adapter_error", error_message="Failed to build provider adapter.", ) return {"status": "failed", "reason": "adapter_error"} # Build MinIO client for cache byte storage minio_client = _build_minio_client() # Delegate to the processing service from services.cloud_analysis_processing import ( process_job_item, ProcessingResult, _ClassificationError, OwnerValidationError, ) try: result: ProcessingResult = await process_job_item( session, job_id=_uuid.UUID(job_id), item_id=_uuid.UUID(item_id), user_id=_uuid.UUID(user_id), connection_id=_uuid.UUID(connection_id), cloud_item_id=_uuid.UUID(cloud_item_id), minio_client=minio_client, provider_adapter=adapter, ) await session.commit() return { "status": result.status, "error_code": result.error_code, "cache_entry_id": str(result.cache_entry_id) if result.cache_entry_id else None, } except _ClassificationError as exc: # Re-raise as our sentinel so the outer sync layer can self.retry() raise _ClassificationRetry(str(exc)) from exc except Exception as exc: err_str = str(exc).lower() # Detect auth/scope errors — terminal, do not retry if any(kw in err_str for kw in ( "invalid_grant", "unauthorized", "401", "403", "scope", "revoked", "credential_error", "auth_error", )): await _write_item_failure( session, job_id=job_id, item_id=item_id, user_id=user_id, error_code="auth_error", error_message="Authentication failed. Re-connect the account.", ) await session.commit() return {"status": "failed", "reason": "auth_error"} # Transient failure — signal outer layer to retry raise _TransientError(f"Transient error during processing: {exc}") from exc def _build_minio_client(): """Build a MinIO-compatible storage client for cache byte operations. Returns an object with async get_object(key), put_object(key, data, content_type), and delete_object(key) methods backed by the project's MinIO storage backend. """ from storage import get_storage_backend class _MinIOClientWrapper: def __init__(self, backend): self._backend = backend async def get_object(self, key: str) -> bytes: return await self._backend.get_object(key) async def put_object(self, key: str, data: bytes, content_type: str = None) -> None: await self._backend.put_object(key, data, content_type=content_type or "application/octet-stream") async def delete_object(self, key: str) -> None: try: await self._backend.delete_object(key) except Exception: pass return _MinIOClientWrapper(get_storage_backend()) async def _write_item_failure( session, *, job_id: str, item_id: str, user_id: str, error_code: str, error_message: str, ) -> None: """Write a failure status to a CloudAnalysisJobItem without leaking exceptions. Used for terminal errors (credential, adapter) where the processing service could not run its normal status-transition path. """ import uuid as _uuid from sqlalchemy import select from db.models import CloudAnalysisJob, CloudAnalysisJobItem try: uid = _uuid.UUID(user_id) jid = _uuid.UUID(job_id) iid = _uuid.UUID(item_id) job = (await session.execute( select(CloudAnalysisJob).where( CloudAnalysisJob.id == jid, CloudAnalysisJob.user_id == uid, ) )).scalars().first() job_item = (await session.execute( select(CloudAnalysisJobItem).where( CloudAnalysisJobItem.id == iid, CloudAnalysisJobItem.job_id == jid, ) )).scalars().first() if job_item is not None and job is not None: from services.cloud_analysis_processing import _set_item_status, _update_job_completion old_status = job_item.status await _set_item_status( session, job_item=job_item, job=job, new_status="failed", old_status=old_status, error_code=error_code, error_message=error_message, ) await _update_job_completion(session, job=job) except Exception: pass # Best-effort — do not mask the original failure async def _write_max_retry_failure( job_id: str, item_id: str, user_id: str ) -> None: """Write final 'failed' status after all Celery retries are exhausted.""" from db.session import AsyncSessionLocal async with AsyncSessionLocal() as session: await _write_item_failure( session, job_id=job_id, item_id=item_id, user_id=user_id, error_code="max_retries_exceeded", error_message="Processing failed after 3 retry attempts. Use per-item retry to requeue.", ) await session.commit() # ── Celery task ─────────────────────────────────────────────────────────────── @celery_app.task( bind=True, max_retries=3, name="tasks.cloud_analysis_tasks.process_cloud_analysis_item", serializer="json", acks_late=True, reject_on_worker_lost=True, ) def process_cloud_analysis_item( self, job_id: str, item_id: str, user_id: str, connection_id: str, cloud_item_id: str, ) -> dict: """Sync Celery entry-point for processing one cloud analysis job item. Broker payload contains IDs only — no credentials, provider URLs, filenames, or bytes (T-14-10). Credentials are decrypted inside the worker. Retry harness: - Bounded exponential backoff: 30s / 90s / 270s (± 10s jitter). - _TransientError and _ClassificationRetry sentinels escape asyncio.run() and trigger self.retry() here in the sync Celery layer. - MaxRetriesExceededError writes final failure status to DB. - Terminal auth failures are NOT retried. Args: job_id: CloudAnalysisJob UUID string. item_id: CloudAnalysisJobItem UUID string. user_id: Owner User UUID string. connection_id: CloudConnection UUID string. cloud_item_id: CloudItem UUID string. Returns: Result dict: {"status": "indexed"|"failed"|"cancelled"|"stale", ...} """ try: return asyncio.run( _run_processing(job_id, item_id, user_id, connection_id, cloud_item_id) ) except (_TransientError, _ClassificationRetry) as exc: try: # Exponential backoff with jitter: 30s, 90s, 270s (± 10s) jitter = random.randint(-10, 10) countdown = (30 * (3 ** self.request.retries)) + jitter raise self.retry(exc=exc, countdown=countdown) except MaxRetriesExceededError: # All retries exhausted — write final failure status asyncio.run(_write_max_retry_failure(job_id, item_id, user_id)) return { "status": "failed", "reason": "max_retries_exceeded", "job_id": job_id, "item_id": item_id, }