kite/backend/tasks/cloud_analysis_tasks.py

"""
Celery tasks for cloud analysis processing — Phase 14 Plan 05.

Provides the sync Celery bridge to the async processing service.

Task: process_cloud_analysis_item
  - Broker payload: job_id (str), item_id (str), user_id (str), connection_id (str),
    cloud_item_id (str) — IDs only, no credentials, bytes, or filenames (T-14-10).
  - Credentials are decrypted inside the worker after ownership revalidation.
  - Supports bounded transient provider/classifier retry (max_retries=3).
  - Terminal auth failures do NOT retry — they write a controlled failure status.
  - MaxRetriesExceededError writes final "failed" status to DB.
  - Cooperative cancellation is checked between all major processing stages by the
    processing service.

Retry harness (same pattern as cloud_tasks.py / document_tasks.py):
  CRITICAL: self.retry() MUST be raised from the outer sync task, NOT inside
  asyncio.run(). _TransientError and _ClassificationRetry are sentinels raised by
  _run_processing() to signal retryable failures that escape asyncio.run().

Aggregate job counter updates:
  - Handled by services.cloud_analysis_processing._set_item_status and
    _update_job_completion (the processing service owns all counter transitions).
  - The Celery task only records the final outcome returned by process_job_item.

Queue routing:
  - Added to celery_app.py task_routes under "tasks.cloud_analysis_tasks.*" → "documents".

Security:
  - T-14-10: Broker payload contains no credentials, provider URLs, filenames, or bytes.
  - Owner revalidation inside the worker — no trust in the broker message alone.
"""
from __future__ import annotations

import asyncio
import random

from celery.exceptions import MaxRetriesExceededError

from celery_app import celery_app


# ── Retry sentinels ───────────────────────────────────────────────────────────

class _TransientError(Exception):
    """Retryable transient provider or network failure.

    Raised by _run_processing() to escape asyncio.run() so self.retry()
    can be called in the outer sync Celery task layer.
    """


class _ClassificationRetry(Exception):
    """Retryable AI classification failure.

    Separate sentinel from _TransientError so the caller can apply appropriate
    backoff and log the classification failure path distinctly.
    """


# ── Async processing worker ───────────────────────────────────────────────────

async def _run_processing(
    job_id: str,
    item_id: str,
    user_id: str,
    connection_id: str,
    cloud_item_id: str,
) -> dict:
    """Async body of process_cloud_analysis_item.

    Opens its own AsyncSession (never shared with other tasks/requests).
    Decrypts credentials inside the worker — never put them in the broker message.

    Returns a result dict with at least: {"status": "indexed"|"failed"|"cancelled"|"stale"}.

    Raises:
        _TransientError:       Provider/network failure that should be retried.
        _ClassificationRetry:  AI classification failure that should be retried.
    """
    import uuid as _uuid

    from db.session import AsyncSessionLocal
    from services.cloud_items import resolve_owned_connection, ConnectionNotFound
    from storage.cloud_backend_factory import build_cloud_resource_adapter
    from storage.cloud_utils import decrypt_credentials
    from storage import get_storage_backend
    from config import settings

    master_key = settings.cloud_creds_key.encode()

    async with AsyncSessionLocal() as session:
        # Revalidate ownership — worker never trusts broker payload alone (T-14-10)
        try:
            conn = await resolve_owned_connection(
                session,
                connection_id=connection_id,
                user_id=user_id,
            )
        except ConnectionNotFound:
            # Connection deleted or user changed — silently drop, do not retry
            return {"status": "cancelled", "reason": "connection_not_found"}

        # Decrypt credentials inside the worker (T-14-10)
        try:
            credentials = decrypt_credentials(master_key, user_id, conn.credentials_enc)
        except Exception as exc:
            # Decryption failure is terminal — write failure status
            await _write_item_failure(
                session,
                job_id=job_id,
                item_id=item_id,
                user_id=user_id,
                error_code="credential_error",
                error_message="Credential decryption failed. Re-connect the account.",
            )
            return {"status": "failed", "reason": "credential_error"}

        # Build read-only provider adapter
        try:
            adapter = build_cloud_resource_adapter(conn.provider, credentials)
        except Exception as exc:
            await _write_item_failure(
                session,
                job_id=job_id,
                item_id=item_id,
                user_id=user_id,
                error_code="adapter_error",
                error_message="Failed to build provider adapter.",
            )
            return {"status": "failed", "reason": "adapter_error"}

        # Build MinIO client for cache byte storage
        minio_client = _build_minio_client()

        # Delegate to the processing service
        from services.cloud_analysis_processing import (
            process_job_item,
            ProcessingResult,
            _ClassificationError,
            OwnerValidationError,
        )

        try:
            result: ProcessingResult = await process_job_item(
                session,
                job_id=_uuid.UUID(job_id),
                item_id=_uuid.UUID(item_id),
                user_id=_uuid.UUID(user_id),
                connection_id=_uuid.UUID(connection_id),
                cloud_item_id=_uuid.UUID(cloud_item_id),
                minio_client=minio_client,
                provider_adapter=adapter,
            )
            await session.commit()
            return {
                "status": result.status,
                "error_code": result.error_code,
                "cache_entry_id": str(result.cache_entry_id) if result.cache_entry_id else None,
            }

        except _ClassificationError as exc:
            # Re-raise as our sentinel so the outer sync layer can self.retry()
            raise _ClassificationRetry(str(exc)) from exc

        except Exception as exc:
            err_str = str(exc).lower()
            # Detect auth/scope errors — terminal, do not retry
            if any(kw in err_str for kw in (
                "invalid_grant", "unauthorized", "401", "403", "scope", "revoked",
                "credential_error", "auth_error",
            )):
                await _write_item_failure(
                    session,
                    job_id=job_id,
                    item_id=item_id,
                    user_id=user_id,
                    error_code="auth_error",
                    error_message="Authentication failed. Re-connect the account.",
                )
                await session.commit()
                return {"status": "failed", "reason": "auth_error"}
            # Transient failure — signal outer layer to retry
            raise _TransientError(f"Transient error during processing: {exc}") from exc


def _build_minio_client():
    """Build a MinIO-compatible storage client for cache byte operations.

    Returns an object with async get_object(key), put_object(key, data, content_type),
    and delete_object(key) methods backed by the project's MinIO storage backend.
    """
    from storage import get_storage_backend

    class _MinIOClientWrapper:
        def __init__(self, backend):
            self._backend = backend

        async def get_object(self, key: str) -> bytes:
            return await self._backend.get_object(key)

        async def put_object(self, key: str, data: bytes, content_type: str = None) -> None:
            await self._backend.put_object(key, data, content_type=content_type or "application/octet-stream")

        async def delete_object(self, key: str) -> None:
            try:
                await self._backend.delete_object(key)
            except Exception:
                pass

    return _MinIOClientWrapper(get_storage_backend())


async def _write_item_failure(
    session,
    *,
    job_id: str,
    item_id: str,
    user_id: str,
    error_code: str,
    error_message: str,
) -> None:
    """Write a failure status to a CloudAnalysisJobItem without leaking exceptions.

    Used for terminal errors (credential, adapter) where the processing service
    could not run its normal status-transition path.
    """
    import uuid as _uuid
    from sqlalchemy import select
    from db.models import CloudAnalysisJob, CloudAnalysisJobItem

    try:
        uid = _uuid.UUID(user_id)
        jid = _uuid.UUID(job_id)
        iid = _uuid.UUID(item_id)

        job = (await session.execute(
            select(CloudAnalysisJob).where(
                CloudAnalysisJob.id == jid,
                CloudAnalysisJob.user_id == uid,
            )
        )).scalars().first()

        job_item = (await session.execute(
            select(CloudAnalysisJobItem).where(
                CloudAnalysisJobItem.id == iid,
                CloudAnalysisJobItem.job_id == jid,
            )
        )).scalars().first()

        if job_item is not None and job is not None:
            from services.cloud_analysis_processing import _set_item_status, _update_job_completion
            old_status = job_item.status
            await _set_item_status(
                session,
                job_item=job_item,
                job=job,
                new_status="failed",
                old_status=old_status,
                error_code=error_code,
                error_message=error_message,
            )
            await _update_job_completion(session, job=job)
    except Exception:
        pass  # Best-effort — do not mask the original failure


async def _write_max_retry_failure(
    job_id: str, item_id: str, user_id: str
) -> None:
    """Write final 'failed' status after all Celery retries are exhausted."""
    from db.session import AsyncSessionLocal

    async with AsyncSessionLocal() as session:
        await _write_item_failure(
            session,
            job_id=job_id,
            item_id=item_id,
            user_id=user_id,
            error_code="max_retries_exceeded",
            error_message="Processing failed after 3 retry attempts. Use per-item retry to requeue.",
        )
        await session.commit()


# ── Celery task ───────────────────────────────────────────────────────────────

@celery_app.task(
    bind=True,
    max_retries=3,
    name="tasks.cloud_analysis_tasks.process_cloud_analysis_item",
    serializer="json",
    acks_late=True,
    reject_on_worker_lost=True,
)
def process_cloud_analysis_item(
    self,
    job_id: str,
    item_id: str,
    user_id: str,
    connection_id: str,
    cloud_item_id: str,
) -> dict:
    """Sync Celery entry-point for processing one cloud analysis job item.

    Broker payload contains IDs only — no credentials, provider URLs, filenames,
    or bytes (T-14-10). Credentials are decrypted inside the worker.

    Retry harness:
      - Bounded exponential backoff: 30s / 90s / 270s (± 10s jitter).
      - _TransientError and _ClassificationRetry sentinels escape asyncio.run()
        and trigger self.retry() here in the sync Celery layer.
      - MaxRetriesExceededError writes final failure status to DB.
      - Terminal auth failures are NOT retried.

    Args:
        job_id:         CloudAnalysisJob UUID string.
        item_id:        CloudAnalysisJobItem UUID string.
        user_id:        Owner User UUID string.
        connection_id:  CloudConnection UUID string.
        cloud_item_id:  CloudItem UUID string.

    Returns:
        Result dict: {"status": "indexed"|"failed"|"cancelled"|"stale", ...}
    """
    try:
        return asyncio.run(
            _run_processing(job_id, item_id, user_id, connection_id, cloud_item_id)
        )
    except (_TransientError, _ClassificationRetry) as exc:
        try:
            # Exponential backoff with jitter: 30s, 90s, 270s (± 10s)
            jitter = random.randint(-10, 10)
            countdown = (30 * (3 ** self.request.retries)) + jitter
            raise self.retry(exc=exc, countdown=countdown)
        except MaxRetriesExceededError:
            # All retries exhausted — write final failure status
            asyncio.run(_write_max_retry_failure(job_id, item_id, user_id))
            return {
                "status": "failed",
                "reason": "max_retries_exceeded",
                "job_id": job_id,
                "item_id": item_id,
            }