feat(14-05): add Celery cloud analysis task with retry harness (Task 2)
- Implement tasks/cloud_analysis_tasks.py: sync Celery bridge to async processing - Broker payload contains IDs only — credentials decrypted in worker (T-14-10) - _TransientError and _ClassificationRetry sentinels for bounded retry (max=3) - Auth errors are terminal (not retried); MaxRetriesExceeded writes failure status - celery_app.py: add cloud_analysis_tasks route and import registration - Add 6 contract tests: task registration, broker payload shape, queue routing, bounded retries, sentinel pattern, connection-not-found graceful drop
This commit is contained in:
curo1305
@@ -1067,3 +1067,175 @@ async def test_processing_never_calls_provider_mutation_methods(db_session):
|
||||
)
|
||||
# Should succeed or fail gracefully (not from a mutation)
|
||||
assert proc_result.status in ("indexed", "failed", "stale", "cancelled")
|
||||
|
||||
|
||||
# ─── Plan 05 Task 2: Celery task broker payload and retry contract ────────────
|
||||
|
||||
def test_celery_task_registered_in_app():
|
||||
"""ANALYZE-04: process_cloud_analysis_item task is registered in the Celery app.
|
||||
|
||||
Verifies that the task module is imported and the task name is in the registry.
|
||||
"""
|
||||
from celery_app import celery_app
|
||||
import tasks.cloud_analysis_tasks # noqa: F401 — side effect: registers the task
|
||||
|
||||
task_names = list(celery_app.tasks.keys())
|
||||
assert "tasks.cloud_analysis_tasks.process_cloud_analysis_item" in task_names, (
|
||||
"process_cloud_analysis_item must be registered in the Celery task registry"
|
||||
)
|
||||
|
||||
|
||||
def test_celery_broker_payload_contains_ids_only():
|
||||
"""T-14-10: Celery task signature accepts only ID strings — no credentials/bytes.
|
||||
|
||||
Inspects the task function signature to verify it accepts only string ID
|
||||
parameters (job_id, item_id, user_id, connection_id, cloud_item_id).
|
||||
"""
|
||||
import inspect
|
||||
from tasks.cloud_analysis_tasks import process_cloud_analysis_item
|
||||
|
||||
sig = inspect.signature(process_cloud_analysis_item)
|
||||
params = list(sig.parameters.keys())
|
||||
|
||||
# Required ID-only parameters
|
||||
required = ["job_id", "item_id", "user_id", "connection_id", "cloud_item_id"]
|
||||
for param in required:
|
||||
assert param in params, (
|
||||
f"Celery task must accept '{param}' ID parameter — broker payload "
|
||||
"must contain IDs only (T-14-10)"
|
||||
)
|
||||
|
||||
# Forbidden parameters that would indicate credential leakage
|
||||
forbidden = ["credentials", "credentials_enc", "access_token", "file_bytes",
|
||||
"provider_url", "filename", "object_key"]
|
||||
for param in forbidden:
|
||||
assert param not in params, (
|
||||
f"Celery task must NOT accept '{param}' — credentials and bytes "
|
||||
"must not appear in broker payload (T-14-10)"
|
||||
)
|
||||
|
||||
|
||||
def test_celery_task_routes_to_documents_queue():
|
||||
"""ANALYZE-04: cloud_analysis_tasks routes to the 'documents' queue."""
|
||||
from celery_app import celery_app
|
||||
|
||||
routes = celery_app.conf.task_routes or {}
|
||||
assert "tasks.cloud_analysis_tasks.*" in routes, (
|
||||
"cloud_analysis_tasks must have a task route entry"
|
||||
)
|
||||
route = routes["tasks.cloud_analysis_tasks.*"]
|
||||
assert route.get("queue") == "documents", (
|
||||
"cloud_analysis_tasks must route to the 'documents' queue"
|
||||
)
|
||||
|
||||
|
||||
def test_celery_task_max_retries_is_bounded():
|
||||
"""ANALYZE-04: Celery task has bounded max_retries (not unlimited).
|
||||
|
||||
Unlimited retries could cause runaway retry storms on persistent failures.
|
||||
"""
|
||||
from tasks.cloud_analysis_tasks import process_cloud_analysis_item
|
||||
|
||||
# Access the underlying task object's max_retries attribute
|
||||
task_obj = process_cloud_analysis_item
|
||||
assert hasattr(task_obj, "max_retries"), "Task must have max_retries set"
|
||||
assert task_obj.max_retries is not None, "max_retries must not be None (unlimited)"
|
||||
assert 1 <= task_obj.max_retries <= 10, (
|
||||
f"max_retries must be a reasonable bounded value, got {task_obj.max_retries}"
|
||||
)
|
||||
|
||||
|
||||
def test_celery_task_retry_sentinel_escapes_asyncio_run():
|
||||
"""ANALYZE-04: _TransientError and _ClassificationRetry sentinels escape asyncio.run().
|
||||
|
||||
The sentinel pattern requires that exceptions raised inside asyncio.run()
|
||||
propagate out to the sync Celery task layer where self.retry() can be called.
|
||||
This test verifies that the sentinels are plain Exception subclasses (not
|
||||
coroutines or awaitables), so they can propagate through asyncio.run() normally.
|
||||
"""
|
||||
from tasks.cloud_analysis_tasks import _TransientError, _ClassificationRetry
|
||||
|
||||
# The sentinels must be importable and be plain Exception subclasses
|
||||
assert issubclass(_TransientError, Exception), (
|
||||
"_TransientError must be a plain Exception subclass to escape asyncio.run()"
|
||||
)
|
||||
assert issubclass(_ClassificationRetry, Exception), (
|
||||
"_ClassificationRetry must be a plain Exception subclass to escape asyncio.run()"
|
||||
)
|
||||
|
||||
# Verify they are NOT derived from asyncio-specific types that would be swallowed
|
||||
import asyncio
|
||||
assert not issubclass(_TransientError, asyncio.CancelledError), (
|
||||
"_TransientError must not derive from CancelledError (would be swallowed)"
|
||||
)
|
||||
assert not issubclass(_ClassificationRetry, asyncio.CancelledError), (
|
||||
"_ClassificationRetry must not derive from CancelledError (would be swallowed)"
|
||||
)
|
||||
|
||||
# Verify they can be raised and caught normally (synchronous check)
|
||||
caught_transient = False
|
||||
try:
|
||||
raise _TransientError("test")
|
||||
except _TransientError:
|
||||
caught_transient = True
|
||||
assert caught_transient, "_TransientError must be catchable as Exception"
|
||||
|
||||
caught_classification = False
|
||||
try:
|
||||
raise _ClassificationRetry("test")
|
||||
except _ClassificationRetry:
|
||||
caught_classification = True
|
||||
assert caught_classification, "_ClassificationRetry must be catchable as Exception"
|
||||
|
||||
|
||||
async def test_run_processing_returns_cancelled_when_connection_deleted():
|
||||
"""ANALYZE-04: _run_processing returns cancelled/skipped when connection no longer exists.
|
||||
|
||||
The worker revalidates ownership at the start — if the connection is deleted
|
||||
between enqueue and execution, the task silently drops (no retry).
|
||||
|
||||
Uses mocked AsyncSessionLocal to avoid requiring a live PostgreSQL connection
|
||||
in the test environment.
|
||||
"""
|
||||
from tasks.cloud_analysis_tasks import _run_processing
|
||||
from unittest.mock import patch, AsyncMock, MagicMock
|
||||
from services.cloud_items import ConnectionNotFound
|
||||
|
||||
import uuid
|
||||
|
||||
# Mock the AsyncSessionLocal so the task uses an in-memory session
|
||||
mock_session = AsyncMock()
|
||||
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
|
||||
mock_session.__aexit__ = AsyncMock(return_value=False)
|
||||
|
||||
# Mock resolve_owned_connection to raise ConnectionNotFound (deleted connection)
|
||||
with patch("tasks.cloud_analysis_tasks._run_processing") as patched_run:
|
||||
# Verify the function exists and is importable
|
||||
from tasks.cloud_analysis_tasks import _run_processing as real_run
|
||||
assert callable(real_run)
|
||||
|
||||
# Test the connection-not-found path by patching at the service layer
|
||||
with patch("services.cloud_items.resolve_owned_connection",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=ConnectionNotFound("not found")):
|
||||
with patch("db.session.AsyncSessionLocal") as mock_session_local:
|
||||
mock_ctx = AsyncMock()
|
||||
mock_ctx.__aenter__ = AsyncMock(return_value=mock_ctx)
|
||||
mock_ctx.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_session_local.return_value = mock_ctx
|
||||
|
||||
result = await _run_processing(
|
||||
job_id=str(uuid.uuid4()),
|
||||
item_id=str(uuid.uuid4()),
|
||||
user_id=str(uuid.uuid4()),
|
||||
connection_id=str(uuid.uuid4()), # Does not exist in DB
|
||||
cloud_item_id=str(uuid.uuid4()),
|
||||
)
|
||||
|
||||
# Worker must return a graceful status for deleted connection
|
||||
assert result.get("status") in ("cancelled", "failed"), (
|
||||
f"Deleted connection must return cancelled/failed status, got: {result}"
|
||||
)
|
||||
assert result.get("reason") == "connection_not_found", (
|
||||
f"Expected reason='connection_not_found', got: {result.get('reason')}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user