"""Add cloud_byte_cache_entries, cloud_analysis_jobs, cloud_analysis_job_items, and user analysis settings. Revision ID: 0007 Revises: 0006 Create Date: 2026-06-23 Changes: 1. cloud_byte_cache_entries: durable per-item provider byte cache backed by MinIO. Tracks version key, object key, content type, size, pin count, active job count, and LRU access timestamps. Entries are owner-scoped (user_id + connection_id + cloud_item_id + version_key). quota.used_bytes is incremented on retain and decremented on eviction via the atomic UPDATE pattern. 2. cloud_analysis_jobs: user-visible analysis batch rows for file, selection, folder, or connection scope. Aggregate counts (total, queued, downloading, extracting, classifying, indexed, already_current, skipped, cancelled, failed, unsupported) and provider bytes estimate are updated as workers process items. 3. cloud_analysis_job_items: per-item rows that drive idempotent processing. Status, error code, cache entry linkage, and retry counter. Unique constraint on (job_id, cloud_item_id) prevents duplicate item rows per job. 4. user_analysis_settings: one row per user with analysis preferences. progress_detail, failure_behavior, and cloud_cache_limit_bytes. Does NOT create a second settings authority — this is user-scoped, separate from system_settings. Design notes: - Ownership boundary: every row has user_id. Eviction and status queries are always scoped by user_id before any other predicate. - version_key uniqueness: (user_id, connection_id, cloud_item_id, version_key) is the idempotency key for cache entries. - Downgrade drops only Phase 14 objects in reverse-creation order. - D-18 compliance: cloud provider byte sizes flow only to cache entries and quota accounting, never to quotas.used_bytes via cloud_items.provider_size. """ from __future__ import annotations import sqlalchemy as sa from sqlalchemy.dialects.postgresql import UUID as PG_UUID from alembic import op # revision identifiers, used by Alembic. revision = "0007" down_revision = "0006" branch_labels = None depends_on = None def upgrade() -> None: # ── 1. cloud_byte_cache_entries ──────────────────────────────────────────── op.create_table( "cloud_byte_cache_entries", sa.Column( "id", PG_UUID(as_uuid=True), primary_key=True, server_default=sa.text("gen_random_uuid()"), ), sa.Column( "user_id", PG_UUID(as_uuid=True), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False, ), sa.Column( "connection_id", PG_UUID(as_uuid=True), sa.ForeignKey("cloud_connections.id", ondelete="CASCADE"), nullable=False, ), sa.Column( "cloud_item_id", PG_UUID(as_uuid=True), sa.ForeignKey("cloud_items.id", ondelete="CASCADE"), nullable=False, ), # Snapshot of the provider item id — for audit/debug without joins sa.Column("provider_item_id", sa.Text, nullable=False), # Derived from etag / version / metadata fingerprint / content hash (in precedence order) sa.Column("version_key", sa.Text, nullable=False), # Private MinIO object key (UUID-based) — never exposed in API responses sa.Column("object_key", sa.Text, nullable=False), sa.Column("content_type", sa.Text, nullable=True), # Actual byte count stored in MinIO — flows to quota.used_bytes while retained sa.Column("size_bytes", sa.BigInteger, nullable=False), # Optional content hash computed ONLY after bytes are already hydrated sa.Column("content_hash", sa.Text, nullable=True), # pin_count > 0 means a preview or download lease is active — must not be evicted sa.Column("pin_count", sa.Integer, nullable=False, server_default="0"), # active_job_count > 0 means a Celery analysis task is using these bytes sa.Column("active_job_count", sa.Integer, nullable=False, server_default="0"), # LRU eviction timestamps sa.Column("last_accessed_at", sa.TIMESTAMP(timezone=True), nullable=True), sa.Column("evicted_at", sa.TIMESTAMP(timezone=True), nullable=True), sa.Column( "created_at", sa.TIMESTAMP(timezone=True), nullable=False, server_default=sa.text("now()"), ), sa.Column( "updated_at", sa.TIMESTAMP(timezone=True), nullable=False, server_default=sa.text("now()"), ), # Idempotency key: same item + same version must reuse the same cache entry sa.UniqueConstraint( "user_id", "connection_id", "cloud_item_id", "version_key", name="uq_cache_entry_user_connection_item_version", ), ) # LRU eviction scan: scan only non-evicted entries for a user ordered by last_accessed_at op.create_index( "ix_cache_entries_user_evicted_accessed", "cloud_byte_cache_entries", ["user_id", "evicted_at", "last_accessed_at"], ) # Eviction guard: quick check for active pins / jobs before eviction candidate selection op.create_index( "ix_cache_entries_user_active", "cloud_byte_cache_entries", ["user_id", "pin_count", "active_job_count", "last_accessed_at"], ) # ── 2. cloud_analysis_jobs ───────────────────────────────────────────────── op.create_table( "cloud_analysis_jobs", sa.Column( "id", PG_UUID(as_uuid=True), primary_key=True, server_default=sa.text("gen_random_uuid()"), ), sa.Column( "user_id", PG_UUID(as_uuid=True), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False, ), sa.Column( "connection_id", PG_UUID(as_uuid=True), sa.ForeignKey("cloud_connections.id", ondelete="CASCADE"), nullable=False, ), # "file" | "selection" | "folder" | "connection" sa.Column("scope_kind", sa.String(16), nullable=False), # Provider item id of the root scope (null for connection-wide scope) sa.Column("scope_ref", sa.Text, nullable=True), sa.Column("recursive", sa.Boolean, nullable=False, server_default="false"), # "queued" | "running" | "paused" | "completed" | "cancelled" | "failed" sa.Column("status", sa.String(16), nullable=False, server_default="queued"), # Aggregate item counters — kept in sync by Celery workers sa.Column("total_count", sa.Integer, nullable=False, server_default="0"), sa.Column("queued_count", sa.Integer, nullable=False, server_default="0"), sa.Column("downloading_count", sa.Integer, nullable=False, server_default="0"), sa.Column("extracting_count", sa.Integer, nullable=False, server_default="0"), sa.Column("classifying_count", sa.Integer, nullable=False, server_default="0"), sa.Column("indexed_count", sa.Integer, nullable=False, server_default="0"), sa.Column("already_current_count", sa.Integer, nullable=False, server_default="0"), sa.Column("skipped_count", sa.Integer, nullable=False, server_default="0"), sa.Column("cancelled_count", sa.Integer, nullable=False, server_default="0"), sa.Column("failed_count", sa.Integer, nullable=False, server_default="0"), sa.Column("unsupported_count", sa.Integer, nullable=False, server_default="0"), # Provider-reported byte estimate (never used for quota) sa.Column("provider_bytes_estimate", sa.BigInteger, nullable=False, server_default="0"), # "pause_batch" | "continue_item" (default is pause_batch per D-11) sa.Column("failure_behavior", sa.String(16), nullable=False, server_default="pause_batch"), sa.Column("started_at", sa.TIMESTAMP(timezone=True), nullable=True), sa.Column("finished_at", sa.TIMESTAMP(timezone=True), nullable=True), sa.Column( "created_at", sa.TIMESTAMP(timezone=True), nullable=False, server_default=sa.text("now()"), ), sa.Column( "updated_at", sa.TIMESTAMP(timezone=True), nullable=False, server_default=sa.text("now()"), ), ) op.create_index("ix_analysis_jobs_user_id", "cloud_analysis_jobs", ["user_id"]) op.create_index( "ix_analysis_jobs_user_status", "cloud_analysis_jobs", ["user_id", "status"], ) # ── 3. cloud_analysis_job_items ──────────────────────────────────────────── op.create_table( "cloud_analysis_job_items", sa.Column( "id", PG_UUID(as_uuid=True), primary_key=True, server_default=sa.text("gen_random_uuid()"), ), sa.Column( "job_id", PG_UUID(as_uuid=True), sa.ForeignKey("cloud_analysis_jobs.id", ondelete="CASCADE"), nullable=False, ), sa.Column( "user_id", PG_UUID(as_uuid=True), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=False, ), sa.Column( "connection_id", PG_UUID(as_uuid=True), sa.ForeignKey("cloud_connections.id", ondelete="CASCADE"), nullable=False, ), sa.Column( "cloud_item_id", PG_UUID(as_uuid=True), sa.ForeignKey("cloud_items.id", ondelete="CASCADE"), nullable=False, ), # Snapshot for idempotency and debug — avoids joins in hot paths sa.Column("provider_item_id", sa.Text, nullable=False), # version_key at enqueue time — compared to re-resolved key at processing time sa.Column("version_key", sa.Text, nullable=True), # Metadata fingerprint at enqueue time — "item_id:size:modified_at:content_type" sa.Column("fingerprint", sa.Text, nullable=True), # "queued" | "downloading" | "extracting" | "classifying" | "indexed" # | "already_current" | "cancelled" | "failed" | "unsupported" | "stale" sa.Column("status", sa.String(16), nullable=False, server_default="queued"), # Controlled error strings only — never raw provider/exception text sa.Column("error_code", sa.String(64), nullable=True), sa.Column("error_message", sa.Text, nullable=True), # Link to the cache entry used during processing (null until bytes hydrated) sa.Column( "cache_entry_id", PG_UUID(as_uuid=True), sa.ForeignKey("cloud_byte_cache_entries.id", ondelete="SET NULL"), nullable=True, ), sa.Column("retry_count", sa.Integer, nullable=False, server_default="0"), sa.Column("started_at", sa.TIMESTAMP(timezone=True), nullable=True), sa.Column("finished_at", sa.TIMESTAMP(timezone=True), nullable=True), sa.Column( "created_at", sa.TIMESTAMP(timezone=True), nullable=False, server_default=sa.text("now()"), ), sa.Column( "updated_at", sa.TIMESTAMP(timezone=True), nullable=False, server_default=sa.text("now()"), ), # Each item appears once per job sa.UniqueConstraint( "job_id", "cloud_item_id", name="uq_job_items_job_cloud_item", ), ) # Duplicate-current check: fast lookup by (user_id, cloud_item_id, version_key) # across all jobs to detect already-current items without byte download op.create_index( "ix_job_items_user_item_version", "cloud_analysis_job_items", ["user_id", "cloud_item_id", "version_key"], ) op.create_index("ix_job_items_job_id", "cloud_analysis_job_items", ["job_id"]) # ── 4. user_analysis_settings ────────────────────────────────────────────── op.create_table( "user_analysis_settings", sa.Column( "user_id", PG_UUID(as_uuid=True), sa.ForeignKey("users.id", ondelete="CASCADE"), primary_key=True, ), # "simple" (default) | "detailed" sa.Column( "analysis_progress_detail", sa.String(16), nullable=False, server_default="simple", ), # "pause_batch" (default) | "continue_item" sa.Column( "analysis_failure_behavior", sa.String(16), nullable=False, server_default="pause_batch", ), # User-preferred cache byte limit (capped by tier/admin maximum in service layer) # Default: 512 MB sa.Column( "cloud_cache_limit_bytes", sa.BigInteger, nullable=False, server_default=str(512 * 1024 * 1024), ), sa.Column( "created_at", sa.TIMESTAMP(timezone=True), nullable=False, server_default=sa.text("now()"), ), sa.Column( "updated_at", sa.TIMESTAMP(timezone=True), nullable=False, server_default=sa.text("now()"), ), ) def downgrade() -> None: # Drop in reverse-creation order. Phase 14 objects only — no Phase 12/13 changes. op.drop_table("user_analysis_settings") op.drop_index("ix_job_items_job_id", table_name="cloud_analysis_job_items") op.drop_index("ix_job_items_user_item_version", table_name="cloud_analysis_job_items") op.drop_table("cloud_analysis_job_items") op.drop_index("ix_analysis_jobs_user_status", table_name="cloud_analysis_jobs") op.drop_index("ix_analysis_jobs_user_id", table_name="cloud_analysis_jobs") op.drop_table("cloud_analysis_jobs") op.drop_index("ix_cache_entries_user_active", table_name="cloud_byte_cache_entries") op.drop_index("ix_cache_entries_user_evicted_accessed", table_name="cloud_byte_cache_entries") op.drop_table("cloud_byte_cache_entries")