feat(14-02): add migration 0007 and ORM models for cloud analysis cache schema
- 0007_cloud_analysis_cache.py: new Alembic migration adding 4 Phase 14 tables
- cloud_byte_cache_entries: durable MinIO cache with version_key uniqueness,
pin_count/active_job_count guards, LRU eviction indexes
- cloud_analysis_jobs: batch job rows (scope_kind, status, aggregate counters,
provider_bytes_estimate, failure_behavior)
- cloud_analysis_job_items: per-item idempotency rows (status vocabulary,
controlled error fields, cache_entry_id, retry_count, version_key fingerprint)
- user_analysis_settings: per-user preferences (progress_detail, failure_behavior,
cloud_cache_limit_bytes)
- Reversible downgrade drops Phase 14 objects only
- db/models.py: CloudByteCacheEntry, CloudAnalysisJob, CloudAnalysisJobItem,
UserAnalysisSettings ORM models mirroring migration
This commit is contained in:
@@ -457,6 +457,249 @@ class CloudFolderState(Base):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class CloudByteCacheEntry(Base):
|
||||||
|
"""Durable per-item provider byte cache entry backed by MinIO.
|
||||||
|
|
||||||
|
Tracks retained cloud bytes for active open, preview, or analysis work.
|
||||||
|
size_bytes counts against quota.used_bytes while retained; decremented on eviction.
|
||||||
|
|
||||||
|
Eviction rules (D-14, D-16):
|
||||||
|
- Only entries with pin_count == 0 and active_job_count == 0 may be evicted.
|
||||||
|
- Eviction is per-user LRU ordered by last_accessed_at ascending.
|
||||||
|
- object_key is a UUID-based private MinIO key — never exposed in API responses (T-14-02).
|
||||||
|
|
||||||
|
Phase 14: cache lifecycle (create -> pin -> release -> evict) managed by
|
||||||
|
services/cloud_cache.py.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "cloud_byte_cache_entries"
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
|
||||||
|
)
|
||||||
|
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True),
|
||||||
|
ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
connection_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True),
|
||||||
|
ForeignKey("cloud_connections.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
cloud_item_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True),
|
||||||
|
ForeignKey("cloud_items.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
# Snapshot of provider_item_id — for audit/debug without joins
|
||||||
|
provider_item_id: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
# Derived from version / etag / metadata fingerprint (content hash optional, post-hydration only)
|
||||||
|
version_key: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
# Private MinIO object key — UUID-based, never exposed in API responses
|
||||||
|
object_key: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
content_type: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||||
|
# Actual byte count stored in MinIO — debited from quota.used_bytes
|
||||||
|
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||||
|
# Optional content hash — computed ONLY after bytes already hydrated, never as a pre-download requirement
|
||||||
|
content_hash: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||||
|
# pin_count > 0 = active preview/download lease; must not be evicted
|
||||||
|
pin_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
# active_job_count > 0 = Celery analysis task in flight; must not be evicted
|
||||||
|
active_job_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
last_accessed_at: Mapped[Optional[datetime]] = mapped_column(
|
||||||
|
TIMESTAMP(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
evicted_at: Mapped[Optional[datetime]] = mapped_column(
|
||||||
|
TIMESTAMP(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
|
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"user_id",
|
||||||
|
"connection_id",
|
||||||
|
"cloud_item_id",
|
||||||
|
"version_key",
|
||||||
|
name="uq_cache_entry_user_connection_item_version",
|
||||||
|
),
|
||||||
|
Index("ix_cache_entries_user_evicted_accessed", "user_id", "evicted_at", "last_accessed_at"),
|
||||||
|
Index("ix_cache_entries_user_active", "user_id", "pin_count", "active_job_count", "last_accessed_at"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class CloudAnalysisJob(Base):
|
||||||
|
"""User-visible analysis batch row.
|
||||||
|
|
||||||
|
Represents a single enqueue action for file, selection, folder, or connection scope.
|
||||||
|
Aggregate counters are updated by Celery workers as items are processed.
|
||||||
|
|
||||||
|
status: "queued" | "running" | "paused" | "completed" | "cancelled" | "failed"
|
||||||
|
scope_kind: "file" | "selection" | "folder" | "connection"
|
||||||
|
failure_behavior: "pause_batch" (default) | "continue_item" (D-11)
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "cloud_analysis_jobs"
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
|
||||||
|
)
|
||||||
|
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True),
|
||||||
|
ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
connection_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True),
|
||||||
|
ForeignKey("cloud_connections.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
scope_kind: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||||
|
scope_ref: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||||
|
recursive: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||||
|
status: Mapped[str] = mapped_column(String(16), nullable=False, default="queued")
|
||||||
|
# Aggregate item counters
|
||||||
|
total_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
queued_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
downloading_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
extracting_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
classifying_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
indexed_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
already_current_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
skipped_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
cancelled_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
failed_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
unsupported_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
# Provider-reported byte total — for estimates only, never used for quota
|
||||||
|
provider_bytes_estimate: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||||
|
failure_behavior: Mapped[str] = mapped_column(String(16), nullable=False, default="pause_batch")
|
||||||
|
started_at: Mapped[Optional[datetime]] = mapped_column(TIMESTAMP(timezone=True), nullable=True)
|
||||||
|
finished_at: Mapped[Optional[datetime]] = mapped_column(TIMESTAMP(timezone=True), nullable=True)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
|
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
Index("ix_analysis_jobs_user_id", "user_id"),
|
||||||
|
Index("ix_analysis_jobs_user_status", "user_id", "status"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class CloudAnalysisJobItem(Base):
|
||||||
|
"""Per-item row for idempotent processing within a CloudAnalysisJob.
|
||||||
|
|
||||||
|
status vocabulary (PATTERNS.md):
|
||||||
|
queued | downloading | extracting | classifying | indexed |
|
||||||
|
already_current | cancelled | failed | unsupported | stale
|
||||||
|
|
||||||
|
error_code and error_message contain controlled strings only — never raw
|
||||||
|
provider exception text or stack traces.
|
||||||
|
|
||||||
|
cache_entry_id is null until bytes are hydrated. It links to the
|
||||||
|
CloudByteCacheEntry used during processing and is cleared when the cache
|
||||||
|
entry is evicted.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "cloud_analysis_job_items"
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
|
||||||
|
)
|
||||||
|
job_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True),
|
||||||
|
ForeignKey("cloud_analysis_jobs.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True),
|
||||||
|
ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
connection_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True),
|
||||||
|
ForeignKey("cloud_connections.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
cloud_item_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True),
|
||||||
|
ForeignKey("cloud_items.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
provider_item_id: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
# version_key at enqueue time — re-resolved and compared during processing
|
||||||
|
version_key: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||||
|
# Metadata fingerprint: "provider_item_id:size:modified_at:content_type"
|
||||||
|
fingerprint: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||||
|
status: Mapped[str] = mapped_column(String(16), nullable=False, default="queued")
|
||||||
|
# Controlled error strings — never raw provider error text
|
||||||
|
error_code: Mapped[Optional[str]] = mapped_column(String(64), nullable=True)
|
||||||
|
error_message: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||||
|
cache_entry_id: Mapped[Optional[uuid.UUID]] = mapped_column(
|
||||||
|
UUID(as_uuid=True),
|
||||||
|
ForeignKey("cloud_byte_cache_entries.id", ondelete="SET NULL"),
|
||||||
|
nullable=True,
|
||||||
|
)
|
||||||
|
retry_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
started_at: Mapped[Optional[datetime]] = mapped_column(TIMESTAMP(timezone=True), nullable=True)
|
||||||
|
finished_at: Mapped[Optional[datetime]] = mapped_column(TIMESTAMP(timezone=True), nullable=True)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
|
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("job_id", "cloud_item_id", name="uq_job_items_job_cloud_item"),
|
||||||
|
Index("ix_job_items_user_item_version", "user_id", "cloud_item_id", "version_key"),
|
||||||
|
Index("ix_job_items_job_id", "job_id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class UserAnalysisSettings(Base):
|
||||||
|
"""Per-user analysis preferences — one row per user, created on demand.
|
||||||
|
|
||||||
|
Distinct from system_settings (AI provider config) and CloudFolderState.
|
||||||
|
This is the single settings authority for user analysis preferences.
|
||||||
|
|
||||||
|
progress_detail: "simple" (default) | "detailed"
|
||||||
|
analysis_failure_behavior: "pause_batch" (default) | "continue_item" (D-11)
|
||||||
|
cloud_cache_limit_bytes: user preference capped by tier/admin maximum in the service layer
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "user_analysis_settings"
|
||||||
|
|
||||||
|
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True),
|
||||||
|
ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
primary_key=True,
|
||||||
|
)
|
||||||
|
analysis_progress_detail: Mapped[str] = mapped_column(
|
||||||
|
String(16), nullable=False, default="simple"
|
||||||
|
)
|
||||||
|
analysis_failure_behavior: Mapped[str] = mapped_column(
|
||||||
|
String(16), nullable=False, default="pause_batch"
|
||||||
|
)
|
||||||
|
# Default: 512 MB
|
||||||
|
cloud_cache_limit_bytes: Mapped[int] = mapped_column(
|
||||||
|
BigInteger, nullable=False, default=512 * 1024 * 1024
|
||||||
|
)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
|
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Group(Base):
|
class Group(Base):
|
||||||
"""v2 stub — empty table, seeded for schema completeness (PROJECT.md D-02).
|
"""v2 stub — empty table, seeded for schema completeness (PROJECT.md D-02).
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,345 @@
|
|||||||
|
"""Add cloud_byte_cache_entries, cloud_analysis_jobs, cloud_analysis_job_items, and user analysis settings.
|
||||||
|
|
||||||
|
Revision ID: 0007
|
||||||
|
Revises: 0006
|
||||||
|
Create Date: 2026-06-23
|
||||||
|
|
||||||
|
Changes:
|
||||||
|
1. cloud_byte_cache_entries: durable per-item provider byte cache backed by MinIO.
|
||||||
|
Tracks version key, object key, content type, size, pin count, active job count,
|
||||||
|
and LRU access timestamps. Entries are owner-scoped (user_id + connection_id +
|
||||||
|
cloud_item_id + version_key). quota.used_bytes is incremented on retain and
|
||||||
|
decremented on eviction via the atomic UPDATE pattern.
|
||||||
|
|
||||||
|
2. cloud_analysis_jobs: user-visible analysis batch rows for file, selection, folder,
|
||||||
|
or connection scope. Aggregate counts (total, queued, downloading, extracting,
|
||||||
|
classifying, indexed, already_current, skipped, cancelled, failed, unsupported) and
|
||||||
|
provider bytes estimate are updated as workers process items.
|
||||||
|
|
||||||
|
3. cloud_analysis_job_items: per-item rows that drive idempotent processing.
|
||||||
|
Status, error code, cache entry linkage, and retry counter. Unique constraint on
|
||||||
|
(job_id, cloud_item_id) prevents duplicate item rows per job.
|
||||||
|
|
||||||
|
4. user_analysis_settings: one row per user with analysis preferences.
|
||||||
|
progress_detail, failure_behavior, and cloud_cache_limit_bytes. Does NOT create
|
||||||
|
a second settings authority — this is user-scoped, separate from system_settings.
|
||||||
|
|
||||||
|
Design notes:
|
||||||
|
- Ownership boundary: every row has user_id. Eviction and status queries are always
|
||||||
|
scoped by user_id before any other predicate.
|
||||||
|
- version_key uniqueness: (user_id, connection_id, cloud_item_id, version_key) is
|
||||||
|
the idempotency key for cache entries.
|
||||||
|
- Downgrade drops only Phase 14 objects in reverse-creation order.
|
||||||
|
- D-18 compliance: cloud provider byte sizes flow only to cache entries and quota
|
||||||
|
accounting, never to quotas.used_bytes via cloud_items.provider_size.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from sqlalchemy.dialects.postgresql import UUID as PG_UUID
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = "0007"
|
||||||
|
down_revision = "0006"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
# ── 1. cloud_byte_cache_entries ────────────────────────────────────────────
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"cloud_byte_cache_entries",
|
||||||
|
sa.Column(
|
||||||
|
"id",
|
||||||
|
PG_UUID(as_uuid=True),
|
||||||
|
primary_key=True,
|
||||||
|
server_default=sa.text("gen_random_uuid()"),
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"user_id",
|
||||||
|
PG_UUID(as_uuid=True),
|
||||||
|
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"connection_id",
|
||||||
|
PG_UUID(as_uuid=True),
|
||||||
|
sa.ForeignKey("cloud_connections.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"cloud_item_id",
|
||||||
|
PG_UUID(as_uuid=True),
|
||||||
|
sa.ForeignKey("cloud_items.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
# Snapshot of the provider item id — for audit/debug without joins
|
||||||
|
sa.Column("provider_item_id", sa.Text, nullable=False),
|
||||||
|
# Derived from etag / version / metadata fingerprint / content hash (in precedence order)
|
||||||
|
sa.Column("version_key", sa.Text, nullable=False),
|
||||||
|
# Private MinIO object key (UUID-based) — never exposed in API responses
|
||||||
|
sa.Column("object_key", sa.Text, nullable=False),
|
||||||
|
sa.Column("content_type", sa.Text, nullable=True),
|
||||||
|
# Actual byte count stored in MinIO — flows to quota.used_bytes while retained
|
||||||
|
sa.Column("size_bytes", sa.BigInteger, nullable=False),
|
||||||
|
# Optional content hash computed ONLY after bytes are already hydrated
|
||||||
|
sa.Column("content_hash", sa.Text, nullable=True),
|
||||||
|
# pin_count > 0 means a preview or download lease is active — must not be evicted
|
||||||
|
sa.Column("pin_count", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
# active_job_count > 0 means a Celery analysis task is using these bytes
|
||||||
|
sa.Column("active_job_count", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
# LRU eviction timestamps
|
||||||
|
sa.Column("last_accessed_at", sa.TIMESTAMP(timezone=True), nullable=True),
|
||||||
|
sa.Column("evicted_at", sa.TIMESTAMP(timezone=True), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.TIMESTAMP(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.TIMESTAMP(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
),
|
||||||
|
# Idempotency key: same item + same version must reuse the same cache entry
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"user_id",
|
||||||
|
"connection_id",
|
||||||
|
"cloud_item_id",
|
||||||
|
"version_key",
|
||||||
|
name="uq_cache_entry_user_connection_item_version",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# LRU eviction scan: scan only non-evicted entries for a user ordered by last_accessed_at
|
||||||
|
op.create_index(
|
||||||
|
"ix_cache_entries_user_evicted_accessed",
|
||||||
|
"cloud_byte_cache_entries",
|
||||||
|
["user_id", "evicted_at", "last_accessed_at"],
|
||||||
|
)
|
||||||
|
# Eviction guard: quick check for active pins / jobs before eviction candidate selection
|
||||||
|
op.create_index(
|
||||||
|
"ix_cache_entries_user_active",
|
||||||
|
"cloud_byte_cache_entries",
|
||||||
|
["user_id", "pin_count", "active_job_count", "last_accessed_at"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── 2. cloud_analysis_jobs ─────────────────────────────────────────────────
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"cloud_analysis_jobs",
|
||||||
|
sa.Column(
|
||||||
|
"id",
|
||||||
|
PG_UUID(as_uuid=True),
|
||||||
|
primary_key=True,
|
||||||
|
server_default=sa.text("gen_random_uuid()"),
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"user_id",
|
||||||
|
PG_UUID(as_uuid=True),
|
||||||
|
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"connection_id",
|
||||||
|
PG_UUID(as_uuid=True),
|
||||||
|
sa.ForeignKey("cloud_connections.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
# "file" | "selection" | "folder" | "connection"
|
||||||
|
sa.Column("scope_kind", sa.String(16), nullable=False),
|
||||||
|
# Provider item id of the root scope (null for connection-wide scope)
|
||||||
|
sa.Column("scope_ref", sa.Text, nullable=True),
|
||||||
|
sa.Column("recursive", sa.Boolean, nullable=False, server_default="false"),
|
||||||
|
# "queued" | "running" | "paused" | "completed" | "cancelled" | "failed"
|
||||||
|
sa.Column("status", sa.String(16), nullable=False, server_default="queued"),
|
||||||
|
# Aggregate item counters — kept in sync by Celery workers
|
||||||
|
sa.Column("total_count", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
sa.Column("queued_count", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
sa.Column("downloading_count", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
sa.Column("extracting_count", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
sa.Column("classifying_count", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
sa.Column("indexed_count", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
sa.Column("already_current_count", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
sa.Column("skipped_count", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
sa.Column("cancelled_count", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
sa.Column("failed_count", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
sa.Column("unsupported_count", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
# Provider-reported byte estimate (never used for quota)
|
||||||
|
sa.Column("provider_bytes_estimate", sa.BigInteger, nullable=False, server_default="0"),
|
||||||
|
# "pause_batch" | "continue_item" (default is pause_batch per D-11)
|
||||||
|
sa.Column("failure_behavior", sa.String(16), nullable=False, server_default="pause_batch"),
|
||||||
|
sa.Column("started_at", sa.TIMESTAMP(timezone=True), nullable=True),
|
||||||
|
sa.Column("finished_at", sa.TIMESTAMP(timezone=True), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.TIMESTAMP(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.TIMESTAMP(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_index("ix_analysis_jobs_user_id", "cloud_analysis_jobs", ["user_id"])
|
||||||
|
op.create_index(
|
||||||
|
"ix_analysis_jobs_user_status",
|
||||||
|
"cloud_analysis_jobs",
|
||||||
|
["user_id", "status"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── 3. cloud_analysis_job_items ────────────────────────────────────────────
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"cloud_analysis_job_items",
|
||||||
|
sa.Column(
|
||||||
|
"id",
|
||||||
|
PG_UUID(as_uuid=True),
|
||||||
|
primary_key=True,
|
||||||
|
server_default=sa.text("gen_random_uuid()"),
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"job_id",
|
||||||
|
PG_UUID(as_uuid=True),
|
||||||
|
sa.ForeignKey("cloud_analysis_jobs.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"user_id",
|
||||||
|
PG_UUID(as_uuid=True),
|
||||||
|
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"connection_id",
|
||||||
|
PG_UUID(as_uuid=True),
|
||||||
|
sa.ForeignKey("cloud_connections.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"cloud_item_id",
|
||||||
|
PG_UUID(as_uuid=True),
|
||||||
|
sa.ForeignKey("cloud_items.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
# Snapshot for idempotency and debug — avoids joins in hot paths
|
||||||
|
sa.Column("provider_item_id", sa.Text, nullable=False),
|
||||||
|
# version_key at enqueue time — compared to re-resolved key at processing time
|
||||||
|
sa.Column("version_key", sa.Text, nullable=True),
|
||||||
|
# Metadata fingerprint at enqueue time — "item_id:size:modified_at:content_type"
|
||||||
|
sa.Column("fingerprint", sa.Text, nullable=True),
|
||||||
|
# "queued" | "downloading" | "extracting" | "classifying" | "indexed"
|
||||||
|
# | "already_current" | "cancelled" | "failed" | "unsupported" | "stale"
|
||||||
|
sa.Column("status", sa.String(16), nullable=False, server_default="queued"),
|
||||||
|
# Controlled error strings only — never raw provider/exception text
|
||||||
|
sa.Column("error_code", sa.String(64), nullable=True),
|
||||||
|
sa.Column("error_message", sa.Text, nullable=True),
|
||||||
|
# Link to the cache entry used during processing (null until bytes hydrated)
|
||||||
|
sa.Column(
|
||||||
|
"cache_entry_id",
|
||||||
|
PG_UUID(as_uuid=True),
|
||||||
|
sa.ForeignKey("cloud_byte_cache_entries.id", ondelete="SET NULL"),
|
||||||
|
nullable=True,
|
||||||
|
),
|
||||||
|
sa.Column("retry_count", sa.Integer, nullable=False, server_default="0"),
|
||||||
|
sa.Column("started_at", sa.TIMESTAMP(timezone=True), nullable=True),
|
||||||
|
sa.Column("finished_at", sa.TIMESTAMP(timezone=True), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.TIMESTAMP(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.TIMESTAMP(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
),
|
||||||
|
# Each item appears once per job
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"job_id",
|
||||||
|
"cloud_item_id",
|
||||||
|
name="uq_job_items_job_cloud_item",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Duplicate-current check: fast lookup by (user_id, cloud_item_id, version_key)
|
||||||
|
# across all jobs to detect already-current items without byte download
|
||||||
|
op.create_index(
|
||||||
|
"ix_job_items_user_item_version",
|
||||||
|
"cloud_analysis_job_items",
|
||||||
|
["user_id", "cloud_item_id", "version_key"],
|
||||||
|
)
|
||||||
|
op.create_index("ix_job_items_job_id", "cloud_analysis_job_items", ["job_id"])
|
||||||
|
|
||||||
|
# ── 4. user_analysis_settings ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"user_analysis_settings",
|
||||||
|
sa.Column(
|
||||||
|
"user_id",
|
||||||
|
PG_UUID(as_uuid=True),
|
||||||
|
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
primary_key=True,
|
||||||
|
),
|
||||||
|
# "simple" (default) | "detailed"
|
||||||
|
sa.Column(
|
||||||
|
"analysis_progress_detail",
|
||||||
|
sa.String(16),
|
||||||
|
nullable=False,
|
||||||
|
server_default="simple",
|
||||||
|
),
|
||||||
|
# "pause_batch" (default) | "continue_item"
|
||||||
|
sa.Column(
|
||||||
|
"analysis_failure_behavior",
|
||||||
|
sa.String(16),
|
||||||
|
nullable=False,
|
||||||
|
server_default="pause_batch",
|
||||||
|
),
|
||||||
|
# User-preferred cache byte limit (capped by tier/admin maximum in service layer)
|
||||||
|
# Default: 512 MB
|
||||||
|
sa.Column(
|
||||||
|
"cloud_cache_limit_bytes",
|
||||||
|
sa.BigInteger,
|
||||||
|
nullable=False,
|
||||||
|
server_default=str(512 * 1024 * 1024),
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.TIMESTAMP(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at",
|
||||||
|
sa.TIMESTAMP(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
# Drop in reverse-creation order. Phase 14 objects only — no Phase 12/13 changes.
|
||||||
|
op.drop_table("user_analysis_settings")
|
||||||
|
|
||||||
|
op.drop_index("ix_job_items_job_id", table_name="cloud_analysis_job_items")
|
||||||
|
op.drop_index("ix_job_items_user_item_version", table_name="cloud_analysis_job_items")
|
||||||
|
op.drop_table("cloud_analysis_job_items")
|
||||||
|
|
||||||
|
op.drop_index("ix_analysis_jobs_user_status", table_name="cloud_analysis_jobs")
|
||||||
|
op.drop_index("ix_analysis_jobs_user_id", table_name="cloud_analysis_jobs")
|
||||||
|
op.drop_table("cloud_analysis_jobs")
|
||||||
|
|
||||||
|
op.drop_index("ix_cache_entries_user_active", table_name="cloud_byte_cache_entries")
|
||||||
|
op.drop_index("ix_cache_entries_user_evicted_accessed", table_name="cloud_byte_cache_entries")
|
||||||
|
op.drop_table("cloud_byte_cache_entries")
|
||||||
Reference in New Issue
Block a user