feat(14-02): add migration 0007 and ORM models for cloud analysis cache schema
- 0007_cloud_analysis_cache.py: new Alembic migration adding 4 Phase 14 tables
- cloud_byte_cache_entries: durable MinIO cache with version_key uniqueness,
pin_count/active_job_count guards, LRU eviction indexes
- cloud_analysis_jobs: batch job rows (scope_kind, status, aggregate counters,
provider_bytes_estimate, failure_behavior)
- cloud_analysis_job_items: per-item idempotency rows (status vocabulary,
controlled error fields, cache_entry_id, retry_count, version_key fingerprint)
- user_analysis_settings: per-user preferences (progress_detail, failure_behavior,
cloud_cache_limit_bytes)
- Reversible downgrade drops Phase 14 objects only
- db/models.py: CloudByteCacheEntry, CloudAnalysisJob, CloudAnalysisJobItem,
UserAnalysisSettings ORM models mirroring migration
This commit is contained in:
curo1305
@@ -457,6 +457,249 @@ class CloudFolderState(Base):
|
||||
)
|
||||
|
||||
|
||||
class CloudByteCacheEntry(Base):
|
||||
"""Durable per-item provider byte cache entry backed by MinIO.
|
||||
|
||||
Tracks retained cloud bytes for active open, preview, or analysis work.
|
||||
size_bytes counts against quota.used_bytes while retained; decremented on eviction.
|
||||
|
||||
Eviction rules (D-14, D-16):
|
||||
- Only entries with pin_count == 0 and active_job_count == 0 may be evicted.
|
||||
- Eviction is per-user LRU ordered by last_accessed_at ascending.
|
||||
- object_key is a UUID-based private MinIO key — never exposed in API responses (T-14-02).
|
||||
|
||||
Phase 14: cache lifecycle (create -> pin -> release -> evict) managed by
|
||||
services/cloud_cache.py.
|
||||
"""
|
||||
|
||||
__tablename__ = "cloud_byte_cache_entries"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
connection_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("cloud_connections.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
cloud_item_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("cloud_items.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
# Snapshot of provider_item_id — for audit/debug without joins
|
||||
provider_item_id: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
# Derived from version / etag / metadata fingerprint (content hash optional, post-hydration only)
|
||||
version_key: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
# Private MinIO object key — UUID-based, never exposed in API responses
|
||||
object_key: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
content_type: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
# Actual byte count stored in MinIO — debited from quota.used_bytes
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
# Optional content hash — computed ONLY after bytes already hydrated, never as a pre-download requirement
|
||||
content_hash: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
# pin_count > 0 = active preview/download lease; must not be evicted
|
||||
pin_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# active_job_count > 0 = Celery analysis task in flight; must not be evicted
|
||||
active_job_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
last_accessed_at: Mapped[Optional[datetime]] = mapped_column(
|
||||
TIMESTAMP(timezone=True), nullable=True
|
||||
)
|
||||
evicted_at: Mapped[Optional[datetime]] = mapped_column(
|
||||
TIMESTAMP(timezone=True), nullable=True
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"user_id",
|
||||
"connection_id",
|
||||
"cloud_item_id",
|
||||
"version_key",
|
||||
name="uq_cache_entry_user_connection_item_version",
|
||||
),
|
||||
Index("ix_cache_entries_user_evicted_accessed", "user_id", "evicted_at", "last_accessed_at"),
|
||||
Index("ix_cache_entries_user_active", "user_id", "pin_count", "active_job_count", "last_accessed_at"),
|
||||
)
|
||||
|
||||
|
||||
class CloudAnalysisJob(Base):
|
||||
"""User-visible analysis batch row.
|
||||
|
||||
Represents a single enqueue action for file, selection, folder, or connection scope.
|
||||
Aggregate counters are updated by Celery workers as items are processed.
|
||||
|
||||
status: "queued" | "running" | "paused" | "completed" | "cancelled" | "failed"
|
||||
scope_kind: "file" | "selection" | "folder" | "connection"
|
||||
failure_behavior: "pause_batch" (default) | "continue_item" (D-11)
|
||||
"""
|
||||
|
||||
__tablename__ = "cloud_analysis_jobs"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
connection_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("cloud_connections.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
scope_kind: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
scope_ref: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
recursive: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="queued")
|
||||
# Aggregate item counters
|
||||
total_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
queued_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
downloading_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
extracting_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
classifying_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
indexed_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
already_current_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
skipped_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
cancelled_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
failed_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
unsupported_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# Provider-reported byte total — for estimates only, never used for quota
|
||||
provider_bytes_estimate: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
failure_behavior: Mapped[str] = mapped_column(String(16), nullable=False, default="pause_batch")
|
||||
started_at: Mapped[Optional[datetime]] = mapped_column(TIMESTAMP(timezone=True), nullable=True)
|
||||
finished_at: Mapped[Optional[datetime]] = mapped_column(TIMESTAMP(timezone=True), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_analysis_jobs_user_id", "user_id"),
|
||||
Index("ix_analysis_jobs_user_status", "user_id", "status"),
|
||||
)
|
||||
|
||||
|
||||
class CloudAnalysisJobItem(Base):
|
||||
"""Per-item row for idempotent processing within a CloudAnalysisJob.
|
||||
|
||||
status vocabulary (PATTERNS.md):
|
||||
queued | downloading | extracting | classifying | indexed |
|
||||
already_current | cancelled | failed | unsupported | stale
|
||||
|
||||
error_code and error_message contain controlled strings only — never raw
|
||||
provider exception text or stack traces.
|
||||
|
||||
cache_entry_id is null until bytes are hydrated. It links to the
|
||||
CloudByteCacheEntry used during processing and is cleared when the cache
|
||||
entry is evicted.
|
||||
"""
|
||||
|
||||
__tablename__ = "cloud_analysis_job_items"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
|
||||
)
|
||||
job_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("cloud_analysis_jobs.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
connection_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("cloud_connections.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
cloud_item_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("cloud_items.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
provider_item_id: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
# version_key at enqueue time — re-resolved and compared during processing
|
||||
version_key: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
# Metadata fingerprint: "provider_item_id:size:modified_at:content_type"
|
||||
fingerprint: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="queued")
|
||||
# Controlled error strings — never raw provider error text
|
||||
error_code: Mapped[Optional[str]] = mapped_column(String(64), nullable=True)
|
||||
error_message: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
cache_entry_id: Mapped[Optional[uuid.UUID]] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("cloud_byte_cache_entries.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
)
|
||||
retry_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
started_at: Mapped[Optional[datetime]] = mapped_column(TIMESTAMP(timezone=True), nullable=True)
|
||||
finished_at: Mapped[Optional[datetime]] = mapped_column(TIMESTAMP(timezone=True), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("job_id", "cloud_item_id", name="uq_job_items_job_cloud_item"),
|
||||
Index("ix_job_items_user_item_version", "user_id", "cloud_item_id", "version_key"),
|
||||
Index("ix_job_items_job_id", "job_id"),
|
||||
)
|
||||
|
||||
|
||||
class UserAnalysisSettings(Base):
|
||||
"""Per-user analysis preferences — one row per user, created on demand.
|
||||
|
||||
Distinct from system_settings (AI provider config) and CloudFolderState.
|
||||
This is the single settings authority for user analysis preferences.
|
||||
|
||||
progress_detail: "simple" (default) | "detailed"
|
||||
analysis_failure_behavior: "pause_batch" (default) | "continue_item" (D-11)
|
||||
cloud_cache_limit_bytes: user preference capped by tier/admin maximum in the service layer
|
||||
"""
|
||||
|
||||
__tablename__ = "user_analysis_settings"
|
||||
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("users.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
)
|
||||
analysis_progress_detail: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="simple"
|
||||
)
|
||||
analysis_failure_behavior: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="pause_batch"
|
||||
)
|
||||
# Default: 512 MB
|
||||
cloud_cache_limit_bytes: Mapped[int] = mapped_column(
|
||||
BigInteger, nullable=False, default=512 * 1024 * 1024
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
|
||||
class Group(Base):
|
||||
"""v2 stub — empty table, seeded for schema completeness (PROJECT.md D-02).
|
||||
|
||||
|
||||
Reference in New Issue
Block a user