feat(14-02): add version-key, settings helpers and Phase 14 cache service functions
- cloud_analysis_versioning.py: compute_version_key with version>etag>fingerprint precedence (D-19); compute_fingerprint helper; content_hash is optional post- hydration only (D-20); raises ValueError only, no HTTPException - cloud_analysis_settings.py: get_or_create_user_analysis_settings (upsert on first access), update_user_analysis_settings with enum validation and tier-capped cache limit, build_default_settings for API responses; raises ValueError only - cloud_cache.py: Phase 14 durable cache functions added alongside existing Phase 12 in-memory TTL cache — create_cache_entry, list_cache_entries (owner-scoped), evict_lru_entries (LRU, skips pin_count>0 and active_job_count>0), increment/ decrement_quota_for_cache (atomic UPDATE pattern); re-exports compute_version_key from cloud_analysis_versioning so tests have one import site
This commit is contained in:
curo1305
@@ -1,6 +1,7 @@
|
||||
"""
|
||||
Cloud folder listing cache for DocuVault.
|
||||
Cloud cache service for DocuVault — Phase 12 in-memory layer + Phase 14 durable cache.
|
||||
|
||||
── Phase 12: in-memory folder listing cache ─────────────────────────────────────
|
||||
Provides a module-level TTLCache singleton for caching cloud provider folder
|
||||
listings with a 60-second TTL (D-16: live API calls with 60-second in-memory TTL).
|
||||
|
||||
@@ -17,13 +18,39 @@ Cache key scheme: "{user_id}:{provider}:{folder_id}"
|
||||
References:
|
||||
RESEARCH.md Pattern 8 — TTLCache thread safety + asyncio integration
|
||||
D-16 — 60-second TTL, in-memory cache (not Redis)
|
||||
|
||||
── Phase 14: durable byte cache (cloud_byte_cache_entries) ─────────────────────
|
||||
Exports create_cache_entry, evict_lru_entries, list_cache_entries,
|
||||
increment_quota_for_cache, decrement_quota_for_cache for use by Celery tasks
|
||||
and the analysis API.
|
||||
|
||||
Also re-exports compute_version_key from services.cloud_analysis_versioning so
|
||||
tests and service callers have a single import location.
|
||||
|
||||
Rules:
|
||||
- Raises ValueError or domain exceptions only — never HTTPException.
|
||||
- Eviction never removes entries with pin_count > 0 or active_job_count > 0.
|
||||
- Eviction is per-user LRU (oldest last_accessed_at first).
|
||||
- quota.used_bytes is adjusted only via increment_quota_for_cache /
|
||||
decrement_quota_for_cache which use the project's atomic UPDATE pattern.
|
||||
- object_key is never returned in API responses (T-14-02).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from typing import Any, Callable, Awaitable
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Callable, Awaitable, List, Optional, Sequence
|
||||
|
||||
from cachetools import TTLCache
|
||||
from sqlalchemy import select, update, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from db.models import CloudByteCacheEntry
|
||||
from services.cloud_analysis_versioning import compute_version_key as _compute_version_key # noqa: F401
|
||||
|
||||
# Re-export so callers can do: from services.cloud_cache import compute_version_key
|
||||
compute_version_key = _compute_version_key
|
||||
|
||||
|
||||
# Module-level singleton: maxsize=1000 (sufficient for ~50 users × 20 folders),
|
||||
@@ -77,6 +104,228 @@ async def get_cloud_folders_cached(
|
||||
return result
|
||||
|
||||
|
||||
# ══ Phase 14: durable byte cache operations ══════════════════════════════════
|
||||
|
||||
|
||||
class CacheQuotaExceeded(ValueError):
|
||||
"""User's quota cannot accommodate the requested cache entry size."""
|
||||
|
||||
|
||||
async def create_cache_entry(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
user_id: uuid.UUID,
|
||||
connection_id: uuid.UUID,
|
||||
cloud_item_id: uuid.UUID,
|
||||
provider_item_id: str,
|
||||
version_key: str,
|
||||
object_key: str,
|
||||
content_type: Optional[str] = None,
|
||||
size_bytes: int,
|
||||
content_hash: Optional[str] = None,
|
||||
pin_count: int = 0,
|
||||
active_job_count: int = 0,
|
||||
last_accessed_at: Optional[datetime] = None,
|
||||
) -> CloudByteCacheEntry:
|
||||
"""Create and flush a new cloud_byte_cache_entries row.
|
||||
|
||||
Does NOT update quota — call increment_quota_for_cache separately.
|
||||
Does NOT enforce uniqueness beyond the DB constraint; callers must check
|
||||
for an existing entry before calling this function.
|
||||
|
||||
Args:
|
||||
session: Active async SQLAlchemy session.
|
||||
user_id: Owner UUID.
|
||||
connection_id: Cloud connection UUID.
|
||||
cloud_item_id: CloudItem row UUID.
|
||||
provider_item_id: Snapshot of the provider item id for audit/debug.
|
||||
version_key: Idempotency key (from compute_version_key).
|
||||
object_key: Private MinIO object key — UUID-based, never exposed.
|
||||
content_type: Optional MIME type.
|
||||
size_bytes: Actual byte count stored in MinIO.
|
||||
content_hash: Optional post-hydration content hash.
|
||||
pin_count: Active preview/download leases (default 0).
|
||||
active_job_count: Active Celery jobs using these bytes (default 0).
|
||||
last_accessed_at: Optional LRU timestamp; defaults to now().
|
||||
|
||||
Returns:
|
||||
Flushed (but not committed) CloudByteCacheEntry row.
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
entry = CloudByteCacheEntry(
|
||||
user_id=user_id if isinstance(user_id, uuid.UUID) else uuid.UUID(str(user_id)),
|
||||
connection_id=connection_id if isinstance(connection_id, uuid.UUID) else uuid.UUID(str(connection_id)),
|
||||
cloud_item_id=cloud_item_id if isinstance(cloud_item_id, uuid.UUID) else uuid.UUID(str(cloud_item_id)),
|
||||
provider_item_id=provider_item_id,
|
||||
version_key=version_key,
|
||||
object_key=object_key,
|
||||
content_type=content_type,
|
||||
size_bytes=size_bytes,
|
||||
content_hash=content_hash,
|
||||
pin_count=pin_count,
|
||||
active_job_count=active_job_count,
|
||||
last_accessed_at=last_accessed_at or now,
|
||||
)
|
||||
session.add(entry)
|
||||
await session.flush()
|
||||
return entry
|
||||
|
||||
|
||||
async def list_cache_entries(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
user_id: uuid.UUID,
|
||||
include_evicted: bool = False,
|
||||
) -> Sequence[CloudByteCacheEntry]:
|
||||
"""Return cache entries owned by user_id.
|
||||
|
||||
Ownership is strictly enforced — only rows where user_id matches are
|
||||
returned. By default, evicted entries (evicted_at IS NOT NULL) are
|
||||
excluded.
|
||||
|
||||
Args:
|
||||
session: Active async SQLAlchemy session.
|
||||
user_id: Owner UUID.
|
||||
include_evicted: If True, include entries with evicted_at set.
|
||||
|
||||
Returns:
|
||||
Sequence of CloudByteCacheEntry rows (may be empty).
|
||||
"""
|
||||
uid = user_id if isinstance(user_id, uuid.UUID) else uuid.UUID(str(user_id))
|
||||
stmt = select(CloudByteCacheEntry).where(CloudByteCacheEntry.user_id == uid)
|
||||
if not include_evicted:
|
||||
stmt = stmt.where(CloudByteCacheEntry.evicted_at.is_(None))
|
||||
result = await session.execute(stmt)
|
||||
return result.scalars().all()
|
||||
|
||||
|
||||
async def evict_lru_entries(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
user_id: uuid.UUID,
|
||||
max_cache_bytes: int,
|
||||
) -> List[uuid.UUID]:
|
||||
"""Mark least-recently-used unpinned entries as evicted until total is within limit.
|
||||
|
||||
Eviction rules (D-14, D-16):
|
||||
- Only entries with pin_count == 0 and active_job_count == 0 are eligible.
|
||||
- Entries are evicted in ascending last_accessed_at order (oldest first).
|
||||
- Eviction stops when the sum of retained entry size_bytes <= max_cache_bytes.
|
||||
- evicted_at is stamped to now(); the caller is responsible for deleting
|
||||
the MinIO objects and calling decrement_quota_for_cache per evicted entry.
|
||||
|
||||
Args:
|
||||
session: Active async SQLAlchemy session.
|
||||
user_id: Owner UUID.
|
||||
max_cache_bytes: Target byte ceiling (entries evicted until we fit within).
|
||||
|
||||
Returns:
|
||||
List of evicted CloudByteCacheEntry UUIDs (empty if nothing needed eviction).
|
||||
"""
|
||||
uid = user_id if isinstance(user_id, uuid.UUID) else uuid.UUID(str(user_id))
|
||||
|
||||
# Fetch all non-evicted entries for this user ordered by LRU ascending
|
||||
stmt = (
|
||||
select(CloudByteCacheEntry)
|
||||
.where(
|
||||
CloudByteCacheEntry.user_id == uid,
|
||||
CloudByteCacheEntry.evicted_at.is_(None),
|
||||
)
|
||||
.order_by(CloudByteCacheEntry.last_accessed_at.asc().nulls_first())
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
entries = result.scalars().all()
|
||||
|
||||
# Sum total retained bytes
|
||||
total_bytes = sum(e.size_bytes for e in entries)
|
||||
now = datetime.now(timezone.utc)
|
||||
evicted_ids: List[uuid.UUID] = []
|
||||
|
||||
for entry in entries:
|
||||
if total_bytes <= max_cache_bytes:
|
||||
break
|
||||
# Skip pinned or actively-used entries
|
||||
if entry.pin_count > 0 or entry.active_job_count > 0:
|
||||
continue
|
||||
# Evict this entry
|
||||
entry.evicted_at = now
|
||||
entry.updated_at = now
|
||||
total_bytes -= entry.size_bytes
|
||||
evicted_ids.append(entry.id)
|
||||
|
||||
if evicted_ids:
|
||||
await session.flush()
|
||||
|
||||
return evicted_ids
|
||||
|
||||
|
||||
async def increment_quota_for_cache(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
user_id: uuid.UUID,
|
||||
size_bytes: int,
|
||||
) -> None:
|
||||
"""Atomically increment quota.used_bytes for a cached byte entry.
|
||||
|
||||
Uses the project's atomic UPDATE pattern to prevent TOCTOU race conditions:
|
||||
UPDATE quotas SET used_bytes = used_bytes + $delta
|
||||
WHERE user_id = $user_id AND (used_bytes + $delta) <= limit_bytes
|
||||
|
||||
Raises:
|
||||
CacheQuotaExceeded: The increment would exceed the user's quota limit.
|
||||
|
||||
Note:
|
||||
Against SQLite (test environments) this runs as a non-atomic update;
|
||||
the atomic guarantee is provided only by PostgreSQL in production.
|
||||
"""
|
||||
uid = user_id if isinstance(user_id, uuid.UUID) else uuid.UUID(str(user_id))
|
||||
|
||||
# Atomic UPDATE … RETURNING pattern (same as quota enforcement in upload flow)
|
||||
result = await session.execute(
|
||||
text(
|
||||
"UPDATE quotas SET used_bytes = used_bytes + :delta "
|
||||
"WHERE user_id = :uid AND (used_bytes + :delta) <= limit_bytes "
|
||||
"RETURNING used_bytes"
|
||||
),
|
||||
{"delta": size_bytes, "uid": str(uid)},
|
||||
)
|
||||
row = result.fetchone()
|
||||
if row is None:
|
||||
raise CacheQuotaExceeded(
|
||||
f"User {uid} quota exhausted — cannot cache {size_bytes} bytes"
|
||||
)
|
||||
|
||||
|
||||
async def decrement_quota_for_cache(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
user_id: uuid.UUID,
|
||||
size_bytes: int,
|
||||
) -> None:
|
||||
"""Atomically decrement quota.used_bytes when a cached entry is evicted.
|
||||
|
||||
Clamps to 0 to prevent negative quota values (defensive — should not occur
|
||||
in practice if increment/decrement are always called in pairs).
|
||||
|
||||
Args:
|
||||
session: Active async SQLAlchemy session.
|
||||
user_id: Owner UUID.
|
||||
size_bytes: Byte count to release.
|
||||
"""
|
||||
uid = user_id if isinstance(user_id, uuid.UUID) else uuid.UUID(str(user_id))
|
||||
|
||||
await session.execute(
|
||||
text(
|
||||
"UPDATE quotas SET used_bytes = GREATEST(0, used_bytes - :delta) "
|
||||
"WHERE user_id = :uid"
|
||||
),
|
||||
{"delta": size_bytes, "uid": str(uid)},
|
||||
)
|
||||
|
||||
|
||||
# ── Phase 12: in-memory folder listing cache ──────────────────────────────────
|
||||
|
||||
|
||||
def invalidate_provider_cache(user_id: str, provider: str) -> None:
|
||||
"""Invalidate all cached folder listings for a specific user + provider.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user