feat(14-06): route preview/download bytes through durable byte cache
- Add hydrate_and_cache_bytes to services/cloud_cache.py (Plan 06 cache lifecycle) - Cache-hit path: pin existing entry, read from MinIO, release pin in finally - Cache-miss path: fetch from provider, store in MinIO, create/reactivate entry, pin during response, release pin in finally - CacheQuotaExceeded and MinIO failures are non-fatal (bytes returned from provider path) - evict_lru_entries run after pin release to stay within user cache limit - Update preview_cloud_file and download_cloud_file in operations.py to call hydrate_and_cache_bytes - Version key computed from CloudItem metadata (no provider call required for key resolution) - Shared _make_minio_cache_wrapper helper eliminates duplicate adapter code - Falls through to direct provider fetch when no CloudItem row found - Add _make_minio_cache_wrapper helper to operations.py (DRY, shared by preview and download) - Add 5 new tests to test_cloud_cache.py: cache-miss fetches provider, cache-hit skips provider, pin released on hit, pin released on miss, user isolation - Add 3 new security tests to test_cloud_security.py: preview response excludes object_key, download response excludes object_key, foreign-user IDOR via download blocked
This commit is contained in:
curo1305
@@ -749,6 +749,288 @@ async def test_pin_raises_on_foreign_entry(db_session):
|
||||
|
||||
# ─── update_cache_access ─────────────────────────────────────────────────────
|
||||
|
||||
# ─── hydrate_and_cache_bytes (Plan 06) ───────────────────────────────────────
|
||||
|
||||
class _FakeMinIO:
|
||||
"""Minimal MinIO stub for hydrate_and_cache_bytes tests."""
|
||||
def __init__(self, bytes_to_return: bytes = b"fake-content"):
|
||||
self._store: dict = {}
|
||||
self._bytes_to_return = bytes_to_return
|
||||
self.put_call_count = 0
|
||||
self.get_call_count = 0
|
||||
self.delete_call_count = 0
|
||||
|
||||
async def get_object(self, key: str) -> bytes:
|
||||
self.get_call_count += 1
|
||||
if key in self._store:
|
||||
return self._store[key]
|
||||
raise Exception(f"Key not found: {key}")
|
||||
|
||||
async def put_object(self, key: str, data: bytes, content_type: str = None) -> None:
|
||||
self.put_call_count += 1
|
||||
self._store[key] = data
|
||||
|
||||
async def delete_object(self, key: str) -> None:
|
||||
self.delete_call_count += 1
|
||||
self._store.pop(key, None)
|
||||
|
||||
|
||||
async def test_hydrate_cache_miss_fetches_from_provider(db_session):
|
||||
"""CACHE-03/Plan 06: Cache miss calls the provider fetch_fn and stores bytes in MinIO."""
|
||||
from services.cloud_cache import hydrate_and_cache_bytes, list_cache_entries
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
user_id = uuid.uuid4()
|
||||
conn_id = uuid.uuid4()
|
||||
item_id = uuid.uuid4()
|
||||
version_key = "etag-miss-v1"
|
||||
expected_bytes = b"provider-content-hello"
|
||||
fetch_call_count = 0
|
||||
|
||||
async def fetch_fn():
|
||||
nonlocal fetch_call_count
|
||||
fetch_call_count += 1
|
||||
return expected_bytes
|
||||
|
||||
minio = _FakeMinIO()
|
||||
|
||||
# Patch increment_quota_for_cache to bypass quota check (no quota row in test DB)
|
||||
with patch("services.cloud_cache.increment_quota_for_cache", new=AsyncMock()):
|
||||
result_bytes, cache_entry_id = await hydrate_and_cache_bytes(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
connection_id=conn_id,
|
||||
cloud_item_id=item_id,
|
||||
provider_item_id="pitem-miss",
|
||||
version_key=version_key,
|
||||
content_type="application/pdf",
|
||||
fetch_fn=fetch_fn,
|
||||
minio_client=minio,
|
||||
cache_limit_bytes=512 * 1024 * 1024,
|
||||
)
|
||||
|
||||
assert result_bytes == expected_bytes, "Returned bytes must match provider bytes"
|
||||
assert fetch_call_count == 1, "Cache miss must call fetch_fn exactly once"
|
||||
assert minio.put_call_count == 1, "Cache miss must store bytes in MinIO"
|
||||
|
||||
# A cache entry must have been created for the user.
|
||||
entries = await list_cache_entries(db_session, user_id=user_id)
|
||||
assert len(entries) == 1, "A cache entry must be created on cache miss"
|
||||
assert entries[0].version_key == version_key
|
||||
|
||||
|
||||
async def test_hydrate_cache_hit_skips_provider(db_session):
|
||||
"""CACHE-03/Plan 06: Cache hit reads from MinIO and does NOT call fetch_fn (provider).
|
||||
|
||||
The fetch_fn must not be called when a non-evicted entry exists for the version_key.
|
||||
"""
|
||||
from services.cloud_cache import (
|
||||
create_cache_entry,
|
||||
hydrate_and_cache_bytes,
|
||||
)
|
||||
|
||||
user_id = uuid.uuid4()
|
||||
conn_id = uuid.uuid4()
|
||||
item_id = uuid.uuid4()
|
||||
version_key = "etag-hit-v1"
|
||||
cached_bytes = b"minio-cached-bytes"
|
||||
object_key = f"cache/{user_id}/{uuid.uuid4()}.pdf"
|
||||
|
||||
# Pre-populate a cache entry
|
||||
entry = await create_cache_entry(
|
||||
session=db_session,
|
||||
user_id=user_id,
|
||||
connection_id=conn_id,
|
||||
cloud_item_id=item_id,
|
||||
provider_item_id="pitem-hit",
|
||||
version_key=version_key,
|
||||
object_key=object_key,
|
||||
content_type="application/pdf",
|
||||
size_bytes=len(cached_bytes),
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
minio = _FakeMinIO()
|
||||
minio._store[object_key] = cached_bytes # Pre-load the fake MinIO store
|
||||
|
||||
fetch_call_count = 0
|
||||
|
||||
async def fetch_fn():
|
||||
nonlocal fetch_call_count
|
||||
fetch_call_count += 1
|
||||
return b"should-not-be-called"
|
||||
|
||||
result_bytes, cache_entry_id = await hydrate_and_cache_bytes(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
connection_id=conn_id,
|
||||
cloud_item_id=item_id,
|
||||
provider_item_id="pitem-hit",
|
||||
version_key=version_key,
|
||||
content_type="application/pdf",
|
||||
fetch_fn=fetch_fn,
|
||||
minio_client=minio,
|
||||
cache_limit_bytes=512 * 1024 * 1024,
|
||||
)
|
||||
|
||||
assert result_bytes == cached_bytes, "Cache hit must return MinIO bytes, not provider bytes"
|
||||
assert fetch_call_count == 0, "Cache hit must NOT call fetch_fn (provider avoided)"
|
||||
assert minio.put_call_count == 0, "Cache hit must NOT write to MinIO again"
|
||||
|
||||
|
||||
async def test_hydrate_pin_released_after_cache_hit(db_session):
|
||||
"""CACHE-04/Plan 06: Pin is released after hydrate_and_cache_bytes returns on cache hit."""
|
||||
from services.cloud_cache import (
|
||||
create_cache_entry,
|
||||
hydrate_and_cache_bytes,
|
||||
list_cache_entries,
|
||||
)
|
||||
|
||||
user_id = uuid.uuid4()
|
||||
conn_id = uuid.uuid4()
|
||||
item_id = uuid.uuid4()
|
||||
version_key = "etag-pin-release"
|
||||
cached_bytes = b"pin-release-test-bytes"
|
||||
object_key = f"cache/{user_id}/{uuid.uuid4()}.pdf"
|
||||
|
||||
entry = await create_cache_entry(
|
||||
session=db_session,
|
||||
user_id=user_id,
|
||||
connection_id=conn_id,
|
||||
cloud_item_id=item_id,
|
||||
provider_item_id="pitem-pin-rel",
|
||||
version_key=version_key,
|
||||
object_key=object_key,
|
||||
content_type="application/pdf",
|
||||
size_bytes=len(cached_bytes),
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
minio = _FakeMinIO()
|
||||
minio._store[object_key] = cached_bytes
|
||||
|
||||
async def fetch_fn():
|
||||
return b"fallback"
|
||||
|
||||
await hydrate_and_cache_bytes(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
connection_id=conn_id,
|
||||
cloud_item_id=item_id,
|
||||
provider_item_id="pitem-pin-rel",
|
||||
version_key=version_key,
|
||||
content_type="application/pdf",
|
||||
fetch_fn=fetch_fn,
|
||||
minio_client=minio,
|
||||
cache_limit_bytes=512 * 1024 * 1024,
|
||||
)
|
||||
|
||||
# After hydration completes, pin_count must be back to 0.
|
||||
await db_session.refresh(entry)
|
||||
assert entry.pin_count == 0, (
|
||||
"Pin must be released after hydrate_and_cache_bytes returns (PATTERNS.md step 8)"
|
||||
)
|
||||
|
||||
|
||||
async def test_hydrate_cache_miss_pin_released(db_session):
|
||||
"""CACHE-04/Plan 06: Pin is released after cache-miss hydration completes."""
|
||||
from services.cloud_cache import hydrate_and_cache_bytes, list_cache_entries
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
user_id = uuid.uuid4()
|
||||
conn_id = uuid.uuid4()
|
||||
item_id = uuid.uuid4()
|
||||
version_key = "etag-miss-pin-release"
|
||||
expected_bytes = b"miss-pin-release-bytes"
|
||||
|
||||
async def fetch_fn():
|
||||
return expected_bytes
|
||||
|
||||
minio = _FakeMinIO()
|
||||
|
||||
with patch("services.cloud_cache.increment_quota_for_cache", new=AsyncMock()):
|
||||
await hydrate_and_cache_bytes(
|
||||
db_session,
|
||||
user_id=user_id,
|
||||
connection_id=conn_id,
|
||||
cloud_item_id=item_id,
|
||||
provider_item_id="pitem-miss-pin",
|
||||
version_key=version_key,
|
||||
content_type="application/pdf",
|
||||
fetch_fn=fetch_fn,
|
||||
minio_client=minio,
|
||||
cache_limit_bytes=512 * 1024 * 1024,
|
||||
)
|
||||
|
||||
entries = await list_cache_entries(db_session, user_id=user_id)
|
||||
assert all(e.pin_count == 0 for e in entries), (
|
||||
"All cache entries must have pin_count == 0 after hydration (no pin leak)"
|
||||
)
|
||||
|
||||
|
||||
async def test_hydrate_foreign_user_gets_separate_cache_entry(db_session):
|
||||
"""T-14-06/Plan 06: hydrate_and_cache_bytes never reuses another user's cache entry."""
|
||||
from services.cloud_cache import (
|
||||
create_cache_entry,
|
||||
hydrate_and_cache_bytes,
|
||||
list_cache_entries,
|
||||
)
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
owner_id = uuid.uuid4()
|
||||
foreign_id = uuid.uuid4()
|
||||
conn_id = uuid.uuid4()
|
||||
item_id = uuid.uuid4()
|
||||
version_key = "etag-shared-user-isolation"
|
||||
cached_bytes = b"owner-only-bytes"
|
||||
object_key = f"cache/{owner_id}/{uuid.uuid4()}.pdf"
|
||||
|
||||
# Create owner's cache entry
|
||||
await create_cache_entry(
|
||||
session=db_session,
|
||||
user_id=owner_id,
|
||||
connection_id=conn_id,
|
||||
cloud_item_id=item_id,
|
||||
provider_item_id="pitem-owner-iso",
|
||||
version_key=version_key,
|
||||
object_key=object_key,
|
||||
size_bytes=len(cached_bytes),
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
minio = _FakeMinIO(bytes_to_return=b"foreign-bytes")
|
||||
fetch_call_count = 0
|
||||
|
||||
async def fetch_fn():
|
||||
nonlocal fetch_call_count
|
||||
fetch_call_count += 1
|
||||
return b"foreign-bytes"
|
||||
|
||||
# Patch increment_quota_for_cache to bypass quota check (no quota row in test DB)
|
||||
with patch("services.cloud_cache.increment_quota_for_cache", new=AsyncMock()):
|
||||
await hydrate_and_cache_bytes(
|
||||
db_session,
|
||||
user_id=foreign_id,
|
||||
connection_id=conn_id,
|
||||
cloud_item_id=item_id,
|
||||
provider_item_id="pitem-owner-iso",
|
||||
version_key=version_key,
|
||||
content_type="application/pdf",
|
||||
fetch_fn=fetch_fn,
|
||||
minio_client=minio,
|
||||
cache_limit_bytes=512 * 1024 * 1024,
|
||||
)
|
||||
|
||||
# Foreign user must have had a cache miss (fetched from provider)
|
||||
assert fetch_call_count == 1, (
|
||||
"Foreign user must not reuse owner's cache entry (T-14-06)"
|
||||
)
|
||||
# Foreign user now has their own entry
|
||||
foreign_entries = await list_cache_entries(db_session, user_id=foreign_id)
|
||||
assert len(foreign_entries) == 1
|
||||
assert str(foreign_entries[0].user_id) == str(foreign_id)
|
||||
|
||||
|
||||
async def test_update_cache_access_touches_last_accessed(db_session):
|
||||
"""CACHE-04: update_cache_access refreshes last_accessed_at for LRU ordering."""
|
||||
from services.cloud_cache import create_cache_entry, update_cache_access
|
||||
|
||||
Reference in New Issue
Block a user