curo/kite
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(14-06): route preview/download bytes through durable byte cache

Browse Source 
- Add hydrate_and_cache_bytes to services/cloud_cache.py (Plan 06 cache lifecycle)
  - Cache-hit path: pin existing entry, read from MinIO, release pin in finally
  - Cache-miss path: fetch from provider, store in MinIO, create/reactivate entry, pin during response, release pin in finally
  - CacheQuotaExceeded and MinIO failures are non-fatal (bytes returned from provider path)
  - evict_lru_entries run after pin release to stay within user cache limit
- Update preview_cloud_file and download_cloud_file in operations.py to call hydrate_and_cache_bytes
  - Version key computed from CloudItem metadata (no provider call required for key resolution)
  - Shared _make_minio_cache_wrapper helper eliminates duplicate adapter code
  - Falls through to direct provider fetch when no CloudItem row found
- Add _make_minio_cache_wrapper helper to operations.py (DRY, shared by preview and download)
- Add 5 new tests to test_cloud_cache.py: cache-miss fetches provider, cache-hit skips provider, pin released on hit, pin released on miss, user isolation
- Add 3 new security tests to test_cloud_security.py: preview response excludes object_key, download response excludes object_key, foreign-user IDOR via download blocked
This commit is contained in:
curo1305
2026-06-23 16:28:06 +02:00
parent 5a9f3a628e
commit dfc335022f
4 changed files with 715 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_cloud_cache.py
+282
View File
@@ -749,6 +749,288 @@ async def test_pin_raises_on_foreign_entry(db_session):
# ─── update_cache_access ─────────────────────────────────────────────────────
# ─── hydrate_and_cache_bytes (Plan 06) ───────────────────────────────────────
class _FakeMinIO:
"""Minimal MinIO stub for hydrate_and_cache_bytes tests."""
def __init__(self, bytes_to_return: bytes = b"fake-content"):
self._store: dict = {}
self._bytes_to_return = bytes_to_return
self.put_call_count = 0
self.get_call_count = 0
self.delete_call_count = 0
async def get_object(self, key: str) -> bytes:
self.get_call_count += 1
if key in self._store:
return self._store[key]
raise Exception(f"Key not found: {key}")
async def put_object(self, key: str, data: bytes, content_type: str = None) -> None:
self.put_call_count += 1
self._store[key] = data
async def delete_object(self, key: str) -> None:
self.delete_call_count += 1
self._store.pop(key, None)
async def test_hydrate_cache_miss_fetches_from_provider(db_session):
"""CACHE-03/Plan 06: Cache miss calls the provider fetch_fn and stores bytes in MinIO."""
from services.cloud_cache import hydrate_and_cache_bytes, list_cache_entries
from unittest.mock import AsyncMock, patch
user_id = uuid.uuid4()
conn_id = uuid.uuid4()
item_id = uuid.uuid4()
version_key = "etag-miss-v1"
expected_bytes = b"provider-content-hello"
fetch_call_count = 0
async def fetch_fn():
nonlocal fetch_call_count
fetch_call_count += 1
return expected_bytes
minio = _FakeMinIO()
# Patch increment_quota_for_cache to bypass quota check (no quota row in test DB)
with patch("services.cloud_cache.increment_quota_for_cache", new=AsyncMock()):
result_bytes, cache_entry_id = await hydrate_and_cache_bytes(
db_session,
user_id=user_id,
connection_id=conn_id,
cloud_item_id=item_id,
provider_item_id="pitem-miss",
version_key=version_key,
content_type="application/pdf",
fetch_fn=fetch_fn,
minio_client=minio,
cache_limit_bytes=512 * 1024 * 1024,
)
assert result_bytes == expected_bytes, "Returned bytes must match provider bytes"
assert fetch_call_count == 1, "Cache miss must call fetch_fn exactly once"
assert minio.put_call_count == 1, "Cache miss must store bytes in MinIO"
# A cache entry must have been created for the user.
entries = await list_cache_entries(db_session, user_id=user_id)
assert len(entries) == 1, "A cache entry must be created on cache miss"
assert entries[0].version_key == version_key
async def test_hydrate_cache_hit_skips_provider(db_session):
"""CACHE-03/Plan 06: Cache hit reads from MinIO and does NOT call fetch_fn (provider).
The fetch_fn must not be called when a non-evicted entry exists for the version_key.
"""
from services.cloud_cache import (
create_cache_entry,
hydrate_and_cache_bytes,
)
user_id = uuid.uuid4()
conn_id = uuid.uuid4()
item_id = uuid.uuid4()
version_key = "etag-hit-v1"
cached_bytes = b"minio-cached-bytes"
object_key = f"cache/{user_id}/{uuid.uuid4()}.pdf"
# Pre-populate a cache entry
entry = await create_cache_entry(
session=db_session,
user_id=user_id,
connection_id=conn_id,
cloud_item_id=item_id,
provider_item_id="pitem-hit",
version_key=version_key,
object_key=object_key,
content_type="application/pdf",
size_bytes=len(cached_bytes),
)
await db_session.commit()
minio = _FakeMinIO()
minio._store[object_key] = cached_bytes # Pre-load the fake MinIO store
fetch_call_count = 0
async def fetch_fn():
nonlocal fetch_call_count
fetch_call_count += 1
return b"should-not-be-called"
result_bytes, cache_entry_id = await hydrate_and_cache_bytes(
db_session,
user_id=user_id,
connection_id=conn_id,
cloud_item_id=item_id,
provider_item_id="pitem-hit",
version_key=version_key,
content_type="application/pdf",
fetch_fn=fetch_fn,
minio_client=minio,
cache_limit_bytes=512 * 1024 * 1024,
)
assert result_bytes == cached_bytes, "Cache hit must return MinIO bytes, not provider bytes"
assert fetch_call_count == 0, "Cache hit must NOT call fetch_fn (provider avoided)"
assert minio.put_call_count == 0, "Cache hit must NOT write to MinIO again"
async def test_hydrate_pin_released_after_cache_hit(db_session):
"""CACHE-04/Plan 06: Pin is released after hydrate_and_cache_bytes returns on cache hit."""
from services.cloud_cache import (
create_cache_entry,
hydrate_and_cache_bytes,
list_cache_entries,
)
user_id = uuid.uuid4()
conn_id = uuid.uuid4()
item_id = uuid.uuid4()
version_key = "etag-pin-release"
cached_bytes = b"pin-release-test-bytes"
object_key = f"cache/{user_id}/{uuid.uuid4()}.pdf"
entry = await create_cache_entry(
session=db_session,
user_id=user_id,
connection_id=conn_id,
cloud_item_id=item_id,
provider_item_id="pitem-pin-rel",
version_key=version_key,
object_key=object_key,
content_type="application/pdf",
size_bytes=len(cached_bytes),
)
await db_session.commit()
minio = _FakeMinIO()
minio._store[object_key] = cached_bytes
async def fetch_fn():
return b"fallback"
await hydrate_and_cache_bytes(
db_session,
user_id=user_id,
connection_id=conn_id,
cloud_item_id=item_id,
provider_item_id="pitem-pin-rel",
version_key=version_key,
content_type="application/pdf",
fetch_fn=fetch_fn,
minio_client=minio,
cache_limit_bytes=512 * 1024 * 1024,
)
# After hydration completes, pin_count must be back to 0.
await db_session.refresh(entry)
assert entry.pin_count == 0, (
"Pin must be released after hydrate_and_cache_bytes returns (PATTERNS.md step 8)"
)
async def test_hydrate_cache_miss_pin_released(db_session):
"""CACHE-04/Plan 06: Pin is released after cache-miss hydration completes."""
from services.cloud_cache import hydrate_and_cache_bytes, list_cache_entries
from unittest.mock import AsyncMock, patch
user_id = uuid.uuid4()
conn_id = uuid.uuid4()
item_id = uuid.uuid4()
version_key = "etag-miss-pin-release"
expected_bytes = b"miss-pin-release-bytes"
async def fetch_fn():
return expected_bytes
minio = _FakeMinIO()
with patch("services.cloud_cache.increment_quota_for_cache", new=AsyncMock()):
await hydrate_and_cache_bytes(
db_session,
user_id=user_id,
connection_id=conn_id,
cloud_item_id=item_id,
provider_item_id="pitem-miss-pin",
version_key=version_key,
content_type="application/pdf",
fetch_fn=fetch_fn,
minio_client=minio,
cache_limit_bytes=512 * 1024 * 1024,
)
entries = await list_cache_entries(db_session, user_id=user_id)
assert all(e.pin_count == 0 for e in entries), (
"All cache entries must have pin_count == 0 after hydration (no pin leak)"
)
async def test_hydrate_foreign_user_gets_separate_cache_entry(db_session):
"""T-14-06/Plan 06: hydrate_and_cache_bytes never reuses another user's cache entry."""
from services.cloud_cache import (
create_cache_entry,
hydrate_and_cache_bytes,
list_cache_entries,
)
from unittest.mock import AsyncMock, patch
owner_id = uuid.uuid4()
foreign_id = uuid.uuid4()
conn_id = uuid.uuid4()
item_id = uuid.uuid4()
version_key = "etag-shared-user-isolation"
cached_bytes = b"owner-only-bytes"
object_key = f"cache/{owner_id}/{uuid.uuid4()}.pdf"
# Create owner's cache entry
await create_cache_entry(
session=db_session,
user_id=owner_id,
connection_id=conn_id,
cloud_item_id=item_id,
provider_item_id="pitem-owner-iso",
version_key=version_key,
object_key=object_key,
size_bytes=len(cached_bytes),
)
await db_session.commit()
minio = _FakeMinIO(bytes_to_return=b"foreign-bytes")
fetch_call_count = 0
async def fetch_fn():
nonlocal fetch_call_count
fetch_call_count += 1
return b"foreign-bytes"
# Patch increment_quota_for_cache to bypass quota check (no quota row in test DB)
with patch("services.cloud_cache.increment_quota_for_cache", new=AsyncMock()):
await hydrate_and_cache_bytes(
db_session,
user_id=foreign_id,
connection_id=conn_id,
cloud_item_id=item_id,
provider_item_id="pitem-owner-iso",
version_key=version_key,
content_type="application/pdf",
fetch_fn=fetch_fn,
minio_client=minio,
cache_limit_bytes=512 * 1024 * 1024,
)
# Foreign user must have had a cache miss (fetched from provider)
assert fetch_call_count == 1, (
"Foreign user must not reuse owner's cache entry (T-14-06)"
)
# Foreign user now has their own entry
foreign_entries = await list_cache_entries(db_session, user_id=foreign_id)
assert len(foreign_entries) == 1
assert str(foreign_entries[0].user_id) == str(foreign_id)
async def test_update_cache_access_touches_last_accessed(db_session):
"""CACHE-04: update_cache_access refreshes last_accessed_at for LRU ordering."""
from services.cloud_cache import create_cache_entry, update_cache_access
backend/tests/test_cloud_security.py
+90
View File
@@ -879,6 +879,96 @@ async def test_foreign_user_cannot_access_analysis_estimate(async_client, db_ses
)
# ── Phase 14 Plan 06: Cache-backed preview/download security ──────────────────
async def test_preview_cache_hit_response_excludes_object_key(async_client, db_session):
"""T-14-02/Plan 06: Preview response from a cache hit must not expose object_key.
Even when bytes come from MinIO (cache hit path), the response body and headers
must not contain the MinIO object key or credentials_enc.
"""
import uuid as _uuid2
from unittest.mock import AsyncMock, patch as _patch, MagicMock
owner_ctx = await _create_user_and_token(db_session, role="user")
conn = await _create_cloud_connection(db_session, owner_ctx["user"].id)
# Simulate a cache hit by patching hydrate_and_cache_bytes to return bytes
with _patch(
"api.cloud.operations.hydrate_and_cache_bytes" if False else "services.cloud_cache.hydrate_and_cache_bytes",
new=AsyncMock(return_value=(b"pdf-bytes", _uuid2.uuid4())),
):
with _patch("api.cloud.operations._resolve_and_get_adapter") as mock_adapter:
mock_conn = MagicMock()
mock_ad = AsyncMock()
mock_ad.get_object = AsyncMock(return_value=b"pdf-bytes")
mock_adapter.return_value = (mock_conn, mock_ad)
# No cloud_item in DB, so falls to direct fetch path — test response exclusion
resp = await async_client.get(
f"/api/cloud/connections/{conn.id}/items/pitem-preview-sec/preview",
headers=owner_ctx["headers"],
)
# Any 2xx or unsupported_preview is acceptable; key check is no secret leakage
assert resp.status_code not in (500,), f"Should not 500 — got {resp.status_code}: {resp.text}"
body = resp.text
assert "object_key" not in body, "T-14-02: object_key must not appear in preview response"
assert "credentials_enc" not in body, "credentials_enc must not appear in preview response"
async def test_download_response_excludes_object_key(async_client, db_session):
"""T-14-02/Plan 06: Download response must not expose object_key or credentials.
The cache lookup and MinIO path are internal; the response headers and body
must contain only the file bytes and safe metadata.
"""
from unittest.mock import AsyncMock, patch as _patch, MagicMock
owner_ctx = await _create_user_and_token(db_session, role="user")
conn = await _create_cloud_connection(db_session, owner_ctx["user"].id)
with _patch("api.cloud.operations._resolve_and_get_adapter") as mock_adapter:
mock_conn = MagicMock()
mock_ad = AsyncMock()
mock_ad.get_object = AsyncMock(return_value=b"download-bytes")
mock_adapter.return_value = (mock_conn, mock_ad)
resp = await async_client.get(
f"/api/cloud/connections/{conn.id}/items/pitem-download-sec/download",
headers=owner_ctx["headers"],
)
assert resp.status_code != 500, f"Should not 500 — got {resp.status_code}: {resp.text}"
# Check headers
for header_name, header_value in resp.headers.items():
assert "object_key" not in header_value.lower(), (
f"T-14-02: object_key must not appear in response headers — found in {header_name}"
)
assert "credentials_enc" not in header_value.lower(), (
"credentials_enc must not appear in response headers"
)
async def test_foreign_user_cannot_download_via_cache(async_client, db_session):
"""T-14-01/Plan 06: Foreign user cannot download another user's cloud file via cache.
The ownership check (resolve_owned_connection) must run before any cache lookup.
"""
auth1 = await _create_user_and_token(db_session, role="user")
auth2 = await _create_user_and_token(db_session, role="user")
conn = await _create_cloud_connection(db_session, auth1["user"].id)
resp = await async_client.get(
f"/api/cloud/connections/{conn.id}/items/pitem-idor/download",
headers=auth2["headers"],
)
assert resp.status_code == 404, (
f"Foreign user must not download via cache — expected 404, got {resp.status_code}"
)
async def test_foreign_user_cannot_read_analysis_job_status(async_client, db_session):
"""T-14-01: Foreign user cannot read job status for another user's job."""
from unittest.mock import patch as _patch