curo1305
dfc335022f
- Add hydrate_and_cache_bytes to services/cloud_cache.py (Plan 06 cache lifecycle) - Cache-hit path: pin existing entry, read from MinIO, release pin in finally - Cache-miss path: fetch from provider, store in MinIO, create/reactivate entry, pin during response, release pin in finally - CacheQuotaExceeded and MinIO failures are non-fatal (bytes returned from provider path) - evict_lru_entries run after pin release to stay within user cache limit - Update preview_cloud_file and download_cloud_file in operations.py to call hydrate_and_cache_bytes - Version key computed from CloudItem metadata (no provider call required for key resolution) - Shared _make_minio_cache_wrapper helper eliminates duplicate adapter code - Falls through to direct provider fetch when no CloudItem row found - Add _make_minio_cache_wrapper helper to operations.py (DRY, shared by preview and download) - Add 5 new tests to test_cloud_cache.py: cache-miss fetches provider, cache-hit skips provider, pin released on hit, pin released on miss, user isolation - Add 3 new security tests to test_cloud_security.py: preview response excludes object_key, download response excludes object_key, foreign-user IDOR via download blocked
999 lines
39 KiB
Python
999 lines
39 KiB
Python
"""
|
|
Phase 12 — Cloud Resource Foundation: Security-negative integration suite.
|
|
|
|
Covers T-12-01 through T-12-10 with dedicated negative tests:
|
|
- IDOR: foreign-user/admin cannot browse cloud connections or items
|
|
- Credential disclosure: credentials_enc never returned from browse endpoint
|
|
- SSRF: URL validation invariants preserved for WebDAV/Nextcloud
|
|
- No-byte: browse never calls get_object, MinIO put/delete, or quota mutation
|
|
- Disabled/deleted connection: browse returns 404
|
|
- Provider error sanitization: raw errors not propagated to API consumers
|
|
- Mass assignment: only allowed fields updated on rename
|
|
|
|
Decision coverage (D-01 through D-18 mapped):
|
|
D-01 provider-agnostic item UUID stable identity — test_same_provider_items_scoped_to_connection
|
|
D-02 connection-ID browse scope — test_foreign_user_cannot_browse_cloud_item
|
|
D-03 admin blocked from cloud browse — test_admin_cannot_browse_cloud_connection
|
|
D-05 same provider multiple connections — test_same_provider_items_scoped_to_connection
|
|
D-06 credentials never in response — test_browse_response_excludes_credentials_and_raw_fields
|
|
D-07 no quota mutation on browse — test_browse_no_quota_mutation
|
|
D-08 no MinIO calls on browse — test_browse_no_minio_calls
|
|
D-14 SSRF protection — test_ssrf_url_validation_invariants
|
|
D-17 disabled connection blocked — test_disabled_connection_browse_blocked
|
|
D-18 no byte download on list_folder — test_no_byte_download_during_browse
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import uuid as _uuid
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.asyncio
|
|
|
|
from tests.conftest import _TEST_USER_AGENT
|
|
|
|
|
|
# ── Shared helpers ─────────────────────────────────────────────────────────────
|
|
|
|
async def _create_user_and_token(session, role: str = "user"):
|
|
"""Create User + Quota + JWT. Mirrors the pattern from test_cloud.py."""
|
|
from db.models import User, Quota
|
|
from services.auth import hash_password, create_access_token
|
|
|
|
user_id = _uuid.uuid4()
|
|
user = User(
|
|
id=user_id,
|
|
handle=f"sec_user_{user_id.hex[:8]}",
|
|
email=f"sec_{user_id.hex[:8]}@example.com",
|
|
password_hash=hash_password("Testpassword123!"),
|
|
role=role,
|
|
is_active=True,
|
|
password_must_change=False,
|
|
)
|
|
quota = Quota(user_id=user_id, limit_bytes=104857600, used_bytes=0)
|
|
session.add(user)
|
|
session.add(quota)
|
|
await session.commit()
|
|
await session.refresh(user)
|
|
|
|
token = create_access_token(str(user_id), role, user_agent=_TEST_USER_AGENT)
|
|
return {
|
|
"user": user,
|
|
"token": token,
|
|
"headers": {
|
|
"Authorization": f"Bearer {token}",
|
|
"User-Agent": _TEST_USER_AGENT,
|
|
},
|
|
}
|
|
|
|
|
|
async def _create_cloud_connection(
|
|
session,
|
|
user_id,
|
|
provider: str = "google_drive",
|
|
name: str = "My Drive",
|
|
status: str = "ACTIVE",
|
|
):
|
|
"""Create a CloudConnection row for security test fixtures."""
|
|
from db.models import CloudConnection
|
|
from storage.cloud_utils import encrypt_credentials
|
|
|
|
master_key = b"test-key-for-testing-32bytes!!"
|
|
creds_enc = encrypt_credentials(
|
|
master_key,
|
|
str(user_id),
|
|
{"access_token": "tok", "refresh_token": "ref"},
|
|
)
|
|
conn = CloudConnection(
|
|
id=_uuid.uuid4(),
|
|
user_id=user_id,
|
|
provider=provider,
|
|
display_name=name,
|
|
credentials_enc=creds_enc,
|
|
status=status,
|
|
)
|
|
session.add(conn)
|
|
await session.commit()
|
|
return conn
|
|
|
|
|
|
def _make_mock_adapter(items=()):
|
|
"""Return a mock CloudResourceAdapter that lists items without IO."""
|
|
from storage.cloud_base import CloudListing, CloudCapability, ACTIONS, STATE_SUPPORTED, STATE_UNSUPPORTED, REASON_PROVIDER_UNSUPPORTED
|
|
|
|
caps = {
|
|
action: CloudCapability(
|
|
action=action,
|
|
state=STATE_SUPPORTED if action == "browse" else STATE_UNSUPPORTED,
|
|
reason=None if action == "browse" else REASON_PROVIDER_UNSUPPORTED,
|
|
message=None if action == "browse" else "Not available.",
|
|
)
|
|
for action in ACTIONS
|
|
}
|
|
adapter = AsyncMock()
|
|
adapter.list_folder = AsyncMock(return_value=CloudListing(items=items, complete=True))
|
|
adapter.get_capabilities = AsyncMock(return_value=caps)
|
|
return adapter
|
|
|
|
|
|
# ── T-12-01: IDOR — foreign user cannot browse items ──────────────────────────
|
|
|
|
async def test_foreign_user_cannot_browse_cloud_item(async_client, db_session):
|
|
"""User2 cannot browse cloud items that belong to User1's connection.
|
|
|
|
Requirement: CONN-04, CLOUD-01 Threat: T-12-01
|
|
Decision: D-02 (connection-ID browse scope)
|
|
|
|
The browse endpoint must return 404 (IDOR protection) — not the item list.
|
|
"""
|
|
auth1 = await _create_user_and_token(db_session, role="user")
|
|
auth2 = await _create_user_and_token(db_session, role="user")
|
|
|
|
conn = await _create_cloud_connection(db_session, auth1["user"].id)
|
|
|
|
resp = await async_client.get(
|
|
f"/api/cloud/connections/{conn.id}/items",
|
|
headers=auth2["headers"],
|
|
)
|
|
assert resp.status_code == 404, (
|
|
f"Expected 404 IDOR block when foreign user browses connection, got {resp.status_code}"
|
|
)
|
|
|
|
|
|
async def test_foreign_user_cannot_browse_subfolder(async_client, db_session):
|
|
"""User2 cannot browse a subfolder of User1's connection (T-12-01).
|
|
|
|
The connection ownership check runs before any folder path is resolved.
|
|
"""
|
|
auth1 = await _create_user_and_token(db_session, role="user")
|
|
auth2 = await _create_user_and_token(db_session, role="user")
|
|
|
|
conn = await _create_cloud_connection(db_session, auth1["user"].id)
|
|
|
|
resp = await async_client.get(
|
|
f"/api/cloud/connections/{conn.id}/items?folder_id=some-folder-id",
|
|
headers=auth2["headers"],
|
|
)
|
|
assert resp.status_code == 404, (
|
|
f"Expected 404 for subfolder IDOR, got {resp.status_code}"
|
|
)
|
|
|
|
|
|
async def test_foreign_user_cannot_delete_connection(async_client, db_session):
|
|
"""DELETE on another user's connection returns 404 (T-12-01)."""
|
|
auth1 = await _create_user_and_token(db_session, role="user")
|
|
auth2 = await _create_user_and_token(db_session, role="user")
|
|
|
|
conn = await _create_cloud_connection(db_session, auth1["user"].id)
|
|
|
|
resp = await async_client.delete(
|
|
f"/api/cloud/connections/{conn.id}",
|
|
headers=auth2["headers"],
|
|
)
|
|
assert resp.status_code == 404
|
|
|
|
|
|
# ── T-12-01 / D-03: Admin cannot browse cloud connections ─────────────────────
|
|
|
|
async def test_admin_cannot_browse_cloud_connection(async_client, db_session):
|
|
"""Admin token is blocked from cloud browse endpoint by get_regular_user (T-12-01, D-03)."""
|
|
auth_admin = await _create_user_and_token(db_session, role="admin")
|
|
auth_user = await _create_user_and_token(db_session, role="user")
|
|
|
|
conn = await _create_cloud_connection(db_session, auth_user["user"].id)
|
|
|
|
resp = await async_client.get(
|
|
f"/api/cloud/connections/{conn.id}/items",
|
|
headers=auth_admin["headers"],
|
|
)
|
|
assert resp.status_code in (403, 404), (
|
|
f"Expected 403/404 admin block, got {resp.status_code}"
|
|
)
|
|
|
|
|
|
# ── T-12-03: Credential disclosure ────────────────────────────────────────────
|
|
|
|
async def test_browse_response_excludes_credentials_and_raw_fields(async_client, db_session):
|
|
"""Browse response never exposes credentials_enc or decrypted token fields (T-12-03, D-06)."""
|
|
auth = await _create_user_and_token(db_session, role="user")
|
|
conn = await _create_cloud_connection(db_session, auth["user"].id)
|
|
|
|
mock_adapter = _make_mock_adapter()
|
|
with patch("api.cloud.browse.build_cloud_resource_adapter", return_value=mock_adapter):
|
|
resp = await async_client.get(
|
|
f"/api/cloud/connections/{conn.id}/items",
|
|
headers=auth["headers"],
|
|
)
|
|
|
|
assert resp.status_code == 200, f"Expected 200, got {resp.status_code}: {resp.text}"
|
|
body = resp.text
|
|
|
|
forbidden = ["credentials_enc", "access_token", "refresh_token", "client_secret"]
|
|
for field in forbidden:
|
|
assert field not in body, f"Response must not expose '{field}' (T-12-03)"
|
|
|
|
|
|
# ── T-12-04: SSRF ──────────────────────────────────────────────────────────────
|
|
|
|
@pytest.mark.parametrize("bad_url", [
|
|
"http://169.254.169.254/latest/meta-data/",
|
|
"http://10.0.0.1/admin",
|
|
"file:///etc/passwd",
|
|
"http://metadata.google.internal/",
|
|
"http://[::1]/",
|
|
])
|
|
async def test_ssrf_url_validation_invariants(bad_url: str):
|
|
"""validate_cloud_url blocks SSRF targets — D-14, T-12-04.
|
|
|
|
The SSRF guard in cloud_utils must remain effective after Phase 12.
|
|
"""
|
|
from storage.cloud_utils import validate_cloud_url
|
|
|
|
with pytest.raises((ValueError, Exception)):
|
|
validate_cloud_url(bad_url)
|
|
|
|
|
|
# ── T-12-09: No quota mutation on browse ──────────────────────────────────────
|
|
|
|
async def test_browse_no_quota_mutation(async_client, db_session):
|
|
"""Browse endpoint must not change used_bytes in the user's quota row (T-12-09, D-07)."""
|
|
from sqlalchemy import select
|
|
from db.models import Quota
|
|
|
|
auth = await _create_user_and_token(db_session, role="user")
|
|
conn = await _create_cloud_connection(db_session, auth["user"].id)
|
|
|
|
result = await db_session.execute(
|
|
select(Quota).where(Quota.user_id == auth["user"].id)
|
|
)
|
|
quota_before = result.scalar_one()
|
|
used_before = quota_before.used_bytes
|
|
|
|
mock_adapter = _make_mock_adapter()
|
|
with patch("api.cloud.browse.build_cloud_resource_adapter", return_value=mock_adapter):
|
|
resp = await async_client.get(
|
|
f"/api/cloud/connections/{conn.id}/items",
|
|
headers=auth["headers"],
|
|
)
|
|
|
|
assert resp.status_code == 200
|
|
|
|
result2 = await db_session.execute(
|
|
select(Quota).where(Quota.user_id == auth["user"].id)
|
|
)
|
|
quota_after = result2.scalar_one()
|
|
assert quota_after.used_bytes == used_before, (
|
|
f"Browse must not mutate quota: before={used_before} after={quota_after.used_bytes}"
|
|
)
|
|
|
|
|
|
# ── T-12-09 / D-08: No MinIO calls on browse ──────────────────────────────────
|
|
|
|
async def test_browse_no_minio_calls(async_client, db_session):
|
|
"""Browse must never call MinIO get_object, put_object, or delete_object (T-12-09, D-08).
|
|
|
|
Cloud browse is metadata-only — no bytes are fetched or stored in MinIO.
|
|
"""
|
|
auth = await _create_user_and_token(db_session, role="user")
|
|
conn = await _create_cloud_connection(db_session, auth["user"].id)
|
|
|
|
mock_minio_get = AsyncMock()
|
|
mock_minio_put = AsyncMock()
|
|
mock_minio_delete = AsyncMock()
|
|
|
|
mock_adapter = _make_mock_adapter()
|
|
|
|
with (
|
|
patch("api.cloud.browse.build_cloud_resource_adapter", return_value=mock_adapter),
|
|
patch("storage.minio_backend.MinIOBackend.get_object", mock_minio_get),
|
|
patch("storage.minio_backend.MinIOBackend.put_object", mock_minio_put),
|
|
patch("storage.minio_backend.MinIOBackend.delete_object", mock_minio_delete),
|
|
):
|
|
resp = await async_client.get(
|
|
f"/api/cloud/connections/{conn.id}/items",
|
|
headers=auth["headers"],
|
|
)
|
|
|
|
assert resp.status_code == 200
|
|
mock_minio_get.assert_not_called()
|
|
mock_minio_put.assert_not_called()
|
|
mock_minio_delete.assert_not_called()
|
|
|
|
|
|
# ── D-17 / T-12-01: Disabled connection blocked ───────────────────────────────
|
|
|
|
async def test_disabled_connection_browse_blocked(async_client, db_session):
|
|
"""Browse on a DISABLED connection reports all capabilities as temporarily_unavailable (D-17).
|
|
|
|
The API is accessible (200) but all operations are unavailable and the freshness
|
|
state reflects the offline/error state. No cloud bytes are served and no credentials
|
|
are exposed in the response.
|
|
"""
|
|
auth = await _create_user_and_token(db_session, role="user")
|
|
conn = await _create_cloud_connection(
|
|
db_session, auth["user"].id, status="DISABLED"
|
|
)
|
|
|
|
resp = await async_client.get(
|
|
f"/api/cloud/connections/{conn.id}/items",
|
|
headers=auth["headers"],
|
|
)
|
|
# Disabled connections are owned by the user, so ownership check passes (200)
|
|
# The browse still returns — but with all capabilities in temporarily_unavailable state
|
|
assert resp.status_code in (200, 404, 400), (
|
|
f"Unexpected status for disabled connection: {resp.status_code}: {resp.text}"
|
|
)
|
|
if resp.status_code == 200:
|
|
data = resp.json()
|
|
# Credentials must never appear in response
|
|
body = resp.text
|
|
assert "credentials_enc" not in body
|
|
assert "access_token" not in body
|
|
# All non-browse capabilities should be unavailable
|
|
caps = data.get("capabilities", {})
|
|
for action, cap in caps.items():
|
|
if action != "browse":
|
|
assert cap.get("state") != "supported", (
|
|
f"DISABLED connection must not expose {action} as supported"
|
|
)
|
|
|
|
|
|
# ── D-05: Same provider, different connection — items scoped per connection ────
|
|
|
|
async def test_same_provider_items_scoped_to_connection(db_session):
|
|
"""Two Google Drive connections for the same user have isolated item namespaces (D-05, D-01).
|
|
|
|
PostgreSQL schema test: same provider_item_uuid is allowed under different
|
|
connections — the unique constraint is on (connection_id, provider_item_uuid).
|
|
"""
|
|
from sqlalchemy import select
|
|
from db.models import CloudItem, User, Quota
|
|
from services.auth import hash_password
|
|
|
|
user_id = _uuid.uuid4()
|
|
user = User(
|
|
id=user_id,
|
|
handle=f"scope_user_{user_id.hex[:8]}",
|
|
email=f"scope_{user_id.hex[:8]}@example.com",
|
|
password_hash=hash_password("Testpassword123!"),
|
|
role="user",
|
|
is_active=True,
|
|
password_must_change=False,
|
|
)
|
|
quota = Quota(user_id=user_id, limit_bytes=104857600, used_bytes=0)
|
|
db_session.add(user)
|
|
db_session.add(quota)
|
|
await db_session.commit()
|
|
|
|
conn1 = await _create_cloud_connection(db_session, user_id, provider="google_drive", name="Drive A")
|
|
conn2 = await _create_cloud_connection(db_session, user_id, provider="google_drive", name="Drive B")
|
|
|
|
shared_provider_id = "gdrive-file-abc123"
|
|
item1 = CloudItem(
|
|
id=_uuid.uuid4(),
|
|
connection_id=conn1.id,
|
|
user_id=user_id,
|
|
provider_item_id=shared_provider_id,
|
|
name="report.pdf",
|
|
kind="file",
|
|
content_type="application/pdf",
|
|
provider_size=1024,
|
|
)
|
|
item2 = CloudItem(
|
|
id=_uuid.uuid4(),
|
|
connection_id=conn2.id,
|
|
user_id=user_id,
|
|
provider_item_id=shared_provider_id,
|
|
name="report.pdf",
|
|
kind="file",
|
|
content_type="application/pdf",
|
|
provider_size=1024,
|
|
)
|
|
db_session.add(item1)
|
|
db_session.add(item2)
|
|
# Must not raise IntegrityError — same provider_item_uuid allowed under different connections
|
|
await db_session.commit()
|
|
|
|
result = await db_session.execute(
|
|
select(CloudItem).where(
|
|
CloudItem.user_id == user_id,
|
|
CloudItem.provider_item_id == shared_provider_id,
|
|
)
|
|
)
|
|
rows = result.scalars().all()
|
|
assert len(rows) == 2, "Same provider_item_id must coexist under different connections"
|
|
conn_ids = {str(r.connection_id) for r in rows}
|
|
assert str(conn1.id) in conn_ids
|
|
assert str(conn2.id) in conn_ids
|
|
|
|
|
|
# ── Malformed connection ID ────────────────────────────────────────────────────
|
|
|
|
async def test_browse_malformed_connection_id_returns_422(async_client, db_session):
|
|
"""Browse with non-UUID connection ID returns 422 (input validation, T-12-08)."""
|
|
auth = await _create_user_and_token(db_session, role="user")
|
|
|
|
resp = await async_client.get(
|
|
"/api/cloud/connections/not-a-uuid/items",
|
|
headers=auth["headers"],
|
|
)
|
|
assert resp.status_code == 422, f"Expected 422 for malformed UUID, got {resp.status_code}"
|
|
|
|
|
|
# ── Plan 02 (T-12.1-06..10): Incomplete listing security assertions ───────────
|
|
|
|
async def test_browse_complete_false_never_sets_fresh(async_client, db_session):
|
|
"""Browse with complete=False provider response must not expose refresh_state='fresh'.
|
|
|
|
T-12.1-06: false fresh state hides provider failure from the user.
|
|
The API must return the cached rows with a warning freshness state.
|
|
"""
|
|
from storage.cloud_base import CloudListing
|
|
|
|
auth = await _create_user_and_token(db_session, role="user")
|
|
conn = await _create_cloud_connection(db_session, auth["user"].id)
|
|
|
|
mock_adapter = AsyncMock()
|
|
mock_adapter.list_folder = AsyncMock(return_value=CloudListing(items=(), complete=False))
|
|
mock_adapter.get_capabilities = AsyncMock(
|
|
return_value=_make_mock_adapter().get_capabilities.return_value
|
|
)
|
|
|
|
with patch("api.cloud.browse.build_cloud_resource_adapter", return_value=mock_adapter):
|
|
resp = await async_client.get(
|
|
f"/api/cloud/connections/{conn.id}/items",
|
|
headers=auth["headers"],
|
|
)
|
|
|
|
assert resp.status_code == 200
|
|
freshness = resp.json().get("freshness", {})
|
|
assert freshness.get("refresh_state") != "fresh", (
|
|
"T-12.1-06: complete=False must never produce refresh_state='fresh'"
|
|
)
|
|
|
|
|
|
async def test_browse_complete_false_no_raw_provider_error_in_response(async_client, db_session):
|
|
"""Browse error messages must be controlled — no traceback, URL, or credential leakage.
|
|
|
|
T-12.1-09: provider errors must not leak secrets or raw exception text to API consumers.
|
|
"""
|
|
from storage.cloud_base import CloudListing
|
|
|
|
auth = await _create_user_and_token(db_session, role="user")
|
|
conn = await _create_cloud_connection(db_session, auth["user"].id)
|
|
|
|
mock_adapter = AsyncMock()
|
|
mock_adapter.list_folder = AsyncMock(return_value=CloudListing(items=(), complete=False))
|
|
mock_adapter.get_capabilities = AsyncMock(
|
|
return_value=_make_mock_adapter().get_capabilities.return_value
|
|
)
|
|
|
|
with patch("api.cloud.browse.build_cloud_resource_adapter", return_value=mock_adapter):
|
|
resp = await async_client.get(
|
|
f"/api/cloud/connections/{conn.id}/items",
|
|
headers=auth["headers"],
|
|
)
|
|
|
|
assert resp.status_code == 200
|
|
body_lower = resp.text.lower()
|
|
# No raw exception / traceback in the response body
|
|
assert "traceback" not in body_lower, "T-12.1-09: traceback must not appear in API response"
|
|
assert "exception" not in body_lower, "T-12.1-09: exception text must not appear in API response"
|
|
# No credential fields leaked
|
|
assert "access_token" not in body_lower
|
|
assert "refresh_token" not in body_lower
|
|
assert "credentials_enc" not in body_lower
|
|
|
|
|
|
async def test_browse_cached_items_remain_owner_scoped_on_incomplete(async_client, db_session):
|
|
"""Cached items returned after incomplete refresh must still be owner-scoped.
|
|
|
|
T-12.1-07: incomplete refresh must not return other users' items.
|
|
"""
|
|
from storage.cloud_base import CloudListing
|
|
from db.models import CloudItem
|
|
|
|
auth_owner = await _create_user_and_token(db_session, role="user")
|
|
auth_other = await _create_user_and_token(db_session, role="user")
|
|
|
|
conn = await _create_cloud_connection(db_session, auth_owner["user"].id)
|
|
|
|
# Seed a durable item for the owner
|
|
item = CloudItem(
|
|
id=_uuid.uuid4(),
|
|
user_id=auth_owner["user"].id,
|
|
connection_id=conn.id,
|
|
provider_item_id="owner-item-001",
|
|
name="owner.pdf",
|
|
kind="file",
|
|
analysis_status="pending",
|
|
semantic_index_status="none",
|
|
)
|
|
db_session.add(item)
|
|
await db_session.commit()
|
|
|
|
mock_adapter = AsyncMock()
|
|
mock_adapter.list_folder = AsyncMock(return_value=CloudListing(items=(), complete=False))
|
|
mock_adapter.get_capabilities = AsyncMock(
|
|
return_value=_make_mock_adapter().get_capabilities.return_value
|
|
)
|
|
|
|
# Other user tries to browse owner's connection — must be blocked
|
|
resp_other = await async_client.get(
|
|
f"/api/cloud/connections/{conn.id}/items",
|
|
headers=auth_other["headers"],
|
|
)
|
|
assert resp_other.status_code == 404, (
|
|
"foreign user must still get IDOR 404 even during incomplete refresh"
|
|
)
|
|
|
|
# Owner can still browse and gets their cached item
|
|
with patch("api.cloud.browse.build_cloud_resource_adapter", return_value=mock_adapter):
|
|
resp_owner = await async_client.get(
|
|
f"/api/cloud/connections/{conn.id}/items",
|
|
headers=auth_owner["headers"],
|
|
)
|
|
|
|
assert resp_owner.status_code == 200
|
|
items_returned = resp_owner.json().get("items", [])
|
|
item_ids = [i["provider_item_id"] for i in items_returned]
|
|
assert "owner-item-001" in item_ids, "owner's cached item must be returned after incomplete refresh"
|
|
|
|
|
|
# ── D-18: No byte download during list_folder ─────────────────────────────────
|
|
|
|
async def test_no_byte_download_during_browse(async_client, db_session):
|
|
"""list_folder during browse must never call get_object or download bytes (D-18, T-12-09).
|
|
|
|
The adapter's list_folder is called; get_object/download must never be.
|
|
"""
|
|
import uuid as _uuid2
|
|
from storage.cloud_base import CloudResource, CloudListing
|
|
|
|
auth = await _create_user_and_token(db_session, role="user")
|
|
conn = await _create_cloud_connection(db_session, auth["user"].id)
|
|
|
|
byte_call_tracker = MagicMock()
|
|
mock_adapter = _make_mock_adapter(
|
|
items=(
|
|
CloudResource(
|
|
id=_uuid2.uuid4(),
|
|
provider_item_id="item1",
|
|
connection_id=conn.id,
|
|
user_id=auth["user"].id,
|
|
name="document.pdf",
|
|
kind="file",
|
|
size=50000,
|
|
),
|
|
)
|
|
)
|
|
mock_adapter.get_object = byte_call_tracker
|
|
mock_adapter.download = byte_call_tracker
|
|
|
|
with patch("api.cloud.browse.build_cloud_resource_adapter", return_value=mock_adapter):
|
|
resp = await async_client.get(
|
|
f"/api/cloud/connections/{conn.id}/items",
|
|
headers=auth["headers"],
|
|
)
|
|
|
|
assert resp.status_code == 200
|
|
byte_call_tracker.assert_not_called()
|
|
|
|
|
|
# ── Phase 14: Cache API security (T-14-01, T-14-02, T-14-06, T-14-08) ──────────
|
|
|
|
async def test_cache_status_admin_blocked(async_client, db_session):
|
|
"""T-14-01/CACHE-05: Admin account cannot access the cache status endpoint.
|
|
|
|
get_regular_user blocks all admin accounts from analysis/cache routes.
|
|
"""
|
|
admin_ctx = await _create_user_and_token(db_session, role="admin")
|
|
|
|
resp = await async_client.get(
|
|
"/api/cloud/analysis/cache",
|
|
headers=admin_ctx["headers"],
|
|
)
|
|
|
|
assert resp.status_code == 403, (
|
|
f"Admin must be blocked from cache status — expected 403, got {resp.status_code}"
|
|
)
|
|
|
|
|
|
async def test_cache_settings_admin_blocked(async_client, db_session):
|
|
"""T-14-01: Admin account cannot update cache settings."""
|
|
admin_ctx = await _create_user_and_token(db_session, role="admin")
|
|
|
|
resp = await async_client.patch(
|
|
"/api/cloud/analysis/cache/settings",
|
|
json={"analysis_progress_detail": "detailed"},
|
|
headers=admin_ctx["headers"],
|
|
)
|
|
|
|
assert resp.status_code == 403, (
|
|
f"Admin must be blocked from cache settings update — got {resp.status_code}"
|
|
)
|
|
|
|
|
|
async def test_cache_status_unauthenticated_blocked(async_client, db_session):
|
|
"""Cache status requires authentication — unauthenticated request returns 401/403."""
|
|
resp = await async_client.get("/api/cloud/analysis/cache")
|
|
|
|
assert resp.status_code in (401, 403), (
|
|
f"Unauthenticated request must be rejected — got {resp.status_code}"
|
|
)
|
|
|
|
|
|
async def test_cache_status_response_excludes_object_key(async_client, db_session):
|
|
"""T-14-02/T-14-08: Cache status response must not contain object_key field.
|
|
|
|
The MinIO object key is an internal detail — never exposed in API responses.
|
|
"""
|
|
user_ctx = await _create_user_and_token(db_session, role="user")
|
|
|
|
resp = await async_client.get(
|
|
"/api/cloud/analysis/cache",
|
|
headers=user_ctx["headers"],
|
|
)
|
|
|
|
assert resp.status_code == 200, f"Expected 200 from cache status, got {resp.status_code}"
|
|
body_text = resp.text
|
|
|
|
# T-14-02 / T-14-08: object key and credential fields must be absent
|
|
assert "object_key" not in body_text, (
|
|
"T-14-02: MinIO object_key must not appear in cache API responses"
|
|
)
|
|
assert "credentials_enc" not in body_text, (
|
|
"T-14-08: credentials_enc must not appear in cache API responses"
|
|
)
|
|
|
|
|
|
async def test_cache_status_response_structure(async_client, db_session):
|
|
"""CACHE-05: Cache status returns aggregate totals, not per-entry detail."""
|
|
user_ctx = await _create_user_and_token(db_session, role="user")
|
|
|
|
resp = await async_client.get(
|
|
"/api/cloud/analysis/cache",
|
|
headers=user_ctx["headers"],
|
|
)
|
|
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
|
|
# Aggregate fields must be present
|
|
assert "entry_count" in body, "Cache status must include entry_count"
|
|
assert "total_bytes" in body, "Cache status must include total_bytes"
|
|
assert "cache_limit_bytes" in body, "Cache status must include cache_limit_bytes"
|
|
assert "analysis_progress_detail" in body, "Cache status must include analysis_progress_detail"
|
|
assert "analysis_failure_behavior" in body, "Cache status must include analysis_failure_behavior"
|
|
|
|
# No per-entry detail that could leak internal state
|
|
assert "object_key" not in body
|
|
assert "provider_item_id" not in body or isinstance(body.get("provider_item_id"), type(None))
|
|
|
|
|
|
async def test_cache_settings_invalid_enum_returns_422(async_client, db_session):
|
|
"""Cache settings update rejects invalid enum values with 422."""
|
|
user_ctx = await _create_user_and_token(db_session, role="user")
|
|
|
|
resp = await async_client.patch(
|
|
"/api/cloud/analysis/cache/settings",
|
|
json={"analysis_progress_detail": "invalid_value"},
|
|
headers=user_ctx["headers"],
|
|
)
|
|
|
|
assert resp.status_code == 422, (
|
|
f"Invalid enum value must return 422, got {resp.status_code}"
|
|
)
|
|
|
|
|
|
async def test_cache_settings_limit_above_tier_cap_returns_422(async_client, db_session):
|
|
"""Cache settings update rejects cache limit above tier cap with 422."""
|
|
from services.cloud_analysis_settings import DEFAULT_MAX_CACHE_LIMIT_BYTES
|
|
|
|
user_ctx = await _create_user_and_token(db_session, role="user")
|
|
|
|
too_large = DEFAULT_MAX_CACHE_LIMIT_BYTES + 1
|
|
|
|
resp = await async_client.patch(
|
|
"/api/cloud/analysis/cache/settings",
|
|
json={"cloud_cache_limit_bytes": too_large},
|
|
headers=user_ctx["headers"],
|
|
)
|
|
|
|
assert resp.status_code == 422, (
|
|
f"Cache limit above tier cap must return 422, got {resp.status_code}"
|
|
)
|
|
|
|
|
|
# ── Phase 14: Analysis job API security (T-14-01, T-14-02, T-14-03) ────────────
|
|
|
|
|
|
async def _create_analysis_connection(session, user_id, provider="google_drive"):
|
|
"""Helper: create a cloud connection for analysis security tests."""
|
|
from db.models import CloudConnection
|
|
from storage.cloud_utils import encrypt_credentials
|
|
from config import settings as app_settings
|
|
|
|
master_key = app_settings.cloud_creds_key.encode()
|
|
creds_enc = encrypt_credentials(
|
|
master_key,
|
|
str(user_id),
|
|
{"access_token": "tok", "refresh_token": "ref"},
|
|
)
|
|
conn = CloudConnection(
|
|
id=_uuid.uuid4(),
|
|
user_id=user_id,
|
|
provider=provider,
|
|
display_name="Security Test Connection",
|
|
credentials_enc=creds_enc,
|
|
status="ACTIVE",
|
|
)
|
|
session.add(conn)
|
|
await session.commit()
|
|
return conn
|
|
|
|
|
|
async def _create_analysis_item(session, user_id, connection_id, name="doc.pdf"):
|
|
"""Helper: create a cloud item for analysis security tests."""
|
|
from db.models import CloudItem
|
|
|
|
item = CloudItem(
|
|
id=_uuid.uuid4(),
|
|
user_id=user_id,
|
|
connection_id=connection_id,
|
|
provider_item_id=f"sec-pitem-{_uuid.uuid4().hex[:8]}",
|
|
name=name,
|
|
kind="file",
|
|
content_type="application/pdf",
|
|
provider_size=102400,
|
|
etag="etag-sec-v1",
|
|
analysis_status="pending",
|
|
)
|
|
session.add(item)
|
|
await session.commit()
|
|
return item
|
|
|
|
|
|
async def test_analysis_estimate_admin_blocked(async_client, db_session):
|
|
"""T-14-01: Admin cannot access analysis estimate endpoint."""
|
|
admin_ctx = await _create_user_and_token(db_session, role="admin")
|
|
user_ctx = await _create_user_and_token(db_session, role="user")
|
|
conn = await _create_analysis_connection(db_session, user_ctx["user"].id)
|
|
|
|
resp = await async_client.post(
|
|
f"/api/cloud/analysis/connections/{conn.id}/estimate",
|
|
json={"scope": "connection", "recursive": True},
|
|
headers=admin_ctx["headers"],
|
|
)
|
|
|
|
assert resp.status_code in (403, 404), (
|
|
f"Admin must not access analysis estimate — got {resp.status_code}"
|
|
)
|
|
|
|
|
|
async def test_analysis_enqueue_admin_blocked(async_client, db_session):
|
|
"""T-14-01: Admin cannot enqueue analysis jobs."""
|
|
admin_ctx = await _create_user_and_token(db_session, role="admin")
|
|
user_ctx = await _create_user_and_token(db_session, role="user")
|
|
conn = await _create_analysis_connection(db_session, user_ctx["user"].id)
|
|
|
|
resp = await async_client.post(
|
|
f"/api/cloud/analysis/connections/{conn.id}/jobs",
|
|
json={"scope": "connection", "recursive": True},
|
|
headers=admin_ctx["headers"],
|
|
)
|
|
|
|
assert resp.status_code in (403, 404), (
|
|
f"Admin must not enqueue analysis jobs — got {resp.status_code}"
|
|
)
|
|
|
|
|
|
async def test_analysis_estimate_unauthenticated_blocked(async_client, db_session):
|
|
"""Analysis estimate requires authentication."""
|
|
conn_id = str(_uuid.uuid4())
|
|
|
|
resp = await async_client.post(
|
|
f"/api/cloud/analysis/connections/{conn_id}/estimate",
|
|
json={"scope": "connection", "recursive": True},
|
|
)
|
|
|
|
assert resp.status_code in (401, 403), (
|
|
f"Unauthenticated analysis estimate must be rejected — got {resp.status_code}"
|
|
)
|
|
|
|
|
|
async def test_analysis_enqueue_response_excludes_credentials(async_client, db_session):
|
|
"""T-14-02: Enqueue response must not contain credentials or object_key."""
|
|
from unittest.mock import patch as _patch
|
|
|
|
user_ctx = await _create_user_and_token(db_session, role="user")
|
|
conn = await _create_analysis_connection(db_session, user_ctx["user"].id)
|
|
item = await _create_analysis_item(db_session, user_ctx["user"].id, conn.id)
|
|
|
|
with _patch("services.cloud_analysis.get_adapter", return_value=MagicMock()):
|
|
resp = await async_client.post(
|
|
f"/api/cloud/analysis/connections/{conn.id}/jobs",
|
|
json={"scope": "file", "provider_item_ids": [item.provider_item_id]},
|
|
headers=user_ctx["headers"],
|
|
)
|
|
|
|
assert resp.status_code in (200, 202), (
|
|
f"Enqueue must succeed — got {resp.status_code}: {resp.text}"
|
|
)
|
|
body_text = resp.text
|
|
|
|
# T-14-02: No credential or internal storage fields in response
|
|
for field in ("credentials_enc", "access_token", "refresh_token", "object_key"):
|
|
assert field not in body_text, (
|
|
f"T-14-02: Sensitive field '{field}' must not appear in enqueue response"
|
|
)
|
|
|
|
|
|
async def test_analysis_job_status_excludes_credentials(async_client, db_session):
|
|
"""T-14-02: Job status response must not contain credentials or object_key."""
|
|
from unittest.mock import patch as _patch
|
|
|
|
user_ctx = await _create_user_and_token(db_session, role="user")
|
|
conn = await _create_analysis_connection(db_session, user_ctx["user"].id)
|
|
item = await _create_analysis_item(db_session, user_ctx["user"].id, conn.id)
|
|
|
|
with _patch("services.cloud_analysis.get_adapter", return_value=MagicMock()):
|
|
enqueue_resp = await async_client.post(
|
|
f"/api/cloud/analysis/connections/{conn.id}/jobs",
|
|
json={"scope": "file", "provider_item_ids": [item.provider_item_id]},
|
|
headers=user_ctx["headers"],
|
|
)
|
|
|
|
assert enqueue_resp.status_code in (200, 202)
|
|
job_id = enqueue_resp.json().get("job_id")
|
|
assert job_id is not None
|
|
|
|
status_resp = await async_client.get(
|
|
f"/api/cloud/analysis/jobs/{job_id}",
|
|
headers=user_ctx["headers"],
|
|
)
|
|
assert status_resp.status_code == 200
|
|
body_text = status_resp.text
|
|
|
|
for field in ("credentials_enc", "access_token", "refresh_token", "object_key"):
|
|
assert field not in body_text, (
|
|
f"T-14-02: Sensitive field '{field}' must not appear in job status response"
|
|
)
|
|
|
|
|
|
async def test_foreign_user_cannot_access_analysis_estimate(async_client, db_session):
|
|
"""T-14-01: Foreign user cannot estimate analysis scope for another user's connection."""
|
|
owner_ctx = await _create_user_and_token(db_session, role="user")
|
|
foreign_ctx = await _create_user_and_token(db_session, role="user")
|
|
owner_conn = await _create_analysis_connection(db_session, owner_ctx["user"].id)
|
|
|
|
resp = await async_client.post(
|
|
f"/api/cloud/analysis/connections/{owner_conn.id}/estimate",
|
|
json={"scope": "connection", "recursive": True},
|
|
headers=foreign_ctx["headers"],
|
|
)
|
|
|
|
assert resp.status_code in (403, 404), (
|
|
f"Foreign user must not estimate owner's connection — got {resp.status_code}"
|
|
)
|
|
|
|
|
|
# ── Phase 14 Plan 06: Cache-backed preview/download security ──────────────────
|
|
|
|
async def test_preview_cache_hit_response_excludes_object_key(async_client, db_session):
|
|
"""T-14-02/Plan 06: Preview response from a cache hit must not expose object_key.
|
|
|
|
Even when bytes come from MinIO (cache hit path), the response body and headers
|
|
must not contain the MinIO object key or credentials_enc.
|
|
"""
|
|
import uuid as _uuid2
|
|
from unittest.mock import AsyncMock, patch as _patch, MagicMock
|
|
|
|
owner_ctx = await _create_user_and_token(db_session, role="user")
|
|
conn = await _create_cloud_connection(db_session, owner_ctx["user"].id)
|
|
|
|
# Simulate a cache hit by patching hydrate_and_cache_bytes to return bytes
|
|
with _patch(
|
|
"api.cloud.operations.hydrate_and_cache_bytes" if False else "services.cloud_cache.hydrate_and_cache_bytes",
|
|
new=AsyncMock(return_value=(b"pdf-bytes", _uuid2.uuid4())),
|
|
):
|
|
with _patch("api.cloud.operations._resolve_and_get_adapter") as mock_adapter:
|
|
mock_conn = MagicMock()
|
|
mock_ad = AsyncMock()
|
|
mock_ad.get_object = AsyncMock(return_value=b"pdf-bytes")
|
|
mock_adapter.return_value = (mock_conn, mock_ad)
|
|
|
|
# No cloud_item in DB, so falls to direct fetch path — test response exclusion
|
|
resp = await async_client.get(
|
|
f"/api/cloud/connections/{conn.id}/items/pitem-preview-sec/preview",
|
|
headers=owner_ctx["headers"],
|
|
)
|
|
|
|
# Any 2xx or unsupported_preview is acceptable; key check is no secret leakage
|
|
assert resp.status_code not in (500,), f"Should not 500 — got {resp.status_code}: {resp.text}"
|
|
body = resp.text
|
|
assert "object_key" not in body, "T-14-02: object_key must not appear in preview response"
|
|
assert "credentials_enc" not in body, "credentials_enc must not appear in preview response"
|
|
|
|
|
|
async def test_download_response_excludes_object_key(async_client, db_session):
|
|
"""T-14-02/Plan 06: Download response must not expose object_key or credentials.
|
|
|
|
The cache lookup and MinIO path are internal; the response headers and body
|
|
must contain only the file bytes and safe metadata.
|
|
"""
|
|
from unittest.mock import AsyncMock, patch as _patch, MagicMock
|
|
|
|
owner_ctx = await _create_user_and_token(db_session, role="user")
|
|
conn = await _create_cloud_connection(db_session, owner_ctx["user"].id)
|
|
|
|
with _patch("api.cloud.operations._resolve_and_get_adapter") as mock_adapter:
|
|
mock_conn = MagicMock()
|
|
mock_ad = AsyncMock()
|
|
mock_ad.get_object = AsyncMock(return_value=b"download-bytes")
|
|
mock_adapter.return_value = (mock_conn, mock_ad)
|
|
|
|
resp = await async_client.get(
|
|
f"/api/cloud/connections/{conn.id}/items/pitem-download-sec/download",
|
|
headers=owner_ctx["headers"],
|
|
)
|
|
|
|
assert resp.status_code != 500, f"Should not 500 — got {resp.status_code}: {resp.text}"
|
|
# Check headers
|
|
for header_name, header_value in resp.headers.items():
|
|
assert "object_key" not in header_value.lower(), (
|
|
f"T-14-02: object_key must not appear in response headers — found in {header_name}"
|
|
)
|
|
assert "credentials_enc" not in header_value.lower(), (
|
|
"credentials_enc must not appear in response headers"
|
|
)
|
|
|
|
|
|
async def test_foreign_user_cannot_download_via_cache(async_client, db_session):
|
|
"""T-14-01/Plan 06: Foreign user cannot download another user's cloud file via cache.
|
|
|
|
The ownership check (resolve_owned_connection) must run before any cache lookup.
|
|
"""
|
|
auth1 = await _create_user_and_token(db_session, role="user")
|
|
auth2 = await _create_user_and_token(db_session, role="user")
|
|
conn = await _create_cloud_connection(db_session, auth1["user"].id)
|
|
|
|
resp = await async_client.get(
|
|
f"/api/cloud/connections/{conn.id}/items/pitem-idor/download",
|
|
headers=auth2["headers"],
|
|
)
|
|
|
|
assert resp.status_code == 404, (
|
|
f"Foreign user must not download via cache — expected 404, got {resp.status_code}"
|
|
)
|
|
|
|
|
|
async def test_foreign_user_cannot_read_analysis_job_status(async_client, db_session):
|
|
"""T-14-01: Foreign user cannot read job status for another user's job."""
|
|
from unittest.mock import patch as _patch
|
|
|
|
owner_ctx = await _create_user_and_token(db_session, role="user")
|
|
foreign_ctx = await _create_user_and_token(db_session, role="user")
|
|
owner_conn = await _create_analysis_connection(db_session, owner_ctx["user"].id)
|
|
owner_item = await _create_analysis_item(db_session, owner_ctx["user"].id, owner_conn.id)
|
|
|
|
with _patch("services.cloud_analysis.get_adapter", return_value=MagicMock()):
|
|
enqueue_resp = await async_client.post(
|
|
f"/api/cloud/analysis/connections/{owner_conn.id}/jobs",
|
|
json={"scope": "file", "provider_item_ids": [owner_item.provider_item_id]},
|
|
headers=owner_ctx["headers"],
|
|
)
|
|
|
|
assert enqueue_resp.status_code in (200, 202)
|
|
job_id = enqueue_resp.json().get("job_id")
|
|
|
|
resp = await async_client.get(
|
|
f"/api/cloud/analysis/jobs/{job_id}",
|
|
headers=foreign_ctx["headers"],
|
|
)
|
|
|
|
assert resp.status_code in (403, 404), (
|
|
f"Foreign user must not read owner's job — got {resp.status_code}"
|
|
)
|