curo/kite
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor backend and frontend cleanup paths

Browse Source
This commit is contained in:
curo1305
2026-06-16 11:50:17 +02:00
parent 6b56763689
commit e97ca164d7
29 changed files with 1106 additions and 2280 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/api/documents/upload.py
+99 -179
View File
@@ -1,32 +1,11 @@
"""Document upload endpoints — presigned URL flow and direct cloud upload.
Endpoints:
POST /upload-url — create pending Document row, return presigned PUT URL (D-05 step 1)
POST /upload — direct multipart upload supporting cloud backends (D-10, D-14, D-15)
POST /{doc_id}/confirm — stat MinIO for authoritative size, enforce quota atomically (D-05 step 3)
Sub-router carries NO prefix — prefix="/api/documents" lives in __init__.py (D-04).
Security:
T-03-04: object_key computed server-side using str(current_user.id) — never user-supplied.
T-03-05: size from backend.stat_object() — never from client.
T-03-06: atomic SQL UPDATE prevents concurrent over-quota uploads (STORE-03 SC2).
T-03-11: ownership assertion on confirm — cross-user access returns 404.
T-03-15: object_key prefix always the authenticated user's id.
T-05-06-01: target_backend validated against _CLOUD_PROVIDERS allowlist.
T-05-06-02: CloudConnectionError detail never includes provider error detail.
"""
"""Document upload endpoints."""
from __future__ import annotations
import uuid
from pathlib import Path
import structlog as _structlog
_log = _structlog.get_logger(__name__)
from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile, status
from sqlalchemy import text
from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile
from sqlalchemy import select, text
from sqlalchemy.ext.asyncio import AsyncSession
from config import settings
@@ -36,24 +15,91 @@ from deps.db import get_db
from deps.utils import get_client_ip
from services.audit import write_audit_log
from services.rate_limiting import account_limiter
from storage import get_storage_backend, get_storage_backend_for_document
from storage import get_storage_backend
from storage.cloud_backend_factory import build_cloud_backend
from storage.cloud_utils import decrypt_credentials
from storage.exceptions import CloudConnectionError
from tasks.document_tasks import extract_and_classify
try:
from minio.error import S3Error
except ImportError:
S3Error = Exception # type: ignore[assignment,misc]
from sqlalchemy import select
from api.documents.shared import UploadUrlRequest, _CLOUD_PROVIDERS
router = APIRouter()
# ── POST /api/documents/upload-url ───────────────────────────────────────────
def _new_minio_document(user_id: uuid.UUID, filename: str, content_type: str) -> Document:
doc_id = uuid.uuid4()
suffix = Path(filename).suffix.lower()
return Document(
id=doc_id,
user_id=user_id,
filename=filename,
content_type=content_type,
size_bytes=0,
storage_backend="minio",
status="pending",
object_key=f"{user_id}/{doc_id}/{uuid.uuid4()}{suffix}",
)
async def _create_presigned_upload(
session: AsyncSession,
user_id: uuid.UUID,
filename: str,
content_type: str,
) -> dict:
doc = _new_minio_document(user_id, filename, content_type)
session.add(doc)
await session.commit()
upload_url = await get_storage_backend().generate_presigned_put_url(
doc.object_key, expires_minutes=15
)
return {"upload_url": upload_url, "document_id": str(doc.id)}
async def _get_active_cloud_connection(
session: AsyncSession,
user_id: uuid.UUID,
provider: str,
) -> CloudConnection:
result = await session.execute(
select(CloudConnection).where(
CloudConnection.user_id == user_id,
CloudConnection.provider == provider,
CloudConnection.status == "ACTIVE",
)
)
conn = result.scalar_one_or_none()
if conn is None:
raise HTTPException(
status_code=404,
detail=f"No active {provider} connection found. Please connect in Settings.",
)
return conn
def _decrypt_cloud_credentials(conn: CloudConnection, user_id: uuid.UUID) -> dict:
return decrypt_credentials(settings.cloud_creds_key.encode(), str(user_id), conn.credentials_enc)
async def _record_upload(
session: AsyncSession,
request: Request,
current_user: User,
doc: Document,
size_bytes: int,
storage_backend: str,
) -> None:
await write_audit_log(
session,
event_type="document.uploaded",
user_id=current_user.id,
actor_id=current_user.id,
resource_id=doc.id,
ip_address=get_client_ip(request) if request else None,
metadata_={"size_bytes": size_bytes, "storage_backend": storage_backend},
)
@router.post("/upload-url")
@account_limiter.limit("100/minute")
@@ -63,41 +109,15 @@ async def request_upload_url(
session: AsyncSession = Depends(get_db),
current_user: User = Depends(get_regular_user),
):
"""Create a pending Document row and return a presigned PUT URL.
D-05 step 1: FastAPI creates a Document row (status='pending'), generates a
15-minute presigned PUT URL, returns {upload_url, document_id}.
Quota is NOT reserved at this step — quota enforcement happens at /confirm.
T-03-04: object_key is computed server-side using str(current_user.id); filename
stored in DB only (CLAUDE.md MinIO key schema).
T-03-15: object_key prefix is always the authenticated user's id — never user-supplied.
"""
"""Create a pending Document row and return a presigned PUT URL."""
request.state.current_user = current_user
doc_id = uuid.uuid4()
suffix = Path(body.filename).suffix.lower()
object_key = f"{current_user.id}/{doc_id}/{uuid.uuid4()}{suffix}"
doc = Document(
id=doc_id,
user_id=current_user.id,
filename=body.filename,
content_type=body.content_type,
size_bytes=0,
storage_backend="minio",
status="pending",
object_key=object_key,
return await _create_presigned_upload(
session,
current_user.id,
body.filename,
body.content_type,
)
session.add(doc)
await session.commit()
upload_url = await get_storage_backend().generate_presigned_put_url(
object_key, expires_minutes=15
)
return {"upload_url": upload_url, "document_id": str(doc_id)}
# ── POST /api/documents/upload ────────────────────────────────────────────────
@router.post("/upload")
@account_limiter.limit("100/minute")
@@ -109,47 +129,15 @@ async def upload_document(
session: AsyncSession = Depends(get_db),
current_user: User = Depends(get_regular_user),
):
"""Direct multipart upload endpoint supporting cloud backends (D-10, D-14, D-15).
If target_backend == "minio": generates a presigned PUT URL (unchanged MinIO flow).
If target_backend in ("google_drive", "onedrive", "nextcloud", "webdav"):
1. Reads file bytes from UploadFile
2. Loads CloudConnection for current_user.id + target_backend; 404 if not found/not ACTIVE
3. Decrypts credentials and instantiates the correct backend class
4. Calls cloud_backend.put_object() to upload directly to the provider
5. Creates Document with storage_backend=target_backend
6. Returns {document_id, storage_backend} — no upload_url (cloud upload is synchronous)
Cloud uploads do NOT use the atomic quota UPDATE — cloud files are not counted
against MinIO quota (D-11: separate backends; cloud storage quota is provider-side).
Security:
T-05-06-01: target_backend validated against _CLOUD_PROVIDERS allowlist → 422 on invalid value
T-05-06-02: CloudConnectionError detail message never includes provider error detail
"""
"""Direct multipart upload endpoint supporting cloud backends."""
request.state.current_user = current_user
if target_backend == "minio":
doc_id = uuid.uuid4()
suffix = Path(file.filename or "file").suffix.lower()
object_key = f"{current_user.id}/{doc_id}/{uuid.uuid4()}{suffix}"
doc = Document(
id=doc_id,
user_id=current_user.id,
filename=file.filename or "upload",
content_type=file.content_type or "application/octet-stream",
size_bytes=0,
storage_backend="minio",
status="pending",
object_key=object_key,
return await _create_presigned_upload(
session,
current_user.id,
file.filename or "upload",
file.content_type or "application/octet-stream",
)
session.add(doc)
await session.commit()
upload_url = await get_storage_backend().generate_presigned_put_url(
object_key, expires_minutes=15
)
return {"upload_url": upload_url, "document_id": str(doc_id)}
if target_backend not in _CLOUD_PROVIDERS:
raise HTTPException(
@@ -157,23 +145,8 @@ async def upload_document(
detail=f"Invalid target_backend '{target_backend}'. Valid values: minio, {', '.join(sorted(_CLOUD_PROVIDERS))}",
)
# Load active CloudConnection for current user + provider (T-05-06-01: user-scoped query)
result = await session.execute(
select(CloudConnection).where(
CloudConnection.user_id == current_user.id,
CloudConnection.provider == target_backend,
CloudConnection.status == "ACTIVE",
)
)
conn = result.scalar_one_or_none()
if conn is None:
raise HTTPException(
status_code=404,
detail=f"No active {target_backend} connection found. Please connect in Settings.",
)
master_key = settings.cloud_creds_key.encode()
credentials = decrypt_credentials(master_key, str(current_user.id), conn.credentials_enc)
conn = await _get_active_cloud_connection(session, current_user.id, target_backend)
credentials = _decrypt_cloud_credentials(conn, current_user.id)
file_bytes = await file.read()
filename = file.filename or "upload"
@@ -182,26 +155,7 @@ async def upload_document(
doc_id = uuid.uuid4()
if target_backend == "google_drive":
from storage.google_drive_backend import GoogleDriveBackend # lazy import
cloud_backend = GoogleDriveBackend(credentials)
elif target_backend == "onedrive":
from storage.onedrive_backend import OneDriveBackend # lazy import
cloud_backend = OneDriveBackend(credentials)
elif target_backend == "nextcloud":
from storage.nextcloud_backend import NextcloudBackend # lazy import
cloud_backend = NextcloudBackend(
credentials["server_url"],
credentials["username"],
credentials["password"],
)
elif target_backend == "webdav":
from storage.webdav_backend import WebDAVBackend # lazy import
cloud_backend = WebDAVBackend(
credentials["server_url"],
credentials["username"],
credentials["password"],
)
cloud_backend = build_cloud_backend(target_backend, credentials)
try:
object_key = await cloud_backend.put_object(
@@ -219,9 +173,9 @@ async def upload_document(
detail="Cloud connection requires re-authentication. Please reconnect in Settings.",
) from exc
# Bust folder listing cache so the next GET /folders reflects the new file
if cloud_folder_path:
from services.cloud_cache import invalidate_provider_cache # lazy import
invalidate_provider_cache(str(current_user.id), target_backend)
doc = Document(
@@ -236,16 +190,7 @@ async def upload_document(
)
session.add(doc)
_ip = get_client_ip(request) if request else None
await write_audit_log(
session,
event_type="document.uploaded",
user_id=current_user.id,
actor_id=current_user.id,
resource_id=doc.id,
ip_address=_ip,
metadata_={"size_bytes": len(file_bytes), "storage_backend": target_backend},
)
await _record_upload(session, request, current_user, doc, len(file_bytes), target_backend)
await session.commit()
extract_and_classify.delay(str(doc.id))
@@ -253,8 +198,6 @@ async def upload_document(
return {"document_id": str(doc.id), "storage_backend": target_backend}
# ── POST /api/documents/{doc_id}/confirm ─────────────────────────────────────
@router.post("/{doc_id}/confirm")
@account_limiter.limit("100/minute")
async def confirm_upload(
@@ -263,18 +206,7 @@ async def confirm_upload(
session: AsyncSession = Depends(get_db),
current_user: User = Depends(get_regular_user),
):
"""Confirm a presigned PUT upload: stat MinIO for size, enforce quota atomically.
D-05 step 3: FastAPI reads authoritative file size from MinIO stat_object (never
from client), runs atomic quota UPDATE, sets status='uploaded', enqueues Celery task.
Quota exceeded: HTTP 413 with {"used_bytes": N, "limit_bytes": M, "rejected_bytes": K}
Upload not found: HTTP 422 (presigned URL may have expired)
T-03-05: size always comes from backend.stat_object(doc.object_key) — never client.
T-03-06: atomic SQL UPDATE prevents concurrent over-quota uploads (STORE-03 SC2).
T-03-11: ownership assertion — cross-user access returns 404 (D-16).
"""
"""Confirm a presigned PUT upload and enforce quota."""
request.state.current_user = current_user
try:
uid = uuid.UUID(doc_id)
@@ -285,7 +217,6 @@ async def confirm_upload(
if doc is None or doc.user_id != current_user.id:
raise HTTPException(status_code=404, detail="Document not found")
# Get authoritative file size from MinIO (T-03-05 — never trust client-supplied size)
try:
size = await get_storage_backend().stat_object(doc.object_key)
except Exception as exc:
@@ -300,7 +231,6 @@ async def confirm_upload(
doc.size_bytes = size
await session.flush()
# Atomic quota enforcement — user_id is always set post-migration (Plan 03-03+)
result = await session.execute(
text(
"UPDATE quotas "
@@ -323,7 +253,7 @@ async def confirm_upload(
try:
await get_storage_backend().delete_object(doc.object_key)
except Exception:
pass # MinIO cleanup is best-effort; object TTL will eventually expire
pass
await session.commit()
raise HTTPException(
status_code=413,
@@ -337,17 +267,7 @@ async def confirm_upload(
used_bytes = row.used_bytes
doc.status = "uploaded"
# D-13: document uploaded event — size_bytes + storage_backend only, NO filename, NO extracted_text (T-04-07-02)
_ip = get_client_ip(request)
await write_audit_log(
session,
event_type="document.uploaded",
user_id=current_user.id,
actor_id=current_user.id,
resource_id=doc.id,
ip_address=_ip,
metadata_={"size_bytes": size, "storage_backend": "minio"},
)
await _record_upload(session, request, current_user, doc, size, "minio")
await session.commit()
extract_and_classify.delay(str(doc.id))