feat: migrate app_config volume to storage-service config bucket (Phase 3)
All JSON config files (AI settings, doc settings, appearance, themes) now live in the 'config' bucket of storage-service instead of a shared Docker volume. - backend/core/config_storage.py: new async HTTP helpers for config bucket r/w - backend/core/app_config.py: fully async rewrite; all load_*/save_*/seed_* functions use config_storage instead of filesystem - backend/routers/settings.py: all asyncio.to_thread() wrappers removed; direct await calls throughout; update_theme reads via load_theme_by_id() - backend/main.py: await seed_builtin_themes() directly (no to_thread) - ai-service: remove CONFIG_PATH, add STORAGE_SERVICE_URL; config_reader now fetches from storage-service via httpx - doc-service: config_reader rewritten to fetch/write via storage-service - docker-compose: remove app_config volume; add storage-service depends_on for ai-service; remove DATA_DIR and CONFIG_PATH from doc-service Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
curo1305
@@ -1,21 +1,25 @@
|
||||
"""
|
||||
Per-service runtime config helpers.
|
||||
|
||||
Config files live on the shared `app_config` Docker volume at /config/.
|
||||
Each service has its own JSON file.
|
||||
All config files are stored in the 'config' bucket of the storage-service.
|
||||
Every function is async — callers must await them.
|
||||
|
||||
Atomic write pattern: write to .tmp in same dir, then os.replace() so
|
||||
services never read a partial file.
|
||||
Key layout in the config bucket:
|
||||
ai_service_config.json
|
||||
doc_service_config.json
|
||||
appearance_config.json
|
||||
themes/{id}.json
|
||||
"""
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from copy import deepcopy
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
_CONFIG_DIR = Path(os.environ.get("APP_CONFIG_DIR", "/config"))
|
||||
from app.core import config_storage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── AI service config schemas ──────────────────────────────────────────────────
|
||||
|
||||
@@ -108,59 +112,50 @@ def _mask_ai_config(data: dict) -> dict:
|
||||
|
||||
# ── Load / Save ────────────────────────────────────────────────────────────────
|
||||
|
||||
def _config_path(service: str) -> Path:
|
||||
return _CONFIG_DIR / f"{service}_config.json"
|
||||
|
||||
|
||||
def load_service_config(service: str) -> dict:
|
||||
path = _config_path(service)
|
||||
if not path.exists():
|
||||
async def load_service_config(service: str) -> dict:
|
||||
data = await config_storage.read_json(f"{service}_config.json")
|
||||
if data is None:
|
||||
if service == "ai_service":
|
||||
return AIServiceConfig().model_dump()
|
||||
if service == "doc_service":
|
||||
return DocServiceConfig().model_dump()
|
||||
return {}
|
||||
with path.open() as f:
|
||||
return json.load(f)
|
||||
return data
|
||||
|
||||
|
||||
def save_service_config(service: str, data: dict) -> None:
|
||||
path = _config_path(service)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(data, indent=2))
|
||||
os.replace(tmp, path)
|
||||
async def save_service_config(service: str, data: dict) -> None:
|
||||
await config_storage.write_json(f"{service}_config.json", data)
|
||||
|
||||
|
||||
# AI service helpers
|
||||
|
||||
def load_ai_service_config() -> AIServiceConfig:
|
||||
raw = load_service_config("ai_service")
|
||||
async def load_ai_service_config() -> AIServiceConfig:
|
||||
raw = await load_service_config("ai_service")
|
||||
return AIServiceConfig.model_validate(raw)
|
||||
|
||||
|
||||
def save_ai_service_config(config: AIServiceConfig) -> None:
|
||||
save_service_config("ai_service", config.model_dump())
|
||||
async def save_ai_service_config(config: AIServiceConfig) -> None:
|
||||
await save_service_config("ai_service", config.model_dump())
|
||||
|
||||
|
||||
def load_ai_service_config_masked() -> dict:
|
||||
raw = load_service_config("ai_service")
|
||||
async def load_ai_service_config_masked() -> dict:
|
||||
raw = await load_service_config("ai_service")
|
||||
return _mask_ai_config(raw)
|
||||
|
||||
|
||||
# Doc service helpers
|
||||
|
||||
def load_doc_service_config() -> DocServiceConfig:
|
||||
raw = load_service_config("doc_service")
|
||||
async def load_doc_service_config() -> DocServiceConfig:
|
||||
raw = await load_service_config("doc_service")
|
||||
return DocServiceConfig.model_validate(raw)
|
||||
|
||||
|
||||
def save_doc_service_config(config: DocServiceConfig) -> None:
|
||||
save_service_config("doc_service", config.model_dump())
|
||||
async def save_doc_service_config(config: DocServiceConfig) -> None:
|
||||
await save_service_config("doc_service", config.model_dump())
|
||||
|
||||
|
||||
def load_doc_service_config_masked() -> dict:
|
||||
return load_service_config("doc_service")
|
||||
async def load_doc_service_config_masked() -> dict:
|
||||
return await load_service_config("doc_service")
|
||||
|
||||
|
||||
def _merge_api_key(new_key: str, existing_key: str) -> str:
|
||||
@@ -172,18 +167,16 @@ def _merge_api_key(new_key: str, existing_key: str) -> str:
|
||||
|
||||
# ── System prompts helpers ─────────────────────────────────────────────────────
|
||||
|
||||
# Registry of all services that have editable system prompts.
|
||||
# key = service identifier, value = human-readable label
|
||||
SYSTEM_PROMPT_SERVICES: dict[str, str] = {
|
||||
"doc_service": "Document Service",
|
||||
}
|
||||
|
||||
|
||||
def load_all_system_prompts() -> dict:
|
||||
async def load_all_system_prompts() -> dict:
|
||||
"""Return {service_id: {label, system, user_template, default_system, default_user_template}}."""
|
||||
result: dict = {}
|
||||
for service_id, label in SYSTEM_PROMPT_SERVICES.items():
|
||||
config = load_service_config(service_id)
|
||||
config = await load_service_config(service_id)
|
||||
prompts = config.get("system_prompts", {})
|
||||
defaults = _get_service_prompt_defaults(service_id)
|
||||
result[service_id] = {
|
||||
@@ -196,15 +189,14 @@ def load_all_system_prompts() -> dict:
|
||||
return result
|
||||
|
||||
|
||||
def save_service_system_prompts(service_id: str, system: str, user_template: str) -> None:
|
||||
"""Persist updated system prompts into the service's config file."""
|
||||
async def save_service_system_prompts(service_id: str, system: str, user_template: str) -> None:
|
||||
if service_id not in SYSTEM_PROMPT_SERVICES:
|
||||
raise ValueError(f"Unknown service: {service_id!r}")
|
||||
config = load_service_config(service_id)
|
||||
config = await load_service_config(service_id)
|
||||
config.setdefault("system_prompts", {})
|
||||
config["system_prompts"]["system"] = system
|
||||
config["system_prompts"]["user_template"] = user_template
|
||||
save_service_config(service_id, config)
|
||||
await save_service_config(service_id, config)
|
||||
|
||||
|
||||
def _get_service_prompt_defaults(service_id: str) -> dict:
|
||||
@@ -221,26 +213,19 @@ class AppearanceConfig(BaseModel):
|
||||
default_mode: str = "system"
|
||||
|
||||
|
||||
def load_appearance_config() -> AppearanceConfig:
|
||||
path = _CONFIG_DIR / "appearance_config.json"
|
||||
if not path.exists():
|
||||
async def load_appearance_config() -> AppearanceConfig:
|
||||
data = await config_storage.read_json("appearance_config.json")
|
||||
if data is None:
|
||||
return AppearanceConfig()
|
||||
with path.open() as f:
|
||||
return AppearanceConfig.model_validate(json.load(f))
|
||||
return AppearanceConfig.model_validate(data)
|
||||
|
||||
|
||||
def save_appearance_config(config: AppearanceConfig) -> None:
|
||||
path = _CONFIG_DIR / "appearance_config.json"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(config.model_dump(), indent=2))
|
||||
os.replace(tmp, path)
|
||||
async def save_appearance_config(config: AppearanceConfig) -> None:
|
||||
await config_storage.write_json("appearance_config.json", config.model_dump())
|
||||
|
||||
|
||||
# ── Theme file management ──────────────────────────────────────────────────────
|
||||
|
||||
_THEMES_DIR = _CONFIG_DIR / "themes"
|
||||
|
||||
# 9 required colour tokens per mode
|
||||
_REQUIRED_TOKENS = frozenset({
|
||||
"primary", "primary_hover", "accent", "accent_hover",
|
||||
@@ -361,36 +346,57 @@ _BUILTIN_THEMES: list[dict] = [
|
||||
]
|
||||
|
||||
|
||||
def seed_builtin_themes() -> None:
|
||||
"""Create /config/themes/ and write built-in theme files if missing."""
|
||||
_THEMES_DIR.mkdir(parents=True, exist_ok=True)
|
||||
async def seed_builtin_themes() -> None:
|
||||
"""Write built-in theme files to storage-service if they are not already there."""
|
||||
existing_keys = await config_storage.list_keys(prefix="themes/")
|
||||
existing_ids = {k.removeprefix("themes/").removesuffix(".json") for k in existing_keys}
|
||||
for theme in _BUILTIN_THEMES:
|
||||
path = _THEMES_DIR / f"{theme['id']}.json"
|
||||
if not path.exists():
|
||||
path.write_text(json.dumps(theme, indent=2))
|
||||
if theme["id"] not in existing_ids:
|
||||
await config_storage.write_json(f"themes/{theme['id']}.json", theme)
|
||||
logger.info("Built-in themes seeded (%d themes)", len(_BUILTIN_THEMES))
|
||||
|
||||
|
||||
def load_all_themes() -> list[dict]:
|
||||
"""Return all themes from /config/themes/*.json, built-ins first."""
|
||||
if not _THEMES_DIR.exists():
|
||||
seed_builtin_themes()
|
||||
themes = []
|
||||
for f in sorted(_THEMES_DIR.glob("*.json")):
|
||||
try:
|
||||
themes.append(json.loads(f.read_text()))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
# Sort: built-ins first (preserving their original order), then custom by label
|
||||
async def load_all_themes() -> list[dict]:
|
||||
"""Return all themes from storage-service, built-ins first then custom by label."""
|
||||
keys = await config_storage.list_keys(prefix="themes/")
|
||||
themes: list[dict] = []
|
||||
for key in keys:
|
||||
data = await config_storage.read_json(key)
|
||||
if data:
|
||||
themes.append(data)
|
||||
|
||||
builtin_ids = [t["id"] for t in _BUILTIN_THEMES]
|
||||
|
||||
def sort_key(t: dict) -> tuple:
|
||||
tid = t.get("id", "")
|
||||
try:
|
||||
return (0, builtin_ids.index(tid))
|
||||
except ValueError:
|
||||
return (1, t.get("label", tid).lower())
|
||||
|
||||
return sorted(themes, key=sort_key)
|
||||
|
||||
|
||||
async def load_theme_by_id(theme_id: str) -> dict | None:
|
||||
"""Return a single theme dict, or None if not found."""
|
||||
return await config_storage.read_json(f"themes/{theme_id}.json")
|
||||
|
||||
|
||||
async def save_theme(theme: dict) -> None:
|
||||
"""Write a theme to storage-service."""
|
||||
await config_storage.write_json(f"themes/{theme['id']}.json", theme)
|
||||
|
||||
|
||||
async def delete_theme(theme_id: str) -> None:
|
||||
"""Delete a custom theme. Raises ValueError for built-ins, KeyError if not found."""
|
||||
data = await config_storage.read_json(f"themes/{theme_id}.json")
|
||||
if data is None:
|
||||
raise FileNotFoundError(theme_id)
|
||||
if data.get("builtin"):
|
||||
raise ValueError("Cannot delete a built-in theme")
|
||||
await config_storage.delete_key(f"themes/{theme_id}.json")
|
||||
|
||||
|
||||
def validate_theme_tokens(colors: dict) -> list[str]:
|
||||
"""Return a list of validation error messages, empty if valid."""
|
||||
errors = []
|
||||
@@ -401,23 +407,3 @@ def validate_theme_tokens(colors: dict) -> list[str]:
|
||||
if key in _REQUIRED_TOKENS and not _RGB_RE.match(str(val)):
|
||||
errors.append(f"Token '{key}' must be an RGB triplet like '37 99 235', got: {val!r}")
|
||||
return errors
|
||||
|
||||
|
||||
def save_theme(theme: dict) -> None:
|
||||
"""Write a theme file atomically."""
|
||||
_THEMES_DIR.mkdir(parents=True, exist_ok=True)
|
||||
path = _THEMES_DIR / f"{theme['id']}.json"
|
||||
tmp = path.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(theme, indent=2))
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def delete_theme(theme_id: str) -> None:
|
||||
"""Delete a custom theme file. Raises ValueError for built-ins, FileNotFoundError if missing."""
|
||||
path = _THEMES_DIR / f"{theme_id}.json"
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(theme_id)
|
||||
data = json.loads(path.read_text())
|
||||
if data.get("builtin"):
|
||||
raise ValueError("Cannot delete a built-in theme")
|
||||
path.unlink()
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
"""
|
||||
Async HTTP client for the 'config' bucket in storage-service.
|
||||
|
||||
All JSON config files (AI settings, doc settings, appearance, themes, …) are stored
|
||||
in the 'config' bucket under the storage-service. This module provides thin
|
||||
async helpers so app_config.py does not depend on the filesystem at all.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_BUCKET = "config"
|
||||
_TIMEOUT = 10.0
|
||||
|
||||
|
||||
def _url(key: str) -> str:
|
||||
return f"{settings.STORAGE_SERVICE_URL}/objects/{_BUCKET}/{key}"
|
||||
|
||||
|
||||
async def read_json(key: str) -> dict | None:
|
||||
"""Return parsed JSON from the config bucket, or None if the key does not exist."""
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
resp = await client.get(_url(key))
|
||||
if resp.status_code == 404:
|
||||
return None
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
async def write_json(key: str, data: dict) -> None:
|
||||
"""Serialise *data* to JSON and PUT it into the config bucket."""
|
||||
payload = json.dumps(data, indent=2).encode()
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
resp = await client.put(
|
||||
_url(key),
|
||||
content=payload,
|
||||
headers={"Content-Type": "application/octet-stream"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
async def delete_key(key: str) -> None:
|
||||
"""Delete a key from the config bucket. No-op if it does not exist."""
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
resp = await client.delete(_url(key))
|
||||
if resp.status_code not in (204, 404):
|
||||
resp.raise_for_status()
|
||||
|
||||
|
||||
async def list_keys(prefix: str = "") -> list[str]:
|
||||
"""List all keys in the config bucket, optionally filtered by *prefix*."""
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
resp = await client.get(f"{settings.STORAGE_SERVICE_URL}/objects/{_BUCKET}")
|
||||
resp.raise_for_status()
|
||||
keys: list[str] = resp.json().get("keys", [])
|
||||
if prefix:
|
||||
keys = [k for k in keys if k.startswith(prefix)]
|
||||
return keys
|
||||
Reference in New Issue
Block a user