feat: add Settings and Suite pages with comprehensive settings management and question suite builder
- Implemented SettingsPage for configuring default provider, temperature, and model paths. - Added SuitePage for managing a suite of questions with features to add, edit, duplicate, and delete questions. - Introduced TypeScript configuration files for app and node environments. - Set up Vite configuration for development server with API proxying to backend.
This commit is contained in:
Christopher Clendening
@@ -0,0 +1,9 @@
|
||||
"""LM-Gambit web backend.
|
||||
|
||||
Wraps the existing ``.core`` diagnostic engine in a FastAPI application so the
|
||||
React frontend can drive it. The engine itself is untouched: providers, the
|
||||
runner and the markdown reporting pipeline behave exactly as they do for the
|
||||
``auto-test.py`` CLI.
|
||||
"""
|
||||
|
||||
__version__ = "2.0.0"
|
||||
+464
@@ -0,0 +1,464 @@
|
||||
"""HTTP surface for the LM-Gambit frontend."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import platform
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from . import __version__
|
||||
from .core_bridge import (
|
||||
DEFAULT_PROVIDER_NAME,
|
||||
DEFAULT_TEMPERATURE,
|
||||
MODELS_DIR,
|
||||
RESULTS_DIR,
|
||||
TEMPLATE_PATH,
|
||||
TESTS_DIR,
|
||||
EngineLoadError,
|
||||
ProviderError,
|
||||
build_provider,
|
||||
detect_architecture,
|
||||
list_provider_names,
|
||||
load_engine_class,
|
||||
load_settings,
|
||||
save_settings,
|
||||
)
|
||||
from .plugins import get_plugin_manager
|
||||
from .run_manager import run_manager
|
||||
from .schemas import (
|
||||
PluginSummary,
|
||||
ModelListResponse,
|
||||
ModelPathEntry,
|
||||
ModelSummary,
|
||||
PlaygroundRequest,
|
||||
PlaygroundResponse,
|
||||
ProviderSummary,
|
||||
ReportDetail,
|
||||
ReportSummary,
|
||||
RunRequest,
|
||||
RunResponse,
|
||||
SaveSuiteRequest,
|
||||
SettingsResponse,
|
||||
SettingsUpdate,
|
||||
SystemInfo,
|
||||
TestPrompt,
|
||||
TestSuiteResponse,
|
||||
)
|
||||
from .suite import SuiteError, list_tests, find_tests, save_suite
|
||||
|
||||
router = APIRouter(prefix="/api")
|
||||
|
||||
_REPORT_NAME_RE = re.compile(r"^[A-Za-z0-9._-]+\.md$")
|
||||
_REPORT_TITLE_RE = re.compile(r"^#\s*Automated Diagnostic Report:\s*(.+)$", re.MULTILINE)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- system
|
||||
|
||||
|
||||
@router.get("/system", response_model=SystemInfo)
|
||||
async def get_system() -> SystemInfo:
|
||||
descriptor = detect_architecture()
|
||||
runtime_name = "unavailable"
|
||||
try:
|
||||
runtime_cls = load_engine_class(descriptor)
|
||||
runtime_name = getattr(runtime_cls, "name", runtime_cls.__name__)
|
||||
except EngineLoadError as exc:
|
||||
runtime_name = f"unavailable ({exc})"
|
||||
|
||||
return SystemInfo(
|
||||
version=__version__,
|
||||
engine_architecture=descriptor.architecture,
|
||||
engine_runtime=runtime_name,
|
||||
template_ok=TEMPLATE_PATH.exists(),
|
||||
python_version=platform.python_version(),
|
||||
metrics={
|
||||
"platform": f"{platform.system()} {platform.machine()}",
|
||||
"models_dir": str(MODELS_DIR),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- plugins
|
||||
|
||||
|
||||
@router.get("/plugins", response_model=List[PluginSummary])
|
||||
async def get_plugins() -> List[PluginSummary]:
|
||||
"""Everything discovered in plugins/, including files that failed to load."""
|
||||
return [PluginSummary(**vars(info)) for info in get_plugin_manager().info()]
|
||||
|
||||
|
||||
@router.post("/plugins/reload", response_model=List[PluginSummary])
|
||||
async def reload_plugins_endpoint() -> List[PluginSummary]:
|
||||
"""Re-scan plugins/. Newly added HTTP routes still need a server restart."""
|
||||
if run_manager.active() is not None:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
detail="A run is in progress. Reload plugins once it finishes.",
|
||||
)
|
||||
manager = get_plugin_manager()
|
||||
manager.load()
|
||||
return [PluginSummary(**vars(info)) for info in manager.info()]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ providers
|
||||
|
||||
|
||||
@router.get("/providers", response_model=List[ProviderSummary])
|
||||
async def get_providers() -> List[ProviderSummary]:
|
||||
settings = load_settings()
|
||||
preferred = str(settings.get("default_provider") or DEFAULT_PROVIDER_NAME)
|
||||
return [
|
||||
ProviderSummary(name=name, is_default=name == preferred)
|
||||
for name in list_provider_names()
|
||||
]
|
||||
|
||||
|
||||
@router.get("/providers/{provider_name}/models", response_model=ModelListResponse)
|
||||
async def get_models(provider_name: str) -> ModelListResponse:
|
||||
def _load() -> List[ModelSummary]:
|
||||
provider = build_provider(provider_name)
|
||||
return [
|
||||
ModelSummary(id=model.id, display_name=model.display_name)
|
||||
for model in provider.list_models()
|
||||
]
|
||||
|
||||
try:
|
||||
models = await asyncio.to_thread(_load)
|
||||
except ProviderError as exc:
|
||||
raise HTTPException(status.HTTP_502_BAD_GATEWAY, detail=str(exc)) from exc
|
||||
except Exception as exc: # noqa: BLE001 - engine/hardware faults reach the UI
|
||||
raise HTTPException(
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"{type(exc).__name__}: {exc}",
|
||||
) from exc
|
||||
|
||||
return ModelListResponse(provider=provider_name, models=models)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- test suite
|
||||
|
||||
|
||||
@router.get("/tests", response_model=TestSuiteResponse)
|
||||
async def get_tests() -> TestSuiteResponse:
|
||||
tests = [TestPrompt(**prompt) for prompt in list_tests()]
|
||||
return TestSuiteResponse(tests=tests, directory=str(TESTS_DIR))
|
||||
|
||||
|
||||
@router.put("/tests", response_model=TestSuiteResponse)
|
||||
async def put_tests(payload: SaveSuiteRequest) -> TestSuiteResponse:
|
||||
if run_manager.active() is not None:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
detail="A run is in progress. Wait for it to finish before editing the suite.",
|
||||
)
|
||||
try:
|
||||
tests = save_suite([draft.prompt for draft in payload.tests])
|
||||
except SuiteError as exc:
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
|
||||
|
||||
return TestSuiteResponse(
|
||||
tests=[TestPrompt(**prompt) for prompt in tests],
|
||||
directory=str(TESTS_DIR),
|
||||
)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------- runs
|
||||
|
||||
|
||||
@router.post("/runs", response_model=RunResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def create_run(payload: RunRequest) -> RunResponse:
|
||||
if not TEMPLATE_PATH.exists():
|
||||
raise HTTPException(
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
detail="Report template missing at .core/templates/test-block.md.",
|
||||
)
|
||||
|
||||
try:
|
||||
prompts = find_tests(payload.filenames) if payload.filenames else list_tests()
|
||||
except SuiteError as exc:
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
|
||||
|
||||
if not prompts:
|
||||
raise HTTPException(
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
detail="No questions in the suite. Add at least one in the Suite Builder.",
|
||||
)
|
||||
|
||||
def _resolve_label() -> str:
|
||||
provider = build_provider(payload.provider)
|
||||
for model in provider.list_models():
|
||||
if model.id == payload.model_id:
|
||||
return model.display_name
|
||||
raise ProviderError(f"Model '{payload.model_id}' not found for {payload.provider}.")
|
||||
|
||||
try:
|
||||
model_label = await asyncio.to_thread(_resolve_label)
|
||||
except ProviderError as exc:
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
|
||||
|
||||
try:
|
||||
run = run_manager.start(
|
||||
provider_name=payload.provider,
|
||||
model_id=payload.model_id,
|
||||
model_label=model_label,
|
||||
temperature=payload.temperature,
|
||||
prompts=prompts,
|
||||
loop=asyncio.get_running_loop(),
|
||||
)
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status.HTTP_409_CONFLICT, detail=str(exc)) from exc
|
||||
|
||||
return RunResponse(**run.as_dict())
|
||||
|
||||
|
||||
@router.get("/runs", response_model=List[RunResponse])
|
||||
async def get_runs() -> List[RunResponse]:
|
||||
return [RunResponse(**run.as_dict()) for run in run_manager.history()]
|
||||
|
||||
|
||||
@router.get("/runs/active", response_model=Optional[RunResponse])
|
||||
async def get_active_run() -> Optional[RunResponse]:
|
||||
run = run_manager.active()
|
||||
return RunResponse(**run.as_dict()) if run else None
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}", response_model=RunResponse)
|
||||
async def get_run(run_id: str) -> RunResponse:
|
||||
run = run_manager.get(run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="Unknown run id.")
|
||||
return RunResponse(**run.as_dict())
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/events")
|
||||
async def stream_run(run_id: str) -> StreamingResponse:
|
||||
if run_manager.get(run_id) is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="Unknown run id.")
|
||||
return StreamingResponse(
|
||||
run_manager.stream(run_id),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache, no-transform",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/runs/{run_id}/cancel", response_model=RunResponse)
|
||||
async def cancel_run(run_id: str) -> RunResponse:
|
||||
run = run_manager.get(run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="Unknown run id.")
|
||||
if not run_manager.cancel(run_id):
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
detail=f"Run is already {run.status}.",
|
||||
)
|
||||
return RunResponse(**run.as_dict())
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- reports
|
||||
|
||||
|
||||
def _report_path(name: str) -> Path:
|
||||
if not _REPORT_NAME_RE.match(name):
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail="Invalid report name.")
|
||||
path = (RESULTS_DIR / name).resolve()
|
||||
if path.parent != RESULTS_DIR.resolve() or not path.is_file():
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, detail="Report not found.")
|
||||
return path
|
||||
|
||||
|
||||
def _model_label_from(content: str, fallback: str) -> str:
|
||||
match = _REPORT_TITLE_RE.search(content)
|
||||
return match.group(1).strip() if match else fallback
|
||||
|
||||
|
||||
@router.get("/reports", response_model=List[ReportSummary])
|
||||
async def get_reports() -> List[ReportSummary]:
|
||||
if not RESULTS_DIR.exists():
|
||||
return []
|
||||
|
||||
summaries: List[ReportSummary] = []
|
||||
for path in RESULTS_DIR.glob("*.md"):
|
||||
try:
|
||||
stat = path.stat()
|
||||
head = path.read_text(encoding="utf-8", errors="replace")[:400]
|
||||
except OSError:
|
||||
continue
|
||||
summaries.append(
|
||||
ReportSummary(
|
||||
name=path.name,
|
||||
model_label=_model_label_from(head, path.stem),
|
||||
size_bytes=stat.st_size,
|
||||
modified_at=stat.st_mtime,
|
||||
)
|
||||
)
|
||||
summaries.sort(key=lambda item: item.modified_at, reverse=True)
|
||||
return summaries
|
||||
|
||||
|
||||
@router.get("/reports/{name}", response_model=ReportDetail)
|
||||
async def get_report(name: str) -> ReportDetail:
|
||||
path = _report_path(name)
|
||||
try:
|
||||
content = path.read_text(encoding="utf-8", errors="replace")
|
||||
stat = path.stat()
|
||||
except OSError as exc:
|
||||
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc)) from exc
|
||||
|
||||
return ReportDetail(
|
||||
name=path.name,
|
||||
model_label=_model_label_from(content, path.stem),
|
||||
size_bytes=stat.st_size,
|
||||
modified_at=stat.st_mtime,
|
||||
content=content,
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/reports/{name}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_report(name: str) -> None:
|
||||
path = _report_path(name)
|
||||
try:
|
||||
path.unlink()
|
||||
except OSError as exc:
|
||||
raise HTTPException(status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc)) from exc
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- playground
|
||||
|
||||
|
||||
@router.post("/playground", response_model=PlaygroundResponse)
|
||||
async def run_playground(payload: PlaygroundRequest) -> PlaygroundResponse:
|
||||
if run_manager.active() is not None:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
detail="A diagnostic run is in progress. The engine can only serve one at a time.",
|
||||
)
|
||||
|
||||
def _execute() -> Dict[str, Any]:
|
||||
provider = build_provider(payload.provider)
|
||||
return provider.run_prompt(
|
||||
payload.model_id,
|
||||
payload.prompt,
|
||||
temperature=payload.temperature,
|
||||
)
|
||||
|
||||
started = time.time()
|
||||
try:
|
||||
result = await asyncio.to_thread(_execute)
|
||||
except ProviderError as exc:
|
||||
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
|
||||
except Exception as exc: # noqa: BLE001 - engine faults are user-facing here
|
||||
return PlaygroundResponse(
|
||||
error=f"{type(exc).__name__}: {exc}",
|
||||
elapsed=round(time.time() - started, 2),
|
||||
)
|
||||
|
||||
elapsed = round(time.time() - started, 2)
|
||||
if "error" in result:
|
||||
return PlaygroundResponse(error=str(result["error"]), elapsed=elapsed)
|
||||
|
||||
return PlaygroundResponse(
|
||||
response=str(result.get("response", "")),
|
||||
metrics=result.get("metrics") or None,
|
||||
elapsed=elapsed,
|
||||
)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- settings
|
||||
|
||||
_NICKNAME_KEY = "local_model_path_nicknames"
|
||||
|
||||
|
||||
def _normalized_paths(settings: Dict[str, Any]) -> List[str]:
|
||||
"""Read model paths tolerating the legacy list-of-dicts format."""
|
||||
raw = settings.get("local_model_paths", [])
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
paths: List[str] = []
|
||||
for entry in raw:
|
||||
if isinstance(entry, str) and entry.strip():
|
||||
candidate = str(Path(entry).expanduser())
|
||||
elif isinstance(entry, dict) and str(entry.get("path", "")).strip():
|
||||
candidate = str(Path(str(entry["path"])).expanduser())
|
||||
else:
|
||||
continue
|
||||
if candidate not in paths:
|
||||
paths.append(candidate)
|
||||
return paths
|
||||
|
||||
|
||||
def _nicknames(settings: Dict[str, Any]) -> Dict[str, str]:
|
||||
"""Nicknames live in their own key so the engine's path list stays strings."""
|
||||
nicknames: Dict[str, str] = {}
|
||||
stored = settings.get(_NICKNAME_KEY)
|
||||
if isinstance(stored, dict):
|
||||
for key, value in stored.items():
|
||||
if isinstance(key, str) and isinstance(value, str):
|
||||
nicknames[str(Path(key).expanduser())] = value
|
||||
|
||||
# Recover nicknames from the legacy list-of-dicts shape.
|
||||
raw = settings.get("local_model_paths", [])
|
||||
if isinstance(raw, list):
|
||||
for entry in raw:
|
||||
if isinstance(entry, dict) and entry.get("path"):
|
||||
key = str(Path(str(entry["path"])).expanduser())
|
||||
nicknames.setdefault(key, str(entry.get("nickname", "")))
|
||||
return nicknames
|
||||
|
||||
|
||||
@router.get("/settings", response_model=SettingsResponse)
|
||||
async def get_settings() -> SettingsResponse:
|
||||
settings = load_settings()
|
||||
nicknames = _nicknames(settings)
|
||||
return SettingsResponse(
|
||||
default_provider=str(settings.get("default_provider") or DEFAULT_PROVIDER_NAME),
|
||||
default_temperature=float(settings.get("default_temperature") or DEFAULT_TEMPERATURE),
|
||||
local_model_paths=[
|
||||
ModelPathEntry(nickname=nicknames.get(path, ""), path=path)
|
||||
for path in _normalized_paths(settings)
|
||||
],
|
||||
tests_dir=str(TESTS_DIR),
|
||||
results_dir=str(RESULTS_DIR),
|
||||
models_dir=str(MODELS_DIR),
|
||||
)
|
||||
|
||||
|
||||
@router.put("/settings", response_model=SettingsResponse)
|
||||
async def put_settings(payload: SettingsUpdate) -> SettingsResponse:
|
||||
settings = load_settings()
|
||||
|
||||
if payload.default_provider is not None:
|
||||
if payload.default_provider not in list_provider_names():
|
||||
raise HTTPException(
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Unknown provider '{payload.default_provider}'.",
|
||||
)
|
||||
settings["default_provider"] = payload.default_provider
|
||||
|
||||
if payload.default_temperature is not None:
|
||||
settings["default_temperature"] = payload.default_temperature
|
||||
|
||||
if payload.local_model_paths is not None:
|
||||
ordered: List[str] = []
|
||||
nicknames: Dict[str, str] = {}
|
||||
for entry in payload.local_model_paths:
|
||||
candidate = str(Path(entry.path).expanduser())
|
||||
if not candidate.strip() or candidate in ordered:
|
||||
continue
|
||||
ordered.append(candidate)
|
||||
if entry.nickname.strip():
|
||||
nicknames[candidate] = entry.nickname.strip()
|
||||
settings["local_model_paths"] = ordered
|
||||
settings[_NICKNAME_KEY] = nicknames
|
||||
|
||||
save_settings(settings)
|
||||
return await get_settings()
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Import shim for the ``.core`` engine package.
|
||||
|
||||
``.core`` is a hidden directory, so it cannot be imported as a normal package.
|
||||
The CLI (``auto-test.py``) works around this by putting the directory on
|
||||
``sys.path`` and importing its modules flat. We do the same thing here, in one
|
||||
place, so the rest of the server can just ``from .core_bridge import run_suite``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT_DIR = Path(__file__).resolve().parent.parent
|
||||
CORE_DIR = ROOT_DIR / ".core"
|
||||
|
||||
if str(CORE_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(CORE_DIR))
|
||||
|
||||
# ruff: noqa: E402 (imports must follow the sys.path mutation above)
|
||||
from config import ( # type: ignore[import-not-found]
|
||||
DEFAULT_PROVIDER_NAME,
|
||||
DEFAULT_TEMPERATURE,
|
||||
MODELS_DIR,
|
||||
RESULTS_DIR,
|
||||
TEMPLATE_PATH,
|
||||
TESTS_DIR,
|
||||
ensure_directories,
|
||||
)
|
||||
from engine_loader import ( # type: ignore[import-not-found]
|
||||
EngineLoadError,
|
||||
detect_architecture,
|
||||
load_engine_class,
|
||||
)
|
||||
from prompts import load_test_prompts # type: ignore[import-not-found]
|
||||
from providers import ( # type: ignore[import-not-found]
|
||||
LocalEngineProvider,
|
||||
ModelInfo,
|
||||
Provider,
|
||||
ProviderError,
|
||||
get_provider,
|
||||
list_provider_names,
|
||||
)
|
||||
from reporting import ( # type: ignore[import-not-found]
|
||||
TemplateNotFoundError,
|
||||
append_sections,
|
||||
finalize_report_summary,
|
||||
replace_analysis_section,
|
||||
sanitize_model_name,
|
||||
)
|
||||
from runner import TestRunError, run_suite # type: ignore[import-not-found]
|
||||
from settings import ( # type: ignore[import-not-found]
|
||||
get_local_model_paths,
|
||||
load_settings,
|
||||
save_settings,
|
||||
set_local_model_paths,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"CORE_DIR",
|
||||
"DEFAULT_PROVIDER_NAME",
|
||||
"EngineLoadError",
|
||||
"detect_architecture",
|
||||
"load_engine_class",
|
||||
"DEFAULT_TEMPERATURE",
|
||||
"MODELS_DIR",
|
||||
"ModelInfo",
|
||||
"Provider",
|
||||
"ProviderError",
|
||||
"RESULTS_DIR",
|
||||
"ROOT_DIR",
|
||||
"TEMPLATE_PATH",
|
||||
"TESTS_DIR",
|
||||
"TemplateNotFoundError",
|
||||
"TestRunError",
|
||||
"LocalEngineProvider",
|
||||
"append_sections",
|
||||
"ensure_directories",
|
||||
"finalize_report_summary",
|
||||
"replace_analysis_section",
|
||||
"get_local_model_paths",
|
||||
"get_provider",
|
||||
"list_provider_names",
|
||||
"load_settings",
|
||||
"load_test_prompts",
|
||||
"run_suite",
|
||||
"sanitize_model_name",
|
||||
"save_settings",
|
||||
"set_local_model_paths",
|
||||
]
|
||||
|
||||
|
||||
def provider_kwargs(provider_name: str) -> dict:
|
||||
"""Build constructor kwargs for a provider, mirroring the runner's logic."""
|
||||
if provider_name == LocalEngineProvider.name:
|
||||
custom_paths = [Path(p).expanduser() for p in get_local_model_paths()]
|
||||
return {"search_paths": [path for path in custom_paths if path]}
|
||||
return {}
|
||||
|
||||
|
||||
def build_provider(provider_name: str) -> Provider:
|
||||
"""Instantiate a provider with the correct per-provider configuration."""
|
||||
return get_provider(provider_name, **provider_kwargs(provider_name))
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
"""FastAPI application factory.
|
||||
|
||||
In production the same process serves the JSON API and the compiled React
|
||||
bundle from ``web/dist``. During frontend development Vite serves the UI on its
|
||||
own port and proxies ``/api`` back here, so CORS is opened for localhost only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import sys
|
||||
|
||||
from fastapi import APIRouter, FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from starlette.requests import Request
|
||||
|
||||
from . import __version__
|
||||
from .api import router
|
||||
from .core_bridge import ROOT_DIR, ensure_directories
|
||||
from .plugins import get_plugin_manager
|
||||
|
||||
WEB_DIST = ROOT_DIR / "web" / "dist"
|
||||
|
||||
DEV_ORIGINS = [
|
||||
"http://localhost:5173",
|
||||
"http://127.0.0.1:5173",
|
||||
]
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
ensure_directories()
|
||||
|
||||
app = FastAPI(
|
||||
title="LM-Gambit",
|
||||
description="Automated LLM diagnostic suite.",
|
||||
version=__version__,
|
||||
docs_url="/api/docs",
|
||||
openapi_url="/api/openapi.json",
|
||||
)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=DEV_ORIGINS,
|
||||
allow_credentials=False,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(router)
|
||||
_mount_plugin_routes(app)
|
||||
|
||||
@app.get("/api/health")
|
||||
async def health() -> dict:
|
||||
return {"status": "ok", "version": __version__, "ui_built": WEB_DIST.is_dir()}
|
||||
|
||||
_mount_frontend(app)
|
||||
return app
|
||||
|
||||
|
||||
def _mount_plugin_routes(app: FastAPI) -> None:
|
||||
"""Give each plugin defining register_routes its own /api/plugins/<slug>.
|
||||
|
||||
Routes are bound at startup, so a plugin that adds or changes endpoints
|
||||
needs a server restart — unlike its other hooks, which reload live.
|
||||
"""
|
||||
manager = get_plugin_manager()
|
||||
for plugin in manager.with_hook("register_routes"):
|
||||
plugin_router = APIRouter(
|
||||
prefix=f"/api/plugins/{plugin.slug}",
|
||||
tags=[f"plugin:{plugin.slug}"],
|
||||
)
|
||||
try:
|
||||
plugin.module.register_routes(plugin_router) # type: ignore[union-attr]
|
||||
except Exception as exc: # noqa: BLE001 - never let a plugin block startup
|
||||
print(
|
||||
f"[plugin:{plugin.slug}] register_routes() failed: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
continue
|
||||
if plugin_router.routes:
|
||||
app.include_router(plugin_router)
|
||||
|
||||
|
||||
def _mount_frontend(app: FastAPI) -> None:
|
||||
assets_dir = WEB_DIST / "assets"
|
||||
if assets_dir.is_dir():
|
||||
app.mount("/assets", StaticFiles(directory=assets_dir), name="assets")
|
||||
|
||||
@app.get("/{full_path:path}", include_in_schema=False)
|
||||
async def spa(request: Request, full_path: str) -> object:
|
||||
if full_path.startswith("api/"):
|
||||
return JSONResponse({"detail": "Not found."}, status_code=404)
|
||||
|
||||
index_file = WEB_DIST / "index.html"
|
||||
if not index_file.is_file():
|
||||
return JSONResponse(
|
||||
{
|
||||
"detail": "Frontend not built.",
|
||||
"fix": "Run 'npm install && npm run build' inside web/, "
|
||||
"or start the Vite dev server with 'npm run dev'.",
|
||||
},
|
||||
status_code=503,
|
||||
)
|
||||
|
||||
# Serve real files (favicon, manifest, …) directly; everything else
|
||||
# falls through to index.html so client-side routing works on reload.
|
||||
if full_path:
|
||||
candidate = (WEB_DIST / full_path).resolve()
|
||||
if candidate.is_file() and WEB_DIST.resolve() in candidate.parents:
|
||||
return FileResponse(candidate)
|
||||
|
||||
return FileResponse(index_file)
|
||||
|
||||
|
||||
app = create_app()
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Bridge between the server's internals and the plugin API.
|
||||
|
||||
Plugins only ever see the frozen dataclasses in ``plugin_api``. This module does
|
||||
the translation and owns the markdown rendering of grades, so nothing in the
|
||||
plugin contract is coupled to how runs happen to be stored.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
||||
|
||||
ROOT_DIR = Path(__file__).resolve().parent.parent
|
||||
if str(ROOT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT_DIR))
|
||||
|
||||
from plugin_api import Grade, GradeEntry, RunRecord, TestRecord # noqa: E402
|
||||
from plugin_system import ( # noqa: E402
|
||||
PluginManager,
|
||||
collect_report_sections,
|
||||
get_plugin_manager,
|
||||
letter_grade,
|
||||
render_grade_section,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from .run_manager import RunState, TestOutcome
|
||||
|
||||
__all__ = [
|
||||
"Grade",
|
||||
"GradeEntry",
|
||||
"PluginManager",
|
||||
"RunRecord",
|
||||
"TestRecord",
|
||||
"build_run_record",
|
||||
"build_test_record",
|
||||
"collect_report_sections",
|
||||
"get_plugin_manager",
|
||||
"letter_grade",
|
||||
"render_grade_section",
|
||||
]
|
||||
|
||||
|
||||
def build_test_record(
|
||||
outcome: "TestOutcome",
|
||||
*,
|
||||
total: int,
|
||||
prompt_text: str,
|
||||
) -> TestRecord:
|
||||
return TestRecord(
|
||||
index=outcome.index,
|
||||
total=total,
|
||||
title=outcome.title,
|
||||
filename=outcome.filename,
|
||||
prompt=prompt_text,
|
||||
ok=outcome.status == "ok",
|
||||
response=outcome.response,
|
||||
error=outcome.error,
|
||||
metrics=dict(outcome.metrics or {}),
|
||||
elapsed=outcome.elapsed,
|
||||
)
|
||||
|
||||
|
||||
def build_run_record(
|
||||
run: "RunState",
|
||||
*,
|
||||
prompts_by_filename: Dict[str, str],
|
||||
report_path: Optional[Path] = None,
|
||||
) -> RunRecord:
|
||||
tests = [
|
||||
build_test_record(
|
||||
outcome,
|
||||
total=run.total,
|
||||
prompt_text=prompts_by_filename.get(outcome.filename, ""),
|
||||
)
|
||||
for outcome in run.outcomes
|
||||
]
|
||||
return RunRecord(
|
||||
id=run.id,
|
||||
provider=run.provider,
|
||||
model_id=run.model_id,
|
||||
model_label=run.model_label,
|
||||
temperature=run.temperature,
|
||||
total=run.total,
|
||||
status=run.status,
|
||||
started_at=run.started_at,
|
||||
finished_at=run.finished_at,
|
||||
report_path=report_path,
|
||||
summary=run.summary(),
|
||||
tests=tests,
|
||||
grades={index: list(entries) for index, entries in run.grades.items()},
|
||||
)
|
||||
@@ -0,0 +1,523 @@
|
||||
"""Background execution of diagnostic runs with server-sent-event streaming.
|
||||
|
||||
The engine's ``run_suite`` is synchronous and blocking, so each run executes on
|
||||
a worker thread. Progress is pushed back onto the asyncio loop through
|
||||
``loop.call_soon_threadsafe`` and fanned out to any connected SSE subscribers.
|
||||
|
||||
Every event is also retained on the run, so a client that connects late (or
|
||||
reconnects after a dropped connection) replays the full history before it
|
||||
starts following the live feed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, AsyncIterator, Dict, List, Optional
|
||||
|
||||
from .core_bridge import (
|
||||
RESULTS_DIR,
|
||||
TemplateNotFoundError,
|
||||
TestRunError,
|
||||
append_sections,
|
||||
finalize_report_summary,
|
||||
replace_analysis_section,
|
||||
run_suite,
|
||||
sanitize_model_name,
|
||||
)
|
||||
from .plugins import (
|
||||
GradeEntry,
|
||||
build_run_record,
|
||||
build_test_record,
|
||||
collect_report_sections,
|
||||
get_plugin_manager,
|
||||
render_grade_section,
|
||||
)
|
||||
|
||||
TERMINAL_EVENTS = {"run.completed", "run.failed", "run.cancelled"}
|
||||
MAX_RUN_HISTORY = 20
|
||||
KEEPALIVE_SECONDS = 15.0
|
||||
|
||||
|
||||
class RunCancelled(Exception):
|
||||
"""Raised inside the engine's progress callback to unwind a running suite.
|
||||
|
||||
The engine has no cancellation hook of its own. Raising from the callback
|
||||
stops the loop cleanly between tests -- results already written to the
|
||||
report are kept and the summary is finalized with whatever completed.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestOutcome:
|
||||
index: int
|
||||
title: str
|
||||
filename: str
|
||||
status: str
|
||||
elapsed: float
|
||||
response: Optional[str] = None
|
||||
error: Optional[str] = None
|
||||
metrics: Optional[Dict[str, Any]] = None
|
||||
grades: List[Dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def score(self) -> Optional[float]:
|
||||
if not self.grades:
|
||||
return None
|
||||
return sum(g["score"] for g in self.grades) / len(self.grades)
|
||||
|
||||
def as_event(self, total: int) -> Dict[str, Any]:
|
||||
return {
|
||||
"type": "test.completed",
|
||||
"index": self.index,
|
||||
"total": total,
|
||||
"title": self.title,
|
||||
"filename": self.filename,
|
||||
"status": self.status,
|
||||
"elapsed": round(self.elapsed, 2),
|
||||
"response": self.response,
|
||||
"error": self.error,
|
||||
"metrics": self.metrics,
|
||||
"grades": self.grades,
|
||||
"score": self.score,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunState:
|
||||
id: str
|
||||
provider: str
|
||||
model_id: str
|
||||
model_label: str
|
||||
temperature: float
|
||||
total: int
|
||||
status: str = "running"
|
||||
started_at: float = field(default_factory=time.time)
|
||||
finished_at: Optional[float] = None
|
||||
report_name: Optional[str] = None
|
||||
error: Optional[str] = None
|
||||
outcomes: List[TestOutcome] = field(default_factory=list)
|
||||
events: List[Dict[str, Any]] = field(default_factory=list)
|
||||
subscribers: List["asyncio.Queue[Dict[str, Any]]"] = field(default_factory=list)
|
||||
cancel_event: threading.Event = field(default_factory=threading.Event)
|
||||
grades: Dict[int, List[GradeEntry]] = field(default_factory=dict)
|
||||
prompts_by_filename: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def completed(self) -> int:
|
||||
return len(self.outcomes)
|
||||
|
||||
@property
|
||||
def overall_score(self) -> Optional[float]:
|
||||
scores = [o.score for o in self.outcomes if o.score is not None]
|
||||
return sum(scores) / len(scores) if scores else None
|
||||
|
||||
def summary(self) -> Dict[str, Any]:
|
||||
ok = [o for o in self.outcomes if o.status == "ok" and o.metrics]
|
||||
if ok:
|
||||
avg_tps = sum(float(o.metrics.get("tokens_per_second", 0)) for o in ok) / len(ok)
|
||||
avg_ttft = sum(float(o.metrics.get("time_to_first_token", 0)) for o in ok) / len(ok)
|
||||
total_tokens = sum(int(o.metrics.get("total_tokens", 0)) for o in ok)
|
||||
else:
|
||||
avg_tps = avg_ttft = 0.0
|
||||
total_tokens = 0
|
||||
overall = self.overall_score
|
||||
return {
|
||||
"average_tokens_per_second": round(avg_tps, 2),
|
||||
"average_time_to_first_token": round(avg_ttft, 2),
|
||||
"total_tokens": total_tokens,
|
||||
"passed": len([o for o in self.outcomes if o.status == "ok"]),
|
||||
"failed": len([o for o in self.outcomes if o.status == "error"]),
|
||||
"overall_score": round(overall, 4) if overall is not None else None,
|
||||
"graded": len([o for o in self.outcomes if o.score is not None]),
|
||||
}
|
||||
|
||||
def as_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": self.id,
|
||||
"status": self.status,
|
||||
"provider": self.provider,
|
||||
"model_id": self.model_id,
|
||||
"model_label": self.model_label,
|
||||
"temperature": self.temperature,
|
||||
"total": self.total,
|
||||
"completed": self.completed,
|
||||
"started_at": self.started_at,
|
||||
"finished_at": self.finished_at,
|
||||
"report_name": self.report_name,
|
||||
"error": self.error,
|
||||
"summary": self.summary(),
|
||||
}
|
||||
|
||||
|
||||
def _format_sse(event: Dict[str, Any]) -> str:
|
||||
return f"event: {event['type']}\ndata: {json.dumps(event)}\n\n"
|
||||
|
||||
|
||||
class RunManager:
|
||||
"""Owns the lifecycle of diagnostic runs.
|
||||
|
||||
Only one run may be active at a time: the local engine loads model weights
|
||||
into memory, so overlapping runs would compete for RAM and produce
|
||||
meaningless throughput numbers.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._runs: "OrderedDict[str, RunState]" = OrderedDict()
|
||||
self._active_id: Optional[str] = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# ------------------------------------------------------------------ query
|
||||
|
||||
def get(self, run_id: str) -> Optional[RunState]:
|
||||
with self._lock:
|
||||
return self._runs.get(run_id)
|
||||
|
||||
def active(self) -> Optional[RunState]:
|
||||
with self._lock:
|
||||
if self._active_id is None:
|
||||
return None
|
||||
return self._runs.get(self._active_id)
|
||||
|
||||
def history(self) -> List[RunState]:
|
||||
with self._lock:
|
||||
return list(reversed(self._runs.values()))
|
||||
|
||||
# ------------------------------------------------------------------ start
|
||||
|
||||
def start(
|
||||
self,
|
||||
*,
|
||||
provider_name: str,
|
||||
model_id: str,
|
||||
model_label: str,
|
||||
temperature: float,
|
||||
prompts: List[Dict[str, str]],
|
||||
loop: asyncio.AbstractEventLoop,
|
||||
) -> RunState:
|
||||
with self._lock:
|
||||
if self._active_id is not None:
|
||||
active = self._runs.get(self._active_id)
|
||||
if active is not None and active.status == "running":
|
||||
raise RuntimeError(
|
||||
f"A run is already in progress ({active.model_label}). "
|
||||
"Cancel it before starting another."
|
||||
)
|
||||
|
||||
run = RunState(
|
||||
id=uuid.uuid4().hex[:12],
|
||||
provider=provider_name,
|
||||
model_id=model_id,
|
||||
model_label=model_label,
|
||||
temperature=temperature,
|
||||
total=len(prompts),
|
||||
prompts_by_filename={
|
||||
p.get("filename", ""): p.get("prompt", "") for p in prompts
|
||||
},
|
||||
)
|
||||
self._runs[run.id] = run
|
||||
self._active_id = run.id
|
||||
while len(self._runs) > MAX_RUN_HISTORY:
|
||||
oldest, _ = next(iter(self._runs.items()))
|
||||
if oldest == self._active_id:
|
||||
break
|
||||
self._runs.pop(oldest)
|
||||
|
||||
run.events.append(
|
||||
{
|
||||
"type": "run.started",
|
||||
"run": run.as_dict(),
|
||||
"tests": [
|
||||
{"index": i, "title": p["title"], "filename": p["filename"]}
|
||||
for i, p in enumerate(prompts, start=1)
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
thread = threading.Thread(
|
||||
target=self._worker,
|
||||
args=(run, prompts, loop),
|
||||
name=f"lm-gambit-run-{run.id}",
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
return run
|
||||
|
||||
def cancel(self, run_id: str) -> bool:
|
||||
run = self.get(run_id)
|
||||
if run is None or run.status != "running":
|
||||
return False
|
||||
run.cancel_event.set()
|
||||
return True
|
||||
|
||||
# ----------------------------------------------------------------- worker
|
||||
|
||||
def _worker(
|
||||
self,
|
||||
run: RunState,
|
||||
prompts: List[Dict[str, str]],
|
||||
loop: asyncio.AbstractEventLoop,
|
||||
) -> None:
|
||||
last_tick = time.time()
|
||||
|
||||
def progress_callback(
|
||||
index: int,
|
||||
total: int,
|
||||
prompt: Dict[str, str],
|
||||
result: Dict[str, Any],
|
||||
) -> None:
|
||||
nonlocal last_tick
|
||||
now = time.time()
|
||||
elapsed = now - last_tick
|
||||
last_tick = now
|
||||
|
||||
if "error" in result:
|
||||
outcome = TestOutcome(
|
||||
index=index,
|
||||
title=prompt.get("title", f"Test {index}"),
|
||||
filename=prompt.get("filename", f"test{index}"),
|
||||
status="error",
|
||||
elapsed=elapsed,
|
||||
error=str(result["error"]),
|
||||
)
|
||||
else:
|
||||
metrics = result.get("metrics") or {}
|
||||
outcome = TestOutcome(
|
||||
index=index,
|
||||
title=prompt.get("title", f"Test {index}"),
|
||||
filename=prompt.get("filename", f"test{index}"),
|
||||
status="ok",
|
||||
elapsed=elapsed,
|
||||
response=str(result.get("response", "")),
|
||||
metrics=dict(metrics),
|
||||
)
|
||||
|
||||
run.outcomes.append(outcome)
|
||||
self._grade(run, outcome)
|
||||
|
||||
# Plugins observe the question after grading, so on_test_complete
|
||||
# sees the same record the report will.
|
||||
record = build_test_record(
|
||||
outcome,
|
||||
total=run.total,
|
||||
prompt_text=run.prompts_by_filename.get(outcome.filename, ""),
|
||||
)
|
||||
get_plugin_manager().emit("on_test_complete", record)
|
||||
|
||||
self._publish(loop, run, outcome.as_event(total))
|
||||
|
||||
if run.cancel_event.is_set():
|
||||
raise RunCancelled
|
||||
|
||||
get_plugin_manager().emit(
|
||||
"on_run_start",
|
||||
build_run_record(run, prompts_by_filename=run.prompts_by_filename),
|
||||
)
|
||||
|
||||
try:
|
||||
report_path = run_suite(
|
||||
provider_name=run.provider,
|
||||
model_id=run.model_id,
|
||||
temperature=run.temperature,
|
||||
progress_callback=progress_callback,
|
||||
prompts=prompts,
|
||||
)
|
||||
except RunCancelled:
|
||||
self._finalize_partial_report(run)
|
||||
run.status = "cancelled"
|
||||
run.finished_at = time.time()
|
||||
self._apply_plugin_report(run)
|
||||
self._publish(
|
||||
loop,
|
||||
run,
|
||||
{"type": "run.cancelled", "run": run.as_dict()},
|
||||
)
|
||||
except (TestRunError, TemplateNotFoundError) as exc:
|
||||
run.status = "failed"
|
||||
run.error = str(exc)
|
||||
run.finished_at = time.time()
|
||||
self._publish(
|
||||
loop,
|
||||
run,
|
||||
{"type": "run.failed", "message": str(exc), "run": run.as_dict()},
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - surface engine faults to the UI
|
||||
run.status = "failed"
|
||||
run.error = f"{type(exc).__name__}: {exc}"
|
||||
run.finished_at = time.time()
|
||||
self._publish(
|
||||
loop,
|
||||
run,
|
||||
{"type": "run.failed", "message": run.error, "run": run.as_dict()},
|
||||
)
|
||||
else:
|
||||
run.status = "completed"
|
||||
run.report_name = report_path.name
|
||||
run.finished_at = time.time()
|
||||
self._apply_plugin_report(run)
|
||||
self._publish(
|
||||
loop,
|
||||
run,
|
||||
{"type": "run.completed", "run": run.as_dict()},
|
||||
)
|
||||
finally:
|
||||
with self._lock:
|
||||
if self._active_id == run.id:
|
||||
self._active_id = None
|
||||
get_plugin_manager().emit("on_run_complete", self._record(run))
|
||||
|
||||
# ---------------------------------------------------------------- plugins
|
||||
|
||||
def _report_path(self, run: RunState) -> Path:
|
||||
return RESULTS_DIR / f"automated_report_{sanitize_model_name(run.model_label)}.md"
|
||||
|
||||
def _record(self, run: RunState):
|
||||
path = self._report_path(run)
|
||||
return build_run_record(
|
||||
run,
|
||||
prompts_by_filename=run.prompts_by_filename,
|
||||
report_path=path if path.exists() else None,
|
||||
)
|
||||
|
||||
def _grade(self, run: RunState, outcome: TestOutcome) -> None:
|
||||
"""Run grader plugins over one answer and attach the results.
|
||||
|
||||
Graders run on this worker thread, so a slow grader delays the next
|
||||
question. That is deliberate: grading is part of the run, and the
|
||||
report should not be finalized before it lands.
|
||||
|
||||
Questions the provider failed are never graded — there is no answer to
|
||||
judge, and scoring them would let one careless plugin drag down the
|
||||
overall score. Plugins that want to react to failures use
|
||||
``on_test_complete``, which fires for every question.
|
||||
"""
|
||||
if outcome.status != "ok":
|
||||
return
|
||||
|
||||
record = build_test_record(
|
||||
outcome,
|
||||
total=run.total,
|
||||
prompt_text=run.prompts_by_filename.get(outcome.filename, ""),
|
||||
)
|
||||
graded = get_plugin_manager().grade(record)
|
||||
if not graded:
|
||||
return
|
||||
|
||||
run.grades[outcome.index] = [
|
||||
GradeEntry(grader=name, grade=grade) for name, grade in graded
|
||||
]
|
||||
outcome.grades = [
|
||||
{
|
||||
"grader": name,
|
||||
"score": grade.score,
|
||||
"label": grade.label,
|
||||
"notes": grade.notes,
|
||||
}
|
||||
for name, grade in graded
|
||||
]
|
||||
|
||||
def _apply_plugin_report(self, run: RunState) -> None:
|
||||
"""Write grades and plugin sections into the finished report."""
|
||||
report_path = self._report_path(run)
|
||||
if not report_path.exists():
|
||||
return
|
||||
|
||||
record = self._record(run)
|
||||
manager = get_plugin_manager()
|
||||
|
||||
grade_markdown = render_grade_section(record)
|
||||
if grade_markdown:
|
||||
try:
|
||||
replace_analysis_section(report_path, grade_markdown)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
try:
|
||||
sections = collect_report_sections(record, manager)
|
||||
except Exception: # noqa: BLE001 - a plugin fault must not fail the run
|
||||
sections = []
|
||||
|
||||
if sections:
|
||||
try:
|
||||
append_sections(report_path, sections)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _finalize_partial_report(self, run: RunState) -> None:
|
||||
"""Write the summary block for a run that stopped early."""
|
||||
report_path = RESULTS_DIR / f"automated_report_{sanitize_model_name(run.model_label)}.md"
|
||||
if not report_path.exists():
|
||||
return
|
||||
results: List[Dict[str, Any]] = []
|
||||
for outcome in run.outcomes:
|
||||
if outcome.status == "ok" and outcome.metrics:
|
||||
results.append({"response": outcome.response, "metrics": outcome.metrics})
|
||||
else:
|
||||
results.append({"error": outcome.error or "cancelled"})
|
||||
try:
|
||||
finalize_report_summary(report_path, results)
|
||||
except (OSError, KeyError):
|
||||
return
|
||||
run.report_name = report_path.name
|
||||
|
||||
# -------------------------------------------------------------- streaming
|
||||
|
||||
def _publish(
|
||||
self,
|
||||
loop: asyncio.AbstractEventLoop,
|
||||
run: RunState,
|
||||
event: Dict[str, Any],
|
||||
) -> None:
|
||||
"""Hand an event from the worker thread to the event loop."""
|
||||
try:
|
||||
loop.call_soon_threadsafe(self._emit, run, event)
|
||||
except RuntimeError:
|
||||
# Loop already closed (server shutting down); keep the history only.
|
||||
run.events.append(event)
|
||||
|
||||
@staticmethod
|
||||
def _emit(run: RunState, event: Dict[str, Any]) -> None:
|
||||
run.events.append(event)
|
||||
for queue in list(run.subscribers):
|
||||
queue.put_nowait(event)
|
||||
|
||||
async def stream(self, run_id: str) -> AsyncIterator[str]:
|
||||
run = self.get(run_id)
|
||||
if run is None:
|
||||
yield _format_sse({"type": "run.failed", "message": "Unknown run id."})
|
||||
return
|
||||
|
||||
queue: "asyncio.Queue[Dict[str, Any]]" = asyncio.Queue()
|
||||
# Subscribing and snapshotting in the same synchronous block guarantees
|
||||
# no event is delivered twice or dropped between the two.
|
||||
run.subscribers.append(queue)
|
||||
history = list(run.events)
|
||||
|
||||
try:
|
||||
for event in history:
|
||||
yield _format_sse(event)
|
||||
if event["type"] in TERMINAL_EVENTS:
|
||||
return
|
||||
|
||||
while True:
|
||||
try:
|
||||
event = await asyncio.wait_for(queue.get(), timeout=KEEPALIVE_SECONDS)
|
||||
except asyncio.TimeoutError:
|
||||
yield ": keepalive\n\n"
|
||||
continue
|
||||
|
||||
yield _format_sse(event)
|
||||
if event["type"] in TERMINAL_EVENTS:
|
||||
return
|
||||
finally:
|
||||
if queue in run.subscribers:
|
||||
run.subscribers.remove(queue)
|
||||
|
||||
|
||||
run_manager = RunManager()
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Pydantic request/response models for the LM-Gambit API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ProviderSummary(BaseModel):
|
||||
name: str
|
||||
is_default: bool = False
|
||||
|
||||
|
||||
class ModelSummary(BaseModel):
|
||||
id: str
|
||||
display_name: str
|
||||
|
||||
|
||||
class ModelListResponse(BaseModel):
|
||||
provider: str
|
||||
models: List[ModelSummary]
|
||||
|
||||
|
||||
class TestPrompt(BaseModel):
|
||||
"""A single question in the suite.
|
||||
|
||||
``title`` is derived from the first non-empty line of ``prompt`` by the
|
||||
engine, so it is read-only here and returned for display purposes only.
|
||||
"""
|
||||
|
||||
filename: str
|
||||
title: str
|
||||
prompt: str
|
||||
|
||||
|
||||
class TestSuiteResponse(BaseModel):
|
||||
tests: List[TestPrompt]
|
||||
directory: str
|
||||
|
||||
|
||||
class TestDraft(BaseModel):
|
||||
"""One question as submitted by the suite builder form."""
|
||||
|
||||
prompt: str = Field(min_length=1)
|
||||
|
||||
|
||||
class SaveSuiteRequest(BaseModel):
|
||||
tests: List[TestDraft]
|
||||
|
||||
|
||||
class RunMetrics(BaseModel):
|
||||
tokens_per_second: float = 0.0
|
||||
total_tokens: int = 0
|
||||
time_to_first_token: float = 0.0
|
||||
stop_reason: str = "N/A"
|
||||
|
||||
|
||||
class RunRequest(BaseModel):
|
||||
provider: str
|
||||
model_id: str
|
||||
temperature: float = Field(default=0.1, ge=0.0, le=2.0)
|
||||
filenames: Optional[List[str]] = Field(
|
||||
default=None,
|
||||
description="Subset of test filenames to run. Omit to run the whole suite.",
|
||||
)
|
||||
|
||||
|
||||
class RunSummary(BaseModel):
|
||||
average_tokens_per_second: float = 0.0
|
||||
average_time_to_first_token: float = 0.0
|
||||
total_tokens: int = 0
|
||||
passed: int = 0
|
||||
failed: int = 0
|
||||
overall_score: Optional[float] = None
|
||||
graded: int = 0
|
||||
|
||||
|
||||
class PluginSummary(BaseModel):
|
||||
name: str
|
||||
slug: str
|
||||
version: str
|
||||
description: str
|
||||
path: str
|
||||
enabled: bool
|
||||
hooks: List[str] = []
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class RunResponse(BaseModel):
|
||||
id: str
|
||||
status: str
|
||||
provider: str
|
||||
model_id: str
|
||||
model_label: str
|
||||
temperature: float
|
||||
total: int
|
||||
completed: int
|
||||
started_at: float
|
||||
finished_at: Optional[float] = None
|
||||
report_name: Optional[str] = None
|
||||
error: Optional[str] = None
|
||||
summary: RunSummary = RunSummary()
|
||||
|
||||
|
||||
class PlaygroundRequest(BaseModel):
|
||||
provider: str
|
||||
model_id: str
|
||||
prompt: str = Field(min_length=1)
|
||||
temperature: float = Field(default=0.1, ge=0.0, le=2.0)
|
||||
|
||||
|
||||
class PlaygroundResponse(BaseModel):
|
||||
response: Optional[str] = None
|
||||
error: Optional[str] = None
|
||||
metrics: Optional[RunMetrics] = None
|
||||
elapsed: float = 0.0
|
||||
|
||||
|
||||
class ReportSummary(BaseModel):
|
||||
name: str
|
||||
model_label: str
|
||||
size_bytes: int
|
||||
modified_at: float
|
||||
|
||||
|
||||
class ReportDetail(ReportSummary):
|
||||
content: str
|
||||
|
||||
|
||||
class ModelPathEntry(BaseModel):
|
||||
nickname: str = ""
|
||||
path: str
|
||||
|
||||
|
||||
class SettingsResponse(BaseModel):
|
||||
default_provider: str
|
||||
default_temperature: float
|
||||
local_model_paths: List[ModelPathEntry]
|
||||
tests_dir: str
|
||||
results_dir: str
|
||||
models_dir: str
|
||||
|
||||
|
||||
class SettingsUpdate(BaseModel):
|
||||
default_provider: Optional[str] = None
|
||||
default_temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0)
|
||||
local_model_paths: Optional[List[ModelPathEntry]] = None
|
||||
|
||||
|
||||
class SystemInfo(BaseModel):
|
||||
version: str
|
||||
engine_architecture: str
|
||||
engine_runtime: str
|
||||
template_ok: bool
|
||||
python_version: str
|
||||
metrics: Dict[str, str] = {}
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Reading and writing the prompt suite in ``tests/``.
|
||||
|
||||
The engine treats every ``tests/*.txt`` file as one prompt and derives its title
|
||||
from the first non-empty line, ordering files by a natural sort of their names.
|
||||
To keep the web editor and the ``auto-test.py`` CLI in perfect agreement, saving
|
||||
rewrites the directory as ``test1.txt`` … ``testN.txt`` in the order shown in
|
||||
the builder. That makes add / remove / reorder unambiguous for both entrypoints.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
from .core_bridge import TESTS_DIR, load_test_prompts
|
||||
|
||||
|
||||
class SuiteError(Exception):
|
||||
"""Raised when the suite on disk cannot be read or written."""
|
||||
|
||||
|
||||
def derive_title(prompt: str, fallback: str) -> str:
|
||||
for line in prompt.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped:
|
||||
return stripped
|
||||
return fallback
|
||||
|
||||
|
||||
def list_tests() -> List[Dict[str, str]]:
|
||||
"""Return the suite exactly as the engine sees it."""
|
||||
TESTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
return load_test_prompts()
|
||||
|
||||
|
||||
def find_tests(filenames: List[str]) -> List[Dict[str, str]]:
|
||||
"""Return the prompts matching ``filenames``, preserving suite order."""
|
||||
wanted = set(filenames)
|
||||
selected = [prompt for prompt in list_tests() if prompt["filename"] in wanted]
|
||||
missing = wanted - {prompt["filename"] for prompt in selected}
|
||||
if missing:
|
||||
raise SuiteError(f"Unknown test file(s): {', '.join(sorted(missing))}")
|
||||
return selected
|
||||
|
||||
|
||||
def save_suite(prompts: List[str]) -> List[Dict[str, str]]:
|
||||
"""Replace the suite with ``prompts`` and return the resulting tests."""
|
||||
cleaned = [text.strip() for text in prompts]
|
||||
for position, text in enumerate(cleaned, start=1):
|
||||
if not text:
|
||||
raise SuiteError(f"Question {position} is empty. Remove it or add a prompt.")
|
||||
|
||||
TESTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
target_names = {f"test{index}.txt" for index in range(1, len(cleaned) + 1)}
|
||||
written: List[Path] = []
|
||||
|
||||
for index, text in enumerate(cleaned, start=1):
|
||||
path = TESTS_DIR / f"test{index}.txt"
|
||||
try:
|
||||
path.write_text(text + "\n", encoding="utf-8")
|
||||
except OSError as exc:
|
||||
raise SuiteError(f"Could not write {path.name}: {exc}") from exc
|
||||
written.append(path)
|
||||
|
||||
for stale in TESTS_DIR.glob("*.txt"):
|
||||
if stale.name not in target_names:
|
||||
try:
|
||||
stale.unlink()
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
return list_tests()
|
||||
Reference in New Issue
Block a user