christopher/LM-Gambit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add plugin framework with UI contributions and suite selection

Browse Source 
- Implemented PluginBlocks component to render various plugin UI elements.
- Created SuitePicker for selecting test suites and questions.
- Introduced usePluginUI hook for managing plugin UI state and manifest.
- Developed DocsPage for comprehensive plugin framework documentation.
- Added PluginPage to render pages declared by plugins based on the current path.
This commit is contained in:
Netherwarlord
2026-07-28 03:00:55 -04:00
parent adf61ae1a0
commit 8f246fabbc
73 changed files with 4972 additions and 640 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/api.py
+129 -15
View File
@@ -46,11 +46,27 @@ from .schemas import (
SaveSuiteRequest,
SettingsResponse,
SettingsUpdate,
SuiteCreateRequest,
SuiteDetail,
SuiteDuplicateRequest,
SuiteSummary,
SuiteUpdateRequest,
SystemInfo,
TestPrompt,
TestSuiteResponse,
)
from .suite import SuiteError, list_tests, find_tests, save_suite
from .suite import (
ReadOnlySuiteError,
SuiteError,
create_suite,
delete_suite,
duplicate_suite,
get_suite,
list_suites,
load_suite_prompts,
resolve_selections,
save_suite_tests,
update_suite,
)
router = APIRouter(prefix="/api")
@@ -93,6 +109,16 @@ async def get_plugins() -> List[PluginSummary]:
return [PluginSummary(**vars(info)) for info in get_plugin_manager().info()]
@router.get("/plugins/ui")
async def get_plugin_ui() -> dict:
"""The interface plugins declare: nav entries, pages and slot panels.
The frontend renders straight from this, which is why installing a plugin
never requires rebuilding the UI.
"""
return get_plugin_manager().ui_manifest()
@router.post("/plugins/reload", response_model=List[PluginSummary])
async def reload_plugins_endpoint() -> List[PluginSummary]:
"""Re-scan plugins/. Newly added HTTP routes still need a server restart."""
@@ -144,27 +170,115 @@ async def get_models(provider_name: str) -> ModelListResponse:
# ---------------------------------------------------------------- test suite
@router.get("/tests", response_model=TestSuiteResponse)
async def get_tests() -> TestSuiteResponse:
tests = [TestPrompt(**prompt) for prompt in list_tests()]
return TestSuiteResponse(tests=tests, directory=str(TESTS_DIR))
def _as_prompt(entry: Dict[str, str]) -> TestPrompt:
return TestPrompt(
filename=entry["filename"],
title=entry["title"],
prompt=entry["prompt"],
suite=entry.get("suite", ""),
id=entry.get("id", ""),
)
@router.put("/tests", response_model=TestSuiteResponse)
async def put_tests(payload: SaveSuiteRequest) -> TestSuiteResponse:
def _summary(suite) -> SuiteSummary:
return SuiteSummary(
slug=suite.slug,
name=suite.name,
description=suite.description,
order=suite.order,
builtin=suite.builtin,
count=suite.count,
)
def _guard_active_run(action: str) -> None:
if run_manager.active() is not None:
raise HTTPException(
status.HTTP_409_CONFLICT,
detail="A run is in progress. Wait for it to finish before editing the suite.",
detail=f"A run is in progress. Wait for it to finish before {action}.",
)
@router.get("/suites", response_model=List[SuiteSummary])
async def get_suites() -> List[SuiteSummary]:
"""Every suite, built-ins first."""
return [_summary(suite) for suite in list_suites()]
@router.get("/suites/{slug}", response_model=SuiteDetail)
async def get_suite_detail(slug: str) -> SuiteDetail:
try:
tests = save_suite([draft.prompt for draft in payload.tests])
suite = get_suite(slug)
tests = load_suite_prompts(slug)
except SuiteError as exc:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc
return SuiteDetail(**_summary(suite).model_dump(), tests=[_as_prompt(t) for t in tests])
@router.post("/suites", response_model=SuiteDetail, status_code=status.HTTP_201_CREATED)
async def post_suite(payload: SuiteCreateRequest) -> SuiteDetail:
try:
suite = create_suite(payload.name, payload.description, payload.slug)
except SuiteError as exc:
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
return SuiteDetail(**_summary(suite).model_dump(), tests=[])
return TestSuiteResponse(
tests=[TestPrompt(**prompt) for prompt in tests],
directory=str(TESTS_DIR),
@router.put("/suites/{slug}", response_model=SuiteDetail)
async def put_suite(slug: str, payload: SuiteUpdateRequest) -> SuiteDetail:
try:
suite = update_suite(slug, name=payload.name, description=payload.description)
except ReadOnlySuiteError as exc:
raise HTTPException(status.HTTP_409_CONFLICT, detail=str(exc)) from exc
except SuiteError as exc:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc
return SuiteDetail(
**_summary(suite).model_dump(),
tests=[_as_prompt(t) for t in load_suite_prompts(slug)],
)
@router.delete("/suites/{slug}", status_code=status.HTTP_204_NO_CONTENT)
async def remove_suite(slug: str) -> None:
_guard_active_run("deleting a suite")
try:
delete_suite(slug)
except ReadOnlySuiteError as exc:
raise HTTPException(status.HTTP_409_CONFLICT, detail=str(exc)) from exc
except SuiteError as exc:
raise HTTPException(status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc
@router.put("/suites/{slug}/tests", response_model=SuiteDetail)
async def put_suite_tests(slug: str, payload: SaveSuiteRequest) -> SuiteDetail:
_guard_active_run("editing questions")
try:
tests = save_suite_tests(slug, [draft.prompt for draft in payload.tests])
suite = get_suite(slug)
except ReadOnlySuiteError as exc:
raise HTTPException(status.HTTP_409_CONFLICT, detail=str(exc)) from exc
except SuiteError as exc:
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
return SuiteDetail(**_summary(suite).model_dump(), tests=[_as_prompt(t) for t in tests])
@router.post(
"/suites/{slug}/duplicate",
response_model=SuiteDetail,
status_code=status.HTTP_201_CREATED,
)
async def duplicate(slug: str, payload: SuiteDuplicateRequest) -> SuiteDetail:
"""Clone any suite, built-in included, into an editable custom one.
This is what stops read-only from being a dead end.
"""
try:
suite = duplicate_suite(slug, payload.name)
except SuiteError as exc:
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
return SuiteDetail(
**_summary(suite).model_dump(),
tests=[_as_prompt(t) for t in load_suite_prompts(suite.slug)],
)
@@ -180,14 +294,14 @@ async def create_run(payload: RunRequest) -> RunResponse:
)
try:
prompts = find_tests(payload.filenames) if payload.filenames else list_tests()
prompts = resolve_selections(payload.selections, payload.filenames)
except SuiteError as exc:
raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
if not prompts:
raise HTTPException(
status.HTTP_400_BAD_REQUEST,
detail="No questions in the suite. Add at least one in the Suite Builder.",
detail="No questions selected. Pick at least one suite in Testing Suites.",
)
def _resolve_label() -> str:
server/core_bridge.py
+30
View File
@@ -55,6 +55,22 @@ from settings import ( # type: ignore[import-not-found]
save_settings,
set_local_model_paths,
)
from suites import ( # type: ignore[import-not-found]
ReadOnlySuiteError,
Suite,
SuiteError,
create_suite,
delete_suite,
duplicate_suite,
find_prompts,
get_suite,
list_suites,
load_prompts,
load_suite_prompts,
save_suite_tests,
split_question_id,
update_suite,
)
__all__ = [
"CORE_DIR",
@@ -83,6 +99,20 @@ __all__ = [
"list_provider_names",
"load_settings",
"load_test_prompts",
"ReadOnlySuiteError",
"Suite",
"SuiteError",
"create_suite",
"delete_suite",
"duplicate_suite",
"find_prompts",
"get_suite",
"list_suites",
"load_prompts",
"load_suite_prompts",
"save_suite_tests",
"split_question_id",
"update_suite",
"run_suite",
"sanitize_model_name",
"save_settings",
server/plugins.py
+6 -2
View File
@@ -53,6 +53,7 @@ def build_test_record(
total=total,
title=outcome.title,
filename=outcome.filename,
suite=outcome.suite,
prompt=prompt_text,
ok=outcome.status == "ok",
response=outcome.response,
@@ -65,14 +66,17 @@ def build_test_record(
def build_run_record(
run: "RunState",
*,
prompts_by_filename: Dict[str, str],
prompts_by_id: Dict[str, str],
report_path: Optional[Path] = None,
) -> RunRecord:
# Keyed by qualified "<suite>/<file>" ID. Looking a prompt up by bare
# filename would hand a grader the wrong question's text whenever two
# suites in one run share a filename, which is the common case.
tests = [
build_test_record(
outcome,
total=run.total,
prompt_text=prompts_by_filename.get(outcome.filename, ""),
prompt_text=prompts_by_id.get(outcome.question_id, ""),
)
for outcome in run.outcomes
]
server/run_manager.py
+33 -7
View File
@@ -65,6 +65,12 @@ class TestOutcome:
error: Optional[str] = None
metrics: Optional[Dict[str, Any]] = None
grades: List[Dict[str, Any]] = field(default_factory=list)
suite: str = ""
@property
def question_id(self) -> str:
"""Unique across suites. ``filename`` alone is not — see RunState."""
return f"{self.suite}/{self.filename}" if self.suite else self.filename
@property
def score(self) -> Optional[float]:
@@ -79,6 +85,8 @@ class TestOutcome:
"total": total,
"title": self.title,
"filename": self.filename,
"suite": self.suite,
"question_id": self.question_id,
"status": self.status,
"elapsed": round(self.elapsed, 2),
"response": self.response,
@@ -107,7 +115,13 @@ class RunState:
subscribers: List["asyncio.Queue[Dict[str, Any]]"] = field(default_factory=list)
cancel_event: threading.Event = field(default_factory=threading.Event)
grades: Dict[int, List[GradeEntry]] = field(default_factory=dict)
prompts_by_filename: Dict[str, str] = field(default_factory=dict)
#: Prompt text keyed by qualified ``"<suite>/<file>"`` ID.
#:
#: This was once keyed by bare filename. With several suites in one run
#: that silently returns the wrong question's text: graders derive their
#: entire rubric from the prompt, so a collision yields confident,
#: plausible, wrong scores with no error anywhere. Never key by filename.
prompts_by_id: Dict[str, str] = field(default_factory=dict)
@property
def completed(self) -> int:
@@ -156,6 +170,16 @@ class RunState:
}
def _prompt_key(prompt: Dict[str, str]) -> str:
"""Qualified ID for a prompt dict as the loader produces it."""
qualified = prompt.get("id")
if qualified:
return str(qualified)
suite = prompt.get("suite", "")
filename = prompt.get("filename", "")
return f"{suite}/{filename}" if suite else filename
def _format_sse(event: Dict[str, Any]) -> str:
return f"event: {event['type']}\ndata: {json.dumps(event)}\n\n"
@@ -217,8 +241,8 @@ class RunManager:
model_label=model_label,
temperature=temperature,
total=len(prompts),
prompts_by_filename={
p.get("filename", ""): p.get("prompt", "") for p in prompts
prompts_by_id={
_prompt_key(p): p.get("prompt", "") for p in prompts
},
)
self._runs[run.id] = run
@@ -282,6 +306,7 @@ class RunManager:
index=index,
title=prompt.get("title", f"Test {index}"),
filename=prompt.get("filename", f"test{index}"),
suite=prompt.get("suite", ""),
status="error",
elapsed=elapsed,
error=str(result["error"]),
@@ -292,6 +317,7 @@ class RunManager:
index=index,
title=prompt.get("title", f"Test {index}"),
filename=prompt.get("filename", f"test{index}"),
suite=prompt.get("suite", ""),
status="ok",
elapsed=elapsed,
response=str(result.get("response", "")),
@@ -306,7 +332,7 @@ class RunManager:
record = build_test_record(
outcome,
total=run.total,
prompt_text=run.prompts_by_filename.get(outcome.filename, ""),
prompt_text=run.prompts_by_id.get(outcome.question_id, ""),
)
get_plugin_manager().emit("on_test_complete", record)
@@ -317,7 +343,7 @@ class RunManager:
get_plugin_manager().emit(
"on_run_start",
build_run_record(run, prompts_by_filename=run.prompts_by_filename),
build_run_record(run, prompts_by_id=run.prompts_by_id),
)
try:
@@ -381,7 +407,7 @@ class RunManager:
path = self._report_path(run)
return build_run_record(
run,
prompts_by_filename=run.prompts_by_filename,
prompts_by_id=run.prompts_by_id,
report_path=path if path.exists() else None,
)
@@ -403,7 +429,7 @@ class RunManager:
record = build_test_record(
outcome,
total=run.total,
prompt_text=run.prompts_by_filename.get(outcome.filename, ""),
prompt_text=run.prompts_by_id.get(outcome.question_id, ""),
)
graded = get_plugin_manager().grade(record)
if not graded:
server/schemas.py
+53 -6
View File
@@ -32,11 +32,10 @@ class TestPrompt(BaseModel):
filename: str
title: str
prompt: str
class TestSuiteResponse(BaseModel):
tests: List[TestPrompt]
directory: str
#: Owning suite slug, and the globally unique "<suite>/<file>" ID.
#: ``filename`` repeats across suites; ``id`` never does.
suite: str = ""
id: str = ""
class TestDraft(BaseModel):
@@ -49,6 +48,44 @@ class SaveSuiteRequest(BaseModel):
tests: List[TestDraft]
# ------------------------------------------------------------------- suites
class SuiteSummary(BaseModel):
slug: str
name: str
description: str = ""
order: int = 100
builtin: bool = False
count: int = 0
class SuiteDetail(SuiteSummary):
tests: List[TestPrompt] = Field(default_factory=list)
class SuiteCreateRequest(BaseModel):
name: str = Field(min_length=1, max_length=80)
description: str = ""
slug: Optional[str] = None
class SuiteUpdateRequest(BaseModel):
name: Optional[str] = Field(default=None, max_length=80)
description: Optional[str] = None
class SuiteDuplicateRequest(BaseModel):
name: Optional[str] = Field(default=None, max_length=80)
class RunSelection(BaseModel):
"""Questions to draw from one suite. Empty ``filenames`` means all of it."""
suite: str
filenames: Optional[List[str]] = None
class RunMetrics(BaseModel):
tokens_per_second: float = 0.0
total_tokens: int = 0
@@ -60,9 +97,19 @@ class RunRequest(BaseModel):
provider: str
model_id: str
temperature: float = Field(default=0.1, ge=0.0, le=2.0)
selections: Optional[List[RunSelection]] = Field(
default=None,
description=(
"Suites to run, in order, each optionally narrowed to a subset of "
"its questions. Omit to run every built-in suite."
),
)
filenames: Optional[List[str]] = Field(
default=None,
description="Subset of test filenames to run. Omit to run the whole suite.",
description=(
"Deprecated. Qualified '<suite>/<file>' IDs. Bare filenames are "
"rejected because they are ambiguous across suites. Prefer 'selections'."
),
)
server/suite.py
+78 -62
View File
@@ -1,73 +1,89 @@
"""Reading and writing the prompt suite in ``tests/``.
"""Server-side access to named test suites.
The engine treats every ``tests/*.txt`` file as one prompt and derives its title
from the first non-empty line, ordering files by a natural sort of their names.
To keep the web editor and the ``auto-test.py`` CLI in perfect agreement, saving
rewrites the directory as ``test1.txt`` … ``testN.txt`` in the order shown in
the builder. That makes add / remove / reorder unambiguous for both entrypoints.
The engine keeps suites in two tiers — read-only built-ins under
``.core/suites/`` and user-owned custom ones under ``tests/<slug>/`` — and this
module is the thin server layer over :mod:`suites` in the core.
Everything here speaks **qualified question IDs** (``"<suite>/<file>"``). Bare
filenames are ambiguous the moment a run draws from more than one suite, and
resolving one to the wrong question would hand a grader the wrong prompt and
produce a confident, wrong score in silence.
"""
from __future__ import annotations
from pathlib import Path
from typing import Dict, List
from typing import Dict, List, Optional, Sequence
from .core_bridge import TESTS_DIR, load_test_prompts
from .core_bridge import ( # noqa: F401 (several are re-exported for the API)
TESTS_DIR,
ReadOnlySuiteError,
Suite,
SuiteError,
create_suite,
delete_suite,
duplicate_suite,
find_prompts,
get_suite,
list_suites,
load_prompts,
load_suite_prompts,
save_suite_tests,
split_question_id,
update_suite,
)
__all__ = [
"ReadOnlySuiteError",
"Suite",
"SuiteError",
"create_suite",
"delete_suite",
"duplicate_suite",
"get_suite",
"list_suites",
"load_suite_prompts",
"resolve_selections",
"save_suite_tests",
"update_suite",
]
class SuiteError(Exception):
"""Raised when the suite on disk cannot be read or written."""
def resolve_selections(
selections: Optional[Sequence[object]] = None,
question_ids: Optional[Sequence[str]] = None,
) -> List[Dict[str, str]]:
"""Turn a run request into an ordered list of prompts.
``selections`` is a list of objects with ``.suite`` and optional
``.filenames``; a suite with no filenames contributes all of its questions.
``question_ids`` is the older flat form and must be fully qualified.
Passing neither means every built-in suite.
"""
if selections:
prompts: List[Dict[str, str]] = []
seen = set()
for selection in selections:
slug = getattr(selection, "suite", None) or ""
get_suite(slug) # raises SuiteError for an unknown slug
filenames = getattr(selection, "filenames", None)
chosen = (
load_suite_prompts(slug)
if not filenames
else find_prompts([f"{slug}/{name}" for name in filenames])
)
for entry in chosen:
if entry["id"] not in seen:
seen.add(entry["id"])
prompts.append(entry)
return prompts
def derive_title(prompt: str, fallback: str) -> str:
for line in prompt.splitlines():
stripped = line.strip()
if stripped:
return stripped
return fallback
if question_ids:
bare = [qid for qid in question_ids if "/" not in qid]
if bare:
raise SuiteError(
"Bare filenames are ambiguous across suites: "
f"{', '.join(sorted(bare))}. Use '<suite>/<file>' IDs, or 'selections'."
)
return find_prompts(list(question_ids))
def list_tests() -> List[Dict[str, str]]:
"""Return the suite exactly as the engine sees it."""
TESTS_DIR.mkdir(parents=True, exist_ok=True)
return load_test_prompts()
def find_tests(filenames: List[str]) -> List[Dict[str, str]]:
"""Return the prompts matching ``filenames``, preserving suite order."""
wanted = set(filenames)
selected = [prompt for prompt in list_tests() if prompt["filename"] in wanted]
missing = wanted - {prompt["filename"] for prompt in selected}
if missing:
raise SuiteError(f"Unknown test file(s): {', '.join(sorted(missing))}")
return selected
def save_suite(prompts: List[str]) -> List[Dict[str, str]]:
"""Replace the suite with ``prompts`` and return the resulting tests."""
cleaned = [text.strip() for text in prompts]
for position, text in enumerate(cleaned, start=1):
if not text:
raise SuiteError(f"Question {position} is empty. Remove it or add a prompt.")
TESTS_DIR.mkdir(parents=True, exist_ok=True)
target_names = {f"test{index}.txt" for index in range(1, len(cleaned) + 1)}
written: List[Path] = []
for index, text in enumerate(cleaned, start=1):
path = TESTS_DIR / f"test{index}.txt"
try:
path.write_text(text + "\n", encoding="utf-8")
except OSError as exc:
raise SuiteError(f"Could not write {path.name}: {exc}") from exc
written.append(path)
for stale in TESTS_DIR.glob("*.txt"):
if stale.name not in target_names:
try:
stale.unlink()
except OSError:
continue
return list_tests()
return load_prompts(None)