feat: add Settings and Suite pages with comprehensive settings management and question suite builder

- Implemented SettingsPage for configuring default provider, temperature, and model paths.
- Added SuitePage for managing a suite of questions with features to add, edit, duplicate, and delete questions.
- Introduced TypeScript configuration files for app and node environments.
- Set up Vite configuration for development server with API proxying to backend.
This commit is contained in:
Christopher Clendening
2026-07-27 17:41:08 -04:00
parent ce0e022f11
commit adf61ae1a0
50 changed files with 10952 additions and 164 deletions
+117
View File
@@ -0,0 +1,117 @@
"""Stable types passed to LM-Gambit plugins.
Plugins import from this module and nothing else in the project::
from plugin_api import Grade, RunRecord, TestRecord
Everything here is a plain frozen dataclass, so plugins never touch engine or
server internals and keep working across refactors of either.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, Optional
__all__ = ["Grade", "GradeEntry", "RunRecord", "TestRecord", "PluginInfo"]
@dataclass(frozen=True)
class TestRecord:
"""One question and the model's answer to it."""
index: int
total: int
title: str
filename: str
prompt: str
ok: bool
response: Optional[str] = None
error: Optional[str] = None
metrics: Dict[str, Any] = field(default_factory=dict)
elapsed: float = 0.0
@property
def tokens_per_second(self) -> float:
return float(self.metrics.get("tokens_per_second", 0) or 0)
@property
def total_tokens(self) -> int:
return int(self.metrics.get("total_tokens", 0) or 0)
@property
def time_to_first_token(self) -> float:
return float(self.metrics.get("time_to_first_token", 0) or 0)
@dataclass(frozen=True)
class Grade:
"""A score for one answer.
``score`` is clamped to 0.01.0. Return ``None`` from a grader instead of a
``Grade`` to abstain — abstentions never count against the average.
"""
score: float
label: str = ""
notes: str = ""
def __post_init__(self) -> None:
object.__setattr__(self, "score", max(0.0, min(1.0, float(self.score))))
@dataclass(frozen=True)
class GradeEntry:
"""A grade together with the plugin that produced it."""
grader: str
grade: Grade
@dataclass(frozen=True)
class RunRecord:
"""A whole diagnostic run, handed to lifecycle and report hooks."""
id: str
provider: str
model_id: str
model_label: str
temperature: float
total: int
status: str
started_at: float
finished_at: Optional[float] = None
report_path: Optional[Path] = None
summary: Dict[str, Any] = field(default_factory=dict)
tests: List[TestRecord] = field(default_factory=list)
grades: Dict[int, List[GradeEntry]] = field(default_factory=dict)
def score_for(self, test_index: int) -> Optional[float]:
"""Mean score across every grader that scored this question."""
entries = self.grades.get(test_index) or []
if not entries:
return None
return sum(entry.grade.score for entry in entries) / len(entries)
@property
def overall_score(self) -> Optional[float]:
"""Mean of the per-question scores, over graded questions only."""
scores = [s for s in (self.score_for(t.index) for t in self.tests) if s is not None]
if not scores:
return None
return sum(scores) / len(scores)
@dataclass(frozen=True)
class PluginInfo:
"""What the server reports about a discovered plugin."""
name: str
slug: str
version: str
description: str
path: str
enabled: bool
hooks: List[str] = field(default_factory=list)
error: Optional[str] = None