feat: add Settings and Suite pages with comprehensive settings management and question suite builder
- Implemented SettingsPage for configuring default provider, temperature, and model paths. - Added SuitePage for managing a suite of questions with features to add, edit, duplicate, and delete questions. - Introduced TypeScript configuration files for app and node environments. - Set up Vite configuration for development server with API proxying to backend.
This commit is contained in:
+117
@@ -0,0 +1,117 @@
|
||||
"""Stable types passed to LM-Gambit plugins.
|
||||
|
||||
Plugins import from this module and nothing else in the project::
|
||||
|
||||
from plugin_api import Grade, RunRecord, TestRecord
|
||||
|
||||
Everything here is a plain frozen dataclass, so plugins never touch engine or
|
||||
server internals and keep working across refactors of either.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
__all__ = ["Grade", "GradeEntry", "RunRecord", "TestRecord", "PluginInfo"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TestRecord:
|
||||
"""One question and the model's answer to it."""
|
||||
|
||||
index: int
|
||||
total: int
|
||||
title: str
|
||||
filename: str
|
||||
prompt: str
|
||||
ok: bool
|
||||
response: Optional[str] = None
|
||||
error: Optional[str] = None
|
||||
metrics: Dict[str, Any] = field(default_factory=dict)
|
||||
elapsed: float = 0.0
|
||||
|
||||
@property
|
||||
def tokens_per_second(self) -> float:
|
||||
return float(self.metrics.get("tokens_per_second", 0) or 0)
|
||||
|
||||
@property
|
||||
def total_tokens(self) -> int:
|
||||
return int(self.metrics.get("total_tokens", 0) or 0)
|
||||
|
||||
@property
|
||||
def time_to_first_token(self) -> float:
|
||||
return float(self.metrics.get("time_to_first_token", 0) or 0)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Grade:
|
||||
"""A score for one answer.
|
||||
|
||||
``score`` is clamped to 0.0–1.0. Return ``None`` from a grader instead of a
|
||||
``Grade`` to abstain — abstentions never count against the average.
|
||||
"""
|
||||
|
||||
score: float
|
||||
label: str = ""
|
||||
notes: str = ""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "score", max(0.0, min(1.0, float(self.score))))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GradeEntry:
|
||||
"""A grade together with the plugin that produced it."""
|
||||
|
||||
grader: str
|
||||
grade: Grade
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunRecord:
|
||||
"""A whole diagnostic run, handed to lifecycle and report hooks."""
|
||||
|
||||
id: str
|
||||
provider: str
|
||||
model_id: str
|
||||
model_label: str
|
||||
temperature: float
|
||||
total: int
|
||||
status: str
|
||||
started_at: float
|
||||
finished_at: Optional[float] = None
|
||||
report_path: Optional[Path] = None
|
||||
summary: Dict[str, Any] = field(default_factory=dict)
|
||||
tests: List[TestRecord] = field(default_factory=list)
|
||||
grades: Dict[int, List[GradeEntry]] = field(default_factory=dict)
|
||||
|
||||
def score_for(self, test_index: int) -> Optional[float]:
|
||||
"""Mean score across every grader that scored this question."""
|
||||
entries = self.grades.get(test_index) or []
|
||||
if not entries:
|
||||
return None
|
||||
return sum(entry.grade.score for entry in entries) / len(entries)
|
||||
|
||||
@property
|
||||
def overall_score(self) -> Optional[float]:
|
||||
"""Mean of the per-question scores, over graded questions only."""
|
||||
scores = [s for s in (self.score_for(t.index) for t in self.tests) if s is not None]
|
||||
if not scores:
|
||||
return None
|
||||
return sum(scores) / len(scores)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PluginInfo:
|
||||
"""What the server reports about a discovered plugin."""
|
||||
|
||||
name: str
|
||||
slug: str
|
||||
version: str
|
||||
description: str
|
||||
path: str
|
||||
enabled: bool
|
||||
hooks: List[str] = field(default_factory=list)
|
||||
error: Optional[str] = None
|
||||
Reference in New Issue
Block a user