Netherwarlord
8f246fabbc
- Implemented PluginBlocks component to render various plugin UI elements. - Created SuitePicker for selecting test suites and questions. - Introduced usePluginUI hook for managing plugin UI state and manifest. - Developed DocsPage for comprehensive plugin framework documentation. - Added PluginPage to render pages declared by plugins based on the current path.
288 lines
7.8 KiB
Python
288 lines
7.8 KiB
Python
"""Stable types passed to LM-Gambit plugins.
|
||
|
||
Plugins import from this module and nothing else in the project::
|
||
|
||
from plugin_api import Grade, RunRecord, TestRecord
|
||
|
||
Everything here is a plain frozen dataclass, so plugins never touch engine or
|
||
server internals and keep working across refactors of either.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from dataclasses import dataclass, field, is_dataclass, asdict
|
||
from pathlib import Path
|
||
from typing import Any, Dict, List, Optional, Sequence, Union
|
||
|
||
__all__ = [
|
||
# grading
|
||
"Grade",
|
||
"GradeEntry",
|
||
"RunRecord",
|
||
"TestRecord",
|
||
"PluginInfo",
|
||
# ui blocks
|
||
"Stat",
|
||
"StatRow",
|
||
"Table",
|
||
"Markdown",
|
||
"Action",
|
||
"Panel",
|
||
# ui surfaces
|
||
"NavItem",
|
||
"Page",
|
||
"SlotPanel",
|
||
"UI_SLOTS",
|
||
"ui_payload",
|
||
]
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class TestRecord:
|
||
"""One question and the model's answer to it."""
|
||
|
||
index: int
|
||
total: int
|
||
title: str
|
||
filename: str
|
||
prompt: str
|
||
ok: bool
|
||
response: Optional[str] = None
|
||
error: Optional[str] = None
|
||
metrics: Dict[str, Any] = field(default_factory=dict)
|
||
elapsed: float = 0.0
|
||
#: Slug of the suite this question came from. Empty only for records built
|
||
#: by older callers. ``filename`` is unique *within* a suite but not across
|
||
#: them — several suites ship a ``test1.txt`` — so identify a question by
|
||
#: ``question_id``, never by ``filename`` alone.
|
||
suite: str = ""
|
||
|
||
@property
|
||
def question_id(self) -> str:
|
||
"""Globally unique ``"<suite>/<file>"`` identifier."""
|
||
return f"{self.suite}/{self.filename}" if self.suite else self.filename
|
||
|
||
@property
|
||
def tokens_per_second(self) -> float:
|
||
return float(self.metrics.get("tokens_per_second", 0) or 0)
|
||
|
||
@property
|
||
def total_tokens(self) -> int:
|
||
return int(self.metrics.get("total_tokens", 0) or 0)
|
||
|
||
@property
|
||
def time_to_first_token(self) -> float:
|
||
return float(self.metrics.get("time_to_first_token", 0) or 0)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Grade:
|
||
"""A score for one answer.
|
||
|
||
``score`` is clamped to 0.0–1.0. Return ``None`` from a grader instead of a
|
||
``Grade`` to abstain — abstentions never count against the average.
|
||
"""
|
||
|
||
score: float
|
||
label: str = ""
|
||
notes: str = ""
|
||
|
||
def __post_init__(self) -> None:
|
||
object.__setattr__(self, "score", max(0.0, min(1.0, float(self.score))))
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class GradeEntry:
|
||
"""A grade together with the plugin that produced it."""
|
||
|
||
grader: str
|
||
grade: Grade
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class RunRecord:
|
||
"""A whole diagnostic run, handed to lifecycle and report hooks."""
|
||
|
||
id: str
|
||
provider: str
|
||
model_id: str
|
||
model_label: str
|
||
temperature: float
|
||
total: int
|
||
status: str
|
||
started_at: float
|
||
finished_at: Optional[float] = None
|
||
report_path: Optional[Path] = None
|
||
summary: Dict[str, Any] = field(default_factory=dict)
|
||
tests: List[TestRecord] = field(default_factory=list)
|
||
grades: Dict[int, List[GradeEntry]] = field(default_factory=dict)
|
||
|
||
def score_for(self, test_index: int) -> Optional[float]:
|
||
"""Mean score across every grader that scored this question."""
|
||
entries = self.grades.get(test_index) or []
|
||
if not entries:
|
||
return None
|
||
return sum(entry.grade.score for entry in entries) / len(entries)
|
||
|
||
@property
|
||
def overall_score(self) -> Optional[float]:
|
||
"""Mean of the per-question scores, over graded questions only."""
|
||
scores = [s for s in (self.score_for(t.index) for t in self.tests) if s is not None]
|
||
if not scores:
|
||
return None
|
||
return sum(scores) / len(scores)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class PluginInfo:
|
||
"""What the server reports about a discovered plugin."""
|
||
|
||
name: str
|
||
slug: str
|
||
version: str
|
||
description: str
|
||
path: str
|
||
enabled: bool
|
||
hooks: List[str] = field(default_factory=list)
|
||
error: Optional[str] = None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# UI contributions
|
||
# ---------------------------------------------------------------------------
|
||
# A plugin describes interface it wants to add; the React app renders it. The
|
||
# plugin never ships JavaScript and the frontend is never rebuilt to install
|
||
# one — the manifest is fetched at runtime from /api/plugins/ui.
|
||
#
|
||
# Every block may carry inline data *or* a ``source`` URL. With a source, the
|
||
# frontend fetches it and expects the same field names back as JSON, so a panel
|
||
# can be static, live, or refreshed by an Action without changing shape.
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Stat:
|
||
"""One number with a caption, for a headline row."""
|
||
|
||
label: str
|
||
value: Union[str, int, float]
|
||
hint: str = ""
|
||
#: "default" | "good" | "warn" | "bad" — colours the value only.
|
||
tone: str = "default"
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class StatRow:
|
||
"""A row of headline numbers."""
|
||
|
||
stats: Sequence[Stat] = field(default_factory=tuple)
|
||
source: Optional[str] = None
|
||
kind: str = field(default="stat_row", init=False)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Table:
|
||
"""A column-headed table. ``rows`` are lists of cell strings."""
|
||
|
||
columns: Sequence[str] = field(default_factory=tuple)
|
||
rows: Sequence[Sequence[Any]] = field(default_factory=tuple)
|
||
source: Optional[str] = None
|
||
empty: str = "Nothing to show yet."
|
||
kind: str = field(default="table", init=False)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Markdown:
|
||
"""Rendered markdown — the same renderer the reports use."""
|
||
|
||
text: str = ""
|
||
source: Optional[str] = None
|
||
kind: str = field(default="markdown", init=False)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Action:
|
||
"""A button that POSTs to one of the plugin's own routes.
|
||
|
||
The response may return ``{"message": ...}`` to show a toast, and/or
|
||
``{"refresh": true}`` to make sibling blocks re-fetch their sources.
|
||
"""
|
||
|
||
label: str
|
||
post: str
|
||
confirm: Optional[str] = None
|
||
#: "primary" | "ghost" | "danger"
|
||
style: str = "primary"
|
||
icon: Optional[str] = None
|
||
kind: str = field(default="action", init=False)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Panel:
|
||
"""A titled card grouping other blocks."""
|
||
|
||
title: str
|
||
blocks: Sequence[Any] = field(default_factory=tuple)
|
||
subtitle: str = ""
|
||
kind: str = field(default="panel", init=False)
|
||
|
||
|
||
Block = Union[StatRow, Table, Markdown, Action, Panel]
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class NavItem:
|
||
"""An entry in the left rail."""
|
||
|
||
label: str
|
||
path: str
|
||
#: Any lucide icon name the host allowlists; falls back to a puzzle piece.
|
||
icon: str = "puzzle"
|
||
hint: str = ""
|
||
order: int = 100
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Page:
|
||
"""A full page at ``path``, reachable from a NavItem or a direct link."""
|
||
|
||
path: str
|
||
title: str
|
||
blocks: Sequence[Any] = field(default_factory=tuple)
|
||
subtitle: str = ""
|
||
|
||
|
||
#: Injection points on the built-in pages.
|
||
UI_SLOTS = (
|
||
"run.aside",
|
||
"reports.aside",
|
||
"suite.aside",
|
||
"settings.section",
|
||
)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class SlotPanel:
|
||
"""A Panel injected into a built-in page. ``slot`` must be in UI_SLOTS."""
|
||
|
||
slot: str
|
||
panel: Panel
|
||
order: int = 100
|
||
|
||
|
||
def ui_payload(value: Any) -> Any:
|
||
"""Convert contribution dataclasses into JSON-safe structures.
|
||
|
||
Recurses through sequences and nested blocks. Anything that is not a
|
||
dataclass is passed through untouched, so plugins may hand back plain
|
||
dicts and lists where that is simpler.
|
||
"""
|
||
if is_dataclass(value) and not isinstance(value, type):
|
||
return {key: ui_payload(item) for key, item in asdict(value).items()}
|
||
if isinstance(value, dict):
|
||
return {key: ui_payload(item) for key, item in value.items()}
|
||
if isinstance(value, (list, tuple)):
|
||
return [ui_payload(item) for item in value]
|
||
if isinstance(value, Path):
|
||
return str(value)
|
||
return value
|