feat: add Settings and Suite pages with comprehensive settings management and question suite builder
- Implemented SettingsPage for configuring default provider, temperature, and model paths. - Added SuitePage for managing a suite of questions with features to add, edit, duplicate, and delete questions. - Introduced TypeScript configuration files for app and node environments. - Set up Vite configuration for development server with API proxying to backend.
This commit is contained in:
Christopher Clendening
@@ -0,0 +1,161 @@
|
||||
"""LM-Gambit plugin skeleton — copy this file to start a new plugin.
|
||||
|
||||
cp plugins/_skeleton.py plugins/my_plugin.py
|
||||
|
||||
Files starting with ``_`` are ignored by the loader, so this template never
|
||||
runs. Rename it (no leading underscore) and it loads on the next server start.
|
||||
|
||||
Every hook below is OPTIONAL. Delete the ones you don't need — a plugin that
|
||||
only defines ``grade`` is perfectly valid. Anything you leave undefined is
|
||||
simply never called.
|
||||
|
||||
If a hook raises, LM-Gambit logs it and carries on. Your plugin stays loaded
|
||||
and its other hooks keep firing, so a bug in one grader can't break a run.
|
||||
"""
|
||||
|
||||
from plugin_api import Grade, RunRecord, TestRecord
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Metadata — all optional. NAME is what shows up in Settings → Plugins.
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
NAME = "My Plugin"
|
||||
VERSION = "1.0.0"
|
||||
DESCRIPTION = "One line describing what this plugin does."
|
||||
ENABLED = True # flip to False to keep the file but stop loading it
|
||||
|
||||
|
||||
def register() -> None:
|
||||
"""One-time setup, called once when the plugin loads.
|
||||
|
||||
Open a database, read a config file, check that a CLI tool you depend on
|
||||
exists. Raising here disables the plugin and shows the error in Settings.
|
||||
"""
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Grading — score an answer
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def grade(test: TestRecord):
|
||||
"""Score one answer, or return None to abstain.
|
||||
|
||||
Abstaining is normal and costs nothing: a grader that only understands
|
||||
SwiftUI questions should return None for everything else, and those
|
||||
questions simply won't count toward the average.
|
||||
|
||||
This is never called for a question the provider failed on — there is no
|
||||
answer to judge. Use `on_test_complete` if you need to see failures too.
|
||||
|
||||
You may return:
|
||||
None abstain — this grader has no opinion
|
||||
0.0 – 1.0 a bare score
|
||||
True / False shorthand for 1.0 / 0.0
|
||||
Grade(...) a score plus a label and notes shown in the report
|
||||
|
||||
`test` gives you:
|
||||
test.index 1-based position in the run
|
||||
test.total how many questions the run covers
|
||||
test.title first line of the prompt
|
||||
test.filename e.g. "test3.txt"
|
||||
test.prompt the full prompt text sent to the model
|
||||
test.ok False if the provider errored
|
||||
test.response the model's answer (None when ok is False)
|
||||
test.error the provider error (None when ok is True)
|
||||
test.metrics raw provider metrics dict
|
||||
test.tokens_per_second convenience accessors over metrics
|
||||
test.total_tokens
|
||||
test.time_to_first_token
|
||||
"""
|
||||
# Only judge questions this plugin actually understands.
|
||||
if "swift" not in test.prompt.lower():
|
||||
return None
|
||||
|
||||
has_code_block = "```" in test.response
|
||||
return Grade(
|
||||
score=1.0 if has_code_block else 0.3,
|
||||
label="has code" if has_code_block else "prose only",
|
||||
notes="Checked that the answer contains a fenced code block.",
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Run lifecycle — react to a run starting, progressing and finishing
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def on_run_start(run: RunRecord) -> None:
|
||||
"""Fired once, before the first question is sent."""
|
||||
print(f"[my_plugin] starting {run.total} questions against {run.model_label}")
|
||||
|
||||
|
||||
def on_test_complete(test: TestRecord) -> None:
|
||||
"""Fired after each question, before the next one starts.
|
||||
|
||||
This runs on the run's worker thread, so slow work here delays the next
|
||||
question. Keep it quick, or hand it off to a thread of your own.
|
||||
"""
|
||||
|
||||
|
||||
def on_run_complete(run: RunRecord) -> None:
|
||||
"""Fired once when the run ends — including when cancelled or failed.
|
||||
|
||||
Check `run.status`: "completed", "cancelled" or "failed". Useful for
|
||||
notifications, CSV logs, or copying `run.report_path` somewhere else.
|
||||
|
||||
run.summary {"average_tokens_per_second": ..., "passed": ...}
|
||||
run.overall_score mean of graded questions, or None if nothing graded
|
||||
run.score_for(3) score for question 3, or None
|
||||
run.tests every TestRecord from the run
|
||||
run.report_path Path to the generated markdown report
|
||||
"""
|
||||
score = run.overall_score
|
||||
if score is not None:
|
||||
print(f"[my_plugin] {run.model_label} scored {score:.0%}")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Report — add your own markdown to the generated report
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def report_sections(run: RunRecord):
|
||||
"""Return extra markdown appended to the report, or None to add nothing.
|
||||
|
||||
Return either a markdown string, or a list of (heading, body) pairs.
|
||||
Headings should start at `##` to sit alongside the built-in sections.
|
||||
"""
|
||||
slowest = min(
|
||||
(t for t in run.tests if t.ok and t.tokens_per_second),
|
||||
key=lambda t: t.tokens_per_second,
|
||||
default=None,
|
||||
)
|
||||
if slowest is None:
|
||||
return None
|
||||
|
||||
return [
|
||||
(
|
||||
"## Slowest question",
|
||||
f"`{slowest.filename}` — {slowest.title}\n\n"
|
||||
f"* **Throughput:** {slowest.tokens_per_second} tok/s",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# HTTP — expose your own API endpoints
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def register_routes(router) -> None:
|
||||
"""Add FastAPI routes, mounted under /api/plugins/<filename-without-.py>.
|
||||
|
||||
For a plugin saved as `plugins/my_plugin.py`, the route below is served at
|
||||
/api/plugins/my_plugin/ping. Routes are registered when the server starts,
|
||||
so adding or changing them requires a restart.
|
||||
"""
|
||||
|
||||
@router.get("/ping")
|
||||
async def ping() -> dict:
|
||||
return {"plugin": NAME, "version": VERSION, "status": "ok"}
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Requirement coverage grader.
|
||||
|
||||
Scores an answer on whether it actually used the API symbols the question asked
|
||||
for. Prompts in this suite name their requirements in backticks —
|
||||
``func generateFizzBuzz(upTo max: Int) -> [String]``, ``actor``, ``@Observable``
|
||||
— which makes them checkable without any language-specific parsing.
|
||||
|
||||
Two wrinkles this handles:
|
||||
|
||||
* **Negated requirements.** "Do not use the old ``INIntent`` based system" names
|
||||
a symbol that must be *absent*. Treating every backticked span as required
|
||||
would penalise a correct answer, so the sentence around each span is checked
|
||||
for a negation first.
|
||||
* **Abstaining.** A question with no backticked requirements gets no score
|
||||
rather than a bad one. Abstentions are excluded from the average.
|
||||
|
||||
This is a starting point, not a final rubric — copy it and encode whatever
|
||||
"correct" means for your own suite.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from plugin_api import Grade, RunRecord, TestRecord
|
||||
|
||||
NAME = "Requirement coverage"
|
||||
VERSION = "1.0.0"
|
||||
DESCRIPTION = "Checks that answers use the API symbols each question asks for."
|
||||
ENABLED = True
|
||||
|
||||
# A backticked span is a requirement only if it looks like an identifier or a
|
||||
# signature — this filters out prose asides and single punctuation marks.
|
||||
_BACKTICK = re.compile(r"`([^`\n]{2,120})`")
|
||||
|
||||
# Markers that flip a requirement into a prohibition. Both an auxiliary-verb
|
||||
# negation ("should not use", "must never rely on", "doesn't need") and a set of
|
||||
# fixed phrases, since prompts word this many different ways.
|
||||
_NEGATION_RE = re.compile(
|
||||
r"\b(?:do|does|did|should|shall|must|will|would|can|could|may)\s*(?:n[o']t|not|never)\b"
|
||||
r"|\bn't\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_NEGATION_PHRASES = (
|
||||
"avoid",
|
||||
"instead of",
|
||||
"rather than",
|
||||
"without using",
|
||||
"no longer",
|
||||
"deprecated",
|
||||
"stop using",
|
||||
"get rid of",
|
||||
"remove the",
|
||||
"not the old",
|
||||
"old system",
|
||||
)
|
||||
|
||||
|
||||
def _is_negated(sentence: str) -> bool:
|
||||
lowered = sentence.lower()
|
||||
return bool(_NEGATION_RE.search(lowered)) or any(p in lowered for p in _NEGATION_PHRASES)
|
||||
|
||||
_CODE_VERBS = ("write", "implement", "create", "refactor", "provide", "build", "generate")
|
||||
|
||||
|
||||
def _sentences(text: str):
|
||||
return re.split(r"(?<=[.!?])\s+|\n{2,}", text)
|
||||
|
||||
|
||||
def _requirements(prompt: str):
|
||||
"""Split backticked spans into (required, forbidden) symbol lists."""
|
||||
required: list[str] = []
|
||||
forbidden: list[str] = []
|
||||
|
||||
for sentence in _sentences(prompt):
|
||||
negated = _is_negated(sentence)
|
||||
for raw in _BACKTICK.findall(sentence):
|
||||
symbol = raw.strip()
|
||||
# Skip spans that are only punctuation or whitespace.
|
||||
if not symbol or not any(ch.isalnum() or ch in "@\\/" for ch in symbol):
|
||||
continue
|
||||
bucket = forbidden if negated else required
|
||||
if symbol not in bucket:
|
||||
bucket.append(symbol)
|
||||
|
||||
# A symbol demanded somewhere and banned elsewhere is ambiguous — drop it.
|
||||
contested = set(required) & set(forbidden)
|
||||
return (
|
||||
[s for s in required if s not in contested],
|
||||
[s for s in forbidden if s not in contested],
|
||||
)
|
||||
|
||||
|
||||
def grade(test: TestRecord):
|
||||
if not test.response:
|
||||
return None
|
||||
|
||||
required, forbidden = _requirements(test.prompt)
|
||||
wants_code = any(verb in test.prompt.lower() for verb in _CODE_VERBS)
|
||||
|
||||
if not required and not forbidden and not wants_code:
|
||||
return None # nothing checkable — abstain
|
||||
|
||||
response = test.response
|
||||
checks: list[bool] = []
|
||||
problems: list[str] = []
|
||||
|
||||
for symbol in required:
|
||||
present = symbol in response
|
||||
checks.append(present)
|
||||
if not present:
|
||||
problems.append(f"missing `{symbol}`")
|
||||
|
||||
for symbol in forbidden:
|
||||
absent = symbol not in response
|
||||
checks.append(absent)
|
||||
if not absent:
|
||||
problems.append(f"used forbidden `{symbol}`")
|
||||
|
||||
if wants_code:
|
||||
has_block = "```" in response
|
||||
checks.append(has_block)
|
||||
if not has_block:
|
||||
problems.append("no fenced code block")
|
||||
|
||||
if not checks:
|
||||
return None
|
||||
|
||||
score = sum(checks) / len(checks)
|
||||
met = sum(checks)
|
||||
|
||||
return Grade(
|
||||
score=score,
|
||||
label=f"{met}/{len(checks)} checks",
|
||||
notes="; ".join(problems) if problems else "All stated requirements met.",
|
||||
)
|
||||
|
||||
|
||||
def report_sections(run: RunRecord):
|
||||
"""Call out the questions where requirements were missed."""
|
||||
misses = []
|
||||
for test in run.tests:
|
||||
for entry in run.grades.get(test.index, []):
|
||||
if entry.grader == NAME and entry.grade.score < 1.0:
|
||||
misses.append((test, entry.grade))
|
||||
|
||||
if not misses:
|
||||
return None
|
||||
|
||||
lines = [
|
||||
"These questions did not use every API symbol the prompt asked for. "
|
||||
"Verify by hand before treating them as failures — a model may solve the "
|
||||
"problem a different but valid way.",
|
||||
"",
|
||||
"| # | Question | Score | Problem |",
|
||||
"| --- | --- | --- | --- |",
|
||||
]
|
||||
for test, grade in misses:
|
||||
problem = grade.notes.replace("|", "\\|")
|
||||
title = test.title.replace("|", "\\|")
|
||||
lines.append(f"| {test.index} | {title} | {grade.score:.0%} | {problem} |")
|
||||
|
||||
return [("## Requirement coverage", "\n".join(lines))]
|
||||
Reference in New Issue
Block a user