feat: add plugin framework with UI contributions and suite selection
- Implemented PluginBlocks component to render various plugin UI elements. - Created SuitePicker for selecting test suites and questions. - Introduced usePluginUI hook for managing plugin UI state and manifest. - Developed DocsPage for comprehensive plugin framework documentation. - Added PluginPage to render pages declared by plugins based on the current path.
This commit is contained in:
Netherwarlord
+113
-2
@@ -13,7 +13,20 @@ If a hook raises, LM-Gambit logs it and carries on. Your plugin stays loaded
|
||||
and its other hooks keep firing, so a bug in one grader can't break a run.
|
||||
"""
|
||||
|
||||
from plugin_api import Grade, RunRecord, TestRecord
|
||||
from plugin_api import (
|
||||
Action,
|
||||
Grade,
|
||||
Markdown,
|
||||
NavItem,
|
||||
Page,
|
||||
Panel,
|
||||
RunRecord,
|
||||
SlotPanel,
|
||||
Stat,
|
||||
StatRow,
|
||||
Table,
|
||||
TestRecord,
|
||||
)
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Metadata — all optional. NAME is what shows up in Settings → Plugins.
|
||||
@@ -58,7 +71,9 @@ def grade(test: TestRecord):
|
||||
test.index 1-based position in the run
|
||||
test.total how many questions the run covers
|
||||
test.title first line of the prompt
|
||||
test.filename e.g. "test3.txt"
|
||||
test.filename e.g. "test3.txt" — NOT unique across suites
|
||||
test.suite owning suite slug, e.g. "math-code"
|
||||
test.question_id "<suite>/<file>" — the unique identifier
|
||||
test.prompt the full prompt text sent to the model
|
||||
test.ok False if the provider errored
|
||||
test.response the model's answer (None when ok is False)
|
||||
@@ -154,8 +169,104 @@ def register_routes(router) -> None:
|
||||
For a plugin saved as `plugins/my_plugin.py`, the route below is served at
|
||||
/api/plugins/my_plugin/ping. Routes are registered when the server starts,
|
||||
so adding or changing them requires a restart.
|
||||
|
||||
Careful: reloading does not rebind routes. These handlers close over the
|
||||
module object that existed at startup, so after a plugin reload the new
|
||||
module handles grade() while these routes still read the old module's
|
||||
globals — serving stale state with no error to warn you. Restart the
|
||||
server after editing a plugin that defines this hook.
|
||||
"""
|
||||
|
||||
@router.get("/ping")
|
||||
async def ping() -> dict:
|
||||
return {"plugin": NAME, "version": VERSION, "status": "ok"}
|
||||
|
||||
@router.get("/stats")
|
||||
async def stats() -> dict:
|
||||
"""Shape a StatRow(source=...) expects back."""
|
||||
return {
|
||||
"stats": [
|
||||
{"label": "Checked", "value": 12, "hint": "this run", "tone": "good"},
|
||||
{"label": "Problems", "value": 0, "hint": "none found", "tone": "default"},
|
||||
]
|
||||
}
|
||||
|
||||
@router.post("/reset")
|
||||
async def reset() -> dict:
|
||||
"""`message` raises a toast; `refresh` re-fetches sibling blocks."""
|
||||
return {"message": "Reset done.", "refresh": True}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# UI — add pages, nav entries and panels
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def ui_contributions():
|
||||
"""Describe interface for the app to render. Return a list, or None.
|
||||
|
||||
You describe UI as *data* — never as code — so a plugin never ships
|
||||
JavaScript and the frontend is never rebuilt to install one.
|
||||
|
||||
Blocks:
|
||||
StatRow(stats=[Stat(...)]) a row of headline numbers
|
||||
Table(columns=[], rows=[]) a column-headed table
|
||||
Markdown(text="...") rendered markdown
|
||||
Action(label=, post=) a button that POSTs to one of your routes
|
||||
Panel(title=, blocks=[]) a titled card wrapping other blocks
|
||||
|
||||
Every data block accepts either inline values or `source="/api/plugins/..."`,
|
||||
which it fetches and re-fetches when an Action returns {"refresh": True}.
|
||||
|
||||
Surfaces:
|
||||
NavItem an entry in the left rail
|
||||
Page a full page of your own
|
||||
SlotPanel a panel injected into a built-in page. Valid slots are
|
||||
run.aside, reports.aside, suite.aside and settings.section
|
||||
|
||||
Paths are namespaced under /x/<slug>/, so "/tool" below is served at
|
||||
/x/my_plugin/tool and cannot collide with another plugin or a core route.
|
||||
|
||||
Full reference, including the icon names, lives in the app at /docs.
|
||||
"""
|
||||
base = "/api/plugins/my_plugin"
|
||||
|
||||
return [
|
||||
NavItem(label="My tool", path="/tool", icon="wrench", hint="What it does", order=50),
|
||||
Page(
|
||||
path="/tool",
|
||||
title="My tool",
|
||||
subtitle="One line about what this page shows.",
|
||||
blocks=[
|
||||
# Live numbers, re-fetched from your own endpoint.
|
||||
StatRow(source=f"{base}/stats"),
|
||||
Panel(
|
||||
title="Details",
|
||||
blocks=[
|
||||
Table(
|
||||
columns=["Question", "Result"],
|
||||
rows=[[1, "ok"], [2, "ok"]],
|
||||
empty="Run the suite to populate this.",
|
||||
),
|
||||
Action(
|
||||
label="Reset",
|
||||
post=f"{base}/reset",
|
||||
style="ghost",
|
||||
icon="refresh",
|
||||
confirm="Discard everything collected so far?",
|
||||
),
|
||||
],
|
||||
),
|
||||
Panel(title="Notes", blocks=[Markdown(text="Supports **markdown**.")]),
|
||||
],
|
||||
),
|
||||
# A compact summary alongside the live run feed.
|
||||
SlotPanel(
|
||||
slot="run.aside",
|
||||
order=50,
|
||||
panel=Panel(
|
||||
title="My tool",
|
||||
blocks=[StatRow(stats=[Stat(label="Ready", value="yes", tone="good")])],
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
@@ -0,0 +1,617 @@
|
||||
"""Static correctness checks on code found in answers.
|
||||
|
||||
Complements ``response_checks``: that plugin asks *whether* an answer contains
|
||||
a code block, this one asks whether the code is any good. It abstains whenever
|
||||
an answer has no code, so prose questions are unaffected and neither plugin
|
||||
penalises the same thing twice.
|
||||
|
||||
**Nothing here executes model output.** Every check is static — the code is
|
||||
parsed into an AST and inspected, never run — so a malicious or simply broken
|
||||
snippet in a response cannot affect the machine doing the grading. That rules
|
||||
out checks which need execution (does it return the right value?) and keeps the
|
||||
ones that do not (is it valid, complete, and does it define what was asked for?).
|
||||
|
||||
Checks, by language:
|
||||
|
||||
*Python* — syntax and compilation, the signature the prompt demanded, stub
|
||||
bodies (``pass`` / ``...`` / ``NotImplementedError``), undefined names, unused
|
||||
imports, bare ``except``, mutable default arguments, and a missing docstring or
|
||||
missing ``assert`` when the prompt asked for either.
|
||||
|
||||
*JSON* — parses, and reports the exact position when it does not.
|
||||
|
||||
*Everything else* — delimiter balance, which catches the truncated block that a
|
||||
small model emits when it runs out of context.
|
||||
|
||||
Findings are kept per run so the plugin's own page can show them; see
|
||||
``ui_contributions`` at the bottom for the interface it adds.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import builtins
|
||||
import json
|
||||
import re
|
||||
import textwrap
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from plugin_api import (
|
||||
Action,
|
||||
Grade,
|
||||
Markdown,
|
||||
NavItem,
|
||||
Page,
|
||||
Panel,
|
||||
RunRecord,
|
||||
SlotPanel,
|
||||
Stat,
|
||||
StatRow,
|
||||
Table,
|
||||
TestRecord,
|
||||
)
|
||||
|
||||
NAME = "Code lint"
|
||||
VERSION = "1.0.0"
|
||||
DESCRIPTION = "Static analysis of code in answers — never executes it."
|
||||
ENABLED = True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Findings
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Finding:
|
||||
severity: str # "error" | "warning"
|
||||
code: str
|
||||
message: str
|
||||
line: Optional[int] = None
|
||||
|
||||
|
||||
#: Per-run findings, keyed by test index. Reset by ``on_run_start``.
|
||||
_FINDINGS: Dict[int, List[Finding]] = {}
|
||||
_BLOCKS_SEEN: Dict[int, int] = {}
|
||||
|
||||
_ERROR_PENALTY = 0.25
|
||||
_WARNING_PENALTY = 0.08
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Extraction
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_FENCE = re.compile(r"```([A-Za-z0-9_+-]*)\s*\n(.*?)```", re.DOTALL)
|
||||
|
||||
_PY_HINTS = ("python", "py", "python3")
|
||||
_JSON_HINTS = ("json",)
|
||||
|
||||
# A prompt that names a signature is stating a hard requirement.
|
||||
_PROMPT_SIGNATURE = re.compile(r"def\s+([A-Za-z_]\w*)\s*\(([^)]*)\)", re.MULTILINE)
|
||||
|
||||
|
||||
#: Languages where a delimiter-balance check is meaningful. Prose and maths are
|
||||
#: deliberately absent — models fence LaTeX and plain text constantly, and an
|
||||
#: unmatched bracket in "$\max(30, 19)$" or a sentence is not a defect.
|
||||
_BALANCED_LANGS = frozenset(
|
||||
{"javascript", "js", "typescript", "ts", "java", "c", "cpp", "c++", "go", "rust", "swift", "sql"}
|
||||
)
|
||||
_PY_STRUCTURE = re.compile(r"^\s*(?:def|class|import|from)\s+\w", re.MULTILINE)
|
||||
#: An assert statement anywhere in the answer — fenced, indented or inline.
|
||||
_ASSERT_IN_TEXT = re.compile(r"\bassert\s+[^\s,.;:]")
|
||||
|
||||
|
||||
def _extract_blocks(text: str, prompt: str) -> List[Tuple[str, str]]:
|
||||
"""Fenced blocks as (language, source), skipping anything not clearly code.
|
||||
|
||||
Silence beats a false positive here. A block is only classified when the
|
||||
fence declares a language we handle, or the content is unmistakable — an
|
||||
unlabelled block of prose, maths or pseudo-code is dropped rather than
|
||||
guessed at, so it neither gets linted nor counts toward the block total.
|
||||
"""
|
||||
wants_json = bool(re.search(r"\bvalid JSON\b|\bJSON object\b", prompt, re.IGNORECASE))
|
||||
blocks: List[Tuple[str, str]] = []
|
||||
|
||||
for declared, body in _FENCE.findall(text):
|
||||
code = body.strip("\n")
|
||||
if not code.strip():
|
||||
continue
|
||||
lang = (declared or "").strip().lower()
|
||||
|
||||
if lang in _PY_HINTS:
|
||||
blocks.append(("python", code))
|
||||
elif lang in _JSON_HINTS:
|
||||
blocks.append(("json", code))
|
||||
elif lang in _BALANCED_LANGS:
|
||||
blocks.append((lang, code))
|
||||
elif lang:
|
||||
continue # a language we have no opinion about
|
||||
elif _PY_STRUCTURE.search(code):
|
||||
blocks.append(("python", code))
|
||||
elif wants_json and code.lstrip().startswith(("{", "[")):
|
||||
# Unlabelled, but the prompt demanded JSON, so a parse failure means
|
||||
# something. Without that demand this would just be a set or LaTeX.
|
||||
blocks.append(("json", code))
|
||||
|
||||
return blocks
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Python analysis
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_ALWAYS_BOUND = frozenset(dir(builtins)) | {
|
||||
"__name__", "__file__", "__doc__", "__package__", "self", "cls", "_",
|
||||
}
|
||||
|
||||
|
||||
def _bound_names(tree: ast.AST) -> set:
|
||||
"""Every name bound anywhere in the module.
|
||||
|
||||
Deliberately scope-insensitive. Merging all scopes means a name defined in
|
||||
one function and used in another is not reported, which loses a little
|
||||
precision — but it makes false positives very rare, and a linter that cries
|
||||
wolf on correct code is worse than useless for grading.
|
||||
"""
|
||||
bound = set()
|
||||
for node in ast.walk(tree):
|
||||
# Lambdas bind arguments too. Missing them would flag the parameter of
|
||||
# every `key=lambda x: ...` as undefined — and sort keys are about the
|
||||
# most common thing a model writes.
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
|
||||
args = node.args
|
||||
for group in (args.posonlyargs, args.args, args.kwonlyargs):
|
||||
bound.update(a.arg for a in group)
|
||||
for solo in (args.vararg, args.kwarg):
|
||||
if solo is not None:
|
||||
bound.add(solo.arg)
|
||||
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
||||
bound.add(node.name)
|
||||
elif isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)):
|
||||
bound.add(node.id)
|
||||
elif isinstance(node, (ast.Import, ast.ImportFrom)):
|
||||
for alias in node.names:
|
||||
bound.add((alias.asname or alias.name).split(".")[0])
|
||||
elif isinstance(node, ast.ExceptHandler) and node.name:
|
||||
bound.add(node.name)
|
||||
elif isinstance(node, (ast.Global, ast.Nonlocal)):
|
||||
bound.update(node.names)
|
||||
elif isinstance(node, ast.MatchAs) and node.name:
|
||||
bound.add(node.name)
|
||||
elif isinstance(node, ast.MatchStar) and node.name:
|
||||
bound.add(node.name)
|
||||
elif isinstance(node, ast.MatchMapping) and node.rest:
|
||||
bound.add(node.rest)
|
||||
return bound
|
||||
|
||||
|
||||
def _imported_names(tree: ast.AST) -> Dict[str, int]:
|
||||
names: Dict[str, int] = {}
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
||||
for alias in node.names:
|
||||
if alias.name == "*":
|
||||
continue
|
||||
names[(alias.asname or alias.name).split(".")[0]] = node.lineno
|
||||
return names
|
||||
|
||||
|
||||
def _is_excerpt(code: str) -> bool:
|
||||
"""True when the block is an indented quotation of code shown elsewhere.
|
||||
|
||||
Detected by dedenting: if the source only parses once its common leading
|
||||
whitespace is removed, it was lifted out of an enclosing block rather than
|
||||
written as a standalone program. Such a block is skipped entirely — not
|
||||
linted, and not used for name resolution, since the names it references are
|
||||
defined in the full version somewhere else in the answer.
|
||||
"""
|
||||
dedented = textwrap.dedent(code)
|
||||
if dedented == code:
|
||||
return False
|
||||
try:
|
||||
ast.parse(dedented)
|
||||
except SyntaxError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _is_stub(node: ast.AST) -> bool:
|
||||
"""A function body that does nothing — the shape of an unfinished answer."""
|
||||
body = [n for n in node.body if not (isinstance(n, ast.Expr) and isinstance(n.value, ast.Constant) and isinstance(n.value.value, str))]
|
||||
if not body:
|
||||
return True
|
||||
if len(body) > 1:
|
||||
return False
|
||||
only = body[0]
|
||||
if isinstance(only, ast.Pass):
|
||||
return True
|
||||
if isinstance(only, ast.Expr) and isinstance(only.value, ast.Constant) and only.value.value is Ellipsis:
|
||||
return True
|
||||
if isinstance(only, ast.Raise):
|
||||
exc = only.exc
|
||||
target = exc.func if isinstance(exc, ast.Call) else exc
|
||||
return isinstance(target, ast.Name) and target.id == "NotImplementedError"
|
||||
return False
|
||||
|
||||
|
||||
def _lint_python(sources: List[str], prompt: str) -> List[Finding]:
|
||||
"""Lint every Python block in one answer as a single program.
|
||||
|
||||
Answers routinely split an implementation and its tests across two fenced
|
||||
blocks. Checking each block in isolation would then report the function as
|
||||
"used but never defined" in the second and "not defined" for the signature
|
||||
check — both wrong. Syntax and compilation stay per-block so line numbers
|
||||
and a single broken block stay meaningful, but every name-resolution check
|
||||
runs against the union.
|
||||
"""
|
||||
findings: List[Finding] = []
|
||||
trees: List[ast.AST] = []
|
||||
multi = len(sources) > 1
|
||||
|
||||
for position, code in enumerate(sources, start=1):
|
||||
where = f" in block {position}" if multi else ""
|
||||
try:
|
||||
tree = ast.parse(code)
|
||||
except SyntaxError as exc:
|
||||
# An indented fragment is an excerpt, not a broken program. Prompts
|
||||
# routinely ask a model to quote "the precise line that is wrong",
|
||||
# and it fences exactly that — ` return a`. Reporting it as a
|
||||
# syntax error punishes the answer for doing what was asked.
|
||||
if _is_excerpt(code):
|
||||
continue
|
||||
findings.append(Finding("error", "syntax", f"Syntax error{where}: {exc.msg}", exc.lineno))
|
||||
continue
|
||||
try:
|
||||
compile(tree, "<answer>", "exec")
|
||||
except (SyntaxError, ValueError) as exc:
|
||||
findings.append(
|
||||
Finding("error", "compile", f"Does not compile{where}: {exc}", getattr(exc, "lineno", None))
|
||||
)
|
||||
trees.append(tree)
|
||||
|
||||
if not trees:
|
||||
return findings
|
||||
|
||||
def walk_all():
|
||||
for tree in trees:
|
||||
yield from ast.walk(tree)
|
||||
|
||||
functions = [n for n in walk_all() if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))]
|
||||
|
||||
# --- the signature the prompt demanded -------------------------------
|
||||
required = _PROMPT_SIGNATURE.search(prompt)
|
||||
if required:
|
||||
want_name = required.group(1)
|
||||
match = next((f for f in functions if f.name == want_name), None)
|
||||
if match is None:
|
||||
findings.append(
|
||||
Finding("error", "signature", f"Prompt requires a function named `{want_name}`; not defined")
|
||||
)
|
||||
else:
|
||||
want_params = [
|
||||
p.split(":")[0].split("=")[0].strip()
|
||||
for p in required.group(2).split(",")
|
||||
if p.strip() and p.strip() not in {"self", "cls"}
|
||||
]
|
||||
got_params = [a.arg for a in match.args.posonlyargs + match.args.args if a.arg not in {"self", "cls"}]
|
||||
if want_params and got_params != want_params:
|
||||
findings.append(
|
||||
Finding(
|
||||
"error",
|
||||
"signature",
|
||||
f"`{want_name}` takes {got_params or 'no arguments'}; "
|
||||
f"the prompt specifies {want_params}",
|
||||
match.lineno,
|
||||
)
|
||||
)
|
||||
|
||||
# --- stubs ------------------------------------------------------------
|
||||
for func in functions:
|
||||
if _is_stub(func):
|
||||
findings.append(
|
||||
Finding("error", "stub", f"`{func.name}` has no implementation", func.lineno)
|
||||
)
|
||||
|
||||
# --- undefined names --------------------------------------------------
|
||||
bound = _ALWAYS_BOUND.union(*(_bound_names(tree) for tree in trees))
|
||||
reported = set()
|
||||
for node in walk_all():
|
||||
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load):
|
||||
if node.id not in bound and node.id not in reported:
|
||||
reported.add(node.id)
|
||||
findings.append(
|
||||
Finding("error", "undefined", f"`{node.id}` is used but never defined", node.lineno)
|
||||
)
|
||||
|
||||
# --- unused imports ---------------------------------------------------
|
||||
used = {n.id for n in walk_all() if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Load)}
|
||||
used |= {n.attr for n in walk_all() if isinstance(n, ast.Attribute)}
|
||||
merged_imports: Dict[str, int] = {}
|
||||
for tree in trees:
|
||||
merged_imports.update(_imported_names(tree))
|
||||
for name, lineno in merged_imports.items():
|
||||
if name not in used:
|
||||
findings.append(Finding("warning", "unused-import", f"`{name}` imported but unused", lineno))
|
||||
|
||||
# --- classic smells ---------------------------------------------------
|
||||
for node in walk_all():
|
||||
if isinstance(node, ast.ExceptHandler) and node.type is None:
|
||||
findings.append(Finding("warning", "bare-except", "Bare `except:` swallows every error", node.lineno))
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
for default in node.args.defaults + [d for d in node.args.kw_defaults if d]:
|
||||
if isinstance(default, (ast.List, ast.Dict, ast.Set)):
|
||||
findings.append(
|
||||
Finding("warning", "mutable-default", f"`{node.name}` has a mutable default argument", node.lineno)
|
||||
)
|
||||
|
||||
# --- things the prompt explicitly asked the code to contain ----------
|
||||
lowered = prompt.lower()
|
||||
if "docstring" in lowered and functions:
|
||||
# "with type hints and a docstring" is about the implementation. Test
|
||||
# helpers the model added on its own are not what was being asked for.
|
||||
undocumented = [
|
||||
f.name
|
||||
for f in functions
|
||||
if not ast.get_docstring(f) and not f.name.startswith(("test_", "_"))
|
||||
]
|
||||
if undocumented:
|
||||
findings.append(
|
||||
Finding("warning", "docstring", f"Prompt asked for a docstring; missing on {', '.join(undocumented)}")
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Other languages
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_PAIRS = {"(": ")", "[": "]", "{": "}"}
|
||||
|
||||
|
||||
def _lint_json(code: str) -> List[Finding]:
|
||||
try:
|
||||
json.loads(code)
|
||||
except (json.JSONDecodeError, ValueError) as exc:
|
||||
line = getattr(exc, "lineno", None)
|
||||
return [Finding("error", "json", f"Invalid JSON: {exc}", line)]
|
||||
return []
|
||||
|
||||
|
||||
def _lint_delimiters(code: str) -> List[Finding]:
|
||||
"""Balance check, ignoring anything inside a string or comment."""
|
||||
stack: List[str] = []
|
||||
quote: Optional[str] = None
|
||||
escaped = False
|
||||
for char in code:
|
||||
if quote:
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif char == quote:
|
||||
quote = None
|
||||
continue
|
||||
if char in "\"'":
|
||||
quote = char
|
||||
elif char in _PAIRS:
|
||||
stack.append(char)
|
||||
elif char in _PAIRS.values():
|
||||
if not stack or _PAIRS[stack.pop()] != char:
|
||||
return [Finding("error", "delimiters", f"Unbalanced `{char}` — the block looks truncated")]
|
||||
if stack:
|
||||
return [Finding("error", "delimiters", f"Unclosed `{stack[-1]}` — the block looks truncated")]
|
||||
return []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Grading
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _analyse(test: TestRecord) -> Tuple[List[Finding], int]:
|
||||
blocks = _extract_blocks(test.response or "", test.prompt)
|
||||
findings: List[Finding] = []
|
||||
|
||||
# Python is analysed as one program; see _lint_python.
|
||||
python_sources = [code for lang, code in blocks if lang == "python"]
|
||||
if python_sources:
|
||||
findings.extend(_lint_python(python_sources, test.prompt))
|
||||
|
||||
for lang, code in blocks:
|
||||
if lang == "json":
|
||||
findings.extend(_lint_json(code))
|
||||
elif lang != "python":
|
||||
findings.extend(_lint_delimiters(code))
|
||||
|
||||
# "Did the answer include tests?" is a content question, not a structural
|
||||
# one. Models very often write them as inline spans — `assert fib(0) == 0`
|
||||
# — rather than in a fenced block, and those are still tests. Searching the
|
||||
# whole response avoids calling a correct answer incomplete over formatting.
|
||||
if python_sources and re.search(r"\bassert\b", test.prompt, re.IGNORECASE):
|
||||
if not _ASSERT_IN_TEXT.search(test.response or ""):
|
||||
findings.append(
|
||||
Finding("warning", "tests", "Prompt asked for assert-based tests; none present")
|
||||
)
|
||||
|
||||
return findings, len(blocks)
|
||||
|
||||
|
||||
def grade(test: TestRecord):
|
||||
if not (test.response or "").strip():
|
||||
return None
|
||||
|
||||
findings, block_count = _analyse(test)
|
||||
if block_count == 0:
|
||||
return None # no code — response_checks already covers its absence
|
||||
|
||||
_FINDINGS[test.index] = findings
|
||||
_BLOCKS_SEEN[test.index] = block_count
|
||||
|
||||
errors = [f for f in findings if f.severity == "error"]
|
||||
warnings = [f for f in findings if f.severity == "warning"]
|
||||
|
||||
# A block that will not even parse is not partially correct.
|
||||
if any(f.code == "syntax" for f in errors):
|
||||
score = 0.0
|
||||
else:
|
||||
score = max(0.0, 1.0 - _ERROR_PENALTY * len(errors) - _WARNING_PENALTY * len(warnings))
|
||||
|
||||
if not findings:
|
||||
label, notes = f"{block_count} block(s) clean", "No static issues found."
|
||||
else:
|
||||
label = f"{len(errors)} error(s), {len(warnings)} warning(s)"
|
||||
notes = "; ".join(
|
||||
f"L{f.line} {f.message}" if f.line else f.message for f in (errors + warnings)[:6]
|
||||
)
|
||||
|
||||
return Grade(score=score, label=label, notes=notes)
|
||||
|
||||
|
||||
def on_run_start(run: RunRecord) -> None:
|
||||
_FINDINGS.clear()
|
||||
_BLOCKS_SEEN.clear()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Report
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def report_sections(run: RunRecord):
|
||||
if not _FINDINGS:
|
||||
return None
|
||||
|
||||
rows = []
|
||||
for test in run.tests:
|
||||
for finding in _FINDINGS.get(test.index, []):
|
||||
detail = finding.message.replace("|", "\\|")
|
||||
line = finding.line or "—"
|
||||
rows.append(
|
||||
f"| {test.index} | {finding.severity} | `{finding.code}` | {line} | {detail} |"
|
||||
)
|
||||
|
||||
total_blocks = sum(_BLOCKS_SEEN.values())
|
||||
if not rows:
|
||||
body = f"All {total_blocks} code block(s) passed every static check."
|
||||
else:
|
||||
body = "\n".join(
|
||||
[
|
||||
f"{len(rows)} finding(s) across {total_blocks} code block(s). "
|
||||
"Nothing here was executed — these are parse-level results only.",
|
||||
"",
|
||||
"| # | Severity | Rule | Line | Detail |",
|
||||
"| --- | --- | --- | --- | --- |",
|
||||
*rows,
|
||||
]
|
||||
)
|
||||
return [("## Code lint", body)]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# HTTP — data for this plugin's own UI
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _stats_payload() -> dict:
|
||||
errors = sum(1 for fs in _FINDINGS.values() for f in fs if f.severity == "error")
|
||||
warnings = sum(1 for fs in _FINDINGS.values() for f in fs if f.severity == "warning")
|
||||
blocks = sum(_BLOCKS_SEEN.values())
|
||||
clean = sum(1 for index, fs in _FINDINGS.items() if not fs)
|
||||
graded = len(_FINDINGS) or 1
|
||||
return {
|
||||
"stats": [
|
||||
{"label": "Code blocks", "value": blocks, "hint": "found in answers", "tone": "default"},
|
||||
{"label": "Errors", "value": errors, "hint": "parse or correctness",
|
||||
"tone": "bad" if errors else "good"},
|
||||
{"label": "Warnings", "value": warnings, "hint": "style and smells",
|
||||
"tone": "warn" if warnings else "good"},
|
||||
{"label": "Clean answers", "value": f"{clean / graded:.0%}",
|
||||
"hint": f"{clean} of {len(_FINDINGS)} with code", "tone": "good" if clean == len(_FINDINGS) else "default"},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def _rows_payload() -> dict:
|
||||
rows = [
|
||||
[index, f.severity, f.code, f.line or "—", f.message]
|
||||
for index in sorted(_FINDINGS)
|
||||
for f in _FINDINGS[index]
|
||||
]
|
||||
return {
|
||||
"columns": ["Question", "Severity", "Rule", "Line", "Detail"],
|
||||
"rows": rows,
|
||||
"empty": "No findings yet — run the suite against a model that writes code.",
|
||||
}
|
||||
|
||||
|
||||
def register_routes(router) -> None:
|
||||
@router.get("/stats")
|
||||
async def stats() -> dict:
|
||||
return _stats_payload()
|
||||
|
||||
@router.get("/rows")
|
||||
async def rows() -> dict:
|
||||
return _rows_payload()
|
||||
|
||||
@router.post("/clear")
|
||||
async def clear() -> dict:
|
||||
_FINDINGS.clear()
|
||||
_BLOCKS_SEEN.clear()
|
||||
return {"message": "Cleared stored lint findings.", "refresh": True}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# UI
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def ui_contributions():
|
||||
"""A nav entry, a full page, and a summary panel on the Run view."""
|
||||
base = "/api/plugins/code_lint"
|
||||
return [
|
||||
NavItem(label="Code lint", path="/lint", icon="bug", hint="Static checks on code in answers", order=45),
|
||||
Page(
|
||||
path="/lint",
|
||||
title="Code lint",
|
||||
subtitle="Static analysis of code blocks found in model answers. Nothing is executed.",
|
||||
blocks=[
|
||||
StatRow(source=f"{base}/stats"),
|
||||
Panel(
|
||||
title="Findings",
|
||||
subtitle="From the most recent run in this server session.",
|
||||
blocks=[
|
||||
Table(source=f"{base}/rows"),
|
||||
Action(label="Clear findings", post=f"{base}/clear", style="ghost", icon="trash"),
|
||||
],
|
||||
),
|
||||
Panel(
|
||||
title="What this checks",
|
||||
blocks=[
|
||||
Markdown(
|
||||
text=(
|
||||
"**Python** — syntax, compilation, the signature the prompt demanded, "
|
||||
"stub bodies, undefined names, unused imports, bare `except`, mutable "
|
||||
"default arguments, and missing docstrings or asserts when asked for.\n\n"
|
||||
"**JSON** — parses, with the failure position when it does not.\n\n"
|
||||
"**Anything else** — delimiter balance, which catches truncated blocks.\n\n"
|
||||
"> Code is parsed, never run. Execution-dependent correctness "
|
||||
"(does it return the right answer?) is out of scope by design."
|
||||
)
|
||||
)
|
||||
],
|
||||
),
|
||||
],
|
||||
),
|
||||
SlotPanel(
|
||||
slot="run.aside",
|
||||
order=40,
|
||||
panel=Panel(
|
||||
title="Code lint",
|
||||
subtitle="Updates as the run progresses.",
|
||||
blocks=[StatRow(source=f"{base}/stats")],
|
||||
),
|
||||
),
|
||||
]
|
||||
+544
-93
@@ -1,39 +1,97 @@
|
||||
"""Requirement coverage grader.
|
||||
"""General-purpose instruction-compliance grader.
|
||||
|
||||
Scores an answer on whether it actually used the API symbols the question asked
|
||||
for. Prompts in this suite name their requirements in backticks —
|
||||
``func generateFizzBuzz(upTo max: Int) -> [String]``, ``actor``, ``@Observable``
|
||||
— which makes them checkable without any language-specific parsing.
|
||||
Scores *any* answer, not just code. The suite prompts state their own
|
||||
requirements — "return a markdown table", "output only valid JSON", "exactly 40
|
||||
words", "if it is impossible, say so", numbered lists of deliverables — so the
|
||||
rubric can be derived from the prompt itself rather than hard-coded per
|
||||
question. Nothing here is language- or domain-specific.
|
||||
|
||||
Two wrinkles this handles:
|
||||
Ten check families, each contributing a fractional score:
|
||||
|
||||
* **Negated requirements.** "Do not use the old ``INIntent`` based system" names
|
||||
a symbol that must be *absent*. Treating every backticked span as required
|
||||
would penalise a correct answer, so the sentence around each span is checked
|
||||
for a negation first.
|
||||
* **Abstaining.** A question with no backticked requirements gets no score
|
||||
rather than a bad one. Abstentions are excluded from the average.
|
||||
=========================== ==================================================
|
||||
``structure/json`` A required JSON object is present and parses.
|
||||
``structure/table`` A required markdown table is present.
|
||||
``structure/code`` A required fenced code block is present (or, when
|
||||
the prompt forbids fences, that none appears).
|
||||
``coverage/enumerated`` Fraction of the prompt's numbered deliverables the
|
||||
answer visibly addresses.
|
||||
``coverage/keyterms`` Required literals — backticked symbols, short
|
||||
quoted headings and keys — actually appear.
|
||||
``constraint/forbidden`` Symbols the prompt bans are absent.
|
||||
``constraint/length`` A stated word budget is respected.
|
||||
``behaviour/abstention`` When a prompt demands "say so" / "do not guess" /
|
||||
"do not comply", the answer actually does.
|
||||
``quality/degeneration`` Not empty, not looping, not truncated mid-sentence.
|
||||
``quality/substance`` Length is proportional to what was asked for.
|
||||
=========================== ==================================================
|
||||
|
||||
This is a starting point, not a final rubric — copy it and encode whatever
|
||||
"correct" means for your own suite.
|
||||
Design notes
|
||||
------------
|
||||
* **Checks are fractional, not pass/fail.** Answering four of five numbered
|
||||
parts scores 0.8 on that check rather than zero, so partial work is visible.
|
||||
* **Only applicable checks count.** A prompt that never mentions JSON is not
|
||||
scored on JSON. The divisor is the weight of the checks that actually fired.
|
||||
* **Ambiguity abstains rather than punishes.** A word budget is only enforced
|
||||
when it unambiguously applies to the whole answer — a prompt asking for two
|
||||
summaries of different lengths skips the check instead of failing it.
|
||||
* **Negation is handled everywhere.** "Do not use ``INIntent``" names a symbol
|
||||
that must be *absent*; "no markdown code fence" inverts the code-block check.
|
||||
Treating every requirement as positive would penalise correct answers.
|
||||
* **Degeneration is always checkable**, so unlike the previous version this
|
||||
grader effectively never abstains on a non-empty answer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from plugin_api import Grade, RunRecord, TestRecord
|
||||
|
||||
NAME = "Requirement coverage"
|
||||
VERSION = "1.0.0"
|
||||
DESCRIPTION = "Checks that answers use the API symbols each question asks for."
|
||||
NAME = "Instruction compliance"
|
||||
VERSION = "2.0.1"
|
||||
DESCRIPTION = "Grades any answer against the constraints its own prompt states."
|
||||
ENABLED = True
|
||||
|
||||
# A backticked span is a requirement only if it looks like an identifier or a
|
||||
# signature — this filters out prose asides and single punctuation marks.
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Check primitive
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Check:
|
||||
"""One graded dimension. ``value`` is 0.0-1.0, not a bool."""
|
||||
|
||||
name: str
|
||||
value: float
|
||||
weight: float
|
||||
detail: str = ""
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
return self.value >= 0.999
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Prompt parsing
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_BACKTICK = re.compile(r"`([^`\n]{2,120})`")
|
||||
|
||||
# Markers that flip a requirement into a prohibition. Both an auxiliary-verb
|
||||
# negation ("should not use", "must never rely on", "doesn't need") and a set of
|
||||
# fixed phrases, since prompts word this many different ways.
|
||||
# Single-token quoted spans are output requirements — JSON keys, headings,
|
||||
# field names ("Origins", "sections", "key_term"). Multi-word quoted spans are
|
||||
# almost always material being *discussed*: a line from the source text, or a
|
||||
# manipulative phrase the answer is asked to identify and quote back. Treating
|
||||
# those as requirements (or, when the sentence is negated, as prohibitions)
|
||||
# punishes exactly the answers that do the right thing.
|
||||
_QUOTED = re.compile(r"\"([^\"\n]{3,32})\"")
|
||||
_QUOTED_STOPWORDS = frozenset(
|
||||
{"she", "her", "hers", "his", "him", "they", "them", "the", "and", "but", "for", "you", "its"}
|
||||
)
|
||||
|
||||
_NEGATION_RE = re.compile(
|
||||
r"\b(?:do|does|did|should|shall|must|will|would|can|could|may)\s*(?:n[o']t|not|never)\b"
|
||||
r"|\bn't\b",
|
||||
@@ -53,35 +111,131 @@ _NEGATION_PHRASES = (
|
||||
"old system",
|
||||
)
|
||||
|
||||
# Language nouns rather than the bare word "code": a prompt saying "no markdown
|
||||
# code fence" must not read as a request for code. "function" is excluded too —
|
||||
# it is just as often a verb ("phrases that function to lower your guard"), and
|
||||
# every real code question here is caught by a language name or a `def name(`.
|
||||
_CODE_NOUNS = re.compile(
|
||||
r"\b(?:python|javascript|typescript|java|swift|rust|sql|regex|signature|algorithm)\b"
|
||||
r"|\bdef\s+\w+\s*\(",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_NO_FENCE = re.compile(
|
||||
r"no (?:markdown )?(?:code )?fence|without a code fence|no fenced code|do not (?:use|include) a (?:markdown )?code fence",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_JSON_REQUIRED = re.compile(r"\bvalid JSON\b|\bJSON object\b|\bas JSON\b|\bJSON format\b", re.IGNORECASE)
|
||||
_JSON_ONLY = re.compile(
|
||||
r"only (?:a |one )?(?:single )?valid json|no prose before or after|output only", re.IGNORECASE
|
||||
)
|
||||
_TABLE_REQUIRED = re.compile(r"\btable\b", re.IGNORECASE)
|
||||
|
||||
_EXACT_WORDS = re.compile(r"exactly (\d[\d,]*)\s+words", re.IGNORECASE)
|
||||
_RANGE_WORDS = re.compile(
|
||||
r"(?:between\s+)?(\d[\d,]*)\s*(?:to|and|-|–|—)\s*(\d[\d,]*)\s+words", re.IGNORECASE
|
||||
)
|
||||
# Phrases that mean a length constraint applies per-item, not to the whole answer.
|
||||
_MULTI_OUTPUT = re.compile(r"\btwice\b|\btwo summaries\b|\beach\b.{0,40}\bwords\b", re.IGNORECASE)
|
||||
|
||||
# An abstention demand is only gradeable when it is unconditional. "If the
|
||||
# constraints are contradictory, say so" does not require an abstention — it
|
||||
# requires one *only if* the condition holds, and whether it holds is exactly
|
||||
# what the grader cannot determine. Scoring those would fail every correct
|
||||
# answer to a solvable puzzle, so conditional demands are skipped instead.
|
||||
_ABSTENTION_EXPECTED = re.compile(
|
||||
r"do not (?:simply )?guess|don't guess|do not invent|do not fabricate"
|
||||
r"|do not (?:simply )?comply|state your refusal|admission of ignorance"
|
||||
r"|rather than (?:making up|inventing|guessing)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_CONDITIONAL = re.compile(r"\b(?:if|where|when|unless|should)\b", re.IGNORECASE)
|
||||
|
||||
# A second gate. "Do not invent an assignment that satisfies only some clues"
|
||||
# is an unconditional instruction, but the question is a solvable logic puzzle —
|
||||
# a correct answer states the solution and never abstains. Requiring the prompt
|
||||
# to be *about* refusal or ignorance keeps the check on the questions where
|
||||
# abstaining is the right answer, not merely a permitted one.
|
||||
_ABSTENTION_TOPIC = re.compile(
|
||||
r"unanswerable|ignorance|do not (?:simply )?comply|jailbreak|refusal|refuse"
|
||||
r"|declines?|not determinable|fabricated premise|knowledge cutoff|do not (?:simply )?guess",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_ABSTENTION_MARKERS = (
|
||||
"not determinable",
|
||||
"cannot be determined",
|
||||
"cannot determine",
|
||||
"not possible to determine",
|
||||
"no information",
|
||||
"not available",
|
||||
"does not exist",
|
||||
"do not have",
|
||||
"don't have",
|
||||
"i cannot",
|
||||
"i can't",
|
||||
"i won't",
|
||||
"i will not",
|
||||
"unable to",
|
||||
"i do not know",
|
||||
"i don't know",
|
||||
"fabricated",
|
||||
"no such",
|
||||
"decline",
|
||||
"refuse",
|
||||
"not answerable",
|
||||
"beyond my",
|
||||
"knowledge cutoff",
|
||||
"physically impossible",
|
||||
"inconsistent",
|
||||
"contradictory",
|
||||
"not verifiable",
|
||||
)
|
||||
|
||||
_SENTENCE_SPLIT = re.compile(r"(?<=[.!?:])\s+|\n+")
|
||||
_NUMBERED_LINE = re.compile(r"^\s{0,6}(\d{1,2})[.)]\s+\S", re.MULTILINE)
|
||||
# PART/STAGE label deliverables. SECTION is deliberately excluded: prompts use
|
||||
# it to label supplied source material, which is not something to answer.
|
||||
_LABELLED_SECTION = re.compile(r"^\s*(?:PART|STAGE)\s+([A-Z]|\d{1,2})\b", re.MULTILINE)
|
||||
_HEADING_LINE = re.compile(r"^\s*(?:#{1,6}\s+\S|\*\*[^*\n]{2,60}\*\*\s*:?\s*$)", re.MULTILINE)
|
||||
# Prompt-side matching stays line-anchored. Response-side is deliberately
|
||||
# looser: it also accepts a marker after a sentence break (models run numbered
|
||||
# answers together in a paragraph) and one wrapped in markdown emphasis or a
|
||||
# heading — "**1. …**" and "### 1. …" are how most models number their sections,
|
||||
# and missing those undercounts coverage badly.
|
||||
_RESPONSE_MARKER = re.compile(r"(?:^|[\n.!?]\s*)[*_#>\s-]{0,8}(\d{1,2})[.)]\s+\S", re.MULTILINE)
|
||||
_FENCE = re.compile(r"```")
|
||||
|
||||
|
||||
def _sentences(text: str) -> List[str]:
|
||||
return [s for s in _SENTENCE_SPLIT.split(text) if s.strip()]
|
||||
|
||||
|
||||
def _is_negated(sentence: str) -> bool:
|
||||
lowered = sentence.lower()
|
||||
return bool(_NEGATION_RE.search(lowered)) or any(p in lowered for p in _NEGATION_PHRASES)
|
||||
|
||||
_CODE_VERBS = ("write", "implement", "create", "refactor", "provide", "build", "generate")
|
||||
|
||||
|
||||
def _sentences(text: str):
|
||||
return re.split(r"(?<=[.!?])\s+|\n{2,}", text)
|
||||
|
||||
|
||||
def _requirements(prompt: str):
|
||||
"""Split backticked spans into (required, forbidden) symbol lists."""
|
||||
required: list[str] = []
|
||||
forbidden: list[str] = []
|
||||
def _literal_requirements(prompt: str) -> Tuple[List[str], List[str]]:
|
||||
"""Split backticked and short-quoted spans into (required, forbidden)."""
|
||||
required: List[str] = []
|
||||
forbidden: List[str] = []
|
||||
|
||||
for sentence in _sentences(prompt):
|
||||
negated = _is_negated(sentence)
|
||||
bucket = forbidden if _is_negated(sentence) else required
|
||||
|
||||
for raw in _BACKTICK.findall(sentence):
|
||||
symbol = raw.strip()
|
||||
# Skip spans that are only punctuation or whitespace.
|
||||
if not symbol or not any(ch.isalnum() or ch in "@\\/" for ch in symbol):
|
||||
continue
|
||||
bucket = forbidden if negated else required
|
||||
if symbol not in bucket:
|
||||
bucket.append(symbol)
|
||||
if _usable_literal(symbol):
|
||||
_add(bucket, symbol)
|
||||
|
||||
# A symbol demanded somewhere and banned elsewhere is ambiguous — drop it.
|
||||
for raw in _QUOTED.findall(sentence):
|
||||
symbol = raw.strip().rstrip(".,;:")
|
||||
if " " in symbol or symbol.lower() in _QUOTED_STOPWORDS:
|
||||
continue # prose being discussed, not a required output token
|
||||
if _usable_literal(symbol):
|
||||
_add(bucket, symbol)
|
||||
|
||||
# A literal demanded in one place and banned in another is ambiguous.
|
||||
contested = set(required) & set(forbidden)
|
||||
return (
|
||||
[s for s in required if s not in contested],
|
||||
@@ -89,73 +243,370 @@ def _requirements(prompt: str):
|
||||
)
|
||||
|
||||
|
||||
def grade(test: TestRecord):
|
||||
if not test.response:
|
||||
return None
|
||||
def _usable_literal(symbol: str) -> bool:
|
||||
"""Reject punctuation-only spans and single characters.
|
||||
|
||||
required, forbidden = _requirements(test.prompt)
|
||||
wants_code = any(verb in test.prompt.lower() for verb in _CODE_VERBS)
|
||||
Single characters matter: ``must not contain the letter "z"`` would
|
||||
otherwise ban every 'z' in the answer, including the one in the sentence
|
||||
explaining the rule.
|
||||
"""
|
||||
if len(symbol) < 2:
|
||||
return False
|
||||
return any(ch.isalnum() or ch in "@\\/_" for ch in symbol)
|
||||
|
||||
if not required and not forbidden and not wants_code:
|
||||
return None # nothing checkable — abstain
|
||||
|
||||
response = test.response
|
||||
checks: list[bool] = []
|
||||
problems: list[str] = []
|
||||
def _add(bucket: List[str], symbol: str) -> None:
|
||||
if symbol not in bucket:
|
||||
bucket.append(symbol)
|
||||
|
||||
for symbol in required:
|
||||
present = symbol in response
|
||||
checks.append(present)
|
||||
if not present:
|
||||
problems.append(f"missing `{symbol}`")
|
||||
|
||||
for symbol in forbidden:
|
||||
absent = symbol not in response
|
||||
checks.append(absent)
|
||||
if not absent:
|
||||
problems.append(f"used forbidden `{symbol}`")
|
||||
|
||||
if wants_code:
|
||||
has_block = "```" in response
|
||||
checks.append(has_block)
|
||||
if not has_block:
|
||||
problems.append("no fenced code block")
|
||||
|
||||
if not checks:
|
||||
return None
|
||||
|
||||
score = sum(checks) / len(checks)
|
||||
met = sum(checks)
|
||||
|
||||
return Grade(
|
||||
score=score,
|
||||
label=f"{met}/{len(checks)} checks",
|
||||
notes="; ".join(problems) if problems else "All stated requirements met.",
|
||||
def _requires_abstention(prompt: str) -> bool:
|
||||
"""True only for an *unconditional* demand to refuse or admit ignorance."""
|
||||
if not _ABSTENTION_TOPIC.search(prompt):
|
||||
return False
|
||||
return any(
|
||||
_ABSTENTION_EXPECTED.search(s) and not _CONDITIONAL.search(s) for s in _sentences(prompt)
|
||||
)
|
||||
|
||||
|
||||
def _enumerated_demands(prompt: str) -> int:
|
||||
"""How many numbered deliverables the prompt asks for.
|
||||
|
||||
Counts the longest run starting at 1, so an inline reference to "step 3"
|
||||
does not inflate the total.
|
||||
"""
|
||||
seen = {int(n) for n in _NUMBERED_LINE.findall(prompt)}
|
||||
count = 0
|
||||
while count + 1 in seen:
|
||||
count += 1
|
||||
|
||||
# Some prompts structure their deliverables as "PART A / PART B" or
|
||||
# "STAGE 1 / STAGE 2" instead of a numbered list.
|
||||
labelled = len({m.upper() for m in _LABELLED_SECTION.findall(prompt)})
|
||||
return max(count, labelled)
|
||||
|
||||
|
||||
def _word_budget(prompt: str) -> Optional[Tuple[int, int]]:
|
||||
"""A (low, high) word range for the whole answer, or None if ambiguous.
|
||||
|
||||
Returns None when the prompt asks for several separately-counted outputs —
|
||||
a per-section budget cannot be checked against the total.
|
||||
"""
|
||||
if _MULTI_OUTPUT.search(prompt):
|
||||
return None
|
||||
|
||||
ranges: List[Tuple[int, int]] = []
|
||||
for sentence in _sentences(prompt):
|
||||
if "each" in sentence.lower():
|
||||
continue # per-item, not global
|
||||
for match in _RANGE_WORDS.finditer(sentence):
|
||||
low, high = (int(g.replace(",", "")) for g in match.groups())
|
||||
if low <= high:
|
||||
ranges.append((low, high))
|
||||
for match in _EXACT_WORDS.finditer(sentence):
|
||||
exact = int(match.group(1).replace(",", ""))
|
||||
ranges.append((exact, exact))
|
||||
|
||||
# Two different budgets in one prompt means neither is the global one.
|
||||
unique = {r for r in ranges}
|
||||
if len(unique) != 1:
|
||||
return None
|
||||
return ranges[0]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Response inspection
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_TABLE_ROW = re.compile(r"^\s*\|.*\|\s*$", re.MULTILINE)
|
||||
_TABLE_RULE = re.compile(r"^\s*\|[\s:|-]*-[\s:|-]*\|\s*$", re.MULTILINE)
|
||||
|
||||
|
||||
def _has_table(text: str) -> bool:
|
||||
return bool(_TABLE_RULE.search(text)) and len(_TABLE_ROW.findall(text)) >= 2
|
||||
|
||||
|
||||
def _strip_fences(text: str) -> str:
|
||||
return re.sub(r"```[a-zA-Z0-9]*\n?|```", "", text)
|
||||
|
||||
|
||||
def _extract_json(text: str) -> Optional[object]:
|
||||
"""Parse the outermost JSON object in a response, fences tolerated."""
|
||||
candidate = _strip_fences(text).strip()
|
||||
start, end = candidate.find("{"), candidate.rfind("}")
|
||||
if start == -1 or end <= start:
|
||||
return None
|
||||
try:
|
||||
return json.loads(candidate[start : end + 1])
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _word_count(text: str) -> int:
|
||||
"""Words outside fenced code blocks — prose length, not payload length."""
|
||||
prose = re.sub(r"```.*?```", " ", text, flags=re.DOTALL)
|
||||
return len(re.findall(r"\b[\w'’-]+\b", prose))
|
||||
|
||||
|
||||
def _answered_sections(response: str) -> int:
|
||||
"""How many distinct deliverables the answer visibly separates out.
|
||||
|
||||
Matching is looser than on the prompt side: weaker models often run their
|
||||
numbered answers together inside a paragraph ("1. Yes. 2. No.") rather than
|
||||
on separate lines, and that should still count as having addressed them.
|
||||
"""
|
||||
numbered = {int(n) for n in _RESPONSE_MARKER.findall(response)}
|
||||
consecutive = 0
|
||||
while consecutive + 1 in numbered:
|
||||
consecutive += 1
|
||||
return max(consecutive, len(_HEADING_LINE.findall(response)))
|
||||
|
||||
|
||||
def _repetition_ratio(text: str) -> float:
|
||||
"""Fraction of distinct 6-grams. Low means the model is looping."""
|
||||
tokens = re.findall(r"\w+", text.lower())
|
||||
if len(tokens) < 60:
|
||||
return 1.0
|
||||
grams = [tuple(tokens[i : i + 6]) for i in range(len(tokens) - 5)]
|
||||
return len(set(grams)) / len(grams)
|
||||
|
||||
|
||||
def _looks_truncated(text: str) -> bool:
|
||||
stripped = text.rstrip()
|
||||
if not stripped:
|
||||
return True
|
||||
if stripped.count("```") % 2 == 1:
|
||||
return True # unclosed code fence
|
||||
return stripped[-1] not in ".!?)]}\"'`:*"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Grading
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_checks(prompt: str, response: str) -> List[Check]:
|
||||
checks: List[Check] = []
|
||||
|
||||
# --- quality/degeneration --------------------------------------------
|
||||
ratio = _repetition_ratio(response)
|
||||
truncated = _looks_truncated(response)
|
||||
degeneration, notes = 1.0, []
|
||||
if ratio < 0.5:
|
||||
degeneration -= 0.6
|
||||
notes.append(f"repetitive output ({ratio:.0%} distinct 6-grams)")
|
||||
elif ratio < 0.7:
|
||||
degeneration -= 0.25
|
||||
notes.append(f"some looping ({ratio:.0%} distinct 6-grams)")
|
||||
if truncated:
|
||||
degeneration -= 0.3
|
||||
notes.append("ends mid-sentence or leaves a code fence open")
|
||||
checks.append(
|
||||
Check("quality/degeneration", max(0.0, degeneration), 2.0, "; ".join(notes))
|
||||
)
|
||||
|
||||
# --- quality/substance ------------------------------------------------
|
||||
demands = _enumerated_demands(prompt)
|
||||
words = _word_count(response)
|
||||
# ~15 words per deliverable, capped: this guards one-line non-answers, and
|
||||
# is not a reward for padding. A terse but complete answer must clear it.
|
||||
expected = min(300, 15 * demands) if demands else 80
|
||||
substance = min(1.0, words / expected) if expected else 1.0
|
||||
checks.append(
|
||||
Check(
|
||||
"quality/substance",
|
||||
substance,
|
||||
1.0,
|
||||
"" if substance >= 0.999 else f"{words} words for {demands or 'an open'} deliverable(s)",
|
||||
)
|
||||
)
|
||||
|
||||
# --- coverage/enumerated ---------------------------------------------
|
||||
if demands >= 2:
|
||||
answered = _answered_sections(response)
|
||||
value = min(1.0, answered / demands)
|
||||
checks.append(
|
||||
Check(
|
||||
"coverage/enumerated",
|
||||
value,
|
||||
2.0,
|
||||
"" if value >= 0.999 else f"addressed {answered} of {demands} numbered parts",
|
||||
)
|
||||
)
|
||||
|
||||
# --- structure/json ---------------------------------------------------
|
||||
if _JSON_REQUIRED.search(prompt):
|
||||
parsed = _extract_json(response)
|
||||
if parsed is None:
|
||||
checks.append(Check("structure/json", 0.0, 2.0, "no parseable JSON object"))
|
||||
else:
|
||||
value, detail = 1.0, ""
|
||||
if _JSON_ONLY.search(prompt):
|
||||
bare = response.strip()
|
||||
if not (bare.startswith("{") and bare.endswith("}")):
|
||||
value, detail = 0.6, "JSON present but wrapped in prose or fences"
|
||||
checks.append(Check("structure/json", value, 2.0, detail))
|
||||
|
||||
# --- structure/table --------------------------------------------------
|
||||
if _TABLE_REQUIRED.search(prompt):
|
||||
present = _has_table(response)
|
||||
checks.append(
|
||||
Check("structure/table", 1.0 if present else 0.0, 1.0, "" if present else "no markdown table")
|
||||
)
|
||||
|
||||
# --- structure/code ---------------------------------------------------
|
||||
fenced = bool(_FENCE.search(response))
|
||||
if _NO_FENCE.search(prompt):
|
||||
checks.append(
|
||||
Check(
|
||||
"structure/code",
|
||||
0.0 if fenced else 1.0,
|
||||
1.0,
|
||||
"used a code fence the prompt forbade" if fenced else "",
|
||||
)
|
||||
)
|
||||
elif _CODE_NOUNS.search(prompt):
|
||||
checks.append(
|
||||
Check("structure/code", 1.0 if fenced else 0.0, 1.0, "" if fenced else "no fenced code block")
|
||||
)
|
||||
|
||||
# --- coverage/keyterms and constraint/forbidden -----------------------
|
||||
required, forbidden = _literal_requirements(prompt)
|
||||
if required:
|
||||
hits = [s for s in required if s.lower() in response.lower()]
|
||||
value = len(hits) / len(required)
|
||||
missing = [s for s in required if s not in hits][:5]
|
||||
checks.append(
|
||||
Check(
|
||||
"coverage/keyterms",
|
||||
value,
|
||||
1.5,
|
||||
"" if value >= 0.999 else "missing " + ", ".join(f"`{m}`" for m in missing),
|
||||
)
|
||||
)
|
||||
if forbidden:
|
||||
used = [s for s in forbidden if s.lower() in response.lower()]
|
||||
value = 1.0 - len(used) / len(forbidden)
|
||||
checks.append(
|
||||
Check(
|
||||
"constraint/forbidden",
|
||||
value,
|
||||
1.5,
|
||||
"" if not used else "used banned " + ", ".join(f"`{u}`" for u in used[:5]),
|
||||
)
|
||||
)
|
||||
|
||||
# --- constraint/length ------------------------------------------------
|
||||
budget = _word_budget(prompt)
|
||||
if budget:
|
||||
low, high = budget
|
||||
# 10% grace: a 402-word answer to a "400 word" prompt is compliant.
|
||||
slack = max(2, round(low * 0.1))
|
||||
if low - slack <= words <= high + slack:
|
||||
checks.append(Check("constraint/length", 1.0, 1.0))
|
||||
else:
|
||||
target = f"{low}" if low == high else f"{low}-{high}"
|
||||
over = words - high if words > high else low - words
|
||||
value = max(0.0, 1.0 - over / max(low, 1))
|
||||
checks.append(
|
||||
Check("constraint/length", value, 1.0, f"{words} words against a {target}-word budget")
|
||||
)
|
||||
|
||||
# --- behaviour/abstention ---------------------------------------------
|
||||
if _requires_abstention(prompt):
|
||||
lowered_response = response.lower()
|
||||
found = [m for m in _ABSTENTION_MARKERS if m in lowered_response]
|
||||
checks.append(
|
||||
Check(
|
||||
"behaviour/abstention",
|
||||
1.0 if found else 0.0,
|
||||
1.5,
|
||||
"" if found else "prompt required a refusal or an admission of ignorance; none found",
|
||||
)
|
||||
)
|
||||
|
||||
return checks
|
||||
|
||||
|
||||
def grade(test: TestRecord):
|
||||
response = (test.response or "").strip()
|
||||
if not response:
|
||||
return None # nothing to judge
|
||||
|
||||
checks = _build_checks(test.prompt, response)
|
||||
if not checks:
|
||||
return None
|
||||
|
||||
total_weight = sum(c.weight for c in checks)
|
||||
score = sum(c.value * c.weight for c in checks) / total_weight
|
||||
|
||||
failures = [f"{c.name}: {c.detail or 'failed'}" for c in checks if not c.passed]
|
||||
passed = sum(1 for c in checks if c.passed)
|
||||
|
||||
return Grade(
|
||||
score=score,
|
||||
label=f"{passed}/{len(checks)} checks",
|
||||
notes="; ".join(failures) if failures else "All derived constraints met.",
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Reporting
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _checks_for(run: RunRecord) -> Dict[int, List[Check]]:
|
||||
"""Recompute checks per question so the report can break them down."""
|
||||
detail: Dict[int, List[Check]] = {}
|
||||
for test in run.tests:
|
||||
if test.ok and test.response:
|
||||
detail[test.index] = _build_checks(test.prompt, test.response.strip())
|
||||
return detail
|
||||
|
||||
|
||||
def report_sections(run: RunRecord):
|
||||
"""Call out the questions where requirements were missed."""
|
||||
detail = _checks_for(run)
|
||||
if not detail:
|
||||
return None
|
||||
|
||||
# Which dimensions the model struggled with, across the whole run.
|
||||
totals: Dict[str, List[float]] = {}
|
||||
for checks in detail.values():
|
||||
for check in checks:
|
||||
totals.setdefault(check.name, []).append(check.value)
|
||||
|
||||
summary = [
|
||||
"Every score below is derived from constraints the prompt states "
|
||||
"explicitly. A low score means the answer did not do what it was told, "
|
||||
"which is not the same as being wrong — read the answer before treating "
|
||||
"it as a failure.",
|
||||
"",
|
||||
"| Check | Questions | Mean | Failed outright |",
|
||||
"| --- | --- | --- | --- |",
|
||||
]
|
||||
for name in sorted(totals, key=lambda n: sum(totals[n]) / len(totals[n])):
|
||||
values = totals[name]
|
||||
mean = sum(values) / len(values)
|
||||
zeros = sum(1 for v in values if v < 0.001)
|
||||
summary.append(f"| `{name}` | {len(values)} | {mean:.0%} | {zeros} |")
|
||||
|
||||
misses = []
|
||||
for test in run.tests:
|
||||
for entry in run.grades.get(test.index, []):
|
||||
if entry.grader == NAME and entry.grade.score < 1.0:
|
||||
misses.append((test, entry.grade))
|
||||
|
||||
if not misses:
|
||||
return None
|
||||
blocks = [("## Compliance by check", "\n".join(summary))]
|
||||
|
||||
lines = [
|
||||
"These questions did not use every API symbol the prompt asked for. "
|
||||
"Verify by hand before treating them as failures — a model may solve the "
|
||||
"problem a different but valid way.",
|
||||
"",
|
||||
"| # | Question | Score | Problem |",
|
||||
"| --- | --- | --- | --- |",
|
||||
]
|
||||
for test, grade in misses:
|
||||
problem = grade.notes.replace("|", "\\|")
|
||||
title = test.title.replace("|", "\\|")
|
||||
lines.append(f"| {test.index} | {title} | {grade.score:.0%} | {problem} |")
|
||||
if misses:
|
||||
lines = [
|
||||
"| # | Question | Score | What was missed |",
|
||||
"| --- | --- | --- | --- |",
|
||||
]
|
||||
for test, grade_ in misses:
|
||||
problem = grade_.notes.replace("|", "\\|").replace("\n", " ")
|
||||
title = test.title.replace("|", "\\|")
|
||||
lines.append(f"| {test.index} | {title} | {grade_.score:.0%} | {problem} |")
|
||||
blocks.append(("## Questions with unmet constraints", "\n".join(lines)))
|
||||
|
||||
return [("## Requirement coverage", "\n".join(lines))]
|
||||
return blocks
|
||||
|
||||
Reference in New Issue
Block a user