Netherwarlord
8f246fabbc
- Implemented PluginBlocks component to render various plugin UI elements. - Created SuitePicker for selecting test suites and questions. - Introduced usePluginUI hook for managing plugin UI state and manifest. - Developed DocsPage for comprehensive plugin framework documentation. - Added PluginPage to render pages declared by plugins based on the current path.
613 lines
23 KiB
Python
613 lines
23 KiB
Python
"""General-purpose instruction-compliance grader.
|
||
|
||
Scores *any* answer, not just code. The suite prompts state their own
|
||
requirements — "return a markdown table", "output only valid JSON", "exactly 40
|
||
words", "if it is impossible, say so", numbered lists of deliverables — so the
|
||
rubric can be derived from the prompt itself rather than hard-coded per
|
||
question. Nothing here is language- or domain-specific.
|
||
|
||
Ten check families, each contributing a fractional score:
|
||
|
||
=========================== ==================================================
|
||
``structure/json`` A required JSON object is present and parses.
|
||
``structure/table`` A required markdown table is present.
|
||
``structure/code`` A required fenced code block is present (or, when
|
||
the prompt forbids fences, that none appears).
|
||
``coverage/enumerated`` Fraction of the prompt's numbered deliverables the
|
||
answer visibly addresses.
|
||
``coverage/keyterms`` Required literals — backticked symbols, short
|
||
quoted headings and keys — actually appear.
|
||
``constraint/forbidden`` Symbols the prompt bans are absent.
|
||
``constraint/length`` A stated word budget is respected.
|
||
``behaviour/abstention`` When a prompt demands "say so" / "do not guess" /
|
||
"do not comply", the answer actually does.
|
||
``quality/degeneration`` Not empty, not looping, not truncated mid-sentence.
|
||
``quality/substance`` Length is proportional to what was asked for.
|
||
=========================== ==================================================
|
||
|
||
Design notes
|
||
------------
|
||
* **Checks are fractional, not pass/fail.** Answering four of five numbered
|
||
parts scores 0.8 on that check rather than zero, so partial work is visible.
|
||
* **Only applicable checks count.** A prompt that never mentions JSON is not
|
||
scored on JSON. The divisor is the weight of the checks that actually fired.
|
||
* **Ambiguity abstains rather than punishes.** A word budget is only enforced
|
||
when it unambiguously applies to the whole answer — a prompt asking for two
|
||
summaries of different lengths skips the check instead of failing it.
|
||
* **Negation is handled everywhere.** "Do not use ``INIntent``" names a symbol
|
||
that must be *absent*; "no markdown code fence" inverts the code-block check.
|
||
Treating every requirement as positive would penalise correct answers.
|
||
* **Degeneration is always checkable**, so unlike the previous version this
|
||
grader effectively never abstains on a non-empty answer.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
from dataclasses import dataclass
|
||
from typing import Dict, List, Optional, Tuple
|
||
|
||
from plugin_api import Grade, RunRecord, TestRecord
|
||
|
||
NAME = "Instruction compliance"
|
||
VERSION = "2.0.1"
|
||
DESCRIPTION = "Grades any answer against the constraints its own prompt states."
|
||
ENABLED = True
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Check primitive
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@dataclass
|
||
class Check:
|
||
"""One graded dimension. ``value`` is 0.0-1.0, not a bool."""
|
||
|
||
name: str
|
||
value: float
|
||
weight: float
|
||
detail: str = ""
|
||
|
||
@property
|
||
def passed(self) -> bool:
|
||
return self.value >= 0.999
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Prompt parsing
|
||
# --------------------------------------------------------------------------
|
||
|
||
_BACKTICK = re.compile(r"`([^`\n]{2,120})`")
|
||
|
||
# Single-token quoted spans are output requirements — JSON keys, headings,
|
||
# field names ("Origins", "sections", "key_term"). Multi-word quoted spans are
|
||
# almost always material being *discussed*: a line from the source text, or a
|
||
# manipulative phrase the answer is asked to identify and quote back. Treating
|
||
# those as requirements (or, when the sentence is negated, as prohibitions)
|
||
# punishes exactly the answers that do the right thing.
|
||
_QUOTED = re.compile(r"\"([^\"\n]{3,32})\"")
|
||
_QUOTED_STOPWORDS = frozenset(
|
||
{"she", "her", "hers", "his", "him", "they", "them", "the", "and", "but", "for", "you", "its"}
|
||
)
|
||
|
||
_NEGATION_RE = re.compile(
|
||
r"\b(?:do|does|did|should|shall|must|will|would|can|could|may)\s*(?:n[o']t|not|never)\b"
|
||
r"|\bn't\b",
|
||
re.IGNORECASE,
|
||
)
|
||
_NEGATION_PHRASES = (
|
||
"avoid",
|
||
"instead of",
|
||
"rather than",
|
||
"without using",
|
||
"no longer",
|
||
"deprecated",
|
||
"stop using",
|
||
"get rid of",
|
||
"remove the",
|
||
"not the old",
|
||
"old system",
|
||
)
|
||
|
||
# Language nouns rather than the bare word "code": a prompt saying "no markdown
|
||
# code fence" must not read as a request for code. "function" is excluded too —
|
||
# it is just as often a verb ("phrases that function to lower your guard"), and
|
||
# every real code question here is caught by a language name or a `def name(`.
|
||
_CODE_NOUNS = re.compile(
|
||
r"\b(?:python|javascript|typescript|java|swift|rust|sql|regex|signature|algorithm)\b"
|
||
r"|\bdef\s+\w+\s*\(",
|
||
re.IGNORECASE,
|
||
)
|
||
_NO_FENCE = re.compile(
|
||
r"no (?:markdown )?(?:code )?fence|without a code fence|no fenced code|do not (?:use|include) a (?:markdown )?code fence",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
_JSON_REQUIRED = re.compile(r"\bvalid JSON\b|\bJSON object\b|\bas JSON\b|\bJSON format\b", re.IGNORECASE)
|
||
_JSON_ONLY = re.compile(
|
||
r"only (?:a |one )?(?:single )?valid json|no prose before or after|output only", re.IGNORECASE
|
||
)
|
||
_TABLE_REQUIRED = re.compile(r"\btable\b", re.IGNORECASE)
|
||
|
||
_EXACT_WORDS = re.compile(r"exactly (\d[\d,]*)\s+words", re.IGNORECASE)
|
||
_RANGE_WORDS = re.compile(
|
||
r"(?:between\s+)?(\d[\d,]*)\s*(?:to|and|-|–|—)\s*(\d[\d,]*)\s+words", re.IGNORECASE
|
||
)
|
||
# Phrases that mean a length constraint applies per-item, not to the whole answer.
|
||
_MULTI_OUTPUT = re.compile(r"\btwice\b|\btwo summaries\b|\beach\b.{0,40}\bwords\b", re.IGNORECASE)
|
||
|
||
# An abstention demand is only gradeable when it is unconditional. "If the
|
||
# constraints are contradictory, say so" does not require an abstention — it
|
||
# requires one *only if* the condition holds, and whether it holds is exactly
|
||
# what the grader cannot determine. Scoring those would fail every correct
|
||
# answer to a solvable puzzle, so conditional demands are skipped instead.
|
||
_ABSTENTION_EXPECTED = re.compile(
|
||
r"do not (?:simply )?guess|don't guess|do not invent|do not fabricate"
|
||
r"|do not (?:simply )?comply|state your refusal|admission of ignorance"
|
||
r"|rather than (?:making up|inventing|guessing)",
|
||
re.IGNORECASE,
|
||
)
|
||
_CONDITIONAL = re.compile(r"\b(?:if|where|when|unless|should)\b", re.IGNORECASE)
|
||
|
||
# A second gate. "Do not invent an assignment that satisfies only some clues"
|
||
# is an unconditional instruction, but the question is a solvable logic puzzle —
|
||
# a correct answer states the solution and never abstains. Requiring the prompt
|
||
# to be *about* refusal or ignorance keeps the check on the questions where
|
||
# abstaining is the right answer, not merely a permitted one.
|
||
_ABSTENTION_TOPIC = re.compile(
|
||
r"unanswerable|ignorance|do not (?:simply )?comply|jailbreak|refusal|refuse"
|
||
r"|declines?|not determinable|fabricated premise|knowledge cutoff|do not (?:simply )?guess",
|
||
re.IGNORECASE,
|
||
)
|
||
_ABSTENTION_MARKERS = (
|
||
"not determinable",
|
||
"cannot be determined",
|
||
"cannot determine",
|
||
"not possible to determine",
|
||
"no information",
|
||
"not available",
|
||
"does not exist",
|
||
"do not have",
|
||
"don't have",
|
||
"i cannot",
|
||
"i can't",
|
||
"i won't",
|
||
"i will not",
|
||
"unable to",
|
||
"i do not know",
|
||
"i don't know",
|
||
"fabricated",
|
||
"no such",
|
||
"decline",
|
||
"refuse",
|
||
"not answerable",
|
||
"beyond my",
|
||
"knowledge cutoff",
|
||
"physically impossible",
|
||
"inconsistent",
|
||
"contradictory",
|
||
"not verifiable",
|
||
)
|
||
|
||
_SENTENCE_SPLIT = re.compile(r"(?<=[.!?:])\s+|\n+")
|
||
_NUMBERED_LINE = re.compile(r"^\s{0,6}(\d{1,2})[.)]\s+\S", re.MULTILINE)
|
||
# PART/STAGE label deliverables. SECTION is deliberately excluded: prompts use
|
||
# it to label supplied source material, which is not something to answer.
|
||
_LABELLED_SECTION = re.compile(r"^\s*(?:PART|STAGE)\s+([A-Z]|\d{1,2})\b", re.MULTILINE)
|
||
_HEADING_LINE = re.compile(r"^\s*(?:#{1,6}\s+\S|\*\*[^*\n]{2,60}\*\*\s*:?\s*$)", re.MULTILINE)
|
||
# Prompt-side matching stays line-anchored. Response-side is deliberately
|
||
# looser: it also accepts a marker after a sentence break (models run numbered
|
||
# answers together in a paragraph) and one wrapped in markdown emphasis or a
|
||
# heading — "**1. …**" and "### 1. …" are how most models number their sections,
|
||
# and missing those undercounts coverage badly.
|
||
_RESPONSE_MARKER = re.compile(r"(?:^|[\n.!?]\s*)[*_#>\s-]{0,8}(\d{1,2})[.)]\s+\S", re.MULTILINE)
|
||
_FENCE = re.compile(r"```")
|
||
|
||
|
||
def _sentences(text: str) -> List[str]:
|
||
return [s for s in _SENTENCE_SPLIT.split(text) if s.strip()]
|
||
|
||
|
||
def _is_negated(sentence: str) -> bool:
|
||
lowered = sentence.lower()
|
||
return bool(_NEGATION_RE.search(lowered)) or any(p in lowered for p in _NEGATION_PHRASES)
|
||
|
||
|
||
def _literal_requirements(prompt: str) -> Tuple[List[str], List[str]]:
|
||
"""Split backticked and short-quoted spans into (required, forbidden)."""
|
||
required: List[str] = []
|
||
forbidden: List[str] = []
|
||
|
||
for sentence in _sentences(prompt):
|
||
bucket = forbidden if _is_negated(sentence) else required
|
||
|
||
for raw in _BACKTICK.findall(sentence):
|
||
symbol = raw.strip()
|
||
if _usable_literal(symbol):
|
||
_add(bucket, symbol)
|
||
|
||
for raw in _QUOTED.findall(sentence):
|
||
symbol = raw.strip().rstrip(".,;:")
|
||
if " " in symbol or symbol.lower() in _QUOTED_STOPWORDS:
|
||
continue # prose being discussed, not a required output token
|
||
if _usable_literal(symbol):
|
||
_add(bucket, symbol)
|
||
|
||
# A literal demanded in one place and banned in another is ambiguous.
|
||
contested = set(required) & set(forbidden)
|
||
return (
|
||
[s for s in required if s not in contested],
|
||
[s for s in forbidden if s not in contested],
|
||
)
|
||
|
||
|
||
def _usable_literal(symbol: str) -> bool:
|
||
"""Reject punctuation-only spans and single characters.
|
||
|
||
Single characters matter: ``must not contain the letter "z"`` would
|
||
otherwise ban every 'z' in the answer, including the one in the sentence
|
||
explaining the rule.
|
||
"""
|
||
if len(symbol) < 2:
|
||
return False
|
||
return any(ch.isalnum() or ch in "@\\/_" for ch in symbol)
|
||
|
||
|
||
def _add(bucket: List[str], symbol: str) -> None:
|
||
if symbol not in bucket:
|
||
bucket.append(symbol)
|
||
|
||
|
||
def _requires_abstention(prompt: str) -> bool:
|
||
"""True only for an *unconditional* demand to refuse or admit ignorance."""
|
||
if not _ABSTENTION_TOPIC.search(prompt):
|
||
return False
|
||
return any(
|
||
_ABSTENTION_EXPECTED.search(s) and not _CONDITIONAL.search(s) for s in _sentences(prompt)
|
||
)
|
||
|
||
|
||
def _enumerated_demands(prompt: str) -> int:
|
||
"""How many numbered deliverables the prompt asks for.
|
||
|
||
Counts the longest run starting at 1, so an inline reference to "step 3"
|
||
does not inflate the total.
|
||
"""
|
||
seen = {int(n) for n in _NUMBERED_LINE.findall(prompt)}
|
||
count = 0
|
||
while count + 1 in seen:
|
||
count += 1
|
||
|
||
# Some prompts structure their deliverables as "PART A / PART B" or
|
||
# "STAGE 1 / STAGE 2" instead of a numbered list.
|
||
labelled = len({m.upper() for m in _LABELLED_SECTION.findall(prompt)})
|
||
return max(count, labelled)
|
||
|
||
|
||
def _word_budget(prompt: str) -> Optional[Tuple[int, int]]:
|
||
"""A (low, high) word range for the whole answer, or None if ambiguous.
|
||
|
||
Returns None when the prompt asks for several separately-counted outputs —
|
||
a per-section budget cannot be checked against the total.
|
||
"""
|
||
if _MULTI_OUTPUT.search(prompt):
|
||
return None
|
||
|
||
ranges: List[Tuple[int, int]] = []
|
||
for sentence in _sentences(prompt):
|
||
if "each" in sentence.lower():
|
||
continue # per-item, not global
|
||
for match in _RANGE_WORDS.finditer(sentence):
|
||
low, high = (int(g.replace(",", "")) for g in match.groups())
|
||
if low <= high:
|
||
ranges.append((low, high))
|
||
for match in _EXACT_WORDS.finditer(sentence):
|
||
exact = int(match.group(1).replace(",", ""))
|
||
ranges.append((exact, exact))
|
||
|
||
# Two different budgets in one prompt means neither is the global one.
|
||
unique = {r for r in ranges}
|
||
if len(unique) != 1:
|
||
return None
|
||
return ranges[0]
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Response inspection
|
||
# --------------------------------------------------------------------------
|
||
|
||
_TABLE_ROW = re.compile(r"^\s*\|.*\|\s*$", re.MULTILINE)
|
||
_TABLE_RULE = re.compile(r"^\s*\|[\s:|-]*-[\s:|-]*\|\s*$", re.MULTILINE)
|
||
|
||
|
||
def _has_table(text: str) -> bool:
|
||
return bool(_TABLE_RULE.search(text)) and len(_TABLE_ROW.findall(text)) >= 2
|
||
|
||
|
||
def _strip_fences(text: str) -> str:
|
||
return re.sub(r"```[a-zA-Z0-9]*\n?|```", "", text)
|
||
|
||
|
||
def _extract_json(text: str) -> Optional[object]:
|
||
"""Parse the outermost JSON object in a response, fences tolerated."""
|
||
candidate = _strip_fences(text).strip()
|
||
start, end = candidate.find("{"), candidate.rfind("}")
|
||
if start == -1 or end <= start:
|
||
return None
|
||
try:
|
||
return json.loads(candidate[start : end + 1])
|
||
except (json.JSONDecodeError, ValueError):
|
||
return None
|
||
|
||
|
||
def _word_count(text: str) -> int:
|
||
"""Words outside fenced code blocks — prose length, not payload length."""
|
||
prose = re.sub(r"```.*?```", " ", text, flags=re.DOTALL)
|
||
return len(re.findall(r"\b[\w'’-]+\b", prose))
|
||
|
||
|
||
def _answered_sections(response: str) -> int:
|
||
"""How many distinct deliverables the answer visibly separates out.
|
||
|
||
Matching is looser than on the prompt side: weaker models often run their
|
||
numbered answers together inside a paragraph ("1. Yes. 2. No.") rather than
|
||
on separate lines, and that should still count as having addressed them.
|
||
"""
|
||
numbered = {int(n) for n in _RESPONSE_MARKER.findall(response)}
|
||
consecutive = 0
|
||
while consecutive + 1 in numbered:
|
||
consecutive += 1
|
||
return max(consecutive, len(_HEADING_LINE.findall(response)))
|
||
|
||
|
||
def _repetition_ratio(text: str) -> float:
|
||
"""Fraction of distinct 6-grams. Low means the model is looping."""
|
||
tokens = re.findall(r"\w+", text.lower())
|
||
if len(tokens) < 60:
|
||
return 1.0
|
||
grams = [tuple(tokens[i : i + 6]) for i in range(len(tokens) - 5)]
|
||
return len(set(grams)) / len(grams)
|
||
|
||
|
||
def _looks_truncated(text: str) -> bool:
|
||
stripped = text.rstrip()
|
||
if not stripped:
|
||
return True
|
||
if stripped.count("```") % 2 == 1:
|
||
return True # unclosed code fence
|
||
return stripped[-1] not in ".!?)]}\"'`:*"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Grading
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def _build_checks(prompt: str, response: str) -> List[Check]:
|
||
checks: List[Check] = []
|
||
|
||
# --- quality/degeneration --------------------------------------------
|
||
ratio = _repetition_ratio(response)
|
||
truncated = _looks_truncated(response)
|
||
degeneration, notes = 1.0, []
|
||
if ratio < 0.5:
|
||
degeneration -= 0.6
|
||
notes.append(f"repetitive output ({ratio:.0%} distinct 6-grams)")
|
||
elif ratio < 0.7:
|
||
degeneration -= 0.25
|
||
notes.append(f"some looping ({ratio:.0%} distinct 6-grams)")
|
||
if truncated:
|
||
degeneration -= 0.3
|
||
notes.append("ends mid-sentence or leaves a code fence open")
|
||
checks.append(
|
||
Check("quality/degeneration", max(0.0, degeneration), 2.0, "; ".join(notes))
|
||
)
|
||
|
||
# --- quality/substance ------------------------------------------------
|
||
demands = _enumerated_demands(prompt)
|
||
words = _word_count(response)
|
||
# ~15 words per deliverable, capped: this guards one-line non-answers, and
|
||
# is not a reward for padding. A terse but complete answer must clear it.
|
||
expected = min(300, 15 * demands) if demands else 80
|
||
substance = min(1.0, words / expected) if expected else 1.0
|
||
checks.append(
|
||
Check(
|
||
"quality/substance",
|
||
substance,
|
||
1.0,
|
||
"" if substance >= 0.999 else f"{words} words for {demands or 'an open'} deliverable(s)",
|
||
)
|
||
)
|
||
|
||
# --- coverage/enumerated ---------------------------------------------
|
||
if demands >= 2:
|
||
answered = _answered_sections(response)
|
||
value = min(1.0, answered / demands)
|
||
checks.append(
|
||
Check(
|
||
"coverage/enumerated",
|
||
value,
|
||
2.0,
|
||
"" if value >= 0.999 else f"addressed {answered} of {demands} numbered parts",
|
||
)
|
||
)
|
||
|
||
# --- structure/json ---------------------------------------------------
|
||
if _JSON_REQUIRED.search(prompt):
|
||
parsed = _extract_json(response)
|
||
if parsed is None:
|
||
checks.append(Check("structure/json", 0.0, 2.0, "no parseable JSON object"))
|
||
else:
|
||
value, detail = 1.0, ""
|
||
if _JSON_ONLY.search(prompt):
|
||
bare = response.strip()
|
||
if not (bare.startswith("{") and bare.endswith("}")):
|
||
value, detail = 0.6, "JSON present but wrapped in prose or fences"
|
||
checks.append(Check("structure/json", value, 2.0, detail))
|
||
|
||
# --- structure/table --------------------------------------------------
|
||
if _TABLE_REQUIRED.search(prompt):
|
||
present = _has_table(response)
|
||
checks.append(
|
||
Check("structure/table", 1.0 if present else 0.0, 1.0, "" if present else "no markdown table")
|
||
)
|
||
|
||
# --- structure/code ---------------------------------------------------
|
||
fenced = bool(_FENCE.search(response))
|
||
if _NO_FENCE.search(prompt):
|
||
checks.append(
|
||
Check(
|
||
"structure/code",
|
||
0.0 if fenced else 1.0,
|
||
1.0,
|
||
"used a code fence the prompt forbade" if fenced else "",
|
||
)
|
||
)
|
||
elif _CODE_NOUNS.search(prompt):
|
||
checks.append(
|
||
Check("structure/code", 1.0 if fenced else 0.0, 1.0, "" if fenced else "no fenced code block")
|
||
)
|
||
|
||
# --- coverage/keyterms and constraint/forbidden -----------------------
|
||
required, forbidden = _literal_requirements(prompt)
|
||
if required:
|
||
hits = [s for s in required if s.lower() in response.lower()]
|
||
value = len(hits) / len(required)
|
||
missing = [s for s in required if s not in hits][:5]
|
||
checks.append(
|
||
Check(
|
||
"coverage/keyterms",
|
||
value,
|
||
1.5,
|
||
"" if value >= 0.999 else "missing " + ", ".join(f"`{m}`" for m in missing),
|
||
)
|
||
)
|
||
if forbidden:
|
||
used = [s for s in forbidden if s.lower() in response.lower()]
|
||
value = 1.0 - len(used) / len(forbidden)
|
||
checks.append(
|
||
Check(
|
||
"constraint/forbidden",
|
||
value,
|
||
1.5,
|
||
"" if not used else "used banned " + ", ".join(f"`{u}`" for u in used[:5]),
|
||
)
|
||
)
|
||
|
||
# --- constraint/length ------------------------------------------------
|
||
budget = _word_budget(prompt)
|
||
if budget:
|
||
low, high = budget
|
||
# 10% grace: a 402-word answer to a "400 word" prompt is compliant.
|
||
slack = max(2, round(low * 0.1))
|
||
if low - slack <= words <= high + slack:
|
||
checks.append(Check("constraint/length", 1.0, 1.0))
|
||
else:
|
||
target = f"{low}" if low == high else f"{low}-{high}"
|
||
over = words - high if words > high else low - words
|
||
value = max(0.0, 1.0 - over / max(low, 1))
|
||
checks.append(
|
||
Check("constraint/length", value, 1.0, f"{words} words against a {target}-word budget")
|
||
)
|
||
|
||
# --- behaviour/abstention ---------------------------------------------
|
||
if _requires_abstention(prompt):
|
||
lowered_response = response.lower()
|
||
found = [m for m in _ABSTENTION_MARKERS if m in lowered_response]
|
||
checks.append(
|
||
Check(
|
||
"behaviour/abstention",
|
||
1.0 if found else 0.0,
|
||
1.5,
|
||
"" if found else "prompt required a refusal or an admission of ignorance; none found",
|
||
)
|
||
)
|
||
|
||
return checks
|
||
|
||
|
||
def grade(test: TestRecord):
|
||
response = (test.response or "").strip()
|
||
if not response:
|
||
return None # nothing to judge
|
||
|
||
checks = _build_checks(test.prompt, response)
|
||
if not checks:
|
||
return None
|
||
|
||
total_weight = sum(c.weight for c in checks)
|
||
score = sum(c.value * c.weight for c in checks) / total_weight
|
||
|
||
failures = [f"{c.name}: {c.detail or 'failed'}" for c in checks if not c.passed]
|
||
passed = sum(1 for c in checks if c.passed)
|
||
|
||
return Grade(
|
||
score=score,
|
||
label=f"{passed}/{len(checks)} checks",
|
||
notes="; ".join(failures) if failures else "All derived constraints met.",
|
||
)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Reporting
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def _checks_for(run: RunRecord) -> Dict[int, List[Check]]:
|
||
"""Recompute checks per question so the report can break them down."""
|
||
detail: Dict[int, List[Check]] = {}
|
||
for test in run.tests:
|
||
if test.ok and test.response:
|
||
detail[test.index] = _build_checks(test.prompt, test.response.strip())
|
||
return detail
|
||
|
||
|
||
def report_sections(run: RunRecord):
|
||
detail = _checks_for(run)
|
||
if not detail:
|
||
return None
|
||
|
||
# Which dimensions the model struggled with, across the whole run.
|
||
totals: Dict[str, List[float]] = {}
|
||
for checks in detail.values():
|
||
for check in checks:
|
||
totals.setdefault(check.name, []).append(check.value)
|
||
|
||
summary = [
|
||
"Every score below is derived from constraints the prompt states "
|
||
"explicitly. A low score means the answer did not do what it was told, "
|
||
"which is not the same as being wrong — read the answer before treating "
|
||
"it as a failure.",
|
||
"",
|
||
"| Check | Questions | Mean | Failed outright |",
|
||
"| --- | --- | --- | --- |",
|
||
]
|
||
for name in sorted(totals, key=lambda n: sum(totals[n]) / len(totals[n])):
|
||
values = totals[name]
|
||
mean = sum(values) / len(values)
|
||
zeros = sum(1 for v in values if v < 0.001)
|
||
summary.append(f"| `{name}` | {len(values)} | {mean:.0%} | {zeros} |")
|
||
|
||
misses = []
|
||
for test in run.tests:
|
||
for entry in run.grades.get(test.index, []):
|
||
if entry.grader == NAME and entry.grade.score < 1.0:
|
||
misses.append((test, entry.grade))
|
||
|
||
blocks = [("## Compliance by check", "\n".join(summary))]
|
||
|
||
if misses:
|
||
lines = [
|
||
"| # | Question | Score | What was missed |",
|
||
"| --- | --- | --- | --- |",
|
||
]
|
||
for test, grade_ in misses:
|
||
problem = grade_.notes.replace("|", "\\|").replace("\n", " ")
|
||
title = test.title.replace("|", "\\|")
|
||
lines.append(f"| {test.index} | {title} | {grade_.score:.0%} | {problem} |")
|
||
blocks.append(("## Questions with unmet constraints", "\n".join(lines)))
|
||
|
||
return blocks
|