feat: add testing framework and initial test cases
- Updated package.json to include Vitest and testing dependencies. - Created test cases for SuitePicker component to validate selection logic. - Added tests for ReportsPage to ensure correct report grouping and ordering. - Implemented read-only enforcement tests for SuitePage to prevent actions on built-in suites. - Introduced a setup file for Vitest to include jest-dom matchers and cleanup after tests. - Modified ReportsPage to group reports by model and display them accordingly. - Enhanced API types to include suite_scope and question_count for better report handling.
This commit is contained in:
Netherwarlord
+138
-98
@@ -1,11 +1,36 @@
|
||||
"""Tidy a model's answer for display in the report.
|
||||
|
||||
This only ever affects the **report**. Graders receive the raw provider output,
|
||||
never anything from this module — see ``reporting.render_response_block``, the
|
||||
sole caller.
|
||||
|
||||
The module exists because early models emitted code as bare indented text with
|
||||
no markdown fences, which rendered as an unreadable wall. Two things have
|
||||
changed since: models now fence their own code reliably, and the question suite
|
||||
is mostly prose, mathematics and JSON rather than code.
|
||||
|
||||
That makes aggressive auto-fencing a liability. The previous implementation
|
||||
classified **line by line** and started a new fence whenever the classification
|
||||
flipped, so a bare JSON object — indented lines alternating with unindented
|
||||
ones — came out shredded across several bogus code blocks, and any sentence
|
||||
containing parentheses was fenced as if it were source.
|
||||
|
||||
So the rule now is conservative: an answer is either wholly code or wholly not,
|
||||
and anything ambiguous is left exactly as the model wrote it. A report that
|
||||
under-formats is honest; one that mangles the answer is worse than useless,
|
||||
because it misrepresents what the model actually produced.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Iterable, List, Tuple
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
__all__ = ["infer_language_from_prompt", "lint_response_markdown"]
|
||||
|
||||
|
||||
# Ordered: the first match wins, so "swiftui" must precede "swift".
|
||||
_LANGUAGE_HINTS: List[Tuple[str, str]] = [
|
||||
("swiftui", "swift"),
|
||||
("swift", "swift"),
|
||||
@@ -24,54 +49,64 @@ _LANGUAGE_HINTS: List[Tuple[str, str]] = [
|
||||
("sql", "sql"),
|
||||
]
|
||||
|
||||
_CODE_SYMBOLS = set("{}();[]=<>+-*/.&|%!:@")
|
||||
_CODE_PREFIXES = (
|
||||
"func ",
|
||||
"class ",
|
||||
"struct ",
|
||||
"enum ",
|
||||
"protocol ",
|
||||
"extension ",
|
||||
"import ",
|
||||
"let ",
|
||||
"var ",
|
||||
"guard ",
|
||||
"if ",
|
||||
"for ",
|
||||
"while ",
|
||||
"switch ",
|
||||
"case ",
|
||||
"return ",
|
||||
"init(",
|
||||
"init ",
|
||||
"deinit",
|
||||
"public ",
|
||||
"private ",
|
||||
"internal ",
|
||||
"fileprivate ",
|
||||
"override ",
|
||||
"@",
|
||||
"#if",
|
||||
"#endif",
|
||||
"#warning",
|
||||
"#error",
|
||||
)
|
||||
#: Language names that are also ordinary English words. "Do not go back and
|
||||
#: edit Stage 1" is not a request for Go, and "a swift response" is not Swift.
|
||||
#: These match only capitalised, which is how the language is written in
|
||||
#: practice, or via an unambiguous ``-lang`` alias.
|
||||
_AMBIGUOUS = {"go", "rust", "swift"}
|
||||
|
||||
_COMMENT_PREFIXES = ("///", "//", "/*", "*/", "* ")
|
||||
_BULLET_PREFIXES = ("- ", "* ", "+ ", "• ")
|
||||
#: Whole-word matchers for the table above.
|
||||
#:
|
||||
#: Substring matching was silently wrong in a way that only showed on prose:
|
||||
#: "algorithm", "ago" and "cargo" all contain "go", and "rusty" contains
|
||||
#: "rust", so an arithmetic question was labelled as Go source. Tokens carrying
|
||||
#: punctuation (``c++``, ``c#``, ``node.js``) cannot take a trailing ``\b``,
|
||||
#: since ``\b`` needs a word character on the inside of the boundary.
|
||||
_LANGUAGE_PATTERNS: List[Tuple[re.Pattern, str]] = []
|
||||
for _token, _language in _LANGUAGE_HINTS:
|
||||
_lead = r"\b" if _token[0].isalnum() else ""
|
||||
_trail = r"\b" if _token[-1].isalnum() else ""
|
||||
if _token in _AMBIGUOUS:
|
||||
_body = f"(?:{re.escape(_token.capitalize())}|{re.escape(_token)}lang)"
|
||||
_LANGUAGE_PATTERNS.append((re.compile(_lead + _body + _trail), _language))
|
||||
else:
|
||||
_LANGUAGE_PATTERNS.append(
|
||||
(re.compile(_lead + re.escape(_token) + _trail, re.IGNORECASE), _language)
|
||||
)
|
||||
|
||||
|
||||
_CODE_SYMBOLS = set("{}();[]=<>+-*/&|%!@")
|
||||
_CODE_PREFIXES = (
|
||||
"func ", "def ", "class ", "struct ", "enum ", "protocol ", "extension ",
|
||||
"import ", "from ", "let ", "var ", "const ", "guard ", "switch ",
|
||||
"public ", "private ", "internal ", "fileprivate ", "override ",
|
||||
"async ", "await ", "@", "#if", "#endif", "#include", "#define",
|
||||
)
|
||||
_COMMENT_PREFIXES = ("///", "//", "/*", "*/")
|
||||
_BULLET_PREFIXES = ("- ", "* ", "+ ", "• ", "> ", "#")
|
||||
|
||||
#: Share of non-blank lines that must look like code before the whole answer is
|
||||
#: fenced. Deliberately high: the cost of missing a fence is cosmetic, the cost
|
||||
#: of fencing prose is a report that lies about the answer.
|
||||
_CODE_RATIO = 0.85
|
||||
_MIN_CODE_LINES = 2
|
||||
|
||||
|
||||
def infer_language_from_prompt(prompt_text: str) -> str:
|
||||
"""Infer a reasonable language hint from the prompt text."""
|
||||
lowered = prompt_text.lower()
|
||||
for token, language in _LANGUAGE_HINTS:
|
||||
if token in lowered:
|
||||
"""Infer a language hint from the prompt, or ``"text"`` when unsure.
|
||||
|
||||
Only used to label fences the model left unlabelled, so a wrong answer is
|
||||
cosmetic — but it should not be confidently wrong on a question with no
|
||||
code in it at all.
|
||||
"""
|
||||
for pattern, language in _LANGUAGE_PATTERNS:
|
||||
if pattern.search(prompt_text):
|
||||
return language
|
||||
return "text"
|
||||
|
||||
|
||||
def lint_response_markdown(raw_text: str, *, language_hint: str = "text") -> str:
|
||||
"""Normalize markdown so that only code segments are fenced and text stays plain."""
|
||||
"""Normalise an answer's markdown without changing what it says."""
|
||||
if not raw_text:
|
||||
return ""
|
||||
|
||||
@@ -82,10 +117,11 @@ def lint_response_markdown(raw_text: str, *, language_hint: str = "text") -> str
|
||||
if "```" in text:
|
||||
return _normalize_existing_fences(text, language_hint)
|
||||
|
||||
return _auto_fence_code_segments(text, language_hint)
|
||||
return _wrap_if_wholly_code(text, language_hint)
|
||||
|
||||
|
||||
def _normalize_existing_fences(text: str, language_hint: str) -> str:
|
||||
"""Even up fence markers and label unlabelled ones. Content is untouched."""
|
||||
lines = text.split("\n")
|
||||
output: List[str] = []
|
||||
in_fence = False
|
||||
@@ -103,67 +139,75 @@ def _normalize_existing_fences(text: str, language_hint: str) -> str:
|
||||
else:
|
||||
output.append(line)
|
||||
|
||||
# A model that runs out of tokens mid-block leaves the fence open, which
|
||||
# would swallow everything rendered after it.
|
||||
if in_fence:
|
||||
output.append("```")
|
||||
|
||||
result = "\n".join(output).strip()
|
||||
return result
|
||||
return "\n".join(output).strip()
|
||||
|
||||
|
||||
def _auto_fence_code_segments(text: str, language_hint: str) -> str:
|
||||
lines = text.split("\n")
|
||||
segments: List[Tuple[str, List[str]]] = []
|
||||
current_lines: List[str] = []
|
||||
current_mode: str | None = None
|
||||
def _wrap_if_wholly_code(text: str, language_hint: str) -> str:
|
||||
"""Fence the answer only if *all* of it is code. Otherwise return it as-is.
|
||||
|
||||
for line in lines:
|
||||
if _is_blank(line):
|
||||
if current_lines:
|
||||
current_lines.append(line)
|
||||
continue
|
||||
Never emits more than one fence and never interleaves fenced and unfenced
|
||||
runs — that interleaving is precisely what shredded JSON answers before.
|
||||
"""
|
||||
as_json = _as_json_block(text)
|
||||
if as_json is not None:
|
||||
return as_json
|
||||
|
||||
classification = "code" if _is_likely_code_line(line) else "text"
|
||||
lines = [line for line in text.split("\n") if line.strip()]
|
||||
if len(lines) < _MIN_CODE_LINES:
|
||||
return text
|
||||
|
||||
if current_mode is None:
|
||||
current_mode = classification
|
||||
current_lines.append(line)
|
||||
continue
|
||||
code_lines = [line for line in lines if _is_likely_code_line(line)]
|
||||
if len(code_lines) / len(lines) < _CODE_RATIO:
|
||||
return text
|
||||
|
||||
if classification == current_mode:
|
||||
current_lines.append(line)
|
||||
continue
|
||||
# Ratio alone is not enough: a dense block of symbol-heavy prose can clear
|
||||
# it. Require at least one unambiguous structural marker as well.
|
||||
if not any(_has_strong_code_signal(line) for line in code_lines):
|
||||
return text
|
||||
|
||||
segments.append((current_mode, current_lines))
|
||||
current_mode = classification
|
||||
current_lines = [line]
|
||||
|
||||
if current_lines:
|
||||
segments.append((current_mode or "text", current_lines))
|
||||
|
||||
cleaned_parts: List[str] = []
|
||||
for mode, block_lines in segments:
|
||||
block_text = "\n".join(block_lines).strip("\n")
|
||||
if not block_text:
|
||||
continue
|
||||
if mode == "code":
|
||||
cleaned_parts.append(_format_code_block(block_text, language_hint))
|
||||
else:
|
||||
cleaned_parts.append(block_text)
|
||||
|
||||
return "\n\n".join(part for part in cleaned_parts if part).strip()
|
||||
|
||||
|
||||
def _format_code_block(content: str, language_hint: str) -> str:
|
||||
language = language_hint if language_hint and language_hint != "text" else ""
|
||||
inner = content.strip("\n")
|
||||
return f"```{language}\n{inner}\n```"
|
||||
return f"```{language}\n{text.strip()}\n```"
|
||||
|
||||
|
||||
def _is_blank(line: str) -> bool:
|
||||
return not line.strip()
|
||||
def _as_json_block(text: str) -> Optional[str]:
|
||||
"""One ```json fence when the answer is a single JSON value, else None.
|
||||
|
||||
Several questions ask for bare JSON with no fence, and the model complies.
|
||||
Treating that as an unfenced blob is what caused it to be split apart.
|
||||
"""
|
||||
candidate = text.strip()
|
||||
if not candidate.startswith(("{", "[")):
|
||||
return None
|
||||
try:
|
||||
json.loads(candidate)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return None
|
||||
return f"```json\n{candidate}\n```"
|
||||
|
||||
|
||||
def _has_strong_code_signal(line: str) -> bool:
|
||||
stripped = line.strip()
|
||||
return (
|
||||
any(stripped.startswith(prefix) for prefix in _CODE_PREFIXES)
|
||||
or stripped.startswith(_COMMENT_PREFIXES)
|
||||
or stripped.endswith(("{", "}", ";", ":"))
|
||||
or bool(re.search(r"\)\s*(->|=>|\{)", stripped))
|
||||
)
|
||||
|
||||
|
||||
def _is_likely_code_line(line: str) -> bool:
|
||||
"""Whether one line looks like source.
|
||||
|
||||
Two rules were removed from the original: "contains both parentheses" and
|
||||
"contains an equals sign". Both fire on ordinary English — "the vote (a
|
||||
delay)", "x = 5 in the worked example" — and between them they were enough
|
||||
to fence plain prose as code.
|
||||
"""
|
||||
if line.startswith((" ", "\t")):
|
||||
return True
|
||||
|
||||
@@ -171,8 +215,11 @@ def _is_likely_code_line(line: str) -> bool:
|
||||
if not stripped:
|
||||
return False
|
||||
|
||||
# Markdown structure is prose, whatever punctuation it carries.
|
||||
if stripped.startswith(_BULLET_PREFIXES):
|
||||
return False
|
||||
if re.match(r"^\d+[.)]\s", stripped):
|
||||
return False
|
||||
|
||||
if stripped.startswith(_COMMENT_PREFIXES):
|
||||
return True
|
||||
@@ -183,21 +230,14 @@ def _is_likely_code_line(line: str) -> bool:
|
||||
if stripped.endswith(("{", "}", ";")):
|
||||
return True
|
||||
|
||||
if stripped.startswith(("}", "{", "case ", "default:")):
|
||||
if stripped.startswith(("}", "{")):
|
||||
return True
|
||||
|
||||
if "(" in stripped and ")" in stripped:
|
||||
return True
|
||||
|
||||
if "=" in stripped:
|
||||
if re.search(r"\)\s*(->|=>)", stripped):
|
||||
return True
|
||||
|
||||
# Symbol density, as a last resort. Needs to be genuinely dense: prose with
|
||||
# a couple of brackets should not qualify.
|
||||
symbol_count = sum(1 for ch in stripped if ch in _CODE_SYMBOLS)
|
||||
letter_count = sum(1 for ch in stripped if ch.isalpha())
|
||||
if symbol_count >= 2 and symbol_count >= max(1, letter_count * 0.3):
|
||||
return True
|
||||
|
||||
if re.search(r"\)\s*->", stripped):
|
||||
return True
|
||||
|
||||
return False
|
||||
return symbol_count >= 3 and symbol_count >= max(2, letter_count * 0.5)
|
||||
|
||||
Reference in New Issue
Block a user