Netherwarlord
f77f0af33a
- Introduced `test_answer_key.py` to validate the correctness of answer keys against expected responses, ensuring both false and true answers are graded appropriately. - Created `test_answer_values.py` to verify the accuracy of answer keys by recomputing values and comparing them against the provided `answers.json`. - Implemented `test_api.py` to test the HTTP API for suite management, ensuring read-only enforcement and proper handling of custom suites. - Added `test_code_lint.py` to perform static code analysis, ensuring that all linting rules are enforced without false positives on correct answers. - Developed `test_collision.py` to guard against prompt mismatches due to filename collisions across suites. - Created `test_gguf.py` to validate GGUF header parsing and model classification, including handling of malformed input. - Introduced `test_linter.py` to ensure report rendering preserves content integrity and correctly fences code. - Added `test_reporting.py` to verify report naming conventions and the accuracy of per-suite score breakdowns.
121 lines
5.0 KiB
Python
121 lines
5.0 KiB
Python
"""The suites HTTP API, including every read-only guard.
|
|
|
|
Needs a running server (``python app.py``) and the ``requests`` package, which
|
|
the LM Studio provider already depends on. Skips cleanly if the server is not
|
|
up rather than reporting a failure, so ``run_all.py`` stays useful offline.
|
|
|
|
Read-only enforcement is asserted at the API, not in the UI: a disabled button
|
|
is a courtesy, a 409 is the actual protection.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import shutil
|
|
import sys
|
|
|
|
from _harness import ROOT, Results, server_available
|
|
|
|
BASE = "http://localhost:8765"
|
|
|
|
unavailable = server_available(BASE)
|
|
if unavailable:
|
|
print(f"\nSKIP suites API — {unavailable}")
|
|
print(" start the server with 'python app.py' to run these.")
|
|
raise SystemExit(0)
|
|
|
|
import requests # noqa: E402 (guarded by server_available above)
|
|
|
|
r = Results("Suites API")
|
|
|
|
|
|
def get(path):
|
|
return requests.get(BASE + path, timeout=20)
|
|
|
|
|
|
def post(path, **kw):
|
|
return requests.post(BASE + path, timeout=20, **kw)
|
|
|
|
|
|
def put(path, **kw):
|
|
return requests.put(BASE + path, timeout=20, **kw)
|
|
|
|
|
|
def delete(path):
|
|
return requests.delete(BASE + path, timeout=20)
|
|
|
|
|
|
# Clear anything a previous interrupted run left behind.
|
|
for leftover in ("scratch-suite", "language-copy", "renamed-suite"):
|
|
shutil.rmtree(ROOT / "tests" / leftover, ignore_errors=True)
|
|
|
|
r.section("listing")
|
|
listing = get("/api/suites")
|
|
suites = listing.json()
|
|
r.equal("GET /api/suites", listing.status_code, 200)
|
|
r.check("five built-in suites", len(suites) == 5 and all(s["builtin"] for s in suites),
|
|
[s["slug"] for s in suites])
|
|
r.equal("question counts", [s["count"] for s in suites], [6, 6, 6, 4, 4])
|
|
|
|
detail = get("/api/suites/math-code")
|
|
r.equal("GET one suite", detail.status_code, 200)
|
|
r.check("question ids are qualified",
|
|
all(t["id"].startswith("math-code/") for t in detail.json()["tests"]))
|
|
r.equal("unknown suite is 404", get("/api/suites/does-not-exist").status_code, 404)
|
|
|
|
r.section("built-ins are read-only, enforced by the API")
|
|
r.equal("rename", put("/api/suites/safety", json={"name": "X"}).status_code, 409)
|
|
r.equal("delete", delete("/api/suites/safety").status_code, 409)
|
|
r.equal("replace questions",
|
|
put("/api/suites/safety/tests", json={"tests": [{"prompt": "hi"}]}).status_code, 409)
|
|
|
|
r.section("the legacy /api/tests alias is gone")
|
|
r.equal("GET", get("/api/tests").status_code, 404)
|
|
r.check("PUT", put("/api/tests", json={"tests": [{"prompt": "x"}]}).status_code in (404, 405))
|
|
|
|
r.section("custom suite lifecycle")
|
|
created = post("/api/suites", json={"name": "Scratch Suite", "description": "d"})
|
|
r.equal("create", created.status_code, 201)
|
|
slug = created.json()["slug"] if created.status_code == 201 else "scratch-suite"
|
|
r.equal("name is slugified", slug, "scratch-suite")
|
|
r.equal("a reserved slug is refused",
|
|
post("/api/suites", json={"name": "Safety", "slug": "safety"}).status_code, 400)
|
|
r.equal("a duplicate name is refused",
|
|
post("/api/suites", json={"name": "Scratch Suite"}).status_code, 400)
|
|
|
|
saved = put(f"/api/suites/{slug}/tests", json={"tests": [{"prompt": "Q one"}, {"prompt": "Q two"}]})
|
|
r.equal("save questions", saved.status_code, 200)
|
|
r.equal("two questions stored", len(saved.json()["tests"]), 2)
|
|
r.equal("a whitespace-only question is refused",
|
|
put(f"/api/suites/{slug}/tests",
|
|
json={"tests": [{"prompt": "a"}, {"prompt": " "}]}).status_code, 400)
|
|
r.equal("an empty-string question is refused",
|
|
put(f"/api/suites/{slug}/tests",
|
|
json={"tests": [{"prompt": "a"}, {"prompt": ""}]}).status_code, 422)
|
|
r.equal("rename", put(f"/api/suites/{slug}", json={"name": "Renamed Suite"}).status_code, 200)
|
|
|
|
r.section("duplicate a built-in into an editable copy")
|
|
copy = post("/api/suites/language/duplicate", json={"name": "Language Copy"})
|
|
r.equal("duplicate", copy.status_code, 201)
|
|
copy_slug = copy.json()["slug"] if copy.status_code == 201 else "language-copy"
|
|
r.check("the copy is custom", copy.json().get("builtin") is False)
|
|
r.equal("all six questions copied", len(copy.json().get("tests", [])), 6)
|
|
r.equal("the copy is editable",
|
|
put(f"/api/suites/{copy_slug}/tests", json={"tests": [{"prompt": "edited"}]}).status_code, 200)
|
|
r.check("the original is untouched", get("/api/suites/language").json()["count"] == 6)
|
|
|
|
r.section("run selection")
|
|
bare = post("/api/runs", json={"provider": "Local Engine", "model_id": "x",
|
|
"filenames": ["test1.txt"]})
|
|
r.check("a bare filename is refused as ambiguous",
|
|
bare.status_code == 400 and "ambiguous" in bare.text, bare.text[:110])
|
|
r.equal("an unknown suite is refused",
|
|
post("/api/runs", json={"provider": "Local Engine", "model_id": "x",
|
|
"selections": [{"suite": "nope"}]}).status_code, 400)
|
|
|
|
r.section("cleanup")
|
|
r.equal("delete the scratch suite", delete(f"/api/suites/{slug}").status_code, 204)
|
|
r.equal("delete the copy", delete(f"/api/suites/{copy_slug}").status_code, 204)
|
|
r.equal("back to five suites", len(get("/api/suites").json()), 5)
|
|
|
|
sys.exit(r.finish())
|