Netherwarlord
f77f0af33a
- Introduced `test_answer_key.py` to validate the correctness of answer keys against expected responses, ensuring both false and true answers are graded appropriately. - Created `test_answer_values.py` to verify the accuracy of answer keys by recomputing values and comparing them against the provided `answers.json`. - Implemented `test_api.py` to test the HTTP API for suite management, ensuring read-only enforcement and proper handling of custom suites. - Added `test_code_lint.py` to perform static code analysis, ensuring that all linting rules are enforced without false positives on correct answers. - Developed `test_collision.py` to guard against prompt mismatches due to filename collisions across suites. - Created `test_gguf.py` to validate GGUF header parsing and model classification, including handling of malformed input. - Introduced `test_linter.py` to ensure report rendering preserves content integrity and correctly fences code. - Added `test_reporting.py` to verify report naming conventions and the accuracy of per-suite score breakdowns.
72 lines
2.1 KiB
Python
72 lines
2.1 KiB
Python
"""Run every Python test and summarise. No third-party dependencies.
|
|
|
|
python tests_py/run_all.py # everything
|
|
python tests_py/run_all.py gguf # only files matching "gguf"
|
|
|
|
Each test is a standalone script run in its own process, so one crashing
|
|
cannot take the rest with it, and each starts from a clean import state —
|
|
several load plugins directly, which would otherwise collide in sys.modules.
|
|
|
|
Exits non-zero if any test fails, so this works as a CI entry point.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
|
|
|
|
def main() -> int:
|
|
wanted = sys.argv[1:]
|
|
tests = sorted(p for p in HERE.glob("test_*.py"))
|
|
if wanted:
|
|
tests = [p for p in tests if any(w in p.stem for w in wanted)]
|
|
if not tests:
|
|
print("no matching tests")
|
|
return 1
|
|
|
|
results: list[tuple[str, int, str]] = []
|
|
for path in tests:
|
|
proc = subprocess.run(
|
|
[sys.executable, str(path)],
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
cwd=str(HERE),
|
|
)
|
|
output = (proc.stdout or "") + (proc.stderr or "")
|
|
print(output.rstrip())
|
|
|
|
# The last "N/M passed" line, or a SKIP notice, is the verdict.
|
|
summary = next(
|
|
(line.strip() for line in reversed(output.splitlines())
|
|
if "passed" in line or line.strip().startswith("SKIP")),
|
|
"no summary",
|
|
)
|
|
results.append((path.stem, proc.returncode, summary))
|
|
|
|
print("\n" + "=" * 60)
|
|
print("SUMMARY")
|
|
print("=" * 60)
|
|
failed = 0
|
|
for name, code, summary in results:
|
|
if "SKIP" in summary:
|
|
status = "SKIP"
|
|
elif code == 0:
|
|
status = "ok"
|
|
else:
|
|
status = "FAIL"
|
|
failed += 1
|
|
print(f" {status:<5} {name:<22} {summary}")
|
|
|
|
print(f"\n{len(results) - failed}/{len(results)} files ok")
|
|
return 1 if failed else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|