Netherwarlord
f77f0af33a
- Introduced `test_answer_key.py` to validate the correctness of answer keys against expected responses, ensuring both false and true answers are graded appropriately. - Created `test_answer_values.py` to verify the accuracy of answer keys by recomputing values and comparing them against the provided `answers.json`. - Implemented `test_api.py` to test the HTTP API for suite management, ensuring read-only enforcement and proper handling of custom suites. - Added `test_code_lint.py` to perform static code analysis, ensuring that all linting rules are enforced without false positives on correct answers. - Developed `test_collision.py` to guard against prompt mismatches due to filename collisions across suites. - Created `test_gguf.py` to validate GGUF header parsing and model classification, including handling of malformed input. - Introduced `test_linter.py` to ensure report rendering preserves content integrity and correctly fences code. - Added `test_reporting.py` to verify report naming conventions and the accuracy of per-suite score breakdowns.
96 lines
4.0 KiB
Python
96 lines
4.0 KiB
Python
"""GGUF header parsing and model classification, including malformed input.
|
|
|
|
Guards the model picker: vision projectors and embedding models are valid GGUF
|
|
files that cannot answer a prompt, and selecting one used to produce an empty
|
|
report with no error at all.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import struct
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
from _harness import Results, bootstrap
|
|
|
|
bootstrap()
|
|
from gguf import ( # noqa: E402
|
|
EMBEDDING, GENERATIVE, MAGIC, PROJECTOR, UNKNOWN,
|
|
architecture, classify, read_metadata,
|
|
)
|
|
|
|
r = Results("GGUF parsing and classification")
|
|
|
|
|
|
def synth(pairs, tmp: str, magic: int = MAGIC) -> Path:
|
|
"""Write a minimal GGUF file with the given (key, type, value) triples."""
|
|
out = bytearray(struct.pack("<IIQQ", magic, 3, 0, len(pairs)))
|
|
for key, vtype, value in pairs:
|
|
kb = key.encode()
|
|
out += struct.pack("<Q", len(kb)) + kb + struct.pack("<I", vtype)
|
|
if vtype == 8: # string
|
|
vb = value.encode()
|
|
out += struct.pack("<Q", len(vb)) + vb
|
|
elif vtype == 4: # uint32
|
|
out += struct.pack("<I", value)
|
|
elif vtype == 9: # array of uint32
|
|
out += struct.pack("<IQ", 4, len(value))
|
|
for item in value:
|
|
out += struct.pack("<I", item)
|
|
path = Path(tmp) / f"{abs(hash(str(pairs))) % 10**8}.gguf"
|
|
path.write_bytes(bytes(out))
|
|
return path
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
r.section("classification from metadata, not filename")
|
|
generative = synth([("general.architecture", 8, "llama"),
|
|
("llama.block_count", 4, 32)], tmp)
|
|
r.equal("generative model", classify(generative), GENERATIVE)
|
|
|
|
embedding = synth([("general.architecture", 8, "bert"),
|
|
("bert.block_count", 4, 12),
|
|
("bert.pooling_type", 4, 1)], tmp)
|
|
r.equal("embedding model, detected by pooling_type", classify(embedding), EMBEDDING)
|
|
|
|
projector = synth([("general.architecture", 8, "clip")], tmp)
|
|
r.equal("vision projector, detected by clip architecture", classify(projector), PROJECTOR)
|
|
|
|
r.section("value skipping")
|
|
with_array = synth([("general.architecture", 8, "llama"),
|
|
("llama.some_list", 9, [1, 2, 3, 4, 5]),
|
|
("llama.block_count", 4, 32)], tmp)
|
|
r.equal("a key after an array is still reachable",
|
|
read_metadata(with_array, ("llama.block_count",)).get("llama.block_count"), 32)
|
|
|
|
r.section("malformed input degrades, never raises")
|
|
r.equal("wrong magic", classify(synth([("general.architecture", 8, "llama")], tmp,
|
|
magic=0xDEADBEEF)), UNKNOWN)
|
|
|
|
truncated = Path(tmp) / "truncated.gguf"
|
|
truncated.write_bytes(struct.pack("<IIQQ", MAGIC, 3, 0, 99) + b"\x05\x00")
|
|
r.equal("truncated mid-header", classify(truncated), UNKNOWN)
|
|
|
|
empty = Path(tmp) / "empty.gguf"
|
|
empty.write_bytes(b"")
|
|
r.equal("empty file", classify(empty), UNKNOWN)
|
|
r.equal("missing file", classify(Path(tmp) / "absent.gguf"), UNKNOWN)
|
|
r.equal("non-GGUF file", classify(Path(__file__)), UNKNOWN)
|
|
|
|
r.section("real models, if any are installed")
|
|
real = sorted(Path.home().joinpath(".lmstudio/models").rglob("*.gguf"))
|
|
if not real:
|
|
print(" SKIP no local models found under ~/.lmstudio/models")
|
|
else:
|
|
kinds = {path.name: classify(path) for path in real}
|
|
projectors = [n for n, k in kinds.items() if k == PROJECTOR]
|
|
generatives = [n for n, k in kinds.items() if k == GENERATIVE]
|
|
r.check("every mmproj-* file classifies as a projector",
|
|
projectors and all(n.startswith("mmproj-") for n in projectors), projectors)
|
|
r.check("no generative model is an mmproj-*",
|
|
not any(n.startswith("mmproj-") for n in generatives), generatives)
|
|
r.check("architecture readable for every real file",
|
|
all(architecture(path) for path in real))
|
|
|
|
raise SystemExit(r.finish())
|