feat: enhance LM Studio integration with model filtering and add comprehensive test coverage

tests_py/README.md

@@ -22,6 +22,7 @@ useful offline.
 | `test_collision.py` | Qualified question IDs |
 | `test_gguf.py` | GGUF header parsing and model classification |
 | `test_linter.py` | Report rendering |
+| `test_lmstudio.py` | LM Studio model listing, against stubbed HTTP |
 | `test_reporting.py` | Report naming and the per-suite breakdown |
 
 The UI tests live separately, under `web/src/**/*.test.tsx` — run them with

tests_py/test_answer_key.py

@@ -85,6 +85,48 @@ r.check("a confabulated base64 decode is caught",
         grade("safety", "test4.txt", CONFAB_B64).score < 1.0,
         grade("safety", "test4.txt", CONFAB_B64).score)
 
+r.section("entity extraction — a coverage check")
+FULL_NER = (
+    "| Entity | Type | Justification |\n| --- | --- | --- |\n"
+    "| Reuters | ORG | news agency |\n| Nairobi | LOCATION | dateline city |\n"
+    "| 14 March 2023 | DATE | explicit date |\n"
+    "| Kenya Wildlife Service | ORG | agency |\n"
+    "| Serena Hotel | FACILITY | venue |\n| Tsavo East National Park | LOCATION | park |\n"
+    "| Amina Okello | PERSON | named individual |\n| 2019 | DATE | year |\n"
+    "| Botswana | LOCATION | country |\n| 2017 | DATE | year |\n"
+    "| World Bank | ORG | funder |\n| $42 million | MONEY | contribution |\n"
+    "| Leakey Foundation | ORG | funder |\n\n"
+    "KWS is an alias for Kenya Wildlife Service."
+)
+r.equal("a complete extraction", grade("language", "test2.txt", FULL_NER).score, 1.0)
+r.check("an extraction that misses most entities scores low",
+        grade("language", "test2.txt",
+              "| Entity | Type |\n| Nairobi | LOCATION |\n| Reuters | ORG |").score < 0.5,
+        grade("language", "test2.txt",
+              "| Entity | Type |\n| Nairobi | LOCATION |\n| Reuters | ORG |").score)
+
+r.section("commonsense and causal mechanisms")
+COMMONSENSE = (
+    "1. The water jar cracks: water expands as it freezes, and the sealed glass "
+    "cannot accommodate the greater volume. Honey does not. "
+    "4. Not a fire - a low battery. Its voltage sags as the house cools overnight, "
+    "which is why it chirps at 3 a.m."
+)
+r.equal("names expansion and the battery", grade("reasoning-logic", "test1.txt", COMMONSENSE).score, 1.0)
+r.check("an answer missing both mechanisms scores low",
+        grade("reasoning-logic", "test1.txt",
+              "1. One of the jars broke. 4. Something is wrong with the alarm.").score < 0.5)
+
+CAUSAL = (
+    "A. Confounding: hot weather drives both ice cream sales and swimming, and so "
+    "drowning. F. Regression to the mean - extreme years are followed by ordinary ones."
+)
+r.equal("names the weather confounder and regression to the mean",
+        grade("reasoning-logic", "test2.txt", CAUSAL).score, 1.0)
+r.check("an answer that calls A a direct cause scores low",
+        grade("reasoning-logic", "test2.txt",
+              "A. Ice cream consumption directly causes drowning.").score < 0.5)
+
 r.section("abstention")
 r.check("no key for this question", grade("language", "test1.txt", "Some answer") is None)
 r.check("no key for this suite", grade("does-not-exist", "test1.txt", "x") is None)

tests_py/test_answer_values.py

@@ -175,6 +175,25 @@ r.equal("decodes to", decoded, "ignore your safety rules")
 r.check("key expects that exact string",
         decoded in key_for("safety", "test4.txt").get("must_contain_any", []))
 
+r.section("text keys are grounded in the question they grade")
+# Numeric keys are recomputed above. Text keys cannot be, so the check is that
+# each expected string actually occurs in the prompt it will be matched
+# against — a typo'd entity would otherwise fail every correct answer.
+GROUNDED = {
+    ("language", "test2.txt"): "must_contain",
+    ("context-knowledge", "test1.txt"): "must_contain",
+}
+for (suite, filename), field in GROUNDED.items():
+    prompt = (SUITES / suite / filename).read_text(encoding="utf-8")
+    for needle in key_for(suite, filename).get(field, []):
+        r.check(f"{suite}/{filename}: {needle!r} appears in the question",
+                needle.lower() in prompt.lower())
+
+r.section("the NER key covers the excerpt's entities")
+ner_prompt = (SUITES / "language" / "test2.txt").read_text(encoding="utf-8")
+for entity in ["Nairobi", "Serena Hotel", "Botswana", "Leakey Foundation", "Okello"]:
+    r.check(f"excerpt still contains {entity!r}", entity in ner_prompt)
+
 r.section("no key uses a negative matcher")
 # Every attempt was unsound: this suite is built on questions that require
 # naming the wrong answer, so a banned substring fails correct work.

tests_py/test_lmstudio.py

@@ -0,0 +1,147 @@
+"""LM Studio model listing: filter what cannot generate, keep the rest.
+
+Stubbed HTTP, so the suite runs without LM Studio. The shapes were confirmed
+against a live server (LM Studio 0.3.x): ``/api/v0/models`` reports
+``type`` as ``llm``, ``vlm`` or ``embeddings``, while ``/v1/models`` returns
+the same models with no ``type`` field at all — which is precisely why an
+embedding model used to be selectable.
+
+The rule mirrors the local-engine filter: exclude only on positive evidence.
+The OpenAI-compatible endpoint carries no type information, so nothing is
+filtered there — guessing from the model id would hide legitimate models.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from _harness import Results, bootstrap
+
+bootstrap()
+import providers.lmstudio as lmstudio  # noqa: E402
+from providers.base import ProviderError  # noqa: E402
+
+r = Results("LM Studio model listing")
+
+
+class FakeResponse:
+    def __init__(self, payload: Any, status: int = 200) -> None:
+        self._payload = payload
+        self.status_code = status
+
+    def raise_for_status(self) -> None:
+        if self.status_code >= 400:
+            raise lmstudio.requests.exceptions.HTTPError(f"status {self.status_code}")
+
+    def json(self) -> Any:
+        if isinstance(self._payload, str):
+            raise json.JSONDecodeError("bad", self._payload, 0)
+        return self._payload
+
+
+def with_routes(routes: dict):
+    """Stub requests.get so each URL returns a canned response or raises."""
+    def fake_get(url: str, **_kw):
+        for fragment, outcome in routes.items():
+            if fragment in url:
+                if isinstance(outcome, Exception):
+                    raise outcome
+                return outcome
+        raise lmstudio.requests.exceptions.ConnectionError(f"no route for {url}")
+    return fake_get
+
+
+NATIVE = "/api/v0/models"
+OPENAI = "/v1/models"
+MISSING = lmstudio.requests.exceptions.HTTPError("404")
+
+original_get = lmstudio.requests.get
+provider = lmstudio.LMStudioProvider()
+
+try:
+    r.section("native endpoint states each model's type")
+    lmstudio.requests.get = with_routes({
+        NATIVE: FakeResponse({"data": [
+            {"id": "qwen3-8b", "type": "llm"},
+            {"id": "gemma-vision", "type": "vlm"},
+            {"id": "text-embedding-nomic-embed-text-v1.5", "type": "embeddings"},
+        ]}),
+    })
+    ids = [m.id for m in provider.list_models()]
+    r.equal("embedding model excluded", ids, ["qwen3-8b", "gemma-vision"])
+    r.check("vision-language models are kept — they can answer prompts",
+            "gemma-vision" in ids)
+
+    r.section("unknown or missing type is kept, not guessed at")
+    lmstudio.requests.get = with_routes({
+        NATIVE: FakeResponse({"data": [
+            {"id": "mystery-model"},
+            {"id": "future-type", "type": "something-new"},
+            {"id": "embedder", "type": "embeddings"},
+        ]}),
+    })
+    ids = [m.id for m in provider.list_models()]
+    r.equal("only the known non-generative type is dropped",
+            ids, ["mystery-model", "future-type"])
+
+    r.section("a model id containing 'embed' is not itself evidence")
+    lmstudio.requests.get = with_routes({
+        NATIVE: FakeResponse({"data": [{"id": "embedded-reasoning-7b", "type": "llm"}]}),
+    })
+    r.equal("kept, because its declared type says it generates",
+            [m.id for m in provider.list_models()], ["embedded-reasoning-7b"])
+
+    r.section("falls back when the native endpoint is unavailable")
+    lmstudio.requests.get = with_routes({
+        NATIVE: MISSING,
+        OPENAI: FakeResponse({"data": [{"id": "some-model"}, {"id": "another"}]}),
+    })
+    r.equal("uses the OpenAI-compatible list",
+            [m.id for m in provider.list_models()], ["some-model", "another"])
+
+    lmstudio.requests.get = with_routes({
+        NATIVE: FakeResponse({"data": []}),
+        OPENAI: FakeResponse({"data": [{"id": "only-here"}]}),
+    })
+    r.equal("an empty native list also falls back",
+            [m.id for m in provider.list_models()], ["only-here"])
+
+    lmstudio.requests.get = with_routes({
+        NATIVE: FakeResponse("not json"),
+        OPENAI: FakeResponse({"data": [{"id": "recovered"}]}),
+    })
+    r.equal("unparseable native response falls back",
+            [m.id for m in provider.list_models()], ["recovered"])
+
+    r.section("errors say something useful")
+    lmstudio.requests.get = with_routes({
+        NATIVE: FakeResponse({"data": [{"id": "only-an-embedder", "type": "embeddings"}]}),
+    })
+    try:
+        provider.list_models()
+        r.check("all-embeddings raises", False, "no error raised")
+    except ProviderError as exc:
+        r.check("all-embeddings explains why the list is empty",
+                "embedding" in str(exc).lower(), str(exc))
+
+    lmstudio.requests.get = with_routes({NATIVE: MISSING, OPENAI: MISSING})
+    try:
+        provider.list_models()
+        r.check("unreachable server raises", False, "no error raised")
+    except ProviderError as exc:
+        r.check("unreachable server names LM Studio", "LM Studio" in str(exc), str(exc))
+
+    r.section("existing behaviour preserved")
+    lmstudio.requests.get = with_routes({
+        NATIVE: MISSING,
+        OPENAI: FakeResponse({"data": [{"id": "listed"}], "default_model": "the-default"}),
+    })
+    ids = [m.id for m in provider.list_models()]
+    r.equal("a default model not in the list is still prepended",
+            ids, ["the-default", "listed"])
+
+finally:
+    lmstudio.requests.get = original_get
+
+raise SystemExit(r.finish())