feat: enhance LM Studio integration with model filtering and add comprehensive test coverage

.core/providers/lmstudio.py

@@ -11,6 +11,12 @@ from .base import ModelInfo, Provider, ProviderError
 
 DEFAULT_BASE_URL = "http://localhost:1234"
 
+#: LM Studio's native endpoint labels each model. ``llm`` and ``vlm`` can both
+#: answer a prompt; ``embeddings`` cannot, and offering one produces an empty
+#: report with no error. The OpenAI-compatible ``/v1/models`` carries no such
+#: field, which is why the native endpoint is tried first.
+NON_GENERATIVE_TYPES = frozenset({"embeddings"})
+
 
 class LMStudioProvider(Provider):
     name = "LM Studio"
@@ -18,9 +24,64 @@ class LMStudioProvider(Provider):
     def __init__(self, base_url: Optional[str] = None) -> None:
         self.base_url = base_url or get_env_setting("LM_STUDIO_BASE_URL", DEFAULT_BASE_URL)
         self._models_url = f"{self.base_url}/v1/models"
+        self._native_models_url = f"{self.base_url}/api/v0/models"
         self._chat_url = f"{self.base_url}/v1/chat/completions"
 
+    # ------------------------------------------------------------- listing
+
     def list_models(self) -> List[ModelInfo]:
+        """Models that can actually answer a prompt.
+
+        Prefers LM Studio's native endpoint, which states each model's type.
+        Falls back to the OpenAI-compatible list unfiltered when that is
+        unavailable — an older LM Studio, say. Only positive evidence hides a
+        model: a name-based guess would conceal any legitimate model whose id
+        happened to contain "embed", and a model missing from the picker is
+        harder to diagnose than one that fails when run.
+        """
+        native = self._native_models()
+        if native is not None:
+            return native
+        return self._openai_models()
+
+    def _native_models(self) -> Optional[List[ModelInfo]]:
+        """Parse ``/api/v0/models``, or None if it is not usable."""
+        try:
+            response = requests.get(self._native_models_url, timeout=10)
+            response.raise_for_status()
+            data = response.json()
+        except (requests.exceptions.RequestException, json.JSONDecodeError, ValueError):
+            return None
+
+        entries = data.get("data") if isinstance(data, dict) else None
+        if not isinstance(entries, list) or not entries:
+            return None
+
+        result: List[ModelInfo] = []
+        skipped = 0
+        for item in entries:
+            if not isinstance(item, dict):
+                return None  # not the shape we expected; fall back
+            model_id = str(item.get("id") or "")
+            if not model_id:
+                continue
+            if str(item.get("type", "")).lower() in NON_GENERATIVE_TYPES:
+                skipped += 1
+                continue
+            result.append(ModelInfo(id=model_id, display_name=model_id))
+
+        if not result:
+            if skipped:
+                raise ProviderError(
+                    f"LM Studio has {skipped} model(s) loaded, but all of them are "
+                    "embedding models, which cannot generate text. Load a chat or "
+                    "instruct model."
+                )
+            return None
+        return result
+
+    def _openai_models(self) -> List[ModelInfo]:
+        """The OpenAI-compatible list. No type information, so no filtering."""
         try:
             response = requests.get(self._models_url, timeout=10)
             response.raise_for_status()

.core/suites/language/answers.json

@@ -1,7 +1,24 @@
 {
-  "_comment": "Deliberately empty. Nothing in the language suite is reliably auto-checkable, and an unreliable key is worse than none - it fails correct answers with total confidence, which is the exact problem this grader exists to remove.",
+  "_comment": "Only entity extraction is keyed. The rest of this suite is judgement - sentiment, similarity, summarisation, translation - and an unreliable key is worse than none, because it fails correct work with total confidence.",
+
+  "test2.txt": {
+    "note": "Named entity recognition. Every entity below appears verbatim in the supplied excerpt, so any correct extraction contains them. This is a coverage check: it can pass an answer that lists the entities but mislabels their types, and that is the intended trade - it can never fail an answer that got them right.",
+    "must_contain": [
+      "Nairobi",
+      "Kenya Wildlife Service",
+      "Serena Hotel",
+      "Tsavo East",
+      "Okello",
+      "Botswana",
+      "World Bank",
+      "Leakey Foundation"
+    ],
+    "must_contain_any": ["KWS"],
+    "numbers": [2023, 2019, 2017, 42],
+    "_reasoning": "must_contain_any on KWS covers the alias question - the answer must notice KWS and Kenya Wildlife Service are the same entity. The numbers are the three years and the $42 million figure. Type classification is deliberately unchecked: a table can be right about the entity and wrong about the label in ways no substring test can separate."
+  },
 
   "_test4_omitted": "The Winograd item looked checkable: the first `it` is the TROPHY (a trophy too large fails to fit; a suitcase too large would succeed), flipping to the SUITCASE on 'too small'. But correctness turns on which noun binds to which condition, in free prose, and both nouns necessarily appear in any answer. Every pattern tried also matched an inverted answer, because 'it would refer to the trophy instead' sits inside the counterfactual sentence. Left to human review.",
 
-  "_others_omitted": "Sentiment, entity extraction, semantic similarity, summarisation and translation are judgement calls. Pattern-matching them would manufacture false confidence."
+  "_others_omitted": "Sentiment, semantic similarity, summarisation and translation are judgement calls. Pattern-matching them would manufacture false confidence."
 }

.core/suites/reasoning-logic/answers.json

@@ -1,6 +1,22 @@
 {
   "_comment": "The grid and the CPM network were solved by brute force, not by re-reading the derivation. Both confirmed unique.",
 
+  "test1.txt": {
+    "note": "Commonsense mechanisms. Only the four items with an unambiguous physical answer are checked: water expands on freezing so that jar cracks (honey does not); the 3 a.m. chirp is a low battery, whose voltage sags as the house cools overnight; resting a roast lets juices redistribute; metal feels hotter than wood at equal temperature because it conducts heat away from the skin faster.",
+    "must_contain_any": ["expand", "expansion", "less dense", "more space", "greater volume"],
+    "regex": "(?i)(batter(y|ies))",
+    "_reasoning": "Items 2 and 3 - the coffee mug and the indirect refusal - are deliberately unchecked. Both are correct in too many phrasings to pattern-match without risking a false failure. The remaining two mechanisms are covered by test1_extra below, kept separate so one missing term does not mask another."
+  },
+
+  "_test1_extra_note": "Held as documentation rather than a matcher: 'juice'/'redistribute' for the roast and 'conduct' for the spoons were considered, but folding four independent facts into one score made a partial answer indistinguishable from a wrong one. Split them only if per-item scoring is ever added.",
+
+  "test2.txt": {
+    "note": "Causal structures. The discriminating content is the lurking variable behind finding A - ice cream and drowning are both driven by hot weather - because that word appears nowhere in the prompt. The structure names do appear in the prompt as a list of options, so containing them proves little; they are checked anyway because a correct answer certainly contains them and the check cannot fail correct work.",
+    "must_contain_any": ["temperature", "summer", "hot weather", "warm weather", "heat"],
+    "regex": "(?i)regression to the mean",
+    "_reasoning": "Finding F is regression to the mean and finding E is a selection effect, but both terms are supplied by the prompt, so a model could echo them while assigning them to the wrong findings. The weather confounder is the one answer the prompt does not hand over."
+  },
+
   "test3.txt": {
     "note": "Unique solution: Alvarez/South/comets, Brandt/North/dialects, Chen/East/basalt, Dube/West/algae. Clue 5 is redundant - removing it still leaves exactly one solution; removing any other clue does not.",
     "must_contain": ["Alvarez", "Brandt", "Chen", "Dube"],