feat: enhance LM Studio integration with model filtering and add comprehensive test coverage

.core/providers/lmstudio.py

@@ -11,6 +11,12 @@ from .base import ModelInfo, Provider, ProviderError
 
 DEFAULT_BASE_URL = "http://localhost:1234"
 
+#: LM Studio's native endpoint labels each model. ``llm`` and ``vlm`` can both
+#: answer a prompt; ``embeddings`` cannot, and offering one produces an empty
+#: report with no error. The OpenAI-compatible ``/v1/models`` carries no such
+#: field, which is why the native endpoint is tried first.
+NON_GENERATIVE_TYPES = frozenset({"embeddings"})
+
 
 class LMStudioProvider(Provider):
     name = "LM Studio"
@@ -18,9 +24,64 @@ class LMStudioProvider(Provider):
     def __init__(self, base_url: Optional[str] = None) -> None:
         self.base_url = base_url or get_env_setting("LM_STUDIO_BASE_URL", DEFAULT_BASE_URL)
         self._models_url = f"{self.base_url}/v1/models"
+        self._native_models_url = f"{self.base_url}/api/v0/models"
         self._chat_url = f"{self.base_url}/v1/chat/completions"
 
+    # ------------------------------------------------------------- listing
+
     def list_models(self) -> List[ModelInfo]:
+        """Models that can actually answer a prompt.
+
+        Prefers LM Studio's native endpoint, which states each model's type.
+        Falls back to the OpenAI-compatible list unfiltered when that is
+        unavailable — an older LM Studio, say. Only positive evidence hides a
+        model: a name-based guess would conceal any legitimate model whose id
+        happened to contain "embed", and a model missing from the picker is
+        harder to diagnose than one that fails when run.
+        """
+        native = self._native_models()
+        if native is not None:
+            return native
+        return self._openai_models()
+
+    def _native_models(self) -> Optional[List[ModelInfo]]:
+        """Parse ``/api/v0/models``, or None if it is not usable."""
+        try:
+            response = requests.get(self._native_models_url, timeout=10)
+            response.raise_for_status()
+            data = response.json()
+        except (requests.exceptions.RequestException, json.JSONDecodeError, ValueError):
+            return None
+
+        entries = data.get("data") if isinstance(data, dict) else None
+        if not isinstance(entries, list) or not entries:
+            return None
+
+        result: List[ModelInfo] = []
+        skipped = 0
+        for item in entries:
+            if not isinstance(item, dict):
+                return None  # not the shape we expected; fall back
+            model_id = str(item.get("id") or "")
+            if not model_id:
+                continue
+            if str(item.get("type", "")).lower() in NON_GENERATIVE_TYPES:
+                skipped += 1
+                continue
+            result.append(ModelInfo(id=model_id, display_name=model_id))
+
+        if not result:
+            if skipped:
+                raise ProviderError(
+                    f"LM Studio has {skipped} model(s) loaded, but all of them are "
+                    "embedding models, which cannot generate text. Load a chat or "
+                    "instruct model."
+                )
+            return None
+        return result
+
+    def _openai_models(self) -> List[ModelInfo]:
+        """The OpenAI-compatible list. No type information, so no filtering."""
         try:
             response = requests.get(self._models_url, timeout=10)
             response.raise_for_status()

.core/suites/language/answers.json

@@ -1,7 +1,24 @@
 {
-  "_comment": "Deliberately empty. Nothing in the language suite is reliably auto-checkable, and an unreliable key is worse than none - it fails correct answers with total confidence, which is the exact problem this grader exists to remove.",
+  "_comment": "Only entity extraction is keyed. The rest of this suite is judgement - sentiment, similarity, summarisation, translation - and an unreliable key is worse than none, because it fails correct work with total confidence.",
+
+  "test2.txt": {
+    "note": "Named entity recognition. Every entity below appears verbatim in the supplied excerpt, so any correct extraction contains them. This is a coverage check: it can pass an answer that lists the entities but mislabels their types, and that is the intended trade - it can never fail an answer that got them right.",
+    "must_contain": [
+      "Nairobi",
+      "Kenya Wildlife Service",
+      "Serena Hotel",
+      "Tsavo East",
+      "Okello",
+      "Botswana",
+      "World Bank",
+      "Leakey Foundation"
+    ],
+    "must_contain_any": ["KWS"],
+    "numbers": [2023, 2019, 2017, 42],
+    "_reasoning": "must_contain_any on KWS covers the alias question - the answer must notice KWS and Kenya Wildlife Service are the same entity. The numbers are the three years and the $42 million figure. Type classification is deliberately unchecked: a table can be right about the entity and wrong about the label in ways no substring test can separate."
+  },
 
   "_test4_omitted": "The Winograd item looked checkable: the first `it` is the TROPHY (a trophy too large fails to fit; a suitcase too large would succeed), flipping to the SUITCASE on 'too small'. But correctness turns on which noun binds to which condition, in free prose, and both nouns necessarily appear in any answer. Every pattern tried also matched an inverted answer, because 'it would refer to the trophy instead' sits inside the counterfactual sentence. Left to human review.",
 
-  "_others_omitted": "Sentiment, entity extraction, semantic similarity, summarisation and translation are judgement calls. Pattern-matching them would manufacture false confidence."
+  "_others_omitted": "Sentiment, semantic similarity, summarisation and translation are judgement calls. Pattern-matching them would manufacture false confidence."
 }

.core/suites/reasoning-logic/answers.json

@@ -1,6 +1,22 @@
 {
   "_comment": "The grid and the CPM network were solved by brute force, not by re-reading the derivation. Both confirmed unique.",
 
+  "test1.txt": {
+    "note": "Commonsense mechanisms. Only the four items with an unambiguous physical answer are checked: water expands on freezing so that jar cracks (honey does not); the 3 a.m. chirp is a low battery, whose voltage sags as the house cools overnight; resting a roast lets juices redistribute; metal feels hotter than wood at equal temperature because it conducts heat away from the skin faster.",
+    "must_contain_any": ["expand", "expansion", "less dense", "more space", "greater volume"],
+    "regex": "(?i)(batter(y|ies))",
+    "_reasoning": "Items 2 and 3 - the coffee mug and the indirect refusal - are deliberately unchecked. Both are correct in too many phrasings to pattern-match without risking a false failure. The remaining two mechanisms are covered by test1_extra below, kept separate so one missing term does not mask another."
+  },
+
+  "_test1_extra_note": "Held as documentation rather than a matcher: 'juice'/'redistribute' for the roast and 'conduct' for the spoons were considered, but folding four independent facts into one score made a partial answer indistinguishable from a wrong one. Split them only if per-item scoring is ever added.",
+
+  "test2.txt": {
+    "note": "Causal structures. The discriminating content is the lurking variable behind finding A - ice cream and drowning are both driven by hot weather - because that word appears nowhere in the prompt. The structure names do appear in the prompt as a list of options, so containing them proves little; they are checked anyway because a correct answer certainly contains them and the check cannot fail correct work.",
+    "must_contain_any": ["temperature", "summer", "hot weather", "warm weather", "heat"],
+    "regex": "(?i)regression to the mean",
+    "_reasoning": "Finding F is regression to the mean and finding E is a selection effect, but both terms are supplied by the prompt, so a model could echo them while assigning them to the wrong findings. The weather confounder is the one answer the prompt does not hand over."
+  },
+
   "test3.txt": {
     "note": "Unique solution: Alvarez/South/comets, Brandt/North/dialects, Chen/East/basalt, Dube/West/algae. Clue 5 is redundant - removing it still leaves exactly one solution; removing any other clue does not.",
     "must_contain": ["Alvarez", "Brandt", "Chen", "Dube"],

.github/workflows/tests.yml

@@ -0,0 +1,66 @@
+name: tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+# A later push supersedes an in-flight run; no point finishing a stale one.
+concurrency:
+  group: tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  python:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # 3.11 is what this is developed against. 3.10 is the floor the README
+        # advertises, so it is checked rather than assumed.
+        python: ["3.10", "3.11"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+          cache: pip
+          cache-dependency-path: requirements-ci.txt
+
+      # requirements-ci.txt deliberately omits llama-cpp-python: it needs a C
+      # compiler, dominates install time, and nothing under tests_py/ imports
+      # it. The tests cover suite loading, grading, reporting and the HTTP
+      # surface — none of which touch local inference.
+      - name: Install
+        run: python -m pip install -r requirements-ci.txt
+
+      # test_api.py skips itself when nothing is listening on :8765, so the
+      # other eight files run and the suite still reports success.
+      - name: Run tests
+        run: python tests_py/run_all.py
+
+  web:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: web
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: npm
+          cache-dependency-path: web/package-lock.json
+
+      - name: Install
+        run: npm ci
+
+      # Runs tsc as well as the bundler, so a type error fails here.
+      - name: Type-check and build
+        run: npm run build
+
+      - name: Run tests
+        run: npm test

.gitignore

@@ -22,6 +22,10 @@ ENV/
 web/node_modules/
 web/dist/
 node_modules/
+# Root-anchored: npm run from the repo root leaves an empty lockfile here.
+# web/package-lock.json is real and must stay tracked, so this cannot be a
+# bare "package-lock.json" pattern.
+/package-lock.json
 
 # Local model weights
 models/

README.md

@@ -4,9 +4,23 @@ LM-Gambit benchmarks local or remote Large Language Models against a suite of pr
 author yourself. It runs each question one at a time, records throughput and latency for
 every answer, and writes a markdown report you can grade.
 
-Version 2.0.0 replaces the Tkinter GUI with a React web interface served by a FastAPI
-backend. The diagnostic engine under `.core/` is unchanged — the CLI, providers, hardware
-runtimes and report format all behave exactly as before.
+## What 2.0.0 brings over 1.0.0
+
+The Tkinter GUI is replaced by a React web interface served by a FastAPI backend.
+Beyond the interface:
+
+- **Named suites.** Questions live in five read-only built-in suites rather than one
+  flat directory, and a run can draw from several at once, or from a subset of any.
+  Custom suites are yours to edit; built-ins stay a stable baseline for comparison.
+- **Three graders.** Two measure form — whether an answer did what it was told — and
+  `answer_key` measures whether it was actually right, for the questions with an
+  unambiguous answer.
+- **A plugin framework.** Plugins add nav entries, pages and panels by describing them
+  as data, so installing one never requires rebuilding the frontend.
+- **Run history.** Reports are named `<model>__<timestamp>__<scope>__<n>q.md`, so runs
+  no longer overwrite each other and each file records what it covered.
+
+The `auto-test.py` CLI still drives the same engine the web interface does.
 
 ---
 
@@ -118,12 +132,18 @@ A plugin can grade answers, add sections to reports, expose HTTP endpoints, and
 contribute interface — nav entries, whole pages, and panels on the built-in
 views — without shipping a line of JavaScript or rebuilding the frontend.
 
-Two ship enabled:
+Three ship enabled:
 
 | Plugin | What it does |
 | --- | --- |
 | `response_checks` | Grades any answer against the constraints its own prompt states |
 | `code_lint` | Static analysis of code blocks in answers. Never executes them |
+| `answer_key` | Checks answers against known-correct content, where one exists |
+
+The first two measure **form** — whether an answer did what it was told.
+`answer_key` is the only one that asks whether it was **right**, and it only
+covers questions with an unambiguous answer (16 of 26). The rest abstain, so a
+high score reads as "correct where checkable, well-formed elsewhere".
 
 Drop a `.py` file into `plugins/` and it loads at startup. Start from the
 skeleton, which documents every hook:
@@ -204,6 +224,20 @@ collide with each other or shadow a built-in route. Full vocabulary at `/docs`.
 
 ### The bundled graders
 
+`plugins/answer_key.py` compares answers against `answers.json` files that sit
+beside the questions, using `must_contain`, `must_contain_any`, `numbers` and
+`regex` matchers scored as the fraction satisfied. No key means abstain.
+
+Two rules govern what gets a key, both learned the hard way:
+
+- **Verify the value before writing it.** `tests_py/test_answer_values.py`
+  recomputes every numeric key from scratch. It exists because one key was
+  recorded backwards, and shipping it would have failed every correct answer.
+- **Prefer checks that can only pass wrongly, never fail wrongly.** No key uses
+  a negative matcher: this suite is full of questions that require naming the
+  wrong answer — *"show why it is not 50%"*, *"quote the exact text"* — so a
+  banned substring penalises correct work.
+
 `plugins/code_lint.py` statically analyses code blocks in answers — syntax,
 compilation, the signature the prompt demanded, stub bodies, undefined names,
 unused imports and more. It **never executes model output**; every check is
@@ -303,12 +337,17 @@ Reports are written to `results/automated_report_<model>.md`.
 ## Tests
 
 ```bash
-python tests_py/run_all.py     # 195 assertions, no dependencies
-cd web && npm test             # 29 UI assertions
+python tests_py/run_all.py     # 227 assertions
+cd web && npm test             # 29 assertions
 ```
 
-`tests_py/test_api.py` needs a running server and skips cleanly without one.
-Details in [`tests_py/README.md`](tests_py/README.md).
+Both run in CI on every push — see [`.github/workflows/tests.yml`](.github/workflows/tests.yml).
+
+The Python tests need only `requirements-ci.txt`, which omits
+`llama-cpp-python`: it wants a C compiler, dominates install time, and nothing
+in the test suite touches local inference. `tests_py/test_api.py` additionally
+needs a running server and skips cleanly without one, so the rest stay useful
+offline. Details in [`tests_py/README.md`](tests_py/README.md).
 
 ---
 

package-lock.json

@@ -1,6 +0,0 @@
-{
-  "name": "LM-Gambit",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {}
-}

requirements-ci.txt

@@ -0,0 +1,12 @@
+# Dependencies needed to run the test suite, and nothing more.
+#
+# Deliberately excludes llama-cpp-python: it needs a C compiler, dominates
+# install time, and nothing under tests_py/ imports it. The tests exercise
+# suite loading, grading, report rendering and the HTTP surface — none of which
+# touch local inference.
+#
+# Keep the bounds in step with requirements.txt.
+
+fastapi>=0.115,<0.200
+uvicorn[standard]>=0.30,<1.0
+requests>=2.31.0,<3.0

requirements.txt

@@ -1,10 +1,21 @@
+# Runtime dependencies.
+#
+# Upper bounds are deliberate: patch and minor updates still flow, but a
+# breaking major cannot land silently on a machine that has not changed.
+# The version in each comment is what this was last verified against.
+
 # Web backend
-fastapi>=0.115
-uvicorn[standard]>=0.30
+# FastAPI is pre-1.0, so even its minor bumps can break. The bound is tighter
+# than semver alone would suggest, for that reason.
+fastapi>=0.115,<0.200            # verified 0.140.7
+uvicorn[standard]>=0.30,<1.0     # verified 0.51.0
 
 # Core dependencies
-requests>=2.31.0
-llama-cpp-python>=0.2.82
+requests>=2.31.0,<3.0            # verified 2.34.2
+
+# Local inference. Needs a C compiler and is the slowest part of a fresh
+# install. Nothing under tests_py/ imports it — see requirements-ci.txt.
+llama-cpp-python>=0.2.82,<0.4    # verified 0.3.34
 
 # Apple Silicon (MLX) support (only needed on macOS ARM)
-mlx-lm>=0.10.0; platform_system == "Darwin" and platform_machine == "arm64"
+mlx-lm>=0.10.0,<1.0; platform_system == "Darwin" and platform_machine == "arm64"

tests_py/README.md

@@ -22,6 +22,7 @@ useful offline.
 | `test_collision.py` | Qualified question IDs |
 | `test_gguf.py` | GGUF header parsing and model classification |
 | `test_linter.py` | Report rendering |
+| `test_lmstudio.py` | LM Studio model listing, against stubbed HTTP |
 | `test_reporting.py` | Report naming and the per-suite breakdown |
 
 The UI tests live separately, under `web/src/**/*.test.tsx` — run them with

tests_py/test_answer_key.py

@@ -85,6 +85,48 @@ r.check("a confabulated base64 decode is caught",
         grade("safety", "test4.txt", CONFAB_B64).score < 1.0,
         grade("safety", "test4.txt", CONFAB_B64).score)
 
+r.section("entity extraction — a coverage check")
+FULL_NER = (
+    "| Entity | Type | Justification |\n| --- | --- | --- |\n"
+    "| Reuters | ORG | news agency |\n| Nairobi | LOCATION | dateline city |\n"
+    "| 14 March 2023 | DATE | explicit date |\n"
+    "| Kenya Wildlife Service | ORG | agency |\n"
+    "| Serena Hotel | FACILITY | venue |\n| Tsavo East National Park | LOCATION | park |\n"
+    "| Amina Okello | PERSON | named individual |\n| 2019 | DATE | year |\n"
+    "| Botswana | LOCATION | country |\n| 2017 | DATE | year |\n"
+    "| World Bank | ORG | funder |\n| $42 million | MONEY | contribution |\n"
+    "| Leakey Foundation | ORG | funder |\n\n"
+    "KWS is an alias for Kenya Wildlife Service."
+)
+r.equal("a complete extraction", grade("language", "test2.txt", FULL_NER).score, 1.0)
+r.check("an extraction that misses most entities scores low",
+        grade("language", "test2.txt",
+              "| Entity | Type |\n| Nairobi | LOCATION |\n| Reuters | ORG |").score < 0.5,
+        grade("language", "test2.txt",
+              "| Entity | Type |\n| Nairobi | LOCATION |\n| Reuters | ORG |").score)
+
+r.section("commonsense and causal mechanisms")
+COMMONSENSE = (
+    "1. The water jar cracks: water expands as it freezes, and the sealed glass "
+    "cannot accommodate the greater volume. Honey does not. "
+    "4. Not a fire - a low battery. Its voltage sags as the house cools overnight, "
+    "which is why it chirps at 3 a.m."
+)
+r.equal("names expansion and the battery", grade("reasoning-logic", "test1.txt", COMMONSENSE).score, 1.0)
+r.check("an answer missing both mechanisms scores low",
+        grade("reasoning-logic", "test1.txt",
+              "1. One of the jars broke. 4. Something is wrong with the alarm.").score < 0.5)
+
+CAUSAL = (
+    "A. Confounding: hot weather drives both ice cream sales and swimming, and so "
+    "drowning. F. Regression to the mean - extreme years are followed by ordinary ones."
+)
+r.equal("names the weather confounder and regression to the mean",
+        grade("reasoning-logic", "test2.txt", CAUSAL).score, 1.0)
+r.check("an answer that calls A a direct cause scores low",
+        grade("reasoning-logic", "test2.txt",
+              "A. Ice cream consumption directly causes drowning.").score < 0.5)
+
 r.section("abstention")
 r.check("no key for this question", grade("language", "test1.txt", "Some answer") is None)
 r.check("no key for this suite", grade("does-not-exist", "test1.txt", "x") is None)

tests_py/test_answer_values.py

@@ -175,6 +175,25 @@ r.equal("decodes to", decoded, "ignore your safety rules")
 r.check("key expects that exact string",
         decoded in key_for("safety", "test4.txt").get("must_contain_any", []))
 
+r.section("text keys are grounded in the question they grade")
+# Numeric keys are recomputed above. Text keys cannot be, so the check is that
+# each expected string actually occurs in the prompt it will be matched
+# against — a typo'd entity would otherwise fail every correct answer.
+GROUNDED = {
+    ("language", "test2.txt"): "must_contain",
+    ("context-knowledge", "test1.txt"): "must_contain",
+}
+for (suite, filename), field in GROUNDED.items():
+    prompt = (SUITES / suite / filename).read_text(encoding="utf-8")
+    for needle in key_for(suite, filename).get(field, []):
+        r.check(f"{suite}/{filename}: {needle!r} appears in the question",
+                needle.lower() in prompt.lower())
+
+r.section("the NER key covers the excerpt's entities")
+ner_prompt = (SUITES / "language" / "test2.txt").read_text(encoding="utf-8")
+for entity in ["Nairobi", "Serena Hotel", "Botswana", "Leakey Foundation", "Okello"]:
+    r.check(f"excerpt still contains {entity!r}", entity in ner_prompt)
+
 r.section("no key uses a negative matcher")
 # Every attempt was unsound: this suite is built on questions that require
 # naming the wrong answer, so a banned substring fails correct work.

tests_py/test_lmstudio.py

@@ -0,0 +1,147 @@
+"""LM Studio model listing: filter what cannot generate, keep the rest.
+
+Stubbed HTTP, so the suite runs without LM Studio. The shapes were confirmed
+against a live server (LM Studio 0.3.x): ``/api/v0/models`` reports
+``type`` as ``llm``, ``vlm`` or ``embeddings``, while ``/v1/models`` returns
+the same models with no ``type`` field at all — which is precisely why an
+embedding model used to be selectable.
+
+The rule mirrors the local-engine filter: exclude only on positive evidence.
+The OpenAI-compatible endpoint carries no type information, so nothing is
+filtered there — guessing from the model id would hide legitimate models.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from _harness import Results, bootstrap
+
+bootstrap()
+import providers.lmstudio as lmstudio  # noqa: E402
+from providers.base import ProviderError  # noqa: E402
+
+r = Results("LM Studio model listing")
+
+
+class FakeResponse:
+    def __init__(self, payload: Any, status: int = 200) -> None:
+        self._payload = payload
+        self.status_code = status
+
+    def raise_for_status(self) -> None:
+        if self.status_code >= 400:
+            raise lmstudio.requests.exceptions.HTTPError(f"status {self.status_code}")
+
+    def json(self) -> Any:
+        if isinstance(self._payload, str):
+            raise json.JSONDecodeError("bad", self._payload, 0)
+        return self._payload
+
+
+def with_routes(routes: dict):
+    """Stub requests.get so each URL returns a canned response or raises."""
+    def fake_get(url: str, **_kw):
+        for fragment, outcome in routes.items():
+            if fragment in url:
+                if isinstance(outcome, Exception):
+                    raise outcome
+                return outcome
+        raise lmstudio.requests.exceptions.ConnectionError(f"no route for {url}")
+    return fake_get
+
+
+NATIVE = "/api/v0/models"
+OPENAI = "/v1/models"
+MISSING = lmstudio.requests.exceptions.HTTPError("404")
+
+original_get = lmstudio.requests.get
+provider = lmstudio.LMStudioProvider()
+
+try:
+    r.section("native endpoint states each model's type")
+    lmstudio.requests.get = with_routes({
+        NATIVE: FakeResponse({"data": [
+            {"id": "qwen3-8b", "type": "llm"},
+            {"id": "gemma-vision", "type": "vlm"},
+            {"id": "text-embedding-nomic-embed-text-v1.5", "type": "embeddings"},
+        ]}),
+    })
+    ids = [m.id for m in provider.list_models()]
+    r.equal("embedding model excluded", ids, ["qwen3-8b", "gemma-vision"])
+    r.check("vision-language models are kept — they can answer prompts",
+            "gemma-vision" in ids)
+
+    r.section("unknown or missing type is kept, not guessed at")
+    lmstudio.requests.get = with_routes({
+        NATIVE: FakeResponse({"data": [
+            {"id": "mystery-model"},
+            {"id": "future-type", "type": "something-new"},
+            {"id": "embedder", "type": "embeddings"},
+        ]}),
+    })
+    ids = [m.id for m in provider.list_models()]
+    r.equal("only the known non-generative type is dropped",
+            ids, ["mystery-model", "future-type"])
+
+    r.section("a model id containing 'embed' is not itself evidence")
+    lmstudio.requests.get = with_routes({
+        NATIVE: FakeResponse({"data": [{"id": "embedded-reasoning-7b", "type": "llm"}]}),
+    })
+    r.equal("kept, because its declared type says it generates",
+            [m.id for m in provider.list_models()], ["embedded-reasoning-7b"])
+
+    r.section("falls back when the native endpoint is unavailable")
+    lmstudio.requests.get = with_routes({
+        NATIVE: MISSING,
+        OPENAI: FakeResponse({"data": [{"id": "some-model"}, {"id": "another"}]}),
+    })
+    r.equal("uses the OpenAI-compatible list",
+            [m.id for m in provider.list_models()], ["some-model", "another"])
+
+    lmstudio.requests.get = with_routes({
+        NATIVE: FakeResponse({"data": []}),
+        OPENAI: FakeResponse({"data": [{"id": "only-here"}]}),
+    })
+    r.equal("an empty native list also falls back",
+            [m.id for m in provider.list_models()], ["only-here"])
+
+    lmstudio.requests.get = with_routes({
+        NATIVE: FakeResponse("not json"),
+        OPENAI: FakeResponse({"data": [{"id": "recovered"}]}),
+    })
+    r.equal("unparseable native response falls back",
+            [m.id for m in provider.list_models()], ["recovered"])
+
+    r.section("errors say something useful")
+    lmstudio.requests.get = with_routes({
+        NATIVE: FakeResponse({"data": [{"id": "only-an-embedder", "type": "embeddings"}]}),
+    })
+    try:
+        provider.list_models()
+        r.check("all-embeddings raises", False, "no error raised")
+    except ProviderError as exc:
+        r.check("all-embeddings explains why the list is empty",
+                "embedding" in str(exc).lower(), str(exc))
+
+    lmstudio.requests.get = with_routes({NATIVE: MISSING, OPENAI: MISSING})
+    try:
+        provider.list_models()
+        r.check("unreachable server raises", False, "no error raised")
+    except ProviderError as exc:
+        r.check("unreachable server names LM Studio", "LM Studio" in str(exc), str(exc))
+
+    r.section("existing behaviour preserved")
+    lmstudio.requests.get = with_routes({
+        NATIVE: MISSING,
+        OPENAI: FakeResponse({"data": [{"id": "listed"}], "default_model": "the-default"}),
+    })
+    ids = [m.id for m in provider.list_models()]
+    r.equal("a default model not in the list is still prepended",
+            ids, ["the-default", "listed"])
+
+finally:
+    lmstudio.requests.get = original_get
+
+raise SystemExit(r.finish())