feat: enhance GPU support detection and user feedback in the engine interface

.core/engine_loader.py

@@ -65,7 +65,17 @@ class EngineDescriptor:
         return f"engine_{self.architecture}_{self.version}"
 
 
+#: Architectures whose runtimes ask llama.cpp to offload layers to a GPU.
+GPU_ARCHITECTURES = frozenset({"cuda", "rocm", "apple_silicon"})
+
+
 def detect_architecture() -> EngineDescriptor:
+    """Which engine runtime this *hardware* calls for.
+
+    This looks at drivers and CPU family only. It deliberately says nothing
+    about whether the installed llama-cpp-python can actually use that
+    hardware — see gpu_offload_supported() for that half.
+    """
     system = platform.system().lower()
     machine = platform.machine().lower()
 
@@ -81,6 +91,58 @@ def detect_architecture() -> EngineDescriptor:
     return EngineDescriptor("cpu")
 
 
+def gpu_offload_supported() -> Optional[bool]:
+    """Whether the installed llama-cpp-python was *built* with GPU offload.
+
+    Detection and capability are two different questions, and they routinely
+    disagree. llama-cpp-python compiles its backend in at install time and the
+    default wheel is CPU-only, so a machine with an RTX card is detected as
+    "cuda", loads the CUDA runtime, sets n_gpu_layers=-1 — and then generates
+    every token on the CPU, because the binary has no CUDA in it. Nothing
+    errors. The card sits at 0% while the interface reports "cuda".
+
+    llama_supports_gpu_offload() answers the question the detection cannot:
+    it reports what the binary can do, not what the machine has.
+
+    Returns None when the answer is unknowable — llama_cpp missing, or too old
+    to expose the symbol — so callers can distinguish "no" from "cannot tell".
+    """
+    try:
+        import llama_cpp
+    except Exception:
+        return None
+
+    probe = getattr(llama_cpp, "llama_supports_gpu_offload", None)
+    if probe is None:
+        return None
+
+    try:
+        return bool(probe())
+    except Exception:
+        return None
+
+
+def engine_warning(
+    descriptor: Optional[EngineDescriptor] = None,
+    *,
+    offload: Optional[bool] = None,
+) -> Optional[str]:
+    """A human-readable warning when hardware and build disagree, else None."""
+    descriptor = descriptor or detect_architecture()
+    if descriptor.architecture not in GPU_ARCHITECTURES:
+        return None
+
+    offload = gpu_offload_supported() if offload is None else offload
+    if offload is False:
+        return (
+            f"{descriptor.architecture} hardware was detected, but the installed "
+            "llama-cpp-python is a CPU-only build, so the GPU will sit idle and "
+            "generation will run on the CPU. See GPU-ACCELERATION.md to install "
+            "a matching wheel."
+        )
+    return None
+
+
 def load_engine_class(descriptor: Optional[EngineDescriptor] = None) -> Type["BaseRuntime"]:
     descriptor = descriptor or detect_architecture()
     module_path = descriptor.module_path

packaging/GPU-ACCELERATION.md

@@ -14,19 +14,21 @@ created `.venv`.
 
 ## NVIDIA (CUDA)
 
-Check your driver's CUDA version with `nvidia-smi`, then match it:
-
 ```bash
 # Windows
 .venv\Scripts\python -m pip install --force-reinstall --no-cache-dir \
   llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu124
 
-# macOS / Linux
+# Linux
 .venv/bin/python -m pip install --force-reinstall --no-cache-dir \
   llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu124
 ```
 
-Swap `cu124` for `cu121` or `cu122` if your driver is older.
+Use `cu124` unless you have a reason not to. It and `cu125` carry current
+builds; `cu121` through `cu123` stopped at 0.3.4 and are far behind. You do
+**not** need to match your driver's CUDA version exactly — CUDA drivers are
+backward compatible, so a 13.x driver runs a cu124 build without complaint.
+Older than CUDA 12.4 is the only case where dropping back helps.
 
 ## Apple Silicon (Metal)
 
@@ -53,11 +55,30 @@ CMAKE_ARGS="-DGGML_HIPBLAS=on" .venv/bin/python -m pip install \
 
 ## Confirming it worked
 
-Start LM-Gambit and look at the **Engine** panel in the bottom-left. It reports
+Start LM-Gambit and open **Settings → Engine**. The row that answers this is
-the detected architecture. Then run one question and compare tokens/second
+**GPU offload**:
-against what you saw before — that number is the real test.
 
-A caveat worth knowing: the engine detects your *hardware*, not what
+- *Supported by this build* — the installed binary can drive your GPU.
-`llama-cpp-python` was built with. On an NVIDIA machine it will report `cuda`
+- *Not in this build (CPU only)* — it cannot, whatever the architecture says.
-even with a CPU wheel installed. If the architecture says `cuda` but throughput
+
-is unchanged, the wheel is still the CPU one.
+The **Architecture** row above it is detected from your *hardware* and will
+happily read `cuda` on any machine with an NVIDIA driver, including one running
+an entirely CPU-only build. It is not evidence of anything. When the two
+disagree the sidebar also shows a warning, and the Engine chip appends
+`· CPU build`.
+
+Then run one question and watch the numbers. On CPU you should expect roughly
+10–13 tok/s for a mid-size quantised model; a discrete GPU is several times
+that. If throughput did not move after installing a GPU wheel, the install did
+not take — check for an error in the pip output, since a failed
+`--force-reinstall` leaves the previous build in place.
+
+The blunt external check, run while a question is generating:
+
+```bash
+nvidia-smi
+```
+
+Your Python process should appear in the process list holding VRAM. If the list
+shows only desktop software and utilisation sits at 0%, nothing is reaching the
+card.

server/api.py

@@ -24,6 +24,8 @@ from .core_bridge import (
     ProviderError,
     build_provider,
     detect_architecture,
+    engine_warning,
+    gpu_offload_supported,
     list_provider_names,
     load_engine_class,
     load_settings,
@@ -87,10 +89,16 @@ async def get_system() -> SystemInfo:
     except EngineLoadError as exc:
         runtime_name = f"unavailable ({exc})"
 
+    # Computed after load_engine_class above, which already imports llama_cpp,
+    # so the probe costs nothing extra here.
+    offload = gpu_offload_supported()
+
     return SystemInfo(
         version=__version__,
         engine_architecture=descriptor.architecture,
         engine_runtime=runtime_name,
+        engine_gpu_offload=offload,
+        engine_warning=engine_warning(descriptor, offload=offload),
         template_ok=TEMPLATE_PATH.exists(),
         python_version=platform.python_version(),
         metrics={

server/core_bridge.py

@@ -30,6 +30,8 @@ from config import (  # type: ignore[import-not-found]
 from engine_loader import (  # type: ignore[import-not-found]
     EngineLoadError,
     detect_architecture,
+    engine_warning,
+    gpu_offload_supported,
     load_engine_class,
 )
 from prompts import load_test_prompts  # type: ignore[import-not-found]
@@ -77,6 +79,8 @@ __all__ = [
     "DEFAULT_PROVIDER_NAME",
     "EngineLoadError",
     "detect_architecture",
+    "engine_warning",
+    "gpu_offload_supported",
     "load_engine_class",
     "DEFAULT_TEMPERATURE",
     "MODELS_DIR",

server/schemas.py

@@ -204,6 +204,13 @@ class SystemInfo(BaseModel):
     version: str
     engine_architecture: str
     engine_runtime: str
+    # None when llama-cpp-python is absent or too old to report it, which is a
+    # different thing from a confirmed False.
+    engine_gpu_offload: Optional[bool] = None
+    # Set when the detected hardware and the installed build disagree — the
+    # case where the interface would otherwise claim "cuda" while every token
+    # is generated on the CPU.
+    engine_warning: Optional[str] = None
     template_ok: bool
     python_version: str
     metrics: Dict[str, str] = {}

tests_py/test_engine_capability.py

@@ -0,0 +1,114 @@
+"""Hardware detection vs. build capability — the two must not be conflated.
+
+detect_architecture() looks at drivers: nvidia-smi present means "cuda". That
+says nothing about the llama-cpp-python actually installed, and the default
+wheel is CPU-only. The pairing produced a silent failure worth guarding
+against: an RTX 4060 machine reported "cuda", loaded the CUDA runtime, set
+n_gpu_layers=-1, and generated every token on the CPU with the card at 0%
+utilisation. Nothing raised, and the interface said "cuda" throughout.
+
+These tests pin the behaviour that surfaces that mismatch instead of hiding it.
+"""
+
+from __future__ import annotations
+
+from _harness import Results, bootstrap
+
+bootstrap()
+from engine_loader import (  # noqa: E402
+    GPU_ARCHITECTURES,
+    EngineDescriptor,
+    detect_architecture,
+    engine_warning,
+    gpu_offload_supported,
+)
+
+r = Results("Engine capability reporting")
+
+
+# ---------------------------------------------------------------- the mismatch
+
+for arch in ("cuda", "rocm", "apple_silicon"):
+    warning = engine_warning(EngineDescriptor(arch), offload=False)
+    r.check(
+        f"{arch} + CPU-only build warns",
+        warning is not None and arch in warning,
+    )
+    r.check(
+        f"{arch} warning names the fix",
+        warning is not None and "GPU-ACCELERATION.md" in warning,
+    )
+    r.check(
+        f"{arch} warning says the GPU is idle",
+        warning is not None and "idle" in warning.lower(),
+    )
+
+r.check(
+    "GPU-capable build produces no warning",
+    engine_warning(EngineDescriptor("cuda"), offload=True) is None,
+)
+
+
+# ------------------------------------------------------- cpu never false-alarms
+
+# A CPU machine running a CPU build is correct, not a misconfiguration. Warning
+# there would train users to ignore the warning that matters.
+for offload in (False, True, None):
+    r.check(
+        f"cpu architecture stays quiet (offload={offload})",
+        engine_warning(EngineDescriptor("cpu"), offload=offload) is None,
+    )
+
+
+# --------------------------------------------------- unknown is not a negative
+
+# None means "cannot tell" — llama_cpp missing, or too old to expose the
+# symbol. Treating that as False would warn every user whose build predates the
+# probe, which is worse than staying silent.
+for arch in ("cuda", "rocm", "apple_silicon", "cpu"):
+    r.check(
+        f"{arch} with unknowable offload stays quiet",
+        engine_warning(EngineDescriptor(arch), offload=None) is None,
+    )
+
+
+# ------------------------------------------------------------------ the probe
+
+offload = gpu_offload_supported()
+r.check(
+    "gpu_offload_supported returns bool or None",
+    offload is None or isinstance(offload, bool),
+)
+
+# Guards against someone "simplifying" the tri-state into a plain bool later:
+# the distinction between False and None is load-bearing above.
+r.check(
+    "probe never returns a truthy non-bool",
+    offload is None or offload is True or offload is False,
+)
+
+
+# ------------------------------------------------------------- wiring sanity
+
+descriptor = detect_architecture()
+r.check(
+    "detect_architecture returns a known architecture",
+    descriptor.architecture in GPU_ARCHITECTURES | {"cpu"},
+)
+r.check(
+    "GPU_ARCHITECTURES excludes cpu",
+    "cpu" not in GPU_ARCHITECTURES,
+)
+r.check(
+    "every GPU architecture has a runtime module",
+    all((EngineDescriptor(a).module_path).exists() for a in GPU_ARCHITECTURES),
+)
+
+# engine_warning() must work with no arguments at all, since api.py may call it
+# that way if the descriptor is not already to hand.
+r.check(
+    "engine_warning is callable with no arguments",
+    engine_warning() is None or isinstance(engine_warning(), str),
+)
+
+raise SystemExit(r.finish())

web/src/components/Shell.tsx

@@ -102,7 +102,22 @@ export function Shell({ children, activeRun }: { children: ReactNode; activeRun:
       <div className="mt-1 truncate text-[0.75rem] font-medium text-ink-200" title={system.engine_runtime}>
         {system.engine_runtime}
       </div>
-      <div className="text-[0.6875rem] text-ink-500">{system.engine_architecture}</div>
+      <div className="text-[0.6875rem] text-ink-500">
+        {system.engine_architecture}
+        {/* The architecture alone is misleading when the installed
+            llama-cpp-python is a CPU build: it reports the hardware, not what
+            the binary can drive. Saying "CPU build" here is the difference
+            between a user seeing "cuda" and assuming the GPU is busy, and
+            knowing their card is idle. */}
+        {system.engine_gpu_offload === false && (
+          <span className="text-amber-400"> · CPU build</span>
+        )}
+      </div>
+      {system.engine_warning && (
+        <div className="mt-1.5 text-[0.6875rem] leading-snug text-amber-400">
+          {system.engine_warning}
+        </div>
+      )}
       {!system.template_ok && (
         <div className="mt-1.5 text-[0.6875rem] text-rose-400">Report template missing</div>
       )}

web/src/lib/api.ts

@@ -214,6 +214,12 @@ export interface SystemInfo {
   version: string
   engine_architecture: string
   engine_runtime: string
+  /** Whether the installed llama-cpp-python was built with GPU offload.
+   *  null when llama-cpp-python is missing or too old to report it — which is
+   *  distinct from a confirmed false. */
+  engine_gpu_offload: boolean | null
+  /** Set when detected hardware and the installed build disagree. */
+  engine_warning: string | null
   template_ok: boolean
   python_version: string
   metrics: Record<string, string>

web/src/pages/SettingsPage.tsx

@@ -213,6 +213,21 @@ export function SettingsPage() {
                   <dl className="space-y-2.5 text-[0.8125rem]">
                     <InfoRow label="Runtime" value={system.data.engine_runtime} mono />
                     <InfoRow label="Architecture" value={system.data.engine_architecture} />
+                    {/* Architecture is detected from hardware; this is what the
+                        installed binary can actually drive. The two disagreeing
+                        is the difference between a GPU working and sitting idle
+                        while the interface still says "cuda". */}
+                    {system.data.engine_gpu_offload !== null && (
+                      <InfoRow
+                        label="GPU offload"
+                        value={
+                          system.data.engine_gpu_offload
+                            ? 'Supported by this build'
+                            : 'Not in this build (CPU only)'
+                        }
+                        tone={system.data.engine_gpu_offload ? 'ok' : 'bad'}
+                      />
+                    )}
                     <InfoRow label="Platform" value={system.data.metrics.platform ?? '—'} />
                     <InfoRow label="Python" value={system.data.python_version} mono />
                     <InfoRow label="LM-Gambit" value={`v${system.data.version}`} mono />