diff --git a/.core/engine_loader.py b/.core/engine_loader.py
index 50e4a98..b6f98b1 100644
--- a/.core/engine_loader.py
+++ b/.core/engine_loader.py
@@ -65,7 +65,17 @@ class EngineDescriptor:
return f"engine_{self.architecture}_{self.version}"
+#: Architectures whose runtimes ask llama.cpp to offload layers to a GPU.
+GPU_ARCHITECTURES = frozenset({"cuda", "rocm", "apple_silicon"})
+
+
def detect_architecture() -> EngineDescriptor:
+ """Which engine runtime this *hardware* calls for.
+
+ This looks at drivers and CPU family only. It deliberately says nothing
+ about whether the installed llama-cpp-python can actually use that
+ hardware — see gpu_offload_supported() for that half.
+ """
system = platform.system().lower()
machine = platform.machine().lower()
@@ -81,6 +91,58 @@ def detect_architecture() -> EngineDescriptor:
return EngineDescriptor("cpu")
+def gpu_offload_supported() -> Optional[bool]:
+ """Whether the installed llama-cpp-python was *built* with GPU offload.
+
+ Detection and capability are two different questions, and they routinely
+ disagree. llama-cpp-python compiles its backend in at install time and the
+ default wheel is CPU-only, so a machine with an RTX card is detected as
+ "cuda", loads the CUDA runtime, sets n_gpu_layers=-1 — and then generates
+ every token on the CPU, because the binary has no CUDA in it. Nothing
+ errors. The card sits at 0% while the interface reports "cuda".
+
+ llama_supports_gpu_offload() answers the question the detection cannot:
+ it reports what the binary can do, not what the machine has.
+
+ Returns None when the answer is unknowable — llama_cpp missing, or too old
+ to expose the symbol — so callers can distinguish "no" from "cannot tell".
+ """
+ try:
+ import llama_cpp
+ except Exception:
+ return None
+
+ probe = getattr(llama_cpp, "llama_supports_gpu_offload", None)
+ if probe is None:
+ return None
+
+ try:
+ return bool(probe())
+ except Exception:
+ return None
+
+
+def engine_warning(
+ descriptor: Optional[EngineDescriptor] = None,
+ *,
+ offload: Optional[bool] = None,
+) -> Optional[str]:
+ """A human-readable warning when hardware and build disagree, else None."""
+ descriptor = descriptor or detect_architecture()
+ if descriptor.architecture not in GPU_ARCHITECTURES:
+ return None
+
+ offload = gpu_offload_supported() if offload is None else offload
+ if offload is False:
+ return (
+ f"{descriptor.architecture} hardware was detected, but the installed "
+ "llama-cpp-python is a CPU-only build, so the GPU will sit idle and "
+ "generation will run on the CPU. See GPU-ACCELERATION.md to install "
+ "a matching wheel."
+ )
+ return None
+
+
def load_engine_class(descriptor: Optional[EngineDescriptor] = None) -> Type["BaseRuntime"]:
descriptor = descriptor or detect_architecture()
module_path = descriptor.module_path
diff --git a/packaging/GPU-ACCELERATION.md b/packaging/GPU-ACCELERATION.md
index 8cbe9ba..4613c91 100644
--- a/packaging/GPU-ACCELERATION.md
+++ b/packaging/GPU-ACCELERATION.md
@@ -14,19 +14,21 @@ created `.venv`.
## NVIDIA (CUDA)
-Check your driver's CUDA version with `nvidia-smi`, then match it:
-
```bash
# Windows
.venv\Scripts\python -m pip install --force-reinstall --no-cache-dir \
llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu124
-# macOS / Linux
+# Linux
.venv/bin/python -m pip install --force-reinstall --no-cache-dir \
llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu124
```
-Swap `cu124` for `cu121` or `cu122` if your driver is older.
+Use `cu124` unless you have a reason not to. It and `cu125` carry current
+builds; `cu121` through `cu123` stopped at 0.3.4 and are far behind. You do
+**not** need to match your driver's CUDA version exactly — CUDA drivers are
+backward compatible, so a 13.x driver runs a cu124 build without complaint.
+Older than CUDA 12.4 is the only case where dropping back helps.
## Apple Silicon (Metal)
@@ -53,11 +55,30 @@ CMAKE_ARGS="-DGGML_HIPBLAS=on" .venv/bin/python -m pip install \
## Confirming it worked
-Start LM-Gambit and look at the **Engine** panel in the bottom-left. It reports
-the detected architecture. Then run one question and compare tokens/second
-against what you saw before — that number is the real test.
+Start LM-Gambit and open **Settings → Engine**. The row that answers this is
+**GPU offload**:
-A caveat worth knowing: the engine detects your *hardware*, not what
-`llama-cpp-python` was built with. On an NVIDIA machine it will report `cuda`
-even with a CPU wheel installed. If the architecture says `cuda` but throughput
-is unchanged, the wheel is still the CPU one.
+- *Supported by this build* — the installed binary can drive your GPU.
+- *Not in this build (CPU only)* — it cannot, whatever the architecture says.
+
+The **Architecture** row above it is detected from your *hardware* and will
+happily read `cuda` on any machine with an NVIDIA driver, including one running
+an entirely CPU-only build. It is not evidence of anything. When the two
+disagree the sidebar also shows a warning, and the Engine chip appends
+`· CPU build`.
+
+Then run one question and watch the numbers. On CPU you should expect roughly
+10–13 tok/s for a mid-size quantised model; a discrete GPU is several times
+that. If throughput did not move after installing a GPU wheel, the install did
+not take — check for an error in the pip output, since a failed
+`--force-reinstall` leaves the previous build in place.
+
+The blunt external check, run while a question is generating:
+
+```bash
+nvidia-smi
+```
+
+Your Python process should appear in the process list holding VRAM. If the list
+shows only desktop software and utilisation sits at 0%, nothing is reaching the
+card.
diff --git a/server/api.py b/server/api.py
index b591864..9fc4009 100644
--- a/server/api.py
+++ b/server/api.py
@@ -24,6 +24,8 @@ from .core_bridge import (
ProviderError,
build_provider,
detect_architecture,
+ engine_warning,
+ gpu_offload_supported,
list_provider_names,
load_engine_class,
load_settings,
@@ -87,10 +89,16 @@ async def get_system() -> SystemInfo:
except EngineLoadError as exc:
runtime_name = f"unavailable ({exc})"
+ # Computed after load_engine_class above, which already imports llama_cpp,
+ # so the probe costs nothing extra here.
+ offload = gpu_offload_supported()
+
return SystemInfo(
version=__version__,
engine_architecture=descriptor.architecture,
engine_runtime=runtime_name,
+ engine_gpu_offload=offload,
+ engine_warning=engine_warning(descriptor, offload=offload),
template_ok=TEMPLATE_PATH.exists(),
python_version=platform.python_version(),
metrics={
diff --git a/server/core_bridge.py b/server/core_bridge.py
index 70cc2a9..0a0a6be 100644
--- a/server/core_bridge.py
+++ b/server/core_bridge.py
@@ -30,6 +30,8 @@ from config import ( # type: ignore[import-not-found]
from engine_loader import ( # type: ignore[import-not-found]
EngineLoadError,
detect_architecture,
+ engine_warning,
+ gpu_offload_supported,
load_engine_class,
)
from prompts import load_test_prompts # type: ignore[import-not-found]
@@ -77,6 +79,8 @@ __all__ = [
"DEFAULT_PROVIDER_NAME",
"EngineLoadError",
"detect_architecture",
+ "engine_warning",
+ "gpu_offload_supported",
"load_engine_class",
"DEFAULT_TEMPERATURE",
"MODELS_DIR",
diff --git a/server/schemas.py b/server/schemas.py
index 4bfafe1..2ea251e 100644
--- a/server/schemas.py
+++ b/server/schemas.py
@@ -204,6 +204,13 @@ class SystemInfo(BaseModel):
version: str
engine_architecture: str
engine_runtime: str
+ # None when llama-cpp-python is absent or too old to report it, which is a
+ # different thing from a confirmed False.
+ engine_gpu_offload: Optional[bool] = None
+ # Set when the detected hardware and the installed build disagree — the
+ # case where the interface would otherwise claim "cuda" while every token
+ # is generated on the CPU.
+ engine_warning: Optional[str] = None
template_ok: bool
python_version: str
metrics: Dict[str, str] = {}
diff --git a/tests_py/test_engine_capability.py b/tests_py/test_engine_capability.py
new file mode 100644
index 0000000..df2f3ae
--- /dev/null
+++ b/tests_py/test_engine_capability.py
@@ -0,0 +1,114 @@
+"""Hardware detection vs. build capability — the two must not be conflated.
+
+detect_architecture() looks at drivers: nvidia-smi present means "cuda". That
+says nothing about the llama-cpp-python actually installed, and the default
+wheel is CPU-only. The pairing produced a silent failure worth guarding
+against: an RTX 4060 machine reported "cuda", loaded the CUDA runtime, set
+n_gpu_layers=-1, and generated every token on the CPU with the card at 0%
+utilisation. Nothing raised, and the interface said "cuda" throughout.
+
+These tests pin the behaviour that surfaces that mismatch instead of hiding it.
+"""
+
+from __future__ import annotations
+
+from _harness import Results, bootstrap
+
+bootstrap()
+from engine_loader import ( # noqa: E402
+ GPU_ARCHITECTURES,
+ EngineDescriptor,
+ detect_architecture,
+ engine_warning,
+ gpu_offload_supported,
+)
+
+r = Results("Engine capability reporting")
+
+
+# ---------------------------------------------------------------- the mismatch
+
+for arch in ("cuda", "rocm", "apple_silicon"):
+ warning = engine_warning(EngineDescriptor(arch), offload=False)
+ r.check(
+ f"{arch} + CPU-only build warns",
+ warning is not None and arch in warning,
+ )
+ r.check(
+ f"{arch} warning names the fix",
+ warning is not None and "GPU-ACCELERATION.md" in warning,
+ )
+ r.check(
+ f"{arch} warning says the GPU is idle",
+ warning is not None and "idle" in warning.lower(),
+ )
+
+r.check(
+ "GPU-capable build produces no warning",
+ engine_warning(EngineDescriptor("cuda"), offload=True) is None,
+)
+
+
+# ------------------------------------------------------- cpu never false-alarms
+
+# A CPU machine running a CPU build is correct, not a misconfiguration. Warning
+# there would train users to ignore the warning that matters.
+for offload in (False, True, None):
+ r.check(
+ f"cpu architecture stays quiet (offload={offload})",
+ engine_warning(EngineDescriptor("cpu"), offload=offload) is None,
+ )
+
+
+# --------------------------------------------------- unknown is not a negative
+
+# None means "cannot tell" — llama_cpp missing, or too old to expose the
+# symbol. Treating that as False would warn every user whose build predates the
+# probe, which is worse than staying silent.
+for arch in ("cuda", "rocm", "apple_silicon", "cpu"):
+ r.check(
+ f"{arch} with unknowable offload stays quiet",
+ engine_warning(EngineDescriptor(arch), offload=None) is None,
+ )
+
+
+# ------------------------------------------------------------------ the probe
+
+offload = gpu_offload_supported()
+r.check(
+ "gpu_offload_supported returns bool or None",
+ offload is None or isinstance(offload, bool),
+)
+
+# Guards against someone "simplifying" the tri-state into a plain bool later:
+# the distinction between False and None is load-bearing above.
+r.check(
+ "probe never returns a truthy non-bool",
+ offload is None or offload is True or offload is False,
+)
+
+
+# ------------------------------------------------------------- wiring sanity
+
+descriptor = detect_architecture()
+r.check(
+ "detect_architecture returns a known architecture",
+ descriptor.architecture in GPU_ARCHITECTURES | {"cpu"},
+)
+r.check(
+ "GPU_ARCHITECTURES excludes cpu",
+ "cpu" not in GPU_ARCHITECTURES,
+)
+r.check(
+ "every GPU architecture has a runtime module",
+ all((EngineDescriptor(a).module_path).exists() for a in GPU_ARCHITECTURES),
+)
+
+# engine_warning() must work with no arguments at all, since api.py may call it
+# that way if the descriptor is not already to hand.
+r.check(
+ "engine_warning is callable with no arguments",
+ engine_warning() is None or isinstance(engine_warning(), str),
+)
+
+raise SystemExit(r.finish())
diff --git a/web/src/components/Shell.tsx b/web/src/components/Shell.tsx
index 6d60987..0cccf29 100644
--- a/web/src/components/Shell.tsx
+++ b/web/src/components/Shell.tsx
@@ -102,7 +102,22 @@ export function Shell({ children, activeRun }: { children: ReactNode; activeRun:
{system.engine_runtime}
- {system.engine_architecture}
+
+ {system.engine_architecture}
+ {/* The architecture alone is misleading when the installed
+ llama-cpp-python is a CPU build: it reports the hardware, not what
+ the binary can drive. Saying "CPU build" here is the difference
+ between a user seeing "cuda" and assuming the GPU is busy, and
+ knowing their card is idle. */}
+ {system.engine_gpu_offload === false && (
+ · CPU build
+ )}
+
+ {system.engine_warning && (
+
+ {system.engine_warning}
+
+ )}
{!system.template_ok && (
Report template missing
)}
diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts
index 153e566..d4f42db 100644
--- a/web/src/lib/api.ts
+++ b/web/src/lib/api.ts
@@ -214,6 +214,12 @@ export interface SystemInfo {
version: string
engine_architecture: string
engine_runtime: string
+ /** Whether the installed llama-cpp-python was built with GPU offload.
+ * null when llama-cpp-python is missing or too old to report it — which is
+ * distinct from a confirmed false. */
+ engine_gpu_offload: boolean | null
+ /** Set when detected hardware and the installed build disagree. */
+ engine_warning: string | null
template_ok: boolean
python_version: string
metrics: Record
diff --git a/web/src/pages/SettingsPage.tsx b/web/src/pages/SettingsPage.tsx
index 5d21e2a..e169df4 100644
--- a/web/src/pages/SettingsPage.tsx
+++ b/web/src/pages/SettingsPage.tsx
@@ -213,6 +213,21 @@ export function SettingsPage() {
+ {/* Architecture is detected from hardware; this is what the
+ installed binary can actually drive. The two disagreeing
+ is the difference between a GPU working and sitting idle
+ while the interface still says "cuda". */}
+ {system.data.engine_gpu_offload !== null && (
+
+ )}