From b78cd43464f8f3f0e96f75c668686901feb7e6d1 Mon Sep 17 00:00:00 2001 From: Christopher Clendening Date: Fri, 3 Oct 2025 05:37:15 -0400 Subject: [PATCH] Release v1.0.0: update README and GUI title bar for version 1.0.0 --- README.md | 184 +++++++++++++++++++++++++++++++++++++++++++----------- index.py | 18 +----- 2 files changed, 152 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 00d98bb..aa7ecd9 100644 --- a/README.md +++ b/README.md @@ -1,62 +1,175 @@ -# LLM Automated Test Suite -This project automates a battery of diagnostic prompts against local or remote language models via a modular core runner and an optional GUI. -## Prerequisites -- Python 3.10 or newer (tkinter included with the standard library on macOS/Linux; install `python3-tk` on Debian-based systems if needed). -- Install dependencies: +# LM-Gambit v1.0.0: Automated LLM Diagnostic Suite + + +**Version 1.0.0** + +LM-Gambit is a modular, extensible framework for benchmarking and analyzing local or remote Large Language Models (LLMs). It supports both a command-line interface (CLI) and a Tkinter-based GUI, enabling rapid, repeatable evaluation of LLMs using customizable prompt suites. + +--- + +## Features + +- **Automated Testing:** Run a suite of prompt-based diagnostics against any supported LLM provider. +- **Provider System:** Out-of-the-box support for LM Studio (API) and a native Local Engine (hardware-optimized, supports Apple Silicon, CUDA, ROCm, CPU). +- **Extensible:** Add new providers or engine runtimes with minimal code changes. +- **GUI & CLI:** Choose between a full-featured GUI or a fast CLI for scripting and automation. +- **Customizable Reports:** Generates detailed Markdown reports with performance metrics and qualitative analysis sections. +- **Prompt Management:** Prompts are simple `.txt` files, editable in the GUI or any text editor. + +--- + +## Installation + +**Requirements:** +- Python 3.10+ +- macOS, Linux, or Windows (Apple Silicon, NVIDIA, AMD, or CPU-only supported) + +**Install dependencies:** ```bash python -m pip install -r requirements.txt ``` -## Project Layout +**(Optional)** For Apple Silicon (MLX), ensure `mlx-lm` is installed. For CUDA/ROCm, ensure your system drivers are set up. -- `.core/` – shared logic modules, templates, engines, and transient workspace used by both CLI and GUI. - - `config.py` – path/setting helpers. - - `prompts.py` – prompt discovery from `tests/`. - - `providers/` – provider adapters (LM Studio and the native Local Engine runtime). - - `reporting.py` – markdown templating, code-fence hygiene, report summaries. - - `runner.py` – orchestrates full test runs with progress callbacks. - - `engine_loader.py` – detects the host platform/GPU and loads the appropriate runtime. - - `.engine/./.py` – runtime implementations per hardware family (Apple Silicon ↦ MLX, CUDA ↦ llama.cpp, ROCm ↦ llama.cpp, CPU fallback). - - `.temp/` – ephemeral staging for rendered blocks. - - `templates/test-block.md` – markdown template for each test result. -- `models/` – drop-in directory for local `.gguf` (and MLX-compatible) weights discovered by the Local Engine provider. -- `tests/` – one prompt per `.txt` file (editable via the GUI or any editor). -- `results/` – generated markdown reports. -- `auto-test.py` – CLI entrypoint that runs the default provider/model. -- `index.py` – Tkinter GUI for provider/model selection, running tests, and opening prompts/results. +--- -## Running the CLI +## Quick Start + +### CLI Usage + +Run all tests with the default provider/model: ```bash python auto-test.py ``` -The CLI selects the first available model for the default provider (LM Studio by default) and streams progress to stdout. Reports are saved in `results/automated_report_.md`. +**Options:** +- `-h, --help` — Show usage instructions +- `-p ` — Select provider (e.g., "Local Engine", "LM Studio") +- `-m ` — Select model by ID or filename +- `-l, --list` — List available providers or models +- `-t ` — (Reserved) Specify a test suite -To switch to the Local Engine runtime, set `AUTO_TEST_PROVIDER="Local Engine"` (or choose it in the GUI) and ensure your models are available in the `models/` directory or any path listed in `LOCAL_LLM_PATHS`. +Example: -## Using the GUI +```bash +python auto-test.py -p "Local Engine" -m Qwen3-Coder-30B.gguf +``` + +Reports are saved to `results/automated_report_.md`. + +### GUI Usage ```bash python index.py ``` -GUI features: +**GUI Features:** +- Provider/model dropdowns with auto-discovery +- Adjustable sampling temperature +- Run, refresh, open prompts/results, and view latest report +- Live log of test progress +- Settings dialog for managing model search paths -- Provider dropdown with automatic model discovery. -- Model dropdown populated per provider. -- Adjustable sampling temperature. -- Buttons to run tests, refresh models, open the `tests/` folder, open the `results/` folder, and open the latest report. -- Live log of test progress and completion status. -- Settings menu → “Configure Model Paths…” to manage additional folders scanned by the Local Engine provider (persisted in `.core/user_settings.json`). +--- -## Configuration +## Project Structure -Environment variables: +- `.core/` — Core logic, providers, engine loader, reporting, templates + - `providers/` — Provider adapters (LM Studio, Local Engine, etc.) + - `.engine/` — Hardware-specific runtime implementations (MLX, CUDA, ROCm, CPU) + - `runner.py` — Orchestrates test runs, progress, and reporting + - `reporting.py` — Markdown report generation, code-fence hygiene + - `prompts.py` — Loads and parses prompt files from `tests/` + - `templates/test-block.md` — Customizable template for each test result +- `models/` — Drop-in directory for `.gguf` or MLX-compatible weights (auto-discovered) +- `tests/` — One prompt per `.txt` file (edit or add your own) +- `results/` — Markdown reports generated after each run +- `auto-test.py` — CLI entrypoint +- `index.py` — Tkinter GUI entrypoint + +--- + +## How the Core Engine Works + +1. **Provider Selection:** + - Providers (e.g., LM Studio, Local Engine) are registered in `.core/providers/`. + - Each provider implements `list_models()` and `run_prompt()`. + - The Local Engine auto-selects the best runtime for your hardware (MLX, CUDA, ROCm, or CPU fallback). + +2. **Prompt Loading:** + - Prompts are loaded from the `tests/` directory (one `.txt` file per test). + - Each prompt file's first non-empty line is used as the test title. + +3. **Test Execution:** + - For each prompt, the selected provider/model is used to generate a response. + - Results are streamed to the CLI or GUI with live progress updates. + +4. **Reporting:** + - Each test result is rendered using a Markdown template (`.core/templates/test-block.md`). + - Performance metrics (tokens/sec, time to first token, etc.) are included. + - A summary and qualitative analysis section are generated at the top of the report. + +--- + +## Configuration & Environment Variables + +You can customize behavior via environment variables: + +| Variable | Purpose | +|-------------------------|--------------------------------------------------------------| +| `LM_STUDIO_BASE_URL` | Override LM Studio API endpoint (default: http://localhost:1234) | +| `AUTO_TEST_TEMPERATURE` | Default sampling temperature for test runs | +| `AUTO_TEST_PROVIDER` | Default provider for CLI runs | +| `LOCAL_LLM_PATHS` | `:`-separated list of extra directories for `.gguf` models | + +Model search paths can also be managed via the GUI settings dialog (persisted in `.core/user_settings.json`). + +--- + + +## Extending LM-Gambit + +**Add a new provider:** +1. Create a new class in `.core/providers/` inheriting from `Provider` (see `base.py`). +2. Implement `list_models()` and `run_prompt()`. +3. Register your provider in `.core/providers/__init__.py`. + +**Add a new engine runtime:** +1. Add a new Python file under `.core/.engine/./.py`. +2. Implement an `EngineRuntime` class inheriting from `BaseRuntime`. +3. The engine loader will auto-detect and use your runtime if the hardware matches. + +--- + + +## Customizing Prompts & Reports in LM-Gambit + +- Add/edit prompt `.txt` files in `tests/` (first non-empty line is the title). +- Edit `.core/templates/test-block.md` to change the report format. + +--- + +## Troubleshooting + +- **No models found?** + - Ensure your `.gguf` files are in `models/` or listed in `LOCAL_LLM_PATHS`. + - For LM Studio, ensure the app is running and the API is enabled. +- **Engine errors?** + - Check your hardware drivers and Python dependencies. +- **GUI not launching?** + - Ensure `tkinter` is installed and available in your Python environment. + +--- + + +## License + +MIT License. See `LICENSE` for details. - `LM_STUDIO_BASE_URL` – override the default `http://localhost:1234` endpoint for the LM Studio provider. - `AUTO_TEST_TEMPERATURE` – default sampling temperature for test runs. @@ -72,6 +185,7 @@ Templates can be customized by editing `.core/templates/test-block.md`. - The engine loader automatically selects the best runtime for your hardware: MLX on Apple Silicon, CUDA on NVIDIA GPUs, ROCm on AMD GPUs, or a CPU fallback via `llama-cpp-python`. - Settings are saved in `.core/user_settings.json`, keeping CLI and GUI runs in sync. -## Extending Providers + +## Extending Providers in LM-Gambit Provider adapters live under `.core/providers/` and are registered in `.core/providers/__init__.py`. Each provider implements `list_models` and `run_prompt` to integrate with the runner and GUI, while hardware-specific runtimes reside under `.core/.engine/`. diff --git a/index.py b/index.py index 9127aa2..ade07e8 100644 --- a/index.py +++ b/index.py @@ -100,7 +100,7 @@ class App(tk.Tk): actual = "" def get_text(title): d = tk.Toplevel(self) - d.title(title) + d.title(f"{title} - LM-Gambit") d.geometry("600x400") t = tk.Text(d, wrap="word") t.pack(fill="both", expand=True) @@ -119,7 +119,7 @@ class App(tk.Tk): messagebox.showinfo("Diff Viewer", "Both expected and actual output are required.") def configure_model_paths(self) -> None: dialog = tk.Toplevel(self) - dialog.title("Configure Model Paths") + dialog.title("Configure Model Paths - LM-Gambit") dialog.geometry("520x320") dialog.transient(self) dialog.grab_set() @@ -183,7 +183,7 @@ class App(tk.Tk): DownloadModelDialog(self, model_dir) def __init__(self) -> None: super().__init__() - self.title("LLM Automated Test Console") + self.title("LM-Gambit v1.0.0") self.geometry("880x620") self.minsize(760, 520) @@ -482,18 +482,6 @@ class App(tk.Tk): self.log = scrolledtext.ScrolledText(self, wrap="word", height=22) self.log.pack(fill="both", expand=True, padx=12, pady=(0, 12)) self.log.configure(state="disabled") - if not RESULTS_DIR.exists(): - messagebox.showinfo("No Reports", "The results directory does not exist yet.") - return - - reports = list(RESULTS_DIR.glob("*.md")) - if not reports: - messagebox.showinfo("No Reports", "No markdown reports found yet.") - return - - latest_report = max(reports, key=lambda p: p.stat().st_mtime) - open_path(latest_report) - self._append_log(f"Opened latest report: {latest_report.name}") if __name__ == "__main__":