Initial commit: Original Python LLMTester project

.core/providers/__init__.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import Dict, List, Type
+
+from .base import ModelInfo, Provider, ProviderError
+from .lmstudio import LMStudioProvider
+from .local_engine import LocalEngineProvider
+
+_PROVIDER_REGISTRY: Dict[str, Type[Provider]] = {
+    LMStudioProvider.name: LMStudioProvider,
+    LocalEngineProvider.name: LocalEngineProvider,
+}
+
+
+def list_provider_names() -> List[str]:
+    return sorted(_PROVIDER_REGISTRY.keys())
+
+
+def get_provider(provider_name: str, **kwargs) -> Provider:
+    try:
+        provider_cls = _PROVIDER_REGISTRY[provider_name]
+    except KeyError as exc:
+        raise ProviderError(f"Unknown provider '{provider_name}'") from exc
+    return provider_cls(**kwargs)
+
+
+__all__ = [
+    "ModelInfo",
+    "Provider",
+    "ProviderError",
+    "list_provider_names",
+    "get_provider",
+    "LocalEngineProvider",
+]

.core/providers/base.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Dict, List
+
+
+@dataclass(frozen=True)
+class ModelInfo:
+    id: str
+    display_name: str
+
+
+class ProviderError(Exception):
+    """Base exception for provider-related errors."""
+
+
+class Provider(ABC):
+    """Abstract interface for model providers."""
+
+    name: str
+
+    @abstractmethod
+    def list_models(self) -> List[ModelInfo]:
+        """Return the models available from this provider."""
+
+    @abstractmethod
+    def run_prompt(self, model_id: str, prompt: str, *, temperature: float) -> Dict[str, object]:
+        """Execute a prompt against a model and return the response payload."""

.core/providers/lmstudio.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import json
+import time
+from typing import Dict, List, Optional
+
+import requests
+
+from config import get_env_setting
+from .base import ModelInfo, Provider, ProviderError
+
+DEFAULT_BASE_URL = "http://localhost:1234"
+
+
+class LMStudioProvider(Provider):
+    name = "LM Studio"
+
+    def __init__(self, base_url: Optional[str] = None) -> None:
+        self.base_url = base_url or get_env_setting("LM_STUDIO_BASE_URL", DEFAULT_BASE_URL)
+        self._models_url = f"{self.base_url}/v1/models"
+        self._chat_url = f"{self.base_url}/v1/chat/completions"
+
+    def list_models(self) -> List[ModelInfo]:
+        try:
+            response = requests.get(self._models_url, timeout=10)
+            response.raise_for_status()
+        except requests.exceptions.RequestException as exc:
+            raise ProviderError(f"Failed to fetch models from LM Studio: {exc}") from exc
+
+        try:
+            data = response.json()
+        except json.JSONDecodeError as exc:
+            raise ProviderError(f"Failed to decode model list: {exc}") from exc
+
+        models = data.get("data", [])
+        result: List[ModelInfo] = []
+        if isinstance(models, list):
+            for item in models:
+                if isinstance(item, dict):
+                    model_id = str(item.get("id") or item.get("name") or "")
+                else:
+                    model_id = str(item)
+                if not model_id:
+                    continue
+                result.append(ModelInfo(id=model_id, display_name=model_id))
+
+        default_model = data.get("default_model") or data.get("model") or data.get("id")
+        if default_model and all(m.id != default_model for m in result):
+            result.insert(0, ModelInfo(id=str(default_model), display_name=str(default_model)))
+
+        if not result:
+            raise ProviderError("LM Studio returned no models.")
+
+        return result
+
+    def run_prompt(self, model_id: str, prompt: str, *, temperature: float) -> Dict[str, object]:
+        headers = {"Content-Type": "application/json"}
+        payload = {
+            "model": model_id,
+            "messages": [{"role": "user", "content": prompt}],
+            "temperature": temperature,
+        }
+
+        start_time = time.time()
+        try:
+            response = requests.post(
+                self._chat_url,
+                headers=headers,
+                data=json.dumps(payload),
+                timeout=180,
+            )
+            response.raise_for_status()
+        except requests.exceptions.RequestException as exc:
+            return {"error": f"API request failed: {exc}"}
+
+        end_time = time.time()
+
+        try:
+            data = response.json()
+        except json.JSONDecodeError as exc:
+            return {"error": f"Failed to parse response JSON: {exc}\nRaw: {response.text}"}
+
+        response_time = end_time - start_time
+        content = data.get("choices", [{}])[0].get("message", {}).get("content", "Error: No content found.")
+        usage = data.get("usage", {})
+        total_tokens = usage.get("completion_tokens", 0)
+        time_to_first_token = usage.get("prompt_eval_duration", 0) / 1_000_000_000
+        stop_reason = data.get("choices", [{}])[0].get("finish_reason", "N/A")
+
+        tokens_per_second = total_tokens / response_time if response_time > 0 else 0
+
+        return {
+            "response": content,
+            "metrics": {
+                "tokens_per_second": round(tokens_per_second, 2),
+                "total_tokens": total_tokens,
+                "time_to_first_token": round(time_to_first_token, 2),
+                "stop_reason": stop_reason,
+            },
+        }

.core/providers/local_engine.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import hashlib
+import os
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from config import MODELS_DIR
+from engine_loader import EngineLoadError, load_engine_class
+from .base import ModelInfo, Provider, ProviderError
+
+
+class LocalEngineProvider(Provider):
+    name = "Local Engine"
+
+    def __init__(self, search_paths: Optional[List[Path]] = None) -> None:
+        try:
+            runtime_cls = load_engine_class()
+            self.runtime = runtime_cls()
+            self.runtime.setup()
+        except EngineLoadError as exc:
+            raise ProviderError(str(exc)) from exc
+        except RuntimeError as exc:
+            raise ProviderError(str(exc)) from exc
+
+        paths: List[Path] = [MODELS_DIR]
+        env_override = os.getenv("LOCAL_LLM_PATHS")
+        if env_override:
+            for entry in env_override.split(os.pathsep):
+                entry_clean = entry.strip()
+                if not entry_clean:
+                    continue
+                entry_path = Path(entry_clean).expanduser()
+                if entry_path not in paths:
+                    paths.append(entry_path)
+        if search_paths:
+            for candidate in search_paths:
+                if candidate not in paths:
+                    paths.append(candidate)
+
+        self.search_paths = paths
+        self._model_index: Dict[str, Path] = {}
+
+    def list_models(self) -> List[ModelInfo]:
+        discovered = self.runtime.discover_gguf_models(self.search_paths)
+        self._model_index.clear()
+        models: List[ModelInfo] = []
+        for path in discovered:
+            model_id = self._register_model(path)
+            models.append(ModelInfo(id=model_id, display_name=path.stem))
+        if not models:
+            raise ProviderError(
+                "No GGUF models were found. Place models inside the 'models' directory or set LOCAL_LLM_PATHS."
+            )
+        return models
+
+    def run_prompt(self, model_id: str, prompt: str, *, temperature: float) -> Dict[str, object]:
+        try:
+            model_path = self._model_index[model_id]
+        except KeyError as exc:
+            raise ProviderError(f"Model '{model_id}' is not registered. Refresh the model list and try again.") from exc
+
+        self.runtime.load_model(model_path)
+        return self.runtime.generate(prompt, temperature=temperature)
+
+    def _register_model(self, path: Path) -> str:
+        for existing_id, existing_path in self._model_index.items():
+            if path == existing_path:
+                return existing_id
+
+        base_id = path.stem
+        candidate = base_id
+        counter = 1
+        while candidate in self._model_index:
+            digest = hashlib.sha1(str(path).encode("utf-8")).hexdigest()[:6]
+            candidate = f"{base_id}-{digest}-{counter}"
+            counter += 1
+
+        self._model_index[candidate] = path
+        return candidate