Initial commit: Original Python LLMTester project
This commit is contained in:
Binary file not shown.
@@ -0,0 +1,170 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
try: # Optional dependency for GGUF execution
|
||||
from llama_cpp import Llama
|
||||
except ImportError: # pragma: no cover
|
||||
Llama = None # type: ignore
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from llama_cpp import Llama as LlamaType
|
||||
else: # pragma: no cover
|
||||
class LlamaType: # type: ignore
|
||||
...
|
||||
|
||||
from engine_loader import BaseRuntime
|
||||
|
||||
|
||||
class EngineRuntime(BaseRuntime):
|
||||
name = "mlx_apple_silicon"
|
||||
|
||||
def __init__(self, *, max_tokens: int = 1024, context_window: int = 4096) -> None:
|
||||
super().__init__()
|
||||
self.max_tokens = max_tokens
|
||||
self.context_window = context_window
|
||||
self.threads = max(os.cpu_count() or 1, 1)
|
||||
|
||||
self._mode: Optional[str] = None # "mlx" or "llama_cpp"
|
||||
self._model = None
|
||||
self._tokenizer = None
|
||||
self._llm: Optional[LlamaType] = None
|
||||
self._model_path: Path | None = None
|
||||
|
||||
def discover_gguf_models(self, search_paths: list[Path]) -> list[Path]:
|
||||
candidates: list[Path] = []
|
||||
seen: set[Path] = set()
|
||||
for root in search_paths:
|
||||
if not root.exists():
|
||||
continue
|
||||
for path in root.rglob("*.gguf"):
|
||||
if path.is_file():
|
||||
resolved = path.resolve()
|
||||
if resolved not in seen:
|
||||
seen.add(resolved)
|
||||
candidates.append(resolved)
|
||||
for path in root.iterdir():
|
||||
if path.is_dir() and (path / "config.json").exists():
|
||||
resolved = path.resolve()
|
||||
if resolved not in seen:
|
||||
seen.add(resolved)
|
||||
candidates.append(resolved)
|
||||
return candidates
|
||||
|
||||
def load_model(self, model_path: Path) -> None:
|
||||
if self._model_path == model_path:
|
||||
if self._mode == "mlx" and self._model is not None:
|
||||
return
|
||||
if self._mode == "llama_cpp" and self._llm is not None:
|
||||
return
|
||||
|
||||
self.unload()
|
||||
resolved_path = model_path.resolve()
|
||||
|
||||
if resolved_path.is_file() and resolved_path.suffix.lower() == ".gguf":
|
||||
if Llama is None:
|
||||
raise RuntimeError(
|
||||
"llama-cpp-python is required to run GGUF models on Apple Silicon. Install it with 'pip install llama-cpp-python'."
|
||||
)
|
||||
self._llm = Llama(
|
||||
model_path=str(resolved_path),
|
||||
n_ctx=self.context_window,
|
||||
n_gpu_layers=-1,
|
||||
n_threads=self.threads,
|
||||
use_gpu=True,
|
||||
verbose=False,
|
||||
)
|
||||
self._mode = "llama_cpp"
|
||||
else:
|
||||
try:
|
||||
from mlx_lm import load # type: ignore import-not-found
|
||||
except ImportError as exc: # pragma: no cover - environment validation
|
||||
raise RuntimeError(
|
||||
"mlx-lm is required for MLX models. Install it with 'pip install mlx-lm'."
|
||||
) from exc
|
||||
|
||||
self._model, self._tokenizer = load(str(resolved_path))
|
||||
self._mode = "mlx"
|
||||
|
||||
self._model_path = resolved_path
|
||||
|
||||
def generate(self, prompt: str, *, temperature: float) -> dict:
|
||||
if self._mode == "llama_cpp":
|
||||
if self._llm is None:
|
||||
raise RuntimeError("Model must be loaded before calling generate().")
|
||||
|
||||
start_time = time.time()
|
||||
response = self._llm.create_chat_completion(
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=temperature,
|
||||
)
|
||||
elapsed = max(time.time() - start_time, 1e-5)
|
||||
|
||||
content = response["choices"][0]["message"]["content"]
|
||||
usage = response.get("usage", {})
|
||||
completion_tokens = usage.get("completion_tokens", 0)
|
||||
total_tokens = usage.get("total_tokens", completion_tokens)
|
||||
|
||||
return {
|
||||
"response": content,
|
||||
"metrics": {
|
||||
"tokens_per_second": round(completion_tokens / elapsed, 2) if completion_tokens else 0,
|
||||
"total_tokens": total_tokens,
|
||||
"time_to_first_token": round(usage.get("prompt_eval_duration", 0) / 1_000_000_000, 2)
|
||||
if "prompt_eval_duration" in usage
|
||||
else 0,
|
||||
"stop_reason": response["choices"][0].get("finish_reason", "unknown"),
|
||||
},
|
||||
}
|
||||
|
||||
if self._mode == "mlx":
|
||||
if self._model is None or self._tokenizer is None:
|
||||
raise RuntimeError("Model must be loaded before calling generate().")
|
||||
|
||||
start_time = time.time()
|
||||
try:
|
||||
from mlx_lm import generate # type: ignore import-not-found
|
||||
except ImportError as exc: # pragma: no cover
|
||||
raise RuntimeError(
|
||||
"mlx-lm is required for MLX models. Install it with 'pip install mlx-lm'."
|
||||
) from exc
|
||||
|
||||
completion = generate(
|
||||
self._model,
|
||||
self._tokenizer,
|
||||
prompt,
|
||||
max_tokens=self.max_tokens,
|
||||
temperature=temperature,
|
||||
)
|
||||
elapsed = max(time.time() - start_time, 1e-5)
|
||||
|
||||
if isinstance(completion, str):
|
||||
text = completion
|
||||
else:
|
||||
text = str(completion)
|
||||
|
||||
prompt_tokens = len(self._tokenizer.encode(prompt))
|
||||
completion_tokens = max(len(self._tokenizer.encode(text)) - prompt_tokens, 0)
|
||||
total_tokens = prompt_tokens + completion_tokens
|
||||
|
||||
return {
|
||||
"response": text,
|
||||
"metrics": {
|
||||
"tokens_per_second": round(completion_tokens / elapsed, 2) if completion_tokens else 0,
|
||||
"total_tokens": total_tokens,
|
||||
"time_to_first_token": 0,
|
||||
"stop_reason": "stop",
|
||||
},
|
||||
}
|
||||
|
||||
raise RuntimeError("No model is currently loaded. Call load_model() first.")
|
||||
|
||||
def unload(self) -> None:
|
||||
self._model = None
|
||||
self._tokenizer = None
|
||||
self._llm = None
|
||||
self._model_path = None
|
||||
self._mode = None
|
||||
@@ -0,0 +1,72 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from llama_cpp import Llama
|
||||
except ImportError as exc: # pragma: no cover - dependency validation
|
||||
raise RuntimeError(
|
||||
"llama-cpp-python is required for the CPU engine runtime. Install it with 'pip install llama-cpp-python'."
|
||||
) from exc
|
||||
|
||||
from engine_loader import BaseRuntime
|
||||
|
||||
|
||||
class EngineRuntime(BaseRuntime):
|
||||
name = "llama_cpp_cpu"
|
||||
|
||||
def __init__(self, *, context_window: int = 4096) -> None:
|
||||
super().__init__()
|
||||
self.context_window = context_window
|
||||
self.threads = max(os.cpu_count() or 1, 1)
|
||||
self._llm: Llama | None = None
|
||||
self._model_path: Path | None = None
|
||||
|
||||
def load_model(self, model_path: Path) -> None:
|
||||
if self._model_path == model_path and self._llm is not None:
|
||||
return
|
||||
|
||||
self.unload()
|
||||
self._llm = Llama(
|
||||
model_path=str(model_path),
|
||||
n_ctx=self.context_window,
|
||||
n_threads=self.threads,
|
||||
n_gpu_layers=0,
|
||||
verbose=False,
|
||||
)
|
||||
self._model_path = model_path
|
||||
|
||||
def generate(self, prompt: str, *, temperature: float) -> dict:
|
||||
if self._llm is None:
|
||||
raise RuntimeError("Model must be loaded before calling generate().")
|
||||
|
||||
start_time = time.time()
|
||||
response = self._llm.create_chat_completion(
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=temperature,
|
||||
)
|
||||
elapsed = max(time.time() - start_time, 1e-5)
|
||||
|
||||
content = response["choices"][0]["message"]["content"]
|
||||
usage = response.get("usage", {})
|
||||
completion_tokens = usage.get("completion_tokens", 0)
|
||||
total_tokens = usage.get("total_tokens", completion_tokens)
|
||||
|
||||
return {
|
||||
"response": content,
|
||||
"metrics": {
|
||||
"tokens_per_second": round(completion_tokens / elapsed, 2) if completion_tokens else 0,
|
||||
"total_tokens": total_tokens,
|
||||
"time_to_first_token": round(usage.get("prompt_eval_duration", 0) / 1_000_000_000, 2)
|
||||
if "prompt_eval_duration" in usage
|
||||
else 0,
|
||||
"stop_reason": response["choices"][0].get("finish_reason", "unknown"),
|
||||
},
|
||||
}
|
||||
|
||||
def unload(self) -> None:
|
||||
if self._llm is not None:
|
||||
self._llm = None
|
||||
self._model_path = None
|
||||
@@ -0,0 +1,74 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from llama_cpp import Llama
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"llama-cpp-python with CUDA support is required for the CUDA engine runtime."
|
||||
) from exc
|
||||
|
||||
from engine_loader import BaseRuntime
|
||||
|
||||
|
||||
class EngineRuntime(BaseRuntime):
|
||||
name = "llama_cpp_cuda"
|
||||
|
||||
def __init__(self, *, context_window: int = 4096, gpu_layers: int = -1) -> None:
|
||||
super().__init__()
|
||||
self.context_window = context_window
|
||||
self.gpu_layers = gpu_layers
|
||||
self.threads = max(os.cpu_count() or 1, 1)
|
||||
self._llm: Llama | None = None
|
||||
self._model_path: Path | None = None
|
||||
|
||||
def load_model(self, model_path: Path) -> None:
|
||||
if self._model_path == model_path and self._llm is not None:
|
||||
return
|
||||
|
||||
self.unload()
|
||||
self._llm = Llama(
|
||||
model_path=str(model_path),
|
||||
n_ctx=self.context_window,
|
||||
n_threads=self.threads,
|
||||
n_gpu_layers=self.gpu_layers,
|
||||
use_gpu=True,
|
||||
verbose=False,
|
||||
)
|
||||
self._model_path = model_path
|
||||
|
||||
def generate(self, prompt: str, *, temperature: float) -> dict:
|
||||
if self._llm is None:
|
||||
raise RuntimeError("Model must be loaded before calling generate().")
|
||||
|
||||
start_time = time.time()
|
||||
response = self._llm.create_chat_completion(
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=temperature,
|
||||
)
|
||||
elapsed = max(time.time() - start_time, 1e-5)
|
||||
|
||||
content = response["choices"][0]["message"]["content"]
|
||||
usage = response.get("usage", {})
|
||||
completion_tokens = usage.get("completion_tokens", 0)
|
||||
total_tokens = usage.get("total_tokens", completion_tokens)
|
||||
|
||||
return {
|
||||
"response": content,
|
||||
"metrics": {
|
||||
"tokens_per_second": round(completion_tokens / elapsed, 2) if completion_tokens else 0,
|
||||
"total_tokens": total_tokens,
|
||||
"time_to_first_token": round(usage.get("prompt_eval_duration", 0) / 1_000_000_000, 2)
|
||||
if "prompt_eval_duration" in usage
|
||||
else 0,
|
||||
"stop_reason": response["choices"][0].get("finish_reason", "unknown"),
|
||||
},
|
||||
}
|
||||
|
||||
def unload(self) -> None:
|
||||
if self._llm is not None:
|
||||
self._llm = None
|
||||
self._model_path = None
|
||||
@@ -0,0 +1,74 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from llama_cpp import Llama
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"llama-cpp-python built with ROCm support is required for the ROCm engine runtime."
|
||||
) from exc
|
||||
|
||||
from engine_loader import BaseRuntime
|
||||
|
||||
|
||||
class EngineRuntime(BaseRuntime):
|
||||
name = "llama_cpp_rocm"
|
||||
|
||||
def __init__(self, *, context_window: int = 4096, gpu_layers: int = -1) -> None:
|
||||
super().__init__()
|
||||
self.context_window = context_window
|
||||
self.gpu_layers = gpu_layers
|
||||
self.threads = max(os.cpu_count() or 1, 1)
|
||||
self._llm: Llama | None = None
|
||||
self._model_path: Path | None = None
|
||||
|
||||
def load_model(self, model_path: Path) -> None:
|
||||
if self._model_path == model_path and self._llm is not None:
|
||||
return
|
||||
|
||||
self.unload()
|
||||
self._llm = Llama(
|
||||
model_path=str(model_path),
|
||||
n_ctx=self.context_window,
|
||||
n_threads=self.threads,
|
||||
n_gpu_layers=self.gpu_layers,
|
||||
use_gpu=True,
|
||||
verbose=False,
|
||||
)
|
||||
self._model_path = model_path
|
||||
|
||||
def generate(self, prompt: str, *, temperature: float) -> dict:
|
||||
if self._llm is None:
|
||||
raise RuntimeError("Model must be loaded before calling generate().")
|
||||
|
||||
start_time = time.time()
|
||||
response = self._llm.create_chat_completion(
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=temperature,
|
||||
)
|
||||
elapsed = max(time.time() - start_time, 1e-5)
|
||||
|
||||
content = response["choices"][0]["message"]["content"]
|
||||
usage = response.get("usage", {})
|
||||
completion_tokens = usage.get("completion_tokens", 0)
|
||||
total_tokens = usage.get("total_tokens", completion_tokens)
|
||||
|
||||
return {
|
||||
"response": content,
|
||||
"metrics": {
|
||||
"tokens_per_second": round(completion_tokens / elapsed, 2) if completion_tokens else 0,
|
||||
"total_tokens": total_tokens,
|
||||
"time_to_first_token": round(usage.get("prompt_eval_duration", 0) / 1_000_000_000, 2)
|
||||
if "prompt_eval_duration" in usage
|
||||
else 0,
|
||||
"stop_reason": response["choices"][0].get("finish_reason", "unknown"),
|
||||
},
|
||||
}
|
||||
|
||||
def unload(self) -> None:
|
||||
if self._llm is not None:
|
||||
self._llm = None
|
||||
self._model_path = None
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,36 @@
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT_DIR = Path(__file__).resolve().parent.parent
|
||||
CORE_DIR = ROOT_DIR / ".core"
|
||||
TESTS_DIR = ROOT_DIR / "tests"
|
||||
RESULTS_DIR = ROOT_DIR / "results"
|
||||
TEMPLATES_DIR = CORE_DIR / "templates"
|
||||
TEMPLATE_PATH = TEMPLATES_DIR / "test-block.md"
|
||||
TEMP_DIR = CORE_DIR / ".temp"
|
||||
MODELS_DIR = ROOT_DIR / "models"
|
||||
|
||||
DEFAULT_TEMPERATURE = float(os.getenv("AUTO_TEST_TEMPERATURE", "0.1"))
|
||||
DEFAULT_PROVIDER_NAME = os.getenv("AUTO_TEST_PROVIDER", "LM Studio")
|
||||
|
||||
|
||||
def ensure_directories() -> None:
|
||||
"""Ensure that shared directories exist."""
|
||||
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
TEMPLATES_DIR.mkdir(parents=True, exist_ok=True)
|
||||
TEMP_DIR.mkdir(parents=True, exist_ok=True)
|
||||
MODELS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def get_env_setting(key: str, default: str) -> str:
|
||||
"""Read an environment variable with a default fallback."""
|
||||
return os.getenv(key, default)
|
||||
|
||||
|
||||
def reset_temp_directory() -> None:
|
||||
"""Clear and recreate the temporary directory used for staging markdown blocks."""
|
||||
if TEMP_DIR.exists():
|
||||
shutil.rmtree(TEMP_DIR)
|
||||
TEMP_DIR.mkdir(parents=True, exist_ok=True)
|
||||
@@ -0,0 +1,108 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.abc
|
||||
import importlib.util
|
||||
import platform
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Optional, Type
|
||||
|
||||
from config import CORE_DIR
|
||||
|
||||
ENGINE_ROOT = CORE_DIR / ".engine"
|
||||
DEFAULT_ENGINE_VERSION = "v1"
|
||||
|
||||
|
||||
class EngineLoadError(RuntimeError):
|
||||
"""Raised when no suitable engine runtime can be loaded."""
|
||||
|
||||
|
||||
class BaseRuntime:
|
||||
"""Base class for local engine runtimes."""
|
||||
|
||||
name: str = "base"
|
||||
|
||||
def discover_gguf_models(self, search_paths: list[Path]) -> list[Path]:
|
||||
gguf_paths: list[Path] = []
|
||||
seen: set[Path] = set()
|
||||
for root in search_paths:
|
||||
if not root.exists():
|
||||
continue
|
||||
for path in root.rglob("*.gguf"):
|
||||
if path.is_file():
|
||||
resolved = path.resolve()
|
||||
if resolved not in seen:
|
||||
seen.add(resolved)
|
||||
gguf_paths.append(resolved)
|
||||
return gguf_paths
|
||||
|
||||
def setup(self) -> None:
|
||||
"""Perform any runtime initialization before loading models."""
|
||||
|
||||
def load_model(self, model_path: Path) -> None:
|
||||
"""Load model into memory. Implementations may cache the loaded model."""
|
||||
|
||||
def generate(self, prompt: str, *, temperature: float) -> dict:
|
||||
"""Run inference against the currently loaded model and return a structured response."""
|
||||
raise NotImplementedError
|
||||
|
||||
def unload(self) -> None:
|
||||
"""Optional hook to release resources."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EngineDescriptor:
|
||||
architecture: str
|
||||
version: str = DEFAULT_ENGINE_VERSION
|
||||
|
||||
@property
|
||||
def module_path(self) -> Path:
|
||||
return ENGINE_ROOT / f".{self.architecture}" / f"{self.version}.py"
|
||||
|
||||
@property
|
||||
def module_name(self) -> str:
|
||||
return f"engine_{self.architecture}_{self.version}"
|
||||
|
||||
|
||||
def detect_architecture() -> EngineDescriptor:
|
||||
system = platform.system().lower()
|
||||
machine = platform.machine().lower()
|
||||
|
||||
if system == "darwin" and machine.startswith("arm"):
|
||||
return EngineDescriptor("apple_silicon")
|
||||
|
||||
if shutil.which("nvidia-smi"):
|
||||
return EngineDescriptor("cuda")
|
||||
|
||||
if shutil.which("rocm-smi") or shutil.which("rocminfo"):
|
||||
return EngineDescriptor("rocm")
|
||||
|
||||
return EngineDescriptor("cpu")
|
||||
|
||||
|
||||
def load_engine_class(descriptor: Optional[EngineDescriptor] = None) -> Type["BaseRuntime"]:
|
||||
descriptor = descriptor or detect_architecture()
|
||||
module_path = descriptor.module_path
|
||||
if not module_path.exists():
|
||||
raise EngineLoadError(f"Engine runtime not found for architecture '{descriptor.architecture}' at {module_path}")
|
||||
|
||||
spec = importlib.util.spec_from_file_location(descriptor.module_name, module_path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise EngineLoadError(f"Unable to load engine module from {module_path}")
|
||||
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
loader = spec.loader
|
||||
assert isinstance(loader, importlib.abc.Loader)
|
||||
try:
|
||||
loader.exec_module(module) # type: ignore[attr-defined]
|
||||
except Exception as exc: # pragma: no cover - import-time validation
|
||||
raise EngineLoadError(f"Failed to initialize engine runtime for '{descriptor.architecture}': {exc}") from exc
|
||||
|
||||
if not hasattr(module, "EngineRuntime"):
|
||||
raise EngineLoadError(f"Engine module {module_path} does not define 'EngineRuntime'")
|
||||
|
||||
runtime_cls = getattr(module, "EngineRuntime")
|
||||
if not issubclass(runtime_cls, BaseRuntime):
|
||||
raise EngineLoadError(f"Engine runtime from {module_path} must inherit from BaseRuntime")
|
||||
return runtime_cls
|
||||
@@ -0,0 +1,203 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Iterable, List, Tuple
|
||||
|
||||
__all__ = ["infer_language_from_prompt", "lint_response_markdown"]
|
||||
|
||||
|
||||
_LANGUAGE_HINTS: List[Tuple[str, str]] = [
|
||||
("swiftui", "swift"),
|
||||
("swift", "swift"),
|
||||
("objective-c", "objective-c"),
|
||||
("objc", "objective-c"),
|
||||
("kotlin", "kotlin"),
|
||||
("typescript", "typescript"),
|
||||
("javascript", "javascript"),
|
||||
("node.js", "javascript"),
|
||||
("python", "python"),
|
||||
("rust", "rust"),
|
||||
("go", "go"),
|
||||
("c#", "csharp"),
|
||||
("c++", "cpp"),
|
||||
("java", "java"),
|
||||
("sql", "sql"),
|
||||
]
|
||||
|
||||
_CODE_SYMBOLS = set("{}();[]=<>+-*/.&|%!:@")
|
||||
_CODE_PREFIXES = (
|
||||
"func ",
|
||||
"class ",
|
||||
"struct ",
|
||||
"enum ",
|
||||
"protocol ",
|
||||
"extension ",
|
||||
"import ",
|
||||
"let ",
|
||||
"var ",
|
||||
"guard ",
|
||||
"if ",
|
||||
"for ",
|
||||
"while ",
|
||||
"switch ",
|
||||
"case ",
|
||||
"return ",
|
||||
"init(",
|
||||
"init ",
|
||||
"deinit",
|
||||
"public ",
|
||||
"private ",
|
||||
"internal ",
|
||||
"fileprivate ",
|
||||
"override ",
|
||||
"@",
|
||||
"#if",
|
||||
"#endif",
|
||||
"#warning",
|
||||
"#error",
|
||||
)
|
||||
|
||||
_COMMENT_PREFIXES = ("///", "//", "/*", "*/", "* ")
|
||||
_BULLET_PREFIXES = ("- ", "* ", "+ ", "• ")
|
||||
|
||||
|
||||
def infer_language_from_prompt(prompt_text: str) -> str:
|
||||
"""Infer a reasonable language hint from the prompt text."""
|
||||
lowered = prompt_text.lower()
|
||||
for token, language in _LANGUAGE_HINTS:
|
||||
if token in lowered:
|
||||
return language
|
||||
return "text"
|
||||
|
||||
|
||||
def lint_response_markdown(raw_text: str, *, language_hint: str = "text") -> str:
|
||||
"""Normalize markdown so that only code segments are fenced and text stays plain."""
|
||||
if not raw_text:
|
||||
return ""
|
||||
|
||||
text = raw_text.replace("\r\n", "\n").replace("\r", "\n").strip("\n")
|
||||
if not text:
|
||||
return ""
|
||||
|
||||
if "```" in text:
|
||||
return _normalize_existing_fences(text, language_hint)
|
||||
|
||||
return _auto_fence_code_segments(text, language_hint)
|
||||
|
||||
|
||||
def _normalize_existing_fences(text: str, language_hint: str) -> str:
|
||||
lines = text.split("\n")
|
||||
output: List[str] = []
|
||||
in_fence = False
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("```"):
|
||||
fence_marker = stripped[3:].strip()
|
||||
if in_fence:
|
||||
output.append("```")
|
||||
in_fence = False
|
||||
else:
|
||||
language = fence_marker or (language_hint if language_hint != "text" else "")
|
||||
output.append(f"```{language}" if language else "```")
|
||||
in_fence = True
|
||||
else:
|
||||
output.append(line)
|
||||
|
||||
if in_fence:
|
||||
output.append("```")
|
||||
|
||||
result = "\n".join(output).strip()
|
||||
return result
|
||||
|
||||
|
||||
def _auto_fence_code_segments(text: str, language_hint: str) -> str:
|
||||
lines = text.split("\n")
|
||||
segments: List[Tuple[str, List[str]]] = []
|
||||
current_lines: List[str] = []
|
||||
current_mode: str | None = None
|
||||
|
||||
for line in lines:
|
||||
if _is_blank(line):
|
||||
if current_lines:
|
||||
current_lines.append(line)
|
||||
continue
|
||||
|
||||
classification = "code" if _is_likely_code_line(line) else "text"
|
||||
|
||||
if current_mode is None:
|
||||
current_mode = classification
|
||||
current_lines.append(line)
|
||||
continue
|
||||
|
||||
if classification == current_mode:
|
||||
current_lines.append(line)
|
||||
continue
|
||||
|
||||
segments.append((current_mode, current_lines))
|
||||
current_mode = classification
|
||||
current_lines = [line]
|
||||
|
||||
if current_lines:
|
||||
segments.append((current_mode or "text", current_lines))
|
||||
|
||||
cleaned_parts: List[str] = []
|
||||
for mode, block_lines in segments:
|
||||
block_text = "\n".join(block_lines).strip("\n")
|
||||
if not block_text:
|
||||
continue
|
||||
if mode == "code":
|
||||
cleaned_parts.append(_format_code_block(block_text, language_hint))
|
||||
else:
|
||||
cleaned_parts.append(block_text)
|
||||
|
||||
return "\n\n".join(part for part in cleaned_parts if part).strip()
|
||||
|
||||
|
||||
def _format_code_block(content: str, language_hint: str) -> str:
|
||||
language = language_hint if language_hint and language_hint != "text" else ""
|
||||
inner = content.strip("\n")
|
||||
return f"```{language}\n{inner}\n```"
|
||||
|
||||
|
||||
def _is_blank(line: str) -> bool:
|
||||
return not line.strip()
|
||||
|
||||
|
||||
def _is_likely_code_line(line: str) -> bool:
|
||||
if line.startswith((" ", "\t")):
|
||||
return True
|
||||
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
return False
|
||||
|
||||
if stripped.startswith(_BULLET_PREFIXES):
|
||||
return False
|
||||
|
||||
if stripped.startswith(_COMMENT_PREFIXES):
|
||||
return True
|
||||
|
||||
if any(stripped.startswith(prefix) for prefix in _CODE_PREFIXES):
|
||||
return True
|
||||
|
||||
if stripped.endswith(("{", "}", ";")):
|
||||
return True
|
||||
|
||||
if stripped.startswith(("}", "{", "case ", "default:")):
|
||||
return True
|
||||
|
||||
if "(" in stripped and ")" in stripped:
|
||||
return True
|
||||
|
||||
if "=" in stripped:
|
||||
return True
|
||||
|
||||
symbol_count = sum(1 for ch in stripped if ch in _CODE_SYMBOLS)
|
||||
letter_count = sum(1 for ch in stripped if ch.isalpha())
|
||||
if symbol_count >= 2 and symbol_count >= max(1, letter_count * 0.3):
|
||||
return True
|
||||
|
||||
if re.search(r"\)\s*->", stripped):
|
||||
return True
|
||||
|
||||
return False
|
||||
@@ -0,0 +1,46 @@
|
||||
import re
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
from config import TESTS_DIR
|
||||
|
||||
|
||||
def _natural_key(path: Path) -> List[object]:
|
||||
parts = re.split(r"(\d+)", path.stem)
|
||||
return [int(part) if part.isdigit() else part.lower() for part in parts]
|
||||
|
||||
|
||||
def load_test_prompts() -> List[Dict[str, str]]:
|
||||
"""Load prompt files from the tests directory."""
|
||||
if not TESTS_DIR.exists():
|
||||
return []
|
||||
|
||||
prompt_entries: List[Dict[str, str]] = []
|
||||
|
||||
for prompt_path in sorted(TESTS_DIR.glob("*.txt"), key=_natural_key):
|
||||
try:
|
||||
raw_text = prompt_path.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
prompt_text = textwrap.dedent(raw_text).strip()
|
||||
if not prompt_text:
|
||||
continue
|
||||
|
||||
title = prompt_path.stem
|
||||
for line in prompt_text.splitlines():
|
||||
stripped_line = line.strip()
|
||||
if stripped_line:
|
||||
title = stripped_line
|
||||
break
|
||||
|
||||
prompt_entries.append(
|
||||
{
|
||||
"title": title,
|
||||
"prompt": prompt_text,
|
||||
"filename": prompt_path.name,
|
||||
}
|
||||
)
|
||||
|
||||
return prompt_entries
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict, List, Type
|
||||
|
||||
from .base import ModelInfo, Provider, ProviderError
|
||||
from .lmstudio import LMStudioProvider
|
||||
from .local_engine import LocalEngineProvider
|
||||
|
||||
_PROVIDER_REGISTRY: Dict[str, Type[Provider]] = {
|
||||
LMStudioProvider.name: LMStudioProvider,
|
||||
LocalEngineProvider.name: LocalEngineProvider,
|
||||
}
|
||||
|
||||
|
||||
def list_provider_names() -> List[str]:
|
||||
return sorted(_PROVIDER_REGISTRY.keys())
|
||||
|
||||
|
||||
def get_provider(provider_name: str, **kwargs) -> Provider:
|
||||
try:
|
||||
provider_cls = _PROVIDER_REGISTRY[provider_name]
|
||||
except KeyError as exc:
|
||||
raise ProviderError(f"Unknown provider '{provider_name}'") from exc
|
||||
return provider_cls(**kwargs)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ModelInfo",
|
||||
"Provider",
|
||||
"ProviderError",
|
||||
"list_provider_names",
|
||||
"get_provider",
|
||||
"LocalEngineProvider",
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,29 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ModelInfo:
|
||||
id: str
|
||||
display_name: str
|
||||
|
||||
|
||||
class ProviderError(Exception):
|
||||
"""Base exception for provider-related errors."""
|
||||
|
||||
|
||||
class Provider(ABC):
|
||||
"""Abstract interface for model providers."""
|
||||
|
||||
name: str
|
||||
|
||||
@abstractmethod
|
||||
def list_models(self) -> List[ModelInfo]:
|
||||
"""Return the models available from this provider."""
|
||||
|
||||
@abstractmethod
|
||||
def run_prompt(self, model_id: str, prompt: str, *, temperature: float) -> Dict[str, object]:
|
||||
"""Execute a prompt against a model and return the response payload."""
|
||||
@@ -0,0 +1,100 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
from config import get_env_setting
|
||||
from .base import ModelInfo, Provider, ProviderError
|
||||
|
||||
DEFAULT_BASE_URL = "http://localhost:1234"
|
||||
|
||||
|
||||
class LMStudioProvider(Provider):
|
||||
name = "LM Studio"
|
||||
|
||||
def __init__(self, base_url: Optional[str] = None) -> None:
|
||||
self.base_url = base_url or get_env_setting("LM_STUDIO_BASE_URL", DEFAULT_BASE_URL)
|
||||
self._models_url = f"{self.base_url}/v1/models"
|
||||
self._chat_url = f"{self.base_url}/v1/chat/completions"
|
||||
|
||||
def list_models(self) -> List[ModelInfo]:
|
||||
try:
|
||||
response = requests.get(self._models_url, timeout=10)
|
||||
response.raise_for_status()
|
||||
except requests.exceptions.RequestException as exc:
|
||||
raise ProviderError(f"Failed to fetch models from LM Studio: {exc}") from exc
|
||||
|
||||
try:
|
||||
data = response.json()
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ProviderError(f"Failed to decode model list: {exc}") from exc
|
||||
|
||||
models = data.get("data", [])
|
||||
result: List[ModelInfo] = []
|
||||
if isinstance(models, list):
|
||||
for item in models:
|
||||
if isinstance(item, dict):
|
||||
model_id = str(item.get("id") or item.get("name") or "")
|
||||
else:
|
||||
model_id = str(item)
|
||||
if not model_id:
|
||||
continue
|
||||
result.append(ModelInfo(id=model_id, display_name=model_id))
|
||||
|
||||
default_model = data.get("default_model") or data.get("model") or data.get("id")
|
||||
if default_model and all(m.id != default_model for m in result):
|
||||
result.insert(0, ModelInfo(id=str(default_model), display_name=str(default_model)))
|
||||
|
||||
if not result:
|
||||
raise ProviderError("LM Studio returned no models.")
|
||||
|
||||
return result
|
||||
|
||||
def run_prompt(self, model_id: str, prompt: str, *, temperature: float) -> Dict[str, object]:
|
||||
headers = {"Content-Type": "application/json"}
|
||||
payload = {
|
||||
"model": model_id,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": temperature,
|
||||
}
|
||||
|
||||
start_time = time.time()
|
||||
try:
|
||||
response = requests.post(
|
||||
self._chat_url,
|
||||
headers=headers,
|
||||
data=json.dumps(payload),
|
||||
timeout=180,
|
||||
)
|
||||
response.raise_for_status()
|
||||
except requests.exceptions.RequestException as exc:
|
||||
return {"error": f"API request failed: {exc}"}
|
||||
|
||||
end_time = time.time()
|
||||
|
||||
try:
|
||||
data = response.json()
|
||||
except json.JSONDecodeError as exc:
|
||||
return {"error": f"Failed to parse response JSON: {exc}\nRaw: {response.text}"}
|
||||
|
||||
response_time = end_time - start_time
|
||||
content = data.get("choices", [{}])[0].get("message", {}).get("content", "Error: No content found.")
|
||||
usage = data.get("usage", {})
|
||||
total_tokens = usage.get("completion_tokens", 0)
|
||||
time_to_first_token = usage.get("prompt_eval_duration", 0) / 1_000_000_000
|
||||
stop_reason = data.get("choices", [{}])[0].get("finish_reason", "N/A")
|
||||
|
||||
tokens_per_second = total_tokens / response_time if response_time > 0 else 0
|
||||
|
||||
return {
|
||||
"response": content,
|
||||
"metrics": {
|
||||
"tokens_per_second": round(tokens_per_second, 2),
|
||||
"total_tokens": total_tokens,
|
||||
"time_to_first_token": round(time_to_first_token, 2),
|
||||
"stop_reason": stop_reason,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from config import MODELS_DIR
|
||||
from engine_loader import EngineLoadError, load_engine_class
|
||||
from .base import ModelInfo, Provider, ProviderError
|
||||
|
||||
|
||||
class LocalEngineProvider(Provider):
|
||||
name = "Local Engine"
|
||||
|
||||
def __init__(self, search_paths: Optional[List[Path]] = None) -> None:
|
||||
try:
|
||||
runtime_cls = load_engine_class()
|
||||
self.runtime = runtime_cls()
|
||||
self.runtime.setup()
|
||||
except EngineLoadError as exc:
|
||||
raise ProviderError(str(exc)) from exc
|
||||
except RuntimeError as exc:
|
||||
raise ProviderError(str(exc)) from exc
|
||||
|
||||
paths: List[Path] = [MODELS_DIR]
|
||||
env_override = os.getenv("LOCAL_LLM_PATHS")
|
||||
if env_override:
|
||||
for entry in env_override.split(os.pathsep):
|
||||
entry_clean = entry.strip()
|
||||
if not entry_clean:
|
||||
continue
|
||||
entry_path = Path(entry_clean).expanduser()
|
||||
if entry_path not in paths:
|
||||
paths.append(entry_path)
|
||||
if search_paths:
|
||||
for candidate in search_paths:
|
||||
if candidate not in paths:
|
||||
paths.append(candidate)
|
||||
|
||||
self.search_paths = paths
|
||||
self._model_index: Dict[str, Path] = {}
|
||||
|
||||
def list_models(self) -> List[ModelInfo]:
|
||||
discovered = self.runtime.discover_gguf_models(self.search_paths)
|
||||
self._model_index.clear()
|
||||
models: List[ModelInfo] = []
|
||||
for path in discovered:
|
||||
model_id = self._register_model(path)
|
||||
models.append(ModelInfo(id=model_id, display_name=path.stem))
|
||||
if not models:
|
||||
raise ProviderError(
|
||||
"No GGUF models were found. Place models inside the 'models' directory or set LOCAL_LLM_PATHS."
|
||||
)
|
||||
return models
|
||||
|
||||
def run_prompt(self, model_id: str, prompt: str, *, temperature: float) -> Dict[str, object]:
|
||||
try:
|
||||
model_path = self._model_index[model_id]
|
||||
except KeyError as exc:
|
||||
raise ProviderError(f"Model '{model_id}' is not registered. Refresh the model list and try again.") from exc
|
||||
|
||||
self.runtime.load_model(model_path)
|
||||
return self.runtime.generate(prompt, temperature=temperature)
|
||||
|
||||
def _register_model(self, path: Path) -> str:
|
||||
for existing_id, existing_path in self._model_index.items():
|
||||
if path == existing_path:
|
||||
return existing_id
|
||||
|
||||
base_id = path.stem
|
||||
candidate = base_id
|
||||
counter = 1
|
||||
while candidate in self._model_index:
|
||||
digest = hashlib.sha1(str(path).encode("utf-8")).hexdigest()[:6]
|
||||
candidate = f"{base_id}-{digest}-{counter}"
|
||||
counter += 1
|
||||
|
||||
self._model_index[candidate] = path
|
||||
return candidate
|
||||
@@ -0,0 +1,206 @@
|
||||
import hashlib
|
||||
import re
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from config import RESULTS_DIR, TEMPLATE_PATH, TEMP_DIR
|
||||
from markdown_linter import infer_language_from_prompt, lint_response_markdown
|
||||
|
||||
_TEMPLATE_CACHE: Optional[str] = None
|
||||
|
||||
|
||||
class TemplateNotFoundError(FileNotFoundError):
|
||||
pass
|
||||
|
||||
|
||||
def load_template() -> str:
|
||||
"""Load the test block template from disk, caching the content."""
|
||||
global _TEMPLATE_CACHE
|
||||
if _TEMPLATE_CACHE is None:
|
||||
if not TEMPLATE_PATH.exists():
|
||||
raise TemplateNotFoundError(
|
||||
f"Template file not found at '{TEMPLATE_PATH}'. Please create it before running tests."
|
||||
)
|
||||
_TEMPLATE_CACHE = TEMPLATE_PATH.read_text(encoding="utf-8")
|
||||
return _TEMPLATE_CACHE
|
||||
|
||||
|
||||
def has_unclosed_code_block(markdown_text: str) -> bool:
|
||||
"""Check if markdown text ends inside an unclosed triple-backtick block."""
|
||||
fence_count = markdown_text.count("```")
|
||||
return fence_count % 2 == 1
|
||||
|
||||
|
||||
def render_response_block(result: Dict[str, object], *, language_hint: str) -> str:
|
||||
"""Render the response portion of the template based on the result payload."""
|
||||
if "error" in result:
|
||||
return "\n".join(["### Response:", f"**ERROR:** {result['error']}", ""])
|
||||
|
||||
cleaned_response = lint_response_markdown(result["response"], language_hint=language_hint)
|
||||
|
||||
section_lines = ["### Response:"]
|
||||
if cleaned_response:
|
||||
section_lines.append("")
|
||||
section_lines.append(cleaned_response)
|
||||
section_lines.append("")
|
||||
return "\n".join(section_lines)
|
||||
|
||||
|
||||
def render_test_block(
|
||||
prompt_info: Dict[str, str],
|
||||
result: Dict[str, object],
|
||||
index: int,
|
||||
) -> str:
|
||||
"""Populate the test block template with data for a single test."""
|
||||
template = load_template()
|
||||
|
||||
metrics = result.get("metrics", {}) if "error" not in result else {}
|
||||
|
||||
ttft_value = metrics.get("time_to_first_token") if metrics else None
|
||||
language_hint = infer_language_from_prompt(prompt_info["prompt"])
|
||||
|
||||
replacements = {
|
||||
"{{TEST_NUMBER}}": str(index),
|
||||
"{{TEST_TITLE}}": prompt_info.get("title", f"Test {index}"),
|
||||
"{{SOURCE_FILENAME}}": prompt_info.get("filename", "unknown"),
|
||||
"{{PROMPT_CONTENT}}": prompt_info["prompt"].strip(),
|
||||
"{{RESPONSE_BLOCK}}": render_response_block(result, language_hint=language_hint).rstrip(),
|
||||
"{{METRIC_TOKENS_PER_SECOND}}": (
|
||||
str(metrics.get("tokens_per_second", "N/A"))
|
||||
if metrics
|
||||
else "N/A"
|
||||
),
|
||||
"{{METRIC_TOTAL_TOKENS}}": (
|
||||
str(metrics.get("total_tokens", "N/A"))
|
||||
if metrics
|
||||
else "N/A"
|
||||
),
|
||||
"{{METRIC_TTFT}}": (
|
||||
f"{ttft_value}s" if isinstance(ttft_value, (int, float)) else "N/A"
|
||||
),
|
||||
"{{METRIC_STOP_REASON}}": (
|
||||
str(metrics.get("stop_reason", "N/A"))
|
||||
if metrics
|
||||
else "N/A"
|
||||
),
|
||||
}
|
||||
|
||||
rendered = template
|
||||
for placeholder, value in replacements.items():
|
||||
rendered = rendered.replace(placeholder, value)
|
||||
|
||||
return rendered.rstrip() + "\n\n"
|
||||
|
||||
|
||||
def sanitize_model_name(model_name: str) -> str:
|
||||
sanitized = re.sub(r"[^A-Za-z0-9._-]+", "_", model_name.strip())
|
||||
return sanitized or "unknown-model"
|
||||
|
||||
|
||||
def initialize_report_file(model_label: str) -> Path:
|
||||
"""Create the markdown report shell and return its path."""
|
||||
sanitized = sanitize_model_name(model_label)
|
||||
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
report_path = RESULTS_DIR / f"automated_report_{sanitized}.md"
|
||||
|
||||
header = textwrap.dedent(
|
||||
f"""
|
||||
# Automated Diagnostic Report: {model_label}
|
||||
|
||||
---
|
||||
|
||||
## Performance Summary
|
||||
|
||||
<!--SUMMARY_START-->
|
||||
* **Average Tokens/s:** TBD
|
||||
* **Average Time to First Token:** TBD
|
||||
* **Total Tokens Generated:** TBD
|
||||
<!--SUMMARY_END-->
|
||||
|
||||
## Qualitative Analysis
|
||||
|
||||
*(Manual grading and analysis of the responses is required to determine the final letter grade.)*
|
||||
|
||||
---
|
||||
|
||||
"""
|
||||
).lstrip()
|
||||
|
||||
report_path.write_text(header, encoding="utf-8")
|
||||
return report_path
|
||||
|
||||
|
||||
def append_test_result(
|
||||
report_path: Path,
|
||||
prompt_info: Dict[str, str],
|
||||
result: Dict[str, object],
|
||||
index: int,
|
||||
) -> None:
|
||||
"""Append a single test section to the markdown report."""
|
||||
block_content = render_test_block(prompt_info, result, index)
|
||||
|
||||
slug_base = re.sub(r"[^A-Za-z0-9._-]+", "-", prompt_info.get("title", f"test-{index}")).strip("-")
|
||||
if not slug_base:
|
||||
slug_base = f"test-{index}"
|
||||
slug_hash = hashlib.sha1(slug_base.encode("utf-8")).hexdigest()[:8]
|
||||
truncated_slug = slug_base[:48]
|
||||
title_slug = f"{truncated_slug}-{slug_hash}"
|
||||
temp_file = TEMP_DIR / f"{index:03d}_{title_slug}.md"
|
||||
|
||||
try:
|
||||
temp_file.write_text(block_content, encoding="utf-8")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
block_text = temp_file.read_text(encoding="utf-8") if temp_file.exists() else block_content
|
||||
if has_unclosed_code_block(block_text):
|
||||
block_text = block_text.rstrip() + "\n```\n"
|
||||
|
||||
separator = "\n\n---\n\n"
|
||||
existing_tail = report_path.read_text(encoding="utf-8") if report_path.exists() else ""
|
||||
needs_separator = existing_tail and not existing_tail.endswith(separator)
|
||||
|
||||
with report_path.open("a", encoding="utf-8") as report_file:
|
||||
if existing_tail and has_unclosed_code_block(existing_tail):
|
||||
report_file.write("\n```\n\n")
|
||||
if needs_separator and existing_tail:
|
||||
report_file.write(separator)
|
||||
report_file.write(block_text)
|
||||
|
||||
if temp_file.exists():
|
||||
try:
|
||||
temp_file.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def finalize_report_summary(report_path: Path, results: List[Dict[str, object]]) -> None:
|
||||
"""Update the performance summary placeholder once all tests have run."""
|
||||
valid_results = [r for r in results if "error" not in r]
|
||||
|
||||
if valid_results:
|
||||
total_tok_s = sum(r["metrics"]["tokens_per_second"] for r in valid_results)
|
||||
total_ttft = sum(r["metrics"]["time_to_first_token"] for r in valid_results)
|
||||
total_tokens = sum(r["metrics"]["total_tokens"] for r in valid_results)
|
||||
avg_tok_s = round(total_tok_s / len(valid_results), 2)
|
||||
avg_ttft = round(total_ttft / len(valid_results), 2)
|
||||
else:
|
||||
avg_tok_s = avg_ttft = total_tokens = 0
|
||||
|
||||
summary_block = textwrap.dedent(
|
||||
f"""
|
||||
* **Average Tokens/s:** {avg_tok_s}
|
||||
* **Average Time to First Token:** {avg_ttft}s
|
||||
* **Total Tokens Generated:** {total_tokens}
|
||||
"""
|
||||
).strip()
|
||||
|
||||
content = report_path.read_text(encoding="utf-8")
|
||||
updated_content = re.sub(
|
||||
r"<!--SUMMARY_START-->.*?<!--SUMMARY_END-->",
|
||||
f"<!--SUMMARY_START-->\n{summary_block}\n<!--SUMMARY_END-->",
|
||||
content,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
report_path.write_text(updated_content, encoding="utf-8")
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Callable, Dict, List, Optional
|
||||
|
||||
from config import (
|
||||
DEFAULT_PROVIDER_NAME,
|
||||
DEFAULT_TEMPERATURE,
|
||||
TEMPLATE_PATH,
|
||||
ensure_directories,
|
||||
reset_temp_directory,
|
||||
)
|
||||
from prompts import load_test_prompts
|
||||
from reporting import (
|
||||
TemplateNotFoundError,
|
||||
append_test_result,
|
||||
finalize_report_summary,
|
||||
initialize_report_file,
|
||||
)
|
||||
from providers import LocalEngineProvider, ModelInfo, ProviderError, get_provider
|
||||
from settings import get_local_model_paths
|
||||
|
||||
|
||||
class TestRunError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _choose_model(models: List[ModelInfo], requested_id: Optional[str]) -> ModelInfo:
|
||||
if not models:
|
||||
raise TestRunError("No models available from provider.")
|
||||
|
||||
if requested_id:
|
||||
for model in models:
|
||||
if model.id == requested_id:
|
||||
return model
|
||||
raise TestRunError(f"Model '{requested_id}' not found for the selected provider.")
|
||||
|
||||
return models[0]
|
||||
|
||||
|
||||
def run_suite(
|
||||
*,
|
||||
provider_name: Optional[str] = None,
|
||||
model_id: Optional[str] = None,
|
||||
temperature: Optional[float] = None,
|
||||
progress_callback: Optional[Callable[[int, int, Dict[str, str], Dict[str, object]], None]] = None,
|
||||
) -> Path:
|
||||
"""Run the diagnostic test suite and return the generated report path."""
|
||||
ensure_directories()
|
||||
reset_temp_directory()
|
||||
|
||||
provider_name = provider_name or DEFAULT_PROVIDER_NAME
|
||||
provider_kwargs = {}
|
||||
if provider_name == LocalEngineProvider.name:
|
||||
custom_paths = [Path(p).expanduser() for p in get_local_model_paths()]
|
||||
provider_kwargs["search_paths"] = [path for path in custom_paths if path]
|
||||
|
||||
try:
|
||||
provider = get_provider(provider_name, **provider_kwargs)
|
||||
except ProviderError as exc:
|
||||
raise TestRunError(str(exc)) from exc
|
||||
|
||||
try:
|
||||
models = provider.list_models()
|
||||
except ProviderError as exc:
|
||||
raise TestRunError(str(exc)) from exc
|
||||
|
||||
model = _choose_model(models, model_id)
|
||||
|
||||
if not TEMPLATE_PATH.exists():
|
||||
raise TestRunError(
|
||||
"Template file missing. Please create '.core/templates/test-block.md' before running tests."
|
||||
)
|
||||
|
||||
prompts = load_test_prompts()
|
||||
if not prompts:
|
||||
raise TestRunError("No test prompts found in the 'tests' directory.")
|
||||
|
||||
report_path = initialize_report_file(model.display_name)
|
||||
all_results: List[Dict[str, object]] = []
|
||||
selected_temperature = temperature if temperature is not None else DEFAULT_TEMPERATURE
|
||||
|
||||
total_tests = len(prompts)
|
||||
|
||||
for index, prompt in enumerate(prompts, start=1):
|
||||
result = provider.run_prompt(model.id, prompt["prompt"], temperature=selected_temperature)
|
||||
all_results.append(result)
|
||||
append_test_result(report_path, prompt, result, index)
|
||||
if progress_callback:
|
||||
progress_callback(index, total_tests, prompt, result)
|
||||
|
||||
finalize_report_summary(report_path, all_results)
|
||||
return report_path
|
||||
|
||||
|
||||
def safe_run() -> int:
|
||||
"""CLI helper for running the suite with basic error handling."""
|
||||
def _print_progress(index: int, total: int, prompt: Dict[str, str], result: Dict[str, object]) -> None:
|
||||
status = "FAILED" if "error" in result else "DONE"
|
||||
filename_label = prompt.get("filename", f"test{index}")
|
||||
title = prompt.get("title", f"Test {index}")
|
||||
print(f"Running {title} [{filename_label}] ({index}/{total})... {status}")
|
||||
|
||||
print(f"Starting automated diagnostic run with provider '{DEFAULT_PROVIDER_NAME}'…")
|
||||
|
||||
try:
|
||||
report_path = run_suite(progress_callback=_print_progress)
|
||||
except TestRunError as exc:
|
||||
print(f"Error: {exc}")
|
||||
return 1
|
||||
except TemplateNotFoundError as exc:
|
||||
print(f"Error: {exc}")
|
||||
return 1
|
||||
|
||||
print(f"\n✅ Success! Report saved to '{report_path.name}'.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(safe_run())
|
||||
@@ -0,0 +1,67 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
from config import CORE_DIR
|
||||
|
||||
SETTINGS_PATH = CORE_DIR / "user_settings.json"
|
||||
|
||||
_DEFAULT_LOCAL_PATHS: List[str] = []
|
||||
for candidate in (
|
||||
Path.home() / ".lmstudio",
|
||||
Path.home() / ".lmstudio/models",
|
||||
Path.home() / "Library/Application Support/lm-studio/models",
|
||||
):
|
||||
normalized = str(candidate)
|
||||
if normalized not in _DEFAULT_LOCAL_PATHS:
|
||||
_DEFAULT_LOCAL_PATHS.append(normalized)
|
||||
|
||||
_DEFAULT_SETTINGS: Dict[str, object] = {
|
||||
"local_model_paths": _DEFAULT_LOCAL_PATHS,
|
||||
}
|
||||
|
||||
|
||||
def load_settings() -> Dict[str, object]:
|
||||
if not SETTINGS_PATH.exists():
|
||||
return dict(_DEFAULT_SETTINGS)
|
||||
try:
|
||||
data = json.loads(SETTINGS_PATH.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return dict(_DEFAULT_SETTINGS)
|
||||
if not isinstance(data, dict):
|
||||
return dict(_DEFAULT_SETTINGS)
|
||||
merged = dict(_DEFAULT_SETTINGS)
|
||||
merged.update(data)
|
||||
return merged
|
||||
|
||||
|
||||
def save_settings(settings: Dict[str, object]) -> None:
|
||||
SETTINGS_PATH.write_text(json.dumps(settings, indent=2, sort_keys=True), encoding="utf-8")
|
||||
|
||||
|
||||
def get_local_model_paths() -> List[str]:
|
||||
settings = load_settings()
|
||||
paths = settings.get("local_model_paths", [])
|
||||
if not isinstance(paths, list):
|
||||
return []
|
||||
result: List[str] = []
|
||||
for entry in paths:
|
||||
if isinstance(entry, str) and entry.strip():
|
||||
normalized = str(Path(entry).expanduser())
|
||||
if normalized not in result:
|
||||
result.append(normalized)
|
||||
return result
|
||||
|
||||
|
||||
def set_local_model_paths(paths: List[str]) -> None:
|
||||
settings = load_settings()
|
||||
normalized: List[str] = []
|
||||
for entry in paths:
|
||||
if isinstance(entry, str) and entry.strip():
|
||||
candidate = str(Path(entry).expanduser())
|
||||
if candidate not in normalized:
|
||||
normalized.append(candidate)
|
||||
settings["local_model_paths"] = normalized
|
||||
save_settings(settings)
|
||||
@@ -0,0 +1,18 @@
|
||||
## Test {{TEST_NUMBER}}: {{TEST_TITLE}}
|
||||
|
||||
*Source:* `{{SOURCE_FILENAME}}`
|
||||
|
||||
### Prompt:
|
||||
```swift
|
||||
{{PROMPT_CONTENT}}
|
||||
```
|
||||
|
||||
{{RESPONSE_BLOCK}}
|
||||
|
||||
### Performance Metrics:
|
||||
* **Tokens/s:** {{METRIC_TOKENS_PER_SECOND}}
|
||||
* **Total Tokens:** {{METRIC_TOTAL_TOKENS}}
|
||||
* **Time to First Token:** {{METRIC_TTFT}}
|
||||
* **Stop Reason:** {{METRIC_STOP_REASON}}
|
||||
|
||||
---
|
||||
Reference in New Issue
Block a user