feat: add testing framework and initial test cases

- Updated package.json to include Vitest and testing dependencies.
- Created test cases for SuitePicker component to validate selection logic.
- Added tests for ReportsPage to ensure correct report grouping and ordering.
- Implemented read-only enforcement tests for SuitePage to prevent actions on built-in suites.
- Introduced a setup file for Vitest to include jest-dom matchers and cleanup after tests.
- Modified ReportsPage to group reports by model and display them accordingly.
- Enhanced API types to include suite_scope and question_count for better report handling.

server/api.py

@@ -395,6 +395,22 @@ def _model_label_from(content: str, fallback: str) -> str:
     return match.group(1).strip() if match else fallback
 
 
+def _parse_report_name(stem: str) -> tuple:
+    """Pull ``(suite_scope, question_count)`` out of a report filename.
+
+    Names look like ``<model>__<timestamp>__<scope>__<n>q``. Reports written
+    before that scheme return ``("", None)`` rather than a guess — claiming a
+    scope for a file that never recorded one is how the old flat names became
+    misleading in the first place.
+    """
+    parts = stem.split("__")
+    if len(parts) < 4:
+        return "", None
+    scope = parts[-2]
+    match = re.fullmatch(r"(\d+)q", parts[-1])
+    return scope, int(match.group(1)) if match else None
+
+
 @router.get("/reports", response_model=List[ReportSummary])
 async def get_reports() -> List[ReportSummary]:
     if not RESULTS_DIR.exists():
@@ -407,12 +423,15 @@ async def get_reports() -> List[ReportSummary]:
             head = path.read_text(encoding="utf-8", errors="replace")[:400]
         except OSError:
             continue
+        scope, count = _parse_report_name(path.stem)
         summaries.append(
             ReportSummary(
                 name=path.name,
                 model_label=_model_label_from(head, path.stem),
                 size_bytes=stat.st_size,
                 modified_at=stat.st_mtime,
+                suite_scope=scope,
+                question_count=count,
             )
         )
     summaries.sort(key=lambda item: item.modified_at, reverse=True)

server/run_manager.py

@@ -22,14 +22,12 @@ from pathlib import Path
 from typing import Any, AsyncIterator, Dict, List, Optional
 
 from .core_bridge import (
-    RESULTS_DIR,
     TemplateNotFoundError,
     TestRunError,
     append_sections,
     finalize_report_summary,
     replace_analysis_section,
     run_suite,
-    sanitize_model_name,
 )
 from .plugins import (
     GradeEntry,
@@ -122,6 +120,12 @@ class RunState:
     #: entire rubric from the prompt, so a collision yields confident,
     #: plausible, wrong scores with no error anywhere. Never key by filename.
     prompts_by_id: Dict[str, str] = field(default_factory=dict)
+    #: Set by the engine the moment the report file is created.
+    #:
+    #: Report names now carry a timestamp, so this can no longer be
+    #: reconstructed from the model label — three places used to rebuild it
+    #: independently, which silently pointed at the wrong file.
+    report_path: Optional[Path] = None
 
     @property
     def completed(self) -> int:
@@ -347,12 +351,16 @@ class RunManager:
         )
 
         try:
+            def _remember_report(path: Path) -> None:
+                run.report_path = path
+
             report_path = run_suite(
                 provider_name=run.provider,
                 model_id=run.model_id,
                 temperature=run.temperature,
                 progress_callback=progress_callback,
                 prompts=prompts,
+                on_report_created=_remember_report,
             )
         except RunCancelled:
             self._finalize_partial_report(run)
@@ -400,15 +408,12 @@ class RunManager:
 
     # ---------------------------------------------------------------- plugins
 
-    def _report_path(self, run: RunState) -> Path:
-        return RESULTS_DIR / f"automated_report_{sanitize_model_name(run.model_label)}.md"
-
     def _record(self, run: RunState):
-        path = self._report_path(run)
+        path = run.report_path
         return build_run_record(
             run,
             prompts_by_id=run.prompts_by_id,
-            report_path=path if path.exists() else None,
+            report_path=path if path and path.exists() else None,
         )
 
     def _grade(self, run: RunState, outcome: TestOutcome) -> None:
@@ -450,8 +455,8 @@ class RunManager:
 
     def _apply_plugin_report(self, run: RunState) -> None:
         """Write grades and plugin sections into the finished report."""
-        report_path = self._report_path(run)
-        if not report_path.exists():
+        report_path = run.report_path
+        if report_path is None or not report_path.exists():
             return
 
         record = self._record(run)
@@ -477,8 +482,8 @@ class RunManager:
 
     def _finalize_partial_report(self, run: RunState) -> None:
         """Write the summary block for a run that stopped early."""
-        report_path = RESULTS_DIR / f"automated_report_{sanitize_model_name(run.model_label)}.md"
-        if not report_path.exists():
+        report_path = run.report_path
+        if report_path is None or not report_path.exists():
             return
         results: List[Dict[str, Any]] = []
         for outcome in run.outcomes:

server/schemas.py

@@ -169,6 +169,11 @@ class ReportSummary(BaseModel):
     model_label: str
     size_bytes: int
     modified_at: float
+    #: Parsed out of the filename so the list can group by model and show what
+    #: each run actually covered. Older reports predate the scheme and leave
+    #: these empty rather than guessing.
+    suite_scope: str = ""
+    question_count: Optional[int] = None
 
 
 class ReportDetail(ReportSummary):