feat: add testing framework and initial test cases
- Updated package.json to include Vitest and testing dependencies. - Created test cases for SuitePicker component to validate selection logic. - Added tests for ReportsPage to ensure correct report grouping and ordering. - Implemented read-only enforcement tests for SuitePage to prevent actions on built-in suites. - Introduced a setup file for Vitest to include jest-dom matchers and cleanup after tests. - Modified ReportsPage to group reports by model and display them accordingly. - Enhanced API types to include suite_scope and question_count for better report handling.
This commit is contained in:
+10
-1
@@ -46,6 +46,7 @@ def run_suite(
|
||||
temperature: Optional[float] = None,
|
||||
progress_callback: Optional[Callable[[int, int, Dict[str, str], Dict[str, object]], None]] = None,
|
||||
prompts: Optional[List[Dict[str, str]]] = None,
|
||||
on_report_created: Optional[Callable[[Path], None]] = None,
|
||||
) -> Path:
|
||||
"""Run the diagnostic test suite and return the generated report path.
|
||||
|
||||
@@ -84,7 +85,15 @@ def run_suite(
|
||||
if not prompts:
|
||||
raise TestRunError("No test prompts found in the 'tests' directory.")
|
||||
|
||||
report_path = initialize_report_file(model.display_name)
|
||||
# Scope comes straight from the prompts, so the report name always matches
|
||||
# what actually ran — including a subset chosen in the UI.
|
||||
suite_slugs = [str(p.get("suite", "")) for p in prompts]
|
||||
report_path = initialize_report_file(model.display_name, suite_slugs, len(prompts))
|
||||
if on_report_created is not None:
|
||||
# The caller needs this before run_suite returns: a cancelled run still
|
||||
# finalizes its partial report, and the path can no longer be
|
||||
# reconstructed from the model name now that it carries a timestamp.
|
||||
on_report_created(report_path)
|
||||
all_results: List[Dict[str, object]] = []
|
||||
selected_temperature = temperature if temperature is not None else DEFAULT_TEMPERATURE
|
||||
|
||||
|
||||
Reference in New Issue
Block a user