feat: add Settings and Suite pages with comprehensive settings management and question suite builder

- Implemented SettingsPage for configuring default provider, temperature, and model paths.
- Added SuitePage for managing a suite of questions with features to add, edit, duplicate, and delete questions.
- Introduced TypeScript configuration files for app and node environments.
- Set up Vite configuration for development server with API proxying to backend.
This commit is contained in:
Christopher Clendening
2026-07-27 17:41:08 -04:00
parent ce0e022f11
commit adf61ae1a0
50 changed files with 10952 additions and 164 deletions
+41
View File
@@ -120,7 +120,9 @@ def initialize_report_file(model_label: str) -> Path:
## Qualitative Analysis
<!--ANALYSIS_START-->
*(Manual grading and analysis of the responses is required to determine the final letter grade.)*
<!--ANALYSIS_END-->
---
@@ -204,3 +206,42 @@ def finalize_report_summary(report_path: Path, results: List[Dict[str, object]])
flags=re.DOTALL,
)
report_path.write_text(updated_content, encoding="utf-8")
def replace_analysis_section(report_path: Path, markdown: str) -> None:
"""Swap the qualitative-analysis placeholder for generated content.
Used when grader plugins produce scores, so the report carries real grades
instead of the "grade this by hand" note. No-op if the markers are absent
(an older report, or a customized header).
"""
if not report_path.exists():
return
content = report_path.read_text(encoding="utf-8")
if "<!--ANALYSIS_START-->" not in content:
return
updated = re.sub(
r"<!--ANALYSIS_START-->.*?<!--ANALYSIS_END-->",
lambda _: f"<!--ANALYSIS_START-->\n{markdown.strip()}\n<!--ANALYSIS_END-->",
content,
flags=re.DOTALL,
)
report_path.write_text(updated, encoding="utf-8")
def append_sections(report_path: Path, sections: List[str]) -> None:
"""Append extra markdown blocks to the end of a finished report."""
blocks = [block.strip() for block in sections if block and block.strip()]
if not blocks or not report_path.exists():
return
existing = report_path.read_text(encoding="utf-8")
with report_path.open("a", encoding="utf-8") as report_file:
if has_unclosed_code_block(existing):
report_file.write("\n```\n")
if not existing.endswith("\n"):
report_file.write("\n")
for block in blocks:
report_file.write(f"\n---\n\n{block}\n")
+9 -2
View File
@@ -45,8 +45,14 @@ def run_suite(
model_id: Optional[str] = None,
temperature: Optional[float] = None,
progress_callback: Optional[Callable[[int, int, Dict[str, str], Dict[str, object]], None]] = None,
prompts: Optional[List[Dict[str, str]]] = None,
) -> Path:
"""Run the diagnostic test suite and return the generated report path."""
"""Run the diagnostic test suite and return the generated report path.
When ``prompts`` is omitted every prompt in the ``tests`` directory is run.
Callers may pass an explicit subset (same shape as ``load_test_prompts``)
to re-run only part of the suite.
"""
ensure_directories()
reset_temp_directory()
@@ -73,7 +79,8 @@ def run_suite(
"Template file missing. Please create '.core/templates/test-block.md' before running tests."
)
prompts = load_test_prompts()
if prompts is None:
prompts = load_test_prompts()
if not prompts:
raise TestRunError("No test prompts found in the 'tests' directory.")