feat: add Settings and Suite pages with comprehensive settings management and question suite builder
- Implemented SettingsPage for configuring default provider, temperature, and model paths. - Added SuitePage for managing a suite of questions with features to add, edit, duplicate, and delete questions. - Introduced TypeScript configuration files for app and node environments. - Set up Vite configuration for development server with API proxying to backend.
This commit is contained in:
Christopher Clendening
@@ -0,0 +1,281 @@
|
||||
/** Typed client for the LM-Gambit Python API. */
|
||||
|
||||
export interface ProviderSummary {
|
||||
name: string
|
||||
is_default: boolean
|
||||
}
|
||||
|
||||
export interface ModelSummary {
|
||||
id: string
|
||||
display_name: string
|
||||
}
|
||||
|
||||
export interface TestPrompt {
|
||||
filename: string
|
||||
title: string
|
||||
prompt: string
|
||||
}
|
||||
|
||||
export interface TestSuite {
|
||||
tests: TestPrompt[]
|
||||
directory: string
|
||||
}
|
||||
|
||||
export interface RunMetrics {
|
||||
tokens_per_second: number
|
||||
total_tokens: number
|
||||
time_to_first_token: number
|
||||
stop_reason: string
|
||||
}
|
||||
|
||||
export interface RunSummary {
|
||||
average_tokens_per_second: number
|
||||
average_time_to_first_token: number
|
||||
total_tokens: number
|
||||
passed: number
|
||||
failed: number
|
||||
overall_score: number | null
|
||||
graded: number
|
||||
}
|
||||
|
||||
/** One plugin's verdict on one answer. */
|
||||
export interface GradeResult {
|
||||
grader: string
|
||||
score: number
|
||||
label: string
|
||||
notes: string
|
||||
}
|
||||
|
||||
export interface PluginSummary {
|
||||
name: string
|
||||
slug: string
|
||||
version: string
|
||||
description: string
|
||||
path: string
|
||||
enabled: boolean
|
||||
hooks: string[]
|
||||
error: string | null
|
||||
}
|
||||
|
||||
export type RunStatus = 'running' | 'completed' | 'failed' | 'cancelled'
|
||||
|
||||
export interface Run {
|
||||
id: string
|
||||
status: RunStatus
|
||||
provider: string
|
||||
model_id: string
|
||||
model_label: string
|
||||
temperature: number
|
||||
total: number
|
||||
completed: number
|
||||
started_at: number
|
||||
finished_at: number | null
|
||||
report_name: string | null
|
||||
error: string | null
|
||||
summary: RunSummary
|
||||
}
|
||||
|
||||
export interface ReportSummary {
|
||||
name: string
|
||||
model_label: string
|
||||
size_bytes: number
|
||||
modified_at: number
|
||||
}
|
||||
|
||||
export interface ReportDetail extends ReportSummary {
|
||||
content: string
|
||||
}
|
||||
|
||||
export interface ModelPathEntry {
|
||||
nickname: string
|
||||
path: string
|
||||
}
|
||||
|
||||
export interface Settings {
|
||||
default_provider: string
|
||||
default_temperature: number
|
||||
local_model_paths: ModelPathEntry[]
|
||||
tests_dir: string
|
||||
results_dir: string
|
||||
models_dir: string
|
||||
}
|
||||
|
||||
export interface SystemInfo {
|
||||
version: string
|
||||
engine_architecture: string
|
||||
engine_runtime: string
|
||||
template_ok: boolean
|
||||
python_version: string
|
||||
metrics: Record<string, string>
|
||||
}
|
||||
|
||||
export interface PlaygroundResult {
|
||||
response: string | null
|
||||
error: string | null
|
||||
metrics: RunMetrics | null
|
||||
elapsed: number
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ events */
|
||||
|
||||
export interface TestCompletedEvent {
|
||||
type: 'test.completed'
|
||||
index: number
|
||||
total: number
|
||||
title: string
|
||||
filename: string
|
||||
status: 'ok' | 'error'
|
||||
elapsed: number
|
||||
response: string | null
|
||||
error: string | null
|
||||
metrics: RunMetrics | null
|
||||
grades: GradeResult[]
|
||||
score: number | null
|
||||
}
|
||||
|
||||
export interface RunStartedEvent {
|
||||
type: 'run.started'
|
||||
run: Run
|
||||
tests: { index: number; title: string; filename: string }[]
|
||||
}
|
||||
|
||||
export interface RunTerminalEvent {
|
||||
type: 'run.completed' | 'run.failed' | 'run.cancelled'
|
||||
run: Run
|
||||
message?: string
|
||||
}
|
||||
|
||||
export type RunEvent = RunStartedEvent | TestCompletedEvent | RunTerminalEvent
|
||||
|
||||
/* ------------------------------------------------------------------ client */
|
||||
|
||||
export class ApiError extends Error {
|
||||
status: number
|
||||
|
||||
constructor(message: string, status: number) {
|
||||
super(message)
|
||||
this.name = 'ApiError'
|
||||
this.status = status
|
||||
}
|
||||
}
|
||||
|
||||
async function request<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
let response: Response
|
||||
try {
|
||||
response = await fetch(`/api${path}`, {
|
||||
headers: init?.body ? { 'Content-Type': 'application/json' } : undefined,
|
||||
...init,
|
||||
})
|
||||
} catch {
|
||||
throw new ApiError('Cannot reach the LM-Gambit server. Is it still running?', 0)
|
||||
}
|
||||
|
||||
if (response.status === 204) return undefined as T
|
||||
|
||||
const raw = await response.text()
|
||||
let payload: unknown = null
|
||||
if (raw) {
|
||||
try {
|
||||
payload = JSON.parse(raw)
|
||||
} catch {
|
||||
payload = raw
|
||||
}
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const detail =
|
||||
payload && typeof payload === 'object' && 'detail' in payload
|
||||
? (payload as { detail: unknown }).detail
|
||||
: payload
|
||||
throw new ApiError(
|
||||
typeof detail === 'string' ? detail : `Request failed (${response.status})`,
|
||||
response.status,
|
||||
)
|
||||
}
|
||||
|
||||
return payload as T
|
||||
}
|
||||
|
||||
export const api = {
|
||||
system: () => request<SystemInfo>('/system'),
|
||||
|
||||
providers: () => request<ProviderSummary[]>('/providers'),
|
||||
models: (provider: string) =>
|
||||
request<{ provider: string; models: ModelSummary[] }>(
|
||||
`/providers/${encodeURIComponent(provider)}/models`,
|
||||
),
|
||||
|
||||
tests: () => request<TestSuite>('/tests'),
|
||||
saveTests: (prompts: string[]) =>
|
||||
request<TestSuite>('/tests', {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify({ tests: prompts.map((prompt) => ({ prompt })) }),
|
||||
}),
|
||||
|
||||
startRun: (body: {
|
||||
provider: string
|
||||
model_id: string
|
||||
temperature: number
|
||||
filenames?: string[]
|
||||
}) => request<Run>('/runs', { method: 'POST', body: JSON.stringify(body) }),
|
||||
runs: () => request<Run[]>('/runs'),
|
||||
activeRun: () => request<Run | null>('/runs/active'),
|
||||
cancelRun: (id: string) => request<Run>(`/runs/${id}/cancel`, { method: 'POST' }),
|
||||
|
||||
reports: () => request<ReportSummary[]>('/reports'),
|
||||
report: (name: string) => request<ReportDetail>(`/reports/${encodeURIComponent(name)}`),
|
||||
deleteReport: (name: string) =>
|
||||
request<void>(`/reports/${encodeURIComponent(name)}`, { method: 'DELETE' }),
|
||||
|
||||
playground: (body: {
|
||||
provider: string
|
||||
model_id: string
|
||||
prompt: string
|
||||
temperature: number
|
||||
}) => request<PlaygroundResult>('/playground', { method: 'POST', body: JSON.stringify(body) }),
|
||||
|
||||
plugins: () => request<PluginSummary[]>('/plugins'),
|
||||
reloadPlugins: () => request<PluginSummary[]>('/plugins/reload', { method: 'POST' }),
|
||||
|
||||
settings: () => request<Settings>('/settings'),
|
||||
saveSettings: (body: {
|
||||
default_provider?: string
|
||||
default_temperature?: number
|
||||
local_model_paths?: ModelPathEntry[]
|
||||
}) => request<Settings>('/settings', { method: 'PUT', body: JSON.stringify(body) }),
|
||||
}
|
||||
|
||||
/** Subscribe to a run's server-sent event feed. Returns an unsubscribe fn. */
|
||||
export function subscribeToRun(
|
||||
runId: string,
|
||||
onEvent: (event: RunEvent) => void,
|
||||
onError?: () => void,
|
||||
): () => void {
|
||||
const source = new EventSource(`/api/runs/${runId}/events`)
|
||||
|
||||
const handle = (raw: MessageEvent) => {
|
||||
try {
|
||||
onEvent(JSON.parse(raw.data) as RunEvent)
|
||||
} catch {
|
||||
/* ignore malformed frames */
|
||||
}
|
||||
}
|
||||
|
||||
for (const name of [
|
||||
'run.started',
|
||||
'test.completed',
|
||||
'run.completed',
|
||||
'run.failed',
|
||||
'run.cancelled',
|
||||
]) {
|
||||
source.addEventListener(name, handle as EventListener)
|
||||
}
|
||||
|
||||
source.onerror = () => {
|
||||
// The server closes the stream once a run reaches a terminal state, which
|
||||
// EventSource reports as an error. Only surface it while still connecting.
|
||||
if (source.readyState === EventSource.CLOSED) onError?.()
|
||||
}
|
||||
|
||||
return () => source.close()
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/** Small display helpers shared across views. */
|
||||
|
||||
export function formatDuration(seconds: number): string {
|
||||
if (!Number.isFinite(seconds) || seconds < 0) return '—'
|
||||
if (seconds < 1) return `${Math.round(seconds * 1000)}ms`
|
||||
if (seconds < 60) return `${seconds.toFixed(1)}s`
|
||||
const minutes = Math.floor(seconds / 60)
|
||||
const rest = Math.round(seconds % 60)
|
||||
if (minutes < 60) return `${minutes}m ${rest}s`
|
||||
return `${Math.floor(minutes / 60)}h ${minutes % 60}m`
|
||||
}
|
||||
|
||||
export function formatBytes(bytes: number): string {
|
||||
if (bytes < 1024) return `${bytes} B`
|
||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`
|
||||
}
|
||||
|
||||
export function formatRelativeTime(epochSeconds: number): string {
|
||||
const delta = Date.now() / 1000 - epochSeconds
|
||||
if (delta < 60) return 'just now'
|
||||
if (delta < 3600) return `${Math.floor(delta / 60)}m ago`
|
||||
if (delta < 86400) return `${Math.floor(delta / 3600)}h ago`
|
||||
if (delta < 604800) return `${Math.floor(delta / 86400)}d ago`
|
||||
return new Date(epochSeconds * 1000).toLocaleDateString()
|
||||
}
|
||||
|
||||
export function formatTimestamp(epochSeconds: number): string {
|
||||
return new Date(epochSeconds * 1000).toLocaleString(undefined, {
|
||||
month: 'short',
|
||||
day: 'numeric',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
})
|
||||
}
|
||||
|
||||
export function clockTime(epochMs: number = Date.now()): string {
|
||||
return new Date(epochMs).toLocaleTimeString(undefined, { hour12: false })
|
||||
}
|
||||
|
||||
/** Title shown for a question — the engine uses the first non-empty line. */
|
||||
export function deriveTitle(prompt: string, fallback = 'Untitled question'): string {
|
||||
for (const line of prompt.split('\n')) {
|
||||
const trimmed = line.trim()
|
||||
if (trimmed) return trimmed
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
export function cx(...values: (string | false | null | undefined)[]): string {
|
||||
return values.filter(Boolean).join(' ')
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* Parses the markdown reports the Python engine writes.
|
||||
*
|
||||
* The shape is fixed by `.core/templates/test-block.md`, so a report can be
|
||||
* read back into structured metrics for charting without the server having to
|
||||
* keep a parallel database of past runs.
|
||||
*/
|
||||
|
||||
export interface ParsedTest {
|
||||
index: number
|
||||
title: string
|
||||
filename: string
|
||||
ok: boolean
|
||||
error: string | null
|
||||
tokensPerSecond: number | null
|
||||
totalTokens: number | null
|
||||
timeToFirstToken: number | null
|
||||
stopReason: string | null
|
||||
/** Mean plugin score for this question, when the report carries grades. */
|
||||
score: number | null
|
||||
}
|
||||
|
||||
export interface ParsedReport {
|
||||
modelLabel: string
|
||||
tests: ParsedTest[]
|
||||
averageTokensPerSecond: number | null
|
||||
averageTimeToFirstToken: number | null
|
||||
totalTokens: number | null
|
||||
overallScore: number | null
|
||||
graders: string[]
|
||||
}
|
||||
|
||||
function numberOrNull(raw: string | undefined): number | null {
|
||||
if (!raw) return null
|
||||
const value = Number.parseFloat(raw.replace(/[^\d.-]/g, ''))
|
||||
return Number.isFinite(value) ? value : null
|
||||
}
|
||||
|
||||
function metric(block: string, label: string): string | undefined {
|
||||
const match = block.match(new RegExp(`\\*\\*${label}:\\*\\*\\s*(.+)`))
|
||||
return match?.[1]?.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Read grades back out of the analysis block that grader plugins write.
|
||||
*
|
||||
* Rows look like: `| 3 | Question title | 80% (4/5 checks) | My Grader | notes |`
|
||||
* A question graded by several plugins gets one row each, so scores are
|
||||
* averaged per question — matching how the run computed them live.
|
||||
*/
|
||||
function parseGrades(markdown: string): {
|
||||
scores: Map<number, number>
|
||||
overall: number | null
|
||||
graders: string[]
|
||||
} {
|
||||
const block = markdown.match(/<!--ANALYSIS_START-->([\s\S]*?)<!--ANALYSIS_END-->/)?.[1]
|
||||
const scores = new Map<number, number>()
|
||||
const graders = new Set<string>()
|
||||
|
||||
if (!block) return { scores, overall: null, graders: [] }
|
||||
|
||||
const overallMatch = block.match(/\*\*Overall score:\s*(\d+(?:\.\d+)?)%/)
|
||||
const overall = overallMatch ? Number.parseFloat(overallMatch[1]) / 100 : null
|
||||
|
||||
const collected = new Map<number, number[]>()
|
||||
const row = /^\|\s*(\d+)\s*\|[^|]*\|\s*(\d+(?:\.\d+)?)%[^|]*\|([^|]*)\|/gm
|
||||
let match: RegExpExecArray | null
|
||||
while ((match = row.exec(block)) !== null) {
|
||||
const index = Number.parseInt(match[1], 10)
|
||||
const value = Number.parseFloat(match[2]) / 100
|
||||
if (!Number.isFinite(index) || !Number.isFinite(value)) continue
|
||||
collected.set(index, [...(collected.get(index) ?? []), value])
|
||||
const grader = match[3]?.trim()
|
||||
if (grader && grader !== '—') graders.add(grader)
|
||||
}
|
||||
|
||||
for (const [index, values] of collected) {
|
||||
scores.set(index, values.reduce((a, b) => a + b, 0) / values.length)
|
||||
}
|
||||
|
||||
return { scores, overall, graders: [...graders] }
|
||||
}
|
||||
|
||||
export function parseReport(markdown: string): ParsedReport {
|
||||
const modelLabel =
|
||||
markdown.match(/^#\s*Automated Diagnostic Report:\s*(.+)$/m)?.[1]?.trim() ?? 'Unknown model'
|
||||
|
||||
const summary = markdown.match(/<!--SUMMARY_START-->([\s\S]*?)<!--SUMMARY_END-->/)?.[1] ?? ''
|
||||
|
||||
const { scores, overall, graders } = parseGrades(markdown)
|
||||
|
||||
const tests: ParsedTest[] = []
|
||||
// Split on the test headings the template emits; the first chunk is the preamble.
|
||||
const chunks = markdown.split(/^##\s+Test\s+(\d+):\s*(.*)$/m)
|
||||
|
||||
for (let i = 1; i < chunks.length; i += 3) {
|
||||
const index = Number.parseInt(chunks[i], 10)
|
||||
const title = (chunks[i + 1] ?? '').trim()
|
||||
const body = chunks[i + 2] ?? ''
|
||||
|
||||
const errorMatch = body.match(/\*\*ERROR:\*\*\s*(.+)/)
|
||||
const stopReason = metric(body, 'Stop Reason') ?? null
|
||||
|
||||
tests.push({
|
||||
index,
|
||||
title: title || `Test ${index}`,
|
||||
filename: body.match(/\*Source:\*\s*`([^`]+)`/)?.[1] ?? '',
|
||||
ok: !errorMatch,
|
||||
error: errorMatch?.[1]?.trim() ?? null,
|
||||
tokensPerSecond: numberOrNull(metric(body, 'Tokens/s')),
|
||||
totalTokens: numberOrNull(metric(body, 'Total Tokens')),
|
||||
timeToFirstToken: numberOrNull(metric(body, 'Time to First Token')),
|
||||
stopReason: stopReason === 'N/A' ? null : stopReason,
|
||||
score: scores.get(index) ?? null,
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
modelLabel,
|
||||
tests,
|
||||
averageTokensPerSecond: numberOrNull(metric(summary, 'Average Tokens/s')),
|
||||
averageTimeToFirstToken: numberOrNull(metric(summary, 'Average Time to First Token')),
|
||||
totalTokens: numberOrNull(metric(summary, 'Total Tokens Generated')),
|
||||
overallScore: overall,
|
||||
graders,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/**
|
||||
* A ~40 line History API router.
|
||||
*
|
||||
* The app has five top-level views and no data loaders, so a routing library
|
||||
* would be pure overhead. This gives real URLs, working back/forward buttons
|
||||
* and deep links (e.g. /reports/automated_report_Qwen.md) with no dependency.
|
||||
*/
|
||||
|
||||
import {
|
||||
createContext,
|
||||
useCallback,
|
||||
useContext,
|
||||
useEffect,
|
||||
useMemo,
|
||||
useState,
|
||||
type ReactNode,
|
||||
} from 'react'
|
||||
|
||||
interface RouterValue {
|
||||
path: string
|
||||
navigate: (to: string, options?: { replace?: boolean }) => void
|
||||
}
|
||||
|
||||
const RouterContext = createContext<RouterValue>({ path: '/', navigate: () => {} })
|
||||
|
||||
export function RouterProvider({ children }: { children: ReactNode }) {
|
||||
const [path, setPath] = useState(() => window.location.pathname || '/')
|
||||
|
||||
useEffect(() => {
|
||||
const onPop = () => setPath(window.location.pathname || '/')
|
||||
window.addEventListener('popstate', onPop)
|
||||
return () => window.removeEventListener('popstate', onPop)
|
||||
}, [])
|
||||
|
||||
const navigate = useCallback((to: string, options?: { replace?: boolean }) => {
|
||||
if (to === window.location.pathname) return
|
||||
window.history[options?.replace ? 'replaceState' : 'pushState']({}, '', to)
|
||||
setPath(to)
|
||||
window.scrollTo({ top: 0 })
|
||||
}, [])
|
||||
|
||||
const value = useMemo(() => ({ path, navigate }), [path, navigate])
|
||||
return <RouterContext.Provider value={value}>{children}</RouterContext.Provider>
|
||||
}
|
||||
|
||||
export function useRouter() {
|
||||
return useContext(RouterContext)
|
||||
}
|
||||
|
||||
export function Link({
|
||||
to,
|
||||
className,
|
||||
children,
|
||||
...rest
|
||||
}: { to: string; className?: string; children: ReactNode } & Omit<
|
||||
React.AnchorHTMLAttributes<HTMLAnchorElement>,
|
||||
'href'
|
||||
>) {
|
||||
const { navigate } = useRouter()
|
||||
return (
|
||||
<a
|
||||
href={to}
|
||||
className={className}
|
||||
onClick={(event) => {
|
||||
if (event.metaKey || event.ctrlKey || event.shiftKey || event.button !== 0) return
|
||||
event.preventDefault()
|
||||
navigate(to)
|
||||
}}
|
||||
{...rest}
|
||||
>
|
||||
{children}
|
||||
</a>
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user