feat: add testing framework and initial test cases

- Updated package.json to include Vitest and testing dependencies.
- Created test cases for SuitePicker component to validate selection logic.
- Added tests for ReportsPage to ensure correct report grouping and ordering.
- Implemented read-only enforcement tests for SuitePage to prevent actions on built-in suites.
- Introduced a setup file for Vitest to include jest-dom matchers and cleanup after tests.
- Modified ReportsPage to group reports by model and display them accordingly.
- Enhanced API types to include suite_scope and question_count for better report handling.
This commit is contained in:
Netherwarlord
2026-07-28 20:06:26 -04:00
parent 8f246fabbc
commit 7a81468925
26 changed files with 2487 additions and 168 deletions
+1188 -1
View File
File diff suppressed because it is too large Load Diff
+9 -2
View File
@@ -7,7 +7,9 @@
"dev": "vite",
"build": "tsc -b && vite build",
"lint": "oxlint",
"preview": "vite preview"
"preview": "vite preview",
"test": "vitest run",
"test:watch": "vitest"
},
"dependencies": {
"@tailwindcss/vite": "^4.3.3",
@@ -20,12 +22,17 @@
"tailwindcss": "^4.3.3"
},
"devDependencies": {
"@testing-library/jest-dom": "^7.0.0",
"@testing-library/react": "^16.3.2",
"@testing-library/user-event": "^14.6.1",
"@types/node": "^24.13.2",
"@types/react": "^19.2.17",
"@types/react-dom": "^19.2.3",
"@vitejs/plugin-react": "^6.0.3",
"jsdom": "^30.0.0",
"oxlint": "^1.71.0",
"typescript": "~6.0.2",
"vite": "^8.1.1"
"vite": "^8.1.1",
"vitest": "^4.1.10"
}
}
+149
View File
@@ -0,0 +1,149 @@
/**
* The run selection model.
*
* This is the piece that decides what a run actually covers, and it was
* browser-verified only. The shape it produces matters as much as the count:
* a whole suite must serialise as `{suite}` with no filenames, so "all of
* math-code" stays all of it even after that suite gains a question. Freezing
* today's filenames into the request would silently narrow future runs.
*/
import { describe, expect, it, vi } from 'vitest'
import { render, screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import {
SuitePicker,
countSelected,
selectionsFrom,
type Picks,
} from './SuitePicker'
import type { SuiteSummary } from '../lib/api'
const SUITES: SuiteSummary[] = [
{ slug: 'language', name: 'Language', description: '', order: 10, builtin: true, count: 6 },
{ slug: 'math-code', name: 'Math & Code', description: '', order: 30, builtin: true, count: 6 },
{ slug: 'safety', name: 'Safety', description: '', order: 50, builtin: true, count: 4 },
{ slug: 'mine', name: 'Mine', description: '', order: 100, builtin: false, count: 2 },
]
describe('selectionsFrom', () => {
it('sends a whole suite as {suite} with no filenames', () => {
expect(selectionsFrom({ 'math-code': 'all' })).toEqual([{ suite: 'math-code' }])
})
it('sends explicit filenames for a partial suite', () => {
expect(selectionsFrom({ 'math-code': ['test3.txt', 'test4.txt'] })).toEqual([
{ suite: 'math-code', filenames: ['test3.txt', 'test4.txt'] },
])
})
it('mixes whole and partial suites in one request', () => {
expect(selectionsFrom({ safety: 'all', 'math-code': ['test4.txt'] })).toEqual([
{ suite: 'safety' },
{ suite: 'math-code', filenames: ['test4.txt'] },
])
})
it('omits suites selected down to nothing', () => {
expect(selectionsFrom({ safety: 'all', 'math-code': [] })).toEqual([{ suite: 'safety' }])
})
it('produces nothing for an empty selection', () => {
expect(selectionsFrom({})).toEqual([])
})
})
describe('countSelected', () => {
it('counts a whole suite by its real size, not by listed filenames', () => {
// The point of 'all': the count tracks the suite, so adding a question
// later is reflected without the selection being rebuilt.
expect(countSelected({ 'math-code': 'all' }, SUITES)).toBe(6)
})
it('counts a partial suite by its chosen files', () => {
expect(countSelected({ 'math-code': ['test1.txt', 'test2.txt'] }, SUITES)).toBe(2)
})
it('sums across suites', () => {
expect(countSelected({ safety: 'all', 'math-code': ['test4.txt'] }, SUITES)).toBe(5)
})
it('is zero for no selection', () => {
expect(countSelected({}, SUITES)).toBe(0)
})
it('ignores a pick for a suite that no longer exists', () => {
expect(countSelected({ 'deleted-suite': 'all' }, SUITES)).toBe(0)
})
})
describe('SuitePicker', () => {
const renderPicker = (picks: Picks = {}) => {
const onChange = vi.fn()
render(<SuitePicker suites={SUITES} picks={picks} onChange={onChange} />)
return { onChange }
}
it('lists every suite', () => {
renderPicker()
for (const suite of SUITES) expect(screen.getByText(suite.name)).toBeInTheDocument()
})
it('selecting a suite yields "all", not a filename list', async () => {
const { onChange } = renderPicker()
await userEvent.click(screen.getByLabelText('Select Safety'))
expect(onChange).toHaveBeenCalledWith({ safety: 'all' })
})
it('deselecting removes the suite entirely rather than emptying it', async () => {
const { onChange } = renderPicker({ safety: 'all' })
await userEvent.click(screen.getByLabelText('Select Safety'))
expect(onChange).toHaveBeenCalledWith({})
})
it('shows a partially-selected suite as indeterminate', () => {
renderPicker({ 'math-code': ['test1.txt', 'test2.txt'] })
const box = screen.getByLabelText('Select Math & Code') as HTMLInputElement
// Neither checked nor unchecked: the visual state that tells you a suite
// is only partly included.
expect(box.indeterminate).toBe(true)
expect(box.checked).toBe(true)
})
it('a fully-selected suite is checked, not indeterminate', () => {
renderPicker({ safety: 'all' })
const box = screen.getByLabelText('Select Safety') as HTMLInputElement
expect(box.indeterminate).toBe(false)
expect(box.checked).toBe(true)
})
it('an unselected suite is neither', () => {
renderPicker()
const box = screen.getByLabelText('Select Language') as HTMLInputElement
expect(box.indeterminate).toBe(false)
expect(box.checked).toBe(false)
})
it('only fetches a suite\'s questions when it is expanded', async () => {
const fetched: string[] = []
vi.spyOn(await import('../lib/api'), 'api', 'get').mockReturnValue({
suite: async (slug: string) => {
fetched.push(slug)
return { slug, name: slug, description: '', order: 1, builtin: true, count: 1, tests: [] }
},
} as never)
renderPicker()
expect(fetched).toEqual([])
await userEvent.click(screen.getByText('Safety'))
await waitFor(() => expect(fetched).toEqual(['safety']))
vi.restoreAllMocks()
})
it('does nothing when disabled', async () => {
const onChange = vi.fn()
render(<SuitePicker suites={SUITES} picks={{}} onChange={onChange} disabled />)
await userEvent.click(screen.getByLabelText('Select Safety'))
expect(onChange).not.toHaveBeenCalled()
})
})
+4
View File
@@ -186,6 +186,10 @@ export interface ReportSummary {
model_label: string
size_bytes: number
modified_at: number
/** Which suites the run covered. Empty for reports predating the scheme. */
suite_scope: string
/** How many questions it actually ran. Null for legacy reports. */
question_count: number | null
}
export interface ReportDetail extends ReportSummary {
+2 -1
View File
@@ -3,7 +3,8 @@
*
* The app has five top-level views and no data loaders, so a routing library
* would be pure overhead. This gives real URLs, working back/forward buttons
* and deep links (e.g. /reports/automated_report_Qwen.md) with no dependency.
* and deep links (e.g. /reports/Qwen3.5-9B__20260728-0318__all__26q.md) with
* no dependency.
*/
import {
+76
View File
@@ -0,0 +1,76 @@
/**
* Report grouping.
*
* Reports used to be one file per model, silently overwritten. Now a model has
* a history, and the list must present it as one thread — with the newest run
* first, since that ordering is what makes a regression visible.
*/
import { describe, expect, it } from 'vitest'
import { groupByModel } from './ReportsPage'
import type { ReportSummary } from '../lib/api'
const report = (
model: string,
modified: number,
scope = 'all',
count: number | null = 26,
): ReportSummary => ({
name: `${model}__${modified}__${scope}__${count}q.md`,
model_label: model,
size_bytes: 1000,
modified_at: modified,
suite_scope: scope,
question_count: count,
})
describe('groupByModel', () => {
it('buckets runs of the same model together', () => {
const grouped = groupByModel([
report('gemma', 300, 'safety', 4),
report('deepseek', 200),
report('gemma', 100, 'math-code', 6),
])
expect(grouped.map(([model, runs]) => [model, runs.length])).toEqual([
['gemma', 2],
['deepseek', 1],
])
})
it('preserves the incoming order, which the API sorts newest-first', () => {
const [, gemmaRuns] = groupByModel([
report('gemma', 300, 'safety', 4),
report('gemma', 100, 'math-code', 6),
])[0]
expect(gemmaRuns.map((r) => r.modified_at)).toEqual([300, 100])
})
it('orders models by their most recent run', () => {
const grouped = groupByModel([
report('newest', 500),
report('older', 400),
report('older', 100),
])
expect(grouped.map(([model]) => model)).toEqual(['newest', 'older'])
})
it('keeps runs of one model distinct rather than collapsing them', () => {
// The bug this whole scheme exists to prevent: a one-question smoke test
// and a 26-question benchmark are different runs, not one file.
const [, runs] = groupByModel([
report('gemma', 300, 'context-knowledge', 1),
report('gemma', 200, 'all', 26),
])[0]
expect(runs.map((r) => r.question_count)).toEqual([1, 26])
expect(new Set(runs.map((r) => r.name)).size).toBe(2)
})
it('handles an empty list', () => {
expect(groupByModel([])).toEqual([])
})
it('carries legacy reports through with no scope', () => {
const [[, runs]] = groupByModel([report('old', 100, 'legacy', 4)])
expect(runs[0].suite_scope).toBe('legacy')
})
})
+92 -48
View File
@@ -24,11 +24,26 @@ import { PluginSlot } from '../components/PluginBlocks'
import { Markdown } from '../components/Markdown'
import { ScoreBadge, scoreTone } from '../components/ScoreBadge'
import { ThroughputChart } from '../components/ThroughputChart'
import { api, ApiError, type ReportDetail } from '../lib/api'
import { api, ApiError, type ReportDetail, type ReportSummary } from '../lib/api'
import { parseReport } from '../lib/report'
import { cx, formatBytes, formatRelativeTime } from '../lib/format'
import { useRouter } from '../lib/router'
/**
* Runs bucketed by model, newest first within each bucket and models ordered
* by their most recent run. The API already sorts by time, so insertion order
* carries that through.
*/
export function groupByModel(reports: ReportSummary[]): [string, ReportSummary[]][] {
const groups = new Map<string, ReportSummary[]>()
for (const report of reports) {
const bucket = groups.get(report.model_label)
if (bucket) bucket.push(report)
else groups.set(report.model_label, [report])
}
return [...groups.entries()]
}
type View = 'chart' | 'table' | 'full'
/**
@@ -137,53 +152,82 @@ export function ReportsPage() {
{reports.loading ? (
<SkeletonRows rows={3} />
) : (
<ul className="max-h-[70vh] divide-y divide-navy-700/60 overflow-y-auto">
{items.map((report) => {
const active = report.name === routeName
return (
<li key={report.name}>
<div
className={cx(
'group relative flex items-center gap-2 px-4 py-3 transition-colors',
active ? 'bg-navy-750/60' : 'hover:bg-navy-800/40',
)}
>
{active && (
<span className="absolute top-1/2 left-0 h-8 w-[3px] -translate-y-1/2 rounded-r-full bg-ember-500" />
)}
<button
className="min-w-0 flex-1 text-left"
onClick={() =>
navigate(`/reports/${encodeURIComponent(report.name)}`)
}
>
<div
className={cx(
'truncate text-[0.8125rem] font-medium',
active ? 'text-ink-100' : 'text-ink-200',
)}
>
{report.model_label}
</div>
<div className="mt-0.5 flex items-center gap-2 text-[0.6875rem] text-ink-500">
<span>{formatRelativeTime(report.modified_at)}</span>
<span>·</span>
<span className="num">{formatBytes(report.size_bytes)}</span>
</div>
</button>
<button
className="shrink-0 rounded-md p-1.5 text-ink-500 opacity-0 transition-all group-hover:opacity-100 hover:bg-rose-500/15 hover:text-rose-400 focus-visible:opacity-100"
onClick={() => setPendingDelete(report.name)}
aria-label={`Delete ${report.name}`}
title="Delete report"
>
<Trash2 size={13} />
</button>
</div>
</li>
)
})}
</ul>
<div className="max-h-[70vh] overflow-y-auto">
{groupByModel(items).map(([model, runs]) => (
<div key={model}>
{/* Runs are grouped by model so a model's history reads
as one thread rather than scattered through the list. */}
<div className="sticky top-0 z-10 flex items-baseline justify-between gap-2 border-b border-navy-700/60 bg-navy-900/95 px-4 py-2 backdrop-blur">
<span className="truncate text-[0.6875rem] font-semibold tracking-[0.06em] text-ink-300 uppercase">
{model}
</span>
<span className="num shrink-0 text-[0.625rem] text-ink-500">
{runs.length} run{runs.length === 1 ? '' : 's'}
</span>
</div>
<ul className="divide-y divide-navy-700/60">
{runs.map((report) => {
const active = report.name === routeName
return (
<li key={report.name}>
<div
className={cx(
'group relative flex items-center gap-2 px-4 py-2.5 transition-colors',
active ? 'bg-navy-750/60' : 'hover:bg-navy-800/40',
)}
>
{active && (
<span className="absolute top-1/2 left-0 h-8 w-[3px] -translate-y-1/2 rounded-r-full bg-ember-500" />
)}
<button
className="min-w-0 flex-1 text-left"
onClick={() =>
navigate(`/reports/${encodeURIComponent(report.name)}`)
}
>
<div className="flex items-center gap-1.5">
{/* Scope and count are what distinguish a
smoke test from a real benchmark. */}
{report.suite_scope ? (
<span className="rounded border border-navy-700 bg-navy-800/70 px-1.5 py-px text-[0.625rem] text-ink-300">
{report.suite_scope}
</span>
) : (
<span
className="rounded border border-navy-700 bg-navy-800/40 px-1.5 py-px text-[0.625rem] text-ink-600"
title="Saved before runs recorded their scope"
>
legacy
</span>
)}
{report.question_count != null && (
<span className="num text-[0.6875rem] text-ink-400">
{report.question_count}q
</span>
)}
</div>
<div className="mt-0.5 flex items-center gap-2 text-[0.6875rem] text-ink-500">
<span>{formatRelativeTime(report.modified_at)}</span>
<span>·</span>
<span className="num">{formatBytes(report.size_bytes)}</span>
</div>
</button>
<button
className="shrink-0 rounded-md p-1.5 text-ink-500 opacity-0 transition-all group-hover:opacity-100 hover:bg-rose-500/15 hover:text-rose-400 focus-visible:opacity-100"
onClick={() => setPendingDelete(report.name)}
aria-label={`Delete ${report.name}`}
title="Delete report"
>
<Trash2 size={13} />
</button>
</div>
</li>
)
})}
</ul>
</div>
))}
</div>
)}
</Card>
<PluginSlot slot="reports.aside" className="mt-4" />
+118
View File
@@ -0,0 +1,118 @@
/**
* Read-only enforcement on the Testing Suites page.
*
* These are presentation guards only — the API returns 409 regardless, and
* that is the real protection. What is asserted here is that the page does not
* *offer* an action it cannot perform, since a Rename button that always fails
* is worse than no button.
*/
import { describe, expect, it, vi, beforeEach } from 'vitest'
import { render, screen, waitFor } from '@testing-library/react'
import { SuitePage } from './SuitePage'
import { RunFeedProvider } from '../hooks/useRunFeed'
import type { SuiteDetail, SuiteSummary } from '../lib/api'
const SUMMARIES: SuiteSummary[] = [
{ slug: 'safety', name: 'Safety', description: 'Guardrails', order: 50, builtin: true, count: 2 },
{ slug: 'mine', name: 'My Suite', description: 'Mine', order: 100, builtin: false, count: 1 },
]
const DETAILS: Record<string, SuiteDetail> = {
safety: {
...SUMMARIES[0],
tests: [
{ filename: 'test1.txt', title: 'Built-in question one', prompt: 'Built-in question one', suite: 'safety', id: 'safety/test1.txt' },
{ filename: 'test2.txt', title: 'Built-in question two', prompt: 'Built-in question two', suite: 'safety', id: 'safety/test2.txt' },
],
},
mine: {
...SUMMARIES[1],
tests: [
{ filename: 'test1.txt', title: 'My question', prompt: 'My question', suite: 'mine', id: 'mine/test1.txt' },
],
},
}
// The page only needs these two calls to render; everything else is user-driven.
vi.mock('../lib/api', async (importOriginal) => {
const actual = await importOriginal<typeof import('../lib/api')>()
return {
...actual,
api: {
...actual.api,
suites: vi.fn(async () => SUMMARIES),
suite: vi.fn(async (slug: string) => DETAILS[slug]),
activeRun: vi.fn(async () => null),
runs: vi.fn(async () => []),
},
}
})
const renderPage = () =>
render(
<RunFeedProvider>
<SuitePage />
</RunFeedProvider>,
)
describe('SuitePage read-only guards', () => {
beforeEach(() => vi.clearAllMocks())
it('offers Duplicate but not Rename or Delete for a built-in suite', async () => {
renderPage()
// The first suite is selected automatically, and it is built-in.
await waitFor(() => expect(screen.getByRole('button', { name: /duplicate/i })).toBeInTheDocument())
expect(screen.queryByRole('button', { name: /^rename$/i })).not.toBeInTheDocument()
expect(screen.queryByRole('button', { name: /delete suite/i })).not.toBeInTheDocument()
})
// The question text appears twice per card - once as the title, once as the
// textarea value - so questions are counted by role rather than by text.
const questionBoxes = () => screen.getAllByRole('textbox')
const waitForQuestions = (count: number) =>
waitFor(() => expect(questionBoxes()).toHaveLength(count))
const selectCustomSuite = async () => {
const { default: userEvent } = await import('@testing-library/user-event')
await waitFor(() => expect(screen.getByRole('button', { name: /My Suite/ })).toBeInTheDocument())
await userEvent.click(screen.getByRole('button', { name: /My Suite/ }))
}
it('marks a built-in suite as read-only and locks its question boxes', async () => {
renderPage()
await waitForQuestions(2)
expect(screen.getByText(/ships with the app/i)).toBeInTheDocument()
for (const box of questionBoxes()) expect(box).toHaveAttribute('readonly')
})
it('offers no save control for a built-in suite', async () => {
renderPage()
await waitForQuestions(2)
expect(screen.queryByRole('button', { name: /save questions/i })).not.toBeInTheDocument()
expect(screen.queryByRole('button', { name: /add question/i })).not.toBeInTheDocument()
})
it('offers Rename and Delete once a custom suite is selected', async () => {
renderPage()
await waitForQuestions(2)
await selectCustomSuite()
await waitFor(() =>
expect(screen.getByRole('button', { name: /^rename$/i })).toBeInTheDocument(),
)
// "Delete suite", not the per-question icon buttons which announce as
// plain "Delete".
expect(screen.getByRole('button', { name: /delete suite/i })).toBeInTheDocument()
expect(screen.queryByText(/ships with the app/i)).not.toBeInTheDocument()
})
it('leaves a custom suite\'s question boxes editable', async () => {
renderPage()
await waitForQuestions(2)
await selectCustomSuite()
await waitForQuestions(1)
for (const box of questionBoxes()) expect(box).not.toHaveAttribute('readonly')
})
})
+3
View File
@@ -383,6 +383,9 @@ export function SuitePage() {
className="btn btn-ghost btn-sm text-rose-400 hover:bg-rose-500/10"
onClick={() => setDeleteSuiteOpen(true)}
disabled={busy || isLive}
// Distinguished from the per-question Delete buttons,
// which are icon-only and announce as just "Delete".
aria-label="Delete suite"
>
<Trash2 size={13} />
Delete
+12
View File
@@ -0,0 +1,12 @@
/**
* Vitest setup, loaded before every test file.
*
* Adds jest-dom's DOM matchers (`toBeChecked`, `toBeDisabled`, …) and clears
* the DOM between tests so one test's render cannot leak into the next.
*/
import '@testing-library/jest-dom/vitest'
import { cleanup } from '@testing-library/react'
import { afterEach } from 'vitest'
afterEach(cleanup)
+10 -1
View File
@@ -1,4 +1,6 @@
import { defineConfig } from 'vite'
// defineConfig comes from vitest/config rather than vite so the `test` block
// below type-checks; it is vite's own, widened with Vitest's options.
import { defineConfig } from 'vitest/config'
import react from '@vitejs/plugin-react'
import tailwindcss from '@tailwindcss/vite'
@@ -28,4 +30,11 @@ export default defineConfig({
emptyOutDir: true,
chunkSizeWarningLimit: 900,
},
test: {
environment: 'jsdom',
globals: true,
setupFiles: ['./src/test/setup.ts'],
// Only our own tests; node_modules and the built bundle are not ours.
include: ['src/**/*.test.{ts,tsx}'],
},
})