feat: add plugin framework with UI contributions and suite selection
- Implemented PluginBlocks component to render various plugin UI elements. - Created SuitePicker for selecting test suites and questions. - Introduced usePluginUI hook for managing plugin UI state and manifest. - Developed DocsPage for comprehensive plugin framework documentation. - Added PluginPage to render pages declared by plugins based on the current path.
This commit is contained in:
Netherwarlord
+15
-1
@@ -1,14 +1,17 @@
|
||||
import { Shell } from './components/Shell'
|
||||
import { useRouter } from './lib/router'
|
||||
import { useRunFeed } from './hooks/useRunFeed'
|
||||
import { PluginUIProvider } from './hooks/usePluginUI'
|
||||
import { RunPage } from './pages/RunPage'
|
||||
import { SuitePage } from './pages/SuitePage'
|
||||
import { ReportsPage } from './pages/ReportsPage'
|
||||
import { PlaygroundPage } from './pages/PlaygroundPage'
|
||||
import { SettingsPage } from './pages/SettingsPage'
|
||||
import { DocsPage } from './pages/DocsPage'
|
||||
import { PluginPage } from './pages/PluginPage'
|
||||
import { NotFoundPage } from './pages/NotFoundPage'
|
||||
|
||||
export function App() {
|
||||
function Routes() {
|
||||
const { path } = useRouter()
|
||||
const { run, isLive } = useRunFeed()
|
||||
|
||||
@@ -18,7 +21,18 @@ export function App() {
|
||||
else if (path.startsWith('/reports')) page = <ReportsPage />
|
||||
else if (path === '/playground') page = <PlaygroundPage />
|
||||
else if (path === '/settings') page = <SettingsPage />
|
||||
else if (path === '/docs') page = <DocsPage />
|
||||
// Everything under /x/ belongs to a plugin; PluginPage resolves which one.
|
||||
else if (path.startsWith('/x/')) page = <PluginPage />
|
||||
else page = <NotFoundPage />
|
||||
|
||||
return <Shell activeRun={isLive ? run : null}>{page}</Shell>
|
||||
}
|
||||
|
||||
export function App() {
|
||||
return (
|
||||
<PluginUIProvider>
|
||||
<Routes />
|
||||
</PluginUIProvider>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
/**
|
||||
* Renders the interface plugins declare.
|
||||
*
|
||||
* A plugin returns data — never code — from its `ui_contributions()` hook, and
|
||||
* this module draws it with the same primitives the built-in pages use, so a
|
||||
* plugin panel is indistinguishable from a native one. That is the whole reason
|
||||
* installing a plugin never requires rebuilding this bundle.
|
||||
*
|
||||
* Blocks may carry inline data or a `source` URL. With a source the block
|
||||
* fetches its own data and re-fetches whenever `refreshKey` changes, which is
|
||||
* how an Action button refreshes its siblings.
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useMemo, useState, type ReactNode } from 'react'
|
||||
import {
|
||||
Activity, AlertTriangle, BarChart3, Bug, CheckCircle2, Database, FileText,
|
||||
Gauge, Hash, Layers, ListChecks, Play, Puzzle, RefreshCw, Search, Settings2,
|
||||
Shield, Sparkles, Table2, Terminal, Trash2, Wrench, Zap,
|
||||
} from 'lucide-react'
|
||||
import { Markdown } from './Markdown'
|
||||
import { Card, CardHeader, ConfirmDialog, EmptyState, ErrorNote, Spinner, StatTile, useToast } from './ui'
|
||||
import { api, type ActionBlock, type MarkdownBlock, type PanelBlock, type PluginBlock, type StatRowBlock, type TableBlock } from '../lib/api'
|
||||
import { usePluginSlot } from '../hooks/usePluginUI'
|
||||
import { cx } from '../lib/format'
|
||||
|
||||
/* Plugins name an icon; only these resolve. An unknown name is not an error —
|
||||
* it falls back to a puzzle piece so a typo never blanks the interface. */
|
||||
const ICONS: Record<string, typeof Puzzle> = {
|
||||
activity: Activity, alert: AlertTriangle, bar: BarChart3, bug: Bug,
|
||||
check: CheckCircle2, database: Database, file: FileText, gauge: Gauge,
|
||||
hash: Hash, layers: Layers, list: ListChecks, play: Play, puzzle: Puzzle,
|
||||
refresh: RefreshCw, search: Search, settings: Settings2, shield: Shield,
|
||||
sparkles: Sparkles, table: Table2, terminal: Terminal, trash: Trash2,
|
||||
wrench: Wrench, zap: Zap,
|
||||
}
|
||||
|
||||
export function pluginIcon(name: string | null | undefined) {
|
||||
return ICONS[(name || '').toLowerCase()] ?? Puzzle
|
||||
}
|
||||
|
||||
const TONE_TO_TILE = {
|
||||
default: 'neutral',
|
||||
good: 'mint',
|
||||
warn: 'gold',
|
||||
bad: 'rose',
|
||||
} as const
|
||||
|
||||
/* ------------------------------------------------------------ data plumbing */
|
||||
|
||||
/** Inline data, or fetched from `source` and refreshed on demand. */
|
||||
function useBlockData<T extends object>(block: T & { source?: string | null }, refreshKey: number) {
|
||||
const [data, setData] = useState<T>(block)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [loading, setLoading] = useState(Boolean(block.source))
|
||||
|
||||
useEffect(() => {
|
||||
if (!block.source) {
|
||||
setData(block)
|
||||
setLoading(false)
|
||||
return
|
||||
}
|
||||
let cancelled = false
|
||||
setLoading(true)
|
||||
api
|
||||
.pluginData<Partial<T>>(block.source)
|
||||
.then((fresh) => {
|
||||
if (!cancelled) {
|
||||
setData({ ...block, ...fresh })
|
||||
setError(null)
|
||||
}
|
||||
})
|
||||
.catch((cause: unknown) => {
|
||||
if (!cancelled) setError(cause instanceof Error ? cause.message : String(cause))
|
||||
})
|
||||
.finally(() => {
|
||||
if (!cancelled) setLoading(false)
|
||||
})
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [block.source, refreshKey])
|
||||
|
||||
return { data, error, loading }
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ blocks */
|
||||
|
||||
function StatRow({ block, refreshKey }: { block: StatRowBlock; refreshKey: number }) {
|
||||
const { data, error, loading } = useBlockData(block, refreshKey)
|
||||
if (error) return <ErrorNote message={error} />
|
||||
const stats = data.stats ?? []
|
||||
if (loading && !stats.length) {
|
||||
return <div className="flex items-center gap-2 text-[0.8125rem] text-ink-500"><Spinner /> Loading…</div>
|
||||
}
|
||||
if (!stats.length) return null
|
||||
|
||||
return (
|
||||
<div className="grid gap-2.5 sm:grid-cols-2 lg:grid-cols-4">
|
||||
{stats.map((stat, index) => (
|
||||
<StatTile
|
||||
key={`${stat.label}-${index}`}
|
||||
label={stat.label}
|
||||
value={stat.value}
|
||||
hint={stat.hint}
|
||||
tone={TONE_TO_TILE[stat.tone] ?? 'neutral'}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function PluginTable({ block, refreshKey }: { block: TableBlock; refreshKey: number }) {
|
||||
const { data, error, loading } = useBlockData(block, refreshKey)
|
||||
if (error) return <ErrorNote message={error} />
|
||||
|
||||
const rows = data.rows ?? []
|
||||
const columns = data.columns ?? []
|
||||
|
||||
if (loading && !rows.length) {
|
||||
return <div className="flex items-center gap-2 px-4 py-6 text-[0.8125rem] text-ink-500"><Spinner /> Loading…</div>
|
||||
}
|
||||
if (!rows.length) {
|
||||
return <EmptyState icon={<Table2 size={18} />} title="Nothing yet" description={data.empty} />
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full text-left text-[0.8125rem]">
|
||||
<thead>
|
||||
<tr className="border-b border-navy-700">
|
||||
{columns.map((column) => (
|
||||
<th key={column} className="px-4 py-2.5 text-[0.6875rem] font-semibold tracking-wide text-ink-500 uppercase">
|
||||
{column}
|
||||
</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((row, rowIndex) => (
|
||||
<tr key={rowIndex} className="border-b border-navy-800/70 last:border-0 hover:bg-navy-800/40">
|
||||
{row.map((cell, cellIndex) => (
|
||||
<td key={cellIndex} className="px-4 py-2.5 align-top text-ink-300">
|
||||
{String(cell)}
|
||||
</td>
|
||||
))}
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function PluginMarkdown({ block, refreshKey }: { block: MarkdownBlock; refreshKey: number }) {
|
||||
const { data, error } = useBlockData(block, refreshKey)
|
||||
if (error) return <ErrorNote message={error} />
|
||||
if (!data.text) return null
|
||||
return (
|
||||
<div className="px-4 py-3">
|
||||
<Markdown>{data.text}</Markdown>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function PluginAction({ block, onRefresh }: { block: ActionBlock; onRefresh: () => void }) {
|
||||
const toast = useToast()
|
||||
const [busy, setBusy] = useState(false)
|
||||
const [confirming, setConfirming] = useState(false)
|
||||
const Icon = pluginIcon(block.icon)
|
||||
|
||||
const fire = useCallback(async () => {
|
||||
setConfirming(false)
|
||||
setBusy(true)
|
||||
try {
|
||||
const result = await api.pluginAction(block.post)
|
||||
if (result?.message) toast(result.message, 'success')
|
||||
if (result?.refresh) onRefresh()
|
||||
} catch (cause: unknown) {
|
||||
toast(cause instanceof Error ? cause.message : String(cause), 'error')
|
||||
} finally {
|
||||
setBusy(false)
|
||||
}
|
||||
}, [block.post, onRefresh, toast])
|
||||
|
||||
const style =
|
||||
block.style === 'danger' ? 'btn-danger' : block.style === 'ghost' ? 'btn-ghost' : 'btn-primary'
|
||||
|
||||
return (
|
||||
<>
|
||||
<button
|
||||
className={cx('btn btn-sm', style)}
|
||||
disabled={busy}
|
||||
onClick={() => (block.confirm ? setConfirming(true) : fire())}
|
||||
>
|
||||
{busy ? <Spinner className="h-3.5 w-3.5" /> : <Icon size={14} />}
|
||||
{block.label}
|
||||
</button>
|
||||
<ConfirmDialog
|
||||
open={confirming}
|
||||
title={block.label}
|
||||
body={block.confirm ?? ''}
|
||||
confirmLabel={block.label}
|
||||
destructive={block.style === 'danger'}
|
||||
onConfirm={fire}
|
||||
onCancel={() => setConfirming(false)}
|
||||
/>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
function PluginPanel({
|
||||
block,
|
||||
refreshKey,
|
||||
onRefresh,
|
||||
}: {
|
||||
block: PanelBlock
|
||||
refreshKey: number
|
||||
onRefresh: () => void
|
||||
}) {
|
||||
const children = block.blocks ?? []
|
||||
// Actions belong in the panel header, everything else in the body.
|
||||
const actions = children.filter((child): child is ActionBlock => child.kind === 'action')
|
||||
const body = children.filter((child) => child.kind !== 'action')
|
||||
// Padding is the block's own job for tables and markdown, which bleed to the edge.
|
||||
const bleeds = (kind: string) => kind === 'table' || kind === 'markdown'
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<CardHeader
|
||||
title={block.title}
|
||||
hint={block.subtitle || undefined}
|
||||
actions={
|
||||
actions.length ? (
|
||||
<div className="flex items-center gap-2">
|
||||
{actions.map((action, index) => (
|
||||
<PluginAction key={index} block={action} onRefresh={onRefresh} />
|
||||
))}
|
||||
</div>
|
||||
) : undefined
|
||||
}
|
||||
/>
|
||||
{body.map((child, index) => (
|
||||
<div key={index} className={bleeds(child.kind) ? '' : 'px-4 py-3'}>
|
||||
<BlockView block={child} refreshKey={refreshKey} onRefresh={onRefresh} />
|
||||
</div>
|
||||
))}
|
||||
</Card>
|
||||
)
|
||||
}
|
||||
|
||||
function BlockView({
|
||||
block,
|
||||
refreshKey,
|
||||
onRefresh,
|
||||
}: {
|
||||
block: PluginBlock
|
||||
refreshKey: number
|
||||
onRefresh: () => void
|
||||
}): ReactNode {
|
||||
switch (block.kind) {
|
||||
case 'stat_row':
|
||||
return <StatRow block={block} refreshKey={refreshKey} />
|
||||
case 'table':
|
||||
return <PluginTable block={block} refreshKey={refreshKey} />
|
||||
case 'markdown':
|
||||
return <PluginMarkdown block={block} refreshKey={refreshKey} />
|
||||
case 'action':
|
||||
return <PluginAction block={block} onRefresh={onRefresh} />
|
||||
case 'panel':
|
||||
return <PluginPanel block={block} refreshKey={refreshKey} onRefresh={onRefresh} />
|
||||
default:
|
||||
// A plugin built against a newer host than this bundle. Say so rather
|
||||
// than rendering nothing, so the cause is obvious.
|
||||
return (
|
||||
<ErrorNote
|
||||
message={`This build does not know how to render a "${(block as { kind: string }).kind}" block.`}
|
||||
/>
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------- public */
|
||||
|
||||
export function PluginBlocks({
|
||||
blocks,
|
||||
className,
|
||||
autoRefreshMs,
|
||||
}: {
|
||||
blocks: PluginBlock[]
|
||||
className?: string
|
||||
autoRefreshMs?: number
|
||||
}) {
|
||||
const [refreshKey, setRefreshKey] = useState(0)
|
||||
const refresh = useCallback(() => setRefreshKey((key) => key + 1), [])
|
||||
|
||||
useEffect(() => {
|
||||
if (!autoRefreshMs) return
|
||||
const timer = setInterval(refresh, autoRefreshMs)
|
||||
return () => clearInterval(timer)
|
||||
}, [autoRefreshMs, refresh])
|
||||
|
||||
const rendered = useMemo(
|
||||
() =>
|
||||
blocks.map((block, index) => (
|
||||
<BlockView key={index} block={block} refreshKey={refreshKey} onRefresh={refresh} />
|
||||
)),
|
||||
[blocks, refreshKey, refresh],
|
||||
)
|
||||
|
||||
return <div className={cx('space-y-4', className)}>{rendered}</div>
|
||||
}
|
||||
|
||||
/**
|
||||
* Every panel plugins contributed to one injection point.
|
||||
*
|
||||
* Renders nothing at all when no plugin uses the slot, so built-in pages carry
|
||||
* these with no visual cost until something opts in.
|
||||
*/
|
||||
export function PluginSlot({
|
||||
slot,
|
||||
className,
|
||||
autoRefreshMs,
|
||||
}: {
|
||||
slot: string
|
||||
className?: string
|
||||
autoRefreshMs?: number
|
||||
}) {
|
||||
const panels = usePluginSlot(slot)
|
||||
if (!panels.length) return null
|
||||
|
||||
return (
|
||||
<div className={cx('space-y-4', className)}>
|
||||
{panels.map((entry, index) => (
|
||||
<PluginBlocks
|
||||
key={`${entry.slug}-${index}`}
|
||||
blocks={[entry.panel]}
|
||||
autoRefreshMs={autoRefreshMs}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import { useEffect, useState, type ReactNode } from 'react'
|
||||
import {
|
||||
BookOpen,
|
||||
Cpu,
|
||||
FileText,
|
||||
FlaskConical,
|
||||
@@ -12,16 +13,20 @@ import {
|
||||
X,
|
||||
} from 'lucide-react'
|
||||
import { Logo } from './Logo'
|
||||
import { pluginIcon } from './PluginBlocks'
|
||||
import { Link, useRouter } from '../lib/router'
|
||||
import { api, type Run, type SystemInfo } from '../lib/api'
|
||||
import { usePluginUI } from '../hooks/usePluginUI'
|
||||
import { cx } from '../lib/format'
|
||||
|
||||
/** Built-ins carry an order so plugin entries can slot between them. */
|
||||
const NAV = [
|
||||
{ to: '/', label: 'Run', icon: Gauge, hint: 'Execute the diagnostic suite' },
|
||||
{ to: '/suite', label: 'Suite', icon: ListChecks, hint: 'Author the questions' },
|
||||
{ to: '/reports', label: 'Reports', icon: FileText, hint: 'Read past results' },
|
||||
{ to: '/playground', label: 'Playground', icon: FlaskConical, hint: 'Try a single prompt' },
|
||||
{ to: '/settings', label: 'Settings', icon: Settings2, hint: 'Paths and defaults' },
|
||||
{ to: '/', label: 'Run', icon: Gauge, hint: 'Execute the diagnostic suite', order: 10 },
|
||||
{ to: '/suite', label: 'Testing Suites', icon: ListChecks, hint: 'Browse and author question suites', order: 20 },
|
||||
{ to: '/reports', label: 'Reports', icon: FileText, hint: 'Read past results', order: 30 },
|
||||
{ to: '/playground', label: 'Playground', icon: FlaskConical, hint: 'Try a single prompt', order: 40 },
|
||||
{ to: '/docs', label: 'Docs', icon: BookOpen, hint: 'Plugin framework reference', order: 80 },
|
||||
{ to: '/settings', label: 'Settings', icon: Settings2, hint: 'Paths and defaults', order: 90 },
|
||||
]
|
||||
|
||||
function isActive(path: string, to: string) {
|
||||
@@ -32,6 +37,20 @@ export function Shell({ children, activeRun }: { children: ReactNode; activeRun:
|
||||
const { path } = useRouter()
|
||||
const [system, setSystem] = useState<SystemInfo | null>(null)
|
||||
const [drawerOpen, setDrawerOpen] = useState(false)
|
||||
const { nav: pluginNav } = usePluginUI()
|
||||
|
||||
// Plugin entries are merged into the rail rather than fenced off in a
|
||||
// section of their own, so a plugin view feels like part of the app.
|
||||
const navItems = [
|
||||
...NAV,
|
||||
...pluginNav.map((item) => ({
|
||||
to: item.path,
|
||||
label: item.label,
|
||||
icon: pluginIcon(item.icon),
|
||||
hint: item.hint || `From the ${item.slug} plugin`,
|
||||
order: item.order,
|
||||
})),
|
||||
].sort((a, b) => a.order - b.order || a.label.localeCompare(b.label))
|
||||
|
||||
useEffect(() => {
|
||||
api.system().then(setSystem).catch(() => setSystem(null))
|
||||
@@ -43,7 +62,7 @@ export function Shell({ children, activeRun }: { children: ReactNode; activeRun:
|
||||
|
||||
const nav = (
|
||||
<nav className="flex flex-col gap-1">
|
||||
{NAV.map(({ to, label, icon: Icon, hint }) => {
|
||||
{navItems.map(({ to, label, icon: Icon, hint }) => {
|
||||
const active = isActive(path, to)
|
||||
return (
|
||||
<Link
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
/**
|
||||
* Pick which suites — and which questions inside them — a run should cover.
|
||||
*
|
||||
* The selection model mirrors the API's `selections` shape rather than a flat
|
||||
* list of question IDs, which buys two things:
|
||||
*
|
||||
* - A whole suite is expressed as `{suite}` with no filenames, so "run all of
|
||||
* math-code" stays true even if that suite gains a question later.
|
||||
* - Selecting a suite needs no knowledge of its contents, so questions are
|
||||
* only fetched for the suites a user actually expands.
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useState } from 'react'
|
||||
import { ChevronDown, ChevronRight, Loader2, Lock } from 'lucide-react'
|
||||
import { api, type RunSelection, type SuiteSummary, type TestPrompt } from '../lib/api'
|
||||
import { cx } from '../lib/format'
|
||||
|
||||
/** `'all'` means the whole suite; an array means those filenames only. */
|
||||
export type Picks = Record<string, 'all' | string[]>
|
||||
|
||||
export function selectionsFrom(picks: Picks): RunSelection[] {
|
||||
return Object.entries(picks)
|
||||
.filter(([, value]) => value === 'all' || value.length > 0)
|
||||
.map(([suite, value]) =>
|
||||
value === 'all' ? { suite } : { suite, filenames: value },
|
||||
)
|
||||
}
|
||||
|
||||
export function countSelected(picks: Picks, suites: SuiteSummary[]): number {
|
||||
return suites.reduce((total, suite) => {
|
||||
const pick = picks[suite.slug]
|
||||
if (pick === 'all') return total + suite.count
|
||||
return total + (pick?.length ?? 0)
|
||||
}, 0)
|
||||
}
|
||||
|
||||
type PickState = 'none' | 'some' | 'all'
|
||||
|
||||
function stateOf(pick: Picks[string] | undefined, count: number): PickState {
|
||||
if (pick === 'all') return 'all'
|
||||
if (!pick || pick.length === 0) return 'none'
|
||||
return pick.length >= count ? 'all' : 'some'
|
||||
}
|
||||
|
||||
export function SuitePicker({
|
||||
suites,
|
||||
picks,
|
||||
onChange,
|
||||
disabled,
|
||||
}: {
|
||||
suites: SuiteSummary[]
|
||||
picks: Picks
|
||||
onChange: (next: Picks) => void
|
||||
disabled?: boolean
|
||||
}) {
|
||||
const [expanded, setExpanded] = useState<string | null>(null)
|
||||
const [questions, setQuestions] = useState<Record<string, TestPrompt[]>>({})
|
||||
const [loading, setLoading] = useState<string | null>(null)
|
||||
|
||||
// Only fetch a suite's questions when it is actually opened.
|
||||
const expand = useCallback(
|
||||
(slug: string) => {
|
||||
if (expanded === slug) {
|
||||
setExpanded(null)
|
||||
return
|
||||
}
|
||||
setExpanded(slug)
|
||||
if (questions[slug]) return
|
||||
setLoading(slug)
|
||||
api
|
||||
.suite(slug)
|
||||
.then((detail) => setQuestions((map) => ({ ...map, [slug]: detail.tests })))
|
||||
.catch(() => setQuestions((map) => ({ ...map, [slug]: [] })))
|
||||
.finally(() => setLoading(null))
|
||||
},
|
||||
[expanded, questions],
|
||||
)
|
||||
|
||||
const toggleSuite = (suite: SuiteSummary) => {
|
||||
const next = { ...picks }
|
||||
if (stateOf(picks[suite.slug], suite.count) === 'none') next[suite.slug] = 'all'
|
||||
else delete next[suite.slug]
|
||||
onChange(next)
|
||||
}
|
||||
|
||||
const toggleQuestion = (suite: SuiteSummary, filename: string) => {
|
||||
const loaded = questions[suite.slug] ?? []
|
||||
const currentPick = picks[suite.slug]
|
||||
// Narrowing an "all" suite needs its concrete filenames first.
|
||||
const current =
|
||||
currentPick === 'all' ? loaded.map((q) => q.filename) : [...(currentPick ?? [])]
|
||||
|
||||
const at = current.indexOf(filename)
|
||||
if (at >= 0) current.splice(at, 1)
|
||||
else current.push(filename)
|
||||
|
||||
const next = { ...picks }
|
||||
if (current.length === 0) delete next[suite.slug]
|
||||
else if (loaded.length > 0 && current.length >= loaded.length) next[suite.slug] = 'all'
|
||||
else next[suite.slug] = current
|
||||
onChange(next)
|
||||
}
|
||||
|
||||
return (
|
||||
<ul className="space-y-1.5">
|
||||
{suites.map((suite) => {
|
||||
const state = stateOf(picks[suite.slug], suite.count)
|
||||
const open = expanded === suite.slug
|
||||
const rows = questions[suite.slug] ?? []
|
||||
const pick = picks[suite.slug]
|
||||
|
||||
return (
|
||||
<li
|
||||
key={suite.slug}
|
||||
className={cx(
|
||||
'overflow-hidden rounded-lg border transition-colors',
|
||||
state === 'none'
|
||||
? 'border-navy-700 bg-navy-900/30'
|
||||
: 'border-gold-500/35 bg-gold-500/8',
|
||||
)}
|
||||
>
|
||||
<div className="flex items-center gap-2 px-2.5 py-2">
|
||||
<input
|
||||
type="checkbox"
|
||||
className="h-3.5 w-3.5 shrink-0 accent-[#e3ad46]"
|
||||
checked={state !== 'none'}
|
||||
ref={(node) => {
|
||||
// Partial selection is neither checked nor unchecked.
|
||||
if (node) node.indeterminate = state === 'some'
|
||||
}}
|
||||
disabled={disabled}
|
||||
onChange={() => toggleSuite(suite)}
|
||||
aria-label={`Select ${suite.name}`}
|
||||
/>
|
||||
|
||||
<button
|
||||
className="flex min-w-0 flex-1 items-center gap-1.5 text-left"
|
||||
onClick={() => expand(suite.slug)}
|
||||
disabled={disabled}
|
||||
>
|
||||
{open ? (
|
||||
<ChevronDown size={12} className="shrink-0 text-ink-500" />
|
||||
) : (
|
||||
<ChevronRight size={12} className="shrink-0 text-ink-500" />
|
||||
)}
|
||||
{suite.builtin && <Lock size={10} className="shrink-0 text-ink-600" />}
|
||||
<span className="truncate text-[0.75rem] text-ink-200">{suite.name}</span>
|
||||
</button>
|
||||
|
||||
<span className="num shrink-0 text-[0.6875rem] text-ink-500">
|
||||
{state === 'all'
|
||||
? suite.count
|
||||
: state === 'some'
|
||||
? `${(pick as string[]).length}/${suite.count}`
|
||||
: suite.count}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{open && (
|
||||
<div className="border-t border-navy-700/70 bg-navy-950/30 px-2.5 py-2">
|
||||
{loading === suite.slug ? (
|
||||
<p className="flex items-center gap-2 py-1 text-[0.75rem] text-ink-500">
|
||||
<Loader2 size={12} className="animate-spin" /> Loading questions…
|
||||
</p>
|
||||
) : rows.length === 0 ? (
|
||||
<p className="py-1 text-[0.75rem] text-ink-500">This suite has no questions.</p>
|
||||
) : (
|
||||
<ul className="space-y-0.5">
|
||||
{rows.map((question, index) => {
|
||||
const checked =
|
||||
pick === 'all' || (pick ?? []).includes(question.filename)
|
||||
return (
|
||||
<li key={question.id}>
|
||||
<label
|
||||
className={cx(
|
||||
'flex cursor-pointer items-start gap-2 rounded px-1.5 py-1 transition-colors hover:bg-navy-800/60',
|
||||
disabled && 'cursor-not-allowed opacity-60',
|
||||
)}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
className="mt-0.5 h-3 w-3 shrink-0 accent-[#e3ad46]"
|
||||
checked={checked}
|
||||
disabled={disabled}
|
||||
onChange={() => toggleQuestion(suite, question.filename)}
|
||||
/>
|
||||
<span className="min-w-0">
|
||||
<span className="num mr-1.5 text-[0.625rem] text-ink-600">
|
||||
{String(index + 1).padStart(2, '0')}
|
||||
</span>
|
||||
<span className="text-[0.6875rem] leading-snug text-ink-300">
|
||||
{question.title}
|
||||
</span>
|
||||
</span>
|
||||
</label>
|
||||
</li>
|
||||
)
|
||||
})}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</li>
|
||||
)
|
||||
})}
|
||||
</ul>
|
||||
)
|
||||
}
|
||||
|
||||
/** Default selection: every built-in suite, matching the CLI's bare run. */
|
||||
export function useDefaultPicks(suites: SuiteSummary[]) {
|
||||
const [picks, setPicks] = useState<Picks>({})
|
||||
const [seeded, setSeeded] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
if (seeded || !suites.length) return
|
||||
const next: Picks = {}
|
||||
for (const suite of suites) if (suite.builtin) next[suite.slug] = 'all'
|
||||
setPicks(next)
|
||||
setSeeded(true)
|
||||
}, [suites, seeded])
|
||||
|
||||
return [picks, setPicks] as const
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* The plugin UI manifest, fetched once and shared.
|
||||
*
|
||||
* Nav entries, pages and slot panels all come from here, so a single fetch at
|
||||
* startup is enough for plugins to appear everywhere they contribute. The
|
||||
* manifest is intentionally forgiving: if it cannot be loaded the app renders
|
||||
* exactly as it did before plugins existed rather than showing an error.
|
||||
*/
|
||||
|
||||
import { createContext, useContext, useEffect, useMemo, useState, type ReactNode } from 'react'
|
||||
import { api, type PluginSlotPanel, type PluginUIManifest } from '../lib/api'
|
||||
|
||||
const EMPTY: PluginUIManifest = { nav: [], pages: [], slots: {} }
|
||||
|
||||
interface PluginUIValue extends PluginUIManifest {
|
||||
reload: () => void
|
||||
}
|
||||
|
||||
const PluginUIContext = createContext<PluginUIValue>({ ...EMPTY, reload: () => {} })
|
||||
|
||||
export function PluginUIProvider({ children }: { children: ReactNode }) {
|
||||
const [manifest, setManifest] = useState<PluginUIManifest>(EMPTY)
|
||||
const [nonce, setNonce] = useState(0)
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false
|
||||
api
|
||||
.pluginUI()
|
||||
.then((next) => {
|
||||
if (!cancelled) setManifest(next ?? EMPTY)
|
||||
})
|
||||
.catch(() => {
|
||||
// A plugin manifest is additive. Failing to load one must never take
|
||||
// the rest of the app down with it.
|
||||
if (!cancelled) setManifest(EMPTY)
|
||||
})
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
}, [nonce])
|
||||
|
||||
const value = useMemo(
|
||||
() => ({ ...manifest, reload: () => setNonce((n) => n + 1) }),
|
||||
[manifest],
|
||||
)
|
||||
return <PluginUIContext.Provider value={value}>{children}</PluginUIContext.Provider>
|
||||
}
|
||||
|
||||
export function usePluginUI() {
|
||||
return useContext(PluginUIContext)
|
||||
}
|
||||
|
||||
/** Panels contributed to one injection point, already ordered. */
|
||||
export function usePluginSlot(slot: string): PluginSlotPanel[] {
|
||||
const { slots } = usePluginUI()
|
||||
return slots?.[slot] ?? []
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
subscribeToRun,
|
||||
type Run,
|
||||
type RunEvent,
|
||||
type RunSelection,
|
||||
type TestCompletedEvent,
|
||||
} from '../lib/api'
|
||||
|
||||
@@ -48,6 +49,7 @@ interface RunFeedValue {
|
||||
provider: string
|
||||
model_id: string
|
||||
temperature: number
|
||||
selections?: RunSelection[]
|
||||
filenames?: string[]
|
||||
}) => Promise<void>
|
||||
cancel: () => Promise<void>
|
||||
@@ -159,6 +161,7 @@ export function RunFeedProvider({ children }: { children: ReactNode }) {
|
||||
provider: string
|
||||
model_id: string
|
||||
temperature: number
|
||||
selections?: RunSelection[]
|
||||
filenames?: string[]
|
||||
}) => {
|
||||
setStarting(true)
|
||||
|
||||
+139
-5
@@ -14,11 +14,29 @@ export interface TestPrompt {
|
||||
filename: string
|
||||
title: string
|
||||
prompt: string
|
||||
/** Owning suite slug. */
|
||||
suite: string
|
||||
/** Globally unique "<suite>/<file>". `filename` repeats across suites. */
|
||||
id: string
|
||||
}
|
||||
|
||||
export interface TestSuite {
|
||||
export interface SuiteSummary {
|
||||
slug: string
|
||||
name: string
|
||||
description: string
|
||||
order: number
|
||||
builtin: boolean
|
||||
count: number
|
||||
}
|
||||
|
||||
export interface SuiteDetail extends SuiteSummary {
|
||||
tests: TestPrompt[]
|
||||
directory: string
|
||||
}
|
||||
|
||||
/** One suite's contribution to a run; omit `filenames` to take all of it. */
|
||||
export interface RunSelection {
|
||||
suite: string
|
||||
filenames?: string[]
|
||||
}
|
||||
|
||||
export interface RunMetrics {
|
||||
@@ -57,6 +75,94 @@ export interface PluginSummary {
|
||||
error: string | null
|
||||
}
|
||||
|
||||
/* -------------------------------------------------------- plugin-declared UI */
|
||||
/* Plugins describe interface as data; this app renders it. Nothing here is
|
||||
* plugin-supplied code — see /docs for the contribution vocabulary. */
|
||||
|
||||
export interface PluginStat {
|
||||
label: string
|
||||
value: string | number
|
||||
hint: string
|
||||
tone: 'default' | 'good' | 'warn' | 'bad'
|
||||
}
|
||||
|
||||
export interface StatRowBlock {
|
||||
kind: 'stat_row'
|
||||
stats: PluginStat[]
|
||||
source: string | null
|
||||
}
|
||||
|
||||
export interface TableBlock {
|
||||
kind: 'table'
|
||||
columns: string[]
|
||||
rows: (string | number)[][]
|
||||
source: string | null
|
||||
empty: string
|
||||
}
|
||||
|
||||
export interface MarkdownBlock {
|
||||
kind: 'markdown'
|
||||
text: string
|
||||
source: string | null
|
||||
}
|
||||
|
||||
export interface ActionBlock {
|
||||
kind: 'action'
|
||||
label: string
|
||||
post: string
|
||||
confirm: string | null
|
||||
style: 'primary' | 'ghost' | 'danger'
|
||||
icon: string | null
|
||||
}
|
||||
|
||||
export interface PanelBlock {
|
||||
kind: 'panel'
|
||||
title: string
|
||||
subtitle: string
|
||||
blocks: PluginBlock[]
|
||||
}
|
||||
|
||||
export type PluginBlock = StatRowBlock | TableBlock | MarkdownBlock | ActionBlock | PanelBlock
|
||||
|
||||
export interface PluginNavItem {
|
||||
slug: string
|
||||
label: string
|
||||
path: string
|
||||
icon: string
|
||||
hint: string
|
||||
order: number
|
||||
}
|
||||
|
||||
export interface PluginPageSpec {
|
||||
slug: string
|
||||
plugin: string
|
||||
path: string
|
||||
title: string
|
||||
subtitle: string
|
||||
blocks: PluginBlock[]
|
||||
}
|
||||
|
||||
export interface PluginSlotPanel {
|
||||
slug: string
|
||||
plugin: string
|
||||
order: number
|
||||
panel: PanelBlock
|
||||
}
|
||||
|
||||
export interface PluginUIManifest {
|
||||
nav: PluginNavItem[]
|
||||
pages: PluginPageSpec[]
|
||||
slots: Record<string, PluginSlotPanel[]>
|
||||
}
|
||||
|
||||
/** Plugin data URLs are confined to this app's own plugin namespace. */
|
||||
function assertPluginUrl(url: string): string {
|
||||
if (!url.startsWith('/api/plugins/')) {
|
||||
throw new ApiError(`Refusing to call ${url} — plugin URLs must start with /api/plugins/.`, 0)
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
export type RunStatus = 'running' | 'completed' | 'failed' | 'cancelled'
|
||||
|
||||
export interface Run {
|
||||
@@ -205,17 +311,35 @@ export const api = {
|
||||
`/providers/${encodeURIComponent(provider)}/models`,
|
||||
),
|
||||
|
||||
tests: () => request<TestSuite>('/tests'),
|
||||
saveTests: (prompts: string[]) =>
|
||||
request<TestSuite>('/tests', {
|
||||
suites: () => request<SuiteSummary[]>('/suites'),
|
||||
suite: (slug: string) => request<SuiteDetail>(`/suites/${encodeURIComponent(slug)}`),
|
||||
createSuite: (body: { name: string; description?: string; slug?: string }) =>
|
||||
request<SuiteDetail>('/suites', { method: 'POST', body: JSON.stringify(body) }),
|
||||
updateSuite: (slug: string, body: { name?: string; description?: string }) =>
|
||||
request<SuiteDetail>(`/suites/${encodeURIComponent(slug)}`, {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify(body),
|
||||
}),
|
||||
deleteSuite: (slug: string) =>
|
||||
request<void>(`/suites/${encodeURIComponent(slug)}`, { method: 'DELETE' }),
|
||||
saveSuiteTests: (slug: string, prompts: string[]) =>
|
||||
request<SuiteDetail>(`/suites/${encodeURIComponent(slug)}/tests`, {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify({ tests: prompts.map((prompt) => ({ prompt })) }),
|
||||
}),
|
||||
duplicateSuite: (slug: string, name?: string) =>
|
||||
request<SuiteDetail>(`/suites/${encodeURIComponent(slug)}/duplicate`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ name }),
|
||||
}),
|
||||
|
||||
startRun: (body: {
|
||||
provider: string
|
||||
model_id: string
|
||||
temperature: number
|
||||
/** Suites to run, each optionally narrowed to some of its questions. */
|
||||
selections?: RunSelection[]
|
||||
/** Deprecated: flat qualified "<suite>/<file>" IDs. Prefer `selections`. */
|
||||
filenames?: string[]
|
||||
}) => request<Run>('/runs', { method: 'POST', body: JSON.stringify(body) }),
|
||||
runs: () => request<Run[]>('/runs'),
|
||||
@@ -236,6 +360,16 @@ export const api = {
|
||||
|
||||
plugins: () => request<PluginSummary[]>('/plugins'),
|
||||
reloadPlugins: () => request<PluginSummary[]>('/plugins/reload', { method: 'POST' }),
|
||||
pluginUI: () => request<PluginUIManifest>('/plugins/ui'),
|
||||
|
||||
/** Read a block's `source`. The URL comes from a plugin, so it is checked. */
|
||||
pluginData: <T>(url: string) => request<T>(assertPluginUrl(url).replace(/^\/api/, '')),
|
||||
/** Fire an Action button. May return a message to toast and a refresh flag. */
|
||||
pluginAction: (url: string) =>
|
||||
request<{ message?: string; refresh?: boolean }>(assertPluginUrl(url).replace(/^\/api/, ''), {
|
||||
method: 'POST',
|
||||
body: '{}',
|
||||
}),
|
||||
|
||||
settings: () => request<Settings>('/settings'),
|
||||
saveSettings: (body: {
|
||||
|
||||
@@ -0,0 +1,335 @@
|
||||
/**
|
||||
* Plugin framework documentation.
|
||||
*
|
||||
* The reference prose is static, but the "Installed" section reads the live
|
||||
* registry so the examples and the reality can never drift apart.
|
||||
*/
|
||||
|
||||
import { useState } from 'react'
|
||||
import { BookOpen, CircleAlert, Puzzle } from 'lucide-react'
|
||||
import { Markdown } from '../components/Markdown'
|
||||
import { pluginIcon } from '../components/PluginBlocks'
|
||||
import { Card, CardHeader, PageHeader, useAsync } from '../components/ui'
|
||||
import { usePluginUI } from '../hooks/usePluginUI'
|
||||
import { api } from '../lib/api'
|
||||
import { cx } from '../lib/format'
|
||||
|
||||
const SECTIONS = [
|
||||
{ id: 'start', label: 'Getting started' },
|
||||
{ id: 'hooks', label: 'Hooks' },
|
||||
{ id: 'grading', label: 'Grading' },
|
||||
{ id: 'ui', label: 'Contributing UI' },
|
||||
{ id: 'http', label: 'HTTP routes' },
|
||||
{ id: 'installed', label: 'Installed' },
|
||||
] as const
|
||||
|
||||
const GETTING_STARTED = `
|
||||
Drop a \`.py\` file into \`plugins/\` and it loads at startup. A directory with an
|
||||
\`__init__.py\` works too, when one file is not enough.
|
||||
|
||||
\`\`\`bash
|
||||
cp plugins/_skeleton.py plugins/my_plugin.py
|
||||
\`\`\`
|
||||
|
||||
Files beginning with \`_\` are ignored, so the skeleton itself never runs.
|
||||
|
||||
Every hook is optional — a plugin defining only \`grade\` is perfectly valid.
|
||||
Plugins import from \`plugin_api\` and nothing else in the project, so they keep
|
||||
working across refactors of the engine or the server.
|
||||
|
||||
\`\`\`python
|
||||
NAME = "My plugin" # shown in Settings and in report tables
|
||||
VERSION = "1.0.0"
|
||||
DESCRIPTION = "One line."
|
||||
ENABLED = True # False keeps the file but stops loading it
|
||||
\`\`\`
|
||||
|
||||
**Reloading.** Settings → Plugins has a refresh button that re-scans the
|
||||
directory and picks up edits live. The one exception is \`register_routes\`:
|
||||
HTTP routes are bound when the server starts, so adding or changing them needs
|
||||
a restart.
|
||||
|
||||
**Failure isolation.** A plugin that raises is logged and skipped for that call
|
||||
only. It stays loaded, its other hooks keep firing, and a broken plugin can
|
||||
never fail a run or stop the server.
|
||||
`
|
||||
|
||||
const HOOKS = `
|
||||
| Hook | Called | Use it for |
|
||||
| --- | --- | --- |
|
||||
| \`grade(test)\` | after each answer | Score it, or \`None\` to abstain |
|
||||
| \`on_run_start(run)\` | once, before the first question | Reset state, open a file |
|
||||
| \`on_test_complete(test)\` | after each question | Log, stream, react to failures |
|
||||
| \`on_run_complete(run)\` | when the run ends | Notify, export — fires on cancel and failure too |
|
||||
| \`report_sections(run)\` | after the report is written | Return extra markdown to append |
|
||||
| \`register_routes(router)\` | at server start | Endpoints under \`/api/plugins/<slug>\` |
|
||||
| \`ui_contributions()\` | when the UI loads | Nav entries, pages and panels |
|
||||
| \`register()\` | once, at load | Setup; raising here disables the plugin |
|
||||
|
||||
\`TestRecord\` gives you \`index\`, \`total\`, \`title\`, \`filename\`, \`suite\`,
|
||||
\`prompt\`, \`ok\`, \`response\`, \`error\`, \`metrics\`, \`elapsed\`, plus
|
||||
\`tokens_per_second\`, \`total_tokens\` and \`time_to_first_token\`.
|
||||
|
||||
> **\`filename\` is not unique.** Questions live in named suites and every suite
|
||||
> has a \`test1.txt\`. Identify a question by \`question_id\` — the qualified
|
||||
> \`"<suite>/<file>"\` — never by \`filename\` alone. A run routinely spans
|
||||
> several suites, and graders derive their whole rubric from \`prompt\`, so
|
||||
> mixing two questions up produces a confident, wrong score in silence.
|
||||
|
||||
\`RunRecord\` gives you \`tests\`, \`grades\`, \`summary\`, \`status\`, \`report_path\`,
|
||||
\`score_for(index)\` and \`overall_score\`.
|
||||
`
|
||||
|
||||
const GRADING = `
|
||||
\`grade\` may return \`None\` to abstain, a float from 0.0 to 1.0, \`True\`/\`False\`,
|
||||
or a \`Grade\` with a label and notes that reach the report.
|
||||
|
||||
\`\`\`python
|
||||
from plugin_api import Grade
|
||||
|
||||
def grade(test):
|
||||
if "json" not in test.prompt.lower():
|
||||
return None # not my kind of question
|
||||
ok = test.response.strip().startswith("{")
|
||||
return Grade(
|
||||
score=1.0 if ok else 0.0,
|
||||
label="valid shape" if ok else "not an object",
|
||||
notes="Checked the answer is a bare JSON object.",
|
||||
)
|
||||
\`\`\`
|
||||
|
||||
- **Abstaining is free.** \`None\` excludes the question from that grader
|
||||
entirely; it never counts as a zero. Prefer it over guessing.
|
||||
- **A question's score is the mean** of every grader that scored it, and the
|
||||
run's score is the mean of those.
|
||||
- **Failed questions are never graded** — there is no answer to judge. Use
|
||||
\`on_test_complete\` to see failures.
|
||||
- **Two graders should not check the same thing.** The bundled pair are split
|
||||
deliberately: \`response_checks\` asks whether an answer contains code,
|
||||
\`code_lint\` asks whether that code is any good, and abstains when there is none.
|
||||
`
|
||||
|
||||
const UI_DOCS = `
|
||||
Plugins describe interface as **data**, and the app renders it. No plugin ships
|
||||
JavaScript, and installing one never requires rebuilding the frontend.
|
||||
|
||||
\`\`\`python
|
||||
from plugin_api import Action, NavItem, Page, Panel, StatRow, Table
|
||||
|
||||
def ui_contributions():
|
||||
base = "/api/plugins/my_plugin"
|
||||
return [
|
||||
NavItem(label="My tool", path="/tool", icon="wrench", order=50),
|
||||
Page(
|
||||
path="/tool",
|
||||
title="My tool",
|
||||
subtitle="What it does.",
|
||||
blocks=[
|
||||
StatRow(source=f"{base}/stats"),
|
||||
Panel(title="Results", blocks=[
|
||||
Table(source=f"{base}/rows"),
|
||||
Action(label="Clear", post=f"{base}/clear", style="ghost"),
|
||||
]),
|
||||
],
|
||||
),
|
||||
]
|
||||
\`\`\`
|
||||
|
||||
### Blocks
|
||||
|
||||
| Block | Renders | Fields |
|
||||
| --- | --- | --- |
|
||||
| \`StatRow\` | a row of headline numbers | \`stats\` of \`Stat(label, value, hint, tone)\` |
|
||||
| \`Table\` | a column-headed table | \`columns\`, \`rows\`, \`empty\` |
|
||||
| \`Markdown\` | rendered markdown | \`text\` |
|
||||
| \`Action\` | a button that POSTs | \`label\`, \`post\`, \`confirm\`, \`style\`, \`icon\` |
|
||||
| \`Panel\` | a titled card wrapping blocks | \`title\`, \`subtitle\`, \`blocks\` |
|
||||
|
||||
\`tone\` is one of \`default\`, \`good\`, \`warn\`, \`bad\`. \`style\` is \`primary\`,
|
||||
\`ghost\` or \`danger\`.
|
||||
|
||||
### Static or live
|
||||
|
||||
Every data block takes either inline values or a \`source\` URL. With a source,
|
||||
the block fetches it and expects the same field names back as JSON — so the
|
||||
same \`Table\` can be a fixed list or a live view without changing shape.
|
||||
|
||||
An \`Action\` may return \`{"message": "..."}\` to raise a toast and
|
||||
\`{"refresh": true}\` to make sibling blocks re-fetch.
|
||||
|
||||
### Surfaces
|
||||
|
||||
| Surface | Effect |
|
||||
| --- | --- |
|
||||
| \`NavItem(label, path, icon, hint, order)\` | an entry in the left rail |
|
||||
| \`Page(path, title, subtitle, blocks)\` | a full page |
|
||||
| \`SlotPanel(slot, panel, order)\` | a panel injected into a built-in page |
|
||||
|
||||
Slots are \`run.aside\`, \`reports.aside\`, \`suite.aside\` and \`settings.section\`.
|
||||
|
||||
**Paths are namespaced.** Whatever you declare is served under \`/x/<slug>/\`, so
|
||||
\`"/tool"\` in the plugin \`my_plugin\` becomes \`/x/my_plugin/tool\`. Two plugins
|
||||
can use the same short path without colliding, and no plugin can ever shadow a
|
||||
built-in route.
|
||||
|
||||
**Icons** are named: \`activity\`, \`alert\`, \`bar\`, \`bug\`, \`check\`, \`database\`,
|
||||
\`file\`, \`gauge\`, \`hash\`, \`layers\`, \`list\`, \`play\`, \`puzzle\`, \`refresh\`,
|
||||
\`search\`, \`settings\`, \`shield\`, \`sparkles\`, \`table\`, \`terminal\`, \`trash\`,
|
||||
\`wrench\`, \`zap\`. An unknown name falls back to a puzzle piece.
|
||||
`
|
||||
|
||||
const HTTP_DOCS = `
|
||||
\`register_routes\` receives a FastAPI router already prefixed with
|
||||
\`/api/plugins/<slug>\`. This is where \`source\` and \`post\` URLs point.
|
||||
|
||||
\`\`\`python
|
||||
def register_routes(router):
|
||||
@router.get("/stats")
|
||||
async def stats() -> dict:
|
||||
return {"stats": [
|
||||
{"label": "Checked", "value": 12, "hint": "this run", "tone": "good"},
|
||||
]}
|
||||
|
||||
@router.post("/clear")
|
||||
async def clear() -> dict:
|
||||
return {"message": "Cleared.", "refresh": True}
|
||||
\`\`\`
|
||||
|
||||
The frontend refuses any block URL that does not begin with \`/api/plugins/\`,
|
||||
so a plugin cannot point the browser at an external host.
|
||||
|
||||
> **Routes bind at server start, and reloading does not rebind them.** The
|
||||
> handlers you register close over the module object that existed at startup.
|
||||
> After a plugin reload, the *new* module handles \`grade\` and the other hooks
|
||||
> while your routes keep reading the *old* module's globals — so an endpoint
|
||||
> can serve stale state with no error to hint at it. Restart the server after
|
||||
> editing a plugin that defines \`register_routes\`.
|
||||
`
|
||||
|
||||
function SectionCard({ id, title, body }: { id: string; title: string; body: string }) {
|
||||
return (
|
||||
<Card className="scroll-mt-6" >
|
||||
<div id={id}>
|
||||
<CardHeader title={title} icon={<BookOpen size={14} />} />
|
||||
<div className="px-4 py-3">
|
||||
<Markdown>{body}</Markdown>
|
||||
</div>
|
||||
</div>
|
||||
</Card>
|
||||
)
|
||||
}
|
||||
|
||||
function Installed() {
|
||||
const plugins = useAsync(() => api.plugins(), [])
|
||||
const { nav, pages, slots } = usePluginUI()
|
||||
|
||||
const contributionsFor = (slug: string) => {
|
||||
const bits: string[] = []
|
||||
const navCount = nav.filter((n) => n.slug === slug).length
|
||||
const pageCount = pages.filter((p) => p.slug === slug).length
|
||||
const slotCount = Object.values(slots ?? {}).flat().filter((s) => s.slug === slug).length
|
||||
if (navCount) bits.push(`${navCount} nav entry`)
|
||||
if (pageCount) bits.push(`${pageCount} page`)
|
||||
if (slotCount) bits.push(`${slotCount} panel`)
|
||||
return bits
|
||||
}
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<div id="installed">
|
||||
<CardHeader
|
||||
title="Installed plugins"
|
||||
icon={<Puzzle size={14} />}
|
||||
hint="Read live from the registry."
|
||||
/>
|
||||
<div className="divide-y divide-navy-800/70">
|
||||
{(plugins.data ?? []).map((plugin) => {
|
||||
const Icon = pluginIcon(nav.find((n) => n.slug === plugin.slug)?.icon)
|
||||
const bits = contributionsFor(plugin.slug)
|
||||
return (
|
||||
<div key={plugin.slug} className="flex items-start gap-3 px-4 py-3">
|
||||
<Icon size={15} className="mt-0.5 shrink-0 text-gold-500" />
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="flex flex-wrap items-baseline gap-2">
|
||||
<span className="text-[0.8125rem] font-semibold text-ink-200">{plugin.name}</span>
|
||||
<span className="num text-[0.6875rem] text-ink-500">v{plugin.version}</span>
|
||||
<code className="text-[0.6875rem] text-ink-500">{plugin.slug}</code>
|
||||
</div>
|
||||
<p className="mt-0.5 text-[0.8125rem] text-ink-400">{plugin.description}</p>
|
||||
<div className="mt-1.5 flex flex-wrap gap-1.5">
|
||||
{plugin.hooks.map((hook) => (
|
||||
<span
|
||||
key={hook}
|
||||
className="rounded border border-navy-700 bg-navy-800/60 px-1.5 py-0.5 text-[0.625rem] text-ink-400"
|
||||
>
|
||||
{hook}
|
||||
</span>
|
||||
))}
|
||||
{bits.map((bit) => (
|
||||
<span
|
||||
key={bit}
|
||||
className="rounded border border-gold-500/30 bg-gold-500/10 px-1.5 py-0.5 text-[0.625rem] text-gold-400"
|
||||
>
|
||||
{bit}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
{plugin.error && (
|
||||
<p className="mt-1.5 flex items-center gap-1.5 text-[0.75rem] text-rose-400">
|
||||
<CircleAlert size={12} /> {plugin.error}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
{!plugins.data?.length && (
|
||||
<p className="px-4 py-6 text-center text-[0.8125rem] text-ink-500">
|
||||
Nothing in <code>plugins/</code> yet.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</Card>
|
||||
)
|
||||
}
|
||||
|
||||
export function DocsPage() {
|
||||
const [active, setActive] = useState<string>('start')
|
||||
|
||||
return (
|
||||
<>
|
||||
<PageHeader
|
||||
title="Plugin framework"
|
||||
subtitle="Everything a plugin can do: grade answers, extend reports, add HTTP routes, and contribute interface — without shipping a line of JavaScript."
|
||||
/>
|
||||
|
||||
<nav className="mb-5 flex flex-wrap gap-1.5">
|
||||
{SECTIONS.map((section) => (
|
||||
<a
|
||||
key={section.id}
|
||||
href={`#${section.id}`}
|
||||
onClick={() => setActive(section.id)}
|
||||
className={cx(
|
||||
'rounded-lg border px-2.5 py-1 text-[0.75rem] transition-colors',
|
||||
active === section.id
|
||||
? 'border-gold-500/40 bg-gold-500/10 text-gold-400'
|
||||
: 'border-navy-700 text-ink-400 hover:text-ink-200',
|
||||
)}
|
||||
>
|
||||
{section.label}
|
||||
</a>
|
||||
))}
|
||||
</nav>
|
||||
|
||||
<div className="space-y-4">
|
||||
<SectionCard id="start" title="Getting started" body={GETTING_STARTED} />
|
||||
<SectionCard id="hooks" title="Hook reference" body={HOOKS} />
|
||||
<SectionCard id="grading" title="Grading" body={GRADING} />
|
||||
<SectionCard id="ui" title="Contributing UI" body={UI_DOCS} />
|
||||
<SectionCard id="http" title="HTTP routes" body={HTTP_DOCS} />
|
||||
<Installed />
|
||||
</div>
|
||||
</>
|
||||
)
|
||||
}
|
||||
@@ -29,6 +29,12 @@ export function PlaygroundPage() {
|
||||
const [prompt, setPrompt] = useState('')
|
||||
const [result, setResult] = useState<PlaygroundResult | null>(null)
|
||||
const [sending, setSending] = useState(false)
|
||||
const [saveTarget, setSaveTarget] = useState<null | true>(null)
|
||||
// Built-ins are read-only, so only custom suites are valid targets.
|
||||
const customSuites = useAsync(
|
||||
() => api.suites().then((all) => all.filter((s) => !s.builtin)),
|
||||
[],
|
||||
)
|
||||
|
||||
useEffect(() => {
|
||||
if (!providers.data?.length || provider) return
|
||||
@@ -58,12 +64,24 @@ export function PlaygroundPage() {
|
||||
}
|
||||
}
|
||||
|
||||
const saveAsQuestion = async () => {
|
||||
/**
|
||||
* Append the prompt to a custom suite.
|
||||
*
|
||||
* There is no longer a single "the suite" to append to, and the built-ins
|
||||
* are read-only, so this needs an explicit target. Only custom suites can
|
||||
* be chosen; if none exist yet the button points at Testing Suites instead.
|
||||
*/
|
||||
const saveAsQuestion = async (slug: string) => {
|
||||
if (!prompt.trim()) return
|
||||
setSaveTarget(null)
|
||||
try {
|
||||
const suite = await api.tests()
|
||||
await api.saveTests([...suite.tests.map((t) => t.prompt), prompt.trim()])
|
||||
toast('Added to the suite as the last question.', 'success')
|
||||
const detail = await api.suite(slug)
|
||||
const saved = await api.saveSuiteTests(slug, [
|
||||
...detail.tests.map((t) => t.prompt),
|
||||
prompt.trim(),
|
||||
])
|
||||
toast(`Added to "${saved.name}" as question ${saved.tests.length}.`, 'success')
|
||||
customSuites.reload()
|
||||
} catch (cause) {
|
||||
toast(cause instanceof ApiError ? cause.message : String(cause), 'error')
|
||||
}
|
||||
@@ -146,15 +164,55 @@ export function PlaygroundPage() {
|
||||
title="Prompt"
|
||||
icon={<FlaskConical size={14} />}
|
||||
actions={
|
||||
<button
|
||||
className="btn btn-ghost btn-sm"
|
||||
onClick={saveAsQuestion}
|
||||
disabled={!prompt.trim() || disabled}
|
||||
title="Append this prompt to the diagnostic suite"
|
||||
>
|
||||
<Plus size={12} />
|
||||
Add to suite
|
||||
</button>
|
||||
<div className="relative">
|
||||
<button
|
||||
className="btn btn-ghost btn-sm"
|
||||
onClick={() => setSaveTarget((open) => (open ? null : true))}
|
||||
disabled={!prompt.trim() || disabled}
|
||||
title="Append this prompt to a custom suite"
|
||||
>
|
||||
<Plus size={12} />
|
||||
Add to suite
|
||||
</button>
|
||||
|
||||
{saveTarget && (
|
||||
<>
|
||||
<div className="fixed inset-0 z-10" onClick={() => setSaveTarget(null)} />
|
||||
<div className="surface-raised animate-fade-up absolute right-0 z-20 mt-1.5 w-60 p-1.5">
|
||||
<p className="px-2 py-1 text-[0.625rem] font-semibold tracking-[0.09em] text-ink-500 uppercase">
|
||||
Add to
|
||||
</p>
|
||||
{customSuites.data?.length ? (
|
||||
customSuites.data.map((suite) => (
|
||||
<button
|
||||
key={suite.slug}
|
||||
className="flex w-full items-center justify-between gap-2 rounded-lg px-2 py-1.5 text-left text-[0.75rem] text-ink-300 transition-colors hover:bg-navy-750 hover:text-ink-100"
|
||||
onClick={() => saveAsQuestion(suite.slug)}
|
||||
>
|
||||
<span className="truncate">{suite.name}</span>
|
||||
<span className="num shrink-0 text-[0.6875rem] text-ink-500">
|
||||
{suite.count}
|
||||
</span>
|
||||
</button>
|
||||
))
|
||||
) : (
|
||||
<div className="px-2 py-2">
|
||||
<p className="text-[0.75rem] leading-relaxed text-ink-500">
|
||||
No custom suites yet. Built-in suites are read-only.
|
||||
</p>
|
||||
<Link
|
||||
to="/suite"
|
||||
className="btn btn-ghost btn-sm mt-2 w-full"
|
||||
onClick={() => setSaveTarget(null)}
|
||||
>
|
||||
Open Testing Suites
|
||||
</Link>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
}
|
||||
/>
|
||||
<div className="p-4">
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
/** Renders a page declared by a plugin, matched on the current path. */
|
||||
|
||||
import { Puzzle } from 'lucide-react'
|
||||
import { PluginBlocks } from '../components/PluginBlocks'
|
||||
import { EmptyState, PageHeader } from '../components/ui'
|
||||
import { usePluginUI } from '../hooks/usePluginUI'
|
||||
import { useRouter } from '../lib/router'
|
||||
import { Link } from '../lib/router'
|
||||
|
||||
export function PluginPage() {
|
||||
const { path } = useRouter()
|
||||
const { pages } = usePluginUI()
|
||||
|
||||
const page = pages.find((candidate) => candidate.path === path)
|
||||
|
||||
if (!page) {
|
||||
return (
|
||||
<EmptyState
|
||||
icon={<Puzzle size={18} />}
|
||||
title="No plugin owns this page"
|
||||
description={`Nothing is registered at ${path}. The plugin that provided it may have been disabled or removed — check Settings, or reload plugins.`}
|
||||
action={
|
||||
<Link to="/settings" className="btn btn-ghost btn-sm">
|
||||
Open Settings
|
||||
</Link>
|
||||
}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<PageHeader
|
||||
title={page.title}
|
||||
subtitle={page.subtitle || undefined}
|
||||
actions={
|
||||
<span className="rounded-lg border border-navy-700 bg-navy-800/60 px-2.5 py-1 text-[0.6875rem] text-ink-500">
|
||||
from {page.plugin}
|
||||
</span>
|
||||
}
|
||||
/>
|
||||
<PluginBlocks blocks={page.blocks} />
|
||||
</>
|
||||
)
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
useAsync,
|
||||
useToast,
|
||||
} from '../components/ui'
|
||||
import { PluginSlot } from '../components/PluginBlocks'
|
||||
import { Markdown } from '../components/Markdown'
|
||||
import { ScoreBadge, scoreTone } from '../components/ScoreBadge'
|
||||
import { ThroughputChart } from '../components/ThroughputChart'
|
||||
@@ -185,6 +186,7 @@ export function ReportsPage() {
|
||||
</ul>
|
||||
)}
|
||||
</Card>
|
||||
<PluginSlot slot="reports.aside" className="mt-4" />
|
||||
</div>
|
||||
|
||||
{/* ----------------------------------------------------- detail */}
|
||||
|
||||
+51
-66
@@ -28,6 +28,13 @@ import {
|
||||
} from '../components/ui'
|
||||
import { Markdown } from '../components/Markdown'
|
||||
import { ScoreBadge, scoreTone } from '../components/ScoreBadge'
|
||||
import { PluginSlot } from '../components/PluginBlocks'
|
||||
import {
|
||||
SuitePicker,
|
||||
countSelected,
|
||||
selectionsFrom,
|
||||
useDefaultPicks,
|
||||
} from '../components/SuitePicker'
|
||||
import { Link } from '../lib/router'
|
||||
import { api, ApiError, type TestCompletedEvent } from '../lib/api'
|
||||
import { clockTime, cx, formatDuration } from '../lib/format'
|
||||
@@ -38,12 +45,11 @@ export function RunPage() {
|
||||
const { run, outcomes, logs, starting, isLive, start, cancel, clear } = useRunFeed()
|
||||
|
||||
const providers = useAsync(() => api.providers(), [])
|
||||
const suite = useAsync(() => api.tests(), [])
|
||||
const suites = useAsync(() => api.suites(), [])
|
||||
|
||||
const [provider, setProvider] = useState('')
|
||||
const [modelId, setModelId] = useState('')
|
||||
const [temperature, setTemperature] = useState(0.1)
|
||||
const [selected, setSelected] = useState<string[] | null>(null)
|
||||
const [pickerOpen, setPickerOpen] = useState(false)
|
||||
|
||||
// Seed provider + temperature from saved settings, falling back to the
|
||||
@@ -72,25 +78,23 @@ export function RunPage() {
|
||||
setModelId((current) => (list.some((m) => m.id === current) ? current : (list[0]?.id ?? '')))
|
||||
}, [models.data])
|
||||
|
||||
const tests = suite.data?.tests ?? []
|
||||
const selectedFilenames = selected ?? tests.map((t) => t.filename)
|
||||
const allSelected = selectedFilenames.length === tests.length
|
||||
const suiteList = suites.data ?? []
|
||||
const [picks, setPicks] = useDefaultPicks(suiteList)
|
||||
|
||||
const toggleTest = (filename: string) => {
|
||||
const next = new Set(selectedFilenames)
|
||||
if (next.has(filename)) next.delete(filename)
|
||||
else next.add(filename)
|
||||
setSelected(tests.filter((t) => next.has(t.filename)).map((t) => t.filename))
|
||||
}
|
||||
const selectedCount = countSelected(picks, suiteList)
|
||||
const totalCount = suiteList.reduce((sum, s) => sum + s.count, 0)
|
||||
const pickedSuites = Object.keys(picks).length
|
||||
|
||||
const launch = async () => {
|
||||
if (!provider || !modelId) return
|
||||
try {
|
||||
// Sent as `selections`, so "all of a suite" stays all of it even if that
|
||||
// suite gains a question between now and the next run.
|
||||
await start({
|
||||
provider,
|
||||
model_id: modelId,
|
||||
temperature,
|
||||
filenames: allSelected ? undefined : selectedFilenames,
|
||||
selections: selectionsFrom(picks),
|
||||
})
|
||||
} catch (cause) {
|
||||
toast(cause instanceof ApiError ? cause.message : String(cause), 'error')
|
||||
@@ -105,8 +109,7 @@ export function RunPage() {
|
||||
}
|
||||
}
|
||||
|
||||
const canLaunch =
|
||||
!!provider && !!modelId && selectedFilenames.length > 0 && !isLive && !starting
|
||||
const canLaunch = !!provider && !!modelId && selectedCount > 0 && !isLive && !starting
|
||||
|
||||
return (
|
||||
<>
|
||||
@@ -189,92 +192,72 @@ export function RunPage() {
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
{/* question selection */}
|
||||
{/* suite + question selection */}
|
||||
<Card raised>
|
||||
<CardHeader
|
||||
title="Questions"
|
||||
title="Suites"
|
||||
icon={<ListChecks size={14} />}
|
||||
actions={
|
||||
<span className="chip chip-gold num">
|
||||
{selectedFilenames.length}/{tests.length}
|
||||
{selectedCount}/{totalCount}
|
||||
</span>
|
||||
}
|
||||
/>
|
||||
<div className="p-4">
|
||||
{suite.error && <ErrorNote message={suite.error} onRetry={suite.reload} />}
|
||||
{suites.error && <ErrorNote message={suites.error} onRetry={suites.reload} />}
|
||||
|
||||
{!suite.error && tests.length === 0 && !suite.loading && (
|
||||
{!suites.error && suiteList.length === 0 && !suites.loading && (
|
||||
<EmptyState
|
||||
icon={<ListChecks size={20} />}
|
||||
title="No questions yet"
|
||||
description="Add questions in the Suite Builder before running diagnostics."
|
||||
title="No suites found"
|
||||
description="Create a suite before running diagnostics."
|
||||
action={
|
||||
<Link to="/suite" className="btn btn-primary">
|
||||
Open Suite Builder
|
||||
Open Testing Suites
|
||||
</Link>
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{tests.length > 0 && (
|
||||
{suiteList.length > 0 && (
|
||||
<>
|
||||
<div className="mb-3 flex items-center gap-2">
|
||||
<button
|
||||
className="btn btn-ghost btn-sm"
|
||||
disabled={isLive}
|
||||
onClick={() => setSelected(allSelected ? [] : tests.map((t) => t.filename))}
|
||||
onClick={() =>
|
||||
setPicks(
|
||||
pickedSuites === suiteList.length
|
||||
? {}
|
||||
: Object.fromEntries(suiteList.map((s) => [s.slug, 'all' as const])),
|
||||
)
|
||||
}
|
||||
>
|
||||
{allSelected ? 'Clear all' : 'Select all'}
|
||||
{pickedSuites === suiteList.length ? 'Clear all' : 'Select all'}
|
||||
</button>
|
||||
<button
|
||||
className="btn btn-ghost btn-sm"
|
||||
onClick={() => setPickerOpen((open) => !open)}
|
||||
>
|
||||
{pickerOpen ? <ChevronDown size={13} /> : <ChevronRight size={13} />}
|
||||
{pickerOpen ? 'Hide list' : 'Choose questions'}
|
||||
{pickerOpen ? 'Hide suites' : 'Choose suites'}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{pickerOpen && (
|
||||
<ul className="animate-fade-in max-h-72 space-y-1 overflow-y-auto pr-1">
|
||||
{tests.map((test, index) => {
|
||||
const checked = selectedFilenames.includes(test.filename)
|
||||
return (
|
||||
<li key={test.filename}>
|
||||
<label
|
||||
className={cx(
|
||||
'flex cursor-pointer items-start gap-2.5 rounded-lg border px-2.5 py-2 transition-colors',
|
||||
checked
|
||||
? 'border-gold-500/35 bg-gold-500/8'
|
||||
: 'border-navy-700 bg-navy-900/30 hover:border-navy-600',
|
||||
isLive && 'cursor-not-allowed opacity-60',
|
||||
)}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
className="mt-0.5 h-3.5 w-3.5 shrink-0 accent-[#e3ad46]"
|
||||
checked={checked}
|
||||
disabled={isLive}
|
||||
onChange={() => toggleTest(test.filename)}
|
||||
/>
|
||||
<span className="min-w-0">
|
||||
<span className="num mr-1.5 text-[0.6875rem] text-ink-500">
|
||||
{String(index + 1).padStart(2, '0')}
|
||||
</span>
|
||||
<span className="text-[0.75rem] text-ink-200">{test.title}</span>
|
||||
</span>
|
||||
</label>
|
||||
</li>
|
||||
)
|
||||
})}
|
||||
</ul>
|
||||
)}
|
||||
|
||||
{!pickerOpen && (
|
||||
<p className="text-[0.75rem] text-ink-500">
|
||||
{allSelected
|
||||
? `The full suite of ${tests.length} questions will run in order.`
|
||||
: `${selectedFilenames.length} of ${tests.length} questions selected.`}
|
||||
{pickerOpen ? (
|
||||
<div className="animate-fade-in max-h-96 overflow-y-auto pr-1">
|
||||
<SuitePicker
|
||||
suites={suiteList}
|
||||
picks={picks}
|
||||
onChange={setPicks}
|
||||
disabled={isLive}
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<p className="text-[0.75rem] leading-relaxed text-ink-500">
|
||||
{selectedCount === 0
|
||||
? 'Nothing selected — pick at least one suite.'
|
||||
: `${selectedCount} question${selectedCount === 1 ? '' : 's'} from ${pickedSuites} suite${pickedSuites === 1 ? '' : 's'}, run in order.`}
|
||||
</p>
|
||||
)}
|
||||
</>
|
||||
@@ -295,6 +278,8 @@ export function RunPage() {
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<PluginSlot slot="run.aside" autoRefreshMs={isLive ? 4000 : undefined} />
|
||||
</div>
|
||||
|
||||
{/* ------------------------------------------------------ live feed */}
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
useAsync,
|
||||
useToast,
|
||||
} from '../components/ui'
|
||||
import { PluginSlot } from '../components/PluginBlocks'
|
||||
import { api, ApiError, type ModelPathEntry, type PluginSummary } from '../lib/api'
|
||||
import { cx } from '../lib/format'
|
||||
|
||||
@@ -203,6 +204,7 @@ export function SettingsPage() {
|
||||
</div>
|
||||
|
||||
<div className="space-y-5 xl:col-span-5">
|
||||
<PluginSlot slot="settings.section" />
|
||||
<Card>
|
||||
<CardHeader title="Engine" icon={<Cpu size={14} />} />
|
||||
<div className="p-4">
|
||||
|
||||
+580
-187
@@ -1,10 +1,23 @@
|
||||
/**
|
||||
* Testing Suites — browse every suite, edit the custom ones.
|
||||
*
|
||||
* Two tiers. Built-ins ship with the app and are read-only at both levels: the
|
||||
* suite cannot be renamed or deleted, and its questions cannot be edited. That
|
||||
* is enforced by the API (409), not by the disabled controls here — this page
|
||||
* only reflects it. "Duplicate" is what keeps read-only from being a dead end.
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'
|
||||
import {
|
||||
ArrowDown,
|
||||
ArrowUp,
|
||||
Copy,
|
||||
FolderPlus,
|
||||
Info,
|
||||
Layers,
|
||||
ListChecks,
|
||||
Lock,
|
||||
Pencil,
|
||||
Plus,
|
||||
RotateCcw,
|
||||
Save,
|
||||
@@ -12,15 +25,18 @@ import {
|
||||
} from 'lucide-react'
|
||||
import {
|
||||
Card,
|
||||
CardHeader,
|
||||
ConfirmDialog,
|
||||
EmptyState,
|
||||
ErrorNote,
|
||||
PageHeader,
|
||||
SkeletonRows,
|
||||
Spinner,
|
||||
useAsync,
|
||||
useToast,
|
||||
} from '../components/ui'
|
||||
import { api, ApiError } from '../lib/api'
|
||||
import { PluginSlot } from '../components/PluginBlocks'
|
||||
import { api, ApiError, type SuiteDetail, type SuiteSummary } from '../lib/api'
|
||||
import { cx, deriveTitle } from '../lib/format'
|
||||
import { useRunFeed } from '../hooks/useRunFeed'
|
||||
|
||||
@@ -33,48 +49,69 @@ interface Draft {
|
||||
let keySeed = 1
|
||||
const newKey = () => `q${keySeed++}`
|
||||
|
||||
const PLACEHOLDER =
|
||||
'Write the question exactly as the model should receive it.\n\n' +
|
||||
'Example: Analyze the sentiment of the following customer review.'
|
||||
|
||||
export function SuitePage() {
|
||||
const toast = useToast()
|
||||
const { isLive } = useRunFeed()
|
||||
|
||||
const suites = useAsync(() => api.suites(), [])
|
||||
const [activeSlug, setActiveSlug] = useState<string | null>(null)
|
||||
const [detail, setDetail] = useState<SuiteDetail | null>(null)
|
||||
const [drafts, setDrafts] = useState<Draft[]>([])
|
||||
const [baseline, setBaseline] = useState<string[]>([])
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [loadError, setLoadError] = useState<string | null>(null)
|
||||
const [loadingDetail, setLoadingDetail] = useState(false)
|
||||
const [detailError, setDetailError] = useState<string | null>(null)
|
||||
const [saving, setSaving] = useState(false)
|
||||
const [confirmDiscard, setConfirmDiscard] = useState(false)
|
||||
const [pendingDelete, setPendingDelete] = useState<string | null>(null)
|
||||
const [focusKey, setFocusKey] = useState<string | null>(null)
|
||||
const [busy, setBusy] = useState(false)
|
||||
|
||||
const load = useCallback(() => {
|
||||
setLoading(true)
|
||||
setLoadError(null)
|
||||
const [focusKey, setFocusKey] = useState<string | null>(null)
|
||||
const [pendingDelete, setPendingDelete] = useState<string | null>(null)
|
||||
const [confirmDiscard, setConfirmDiscard] = useState(false)
|
||||
const [pendingSwitch, setPendingSwitch] = useState<string | null>(null)
|
||||
const [deleteSuiteOpen, setDeleteSuiteOpen] = useState(false)
|
||||
const [dialog, setDialog] = useState<null | { mode: 'create' | 'rename' | 'duplicate' }>(null)
|
||||
|
||||
const list = suites.data ?? []
|
||||
const builtins = list.filter((s) => s.builtin)
|
||||
const customs = list.filter((s) => !s.builtin)
|
||||
const readOnly = detail?.builtin ?? true
|
||||
|
||||
// Select something as soon as the list arrives.
|
||||
useEffect(() => {
|
||||
if (!activeSlug && list.length) setActiveSlug(list[0].slug)
|
||||
}, [list, activeSlug])
|
||||
|
||||
const loadDetail = useCallback((slug: string) => {
|
||||
setLoadingDetail(true)
|
||||
setDetailError(null)
|
||||
api
|
||||
.tests()
|
||||
.then((suite) => {
|
||||
.suite(slug)
|
||||
.then((data) => {
|
||||
setDetail(data)
|
||||
setDrafts(
|
||||
suite.tests.map((test) => ({
|
||||
key: newKey(),
|
||||
prompt: test.prompt,
|
||||
filename: test.filename,
|
||||
})),
|
||||
data.tests.map((t) => ({ key: newKey(), prompt: t.prompt, filename: t.filename })),
|
||||
)
|
||||
setBaseline(suite.tests.map((test) => test.prompt))
|
||||
setBaseline(data.tests.map((t) => t.prompt))
|
||||
})
|
||||
.catch((cause: unknown) =>
|
||||
setLoadError(cause instanceof Error ? cause.message : String(cause)),
|
||||
setDetailError(cause instanceof Error ? cause.message : String(cause)),
|
||||
)
|
||||
.finally(() => setLoading(false))
|
||||
.finally(() => setLoadingDetail(false))
|
||||
}, [])
|
||||
|
||||
useEffect(load, [load])
|
||||
useEffect(() => {
|
||||
if (activeSlug) loadDetail(activeSlug)
|
||||
}, [activeSlug, loadDetail])
|
||||
|
||||
const current = drafts.map((draft) => draft.prompt)
|
||||
const current = drafts.map((d) => d.prompt)
|
||||
const dirty =
|
||||
current.length !== baseline.length ||
|
||||
current.some((prompt, index) => prompt.trim() !== baseline[index]?.trim())
|
||||
!readOnly &&
|
||||
(current.length !== baseline.length ||
|
||||
current.some((p, i) => p.trim() !== baseline[i]?.trim()))
|
||||
|
||||
// Guard against losing edits on reload / close.
|
||||
useEffect(() => {
|
||||
if (!dirty) return
|
||||
const warn = (event: BeforeUnloadEvent) => event.preventDefault()
|
||||
@@ -82,36 +119,45 @@ export function SuitePage() {
|
||||
return () => window.removeEventListener('beforeunload', warn)
|
||||
}, [dirty])
|
||||
|
||||
/** Switching away from unsaved edits asks first. */
|
||||
const selectSuite = (slug: string) => {
|
||||
if (slug === activeSlug) return
|
||||
if (dirty) setPendingSwitch(slug)
|
||||
else setActiveSlug(slug)
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------ questions */
|
||||
|
||||
const update = (key: string, prompt: string) =>
|
||||
setDrafts((list) => list.map((d) => (d.key === key ? { ...d, prompt } : d)))
|
||||
setDrafts((l) => l.map((d) => (d.key === key ? { ...d, prompt } : d)))
|
||||
|
||||
const addQuestion = () => {
|
||||
const key = newKey()
|
||||
setDrafts((list) => [...list, { key, prompt: '', filename: null }])
|
||||
setDrafts((l) => [...l, { key, prompt: '', filename: null }])
|
||||
setFocusKey(key)
|
||||
}
|
||||
|
||||
const duplicate = (key: string) =>
|
||||
setDrafts((list) => {
|
||||
const index = list.findIndex((d) => d.key === key)
|
||||
if (index < 0) return list
|
||||
const copy = { key: newKey(), prompt: list[index].prompt, filename: null }
|
||||
return [...list.slice(0, index + 1), copy, ...list.slice(index + 1)]
|
||||
const duplicateQuestion = (key: string) =>
|
||||
setDrafts((l) => {
|
||||
const i = l.findIndex((d) => d.key === key)
|
||||
if (i < 0) return l
|
||||
return [...l.slice(0, i + 1), { key: newKey(), prompt: l[i].prompt, filename: null }, ...l.slice(i + 1)]
|
||||
})
|
||||
|
||||
const remove = (key: string) => setDrafts((list) => list.filter((d) => d.key !== key))
|
||||
const removeQuestion = (key: string) => setDrafts((l) => l.filter((d) => d.key !== key))
|
||||
|
||||
const move = (key: string, direction: -1 | 1) =>
|
||||
setDrafts((list) => {
|
||||
const index = list.findIndex((d) => d.key === key)
|
||||
const target = index + direction
|
||||
if (index < 0 || target < 0 || target >= list.length) return list
|
||||
const next = [...list]
|
||||
;[next[index], next[target]] = [next[target], next[index]]
|
||||
setDrafts((l) => {
|
||||
const i = l.findIndex((d) => d.key === key)
|
||||
const target = i + direction
|
||||
if (i < 0 || target < 0 || target >= l.length) return l
|
||||
const next = [...l]
|
||||
;[next[i], next[target]] = [next[target], next[i]]
|
||||
return next
|
||||
})
|
||||
|
||||
const save = async () => {
|
||||
if (!detail) return
|
||||
const blank = drafts.findIndex((d) => !d.prompt.trim())
|
||||
if (blank >= 0) {
|
||||
toast(`Question ${blank + 1} is empty. Add a prompt or remove it.`, 'error')
|
||||
@@ -119,19 +165,12 @@ export function SuitePage() {
|
||||
}
|
||||
setSaving(true)
|
||||
try {
|
||||
const suite = await api.saveTests(drafts.map((d) => d.prompt))
|
||||
setDrafts(
|
||||
suite.tests.map((test) => ({
|
||||
key: newKey(),
|
||||
prompt: test.prompt,
|
||||
filename: test.filename,
|
||||
})),
|
||||
)
|
||||
setBaseline(suite.tests.map((test) => test.prompt))
|
||||
toast(
|
||||
`Saved ${suite.tests.length} question${suite.tests.length === 1 ? '' : 's'} to the suite.`,
|
||||
'success',
|
||||
)
|
||||
const saved = await api.saveSuiteTests(detail.slug, drafts.map((d) => d.prompt))
|
||||
setDetail(saved)
|
||||
setDrafts(saved.tests.map((t) => ({ key: newKey(), prompt: t.prompt, filename: t.filename })))
|
||||
setBaseline(saved.tests.map((t) => t.prompt))
|
||||
suites.reload()
|
||||
toast(`Saved ${saved.tests.length} question${saved.tests.length === 1 ? '' : 's'} to ${saved.name}.`, 'success')
|
||||
} catch (cause) {
|
||||
toast(cause instanceof ApiError ? cause.message : String(cause), 'error')
|
||||
} finally {
|
||||
@@ -139,29 +178,72 @@ export function SuitePage() {
|
||||
}
|
||||
}
|
||||
|
||||
const deleting = drafts.find((d) => d.key === pendingDelete)
|
||||
/* --------------------------------------------------------------- suites */
|
||||
|
||||
const submitDialog = async (name: string, description: string) => {
|
||||
if (!name.trim()) return
|
||||
setBusy(true)
|
||||
try {
|
||||
let result: SuiteDetail
|
||||
if (dialog?.mode === 'create') {
|
||||
result = await api.createSuite({ name, description })
|
||||
toast(`Created "${result.name}".`, 'success')
|
||||
} else if (dialog?.mode === 'duplicate') {
|
||||
result = await api.duplicateSuite(detail!.slug, name)
|
||||
toast(`Copied to "${result.name}" — this one is editable.`, 'success')
|
||||
} else {
|
||||
result = await api.updateSuite(detail!.slug, { name, description })
|
||||
toast('Suite renamed.', 'success')
|
||||
}
|
||||
setDialog(null)
|
||||
await suites.reload()
|
||||
setActiveSlug(result.slug)
|
||||
if (result.slug === activeSlug) loadDetail(result.slug)
|
||||
} catch (cause) {
|
||||
toast(cause instanceof ApiError ? cause.message : String(cause), 'error')
|
||||
} finally {
|
||||
setBusy(false)
|
||||
}
|
||||
}
|
||||
|
||||
const deleteSuite = async () => {
|
||||
if (!detail) return
|
||||
setDeleteSuiteOpen(false)
|
||||
setBusy(true)
|
||||
try {
|
||||
await api.deleteSuite(detail.slug)
|
||||
toast(`Deleted "${detail.name}".`, 'success')
|
||||
setActiveSlug(null)
|
||||
setDetail(null)
|
||||
await suites.reload()
|
||||
} catch (cause) {
|
||||
toast(cause instanceof ApiError ? cause.message : String(cause), 'error')
|
||||
} finally {
|
||||
setBusy(false)
|
||||
}
|
||||
}
|
||||
|
||||
const deletingQuestion = drafts.find((d) => d.key === pendingDelete)
|
||||
|
||||
return (
|
||||
<>
|
||||
<PageHeader
|
||||
title="Suite Builder"
|
||||
subtitle="Each box is one question. They are sent to the model one at a time, in this order."
|
||||
title="Testing Suites"
|
||||
subtitle="Questions are grouped into named suites. Built-in suites ship with the app and are read-only; duplicate one to make it yours."
|
||||
actions={
|
||||
<>
|
||||
{dirty && (
|
||||
<button
|
||||
className="btn btn-ghost"
|
||||
onClick={() => setConfirmDiscard(true)}
|
||||
disabled={saving}
|
||||
>
|
||||
<button className="btn btn-ghost" onClick={() => setConfirmDiscard(true)} disabled={saving}>
|
||||
<RotateCcw size={14} />
|
||||
Discard
|
||||
</button>
|
||||
)}
|
||||
<button className="btn btn-primary" onClick={save} disabled={!dirty || saving || isLive}>
|
||||
{saving ? <Spinner /> : <Save size={14} />}
|
||||
{saving ? 'Saving…' : dirty ? 'Save suite' : 'Saved'}
|
||||
</button>
|
||||
{!readOnly && detail && (
|
||||
<button className="btn btn-primary" onClick={save} disabled={!dirty || saving || isLive}>
|
||||
{saving ? <Spinner /> : <Save size={14} />}
|
||||
{saving ? 'Saving…' : dirty ? 'Save questions' : 'Saved'}
|
||||
</button>
|
||||
)}
|
||||
</>
|
||||
}
|
||||
/>
|
||||
@@ -170,108 +252,238 @@ export function SuitePage() {
|
||||
<div className="mb-4 flex items-start gap-2.5 rounded-xl border border-ember-500/35 bg-ember-500/8 px-4 py-3">
|
||||
<Info size={15} className="mt-0.5 shrink-0 text-ember-400" />
|
||||
<p className="text-[0.8125rem] text-ember-300">
|
||||
A run is in progress. You can edit questions, but saving is blocked until it finishes.
|
||||
A run is in progress. Editing is blocked until it finishes.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{loadError && <ErrorNote message={loadError} onRetry={load} />}
|
||||
{suites.error && <ErrorNote message={suites.error} onRetry={suites.reload} />}
|
||||
|
||||
{loading ? (
|
||||
<Card>
|
||||
<SkeletonRows rows={4} />
|
||||
</Card>
|
||||
) : (
|
||||
<div className="grid gap-5 xl:grid-cols-12">
|
||||
<div className="space-y-3 xl:col-span-9">
|
||||
{drafts.length === 0 ? (
|
||||
<Card>
|
||||
<EmptyState
|
||||
icon={<ListChecks size={20} />}
|
||||
title="The suite is empty"
|
||||
description="Add your first question to start benchmarking."
|
||||
action={
|
||||
<button className="btn btn-primary" onClick={addQuestion}>
|
||||
<Plus size={14} />
|
||||
Add question
|
||||
</button>
|
||||
}
|
||||
/>
|
||||
</Card>
|
||||
<div className="grid gap-5 xl:grid-cols-12">
|
||||
{/* ------------------------------------------------- suite list */}
|
||||
<div className="xl:col-span-4">
|
||||
<Card className="xl:sticky xl:top-6">
|
||||
<CardHeader
|
||||
title="Suites"
|
||||
icon={<Layers size={14} />}
|
||||
actions={
|
||||
<button
|
||||
className="btn btn-ghost btn-sm"
|
||||
onClick={() => setDialog({ mode: 'create' })}
|
||||
disabled={busy || isLive}
|
||||
>
|
||||
<FolderPlus size={13} />
|
||||
New
|
||||
</button>
|
||||
}
|
||||
/>
|
||||
|
||||
{suites.loading && !list.length ? (
|
||||
<SkeletonRows rows={4} />
|
||||
) : (
|
||||
drafts.map((draft, index) => (
|
||||
<QuestionCard
|
||||
key={draft.key}
|
||||
draft={draft}
|
||||
index={index}
|
||||
total={drafts.length}
|
||||
autoFocus={focusKey === draft.key}
|
||||
onFocused={() => setFocusKey(null)}
|
||||
onChange={(prompt) => update(draft.key, prompt)}
|
||||
onMoveUp={() => move(draft.key, -1)}
|
||||
onMoveDown={() => move(draft.key, 1)}
|
||||
onDuplicate={() => duplicate(draft.key)}
|
||||
onDelete={() =>
|
||||
draft.prompt.trim() ? setPendingDelete(draft.key) : remove(draft.key)
|
||||
}
|
||||
/>
|
||||
))
|
||||
<div className="p-2">
|
||||
<SuiteGroup label="Built-in" hint="read-only">
|
||||
{builtins.map((s) => (
|
||||
<SuiteRow
|
||||
key={s.slug}
|
||||
suite={s}
|
||||
active={s.slug === activeSlug}
|
||||
onClick={() => selectSuite(s.slug)}
|
||||
/>
|
||||
))}
|
||||
</SuiteGroup>
|
||||
|
||||
<SuiteGroup label="Custom" hint={customs.length ? 'editable' : undefined}>
|
||||
{customs.length ? (
|
||||
customs.map((s) => (
|
||||
<SuiteRow
|
||||
key={s.slug}
|
||||
suite={s}
|
||||
active={s.slug === activeSlug}
|
||||
onClick={() => selectSuite(s.slug)}
|
||||
/>
|
||||
))
|
||||
) : (
|
||||
<p className="px-2.5 py-3 text-[0.75rem] leading-relaxed text-ink-500">
|
||||
None yet. Duplicate a built-in suite, or create an empty one.
|
||||
</p>
|
||||
)}
|
||||
</SuiteGroup>
|
||||
</div>
|
||||
)}
|
||||
</Card>
|
||||
|
||||
{drafts.length > 0 && (
|
||||
<button
|
||||
className="flex w-full items-center justify-center gap-2 rounded-xl border border-dashed border-navy-600 py-3.5 text-[0.8125rem] font-medium text-ink-400 transition-colors hover:border-ember-500/60 hover:bg-ember-500/5 hover:text-ember-400"
|
||||
onClick={addQuestion}
|
||||
>
|
||||
<Plus size={15} />
|
||||
Add question
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<aside className="xl:col-span-3">
|
||||
<div className="sticky top-6 space-y-3">
|
||||
<Card>
|
||||
<div className="p-4">
|
||||
<div className="text-[0.625rem] font-semibold tracking-[0.09em] text-ink-500 uppercase">
|
||||
Suite
|
||||
</div>
|
||||
<div className="num mt-1 text-[1.6rem] font-semibold text-gold-400">
|
||||
{drafts.length}
|
||||
</div>
|
||||
<div className="text-[0.75rem] text-ink-500">
|
||||
question{drafts.length === 1 ? '' : 's'}
|
||||
{dirty && <span className="ml-1.5 text-ember-400">· unsaved</span>}
|
||||
</div>
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
<Card>
|
||||
<div className="space-y-2.5 p-4 text-[0.75rem] leading-relaxed text-ink-400">
|
||||
<p className="flex items-center gap-1.5 font-semibold text-ink-300">
|
||||
<Info size={13} className="text-gold-500" />
|
||||
How saving works
|
||||
</p>
|
||||
<p>
|
||||
The whole prompt is sent to the model. Its{' '}
|
||||
<span className="text-ink-200">first line</span> doubles as the title in reports,
|
||||
so lead with the task.
|
||||
</p>
|
||||
<p>
|
||||
Saving rewrites the suite as{' '}
|
||||
<code className="rounded bg-navy-750 px-1 py-0.5 font-mono text-[0.6875rem] text-gold-300">
|
||||
test1.txt … testN.txt
|
||||
</code>{' '}
|
||||
in this order, so the CLI (
|
||||
<code className="font-mono text-[0.6875rem]">auto-test.py</code>) sees exactly
|
||||
what you see here.
|
||||
</p>
|
||||
</div>
|
||||
</Card>
|
||||
</div>
|
||||
</aside>
|
||||
<PluginSlot slot="suite.aside" className="mt-4" />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* ---------------------------------------------- selected suite */}
|
||||
<div className="space-y-3 xl:col-span-8">
|
||||
{detailError && <ErrorNote message={detailError} onRetry={() => activeSlug && loadDetail(activeSlug)} />}
|
||||
|
||||
{loadingDetail && !detail ? (
|
||||
<Card>
|
||||
<SkeletonRows rows={4} />
|
||||
</Card>
|
||||
) : !detail ? (
|
||||
<Card>
|
||||
<EmptyState
|
||||
icon={<Layers size={20} />}
|
||||
title="No suite selected"
|
||||
description="Pick one on the left, or create a new custom suite."
|
||||
/>
|
||||
</Card>
|
||||
) : (
|
||||
<>
|
||||
<Card raised>
|
||||
<div className="flex flex-wrap items-start justify-between gap-3 p-4">
|
||||
<div className="min-w-0">
|
||||
<div className="flex items-center gap-2">
|
||||
<h2 className="text-[0.9375rem] font-semibold text-ink-100">{detail.name}</h2>
|
||||
{detail.builtin ? (
|
||||
<span className="flex items-center gap-1 rounded border border-gold-500/30 bg-gold-500/10 px-1.5 py-0.5 text-[0.625rem] text-gold-400">
|
||||
<Lock size={9} /> built-in
|
||||
</span>
|
||||
) : (
|
||||
<span className="rounded border border-mint-500/30 bg-mint-500/10 px-1.5 py-0.5 text-[0.625rem] text-mint-400">
|
||||
custom
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<p className="mt-1 max-w-xl text-[0.8125rem] leading-relaxed text-ink-400">
|
||||
{detail.description || 'No description.'}
|
||||
</p>
|
||||
<p className="mt-1.5 text-[0.6875rem] text-ink-500">
|
||||
<code className="font-mono">{detail.slug}</code> · {drafts.length} question
|
||||
{drafts.length === 1 ? '' : 's'}
|
||||
{dirty && <span className="ml-1.5 text-ember-400">· unsaved</span>}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex shrink-0 flex-wrap items-center gap-2">
|
||||
<button
|
||||
className="btn btn-ghost btn-sm"
|
||||
onClick={() => setDialog({ mode: 'duplicate' })}
|
||||
disabled={busy || isLive}
|
||||
title="Clone this suite into an editable custom one"
|
||||
>
|
||||
<Copy size={13} />
|
||||
Duplicate
|
||||
</button>
|
||||
{!readOnly && (
|
||||
<>
|
||||
<button
|
||||
className="btn btn-ghost btn-sm"
|
||||
onClick={() => setDialog({ mode: 'rename' })}
|
||||
disabled={busy || isLive}
|
||||
>
|
||||
<Pencil size={13} />
|
||||
Rename
|
||||
</button>
|
||||
<button
|
||||
className="btn btn-ghost btn-sm text-rose-400 hover:bg-rose-500/10"
|
||||
onClick={() => setDeleteSuiteOpen(true)}
|
||||
disabled={busy || isLive}
|
||||
>
|
||||
<Trash2 size={13} />
|
||||
Delete
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{readOnly && (
|
||||
<div className="flex items-start gap-2.5 border-t border-navy-700 bg-gold-500/5 px-4 py-3">
|
||||
<Lock size={14} className="mt-0.5 shrink-0 text-gold-400" />
|
||||
<p className="text-[0.75rem] leading-relaxed text-gold-300">
|
||||
This suite ships with the app, so its questions cannot be changed. Use{' '}
|
||||
<strong>Duplicate</strong> to get an editable copy — the original stays intact
|
||||
as a stable baseline for comparing models.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</Card>
|
||||
|
||||
{drafts.length === 0 ? (
|
||||
<Card>
|
||||
<EmptyState
|
||||
icon={<ListChecks size={20} />}
|
||||
title="No questions yet"
|
||||
description={
|
||||
readOnly
|
||||
? 'This suite is empty.'
|
||||
: 'Add your first question to start benchmarking.'
|
||||
}
|
||||
action={
|
||||
readOnly ? undefined : (
|
||||
<button className="btn btn-primary" onClick={addQuestion}>
|
||||
<Plus size={14} />
|
||||
Add question
|
||||
</button>
|
||||
)
|
||||
}
|
||||
/>
|
||||
</Card>
|
||||
) : (
|
||||
drafts.map((draft, index) => (
|
||||
<QuestionCard
|
||||
key={draft.key}
|
||||
draft={draft}
|
||||
index={index}
|
||||
total={drafts.length}
|
||||
readOnly={readOnly}
|
||||
autoFocus={focusKey === draft.key}
|
||||
onFocused={() => setFocusKey(null)}
|
||||
onChange={(prompt) => update(draft.key, prompt)}
|
||||
onMoveUp={() => move(draft.key, -1)}
|
||||
onMoveDown={() => move(draft.key, 1)}
|
||||
onDuplicate={() => duplicateQuestion(draft.key)}
|
||||
onDelete={() =>
|
||||
draft.prompt.trim() ? setPendingDelete(draft.key) : removeQuestion(draft.key)
|
||||
}
|
||||
/>
|
||||
))
|
||||
)}
|
||||
|
||||
{!readOnly && drafts.length > 0 && (
|
||||
<button
|
||||
className="flex w-full items-center justify-center gap-2 rounded-xl border border-dashed border-navy-600 py-3.5 text-[0.8125rem] font-medium text-ink-400 transition-colors hover:border-ember-500/60 hover:bg-ember-500/5 hover:text-ember-400"
|
||||
onClick={addQuestion}
|
||||
>
|
||||
<Plus size={15} />
|
||||
Add question
|
||||
</button>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<SuiteDialog
|
||||
open={!!dialog}
|
||||
mode={dialog?.mode ?? 'create'}
|
||||
busy={busy}
|
||||
initialName={
|
||||
dialog?.mode === 'rename'
|
||||
? (detail?.name ?? '')
|
||||
: dialog?.mode === 'duplicate'
|
||||
? `${detail?.name ?? 'Suite'} (copy)`
|
||||
: ''
|
||||
}
|
||||
initialDescription={dialog?.mode === 'rename' ? (detail?.description ?? '') : ''}
|
||||
onSubmit={submitDialog}
|
||||
onCancel={() => setDialog(null)}
|
||||
/>
|
||||
|
||||
<ConfirmDialog
|
||||
open={deleteSuiteOpen}
|
||||
title={`Delete "${detail?.name}"?`}
|
||||
body={`This removes the suite and all ${drafts.length} of its questions from disk. This cannot be undone.`}
|
||||
confirmLabel="Delete suite"
|
||||
destructive
|
||||
onConfirm={deleteSuite}
|
||||
onCancel={() => setDeleteSuiteOpen(false)}
|
||||
/>
|
||||
|
||||
<ConfirmDialog
|
||||
open={confirmDiscard}
|
||||
@@ -281,19 +493,32 @@ export function SuitePage() {
|
||||
destructive
|
||||
onConfirm={() => {
|
||||
setConfirmDiscard(false)
|
||||
load()
|
||||
if (activeSlug) loadDetail(activeSlug)
|
||||
}}
|
||||
onCancel={() => setConfirmDiscard(false)}
|
||||
/>
|
||||
|
||||
<ConfirmDialog
|
||||
open={!!deleting}
|
||||
open={!!pendingSwitch}
|
||||
title="Leave without saving?"
|
||||
body="You have unsaved edits in this suite. Switching will discard them."
|
||||
confirmLabel="Discard and switch"
|
||||
destructive
|
||||
onConfirm={() => {
|
||||
setActiveSlug(pendingSwitch)
|
||||
setPendingSwitch(null)
|
||||
}}
|
||||
onCancel={() => setPendingSwitch(null)}
|
||||
/>
|
||||
|
||||
<ConfirmDialog
|
||||
open={!!deletingQuestion}
|
||||
title="Delete this question?"
|
||||
body={deriveTitle(deleting?.prompt ?? '', 'This question')}
|
||||
body={deriveTitle(deletingQuestion?.prompt ?? '', 'This question')}
|
||||
confirmLabel="Delete"
|
||||
destructive
|
||||
onConfirm={() => {
|
||||
if (pendingDelete) remove(pendingDelete)
|
||||
if (pendingDelete) removeQuestion(pendingDelete)
|
||||
setPendingDelete(null)
|
||||
}}
|
||||
onCancel={() => setPendingDelete(null)}
|
||||
@@ -302,12 +527,178 @@ export function SuitePage() {
|
||||
)
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------- card */
|
||||
/* -------------------------------------------------------------- suite list */
|
||||
|
||||
function SuiteGroup({
|
||||
label,
|
||||
hint,
|
||||
children,
|
||||
}: {
|
||||
label: string
|
||||
hint?: string
|
||||
children: React.ReactNode
|
||||
}) {
|
||||
return (
|
||||
<div className="mb-1 last:mb-0">
|
||||
<div className="flex items-baseline gap-2 px-2.5 pt-2 pb-1">
|
||||
<span className="text-[0.625rem] font-semibold tracking-[0.09em] text-ink-500 uppercase">
|
||||
{label}
|
||||
</span>
|
||||
{hint && <span className="text-[0.625rem] text-ink-600">{hint}</span>}
|
||||
</div>
|
||||
<div className="space-y-0.5">{children}</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function SuiteRow({
|
||||
suite,
|
||||
active,
|
||||
onClick,
|
||||
}: {
|
||||
suite: SuiteSummary
|
||||
active: boolean
|
||||
onClick: () => void
|
||||
}) {
|
||||
return (
|
||||
<button
|
||||
onClick={onClick}
|
||||
className={cx(
|
||||
'flex w-full items-center gap-2.5 rounded-lg px-2.5 py-2 text-left transition-colors',
|
||||
active ? 'bg-navy-750 text-ink-100' : 'text-ink-300 hover:bg-navy-800/60',
|
||||
)}
|
||||
>
|
||||
{suite.builtin ? (
|
||||
<Lock size={12} className={cx('shrink-0', active ? 'text-gold-400' : 'text-ink-500')} />
|
||||
) : (
|
||||
<ListChecks size={12} className={cx('shrink-0', active ? 'text-mint-400' : 'text-ink-500')} />
|
||||
)}
|
||||
<span className="min-w-0 flex-1">
|
||||
<span className="block truncate text-[0.8125rem] font-medium">{suite.name}</span>
|
||||
<span className="block truncate text-[0.6875rem] text-ink-500">{suite.slug}</span>
|
||||
</span>
|
||||
<span className="num shrink-0 text-[0.6875rem] text-ink-500">{suite.count}</span>
|
||||
</button>
|
||||
)
|
||||
}
|
||||
|
||||
/* ----------------------------------------------------------- name dialog */
|
||||
|
||||
function SuiteDialog({
|
||||
open,
|
||||
mode,
|
||||
busy,
|
||||
initialName,
|
||||
initialDescription,
|
||||
onSubmit,
|
||||
onCancel,
|
||||
}: {
|
||||
open: boolean
|
||||
mode: 'create' | 'rename' | 'duplicate'
|
||||
busy: boolean
|
||||
initialName: string
|
||||
initialDescription: string
|
||||
onSubmit: (name: string, description: string) => void
|
||||
onCancel: () => void
|
||||
}) {
|
||||
const [name, setName] = useState(initialName)
|
||||
const [description, setDescription] = useState(initialDescription)
|
||||
|
||||
useEffect(() => {
|
||||
if (open) {
|
||||
setName(initialName)
|
||||
setDescription(initialDescription)
|
||||
}
|
||||
}, [open, initialName, initialDescription])
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) return
|
||||
const onKey = (event: KeyboardEvent) => {
|
||||
if (event.key === 'Escape') onCancel()
|
||||
}
|
||||
window.addEventListener('keydown', onKey)
|
||||
return () => window.removeEventListener('keydown', onKey)
|
||||
}, [open, onCancel])
|
||||
|
||||
if (!open) return null
|
||||
|
||||
const copy = {
|
||||
create: { title: 'New suite', action: 'Create suite' },
|
||||
rename: { title: 'Rename suite', action: 'Save' },
|
||||
duplicate: { title: 'Duplicate to a custom suite', action: 'Duplicate' },
|
||||
}[mode]
|
||||
|
||||
return (
|
||||
<div
|
||||
className="animate-fade-in fixed inset-0 z-50 flex items-center justify-center bg-navy-950/75 p-5 backdrop-blur-sm"
|
||||
onClick={onCancel}
|
||||
>
|
||||
<form
|
||||
className="surface-raised animate-fade-up w-full max-w-md p-5"
|
||||
onClick={(event) => event.stopPropagation()}
|
||||
onSubmit={(event) => {
|
||||
event.preventDefault()
|
||||
onSubmit(name, description)
|
||||
}}
|
||||
>
|
||||
<h3 className="text-[0.9375rem] font-semibold text-ink-100">{copy.title}</h3>
|
||||
|
||||
<label className="label mt-4 block" htmlFor="suite-name">
|
||||
Name
|
||||
</label>
|
||||
<input
|
||||
id="suite-name"
|
||||
className="field"
|
||||
value={name}
|
||||
autoFocus
|
||||
maxLength={80}
|
||||
placeholder="e.g. Domain knowledge"
|
||||
onChange={(event) => setName(event.target.value)}
|
||||
/>
|
||||
|
||||
{mode !== 'duplicate' && (
|
||||
<>
|
||||
<label className="label mt-3 block" htmlFor="suite-description">
|
||||
Description <span className="text-ink-600">(optional)</span>
|
||||
</label>
|
||||
<input
|
||||
id="suite-description"
|
||||
className="field"
|
||||
value={description}
|
||||
placeholder="What this suite covers"
|
||||
onChange={(event) => setDescription(event.target.value)}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
|
||||
{mode === 'duplicate' && (
|
||||
<p className="mt-3 text-[0.75rem] leading-relaxed text-ink-500">
|
||||
Copies every question into a new custom suite you can edit. The original is
|
||||
untouched.
|
||||
</p>
|
||||
)}
|
||||
|
||||
<div className="mt-5 flex justify-end gap-2">
|
||||
<button type="button" className="btn btn-ghost" onClick={onCancel}>
|
||||
Cancel
|
||||
</button>
|
||||
<button type="submit" className="btn btn-primary" disabled={!name.trim() || busy}>
|
||||
{busy && <Spinner className="h-3.5 w-3.5" />}
|
||||
{copy.action}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------- question */
|
||||
|
||||
function QuestionCard({
|
||||
draft,
|
||||
index,
|
||||
total,
|
||||
readOnly,
|
||||
autoFocus,
|
||||
onFocused,
|
||||
onChange,
|
||||
@@ -319,6 +710,7 @@ function QuestionCard({
|
||||
draft: Draft
|
||||
index: number
|
||||
total: number
|
||||
readOnly: boolean
|
||||
autoFocus: boolean
|
||||
onFocused: () => void
|
||||
onChange: (prompt: string) => void
|
||||
@@ -332,7 +724,6 @@ function QuestionCard({
|
||||
const empty = !draft.prompt.trim()
|
||||
const words = draft.prompt.trim() ? draft.prompt.trim().split(/\s+/).length : 0
|
||||
|
||||
// Grow the box to fit its content so long prompts stay readable.
|
||||
useLayoutEffect(() => {
|
||||
const node = textareaRef.current
|
||||
if (!node) return
|
||||
@@ -348,16 +739,12 @@ function QuestionCard({
|
||||
}, [autoFocus, onFocused])
|
||||
|
||||
return (
|
||||
<Card
|
||||
className={cx('animate-fade-up overflow-hidden', empty && 'border-ember-500/40')}
|
||||
>
|
||||
<Card className={cx('animate-fade-up overflow-hidden', empty && !readOnly && 'border-ember-500/40')}>
|
||||
<div className="flex items-center gap-3 border-b border-navy-700 px-3.5 py-2.5">
|
||||
<span
|
||||
className={cx(
|
||||
'num flex h-6 w-6 shrink-0 items-center justify-center rounded-lg text-[0.6875rem] font-semibold',
|
||||
empty
|
||||
? 'bg-ember-500/15 text-ember-400'
|
||||
: 'bg-gold-500/12 text-gold-400',
|
||||
empty && !readOnly ? 'bg-ember-500/15 text-ember-400' : 'bg-gold-500/12 text-gold-400',
|
||||
)}
|
||||
>
|
||||
{index + 1}
|
||||
@@ -378,31 +765,37 @@ function QuestionCard({
|
||||
<span className="num">{words}w</span>
|
||||
</span>
|
||||
|
||||
<div className="flex shrink-0 items-center gap-0.5">
|
||||
<IconButton label="Move up" disabled={index === 0} onClick={onMoveUp}>
|
||||
<ArrowUp size={13} />
|
||||
</IconButton>
|
||||
<IconButton label="Move down" disabled={index === total - 1} onClick={onMoveDown}>
|
||||
<ArrowDown size={13} />
|
||||
</IconButton>
|
||||
<IconButton label="Duplicate" onClick={onDuplicate}>
|
||||
<Copy size={13} />
|
||||
</IconButton>
|
||||
<IconButton label="Delete" danger onClick={onDelete}>
|
||||
<Trash2 size={13} />
|
||||
</IconButton>
|
||||
</div>
|
||||
{readOnly ? (
|
||||
<Lock size={12} className="shrink-0 text-ink-600" aria-label="Read-only" />
|
||||
) : (
|
||||
<div className="flex shrink-0 items-center gap-0.5">
|
||||
<IconButton label="Move up" disabled={index === 0} onClick={onMoveUp}>
|
||||
<ArrowUp size={13} />
|
||||
</IconButton>
|
||||
<IconButton label="Move down" disabled={index === total - 1} onClick={onMoveDown}>
|
||||
<ArrowDown size={13} />
|
||||
</IconButton>
|
||||
<IconButton label="Duplicate" onClick={onDuplicate}>
|
||||
<Copy size={13} />
|
||||
</IconButton>
|
||||
<IconButton label="Delete" danger onClick={onDelete}>
|
||||
<Trash2 size={13} />
|
||||
</IconButton>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<textarea
|
||||
ref={textareaRef}
|
||||
value={draft.prompt}
|
||||
readOnly={readOnly}
|
||||
onChange={(event) => onChange(event.target.value)}
|
||||
placeholder={
|
||||
'Write the question exactly as the model should receive it.\n\nExample: Write a Swift function that solves the FizzBuzz problem.'
|
||||
}
|
||||
placeholder={PLACEHOLDER}
|
||||
spellCheck={false}
|
||||
className="w-full resize-none border-0 bg-transparent px-4 py-3.5 font-mono text-[0.8125rem] leading-relaxed text-ink-200 placeholder:text-ink-500/70 focus:outline-none"
|
||||
className={cx(
|
||||
'w-full resize-none border-0 bg-transparent px-4 py-3.5 font-mono text-[0.8125rem] leading-relaxed placeholder:text-ink-500/70 focus:outline-none',
|
||||
readOnly ? 'cursor-default text-ink-300' : 'text-ink-200',
|
||||
)}
|
||||
/>
|
||||
</Card>
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user