ui: fix avg token/sec calculation on models page (#357)
* ui: use percentiles for token stats * ui: add histogram of metrics * update vite to remove security warnings fixes #355
This commit is contained in:
Benson Wong
GitHub
6
ui/package-lock.json
generated
6
ui/package-lock.json
generated
@@ -3975,9 +3975,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/vite": {
|
"node_modules/vite": {
|
||||||
"version": "6.3.5",
|
"version": "6.4.1",
|
||||||
"resolved": "https://registry.npmjs.org/vite/-/vite-6.3.5.tgz",
|
"resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz",
|
||||||
"integrity": "sha512-cZn6NDFE7wdTpINgs++ZJ4N49W2vRp8LCKrn3Ob1kYNtOo21vfDoaV5GzBfLU4MovSAB8uNRm4jgzVQZ+mBzPQ==",
|
"integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
|||||||
@@ -191,42 +191,300 @@ function ModelsPanel() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface HistogramData {
|
||||||
|
bins: number[];
|
||||||
|
min: number;
|
||||||
|
max: number;
|
||||||
|
binSize: number;
|
||||||
|
p99: number;
|
||||||
|
p95: number;
|
||||||
|
p50: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function TokenHistogram({ data }: { data: HistogramData }) {
|
||||||
|
const { bins, min, max, p50, p95, p99 } = data;
|
||||||
|
const maxCount = Math.max(...bins);
|
||||||
|
|
||||||
|
const height = 120;
|
||||||
|
const padding = { top: 10, right: 15, bottom: 25, left: 45 };
|
||||||
|
|
||||||
|
// Use viewBox for responsive sizing
|
||||||
|
const viewBoxWidth = 600;
|
||||||
|
const chartWidth = viewBoxWidth - padding.left - padding.right;
|
||||||
|
const chartHeight = height - padding.top - padding.bottom;
|
||||||
|
|
||||||
|
const barWidth = chartWidth / bins.length;
|
||||||
|
const range = max - min;
|
||||||
|
|
||||||
|
// Calculate x position for a given value
|
||||||
|
const getXPosition = (value: number) => {
|
||||||
|
return padding.left + ((value - min) / range) * chartWidth;
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="mt-2 w-full">
|
||||||
|
<svg
|
||||||
|
viewBox={`0 0 ${viewBoxWidth} ${height}`}
|
||||||
|
className="w-full h-auto"
|
||||||
|
preserveAspectRatio="xMidYMid meet"
|
||||||
|
>
|
||||||
|
{/* Y-axis */}
|
||||||
|
<line
|
||||||
|
x1={padding.left}
|
||||||
|
y1={padding.top}
|
||||||
|
x2={padding.left}
|
||||||
|
y2={height - padding.bottom}
|
||||||
|
stroke="currentColor"
|
||||||
|
strokeWidth="1"
|
||||||
|
opacity="0.3"
|
||||||
|
/>
|
||||||
|
|
||||||
|
{/* X-axis */}
|
||||||
|
<line
|
||||||
|
x1={padding.left}
|
||||||
|
y1={height - padding.bottom}
|
||||||
|
x2={viewBoxWidth - padding.right}
|
||||||
|
y2={height - padding.bottom}
|
||||||
|
stroke="currentColor"
|
||||||
|
strokeWidth="1"
|
||||||
|
opacity="0.3"
|
||||||
|
/>
|
||||||
|
|
||||||
|
{/* Histogram bars */}
|
||||||
|
{bins.map((count, i) => {
|
||||||
|
const barHeight = maxCount > 0 ? (count / maxCount) * chartHeight : 0;
|
||||||
|
const x = padding.left + i * barWidth;
|
||||||
|
const y = height - padding.bottom - barHeight;
|
||||||
|
const binStart = min + i * data.binSize;
|
||||||
|
const binEnd = binStart + data.binSize;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<g key={i}>
|
||||||
|
<rect
|
||||||
|
x={x}
|
||||||
|
y={y}
|
||||||
|
width={Math.max(barWidth - 1, 1)}
|
||||||
|
height={barHeight}
|
||||||
|
fill="currentColor"
|
||||||
|
opacity="0.6"
|
||||||
|
className="text-blue-500 dark:text-blue-400 hover:opacity-90 transition-opacity cursor-pointer"
|
||||||
|
/>
|
||||||
|
<title>{`${binStart.toFixed(1)} - ${binEnd.toFixed(1)} tokens/sec\nCount: ${count}`}</title>
|
||||||
|
</g>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
|
||||||
|
{/* Percentile lines */}
|
||||||
|
<line
|
||||||
|
x1={getXPosition(p50)}
|
||||||
|
y1={padding.top}
|
||||||
|
x2={getXPosition(p50)}
|
||||||
|
y2={height - padding.bottom}
|
||||||
|
stroke="currentColor"
|
||||||
|
strokeWidth="2"
|
||||||
|
strokeDasharray="4 2"
|
||||||
|
opacity="0.7"
|
||||||
|
className="text-gray-600 dark:text-gray-400"
|
||||||
|
/>
|
||||||
|
|
||||||
|
<line
|
||||||
|
x1={getXPosition(p95)}
|
||||||
|
y1={padding.top}
|
||||||
|
x2={getXPosition(p95)}
|
||||||
|
y2={height - padding.bottom}
|
||||||
|
stroke="currentColor"
|
||||||
|
strokeWidth="2"
|
||||||
|
strokeDasharray="4 2"
|
||||||
|
opacity="0.7"
|
||||||
|
className="text-orange-500 dark:text-orange-400"
|
||||||
|
/>
|
||||||
|
|
||||||
|
<line
|
||||||
|
x1={getXPosition(p99)}
|
||||||
|
y1={padding.top}
|
||||||
|
x2={getXPosition(p99)}
|
||||||
|
y2={height - padding.bottom}
|
||||||
|
stroke="currentColor"
|
||||||
|
strokeWidth="2"
|
||||||
|
strokeDasharray="4 2"
|
||||||
|
opacity="0.7"
|
||||||
|
className="text-green-500 dark:text-green-400"
|
||||||
|
/>
|
||||||
|
|
||||||
|
{/* X-axis labels */}
|
||||||
|
<text
|
||||||
|
x={padding.left}
|
||||||
|
y={height - 5}
|
||||||
|
fontSize="10"
|
||||||
|
fill="currentColor"
|
||||||
|
opacity="0.6"
|
||||||
|
textAnchor="start"
|
||||||
|
>
|
||||||
|
{min.toFixed(1)}
|
||||||
|
</text>
|
||||||
|
|
||||||
|
<text
|
||||||
|
x={viewBoxWidth - padding.right}
|
||||||
|
y={height - 5}
|
||||||
|
fontSize="10"
|
||||||
|
fill="currentColor"
|
||||||
|
opacity="0.6"
|
||||||
|
textAnchor="end"
|
||||||
|
>
|
||||||
|
{max.toFixed(1)}
|
||||||
|
</text>
|
||||||
|
|
||||||
|
{/* X-axis label */}
|
||||||
|
<text
|
||||||
|
x={padding.left + chartWidth / 2}
|
||||||
|
y={height - 2}
|
||||||
|
fontSize="10"
|
||||||
|
fill="currentColor"
|
||||||
|
opacity="0.6"
|
||||||
|
textAnchor="middle"
|
||||||
|
>
|
||||||
|
Tokens/Second Distribution
|
||||||
|
</text>
|
||||||
|
</svg>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function StatsPanel() {
|
function StatsPanel() {
|
||||||
const { metrics } = useAPI();
|
const { metrics } = useAPI();
|
||||||
|
|
||||||
const [totalRequests, totalInputTokens, totalOutputTokens, avgTokensPerSecond] = useMemo(() => {
|
const [totalRequests, totalInputTokens, totalOutputTokens, tokenStats, histogramData] = useMemo(() => {
|
||||||
const totalRequests = metrics.length;
|
const totalRequests = metrics.length;
|
||||||
if (totalRequests === 0) {
|
if (totalRequests === 0) {
|
||||||
return [0, 0, 0];
|
return [0, 0, 0, { p99: 0, p95: 0, p50: 0 }, null];
|
||||||
}
|
}
|
||||||
const totalInputTokens = metrics.reduce((sum, m) => sum + m.input_tokens, 0);
|
const totalInputTokens = metrics.reduce((sum, m) => sum + m.input_tokens, 0);
|
||||||
const totalOutputTokens = metrics.reduce((sum, m) => sum + m.output_tokens, 0);
|
const totalOutputTokens = metrics.reduce((sum, m) => sum + m.output_tokens, 0);
|
||||||
const avgTokensPerSecond = (metrics.reduce((sum, m) => sum + m.tokens_per_second, 0) / totalRequests).toFixed(2);
|
|
||||||
return [totalRequests, totalInputTokens, totalOutputTokens, avgTokensPerSecond];
|
// Calculate token statistics using output_tokens and duration_ms
|
||||||
|
// Filter out metrics with invalid duration or output tokens
|
||||||
|
const validMetrics = metrics.filter((m) => m.duration_ms > 0 && m.output_tokens > 0);
|
||||||
|
if (validMetrics.length === 0) {
|
||||||
|
return [totalRequests, totalInputTokens, totalOutputTokens, { p99: 0, p95: 0, p50: 0 }, null];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate tokens/second for each valid metric
|
||||||
|
const tokensPerSecond = validMetrics.map((m) => m.output_tokens / (m.duration_ms / 1000));
|
||||||
|
|
||||||
|
// Sort for percentile calculation
|
||||||
|
const sortedTokensPerSecond = [...tokensPerSecond].sort((a, b) => a - b);
|
||||||
|
|
||||||
|
// Calculate percentiles - showing speed thresholds where X% of requests are SLOWER (below)
|
||||||
|
// P99: 99% of requests are slower than this speed (99th percentile - fast requests)
|
||||||
|
// P95: 95% of requests are slower than this speed (95th percentile)
|
||||||
|
// P50: 50% of requests are slower than this speed (median)
|
||||||
|
const p99 = sortedTokensPerSecond[Math.floor(sortedTokensPerSecond.length * 0.99)];
|
||||||
|
const p95 = sortedTokensPerSecond[Math.floor(sortedTokensPerSecond.length * 0.95)];
|
||||||
|
const p50 = sortedTokensPerSecond[Math.floor(sortedTokensPerSecond.length * 0.5)];
|
||||||
|
|
||||||
|
// Create histogram data
|
||||||
|
const min = Math.min(...tokensPerSecond);
|
||||||
|
const max = Math.max(...tokensPerSecond);
|
||||||
|
const binCount = Math.min(30, Math.max(10, Math.floor(tokensPerSecond.length / 5))); // Adaptive bin count
|
||||||
|
const binSize = (max - min) / binCount;
|
||||||
|
|
||||||
|
const bins = Array(binCount).fill(0);
|
||||||
|
tokensPerSecond.forEach((value) => {
|
||||||
|
const binIndex = Math.min(Math.floor((value - min) / binSize), binCount - 1);
|
||||||
|
bins[binIndex]++;
|
||||||
|
});
|
||||||
|
|
||||||
|
const histogramData = {
|
||||||
|
bins,
|
||||||
|
min,
|
||||||
|
max,
|
||||||
|
binSize,
|
||||||
|
p99,
|
||||||
|
p95,
|
||||||
|
p50,
|
||||||
|
};
|
||||||
|
|
||||||
|
return [
|
||||||
|
totalRequests,
|
||||||
|
totalInputTokens,
|
||||||
|
totalOutputTokens,
|
||||||
|
{
|
||||||
|
p99: p99.toFixed(2),
|
||||||
|
p95: p95.toFixed(2),
|
||||||
|
p50: p50.toFixed(2),
|
||||||
|
},
|
||||||
|
histogramData,
|
||||||
|
];
|
||||||
}, [metrics]);
|
}, [metrics]);
|
||||||
|
|
||||||
|
const nf = new Intl.NumberFormat();
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="card">
|
<div className="card">
|
||||||
<div className="rounded-lg overflow-hidden border border-gray-200 dark:border-white/10">
|
<div className="rounded-lg overflow-hidden border border-card-border-inner">
|
||||||
<table className="w-full">
|
<table className="min-w-full divide-y divide-card-border-inner">
|
||||||
<thead>
|
<thead className="bg-secondary">
|
||||||
<tr className="border-b border-gray-200 dark:border-white/10 text-right">
|
<tr>
|
||||||
<th>Requests</th>
|
<th className="px-4 py-3 text-left text-xs font-semibold uppercase tracking-wider text-txtmain">
|
||||||
<th className="border-l border-gray-200 dark:border-white/10">Processed</th>
|
Requests
|
||||||
<th className="border-l border-gray-200 dark:border-white/10">Generated</th>
|
</th>
|
||||||
<th className="border-l border-gray-200 dark:border-white/10">Tokens/Sec</th>
|
<th className="px-4 py-3 text-left text-xs font-semibold uppercase tracking-wider text-txtmain border-l border-card-border-inner">
|
||||||
|
Processed
|
||||||
|
</th>
|
||||||
|
<th className="px-4 py-3 text-left text-xs font-semibold uppercase tracking-wider text-txtmain border-l border-card-border-inner">
|
||||||
|
Generated
|
||||||
|
</th>
|
||||||
|
<th className="px-4 py-3 text-left text-xs font-semibold uppercase tracking-wider text-txtmain border-l border-card-border-inner">
|
||||||
|
Token Stats (tokens/sec)
|
||||||
|
</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
|
||||||
<tr className="text-right">
|
<tbody className="bg-surface divide-y divide-card-border-inner">
|
||||||
<td className="border-r border-gray-200 dark:border-white/10">{totalRequests}</td>
|
<tr className="hover:bg-secondary">
|
||||||
<td className="border-r border-gray-200 dark:border-white/10">
|
<td className="px-4 py-4 text-sm font-semibold text-gray-900 dark:text-white">{totalRequests}</td>
|
||||||
{new Intl.NumberFormat().format(totalInputTokens)}
|
|
||||||
|
<td className="px-4 py-4 text-sm text-gray-700 dark:text-gray-300 border-l border-gray-200 dark:border-white/10">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<span className="text-sm font-medium">{nf.format(totalInputTokens)}</span>
|
||||||
|
<span className="text-xs text-gray-500 dark:text-gray-400">tokens</span>
|
||||||
|
</div>
|
||||||
</td>
|
</td>
|
||||||
<td className="border-r border-gray-200 dark:border-white/10">
|
|
||||||
{new Intl.NumberFormat().format(totalOutputTokens)}
|
<td className="px-4 py-4 text-sm text-gray-700 dark:text-gray-300 border-l border-gray-200 dark:border-white/10">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<span className="text-sm font-medium">{nf.format(totalOutputTokens)}</span>
|
||||||
|
<span className="text-xs text-gray-500 dark:text-gray-400">tokens</span>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
|
||||||
|
<td className="px-4 py-4 border-l border-gray-200 dark:border-white/10">
|
||||||
|
<div className="space-y-3">
|
||||||
|
<div className="grid grid-cols-3 gap-2 items-center">
|
||||||
|
<div className="text-center">
|
||||||
|
<div className="text-xs text-gray-500 dark:text-gray-400">P50</div>
|
||||||
|
<div className="mt-1 inline-block rounded-full bg-gray-100 dark:bg-white/5 px-3 py-1 text-sm font-semibold text-gray-800 dark:text-white">
|
||||||
|
{tokenStats.p50}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="text-center">
|
||||||
|
<div className="text-xs text-gray-500 dark:text-gray-400">P95</div>
|
||||||
|
<div className="mt-1 inline-block rounded-full bg-gray-100 dark:bg-white/5 px-3 py-1 text-sm font-semibold text-gray-800 dark:text-white">
|
||||||
|
{tokenStats.p95}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="text-center">
|
||||||
|
<div className="text-xs text-gray-500 dark:text-gray-400">P99</div>
|
||||||
|
<div className="mt-1 inline-block rounded-full bg-gray-100 dark:bg-white/5 px-3 py-1 text-sm font-semibold text-gray-800 dark:text-white">
|
||||||
|
{tokenStats.p99}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{histogramData && <TokenHistogram data={histogramData} />}
|
||||||
|
</div>
|
||||||
</td>
|
</td>
|
||||||
<td>{avgTokensPerSecond}</td>
|
|
||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
|||||||
Reference in New Issue
Block a user