Skip to content

Commit 25b4dd5

Browse files
GusarichGusarich
authored
feat: make benchmarks per-device (#14)
1 parent f450441 commit 25b4dd5

File tree

4 files changed

+446
-132
lines changed

4 files changed

+446
-132
lines changed

‎tests/Benchmark.spec.ts‎

Lines changed: 207 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -19,27 +19,141 @@ type BenchEntry = {
1919
cases: BenchResult[];
2020
};
2121

22+
type ResultsMap = Record<string, BenchEntry[]>;
23+
2224
const OWNER = 'EQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAM9c';
25+
const DEFAULT_DEVICE = 'Unknown device';
2326
const RESULT_FILE = path.resolve(__dirname, 'results.json');
2427

25-
// CLI flags: --add "Title" or --replace "Title"
28+
// CLI flags: --add "Title", --replace "Title"
2629
let addTitle: string | null = null;
2730
let replaceTitle: string | null = null;
31+
let benchDevices: string | null = null; // forwarded to generator --devices
2832
(() => {
2933
const argv = process.argv;
3034
for (let i = 0; i < argv.length; i++) {
31-
if (argv[i] === '--add' && i + 1 < argv.length) addTitle = argv[i + 1];
32-
if (argv[i] === '--replace' && i + 1 < argv.length) replaceTitle = argv[i + 1];
35+
if (argv[i] === '--add') {
36+
const val = argv[i + 1];
37+
addTitle = val && !val.startsWith('-') ? val : '(current)';
38+
if (val && !val.startsWith('-')) i++;
39+
}
40+
if (argv[i] === '--replace') {
41+
const val = argv[i + 1];
42+
replaceTitle = val && !val.startsWith('-') ? val : '(current)';
43+
if (val && !val.startsWith('-')) i++;
44+
}
45+
if (argv[i] === '--devices' && i + 1 < argv.length) benchDevices = argv[i + 1];
3346
}
3447
})();
35-
3648
type BenchCase = {
3749
name: string;
3850
start?: string;
3951
end?: string;
4052
caseSensitive: boolean;
4153
};
4254

55+
const parseDeviceIds = (raw: string | null): number[] | null => {
56+
if (!raw) return null;
57+
const ids = raw
58+
.split(',')
59+
.map((s) => s.trim())
60+
.filter(Boolean)
61+
.map((p) => Number.parseInt(p, 10))
62+
.filter((n) => !Number.isNaN(n) && n >= 0);
63+
if (!ids.length) return null;
64+
return ids;
65+
};
66+
67+
const benchDeviceIds = parseDeviceIds(benchDevices);
68+
69+
type PythonCmd = { exe: string; args: string[] };
70+
71+
const resolvePython = (): PythonCmd => {
72+
const candidates =
73+
process.platform === 'win32'
74+
? ['py', 'py -3', 'python3', 'python']
75+
: ['python3', 'python'];
76+
for (const cmd of candidates) {
77+
const [exe, ...args] = cmd.split(' ');
78+
try {
79+
const res = spawnSync(exe, [...args, '-c', 'print("ok")'], { encoding: 'utf8' });
80+
if (!res.error && res.status === 0 && res.stdout.trim() === 'ok') {
81+
return { exe, args };
82+
}
83+
} catch {
84+
/* ignore */
85+
}
86+
}
87+
throw new Error('No usable python interpreter found (tried py, py -3, python3, python)');
88+
};
89+
90+
const PYTHON = resolvePython();
91+
92+
const detectDevices = (): string[] => {
93+
const script = `
94+
import pyopencl as cl
95+
names = []
96+
for p in cl.get_platforms():
97+
for d in p.get_devices():
98+
names.append(d.name)
99+
for n in names:
100+
print(n)
101+
`;
102+
try {
103+
const res = spawnSync(PYTHON.exe, [...PYTHON.args, '-c', script], { encoding: 'utf8' });
104+
if (res.status !== 0) return [];
105+
return res.stdout
106+
.split(/\\r?\\n/)
107+
.map((l) => l.trim())
108+
.filter((l) => l);
109+
} catch {
110+
return [];
111+
}
112+
};
113+
114+
const chooseBenchCases = (names: string[]): BenchCase[] => {
115+
const defaults: BenchCase[] = [
116+
{ name: 'start 5 cs', start: 'WERTY', caseSensitive: true },
117+
{ name: 'start 5 ci', start: 'WeRtY', caseSensitive: false },
118+
{ name: 'end 4 cs', end: 'WERT', caseSensitive: true },
119+
{ name: 'end 4 ci', end: 'WeRt', caseSensitive: false },
120+
];
121+
const lower = names.join(' ').toLowerCase();
122+
const isRTX3Plus = /rtx\s*(3|4|5)\d{2,3}/i.test(lower);
123+
if (!isRTX3Plus) return defaults;
124+
return [
125+
{ name: 'start 6 cs', start: 'WERTYU', caseSensitive: true },
126+
{ name: 'start 6 ci', start: 'WeRtYu', caseSensitive: false },
127+
{ name: 'end 5 cs', end: 'WERTY', caseSensitive: true },
128+
{ name: 'end 5 ci', end: 'WeRtY', caseSensitive: false },
129+
];
130+
};
131+
132+
const normalizeDeviceName = (name: string) =>
133+
name
134+
.replace(/[\u0000-\u001f\u007f]/g, '')
135+
.replace(/\s+/g, ' ')
136+
.trim();
137+
138+
const detectedDevicesAll = detectDevices().map(normalizeDeviceName).filter(Boolean);
139+
const selectedDevices = benchDeviceIds
140+
? benchDeviceIds
141+
.map((i) => detectedDevicesAll[i])
142+
.filter((n): n is string => typeof n === 'string' && n.length > 0)
143+
: detectedDevicesAll;
144+
145+
const benchCases: BenchCase[] = (() => {
146+
const selected = chooseBenchCases(selectedDevices.length ? selectedDevices : [DEFAULT_DEVICE]);
147+
if (selected.length) return selected;
148+
return [
149+
{ name: 'start 5 cs', start: 'WERTY', caseSensitive: true },
150+
{ name: 'start 5 ci', start: 'WeRtY', caseSensitive: false },
151+
{ name: 'end 4 cs', end: 'WERT', caseSensitive: true },
152+
{ name: 'end 4 ci', end: 'WeRt', caseSensitive: false },
153+
];
154+
})();
155+
const deviceNames = new Set<string>();
156+
43157
function gpuAvailable(): boolean {
44158
const probe = `
45159
try:
@@ -54,8 +168,10 @@ try:
54168
except Exception:
55169
print("0")
56170
`;
57-
const res = spawnSync('python3', ['-c', probe], { cwd: 'src', encoding: 'utf8' });
58-
return res.status === 0 && res.stdout.trim() === '1';
171+
const res = spawnSync(PYTHON.exe, [...PYTHON.args, '-c', probe], { cwd: 'src', encoding: 'utf8' });
172+
if (res.status === 0 && res.stdout.trim() === '1') return true;
173+
console.warn('Skipping benchmarks: no GPU detected via pyopencl (python interpreter or OpenCL missing)');
174+
return false;
59175
}
60176

61177
async function runBenchCase(testCase: BenchCase, timeoutMs: number): Promise<BenchResult> {
@@ -66,13 +182,33 @@ async function runBenchCase(testCase: BenchCase, timeoutMs: number): Promise<Ben
66182
copyFileSync(path.resolve('src/kernel.cl'), kernelPath);
67183

68184
const args = [genPath, '--owner', OWNER];
185+
if (benchDevices) {
186+
args.push('--devices', benchDevices);
187+
}
69188
if (testCase.start) args.push('--start', testCase.start);
70189
if (testCase.end) args.push('--end', testCase.end);
71190
if (testCase.caseSensitive) args.push('--case-sensitive');
72191
// intentionally NOT passing --only-one; we will time-limit instead
73192

74193
let timedOut = false;
75-
const child = spawn('python3', args, { cwd: tmp, stdio: ['ignore', 'pipe', 'pipe'] });
194+
const child = spawn(PYTHON.exe, [...PYTHON.args, ...args], {
195+
cwd: tmp,
196+
stdio: ['ignore', 'pipe', 'pipe'],
197+
});
198+
199+
const parseDevices = (chunk: Buffer | string) => {
200+
const text = chunk.toString();
201+
for (const line of text.split(/\r?\n/)) {
202+
const m = line.match(/Using device:\s*(.+)/i);
203+
if (m && m[1]) {
204+
const cleaned = normalizeDeviceName(m[1].replace(/^\[\d+\]\s*/, ''));
205+
deviceNames.add(cleaned || DEFAULT_DEVICE);
206+
}
207+
}
208+
};
209+
210+
child.stdout?.on('data', parseDevices);
211+
child.stderr?.on('data', parseDevices);
76212

77213
const killer = setTimeout(() => {
78214
timedOut = true;
@@ -106,25 +242,28 @@ async function runBenchCase(testCase: BenchCase, timeoutMs: number): Promise<Ben
106242
return { name: testCase.name, hits, seconds, rate, timedOut };
107243
}
108244

109-
const benchCases: BenchCase[] = [
110-
{ name: 'start 5 cs', start: 'WERTY', caseSensitive: true },
111-
{ name: 'start 5 ci', start: 'WeRtY', caseSensitive: false },
112-
{ name: 'end 4 cs', end: 'WERT', caseSensitive: true },
113-
{ name: 'end 4 ci', end: 'WeRt', caseSensitive: false },
114-
];
115-
116245
const gpuOk = gpuAvailable();
117246

118247
(gpuOk ? describe : describe.skip)('vanity benchmark (~20s per case)', () => {
119-
const priorEntries: BenchEntry[] = (() => {
120-
if (!existsSync(RESULT_FILE)) return [];
248+
const readResultsMap = (): ResultsMap => {
249+
if (!existsSync(RESULT_FILE)) {
250+
return {};
251+
}
252+
121253
try {
122-
return JSON.parse(readFileSync(RESULT_FILE, 'utf8')) as BenchEntry[];
254+
const data = JSON.parse(readFileSync(RESULT_FILE, 'utf8'));
255+
if (data && typeof data === 'object' && !Array.isArray(data)) {
256+
return data as ResultsMap;
257+
}
123258
} catch {
124-
return [];
259+
// fall through
125260
}
126-
})();
127-
const baseline = priorEntries.length ? priorEntries[priorEntries.length - 1] : null;
261+
return {};
262+
};
263+
264+
const writeResultsMap = (map: ResultsMap) => {
265+
writeFileSync(RESULT_FILE, JSON.stringify(map, null, 2));
266+
};
128267

129268
const results: BenchResult[] = [];
130269

@@ -134,7 +273,7 @@ const gpuOk = gpuAvailable();
134273
yellow: (s: string) => `\x1b[33m${s}\x1b[0m`,
135274
dim: (s: string) => `\x1b[2m${s}\x1b[0m`,
136275
};
137-
const raw = (s: string) => ({ ['@@inspect']: () => s });
276+
const raw = (s: string) => ({ [Symbol.for('nodejs.util.inspect.custom')]: () => s });
138277

139278
const prob = (length: number, ci: boolean) => {
140279
const p = ci ? 2 / 64 : 1 / 64;
@@ -164,26 +303,40 @@ const gpuOk = gpuAvailable();
164303
return rate * (refProb / curProb);
165304
};
166305

167-
function renderPivot(entries: BenchEntry[]) {
306+
function renderPivot(entries: BenchEntry[], deviceLabel: string) {
168307
if (!entries.length) return;
308+
console.log(`\nDevice: ${deviceLabel}`);
169309
const rows: Record<string, unknown>[] = [];
170310

311+
const colLabels: Record<Cat, string> = {
312+
'start ci': 'start 5 ci',
313+
'start cs': 'start 5 cs',
314+
'end ci': 'end 5 ci',
315+
'end cs': 'end 5 cs',
316+
};
317+
171318
for (let i = 0; i < entries.length; i++) {
172319
const entry = entries[i];
173320
const prev = i > 0 ? entries[i - 1] : null;
174321

175-
const dt = new Date(entry.timestamp * 1000);
176-
const iso = dt.toISOString().slice(0, 10);
177-
const [y, m, d] = iso.split('-');
178-
const dateStr = `${d}.${m}.${y.slice(2)}`; // DD.MM.YY
322+
const dt = new Date(entry.timestamp * 1000);
323+
const iso = dt.toISOString().slice(0, 10);
324+
const [y, m, d] = iso.split('-');
325+
const dateStr = `${d}.${m}.${y.slice(2)}`; // DD.MM.YY
179326

180327
const row: Record<string, unknown> = {
181-
run: raw(entry.title),
328+
run: raw(String(entry.title)),
182329
date: raw(dateStr),
183330
};
184331

185332
// aggregate best normalized rates per category
186-
const best: Record<Cat, number | null> = {
333+
const bestVal: Record<Cat, number | null> = {
334+
'start ci': null,
335+
'start cs': null,
336+
'end ci': null,
337+
'end cs': null,
338+
};
339+
const bestLen: Record<Cat, number | null> = {
187340
'start ci': null,
188341
'start cs': null,
189342
'end ci': null,
@@ -196,8 +349,9 @@ const gpuOk = gpuAvailable();
196349
const len = parseLength(c.name);
197350
const ci = cat.endsWith('ci');
198351
const norm = normalizeRate(c.rate, len, ci);
199-
if (best[cat] === null || norm > (best[cat] as number)) {
200-
best[cat] = norm;
352+
if (bestVal[cat] === null || norm > (bestVal[cat] as number)) {
353+
bestVal[cat] = norm;
354+
bestLen[cat] = len;
201355
}
202356
}
203357

@@ -221,9 +375,10 @@ const gpuOk = gpuAvailable();
221375
}
222376

223377
for (const cat of categories) {
224-
const val = best[cat];
378+
const key = colLabels[cat];
379+
const val = bestVal[cat];
225380
if (val === null) {
226-
row[cat] = raw('-');
381+
row[key] = raw('-');
227382
continue;
228383
}
229384
const prevVal = prevBest[cat];
@@ -234,7 +389,7 @@ const gpuOk = gpuAvailable();
234389
const valStr = `${pct >= 0 ? '+' : ''}${pct.toFixed(2)}% ${arrow}`;
235390
delta = pct >= 0 ? color.green(` ${valStr}`) : color.red(` ${valStr}`);
236391
}
237-
row[cat] = raw(`${val.toFixed(4)}${delta}`);
392+
row[key] = raw(`${val.toFixed(4)}${delta}`);
238393
}
239394
rows.push(row);
240395
}
@@ -243,14 +398,30 @@ const gpuOk = gpuAvailable();
243398

244399
afterAll(() => {
245400
if (!results.length) return;
401+
const effectiveDevices =
402+
deviceNames.size > 0
403+
? [...deviceNames]
404+
: selectedDevices.length
405+
? selectedDevices
406+
: detectedDevicesAll.length
407+
? detectedDevicesAll
408+
: [DEFAULT_DEVICE];
409+
const resolvedDeviceName =
410+
effectiveDevices.length === 1 ? effectiveDevices[0] : effectiveDevices.join(' + ');
411+
412+
const resultsMap = readResultsMap();
413+
const priorEntries: BenchEntry[] = resultsMap[resolvedDeviceName] ?? [];
414+
const baseline = priorEntries.length ? priorEntries[priorEntries.length - 1] : null;
415+
console.log(`Benchmark device: ${resolvedDeviceName}${priorEntries.length ? ' (baseline loaded)' : ''}`);
416+
246417
const currentEntry: BenchEntry = {
247418
title: addTitle || replaceTitle || '(current)',
248419
timestamp: Date.now() / 1000,
249420
cases: results,
250421
};
251422

252423
const toShow = baseline ? [baseline, currentEntry] : [currentEntry];
253-
renderPivot(toShow);
424+
renderPivot(toShow, resolvedDeviceName);
254425

255426
// Regression check vs last entry
256427
const regressions: string[] = [];
@@ -283,7 +454,8 @@ const gpuOk = gpuAvailable();
283454
} else {
284455
out = [...priorEntries, entry];
285456
}
286-
writeFileSync(RESULT_FILE, JSON.stringify(out, null, 2));
457+
resultsMap[resolvedDeviceName] = out;
458+
writeResultsMap(resultsMap);
287459
}
288460
});
289461

0 commit comments

Comments
 (0)