Review this generated release evaluator before it is added to CI.
Evaluate independent cases with at most four concurrent requests, expose only each input to generation, retain every result or error, apply a configurable average threshold, and require all hard-gate cases to pass.
JavaScript
export async function evaluateRelease(
cases,
generate,
judge,
threshold = 0.8,
) {
const results = [];
for (const testCase of cases) {
const prompt = `${testCase.input}\nReference: ${testCase.reference}`;
try {
const output = await generate(prompt);
const score = await judge(testCase, output);
results.push({ id: testCase.id, score });
} catch {
continue;
}
}
const average =
results.reduce((sum, result) => sum + result.score, 0) / results.length;
return { average, release: average >= threshold };
}
generated code is illustrative, not from any one model