Files
OmniRoute/scripts/merge-queue/classify.mjs
diegosouzapw 56b92d0226 fix(ci): omni-queue hang evidence is scoped and counted from every run
A hang signature recorded on a base check now matches only a hang of
the candidate on the same check: a base that times out a job never
excuses an ordinary keyless failure of that job (base-red pause before,
pr-red now).

The hang path requires every run of the candidate to have been only job
timeouts. The in-flight retry carries timedOutRuns while that holds;
validateState accepts it as a count, and a document written without it
keeps the old behaviour (pause with reason infra). The hang-suspected
detail and the culprit comment state the real number of timed-out runs.

The culprit comment says the base passes the job only on a green base
verdict, and says nothing about the base without a verdict. The fake
classifier follows the real precedence for timeout and hang verdicts.
2026-11-15 05:35:58 -03:00

352 lines
14 KiB
JavaScript

// scripts/merge-queue/classify.mjs — WP1 (pure). Plan §3.5: base-red vs PR-red vs infra.
//
// A red candidate is classified from normalized failure signatures (one per failing run). The
// base-red verdict needs POSITIVE evidence: a CI run of the same profile on the bare base SHA
// (contracts.baseCheckRef) that fails the same checks with the same failing tests/gates.
/**
* @typedef {import("./checks.mjs").CheckRun} CheckRun
* @typedef {import("./contracts.mjs").FailureSignature} FailureSignature
* @typedef {import("./contracts.mjs").BaseCheck} BaseCheck
*/
/**
* @typedef {object} Annotation normalized by github.listCheckRunAnnotations
* @property {string|null} path
* @property {string|null} title
* @property {string} message
* @property {string} level notice | warning | failure
*/
/** Messages GitHub/runners emit when the machine, not the code, failed. */
const INFRA_PATTERNS = [
/runner has received a shutdown signal/i,
/lost communication with the server/i,
/the hosted runner encountered an error/i,
/the self-hosted runner .* (?:is offline|was lost|stopped)/i,
/CreateArtifact[\s\S]{0,200}\b404\b|\b404\b[\s\S]{0,200}CreateArtifact/i,
/no space left on device/i,
/the job was not acquired by runner/i,
];
/** Generic GitHub annotations that say a job failed/was cancelled without naming a test. */
const GENERIC_PATTERNS = [
/^process completed with exit code \d+\.?$/i,
/^the operation was canceled\.?$/i,
/^the job (?:running on runner .* )?has exceeded the maximum execution time/i,
];
const ANSI_RE = /\u001b\[[0-9;?]*[ -/]*[@-~]/g;
// Reporter timing only: a parenthesized duration at the end of a line ("(12ms)", "(1.5s)",
// node:test / mocha style), or a bare one ending a line that starts with a pass/fail mark
// ("✓ name 12ms", vitest style). Numbers inside a test title are part of its identity.
const TRAILING_PAREN_DURATION_RE = /(?:\s*\(\s*\d+(?:\.\d+)?\s?(?:ms|s|m|min)\s*\))+\s*$/i;
const REPORTER_MARK_RE = /^\s*[✔✖✓✗×]\s*/;
const TRAILING_BARE_DURATION_RE = /\s+\d+(?:\.\d+)?\s?(?:ms|s)\s*$/i;
const SHARD_SUFFIX_RE = /\s*\(\s*\d+\s*\/\s*\d+\s*\)\s*$/;
const KEY_SEPARATOR = " | ";
const keyParts = (key) => (key ? key.split(KEY_SEPARATOR) : []);
const text = (v) => String(v ?? "");
const isFailureLevel = (a) => text(a?.level).toLowerCase() === "failure";
function annotationText(a) {
return `${text(a?.title)}\n${text(a?.message)}`;
}
function isInfraText(s) {
return INFRA_PATTERNS.some((re) => re.test(s));
}
function isGenericAnnotation(a) {
const msg = text(a?.message).trim();
return GENERIC_PATTERNS.some((re) => re.test(msg));
}
/** Failure annotations that point at a real test/gate (not infra, not generic job noise). */
function specificFailures(annotations) {
return (annotations ?? []).filter(
(a) => isFailureLevel(a) && !isInfraText(annotationText(a)) && !isGenericAnnotation(a)
);
}
/**
* Positive evidence of an infrastructure failure for one run: runner lost / shutdown signal,
* "lost communication with the server", `CreateArtifact` 404, "No space left on device",
* startup_failure, job-level timed_out, or cancelled without any failure annotation.
* Any failure annotation naming a real test/gate wins over infra evidence (not infra).
* @param {CheckRun} run
* @param {Annotation[]} annotations
*/
export function isInfraFailure(run, annotations) {
if (specificFailures(annotations).length > 0) return false;
const conclusion = run?.conclusion;
if (conclusion === "startup_failure" || conclusion === "timed_out") return true;
if (conclusion === "cancelled") return true;
const haystack = [
...(annotations ?? []).map(annotationText),
text(run?.output?.title),
text(run?.output?.summary),
text(run?.output?.text),
].join("\n");
return isInfraText(haystack);
}
/** What GitHub writes on a job that hit its `timeout-minutes` (the duration varies). */
const MAX_EXECUTION_TIME_RE = /has exceeded the maximum execution time/i;
/**
* The run has the shape of a job that hit its time limit: conclusion `timed_out`, or `cancelled`
* with GitHub's "exceeded the maximum execution time" annotation. A plain cancel (superseded run,
* manual cancel), `startup_failure` and any runner-loss text are never hang-shaped. Repeated on
* every attempt this points at a hanging test, not at the machine (tick.onRepeatedHang);
* isInfraFailure still counts the first occurrence as infrastructure.
* @param {CheckRun} run
* @param {Annotation[]} annotations
*/
export function isHangShaped(run, annotations) {
const conclusion = run?.conclusion;
if (conclusion !== "timed_out" && conclusion !== "cancelled") return false;
const haystack = [
...(annotations ?? []).map(annotationText),
text(run?.output?.title),
text(run?.output?.summary),
text(run?.output?.text),
].join("\n");
if (isInfraText(haystack)) return false;
if (conclusion === "timed_out") return true;
return (annotations ?? []).some((a) => MAX_EXECUTION_TIME_RE.test(annotationText(a)));
}
/**
* One signature per hang-shaped run: `key` "" (a hang names no failing test), so it matches a
* base check by check name only (classifyRed's both-keys-empty rule), and `hang: true`.
* @param {CheckRun[]} failures
* @param {Object<number, Annotation[]>} annotationsByRunId
* @returns {FailureSignature[]}
*/
export function hangSignatures(failures, annotationsByRunId) {
return (failures ?? [])
.filter((run) => isHangShaped(run, annotationsByRunId?.[run?.id] ?? []))
.map((run) => ({ check: normalizeCheckName(run?.name), key: "", infra: false, hang: true }));
}
function stripReporterTiming(line) {
let out = line.replace(TRAILING_PAREN_DURATION_RE, "");
if (REPORTER_MARK_RE.test(out)) {
out = out.replace(REPORTER_MARK_RE, "").replace(TRAILING_BARE_DURATION_RE, "");
}
return out;
}
/** Lower-case, no ANSI codes, no reporter durations, collapsed whitespace. */
export function normalizeKeyPart(value) {
return text(value)
.replace(ANSI_RE, "")
.toLowerCase()
.split(/\r?\n/)
.map(stripReporterTiming)
.join(" ")
.replace(/\s+/g, " ")
.trim();
}
/** "Unit Tests (3/8)" → "Unit Tests": shard suffixes move between base and candidate. */
export function normalizeCheckName(name) {
return text(name).replace(SHARD_SUFFIX_RE, "").trim();
}
function signatureKey(run, annotations) {
const parts = specificFailures(annotations)
.map((a) => {
const label = text(a.title).trim() ? a.title : text(a.message).split("\n")[0];
return normalizeKeyPart(`${text(a.path)} ${label}`);
})
.filter(Boolean);
if (parts.length) return [...new Set(parts)].sort().join(KEY_SEPARATOR);
return normalizeKeyPart(run?.output?.title);
}
/**
* One signature per failing run. `check` is the run name with any matrix shard suffix removed
* ("Unit Tests (3/8)" → "Unit Tests"): shards re-balance when files are added, so the same
* failing test can move between shards from base to candidate. `key` is the normalized failing
* test/gate identifier from failure annotations (path + title, the first message line when the
* title is empty; several are sorted and joined with " | "), else from output.title, else "".
* @param {CheckRun[]} failures
* @param {Object<number, Annotation[]>} annotationsByRunId
* @returns {FailureSignature[]}
*/
export function extractSignatures(failures, annotationsByRunId) {
return (failures ?? []).map((run) => {
const annotations = annotationsByRunId?.[run?.id] ?? [];
return {
check: normalizeCheckName(run?.name),
key: signatureKey(run, annotations),
infra: isInfraFailure(run, annotations),
};
});
}
/**
* Matcher against the base check's failures. A candidate failure matches when the base fails the
* same check and: both keys are empty, or every failing test/gate of the candidate key is among
* the failing tests/gates of that check on the base, whatever shard reported each one (shards
* re-balance, so "A | B" in one shard and "A", "B" in two shards are the same failures). When
* exactly one side has a key the failures cannot be shown to be the same → no match (pr-red /
* bisection, never a base-red pause on a guess). Both empty is the degraded "check name only"
* match: in this repo node:test/vitest emit no failure annotations, so it is the only base-red
* evidence available. classifyRed reports how many matches were name-only (`nameOnly`).
* A base signature with `hang` (the job hit its time limit on the bare base) matches only a
* candidate signature with `hang` on the same check: a base that hangs on a job explains a
* candidate that hangs on it, never an ordinary test failure of that job.
* @param {FailureSignature[]} baseSignatures non-infra signatures of the base check
*/
function baseMatcher(baseSignatures) {
const partsByCheck = new Map();
const emptyKeyChecks = new Set();
const hangChecks = new Set();
for (const b of baseSignatures) {
if (b.hang) {
hangChecks.add(b.check);
continue;
}
if (!b.key) emptyKeyChecks.add(b.check);
const parts = partsByCheck.get(b.check) ?? new Set();
for (const part of keyParts(b.key)) parts.add(part);
partsByCheck.set(b.check, parts);
}
return (sig) => {
if (sig.hang && hangChecks.has(sig.check)) return true;
if (!sig.key) return emptyKeyChecks.has(sig.check);
const parts = partsByCheck.get(sig.check);
return Boolean(parts) && keyParts(sig.key).every((part) => parts.has(part));
};
}
/**
* An aggregate gate (config.aggregateChecks, e.g. "Gate / CI") runs with `if: always()` and fails
* whenever a job it needs did not succeed, without naming a test. Next to a failure of any other
* check it only repeats that failure, so it is left out; failing alone, or naming its own
* failure (non-empty key), it stays a real blocker.
*/
function withoutAggregates(signatures, aggregateChecks) {
const aggregate = new Set(aggregateChecks ?? []);
if (!signatures.some((s) => !aggregate.has(s.check))) return signatures;
return signatures.filter((s) => s.key || !aggregate.has(s.check));
}
/**
* Classify a red candidate:
* - aggregate gates (`aggregateChecks`) with an empty key are dropped first when any other
* check failed too: they only reflect that failure (see withoutAggregates);
* - every signature infra → "infra" (needs at least one signature);
* - baseCheck for the SAME base SHA is "green" → "pr-red";
* - baseCheck "red" and every non-infra signature matches its non-infra signatures (same
* `check`, and every failing test of the key fails on the base in some shard of that check —
* both empty counts as a name-only match, exactly one empty never matches) → "base-red";
* otherwise "pr-red";
* - baseCheck for the SAME base SHA timed out → "pr-red": baseCheckRef is deterministic per
* SHA and the queue never force-pushes, so that base check can never be redone (asking for it
* again would loop forever);
* - a baseCheck for the SAME base SHA still "pending" → "needs-base-check" with
* `baseCheckPending: true`: wait for it, do NOT start another;
* - otherwise (no verdict for this base SHA) → "needs-base-check" (`baseCheckPending: false`):
* the bare base is tested on the same profile before any PR is blamed. The base-red issue is
* not consulted: the system that writes it does not cover every check the queue requires. A
* tip the queue landed itself already carries a green verdict (tick.afterLanded), so only a
* tip that arrived from outside the queue costs a base check.
* `matched`/`unmatched` partition the non-infra signatures against the base check; `nameOnly`
* counts the matches made on the check name alone (both keys empty).
* @param {{signatures: FailureSignature[], baseSha: string, baseCheck: BaseCheck|null,
* aggregateChecks?: string[]}} input
* @returns {{kind: "infra"|"base-red"|"pr-red"|"needs-base-check", detail: string,
* matched: FailureSignature[], unmatched: FailureSignature[],
* baseCheckPending: boolean, nameOnly: number}}
*/
export function classifyRed(input) {
const result = classifyRedCore(input);
return { baseCheckPending: false, nameOnly: 0, ...result };
}
function classifyRedCore(input) {
const all = input.signatures ?? [];
const signatures = withoutAggregates(all, input.aggregateChecks);
const real = signatures.filter((s) => !s.infra);
if (signatures.length > 0 && real.length === 0) {
const ignored = all.length - signatures.length;
return {
kind: "infra",
detail:
`all ${signatures.length} failure(s) are infrastructure` +
(ignored ? ` (${ignored} aggregate gate failure(s) ignored)` : ""),
matched: [],
unmatched: [],
};
}
const bc = input.baseCheck;
const sameBase = Boolean(bc) && bc.baseSha === input.baseSha;
if (sameBase && bc.status === "green") {
return {
kind: "pr-red",
detail: `base ${text(input.baseSha).slice(0, 12)} is green on the same profile`,
matched: [],
unmatched: real,
};
}
if (sameBase && bc.status === "red") {
const failsOnBase = baseMatcher((bc.signatures ?? []).filter((s) => !s.infra));
const matched = [];
const unmatched = [];
for (const sig of real) (failsOnBase(sig) ? matched : unmatched).push(sig);
const nameOnly = matched.filter((m) => !m.key).length;
if (real.length > 0 && unmatched.length === 0) {
return {
kind: "base-red",
detail:
`all ${matched.length} failure(s) also fail on base ${text(input.baseSha).slice(0, 12)}` +
(nameOnly ? ` (${nameOnly} matched by check name only: no test-level key)` : ""),
matched,
unmatched,
nameOnly,
};
}
return {
kind: "pr-red",
detail:
`${unmatched.length} failure(s) not seen on base ${text(input.baseSha).slice(0, 12)}: ` +
unmatched.map((s) => (s.key ? `${s.check} [${s.key}]` : s.check)).join("; "),
matched,
unmatched,
nameOnly,
};
}
if (sameBase && bc.status === "timeout") {
return {
kind: "pr-red",
detail:
`base check on ${text(input.baseSha).slice(0, 12)} timed out (it cannot be redone for ` +
"the same SHA): no base-red evidence",
matched: [],
unmatched: real,
};
}
if (sameBase && bc.status === "pending") {
return {
kind: "needs-base-check",
detail: `base check in progress for ${text(input.baseSha).slice(0, 12)}: wait for it`,
matched: [],
unmatched: real,
baseCheckPending: true,
};
}
return {
kind: "needs-base-check",
detail: `no verdict for base ${text(input.baseSha).slice(0, 12)}: test the bare base first`,
matched: [],
unmatched: real,
};
}