mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-16 17:09:09 +00:00
CodeRabbit review of #20166 (classify front): - Base-red matching compared whole per-run keys ("a | b"), so the same failing tests grouped into different shards on base and candidate never matched and the batch took the PR-red path (culprit / bisection). The matcher now builds, per normalized check, the union of failing test/gate identifiers of the base check and matches a candidate failure when every identifier of its key is in that union. Empty keys keep the name-only rule (empty matches empty only). - The always() gates "Gate / CI" and "Gate / Quality" fail with no test named whenever a job they need fails, so a timed-out job plus its gate failure was classified PR-red instead of infra. New config key aggregateChecks (validated in config.mjs, passed by tick.onRed to every classifyRed call) lists them; an aggregate failure with an empty key is ignored when any other check failed too. A gate failing alone, or one that names its own failure, still blocks. - validateConfig accepted non-string approvers (null, 123, true) because LOGIN_RE.test coerces them; blockLabels had the same gap. Both item validators now require typeof string, matching the Zod schema. No existing assertion changed; the Zod schema in config.test.mjs gains aggregateChecks to mirror the new required key.
310 lines
12 KiB
JavaScript
310 lines
12 KiB
JavaScript
// scripts/merge-queue/classify.mjs — WP1 (pure). Plan §3.5: base-red vs PR-red vs infra.
|
|
//
|
|
// A red candidate is classified from normalized failure signatures (one per failing run). The
|
|
// base-red verdict needs POSITIVE evidence: a CI run of the same profile on the bare base SHA
|
|
// (contracts.baseCheckRef) that fails the same checks with the same failing tests/gates.
|
|
|
|
/**
|
|
* @typedef {import("./checks.mjs").CheckRun} CheckRun
|
|
* @typedef {import("./contracts.mjs").FailureSignature} FailureSignature
|
|
* @typedef {import("./contracts.mjs").BaseCheck} BaseCheck
|
|
*/
|
|
|
|
/**
|
|
* @typedef {object} Annotation normalized by github.listCheckRunAnnotations
|
|
* @property {string|null} path
|
|
* @property {string|null} title
|
|
* @property {string} message
|
|
* @property {string} level notice | warning | failure
|
|
*/
|
|
|
|
/** Messages GitHub/runners emit when the machine, not the code, failed. */
|
|
const INFRA_PATTERNS = [
|
|
/runner has received a shutdown signal/i,
|
|
/lost communication with the server/i,
|
|
/the hosted runner encountered an error/i,
|
|
/the self-hosted runner .* (?:is offline|was lost|stopped)/i,
|
|
/CreateArtifact[\s\S]{0,200}\b404\b|\b404\b[\s\S]{0,200}CreateArtifact/i,
|
|
/no space left on device/i,
|
|
/the job was not acquired by runner/i,
|
|
];
|
|
|
|
/** Generic GitHub annotations that say a job failed/was cancelled without naming a test. */
|
|
const GENERIC_PATTERNS = [
|
|
/^process completed with exit code \d+\.?$/i,
|
|
/^the operation was canceled\.?$/i,
|
|
/^the job (?:running on runner .* )?has exceeded the maximum execution time/i,
|
|
];
|
|
|
|
const ANSI_RE = /\u001b\[[0-9;?]*[ -/]*[@-~]/g;
|
|
// Reporter timing only: a parenthesized duration at the end of a line ("(12ms)", "(1.5s)",
|
|
// node:test / mocha style), or a bare one ending a line that starts with a pass/fail mark
|
|
// ("✓ name 12ms", vitest style). Numbers inside a test title are part of its identity.
|
|
const TRAILING_PAREN_DURATION_RE = /(?:\s*\(\s*\d+(?:\.\d+)?\s?(?:ms|s|m|min)\s*\))+\s*$/i;
|
|
const REPORTER_MARK_RE = /^\s*[✔✖✓✗×]\s*/;
|
|
const TRAILING_BARE_DURATION_RE = /\s+\d+(?:\.\d+)?\s?(?:ms|s)\s*$/i;
|
|
const SHARD_SUFFIX_RE = /\s*\(\s*\d+\s*\/\s*\d+\s*\)\s*$/;
|
|
const KEY_SEPARATOR = " | ";
|
|
const keyParts = (key) => (key ? key.split(KEY_SEPARATOR) : []);
|
|
|
|
const text = (v) => String(v ?? "");
|
|
const isFailureLevel = (a) => text(a?.level).toLowerCase() === "failure";
|
|
|
|
function annotationText(a) {
|
|
return `${text(a?.title)}\n${text(a?.message)}`;
|
|
}
|
|
|
|
function isInfraText(s) {
|
|
return INFRA_PATTERNS.some((re) => re.test(s));
|
|
}
|
|
|
|
function isGenericAnnotation(a) {
|
|
const msg = text(a?.message).trim();
|
|
return GENERIC_PATTERNS.some((re) => re.test(msg));
|
|
}
|
|
|
|
/** Failure annotations that point at a real test/gate (not infra, not generic job noise). */
|
|
function specificFailures(annotations) {
|
|
return (annotations ?? []).filter(
|
|
(a) => isFailureLevel(a) && !isInfraText(annotationText(a)) && !isGenericAnnotation(a)
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Positive evidence of an infrastructure failure for one run: runner lost / shutdown signal,
|
|
* "lost communication with the server", `CreateArtifact` 404, "No space left on device",
|
|
* startup_failure, job-level timed_out, or cancelled without any failure annotation.
|
|
* Any failure annotation naming a real test/gate wins over infra evidence (not infra).
|
|
* @param {CheckRun} run
|
|
* @param {Annotation[]} annotations
|
|
*/
|
|
export function isInfraFailure(run, annotations) {
|
|
if (specificFailures(annotations).length > 0) return false;
|
|
const conclusion = run?.conclusion;
|
|
if (conclusion === "startup_failure" || conclusion === "timed_out") return true;
|
|
if (conclusion === "cancelled") return true;
|
|
const haystack = [
|
|
...(annotations ?? []).map(annotationText),
|
|
text(run?.output?.title),
|
|
text(run?.output?.summary),
|
|
text(run?.output?.text),
|
|
].join("\n");
|
|
return isInfraText(haystack);
|
|
}
|
|
|
|
function stripReporterTiming(line) {
|
|
let out = line.replace(TRAILING_PAREN_DURATION_RE, "");
|
|
if (REPORTER_MARK_RE.test(out)) {
|
|
out = out.replace(REPORTER_MARK_RE, "").replace(TRAILING_BARE_DURATION_RE, "");
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/** Lower-case, no ANSI codes, no reporter durations, collapsed whitespace. */
|
|
export function normalizeKeyPart(value) {
|
|
return text(value)
|
|
.replace(ANSI_RE, "")
|
|
.toLowerCase()
|
|
.split(/\r?\n/)
|
|
.map(stripReporterTiming)
|
|
.join(" ")
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
}
|
|
|
|
/** "Unit Tests (3/8)" → "Unit Tests": shard suffixes move between base and candidate. */
|
|
export function normalizeCheckName(name) {
|
|
return text(name).replace(SHARD_SUFFIX_RE, "").trim();
|
|
}
|
|
|
|
function signatureKey(run, annotations) {
|
|
const parts = specificFailures(annotations)
|
|
.map((a) => {
|
|
const label = text(a.title).trim() ? a.title : text(a.message).split("\n")[0];
|
|
return normalizeKeyPart(`${text(a.path)} ${label}`);
|
|
})
|
|
.filter(Boolean);
|
|
if (parts.length) return [...new Set(parts)].sort().join(KEY_SEPARATOR);
|
|
return normalizeKeyPart(run?.output?.title);
|
|
}
|
|
|
|
/**
|
|
* One signature per failing run. `check` is the run name with any matrix shard suffix removed
|
|
* ("Unit Tests (3/8)" → "Unit Tests"): shards re-balance when files are added, so the same
|
|
* failing test can move between shards from base to candidate. `key` is the normalized failing
|
|
* test/gate identifier from failure annotations (path + title, the first message line when the
|
|
* title is empty; several are sorted and joined with " | "), else from output.title, else "".
|
|
* @param {CheckRun[]} failures
|
|
* @param {Object<number, Annotation[]>} annotationsByRunId
|
|
* @returns {FailureSignature[]}
|
|
*/
|
|
export function extractSignatures(failures, annotationsByRunId) {
|
|
return (failures ?? []).map((run) => {
|
|
const annotations = annotationsByRunId?.[run?.id] ?? [];
|
|
return {
|
|
check: normalizeCheckName(run?.name),
|
|
key: signatureKey(run, annotations),
|
|
infra: isInfraFailure(run, annotations),
|
|
};
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Matcher against the base check's failures. A candidate failure matches when the base fails the
|
|
* same check and: both keys are empty, or every failing test/gate of the candidate key is among
|
|
* the failing tests/gates of that check on the base, whatever shard reported each one (shards
|
|
* re-balance, so "A | B" in one shard and "A", "B" in two shards are the same failures). When
|
|
* exactly one side has a key the failures cannot be shown to be the same → no match (pr-red /
|
|
* bisection, never a base-red pause on a guess). Both empty is the degraded "check name only"
|
|
* match: in this repo node:test/vitest emit no failure annotations, so it is the only base-red
|
|
* evidence available. classifyRed reports how many matches were name-only (`nameOnly`).
|
|
* @param {FailureSignature[]} baseSignatures non-infra signatures of the base check
|
|
*/
|
|
function baseMatcher(baseSignatures) {
|
|
const partsByCheck = new Map();
|
|
const emptyKeyChecks = new Set();
|
|
for (const b of baseSignatures) {
|
|
if (!b.key) emptyKeyChecks.add(b.check);
|
|
const parts = partsByCheck.get(b.check) ?? new Set();
|
|
for (const part of keyParts(b.key)) parts.add(part);
|
|
partsByCheck.set(b.check, parts);
|
|
}
|
|
return (sig) => {
|
|
if (!sig.key) return emptyKeyChecks.has(sig.check);
|
|
const parts = partsByCheck.get(sig.check);
|
|
return Boolean(parts) && keyParts(sig.key).every((part) => parts.has(part));
|
|
};
|
|
}
|
|
|
|
/**
|
|
* An aggregate gate (config.aggregateChecks, e.g. "Gate / CI") runs with `if: always()` and fails
|
|
* whenever a job it needs did not succeed, without naming a test. Next to a failure of any other
|
|
* check it only repeats that failure, so it is left out; failing alone, or naming its own
|
|
* failure (non-empty key), it stays a real blocker.
|
|
*/
|
|
function withoutAggregates(signatures, aggregateChecks) {
|
|
const aggregate = new Set(aggregateChecks ?? []);
|
|
if (!signatures.some((s) => !aggregate.has(s.check))) return signatures;
|
|
return signatures.filter((s) => s.key || !aggregate.has(s.check));
|
|
}
|
|
|
|
/**
|
|
* Classify a red candidate:
|
|
* - aggregate gates (`aggregateChecks`) with an empty key are dropped first when any other
|
|
* check failed too: they only reflect that failure (see withoutAggregates);
|
|
* - every signature infra → "infra" (needs at least one signature);
|
|
* - baseCheck for the SAME base SHA is "green" → "pr-red";
|
|
* - baseCheck "red" and every non-infra signature matches its non-infra signatures (same
|
|
* `check`, and every failing test of the key fails on the base in some shard of that check —
|
|
* both empty counts as a name-only match, exactly one empty never matches) → "base-red";
|
|
* otherwise "pr-red";
|
|
* - baseCheck for the SAME base SHA timed out → "pr-red": baseCheckRef is deterministic per
|
|
* SHA and the queue never force-pushes, so that base check can never be redone (asking for it
|
|
* again would loop forever);
|
|
* - baseRedIssueOpen and a baseCheck for the SAME base SHA still "pending" →
|
|
* "needs-base-check" with `baseCheckPending: true`: wait for it, do NOT start another;
|
|
* - baseRedIssueOpen and no base check for this base SHA → "needs-base-check"
|
|
* (`baseCheckPending: false`);
|
|
* - otherwise → "pr-red".
|
|
* `matched`/`unmatched` partition the non-infra signatures against the base check; `nameOnly`
|
|
* counts the matches made on the check name alone (both keys empty).
|
|
* @param {{signatures: FailureSignature[], baseSha: string, baseCheck: BaseCheck|null,
|
|
* baseRedIssueOpen: boolean, aggregateChecks?: string[]}} input
|
|
* @returns {{kind: "infra"|"base-red"|"pr-red"|"needs-base-check", detail: string,
|
|
* matched: FailureSignature[], unmatched: FailureSignature[],
|
|
* baseCheckPending: boolean, nameOnly: number}}
|
|
*/
|
|
export function classifyRed(input) {
|
|
const result = classifyRedCore(input);
|
|
return { baseCheckPending: false, nameOnly: 0, ...result };
|
|
}
|
|
|
|
function classifyRedCore(input) {
|
|
const all = input.signatures ?? [];
|
|
const signatures = withoutAggregates(all, input.aggregateChecks);
|
|
const real = signatures.filter((s) => !s.infra);
|
|
if (signatures.length > 0 && real.length === 0) {
|
|
const ignored = all.length - signatures.length;
|
|
return {
|
|
kind: "infra",
|
|
detail:
|
|
`all ${signatures.length} failure(s) are infrastructure` +
|
|
(ignored ? ` (${ignored} aggregate gate failure(s) ignored)` : ""),
|
|
matched: [],
|
|
unmatched: [],
|
|
};
|
|
}
|
|
|
|
const bc = input.baseCheck;
|
|
const sameBase = Boolean(bc) && bc.baseSha === input.baseSha;
|
|
if (sameBase && bc.status === "green") {
|
|
return {
|
|
kind: "pr-red",
|
|
detail: `base ${text(input.baseSha).slice(0, 12)} is green on the same profile`,
|
|
matched: [],
|
|
unmatched: real,
|
|
};
|
|
}
|
|
if (sameBase && bc.status === "red") {
|
|
const failsOnBase = baseMatcher((bc.signatures ?? []).filter((s) => !s.infra));
|
|
const matched = [];
|
|
const unmatched = [];
|
|
for (const sig of real) (failsOnBase(sig) ? matched : unmatched).push(sig);
|
|
const nameOnly = matched.filter((m) => !m.key).length;
|
|
if (real.length > 0 && unmatched.length === 0) {
|
|
return {
|
|
kind: "base-red",
|
|
detail:
|
|
`all ${matched.length} failure(s) also fail on base ${text(input.baseSha).slice(0, 12)}` +
|
|
(nameOnly ? ` (${nameOnly} matched by check name only: no test-level key)` : ""),
|
|
matched,
|
|
unmatched,
|
|
nameOnly,
|
|
};
|
|
}
|
|
return {
|
|
kind: "pr-red",
|
|
detail:
|
|
`${unmatched.length} failure(s) not seen on base ${text(input.baseSha).slice(0, 12)}: ` +
|
|
unmatched.map((s) => (s.key ? `${s.check} [${s.key}]` : s.check)).join("; "),
|
|
matched,
|
|
unmatched,
|
|
nameOnly,
|
|
};
|
|
}
|
|
if (sameBase && bc.status === "timeout") {
|
|
return {
|
|
kind: "pr-red",
|
|
detail:
|
|
`base check on ${text(input.baseSha).slice(0, 12)} timed out (it cannot be redone for ` +
|
|
"the same SHA): no base-red evidence",
|
|
matched: [],
|
|
unmatched: real,
|
|
};
|
|
}
|
|
|
|
if (input.baseRedIssueOpen) {
|
|
if (sameBase && bc.status === "pending") {
|
|
return {
|
|
kind: "needs-base-check",
|
|
detail: `base check in progress for ${text(input.baseSha).slice(0, 12)}: wait for it`,
|
|
matched: [],
|
|
unmatched: real,
|
|
baseCheckPending: true,
|
|
};
|
|
}
|
|
return {
|
|
kind: "needs-base-check",
|
|
detail: "base-red issue open and no base check for this base SHA",
|
|
matched: [],
|
|
unmatched: real,
|
|
};
|
|
}
|
|
return {
|
|
kind: "pr-red",
|
|
detail: "no base-red evidence for this base SHA",
|
|
matched: [],
|
|
unmatched: real,
|
|
};
|
|
}
|