test(ci): the omni-queue tests the reviews asked for before live mode

Task 13b, the test-only items.

- The tick fake's eligibility honours the mode as eligibility.mjs does
  (`shadow-processed` only in shadow mode). New: "a PR that produced a
  would-land in shadow is batched again once the mode is live" (failed
  with the old fake).
- The attempts bound and the withdrawal: "an authentication failure on
  the second landing attempt pauses with the status withdrawn" (passed at
  once: pins existing behaviour) and "the tick that hits
  LANDING_ATTEMPTS_LIMIT withdraws a blessing still standing" (seeded with
  the App's success still standing; on 05b9bd4 it failed only on the
  withdrawal's wording, the behaviour was already there).
- End to end on real git: scenario 2e, blessed, then the base moves
  between the landing's tip read and its push (queue-world's new
  beforeBasePush hook wraps the tick's git runners): the push is refused
  (stale info), the status withdrawn, the batch recomposed on the moved
  tip and landed. Scenario 2f: a planted candidate with the same parents,
  the same message and another tree is refused by the real
  verifyComposition through a tick: nothing blessed, paused. Both passed
  at once (existing behaviour); each fails with the code it pins removed
  (revokeLanding after a refused landing; `trees: true`), checked once.
- Re-aimed, scenario 14 of tick-e2e.test.mjs: the deepEqual compared
  baseChecks[0].startedAt with itself; it now compares it with the
  retry's start, read from the clock of the tick that pushed the retry.
- Re-aimed, github.test.mjs "a hung request times out and is retried as
  a network error": the stalled response is held 10 minutes instead of
  10 s and the bound is half of that instead of 5 s, so machine load
  cannot fail it; a client that waits for the stalled answer fails at the
  test's own 2-minute timeout. What it proves is unchanged (the answer
  comes from the retried request, after one 1 s timeout and one 1 s
  backoff).
This commit is contained in:
diegosouzapw
2026-11-16 15:10:42 -03:00
parent 88422cce7d
commit 6b20bd7582
5 changed files with 211 additions and 13 deletions

View File

@@ -684,17 +684,27 @@ test("mergePull gives up after maxRetries of unconfirmed 5xx", async (t) => {
assert.equal(fake.pulls.get(34).merged, false);
});
test("a hung request times out and is retried as a network error", async (t) => {
// 1 s, not 100 ms: on a loaded machine the retried (fast) request must not time out too.
const { fake, gh, sleeps, logs } = await setup(t, {}, { requestTimeoutMs: 1000 });
fake.addPull({ number: 35, title: "slow" });
fake.inject({ method: "GET", path: "/pulls/35", status: 200, body: {}, delayMs: 10_000 });
const started = Date.now();
assert.equal((await gh.getPull(35)).title, "slow");
assert.ok(Date.now() - started < 5000, "the stalled response did not hold the client");
assert.deepEqual(sleeps, [1000]);
assert.ok(logs.some((l) => /timed out after 1000 ms/.test(l)));
});
// The stalled response is held for 10 minutes. The client times out after 1 s and asks again;
// the answer of that second request (title "slow": the stalled one has no title) arriving in less
// than half the stall can only come from the client giving up on the first. Nothing in this bound
// depends on how loaded the machine is: it fails only if the client waits for the stalled answer.
// A client that does wait fails the test at the test's own 2-minute timeout instead of 10 minutes.
const STALL_MS = 600_000;
test(
"a hung request times out and is retried as a network error",
{ timeout: 120_000 },
async (t) => {
// 1 s, not 100 ms: on a loaded machine the retried (fast) request must not time out too.
const { fake, gh, sleeps, logs } = await setup(t, {}, { requestTimeoutMs: 1000 });
fake.addPull({ number: 35, title: "slow" });
fake.inject({ method: "GET", path: "/pulls/35", status: 200, body: {}, delayMs: STALL_MS });
const started = Date.now();
assert.equal((await gh.getPull(35)).title, "slow");
assert.ok(Date.now() - started < STALL_MS / 2, "the stalled response did not hold the client");
assert.deepEqual(sleeps, [1000]);
assert.ok(logs.some((l) => /timed out after 1000 ms/.test(l)));
}
);
test("a request that keeps timing out fails with GitHubError status 0", async (t) => {
const { fake, gh } = await setup(t, {}, { requestTimeoutMs: 50, maxRetries: 1 });

View File

@@ -548,7 +548,11 @@ export function createFakeModules(world) {
const labeled = pr.timeline.filter((e) => e.event === "labeled" && e.label === label);
const labeledAt = labeled.length ? labeled[labeled.length - 1].createdAt : null;
const inFlight = ctx.state?.inFlight?.prs?.some((p) => p.n === pr.number) ?? false;
const processed = ctx.state?.shadow?.processed?.[String(pr.number)] === pr.headSha;
// As eligibility.evaluateEligibility: a head settled in shadow mode is skipped in shadow
// mode only; live mode batches it again.
const processed =
ctx.config.mode === "shadow" &&
ctx.state?.shadow?.processed?.[String(pr.number)] === pr.headSha;
// newest status of the approval context on the head (approve.mjs writes `failure` when the
// label is removed; a re-label by a non-approver writes nothing)
const approval = pr.headStatuses.find((st) => st.context === APPROVAL_CONTEXTS.queue);

View File

@@ -322,6 +322,13 @@ export async function createQueueWorld(t, options = {}) {
});
},
/**
* When set, a function run once right before the tick's next git push to the base branch (the
* merge-ff landing push), after the tick and the landing read the base tip: somebody merges in
* that window, the race the push's lease refuses. Cleared when it runs.
*/
beforeBasePush: null,
/** One tick, built like main() (fresh runner directory every time). */
async tick({ action = "tick" } = {}) {
tickCount += 1;
@@ -330,6 +337,27 @@ export async function createQueueWorld(t, options = {}) {
{ ...env, RUNNER_TEMP: runnerTemp },
{ now: clock.now, gitEnv, log: (line) => logs.push(line) }
);
// The git runners of the tick, unchanged except that a push to the base runs
// world.beforeBasePush first (the App's and, with landing.actor "owner", the land token's).
const baseDst = `:refs/heads/${QBASE}`;
const hooked = (factory) =>
factory &&
((cwd) => {
const git = factory(cwd);
return {
...git,
run(args, opts) {
const hook = world.beforeBasePush;
if (hook && args[0] === "push" && args.some((a) => String(a).endsWith(baseDst))) {
world.beforeBasePush = null;
hook();
}
return git.run(args, opts);
},
};
});
deps.gitFor = hooked(deps.gitFor);
deps.landGitFor = hooked(deps.landGitFor);
const from = fake.requests.length;
const result = await runTick(deps, { base: QBASE, action });
return { result, requests: fake.requests.slice(from) };

View File

@@ -276,6 +276,99 @@ scenario(
}
);
scenario(
"scenario 2e (live): blessed, then the base moves before the landing push: the push is refused, the status withdrawn, the batch recomposed",
async (t) => {
const w = await createQueueWorld(t, { mode: "live" });
const base0 = w.baseTip();
const heads = await approvedPrs(w, [1]);
await w.tick();
const f = w.state().inFlight;
w.runCi(f.candidateSha, f.candidateRef);
// A manual merge lands between the landing's own tip read and its push.
let moved = null;
w.beforeBasePush = () => {
moved = w.sb.advanceBase({ "manual.txt": "merged by hand\n" }, "manual merge");
};
await w.tick();
let state = w.state();
assert.ok(moved, "the base moved right before the push");
assert.equal(w.baseTip(), moved, "the push was refused: the base is the manual merge");
assert.ok(!w.isAncestor(f.candidateSha, w.baseTip()), "the candidate did not land");
assert.deepEqual(landStatuses(w, f.candidateSha), [
["failure", BOT_LOGIN, `omni-queue ${f.batchId}: landing withdrawn (base-moved)`],
["success", BOT_LOGIN, `omni-queue ${f.batchId}: checks green`],
]);
assert.equal(w.events("land-blessed").length, 1);
assert.equal(w.events("landed").length, 0);
const rejected = w.events("land-rejected");
assert.equal(rejected.length, 1);
assert.match(rejected[0].detail, /^base moved: .*stale info.*; recomposing$/);
assert.equal(state.paused, null);
assert.equal(state.inFlight.baseSha, moved, "recomposed on the moved tip");
assert.equal(state.inFlight.stage, "testing");
assert.notEqual(state.inFlight.candidateSha, f.candidateSha);
assert.deepEqual(nums(state.inFlight.prs), [1]);
assert.ok(w.isAncestor(moved, state.inFlight.candidateSha));
assert.equal("landWithdrawals" in state, false);
await drain(w);
state = w.state();
assert.equal(w.events("landed").length, 1);
assert.ok(w.isAncestor(heads[1], w.baseTip()), "#1 landed on the moved base");
assert.ok(w.isAncestor(moved, w.baseTip()));
assert.notEqual(w.baseTip(), base0);
assert.deepEqual(landStatuses(w, f.candidateSha)[0][0], "failure", "still withdrawn");
}
);
scenario(
"scenario 2f (live): a planted candidate with the same parents and another tree is refused by the real composition check",
async (t) => {
const w = await createQueueWorld(t, { mode: "live" });
const base0 = w.baseTip();
const heads = await approvedPrs(w, [1]);
await w.tick();
const state = w.state();
const f = state.inFlight;
const parents = w.sb.origin(["log", "-1", "--format=%P", f.candidateSha]).stdout.split(" ");
assert.deepEqual(parents, [base0, heads[1]], "one PR: the candidate is its merge commit");
// The same parents and the same message (trailers included) on the bare base's tree: the
// merge commit a state edit could plant to land a tree nobody tested.
const message = w.sb.origin(["log", "-1", "--format=%B", f.candidateSha]).stdout;
const planted = w.sb.origin(
["commit-tree", `${base0}^{tree}`, "-p", base0, "-p", heads[1], "-F", "-"],
{ input: `${message}\n` }
).stdout;
assert.notEqual(planted, f.candidateSha);
const ref = candidateRef(QBASE, f.batchId, planted);
w.sb.origin(["update-ref", `refs/heads/${ref}`, planted]);
state.inFlight = { ...f, candidateSha: planted, candidateRef: ref };
w.fake.commitFiles({
branch: STATE_BRANCH,
files: { [stateFilePath(QBASE)]: `${JSON.stringify(state, null, 2)}\n` },
message: "planted",
});
w.runCi(planted, ref); // its CI is green: a planted tree controls its own checks
await w.tick();
const after = w.state();
assert.deepEqual(landStatuses(w, planted), [], "no land status on the planted commit");
assert.deepEqual(landStatuses(w, f.candidateSha), [], "nor on the composed one");
assert.equal(w.baseTip(), base0, "nothing landed");
assert.equal(w.events("land-blessed").length, 0);
assert.equal(after.inFlight, null);
assert.equal(after.paused?.reason, "error");
assert.match(after.paused.detail, /does not match its candidate/);
assert.ok(
after.paused.detail.includes(`merge commit ${head12(planted)} of #1: tree `),
after.paused.detail
);
assert.match(after.paused.detail, /recomposing gives [0-9a-f]{12}/);
}
);
scenario(
"scenario 2b: a pull request opened on the candidate SHA never decides the candidate's verdict",
async (t) => {
@@ -1307,6 +1400,9 @@ scenario(
assert.equal(state.inFlight.batchId, `basecheck-${head12(base0)}-r1`);
assert.match(state.inFlight.candidateRef, baseCheckRefRe(base0, "-r1"));
assert.equal(state.paused, null);
// The retry's ref was pushed by this tick: its runs count from this tick's clock.
const retryStartedAt = w.clock.now();
assert.equal(state.inFlight.startedAt, retryStartedAt);
w.clock.advance(NO_RUN_TIMEOUT_MINUTES + 1); // nor on its retry
await w.tick();
@@ -1315,7 +1411,7 @@ scenario(
baseSha: base0,
ref: baseCheckRef(QBASE, base0, 1),
status: "timeout",
startedAt: state.baseChecks[0].startedAt,
startedAt: retryStartedAt, // the verdict is the retry's: it starts when the retry was pushed
signatures: [],
});
assert.equal(state.paused, null, "no pause on a base check without a result");

View File

@@ -7486,6 +7486,66 @@ test("base-red pause whose repair lane no longer runs: a base-repair batch at th
assert.deepEqual(h.world.lands, []);
});
// ── the attempts bound and the withdrawal ──
test("an authentication failure on the second landing attempt pauses with the status withdrawn", async () => {
const h = createHarness({ mode: "live", actor: "owner" });
const f = await greenBatch(h, [1]);
h.world.landError = notAvailable(1);
await failingTick(h, /not available from origin/);
assert.equal((await h.state()).inFlight.landAttempts, 1);
h.world.landError = new Error(
"landMergeFf: push gave no verdict: remote: Invalid username or token. | fatal: " +
"Authentication failed for 'https://github.com/acme/widgets.git/'"
);
await h.tick();
const state = await h.state();
assert.equal(state.paused?.reason, "error");
assert.match(state.paused.detail, /OMNI_QUEUE_LAND_TOKEN/);
assert.equal(state.inFlight, null);
assert.deepEqual(statusStates(h), landPairs(2), "each attempt blessed, then withdrawn");
assert.equal(newestLand(h, f.candidateSha), "failure");
});
test("the tick that hits LANDING_ATTEMPTS_LIMIT withdraws a blessing still standing", async () => {
const h = createHarness({ mode: "live", actor: "owner" });
const f = await greenBatch(h, [1]);
// The third attempt's tick blessed it and died: no withdrawal ran.
await diedAfterBlessing(h, f, [f.candidateSha]);
await tamperInFlight(h, { landAttempts: LANDING_ATTEMPTS_LIMIT });
await h.tick();
const state = await h.state();
assert.equal(state.paused?.reason, "error");
assert.match(state.paused.detail, /ended without a verdict 3 times in a row/);
assert.deepEqual(
statusCalls(h).map((c) => c.args),
[landStatus(f.batchId, f.candidateSha, { why: LEFT_LANDING })],
"not blessed again; the standing success withdrawn"
);
assert.deepEqual(h.world.lands, []);
});
// ── shadow → live ──
test("a PR that produced a would-land in shadow is batched again once the mode is live", async () => {
const h = createHarness({ mode: "shadow" });
const f = await greenBatch(h, [1]);
await h.tick();
let state = await h.state();
assert.ok(types(state).includes("would-land"));
assert.equal(state.shadow.processed[1], f.prs[0].head);
await h.tick();
assert.equal((await h.state()).inFlight, null, "shadow: the processed head is not batched again");
h.config.mode = "live";
await h.tick();
state = await h.state();
assert.deepEqual(nums(state.inFlight?.prs ?? []), [1], "live batches it again");
h.runCi(state.inFlight.candidateSha);
await h.tick();
assert.ok(types(await h.state()).includes("landed"));
});
test("M1 through main(), shadow mode (the shipped config): a tick that throws after it left a stored landing withdraws the status and saves the batch at testing with its error", async (t) => {
const q = await mainAgainstFake(t);
const now = new Date().toISOString();