mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-18 07:58:27 +00:00
test(ci): the omni-queue tests the reviews asked for before live mode
Task 13b, the test-only items.
- The tick fake's eligibility honours the mode as eligibility.mjs does
(`shadow-processed` only in shadow mode). New: "a PR that produced a
would-land in shadow is batched again once the mode is live" (failed
with the old fake).
- The attempts bound and the withdrawal: "an authentication failure on
the second landing attempt pauses with the status withdrawn" (passed at
once: pins existing behaviour) and "the tick that hits
LANDING_ATTEMPTS_LIMIT withdraws a blessing still standing" (seeded with
the App's success still standing; on 05b9bd4 it failed only on the
withdrawal's wording, the behaviour was already there).
- End to end on real git: scenario 2e, blessed, then the base moves
between the landing's tip read and its push (queue-world's new
beforeBasePush hook wraps the tick's git runners): the push is refused
(stale info), the status withdrawn, the batch recomposed on the moved
tip and landed. Scenario 2f: a planted candidate with the same parents,
the same message and another tree is refused by the real
verifyComposition through a tick: nothing blessed, paused. Both passed
at once (existing behaviour); each fails with the code it pins removed
(revokeLanding after a refused landing; `trees: true`), checked once.
- Re-aimed, scenario 14 of tick-e2e.test.mjs: the deepEqual compared
baseChecks[0].startedAt with itself; it now compares it with the
retry's start, read from the clock of the tick that pushed the retry.
- Re-aimed, github.test.mjs "a hung request times out and is retried as
a network error": the stalled response is held 10 minutes instead of
10 s and the bound is half of that instead of 5 s, so machine load
cannot fail it; a client that waits for the stalled answer fails at the
test's own 2-minute timeout. What it proves is unchanged (the answer
comes from the retried request, after one 1 s timeout and one 1 s
backoff).
This commit is contained in:
@@ -684,17 +684,27 @@ test("mergePull gives up after maxRetries of unconfirmed 5xx", async (t) => {
|
||||
assert.equal(fake.pulls.get(34).merged, false);
|
||||
});
|
||||
|
||||
test("a hung request times out and is retried as a network error", async (t) => {
|
||||
// 1 s, not 100 ms: on a loaded machine the retried (fast) request must not time out too.
|
||||
const { fake, gh, sleeps, logs } = await setup(t, {}, { requestTimeoutMs: 1000 });
|
||||
fake.addPull({ number: 35, title: "slow" });
|
||||
fake.inject({ method: "GET", path: "/pulls/35", status: 200, body: {}, delayMs: 10_000 });
|
||||
const started = Date.now();
|
||||
assert.equal((await gh.getPull(35)).title, "slow");
|
||||
assert.ok(Date.now() - started < 5000, "the stalled response did not hold the client");
|
||||
assert.deepEqual(sleeps, [1000]);
|
||||
assert.ok(logs.some((l) => /timed out after 1000 ms/.test(l)));
|
||||
});
|
||||
// The stalled response is held for 10 minutes. The client times out after 1 s and asks again;
|
||||
// the answer of that second request (title "slow": the stalled one has no title) arriving in less
|
||||
// than half the stall can only come from the client giving up on the first. Nothing in this bound
|
||||
// depends on how loaded the machine is: it fails only if the client waits for the stalled answer.
|
||||
// A client that does wait fails the test at the test's own 2-minute timeout instead of 10 minutes.
|
||||
const STALL_MS = 600_000;
|
||||
test(
|
||||
"a hung request times out and is retried as a network error",
|
||||
{ timeout: 120_000 },
|
||||
async (t) => {
|
||||
// 1 s, not 100 ms: on a loaded machine the retried (fast) request must not time out too.
|
||||
const { fake, gh, sleeps, logs } = await setup(t, {}, { requestTimeoutMs: 1000 });
|
||||
fake.addPull({ number: 35, title: "slow" });
|
||||
fake.inject({ method: "GET", path: "/pulls/35", status: 200, body: {}, delayMs: STALL_MS });
|
||||
const started = Date.now();
|
||||
assert.equal((await gh.getPull(35)).title, "slow");
|
||||
assert.ok(Date.now() - started < STALL_MS / 2, "the stalled response did not hold the client");
|
||||
assert.deepEqual(sleeps, [1000]);
|
||||
assert.ok(logs.some((l) => /timed out after 1000 ms/.test(l)));
|
||||
}
|
||||
);
|
||||
|
||||
test("a request that keeps timing out fails with GitHubError status 0", async (t) => {
|
||||
const { fake, gh } = await setup(t, {}, { requestTimeoutMs: 50, maxRetries: 1 });
|
||||
|
||||
@@ -548,7 +548,11 @@ export function createFakeModules(world) {
|
||||
const labeled = pr.timeline.filter((e) => e.event === "labeled" && e.label === label);
|
||||
const labeledAt = labeled.length ? labeled[labeled.length - 1].createdAt : null;
|
||||
const inFlight = ctx.state?.inFlight?.prs?.some((p) => p.n === pr.number) ?? false;
|
||||
const processed = ctx.state?.shadow?.processed?.[String(pr.number)] === pr.headSha;
|
||||
// As eligibility.evaluateEligibility: a head settled in shadow mode is skipped in shadow
|
||||
// mode only; live mode batches it again.
|
||||
const processed =
|
||||
ctx.config.mode === "shadow" &&
|
||||
ctx.state?.shadow?.processed?.[String(pr.number)] === pr.headSha;
|
||||
// newest status of the approval context on the head (approve.mjs writes `failure` when the
|
||||
// label is removed; a re-label by a non-approver writes nothing)
|
||||
const approval = pr.headStatuses.find((st) => st.context === APPROVAL_CONTEXTS.queue);
|
||||
|
||||
@@ -322,6 +322,13 @@ export async function createQueueWorld(t, options = {}) {
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* When set, a function run once right before the tick's next git push to the base branch (the
|
||||
* merge-ff landing push), after the tick and the landing read the base tip: somebody merges in
|
||||
* that window, the race the push's lease refuses. Cleared when it runs.
|
||||
*/
|
||||
beforeBasePush: null,
|
||||
|
||||
/** One tick, built like main() (fresh runner directory every time). */
|
||||
async tick({ action = "tick" } = {}) {
|
||||
tickCount += 1;
|
||||
@@ -330,6 +337,27 @@ export async function createQueueWorld(t, options = {}) {
|
||||
{ ...env, RUNNER_TEMP: runnerTemp },
|
||||
{ now: clock.now, gitEnv, log: (line) => logs.push(line) }
|
||||
);
|
||||
// The git runners of the tick, unchanged except that a push to the base runs
|
||||
// world.beforeBasePush first (the App's and, with landing.actor "owner", the land token's).
|
||||
const baseDst = `:refs/heads/${QBASE}`;
|
||||
const hooked = (factory) =>
|
||||
factory &&
|
||||
((cwd) => {
|
||||
const git = factory(cwd);
|
||||
return {
|
||||
...git,
|
||||
run(args, opts) {
|
||||
const hook = world.beforeBasePush;
|
||||
if (hook && args[0] === "push" && args.some((a) => String(a).endsWith(baseDst))) {
|
||||
world.beforeBasePush = null;
|
||||
hook();
|
||||
}
|
||||
return git.run(args, opts);
|
||||
},
|
||||
};
|
||||
});
|
||||
deps.gitFor = hooked(deps.gitFor);
|
||||
deps.landGitFor = hooked(deps.landGitFor);
|
||||
const from = fake.requests.length;
|
||||
const result = await runTick(deps, { base: QBASE, action });
|
||||
return { result, requests: fake.requests.slice(from) };
|
||||
|
||||
@@ -276,6 +276,99 @@ scenario(
|
||||
}
|
||||
);
|
||||
|
||||
scenario(
|
||||
"scenario 2e (live): blessed, then the base moves before the landing push: the push is refused, the status withdrawn, the batch recomposed",
|
||||
async (t) => {
|
||||
const w = await createQueueWorld(t, { mode: "live" });
|
||||
const base0 = w.baseTip();
|
||||
const heads = await approvedPrs(w, [1]);
|
||||
await w.tick();
|
||||
const f = w.state().inFlight;
|
||||
w.runCi(f.candidateSha, f.candidateRef);
|
||||
// A manual merge lands between the landing's own tip read and its push.
|
||||
let moved = null;
|
||||
w.beforeBasePush = () => {
|
||||
moved = w.sb.advanceBase({ "manual.txt": "merged by hand\n" }, "manual merge");
|
||||
};
|
||||
|
||||
await w.tick();
|
||||
let state = w.state();
|
||||
assert.ok(moved, "the base moved right before the push");
|
||||
assert.equal(w.baseTip(), moved, "the push was refused: the base is the manual merge");
|
||||
assert.ok(!w.isAncestor(f.candidateSha, w.baseTip()), "the candidate did not land");
|
||||
assert.deepEqual(landStatuses(w, f.candidateSha), [
|
||||
["failure", BOT_LOGIN, `omni-queue ${f.batchId}: landing withdrawn (base-moved)`],
|
||||
["success", BOT_LOGIN, `omni-queue ${f.batchId}: checks green`],
|
||||
]);
|
||||
assert.equal(w.events("land-blessed").length, 1);
|
||||
assert.equal(w.events("landed").length, 0);
|
||||
const rejected = w.events("land-rejected");
|
||||
assert.equal(rejected.length, 1);
|
||||
assert.match(rejected[0].detail, /^base moved: .*stale info.*; recomposing$/);
|
||||
assert.equal(state.paused, null);
|
||||
assert.equal(state.inFlight.baseSha, moved, "recomposed on the moved tip");
|
||||
assert.equal(state.inFlight.stage, "testing");
|
||||
assert.notEqual(state.inFlight.candidateSha, f.candidateSha);
|
||||
assert.deepEqual(nums(state.inFlight.prs), [1]);
|
||||
assert.ok(w.isAncestor(moved, state.inFlight.candidateSha));
|
||||
assert.equal("landWithdrawals" in state, false);
|
||||
|
||||
await drain(w);
|
||||
state = w.state();
|
||||
assert.equal(w.events("landed").length, 1);
|
||||
assert.ok(w.isAncestor(heads[1], w.baseTip()), "#1 landed on the moved base");
|
||||
assert.ok(w.isAncestor(moved, w.baseTip()));
|
||||
assert.notEqual(w.baseTip(), base0);
|
||||
assert.deepEqual(landStatuses(w, f.candidateSha)[0][0], "failure", "still withdrawn");
|
||||
}
|
||||
);
|
||||
|
||||
scenario(
|
||||
"scenario 2f (live): a planted candidate with the same parents and another tree is refused by the real composition check",
|
||||
async (t) => {
|
||||
const w = await createQueueWorld(t, { mode: "live" });
|
||||
const base0 = w.baseTip();
|
||||
const heads = await approvedPrs(w, [1]);
|
||||
await w.tick();
|
||||
const state = w.state();
|
||||
const f = state.inFlight;
|
||||
const parents = w.sb.origin(["log", "-1", "--format=%P", f.candidateSha]).stdout.split(" ");
|
||||
assert.deepEqual(parents, [base0, heads[1]], "one PR: the candidate is its merge commit");
|
||||
// The same parents and the same message (trailers included) on the bare base's tree: the
|
||||
// merge commit a state edit could plant to land a tree nobody tested.
|
||||
const message = w.sb.origin(["log", "-1", "--format=%B", f.candidateSha]).stdout;
|
||||
const planted = w.sb.origin(
|
||||
["commit-tree", `${base0}^{tree}`, "-p", base0, "-p", heads[1], "-F", "-"],
|
||||
{ input: `${message}\n` }
|
||||
).stdout;
|
||||
assert.notEqual(planted, f.candidateSha);
|
||||
const ref = candidateRef(QBASE, f.batchId, planted);
|
||||
w.sb.origin(["update-ref", `refs/heads/${ref}`, planted]);
|
||||
state.inFlight = { ...f, candidateSha: planted, candidateRef: ref };
|
||||
w.fake.commitFiles({
|
||||
branch: STATE_BRANCH,
|
||||
files: { [stateFilePath(QBASE)]: `${JSON.stringify(state, null, 2)}\n` },
|
||||
message: "planted",
|
||||
});
|
||||
w.runCi(planted, ref); // its CI is green: a planted tree controls its own checks
|
||||
|
||||
await w.tick();
|
||||
const after = w.state();
|
||||
assert.deepEqual(landStatuses(w, planted), [], "no land status on the planted commit");
|
||||
assert.deepEqual(landStatuses(w, f.candidateSha), [], "nor on the composed one");
|
||||
assert.equal(w.baseTip(), base0, "nothing landed");
|
||||
assert.equal(w.events("land-blessed").length, 0);
|
||||
assert.equal(after.inFlight, null);
|
||||
assert.equal(after.paused?.reason, "error");
|
||||
assert.match(after.paused.detail, /does not match its candidate/);
|
||||
assert.ok(
|
||||
after.paused.detail.includes(`merge commit ${head12(planted)} of #1: tree `),
|
||||
after.paused.detail
|
||||
);
|
||||
assert.match(after.paused.detail, /recomposing gives [0-9a-f]{12}/);
|
||||
}
|
||||
);
|
||||
|
||||
scenario(
|
||||
"scenario 2b: a pull request opened on the candidate SHA never decides the candidate's verdict",
|
||||
async (t) => {
|
||||
@@ -1307,6 +1400,9 @@ scenario(
|
||||
assert.equal(state.inFlight.batchId, `basecheck-${head12(base0)}-r1`);
|
||||
assert.match(state.inFlight.candidateRef, baseCheckRefRe(base0, "-r1"));
|
||||
assert.equal(state.paused, null);
|
||||
// The retry's ref was pushed by this tick: its runs count from this tick's clock.
|
||||
const retryStartedAt = w.clock.now();
|
||||
assert.equal(state.inFlight.startedAt, retryStartedAt);
|
||||
|
||||
w.clock.advance(NO_RUN_TIMEOUT_MINUTES + 1); // nor on its retry
|
||||
await w.tick();
|
||||
@@ -1315,7 +1411,7 @@ scenario(
|
||||
baseSha: base0,
|
||||
ref: baseCheckRef(QBASE, base0, 1),
|
||||
status: "timeout",
|
||||
startedAt: state.baseChecks[0].startedAt,
|
||||
startedAt: retryStartedAt, // the verdict is the retry's: it starts when the retry was pushed
|
||||
signatures: [],
|
||||
});
|
||||
assert.equal(state.paused, null, "no pause on a base check without a result");
|
||||
|
||||
@@ -7486,6 +7486,66 @@ test("base-red pause whose repair lane no longer runs: a base-repair batch at th
|
||||
assert.deepEqual(h.world.lands, []);
|
||||
});
|
||||
|
||||
// ── the attempts bound and the withdrawal ──
|
||||
|
||||
test("an authentication failure on the second landing attempt pauses with the status withdrawn", async () => {
|
||||
const h = createHarness({ mode: "live", actor: "owner" });
|
||||
const f = await greenBatch(h, [1]);
|
||||
h.world.landError = notAvailable(1);
|
||||
await failingTick(h, /not available from origin/);
|
||||
assert.equal((await h.state()).inFlight.landAttempts, 1);
|
||||
h.world.landError = new Error(
|
||||
"landMergeFf: push gave no verdict: remote: Invalid username or token. | fatal: " +
|
||||
"Authentication failed for 'https://github.com/acme/widgets.git/'"
|
||||
);
|
||||
await h.tick();
|
||||
const state = await h.state();
|
||||
assert.equal(state.paused?.reason, "error");
|
||||
assert.match(state.paused.detail, /OMNI_QUEUE_LAND_TOKEN/);
|
||||
assert.equal(state.inFlight, null);
|
||||
assert.deepEqual(statusStates(h), landPairs(2), "each attempt blessed, then withdrawn");
|
||||
assert.equal(newestLand(h, f.candidateSha), "failure");
|
||||
});
|
||||
|
||||
test("the tick that hits LANDING_ATTEMPTS_LIMIT withdraws a blessing still standing", async () => {
|
||||
const h = createHarness({ mode: "live", actor: "owner" });
|
||||
const f = await greenBatch(h, [1]);
|
||||
// The third attempt's tick blessed it and died: no withdrawal ran.
|
||||
await diedAfterBlessing(h, f, [f.candidateSha]);
|
||||
await tamperInFlight(h, { landAttempts: LANDING_ATTEMPTS_LIMIT });
|
||||
await h.tick();
|
||||
const state = await h.state();
|
||||
assert.equal(state.paused?.reason, "error");
|
||||
assert.match(state.paused.detail, /ended without a verdict 3 times in a row/);
|
||||
assert.deepEqual(
|
||||
statusCalls(h).map((c) => c.args),
|
||||
[landStatus(f.batchId, f.candidateSha, { why: LEFT_LANDING })],
|
||||
"not blessed again; the standing success withdrawn"
|
||||
);
|
||||
assert.deepEqual(h.world.lands, []);
|
||||
});
|
||||
|
||||
// ── shadow → live ──
|
||||
|
||||
test("a PR that produced a would-land in shadow is batched again once the mode is live", async () => {
|
||||
const h = createHarness({ mode: "shadow" });
|
||||
const f = await greenBatch(h, [1]);
|
||||
await h.tick();
|
||||
let state = await h.state();
|
||||
assert.ok(types(state).includes("would-land"));
|
||||
assert.equal(state.shadow.processed[1], f.prs[0].head);
|
||||
await h.tick();
|
||||
assert.equal((await h.state()).inFlight, null, "shadow: the processed head is not batched again");
|
||||
|
||||
h.config.mode = "live";
|
||||
await h.tick();
|
||||
state = await h.state();
|
||||
assert.deepEqual(nums(state.inFlight?.prs ?? []), [1], "live batches it again");
|
||||
h.runCi(state.inFlight.candidateSha);
|
||||
await h.tick();
|
||||
assert.ok(types(await h.state()).includes("landed"));
|
||||
});
|
||||
|
||||
test("M1 through main(), shadow mode (the shipped config): a tick that throws after it left a stored landing withdraws the status and saves the batch at testing with its error", async (t) => {
|
||||
const q = await mainAgainstFake(t);
|
||||
const now = new Date().toISOString();
|
||||
|
||||
Reference in New Issue
Block a user