fix(ci): an omni-queue resume clears only the pause it saw

Follow-up to the review of the pause/resume confirmation. `resume` reported success as soon
as the base was unpaused, also when it already was, while its dispatch could still be
pending in the tick's concurrency group and later clear a pause recorded in between (a
safety pause). Now the CLI sends nothing for a base that is not paused, and a resume carries
the pause it saw (`expect-pause:<since>`); the tick clears only that pause. A second
dispatch of the same request is therefore harmless, which also covers a retry made after
another request's run was mistaken for ours. The existing test that pinned an empty resume
reason now pins the new format.
This commit is contained in:
diegosouzapw
2026-11-14 15:50:15 -03:00
parent a3419c486d
commit 410c8db3ae
6 changed files with 127 additions and 11 deletions

View File

@@ -296,6 +296,13 @@ after that confirmation. It exits `1` when the run ends any other way without ap
(for example `skipped` while the kill switch is off), or after 20 minutes without confirmation; in
that case check `npm run queue -- status --base <base>` and the `omni-queue` runs before retrying.
A `resume` names the pause it saw (`expect-pause:<since>` in the dispatch reason), and the tick
clears only that pause. On a base that is not paused the CLI sends nothing. So a resume that
waited behind a running tick never clears a pause recorded later (a safety pause), and sending the
same request twice is harmless. When the targeted pause is gone but the base was paused again, the
CLI says so and leaves it paused. A resume dispatched by hand in the Actions tab, with no reason,
clears whatever pause is there.
The CLI evaluates eligibility with the local config file (`config/merge-queue/config.json`, or the
file in OMNI_QUEUE_CONFIG, an override for the sandbox and the tests that the workflow never sets),
so `status` shows the mode of your checkout, not necessarily the one on the default branch. `--repo <owner/repo>` points it at another repository (default: `gh repo view`).

View File

@@ -28,6 +28,7 @@ import { promisify } from "node:util";
import { DEFAULT_CONFIG_PATH, isManagedBase, loadConfig } from "./config.mjs";
import {
FREEZE_LABEL,
RESUME_EXPECT_PREFIX,
STATE_BRANCH,
sanitizeTitle,
stateFilePath,
@@ -355,18 +356,31 @@ async function cmdDispatch(ctx, action) {
const repo = (await gh.request("GET", `/repos/${gh.owner}/${gh.repo}`)).data;
const ref = repo?.default_branch;
if (!ref) throw new Error("could not read the default branch of the repository");
const reason = action === "pause" ? (flags.reason ?? "paused from the CLI") : "";
const inputs = { base, action, reason };
const listRuns = () =>
gh.listWorkflowFileRuns(QUEUE_WORKFLOW, { branch: ref, event: "workflow_dispatch" });
// Watermarks taken BEFORE the dispatch, from GitHub's own data (no local clock): the newest
// "paused" event, and the dispatch runs that already exist (none of them is ours).
const pausedBefore = lastEventAt(await loadState(gh, base), "paused");
// Watermarks taken BEFORE the dispatch, from GitHub's own data (no local clock): the state,
// and the dispatch runs that already exist (none of them is ours).
const before = await loadState(gh, base);
if (action === "resume" && !before?.paused) {
// Nothing is sent: a resume left pending in the tick's concurrency group would clear a
// pause recorded later, such as a safety pause.
out(`${base} is not paused: nothing to resume`);
return 0;
}
const pausedBefore = lastEventAt(before, "paused");
// A resume names the pause it saw, and the tick clears only that one. This also makes a
// second dispatch of the same request harmless.
const target = action === "resume" ? before.paused.since : null;
const reason =
action === "pause"
? (flags.reason ?? "paused from the CLI")
: `${RESUME_EXPECT_PREFIX}${target}`;
const inputs = { base, action, reason };
const known = new Set((await listRuns()).map((run) => run.id));
const applied = (state) =>
action === "pause"
? Boolean(state?.paused) && lastEventAt(state, "paused") > pausedBefore
: !state?.paused;
: !state?.paused || state.paused.since !== target;
await gh.dispatchWorkflow(QUEUE_WORKFLOW, ref, inputs);
out(
`${action} requested for ${base} (workflow ${QUEUE_WORKFLOW} on ${ref}); waiting for a tick to apply it`
@@ -376,8 +390,16 @@ async function cmdDispatch(ctx, action) {
await ctx.sleep(ctx.confirmIntervalMs);
// Runs first, then the state: a run already seen completed has saved whatever it saved.
const runs = (await listRuns()).filter((run) => !known.has(run.id));
if (applied(await loadState(gh, base))) {
out(`${action} applied to ${base}`);
const state = await loadState(gh, base);
if (applied(state)) {
if (action === "resume" && state?.paused) {
out(
`the pause this resume targeted is gone, but ${base} was paused again ` +
`(${state.paused.reason}, since ${state.paused.since}) and stays paused`
);
} else {
out(`${action} applied to ${base}`);
}
return 0;
}
const ours = runs.sort((a, b) => b.id - a.id)[0];

View File

@@ -385,6 +385,13 @@ export function candidateRef(base, batchId, candidateSha) {
return `${CANDIDATE_PREFIX}${sanitizeBase(base)}/${batchId}-${candidateSha.slice(0, 12)}`;
}
/**
* Reason prefix of a `resume` sent by the CLI: `expect-pause:<paused.since>`. The tick clears the
* pause only while it is still the one the operator saw, so a resume that waited in the tick's
* concurrency group cannot clear a newer (safety) pause.
*/
export const RESUME_EXPECT_PREFIX = "expect-pause:";
/** Batch id for a sequence number: (42) → "b42"; with an infra retry: (42, 1) → "b42-r1". */
export function batchIdFor(seq, retry = 0) {
if (!isCount(seq)) throw new Error(`invalid batch sequence: ${seq}`);

View File

@@ -62,6 +62,7 @@ import {
DASHBOARD_LABEL,
DASHBOARD_TITLE_PREFIX,
FREEZE_LABEL,
RESUME_EXPECT_PREFIX,
appendEvent,
baseCheckRef,
baseRedTitle,
@@ -526,6 +527,17 @@ class Tick {
this.note(`paused manually: ${detail}`);
} else if (this.action === "resume") {
if (!this.s.paused) return this.note("resume: the queue was not paused");
const reason = String(this.reason ?? "");
if (reason.startsWith(RESUME_EXPECT_PREFIX)) {
// A resume from the CLI names the pause it saw (contracts.RESUME_EXPECT_PREFIX).
const expected = reason.slice(RESUME_EXPECT_PREFIX.length);
if (this.s.paused.since !== expected) {
return this.note(
`resume ignored: it targeted the pause of ${expected}, ` +
`the queue is paused (${this.s.paused.reason}) since ${this.s.paused.since}`
);
}
}
const previous = this.s.paused.reason;
this.s.paused = null;
this.event("resumed", { detail: `manual resume (was ${previous})` });

View File

@@ -63,8 +63,13 @@ async function applyAction(gh, fake, { base, action, reason }) {
s.paused = { reason: "manual", since: at, baseSha: null, detail, issue: null };
s.history = [...s.history, { at, type: "paused", detail }];
} else if (action === "resume" && s.paused) {
s.history = [...s.history, { at, type: "resumed", detail: `manual resume` }];
s.paused = null;
const expected = String(reason ?? "").startsWith("expect-pause:")
? String(reason).slice("expect-pause:".length)
: null;
if (expected === null || s.paused.since === expected) {
s.history = [...s.history, { at, type: "resumed", detail: `manual resume` }];
s.paused = null;
}
}
writeState(fake, s);
}
@@ -310,6 +315,9 @@ test("pause and resume dispatch omni-queue.yml on the default branch", async (t)
assert.equal(await run(["pause", BASE, "--reason", "deploy window"]), 0);
assert.equal(await run(["resume", BASE]), 0);
assert.equal(await run(["pause", BASE]), 0);
// A resume names the pause it saw (`expect-pause:<since>`), so the tick clears only that one.
const resumeReason = fake.dispatches[1].inputs.reason;
assert.match(resumeReason, /^expect-pause:\d{4}-\d{2}-\d{2}T[\d:.]+Z$/);
assert.deepEqual(fake.dispatches, [
{
workflow: "omni-queue.yml",
@@ -319,7 +327,7 @@ test("pause and resume dispatch omni-queue.yml on the default branch", async (t)
{
workflow: "omni-queue.yml",
ref: "release/v3.8.54",
inputs: { base: BASE, action: "resume", reason: "" },
inputs: { base: BASE, action: "resume", reason: resumeReason },
},
{
workflow: "omni-queue.yml",
@@ -430,6 +438,44 @@ test("resume waits until the pause is cleared", async (t) => {
assert.equal(lines.out.at(-1), `resume applied to ${BASE}`);
});
test("resume on a base that is not paused dispatches nothing", async (t) => {
// A resume left pending in the tick's concurrency group would later clear a pause recorded
// after the operator looked (a safety pause): with nothing to resume, nothing is sent.
const { run, lines, fake } = await setup(t);
assert.equal(await run(["resume", BASE]), 0, lines.err.join("\n"));
assert.deepEqual(fake.dispatches, []);
assert.match(lines.out.at(-1), /is not paused: nothing to resume/);
});
test("a resume names the pause it saw and never claims a newer pause was cleared", async (t) => {
const { run, lines, fake, gh } = await setup(t);
await applyAction(gh, fake, { base: BASE, action: "pause", reason: "x" });
const seen = JSON.parse((await gh.getFileContent(stateFilePath(BASE), STATE_BRANCH)).content)
.paused.since;
let polls = 0;
const sleep = async () => {
polls += 1;
if (polls === 1) {
// Meanwhile a tick lifted that pause and recorded a newer one.
await applyAction(gh, fake, { base: BASE, action: "resume" });
await applyAction(gh, fake, { base: BASE, action: "pause", reason: "safety" });
// The resume that was pending runs now: it must not touch the newer pause.
await applyAction(gh, fake, fake.dispatches[0].inputs);
}
};
const confirm = { sleep, confirmTimeoutMs: 10_000, confirmIntervalMs: 1000 };
assert.equal(await run(["resume", BASE], confirm), 0, lines.err.join("\n"));
assert.deepEqual(fake.dispatches[0].inputs, {
base: BASE,
action: "resume",
reason: `expect-pause:${seen}`,
});
const state = JSON.parse((await gh.getFileContent(stateFilePath(BASE), STATE_BRANCH)).content);
assert.equal(state.paused.detail, "safety", "the newer pause stays");
assert.match(lines.out.at(-1), /paused again/);
assert.doesNotMatch(lines.out.join("\n"), /resume applied/);
});
test("API failure during dispatch → exit 1", async (t) => {
const { run, lines, fake } = await setup(t);
fake.inject({

View File

@@ -1868,3 +1868,25 @@ test("a tick that changes the state still loses the compare-and-swap to a concur
assert.deepEqual(nums(state.inFlight.prs), [1]);
assert.ok(types(state).includes("error"), "the concurrent writer's event survived");
});
test("a resume that names a pause clears only that pause", async () => {
// The CLI sends `expect-pause:<since>` with a resume. A resume that waited in the tick's
// concurrency group must not clear a newer pause (a safety pause recorded meanwhile).
const h = createHarness();
await h.tick({ action: "pause", reason: "first" });
const first = (await h.state()).paused.since;
await h.tick({ action: "resume", reason: `expect-pause:${first}` });
assert.equal((await h.state()).paused, null, "the pause it named is cleared");
h.clock.advance(1);
await h.tick({ action: "pause", reason: "second" });
const second = (await h.state()).paused.since;
assert.notEqual(second, first);
await h.tick({ action: "resume", reason: `expect-pause:${first}` });
const state = await h.state();
assert.equal(state.paused?.since, second, "a stale resume leaves the newer pause alone");
assert.equal(state.history.at(-1).type, "paused", "and records no resumed event");
await h.tick({ action: "resume" });
assert.equal((await h.state()).paused, null, "a resume without a target still clears any pause");
});