diff --git a/docs/ops/OMNI_QUEUE.md b/docs/ops/OMNI_QUEUE.md index c11d68a7cff..251ae39424a 100644 --- a/docs/ops/OMNI_QUEUE.md +++ b/docs/ops/OMNI_QUEUE.md @@ -296,6 +296,13 @@ after that confirmation. It exits `1` when the run ends any other way without ap (for example `skipped` while the kill switch is off), or after 20 minutes without confirmation; in that case check `npm run queue -- status --base ` and the `omni-queue` runs before retrying. +A `resume` names the pause it saw (`expect-pause:` in the dispatch reason), and the tick +clears only that pause. On a base that is not paused the CLI sends nothing. So a resume that +waited behind a running tick never clears a pause recorded later (a safety pause), and sending the +same request twice is harmless. When the targeted pause is gone but the base was paused again, the +CLI says so and leaves it paused. A resume dispatched by hand in the Actions tab, with no reason, +clears whatever pause is there. + The CLI evaluates eligibility with the local config file (`config/merge-queue/config.json`, or the file in OMNI_QUEUE_CONFIG, an override for the sandbox and the tests that the workflow never sets), so `status` shows the mode of your checkout, not necessarily the one on the default branch. `--repo ` points it at another repository (default: `gh repo view`). diff --git a/scripts/merge-queue/cli.mjs b/scripts/merge-queue/cli.mjs index 1cfd99a26ec..f6d5038714a 100644 --- a/scripts/merge-queue/cli.mjs +++ b/scripts/merge-queue/cli.mjs @@ -28,6 +28,7 @@ import { promisify } from "node:util"; import { DEFAULT_CONFIG_PATH, isManagedBase, loadConfig } from "./config.mjs"; import { FREEZE_LABEL, + RESUME_EXPECT_PREFIX, STATE_BRANCH, sanitizeTitle, stateFilePath, @@ -355,18 +356,31 @@ async function cmdDispatch(ctx, action) { const repo = (await gh.request("GET", `/repos/${gh.owner}/${gh.repo}`)).data; const ref = repo?.default_branch; if (!ref) throw new Error("could not read the default branch of the repository"); - const reason = action === "pause" ? (flags.reason ?? "paused from the CLI") : ""; - const inputs = { base, action, reason }; const listRuns = () => gh.listWorkflowFileRuns(QUEUE_WORKFLOW, { branch: ref, event: "workflow_dispatch" }); - // Watermarks taken BEFORE the dispatch, from GitHub's own data (no local clock): the newest - // "paused" event, and the dispatch runs that already exist (none of them is ours). - const pausedBefore = lastEventAt(await loadState(gh, base), "paused"); + // Watermarks taken BEFORE the dispatch, from GitHub's own data (no local clock): the state, + // and the dispatch runs that already exist (none of them is ours). + const before = await loadState(gh, base); + if (action === "resume" && !before?.paused) { + // Nothing is sent: a resume left pending in the tick's concurrency group would clear a + // pause recorded later, such as a safety pause. + out(`${base} is not paused: nothing to resume`); + return 0; + } + const pausedBefore = lastEventAt(before, "paused"); + // A resume names the pause it saw, and the tick clears only that one. This also makes a + // second dispatch of the same request harmless. + const target = action === "resume" ? before.paused.since : null; + const reason = + action === "pause" + ? (flags.reason ?? "paused from the CLI") + : `${RESUME_EXPECT_PREFIX}${target}`; + const inputs = { base, action, reason }; const known = new Set((await listRuns()).map((run) => run.id)); const applied = (state) => action === "pause" ? Boolean(state?.paused) && lastEventAt(state, "paused") > pausedBefore - : !state?.paused; + : !state?.paused || state.paused.since !== target; await gh.dispatchWorkflow(QUEUE_WORKFLOW, ref, inputs); out( `${action} requested for ${base} (workflow ${QUEUE_WORKFLOW} on ${ref}); waiting for a tick to apply it` @@ -376,8 +390,16 @@ async function cmdDispatch(ctx, action) { await ctx.sleep(ctx.confirmIntervalMs); // Runs first, then the state: a run already seen completed has saved whatever it saved. const runs = (await listRuns()).filter((run) => !known.has(run.id)); - if (applied(await loadState(gh, base))) { - out(`${action} applied to ${base}`); + const state = await loadState(gh, base); + if (applied(state)) { + if (action === "resume" && state?.paused) { + out( + `the pause this resume targeted is gone, but ${base} was paused again ` + + `(${state.paused.reason}, since ${state.paused.since}) and stays paused` + ); + } else { + out(`${action} applied to ${base}`); + } return 0; } const ours = runs.sort((a, b) => b.id - a.id)[0]; diff --git a/scripts/merge-queue/contracts.mjs b/scripts/merge-queue/contracts.mjs index 00f39764cb3..442f41e3246 100644 --- a/scripts/merge-queue/contracts.mjs +++ b/scripts/merge-queue/contracts.mjs @@ -385,6 +385,13 @@ export function candidateRef(base, batchId, candidateSha) { return `${CANDIDATE_PREFIX}${sanitizeBase(base)}/${batchId}-${candidateSha.slice(0, 12)}`; } +/** + * Reason prefix of a `resume` sent by the CLI: `expect-pause:`. The tick clears the + * pause only while it is still the one the operator saw, so a resume that waited in the tick's + * concurrency group cannot clear a newer (safety) pause. + */ +export const RESUME_EXPECT_PREFIX = "expect-pause:"; + /** Batch id for a sequence number: (42) → "b42"; with an infra retry: (42, 1) → "b42-r1". */ export function batchIdFor(seq, retry = 0) { if (!isCount(seq)) throw new Error(`invalid batch sequence: ${seq}`); diff --git a/scripts/merge-queue/tick.mjs b/scripts/merge-queue/tick.mjs index 019c09dddb5..ee1982af098 100644 --- a/scripts/merge-queue/tick.mjs +++ b/scripts/merge-queue/tick.mjs @@ -62,6 +62,7 @@ import { DASHBOARD_LABEL, DASHBOARD_TITLE_PREFIX, FREEZE_LABEL, + RESUME_EXPECT_PREFIX, appendEvent, baseCheckRef, baseRedTitle, @@ -526,6 +527,17 @@ class Tick { this.note(`paused manually: ${detail}`); } else if (this.action === "resume") { if (!this.s.paused) return this.note("resume: the queue was not paused"); + const reason = String(this.reason ?? ""); + if (reason.startsWith(RESUME_EXPECT_PREFIX)) { + // A resume from the CLI names the pause it saw (contracts.RESUME_EXPECT_PREFIX). + const expected = reason.slice(RESUME_EXPECT_PREFIX.length); + if (this.s.paused.since !== expected) { + return this.note( + `resume ignored: it targeted the pause of ${expected}, ` + + `the queue is paused (${this.s.paused.reason}) since ${this.s.paused.since}` + ); + } + } const previous = this.s.paused.reason; this.s.paused = null; this.event("resumed", { detail: `manual resume (was ${previous})` }); diff --git a/tests/unit/merge-queue/cli.test.mjs b/tests/unit/merge-queue/cli.test.mjs index 522d02b61c1..94d66dd9fd0 100644 --- a/tests/unit/merge-queue/cli.test.mjs +++ b/tests/unit/merge-queue/cli.test.mjs @@ -63,8 +63,13 @@ async function applyAction(gh, fake, { base, action, reason }) { s.paused = { reason: "manual", since: at, baseSha: null, detail, issue: null }; s.history = [...s.history, { at, type: "paused", detail }]; } else if (action === "resume" && s.paused) { - s.history = [...s.history, { at, type: "resumed", detail: `manual resume` }]; - s.paused = null; + const expected = String(reason ?? "").startsWith("expect-pause:") + ? String(reason).slice("expect-pause:".length) + : null; + if (expected === null || s.paused.since === expected) { + s.history = [...s.history, { at, type: "resumed", detail: `manual resume` }]; + s.paused = null; + } } writeState(fake, s); } @@ -310,6 +315,9 @@ test("pause and resume dispatch omni-queue.yml on the default branch", async (t) assert.equal(await run(["pause", BASE, "--reason", "deploy window"]), 0); assert.equal(await run(["resume", BASE]), 0); assert.equal(await run(["pause", BASE]), 0); + // A resume names the pause it saw (`expect-pause:`), so the tick clears only that one. + const resumeReason = fake.dispatches[1].inputs.reason; + assert.match(resumeReason, /^expect-pause:\d{4}-\d{2}-\d{2}T[\d:.]+Z$/); assert.deepEqual(fake.dispatches, [ { workflow: "omni-queue.yml", @@ -319,7 +327,7 @@ test("pause and resume dispatch omni-queue.yml on the default branch", async (t) { workflow: "omni-queue.yml", ref: "release/v3.8.54", - inputs: { base: BASE, action: "resume", reason: "" }, + inputs: { base: BASE, action: "resume", reason: resumeReason }, }, { workflow: "omni-queue.yml", @@ -430,6 +438,44 @@ test("resume waits until the pause is cleared", async (t) => { assert.equal(lines.out.at(-1), `resume applied to ${BASE}`); }); +test("resume on a base that is not paused dispatches nothing", async (t) => { + // A resume left pending in the tick's concurrency group would later clear a pause recorded + // after the operator looked (a safety pause): with nothing to resume, nothing is sent. + const { run, lines, fake } = await setup(t); + assert.equal(await run(["resume", BASE]), 0, lines.err.join("\n")); + assert.deepEqual(fake.dispatches, []); + assert.match(lines.out.at(-1), /is not paused: nothing to resume/); +}); + +test("a resume names the pause it saw and never claims a newer pause was cleared", async (t) => { + const { run, lines, fake, gh } = await setup(t); + await applyAction(gh, fake, { base: BASE, action: "pause", reason: "x" }); + const seen = JSON.parse((await gh.getFileContent(stateFilePath(BASE), STATE_BRANCH)).content) + .paused.since; + let polls = 0; + const sleep = async () => { + polls += 1; + if (polls === 1) { + // Meanwhile a tick lifted that pause and recorded a newer one. + await applyAction(gh, fake, { base: BASE, action: "resume" }); + await applyAction(gh, fake, { base: BASE, action: "pause", reason: "safety" }); + // The resume that was pending runs now: it must not touch the newer pause. + await applyAction(gh, fake, fake.dispatches[0].inputs); + } + }; + const confirm = { sleep, confirmTimeoutMs: 10_000, confirmIntervalMs: 1000 }; + assert.equal(await run(["resume", BASE], confirm), 0, lines.err.join("\n")); + assert.deepEqual(fake.dispatches[0].inputs, { + base: BASE, + action: "resume", + reason: `expect-pause:${seen}`, + }); + const state = JSON.parse((await gh.getFileContent(stateFilePath(BASE), STATE_BRANCH)).content); + assert.equal(state.paused.detail, "safety", "the newer pause stays"); + assert.match(lines.out.at(-1), /paused again/); + assert.doesNotMatch(lines.out.join("\n"), /resume applied/); +}); + test("API failure during dispatch → exit 1", async (t) => { const { run, lines, fake } = await setup(t); fake.inject({ diff --git a/tests/unit/merge-queue/tick.test.mjs b/tests/unit/merge-queue/tick.test.mjs index 38cbc3569e6..3adb4b53d61 100644 --- a/tests/unit/merge-queue/tick.test.mjs +++ b/tests/unit/merge-queue/tick.test.mjs @@ -1868,3 +1868,25 @@ test("a tick that changes the state still loses the compare-and-swap to a concur assert.deepEqual(nums(state.inFlight.prs), [1]); assert.ok(types(state).includes("error"), "the concurrent writer's event survived"); }); + +test("a resume that names a pause clears only that pause", async () => { + // The CLI sends `expect-pause:` with a resume. A resume that waited in the tick's + // concurrency group must not clear a newer pause (a safety pause recorded meanwhile). + const h = createHarness(); + await h.tick({ action: "pause", reason: "first" }); + const first = (await h.state()).paused.since; + await h.tick({ action: "resume", reason: `expect-pause:${first}` }); + assert.equal((await h.state()).paused, null, "the pause it named is cleared"); + + h.clock.advance(1); + await h.tick({ action: "pause", reason: "second" }); + const second = (await h.state()).paused.since; + assert.notEqual(second, first); + await h.tick({ action: "resume", reason: `expect-pause:${first}` }); + const state = await h.state(); + assert.equal(state.paused?.since, second, "a stale resume leaves the newer pause alone"); + assert.equal(state.history.at(-1).type, "paused", "and records no resumed event"); + + await h.tick({ action: "resume" }); + assert.equal((await h.state()).paused, null, "a resume without a target still clears any pause"); +});