diff --git a/docs/ops/OMNI_QUEUE.md b/docs/ops/OMNI_QUEUE.md
index c11d68a7cff..251ae39424a 100644
--- a/docs/ops/OMNI_QUEUE.md
+++ b/docs/ops/OMNI_QUEUE.md
@@ -296,6 +296,13 @@ after that confirmation. It exits `1` when the run ends any other way without ap
(for example `skipped` while the kill switch is off), or after 20 minutes without confirmation; in
that case check `npm run queue -- status --base ` and the `omni-queue` runs before retrying.
+A `resume` names the pause it saw (`expect-pause:` in the dispatch reason), and the tick
+clears only that pause. On a base that is not paused the CLI sends nothing. So a resume that
+waited behind a running tick never clears a pause recorded later (a safety pause), and sending the
+same request twice is harmless. When the targeted pause is gone but the base was paused again, the
+CLI says so and leaves it paused. A resume dispatched by hand in the Actions tab, with no reason,
+clears whatever pause is there.
+
The CLI evaluates eligibility with the local config file (`config/merge-queue/config.json`, or the
file in OMNI_QUEUE_CONFIG, an override for the sandbox and the tests that the workflow never sets),
so `status` shows the mode of your checkout, not necessarily the one on the default branch. `--repo ` points it at another repository (default: `gh repo view`).
diff --git a/scripts/merge-queue/cli.mjs b/scripts/merge-queue/cli.mjs
index 1cfd99a26ec..f6d5038714a 100644
--- a/scripts/merge-queue/cli.mjs
+++ b/scripts/merge-queue/cli.mjs
@@ -28,6 +28,7 @@ import { promisify } from "node:util";
import { DEFAULT_CONFIG_PATH, isManagedBase, loadConfig } from "./config.mjs";
import {
FREEZE_LABEL,
+ RESUME_EXPECT_PREFIX,
STATE_BRANCH,
sanitizeTitle,
stateFilePath,
@@ -355,18 +356,31 @@ async function cmdDispatch(ctx, action) {
const repo = (await gh.request("GET", `/repos/${gh.owner}/${gh.repo}`)).data;
const ref = repo?.default_branch;
if (!ref) throw new Error("could not read the default branch of the repository");
- const reason = action === "pause" ? (flags.reason ?? "paused from the CLI") : "";
- const inputs = { base, action, reason };
const listRuns = () =>
gh.listWorkflowFileRuns(QUEUE_WORKFLOW, { branch: ref, event: "workflow_dispatch" });
- // Watermarks taken BEFORE the dispatch, from GitHub's own data (no local clock): the newest
- // "paused" event, and the dispatch runs that already exist (none of them is ours).
- const pausedBefore = lastEventAt(await loadState(gh, base), "paused");
+ // Watermarks taken BEFORE the dispatch, from GitHub's own data (no local clock): the state,
+ // and the dispatch runs that already exist (none of them is ours).
+ const before = await loadState(gh, base);
+ if (action === "resume" && !before?.paused) {
+ // Nothing is sent: a resume left pending in the tick's concurrency group would clear a
+ // pause recorded later, such as a safety pause.
+ out(`${base} is not paused: nothing to resume`);
+ return 0;
+ }
+ const pausedBefore = lastEventAt(before, "paused");
+ // A resume names the pause it saw, and the tick clears only that one. This also makes a
+ // second dispatch of the same request harmless.
+ const target = action === "resume" ? before.paused.since : null;
+ const reason =
+ action === "pause"
+ ? (flags.reason ?? "paused from the CLI")
+ : `${RESUME_EXPECT_PREFIX}${target}`;
+ const inputs = { base, action, reason };
const known = new Set((await listRuns()).map((run) => run.id));
const applied = (state) =>
action === "pause"
? Boolean(state?.paused) && lastEventAt(state, "paused") > pausedBefore
- : !state?.paused;
+ : !state?.paused || state.paused.since !== target;
await gh.dispatchWorkflow(QUEUE_WORKFLOW, ref, inputs);
out(
`${action} requested for ${base} (workflow ${QUEUE_WORKFLOW} on ${ref}); waiting for a tick to apply it`
@@ -376,8 +390,16 @@ async function cmdDispatch(ctx, action) {
await ctx.sleep(ctx.confirmIntervalMs);
// Runs first, then the state: a run already seen completed has saved whatever it saved.
const runs = (await listRuns()).filter((run) => !known.has(run.id));
- if (applied(await loadState(gh, base))) {
- out(`${action} applied to ${base}`);
+ const state = await loadState(gh, base);
+ if (applied(state)) {
+ if (action === "resume" && state?.paused) {
+ out(
+ `the pause this resume targeted is gone, but ${base} was paused again ` +
+ `(${state.paused.reason}, since ${state.paused.since}) and stays paused`
+ );
+ } else {
+ out(`${action} applied to ${base}`);
+ }
return 0;
}
const ours = runs.sort((a, b) => b.id - a.id)[0];
diff --git a/scripts/merge-queue/contracts.mjs b/scripts/merge-queue/contracts.mjs
index 00f39764cb3..442f41e3246 100644
--- a/scripts/merge-queue/contracts.mjs
+++ b/scripts/merge-queue/contracts.mjs
@@ -385,6 +385,13 @@ export function candidateRef(base, batchId, candidateSha) {
return `${CANDIDATE_PREFIX}${sanitizeBase(base)}/${batchId}-${candidateSha.slice(0, 12)}`;
}
+/**
+ * Reason prefix of a `resume` sent by the CLI: `expect-pause:`. The tick clears the
+ * pause only while it is still the one the operator saw, so a resume that waited in the tick's
+ * concurrency group cannot clear a newer (safety) pause.
+ */
+export const RESUME_EXPECT_PREFIX = "expect-pause:";
+
/** Batch id for a sequence number: (42) → "b42"; with an infra retry: (42, 1) → "b42-r1". */
export function batchIdFor(seq, retry = 0) {
if (!isCount(seq)) throw new Error(`invalid batch sequence: ${seq}`);
diff --git a/scripts/merge-queue/tick.mjs b/scripts/merge-queue/tick.mjs
index 019c09dddb5..ee1982af098 100644
--- a/scripts/merge-queue/tick.mjs
+++ b/scripts/merge-queue/tick.mjs
@@ -62,6 +62,7 @@ import {
DASHBOARD_LABEL,
DASHBOARD_TITLE_PREFIX,
FREEZE_LABEL,
+ RESUME_EXPECT_PREFIX,
appendEvent,
baseCheckRef,
baseRedTitle,
@@ -526,6 +527,17 @@ class Tick {
this.note(`paused manually: ${detail}`);
} else if (this.action === "resume") {
if (!this.s.paused) return this.note("resume: the queue was not paused");
+ const reason = String(this.reason ?? "");
+ if (reason.startsWith(RESUME_EXPECT_PREFIX)) {
+ // A resume from the CLI names the pause it saw (contracts.RESUME_EXPECT_PREFIX).
+ const expected = reason.slice(RESUME_EXPECT_PREFIX.length);
+ if (this.s.paused.since !== expected) {
+ return this.note(
+ `resume ignored: it targeted the pause of ${expected}, ` +
+ `the queue is paused (${this.s.paused.reason}) since ${this.s.paused.since}`
+ );
+ }
+ }
const previous = this.s.paused.reason;
this.s.paused = null;
this.event("resumed", { detail: `manual resume (was ${previous})` });
diff --git a/tests/unit/merge-queue/cli.test.mjs b/tests/unit/merge-queue/cli.test.mjs
index 522d02b61c1..94d66dd9fd0 100644
--- a/tests/unit/merge-queue/cli.test.mjs
+++ b/tests/unit/merge-queue/cli.test.mjs
@@ -63,8 +63,13 @@ async function applyAction(gh, fake, { base, action, reason }) {
s.paused = { reason: "manual", since: at, baseSha: null, detail, issue: null };
s.history = [...s.history, { at, type: "paused", detail }];
} else if (action === "resume" && s.paused) {
- s.history = [...s.history, { at, type: "resumed", detail: `manual resume` }];
- s.paused = null;
+ const expected = String(reason ?? "").startsWith("expect-pause:")
+ ? String(reason).slice("expect-pause:".length)
+ : null;
+ if (expected === null || s.paused.since === expected) {
+ s.history = [...s.history, { at, type: "resumed", detail: `manual resume` }];
+ s.paused = null;
+ }
}
writeState(fake, s);
}
@@ -310,6 +315,9 @@ test("pause and resume dispatch omni-queue.yml on the default branch", async (t)
assert.equal(await run(["pause", BASE, "--reason", "deploy window"]), 0);
assert.equal(await run(["resume", BASE]), 0);
assert.equal(await run(["pause", BASE]), 0);
+ // A resume names the pause it saw (`expect-pause:`), so the tick clears only that one.
+ const resumeReason = fake.dispatches[1].inputs.reason;
+ assert.match(resumeReason, /^expect-pause:\d{4}-\d{2}-\d{2}T[\d:.]+Z$/);
assert.deepEqual(fake.dispatches, [
{
workflow: "omni-queue.yml",
@@ -319,7 +327,7 @@ test("pause and resume dispatch omni-queue.yml on the default branch", async (t)
{
workflow: "omni-queue.yml",
ref: "release/v3.8.54",
- inputs: { base: BASE, action: "resume", reason: "" },
+ inputs: { base: BASE, action: "resume", reason: resumeReason },
},
{
workflow: "omni-queue.yml",
@@ -430,6 +438,44 @@ test("resume waits until the pause is cleared", async (t) => {
assert.equal(lines.out.at(-1), `resume applied to ${BASE}`);
});
+test("resume on a base that is not paused dispatches nothing", async (t) => {
+ // A resume left pending in the tick's concurrency group would later clear a pause recorded
+ // after the operator looked (a safety pause): with nothing to resume, nothing is sent.
+ const { run, lines, fake } = await setup(t);
+ assert.equal(await run(["resume", BASE]), 0, lines.err.join("\n"));
+ assert.deepEqual(fake.dispatches, []);
+ assert.match(lines.out.at(-1), /is not paused: nothing to resume/);
+});
+
+test("a resume names the pause it saw and never claims a newer pause was cleared", async (t) => {
+ const { run, lines, fake, gh } = await setup(t);
+ await applyAction(gh, fake, { base: BASE, action: "pause", reason: "x" });
+ const seen = JSON.parse((await gh.getFileContent(stateFilePath(BASE), STATE_BRANCH)).content)
+ .paused.since;
+ let polls = 0;
+ const sleep = async () => {
+ polls += 1;
+ if (polls === 1) {
+ // Meanwhile a tick lifted that pause and recorded a newer one.
+ await applyAction(gh, fake, { base: BASE, action: "resume" });
+ await applyAction(gh, fake, { base: BASE, action: "pause", reason: "safety" });
+ // The resume that was pending runs now: it must not touch the newer pause.
+ await applyAction(gh, fake, fake.dispatches[0].inputs);
+ }
+ };
+ const confirm = { sleep, confirmTimeoutMs: 10_000, confirmIntervalMs: 1000 };
+ assert.equal(await run(["resume", BASE], confirm), 0, lines.err.join("\n"));
+ assert.deepEqual(fake.dispatches[0].inputs, {
+ base: BASE,
+ action: "resume",
+ reason: `expect-pause:${seen}`,
+ });
+ const state = JSON.parse((await gh.getFileContent(stateFilePath(BASE), STATE_BRANCH)).content);
+ assert.equal(state.paused.detail, "safety", "the newer pause stays");
+ assert.match(lines.out.at(-1), /paused again/);
+ assert.doesNotMatch(lines.out.join("\n"), /resume applied/);
+});
+
test("API failure during dispatch → exit 1", async (t) => {
const { run, lines, fake } = await setup(t);
fake.inject({
diff --git a/tests/unit/merge-queue/tick.test.mjs b/tests/unit/merge-queue/tick.test.mjs
index 38cbc3569e6..3adb4b53d61 100644
--- a/tests/unit/merge-queue/tick.test.mjs
+++ b/tests/unit/merge-queue/tick.test.mjs
@@ -1868,3 +1868,25 @@ test("a tick that changes the state still loses the compare-and-swap to a concur
assert.deepEqual(nums(state.inFlight.prs), [1]);
assert.ok(types(state).includes("error"), "the concurrent writer's event survived");
});
+
+test("a resume that names a pause clears only that pause", async () => {
+ // The CLI sends `expect-pause:` with a resume. A resume that waited in the tick's
+ // concurrency group must not clear a newer pause (a safety pause recorded meanwhile).
+ const h = createHarness();
+ await h.tick({ action: "pause", reason: "first" });
+ const first = (await h.state()).paused.since;
+ await h.tick({ action: "resume", reason: `expect-pause:${first}` });
+ assert.equal((await h.state()).paused, null, "the pause it named is cleared");
+
+ h.clock.advance(1);
+ await h.tick({ action: "pause", reason: "second" });
+ const second = (await h.state()).paused.since;
+ assert.notEqual(second, first);
+ await h.tick({ action: "resume", reason: `expect-pause:${first}` });
+ const state = await h.state();
+ assert.equal(state.paused?.since, second, "a stale resume leaves the newer pause alone");
+ assert.equal(state.history.at(-1).type, "paused", "and records no resumed event");
+
+ await h.tick({ action: "resume" });
+ assert.equal((await h.state()).paused, null, "a resume without a target still clears any pause");
+});