mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-11-16 17:09:09 +00:00
fix(ci): an omni-queue resume clears only the pause it saw
Follow-up to the review of the pause/resume confirmation. `resume` reported success as soon as the base was unpaused, also when it already was, while its dispatch could still be pending in the tick's concurrency group and later clear a pause recorded in between (a safety pause). Now the CLI sends nothing for a base that is not paused, and a resume carries the pause it saw (`expect-pause:<since>`); the tick clears only that pause. A second dispatch of the same request is therefore harmless, which also covers a retry made after another request's run was mistaken for ours. The existing test that pinned an empty resume reason now pins the new format.
This commit is contained in:
@@ -296,6 +296,13 @@ after that confirmation. It exits `1` when the run ends any other way without ap
|
||||
(for example `skipped` while the kill switch is off), or after 20 minutes without confirmation; in
|
||||
that case check `npm run queue -- status --base <base>` and the `omni-queue` runs before retrying.
|
||||
|
||||
A `resume` names the pause it saw (`expect-pause:<since>` in the dispatch reason), and the tick
|
||||
clears only that pause. On a base that is not paused the CLI sends nothing. So a resume that
|
||||
waited behind a running tick never clears a pause recorded later (a safety pause), and sending the
|
||||
same request twice is harmless. When the targeted pause is gone but the base was paused again, the
|
||||
CLI says so and leaves it paused. A resume dispatched by hand in the Actions tab, with no reason,
|
||||
clears whatever pause is there.
|
||||
|
||||
The CLI evaluates eligibility with the local config file (`config/merge-queue/config.json`, or the
|
||||
file in OMNI_QUEUE_CONFIG, an override for the sandbox and the tests that the workflow never sets),
|
||||
so `status` shows the mode of your checkout, not necessarily the one on the default branch. `--repo <owner/repo>` points it at another repository (default: `gh repo view`).
|
||||
|
||||
@@ -28,6 +28,7 @@ import { promisify } from "node:util";
|
||||
import { DEFAULT_CONFIG_PATH, isManagedBase, loadConfig } from "./config.mjs";
|
||||
import {
|
||||
FREEZE_LABEL,
|
||||
RESUME_EXPECT_PREFIX,
|
||||
STATE_BRANCH,
|
||||
sanitizeTitle,
|
||||
stateFilePath,
|
||||
@@ -355,18 +356,31 @@ async function cmdDispatch(ctx, action) {
|
||||
const repo = (await gh.request("GET", `/repos/${gh.owner}/${gh.repo}`)).data;
|
||||
const ref = repo?.default_branch;
|
||||
if (!ref) throw new Error("could not read the default branch of the repository");
|
||||
const reason = action === "pause" ? (flags.reason ?? "paused from the CLI") : "";
|
||||
const inputs = { base, action, reason };
|
||||
const listRuns = () =>
|
||||
gh.listWorkflowFileRuns(QUEUE_WORKFLOW, { branch: ref, event: "workflow_dispatch" });
|
||||
// Watermarks taken BEFORE the dispatch, from GitHub's own data (no local clock): the newest
|
||||
// "paused" event, and the dispatch runs that already exist (none of them is ours).
|
||||
const pausedBefore = lastEventAt(await loadState(gh, base), "paused");
|
||||
// Watermarks taken BEFORE the dispatch, from GitHub's own data (no local clock): the state,
|
||||
// and the dispatch runs that already exist (none of them is ours).
|
||||
const before = await loadState(gh, base);
|
||||
if (action === "resume" && !before?.paused) {
|
||||
// Nothing is sent: a resume left pending in the tick's concurrency group would clear a
|
||||
// pause recorded later, such as a safety pause.
|
||||
out(`${base} is not paused: nothing to resume`);
|
||||
return 0;
|
||||
}
|
||||
const pausedBefore = lastEventAt(before, "paused");
|
||||
// A resume names the pause it saw, and the tick clears only that one. This also makes a
|
||||
// second dispatch of the same request harmless.
|
||||
const target = action === "resume" ? before.paused.since : null;
|
||||
const reason =
|
||||
action === "pause"
|
||||
? (flags.reason ?? "paused from the CLI")
|
||||
: `${RESUME_EXPECT_PREFIX}${target}`;
|
||||
const inputs = { base, action, reason };
|
||||
const known = new Set((await listRuns()).map((run) => run.id));
|
||||
const applied = (state) =>
|
||||
action === "pause"
|
||||
? Boolean(state?.paused) && lastEventAt(state, "paused") > pausedBefore
|
||||
: !state?.paused;
|
||||
: !state?.paused || state.paused.since !== target;
|
||||
await gh.dispatchWorkflow(QUEUE_WORKFLOW, ref, inputs);
|
||||
out(
|
||||
`${action} requested for ${base} (workflow ${QUEUE_WORKFLOW} on ${ref}); waiting for a tick to apply it`
|
||||
@@ -376,8 +390,16 @@ async function cmdDispatch(ctx, action) {
|
||||
await ctx.sleep(ctx.confirmIntervalMs);
|
||||
// Runs first, then the state: a run already seen completed has saved whatever it saved.
|
||||
const runs = (await listRuns()).filter((run) => !known.has(run.id));
|
||||
if (applied(await loadState(gh, base))) {
|
||||
out(`${action} applied to ${base}`);
|
||||
const state = await loadState(gh, base);
|
||||
if (applied(state)) {
|
||||
if (action === "resume" && state?.paused) {
|
||||
out(
|
||||
`the pause this resume targeted is gone, but ${base} was paused again ` +
|
||||
`(${state.paused.reason}, since ${state.paused.since}) and stays paused`
|
||||
);
|
||||
} else {
|
||||
out(`${action} applied to ${base}`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
const ours = runs.sort((a, b) => b.id - a.id)[0];
|
||||
|
||||
@@ -385,6 +385,13 @@ export function candidateRef(base, batchId, candidateSha) {
|
||||
return `${CANDIDATE_PREFIX}${sanitizeBase(base)}/${batchId}-${candidateSha.slice(0, 12)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reason prefix of a `resume` sent by the CLI: `expect-pause:<paused.since>`. The tick clears the
|
||||
* pause only while it is still the one the operator saw, so a resume that waited in the tick's
|
||||
* concurrency group cannot clear a newer (safety) pause.
|
||||
*/
|
||||
export const RESUME_EXPECT_PREFIX = "expect-pause:";
|
||||
|
||||
/** Batch id for a sequence number: (42) → "b42"; with an infra retry: (42, 1) → "b42-r1". */
|
||||
export function batchIdFor(seq, retry = 0) {
|
||||
if (!isCount(seq)) throw new Error(`invalid batch sequence: ${seq}`);
|
||||
|
||||
@@ -62,6 +62,7 @@ import {
|
||||
DASHBOARD_LABEL,
|
||||
DASHBOARD_TITLE_PREFIX,
|
||||
FREEZE_LABEL,
|
||||
RESUME_EXPECT_PREFIX,
|
||||
appendEvent,
|
||||
baseCheckRef,
|
||||
baseRedTitle,
|
||||
@@ -526,6 +527,17 @@ class Tick {
|
||||
this.note(`paused manually: ${detail}`);
|
||||
} else if (this.action === "resume") {
|
||||
if (!this.s.paused) return this.note("resume: the queue was not paused");
|
||||
const reason = String(this.reason ?? "");
|
||||
if (reason.startsWith(RESUME_EXPECT_PREFIX)) {
|
||||
// A resume from the CLI names the pause it saw (contracts.RESUME_EXPECT_PREFIX).
|
||||
const expected = reason.slice(RESUME_EXPECT_PREFIX.length);
|
||||
if (this.s.paused.since !== expected) {
|
||||
return this.note(
|
||||
`resume ignored: it targeted the pause of ${expected}, ` +
|
||||
`the queue is paused (${this.s.paused.reason}) since ${this.s.paused.since}`
|
||||
);
|
||||
}
|
||||
}
|
||||
const previous = this.s.paused.reason;
|
||||
this.s.paused = null;
|
||||
this.event("resumed", { detail: `manual resume (was ${previous})` });
|
||||
|
||||
@@ -63,8 +63,13 @@ async function applyAction(gh, fake, { base, action, reason }) {
|
||||
s.paused = { reason: "manual", since: at, baseSha: null, detail, issue: null };
|
||||
s.history = [...s.history, { at, type: "paused", detail }];
|
||||
} else if (action === "resume" && s.paused) {
|
||||
s.history = [...s.history, { at, type: "resumed", detail: `manual resume` }];
|
||||
s.paused = null;
|
||||
const expected = String(reason ?? "").startsWith("expect-pause:")
|
||||
? String(reason).slice("expect-pause:".length)
|
||||
: null;
|
||||
if (expected === null || s.paused.since === expected) {
|
||||
s.history = [...s.history, { at, type: "resumed", detail: `manual resume` }];
|
||||
s.paused = null;
|
||||
}
|
||||
}
|
||||
writeState(fake, s);
|
||||
}
|
||||
@@ -310,6 +315,9 @@ test("pause and resume dispatch omni-queue.yml on the default branch", async (t)
|
||||
assert.equal(await run(["pause", BASE, "--reason", "deploy window"]), 0);
|
||||
assert.equal(await run(["resume", BASE]), 0);
|
||||
assert.equal(await run(["pause", BASE]), 0);
|
||||
// A resume names the pause it saw (`expect-pause:<since>`), so the tick clears only that one.
|
||||
const resumeReason = fake.dispatches[1].inputs.reason;
|
||||
assert.match(resumeReason, /^expect-pause:\d{4}-\d{2}-\d{2}T[\d:.]+Z$/);
|
||||
assert.deepEqual(fake.dispatches, [
|
||||
{
|
||||
workflow: "omni-queue.yml",
|
||||
@@ -319,7 +327,7 @@ test("pause and resume dispatch omni-queue.yml on the default branch", async (t)
|
||||
{
|
||||
workflow: "omni-queue.yml",
|
||||
ref: "release/v3.8.54",
|
||||
inputs: { base: BASE, action: "resume", reason: "" },
|
||||
inputs: { base: BASE, action: "resume", reason: resumeReason },
|
||||
},
|
||||
{
|
||||
workflow: "omni-queue.yml",
|
||||
@@ -430,6 +438,44 @@ test("resume waits until the pause is cleared", async (t) => {
|
||||
assert.equal(lines.out.at(-1), `resume applied to ${BASE}`);
|
||||
});
|
||||
|
||||
test("resume on a base that is not paused dispatches nothing", async (t) => {
|
||||
// A resume left pending in the tick's concurrency group would later clear a pause recorded
|
||||
// after the operator looked (a safety pause): with nothing to resume, nothing is sent.
|
||||
const { run, lines, fake } = await setup(t);
|
||||
assert.equal(await run(["resume", BASE]), 0, lines.err.join("\n"));
|
||||
assert.deepEqual(fake.dispatches, []);
|
||||
assert.match(lines.out.at(-1), /is not paused: nothing to resume/);
|
||||
});
|
||||
|
||||
test("a resume names the pause it saw and never claims a newer pause was cleared", async (t) => {
|
||||
const { run, lines, fake, gh } = await setup(t);
|
||||
await applyAction(gh, fake, { base: BASE, action: "pause", reason: "x" });
|
||||
const seen = JSON.parse((await gh.getFileContent(stateFilePath(BASE), STATE_BRANCH)).content)
|
||||
.paused.since;
|
||||
let polls = 0;
|
||||
const sleep = async () => {
|
||||
polls += 1;
|
||||
if (polls === 1) {
|
||||
// Meanwhile a tick lifted that pause and recorded a newer one.
|
||||
await applyAction(gh, fake, { base: BASE, action: "resume" });
|
||||
await applyAction(gh, fake, { base: BASE, action: "pause", reason: "safety" });
|
||||
// The resume that was pending runs now: it must not touch the newer pause.
|
||||
await applyAction(gh, fake, fake.dispatches[0].inputs);
|
||||
}
|
||||
};
|
||||
const confirm = { sleep, confirmTimeoutMs: 10_000, confirmIntervalMs: 1000 };
|
||||
assert.equal(await run(["resume", BASE], confirm), 0, lines.err.join("\n"));
|
||||
assert.deepEqual(fake.dispatches[0].inputs, {
|
||||
base: BASE,
|
||||
action: "resume",
|
||||
reason: `expect-pause:${seen}`,
|
||||
});
|
||||
const state = JSON.parse((await gh.getFileContent(stateFilePath(BASE), STATE_BRANCH)).content);
|
||||
assert.equal(state.paused.detail, "safety", "the newer pause stays");
|
||||
assert.match(lines.out.at(-1), /paused again/);
|
||||
assert.doesNotMatch(lines.out.join("\n"), /resume applied/);
|
||||
});
|
||||
|
||||
test("API failure during dispatch → exit 1", async (t) => {
|
||||
const { run, lines, fake } = await setup(t);
|
||||
fake.inject({
|
||||
|
||||
@@ -1868,3 +1868,25 @@ test("a tick that changes the state still loses the compare-and-swap to a concur
|
||||
assert.deepEqual(nums(state.inFlight.prs), [1]);
|
||||
assert.ok(types(state).includes("error"), "the concurrent writer's event survived");
|
||||
});
|
||||
|
||||
test("a resume that names a pause clears only that pause", async () => {
|
||||
// The CLI sends `expect-pause:<since>` with a resume. A resume that waited in the tick's
|
||||
// concurrency group must not clear a newer pause (a safety pause recorded meanwhile).
|
||||
const h = createHarness();
|
||||
await h.tick({ action: "pause", reason: "first" });
|
||||
const first = (await h.state()).paused.since;
|
||||
await h.tick({ action: "resume", reason: `expect-pause:${first}` });
|
||||
assert.equal((await h.state()).paused, null, "the pause it named is cleared");
|
||||
|
||||
h.clock.advance(1);
|
||||
await h.tick({ action: "pause", reason: "second" });
|
||||
const second = (await h.state()).paused.since;
|
||||
assert.notEqual(second, first);
|
||||
await h.tick({ action: "resume", reason: `expect-pause:${first}` });
|
||||
const state = await h.state();
|
||||
assert.equal(state.paused?.since, second, "a stale resume leaves the newer pause alone");
|
||||
assert.equal(state.history.at(-1).type, "paused", "and records no resumed event");
|
||||
|
||||
await h.tick({ action: "resume" });
|
||||
assert.equal((await h.state()).paused, null, "a resume without a target still clears any pause");
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user