feat(release): refresh-derived verifies the generated files after it writes (#20310)

* feat(release): refresh-derived verifies the generated files after it writes

Agreed between the phase-1 and phase-4/5 owners of the plan 2026-11-13 and approved by the owner on 2026-11-15: release:refresh-derived ends with a verify-generated step that runs check:generated-artifacts (README SVGs, e2e-timings.json) and check:agent-skills-sync (generated SKILL.md) in release mode once the refresh has written. A generated file that still differs from its generator fails the step (exit 2). The step is skipped in --check (nothing reached the disk) and when --only leaves it out (a partial refresh leaves the other artifacts alone). refreshDerived() takes an injectable runNode for the tests.

* docs(changelog): number the refresh fragment after PR #20310
This commit is contained in:
Diego Rodrigues de Sa e Souza
2026-11-16 10:08:35 -03:00
committed by GitHub
parent 3a50632917
commit 8c60cc2d6e
4 changed files with 107 additions and 2 deletions

View File

@@ -0,0 +1 @@
- **chore(release):** `release:refresh-derived` ends with a `verify-generated` step: after it writes, `check:generated-artifacts` (README SVGs, `e2e-timings.json`) and `check:agent-skills-sync` (generated `SKILL.md`) must pass in release mode, so a refresh never leaves a generated file that differs from its generator. Skipped in `--check` and in a partial `--only` run.

View File

@@ -366,7 +366,9 @@ The commands behind it:
`readme-diagrams`, `skills` (generated `SKILL.md`), `e2e-timings` (skipped until the tree
ships `syncE2eTimings`), `ui-keys` (prunes keys `en.json` dropped and, with
`--rewritten-since=<ref>` — default `origin/main` — re-queues English values rewritten since
the last release) and `llm-mirrors`. The
the last release), `llm-mirrors` and `verify-generated` (after a write, `check:generated-artifacts`
and `check:agent-skills-sync` in release mode must pass; skipped in `--check` and when `--only`
leaves it out; a failure is exit 2). The
`changelog` step is opt-in (`--with-changelog`, release profile only). `--profile=nightly`
never touches a version claim. With `-- --check` it writes nothing and exits 1 when any file
would change or a managed value needs a manual edit (listing both); exit 2 on an

View File

@@ -29,6 +29,10 @@
// skills skills/*/SKILL.md (scripts/skills/generate-agent-skills.mjs)
// ui-keys missing UI + CLI locale keys (scripts/i18n/sync-ui-keys.mjs)
// llm-mirrors docs/i18n/*/llm.txt copies of llm.txt (scripts/i18n/sync-llm-mirrors.mjs)
// verify-generated after a write, every generated artifact must equal its generator:
// check:generated-artifacts (README SVGs, e2e-timings.json) and
// check:agent-skills-sync (generated SKILL.md), both --mode=release.
// Skipped in --check (nothing was written) and when --only leaves it out.
//
// --check writes nothing: every step computes its result against an in-memory overlay, so a
// later step sees what an earlier one would have written. --profile=nightly never touches the
@@ -40,7 +44,8 @@
//
// Exit codes: 0 done / nothing to refresh · 1 (--check) something would change, or a managed
// value needs a manual edit · 2 infrastructure (facts could not be collected, a generator
// crashed, the gate exposes no planDerivedRewrites) or a usage error.
// crashed, the gate exposes no planDerivedRewrites, a generated file still differs from its
// generator after the write) or a usage error.
import fs from "node:fs";
import os from "node:os";
@@ -69,6 +74,7 @@ export const STEP_IDS = [
"e2e-timings",
"ui-keys",
"llm-mirrors",
"verify-generated",
];
export const PROFILES = ["release", "nightly"];
const PROVIDER_REFERENCE = "docs/reference/PROVIDER_REFERENCE.md";
@@ -754,6 +760,42 @@ async function stepLlmMirrors(ctx) {
}
}
// Final check (plan 2026-11-13, phases 4/5): after the write, the generated artifacts must equal
// their generators, judged by the same gates CI runs in release mode. In --check nothing reached
// the disk, so the files there are still the old ones and the step is skipped.
const VERIFY_GENERATED = [
{ label: "check:generated-artifacts", script: "scripts/check/check-generated-artifacts.mjs" },
{ label: "check:agent-skills-sync", script: "scripts/skills/generate-agent-skills.mjs" },
];
async function stepVerifyGenerated(ctx) {
if (ctx.check) {
return { status: "skipped", detail: "runs after a write (--check leaves the tree unchanged)" };
}
const checks = VERIFY_GENERATED.filter((c) => fs.existsSync(path.join(ctx.root, c.script)));
if (!checks.length)
return { status: "skipped", detail: "no generated-artifact gate on this tree" };
const run = ctx.runNode ?? runNode;
const failed = [];
for (const c of checks) {
const r = run(["--import", "tsx/esm", c.script, "--mode=release"], { cwd: ctx.root });
if (r.failure || r.status !== 0) {
failed.push(
`${c.label} (${r.failure || `exit ${r.status}`}): ${tailOf(r.stderr || r.stdout)}`
);
}
}
if (failed.length) {
throw new StepFailure(
`a generated file still differs from its generator after the refresh — ${failed.join("; ")}`
);
}
return {
status: "clean",
detail: `${checks.map((c) => c.label).join(" + ")} pass (release mode)`,
};
}
const STEP_IMPL = {
changelog: stepChangelog,
"provider-reference": stepProviderReference,
@@ -764,6 +806,7 @@ const STEP_IMPL = {
"e2e-timings": stepE2eTimings,
"ui-keys": stepUiKeys,
"llm-mirrors": stepLlmMirrors,
"verify-generated": stepVerifyGenerated,
};
/**
@@ -775,6 +818,7 @@ const STEP_IMPL = {
* collectFacts?: Function, // replace the collector (tests)
* planDerivedRewrites?: Function, // inject instead of the gate's export (tests)
* steps?: Record<string, Function>, // replace a step implementation (tests)
* runNode?: Function, // replace the child-process runner (tests)
* log?: (line: string) => void,
* }} [opts]
* @returns {Promise<{ changedFiles: string[], manual: Array<{path:string,label:string,detail:string}>,
@@ -801,6 +845,7 @@ export async function refreshDerived(opts = {}) {
factsError: null,
collect: opts.collectFacts ?? collectFacts,
planDerivedRewrites: opts.planDerivedRewrites ?? null,
runNode: opts.runNode ?? null,
};
const log = opts.log ?? (() => {});
const only = opts.only ? new Set(opts.only) : null;

View File

@@ -628,4 +628,61 @@ describe("phase-4/5 steps", () => {
assert.ok(r.changedFiles.includes("config/quality/e2e-timings.json"));
assert.equal(refreshExitCode(r, { check: true }), 1);
});
it("verify-generated: skipped in --check; after a write both gates run in release mode", async () => {
const root = tree({
"scripts/check/check-generated-artifacts.mjs": "// stub\n",
"scripts/skills/generate-agent-skills.mjs": "// stub\n",
});
const calls: string[][] = [];
const ok = (args: string[]) => {
calls.push(args);
return { status: 0, stdout: "", stderr: "", failure: null };
};
const checked = await refreshDerived({
root,
check: true,
only: ["verify-generated"],
runNode: ok,
});
assert.equal(checked.steps.find((s) => s.id === "verify-generated")?.status, "skipped");
assert.equal(calls.length, 0, "--check writes nothing, so nothing is verified");
const written = await refreshDerived({ root, only: ["verify-generated"], runNode: ok });
assert.equal(written.steps.find((s) => s.id === "verify-generated")?.status, "clean");
assert.deepEqual(calls, [
["--import", "tsx/esm", "scripts/check/check-generated-artifacts.mjs", "--mode=release"],
["--import", "tsx/esm", "scripts/skills/generate-agent-skills.mjs", "--mode=release"],
]);
assert.equal(refreshExitCode(written, { check: false }), 0);
const stale = await refreshDerived({
root,
only: ["verify-generated"],
runNode: (args: string[]) =>
args[2].includes("generated-artifacts")
? {
status: 1,
stdout: "",
stderr: "docs/diagrams/readme-hero.svg is stale",
failure: null,
}
: { status: 0, stdout: "", stderr: "", failure: null },
});
const step = stale.steps.find((s) => s.id === "verify-generated")!;
assert.equal(step.status, "failed");
assert.match(
step.detail,
/check:generated-artifacts \(exit 1\): docs\/diagrams\/readme-hero\.svg is stale/
);
assert.equal(refreshExitCode(stale, { check: false }), 2);
const partial = await refreshDerived({ root, only: ["readme-diagrams"], runNode: ok });
assert.equal(
partial.steps.find((s) => s.id === "verify-generated")?.status,
"skipped",
"a partial refresh (--only) leaves the other artifacts alone, so it is not verified"
);
});
});