From 8c60cc2d6e1c50cd0ebf72a8019e153ba16510a3 Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Mon, 16 Nov 2026 10:08:35 -0300 Subject: [PATCH] feat(release): refresh-derived verifies the generated files after it writes (#20310) * feat(release): refresh-derived verifies the generated files after it writes Agreed between the phase-1 and phase-4/5 owners of the plan 2026-11-13 and approved by the owner on 2026-11-15: release:refresh-derived ends with a verify-generated step that runs check:generated-artifacts (README SVGs, e2e-timings.json) and check:agent-skills-sync (generated SKILL.md) in release mode once the refresh has written. A generated file that still differs from its generator fails the step (exit 2). The step is skipped in --check (nothing reached the disk) and when --only leaves it out (a partial refresh leaves the other artifacts alone). refreshDerived() takes an injectable runNode for the tests. * docs(changelog): number the refresh fragment after PR #20310 --- .../20310-refresh-verify-generated.md | 1 + docs/architecture/QUALITY_GATES.md | 4 +- scripts/release/refresh-derived.mjs | 47 ++++++++++++++- tests/unit/release-refresh-derived.test.ts | 57 +++++++++++++++++++ 4 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 changelog.d/maintenance/20310-refresh-verify-generated.md diff --git a/changelog.d/maintenance/20310-refresh-verify-generated.md b/changelog.d/maintenance/20310-refresh-verify-generated.md new file mode 100644 index 00000000000..12e9d04be2b --- /dev/null +++ b/changelog.d/maintenance/20310-refresh-verify-generated.md @@ -0,0 +1 @@ +- **chore(release):** `release:refresh-derived` ends with a `verify-generated` step: after it writes, `check:generated-artifacts` (README SVGs, `e2e-timings.json`) and `check:agent-skills-sync` (generated `SKILL.md`) must pass in release mode, so a refresh never leaves a generated file that differs from its generator. Skipped in `--check` and in a partial `--only` run. diff --git a/docs/architecture/QUALITY_GATES.md b/docs/architecture/QUALITY_GATES.md index 28cd4717392..7f7f9c38ebf 100644 --- a/docs/architecture/QUALITY_GATES.md +++ b/docs/architecture/QUALITY_GATES.md @@ -366,7 +366,9 @@ The commands behind it: `readme-diagrams`, `skills` (generated `SKILL.md`), `e2e-timings` (skipped until the tree ships `syncE2eTimings`), `ui-keys` (prunes keys `en.json` dropped and, with `--rewritten-since=` — default `origin/main` — re-queues English values rewritten since - the last release) and `llm-mirrors`. The + the last release), `llm-mirrors` and `verify-generated` (after a write, `check:generated-artifacts` + and `check:agent-skills-sync` in release mode must pass; skipped in `--check` and when `--only` + leaves it out; a failure is exit 2). The `changelog` step is opt-in (`--with-changelog`, release profile only). `--profile=nightly` never touches a version claim. With `-- --check` it writes nothing and exits 1 when any file would change or a managed value needs a manual edit (listing both); exit 2 on an diff --git a/scripts/release/refresh-derived.mjs b/scripts/release/refresh-derived.mjs index 8b3e1170481..32b7732a608 100644 --- a/scripts/release/refresh-derived.mjs +++ b/scripts/release/refresh-derived.mjs @@ -29,6 +29,10 @@ // skills skills/*/SKILL.md (scripts/skills/generate-agent-skills.mjs) // ui-keys missing UI + CLI locale keys (scripts/i18n/sync-ui-keys.mjs) // llm-mirrors docs/i18n/*/llm.txt copies of llm.txt (scripts/i18n/sync-llm-mirrors.mjs) +// verify-generated after a write, every generated artifact must equal its generator: +// check:generated-artifacts (README SVGs, e2e-timings.json) and +// check:agent-skills-sync (generated SKILL.md), both --mode=release. +// Skipped in --check (nothing was written) and when --only leaves it out. // // --check writes nothing: every step computes its result against an in-memory overlay, so a // later step sees what an earlier one would have written. --profile=nightly never touches the @@ -40,7 +44,8 @@ // // Exit codes: 0 done / nothing to refresh · 1 (--check) something would change, or a managed // value needs a manual edit · 2 infrastructure (facts could not be collected, a generator -// crashed, the gate exposes no planDerivedRewrites) or a usage error. +// crashed, the gate exposes no planDerivedRewrites, a generated file still differs from its +// generator after the write) or a usage error. import fs from "node:fs"; import os from "node:os"; @@ -69,6 +74,7 @@ export const STEP_IDS = [ "e2e-timings", "ui-keys", "llm-mirrors", + "verify-generated", ]; export const PROFILES = ["release", "nightly"]; const PROVIDER_REFERENCE = "docs/reference/PROVIDER_REFERENCE.md"; @@ -754,6 +760,42 @@ async function stepLlmMirrors(ctx) { } } +// Final check (plan 2026-11-13, phases 4/5): after the write, the generated artifacts must equal +// their generators, judged by the same gates CI runs in release mode. In --check nothing reached +// the disk, so the files there are still the old ones and the step is skipped. +const VERIFY_GENERATED = [ + { label: "check:generated-artifacts", script: "scripts/check/check-generated-artifacts.mjs" }, + { label: "check:agent-skills-sync", script: "scripts/skills/generate-agent-skills.mjs" }, +]; + +async function stepVerifyGenerated(ctx) { + if (ctx.check) { + return { status: "skipped", detail: "runs after a write (--check leaves the tree unchanged)" }; + } + const checks = VERIFY_GENERATED.filter((c) => fs.existsSync(path.join(ctx.root, c.script))); + if (!checks.length) + return { status: "skipped", detail: "no generated-artifact gate on this tree" }; + const run = ctx.runNode ?? runNode; + const failed = []; + for (const c of checks) { + const r = run(["--import", "tsx/esm", c.script, "--mode=release"], { cwd: ctx.root }); + if (r.failure || r.status !== 0) { + failed.push( + `${c.label} (${r.failure || `exit ${r.status}`}): ${tailOf(r.stderr || r.stdout)}` + ); + } + } + if (failed.length) { + throw new StepFailure( + `a generated file still differs from its generator after the refresh — ${failed.join("; ")}` + ); + } + return { + status: "clean", + detail: `${checks.map((c) => c.label).join(" + ")} pass (release mode)`, + }; +} + const STEP_IMPL = { changelog: stepChangelog, "provider-reference": stepProviderReference, @@ -764,6 +806,7 @@ const STEP_IMPL = { "e2e-timings": stepE2eTimings, "ui-keys": stepUiKeys, "llm-mirrors": stepLlmMirrors, + "verify-generated": stepVerifyGenerated, }; /** @@ -775,6 +818,7 @@ const STEP_IMPL = { * collectFacts?: Function, // replace the collector (tests) * planDerivedRewrites?: Function, // inject instead of the gate's export (tests) * steps?: Record, // replace a step implementation (tests) + * runNode?: Function, // replace the child-process runner (tests) * log?: (line: string) => void, * }} [opts] * @returns {Promise<{ changedFiles: string[], manual: Array<{path:string,label:string,detail:string}>, @@ -801,6 +845,7 @@ export async function refreshDerived(opts = {}) { factsError: null, collect: opts.collectFacts ?? collectFacts, planDerivedRewrites: opts.planDerivedRewrites ?? null, + runNode: opts.runNode ?? null, }; const log = opts.log ?? (() => {}); const only = opts.only ? new Set(opts.only) : null; diff --git a/tests/unit/release-refresh-derived.test.ts b/tests/unit/release-refresh-derived.test.ts index 345167f73d7..5365e91bbcf 100644 --- a/tests/unit/release-refresh-derived.test.ts +++ b/tests/unit/release-refresh-derived.test.ts @@ -628,4 +628,61 @@ describe("phase-4/5 steps", () => { assert.ok(r.changedFiles.includes("config/quality/e2e-timings.json")); assert.equal(refreshExitCode(r, { check: true }), 1); }); + + it("verify-generated: skipped in --check; after a write both gates run in release mode", async () => { + const root = tree({ + "scripts/check/check-generated-artifacts.mjs": "// stub\n", + "scripts/skills/generate-agent-skills.mjs": "// stub\n", + }); + const calls: string[][] = []; + const ok = (args: string[]) => { + calls.push(args); + return { status: 0, stdout: "", stderr: "", failure: null }; + }; + + const checked = await refreshDerived({ + root, + check: true, + only: ["verify-generated"], + runNode: ok, + }); + assert.equal(checked.steps.find((s) => s.id === "verify-generated")?.status, "skipped"); + assert.equal(calls.length, 0, "--check writes nothing, so nothing is verified"); + + const written = await refreshDerived({ root, only: ["verify-generated"], runNode: ok }); + assert.equal(written.steps.find((s) => s.id === "verify-generated")?.status, "clean"); + assert.deepEqual(calls, [ + ["--import", "tsx/esm", "scripts/check/check-generated-artifacts.mjs", "--mode=release"], + ["--import", "tsx/esm", "scripts/skills/generate-agent-skills.mjs", "--mode=release"], + ]); + assert.equal(refreshExitCode(written, { check: false }), 0); + + const stale = await refreshDerived({ + root, + only: ["verify-generated"], + runNode: (args: string[]) => + args[2].includes("generated-artifacts") + ? { + status: 1, + stdout: "", + stderr: "docs/diagrams/readme-hero.svg is stale", + failure: null, + } + : { status: 0, stdout: "", stderr: "", failure: null }, + }); + const step = stale.steps.find((s) => s.id === "verify-generated")!; + assert.equal(step.status, "failed"); + assert.match( + step.detail, + /check:generated-artifacts \(exit 1\): docs\/diagrams\/readme-hero\.svg is stale/ + ); + assert.equal(refreshExitCode(stale, { check: false }), 2); + + const partial = await refreshDerived({ root, only: ["readme-diagrams"], runNode: ok }); + assert.equal( + partial.steps.find((s) => s.id === "verify-generated")?.status, + "skipped", + "a partial refresh (--only) leaves the other artifacts alone, so it is not verified" + ); + }); });