From 4eecfc76a5ebc1b327944624cef7b8c7e024b85d Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Thu, 1 Oct 2026 16:43:02 -0700 Subject: [PATCH] feat(promotion): gate promotion out of an org on a passing check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `orgs..check: ` in promotion.yml names a vapi-checks.yml check that must pass in that org before any transition promotes out of it. The gate runs the same inline check as the PR workflow, built from the source org's files at the promoted commit, using that org's key from VAPI_PROMOTION_TOKENS. - Checks are validated before any transition: the named check must exist and read and run in the gated org. - Transitions with no changes skip the gate; plan-only runs print `check would run in ( simulations × targets)` and run nothing. - On --apply the check runs after bindings refresh and before promotionPlanApply writes the target. Any non-pass (failed, incomplete, build error) throws `Promotion out of blocked: check ()`, so the target is untouched and earlier transitions are still committed (previous change). - A pass is reused for later transitions out of the same org in the same run, and dropped once a transition applies into that org. - The "Reconcile configured promotions" step gets timeout-minutes: 90 on the step, not the job, so the always() commit step still runs. - With no check: configured, promotion is unchanged: a test pins the plan output to what the pre-gate code prints, and asserts no check runs and vapi-checks.yml is never read. - Docs: promotion.example.yml and README ("Check before promoting", and a pointer from "PR Checks"). Refs TEST-141 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/promotion.yml | 6 + README.md | 30 +++ promotion.example.yml | 4 + src/promote-cmd.ts | 52 +++- src/promotion-gate.ts | 170 ++++++++++++ src/promotion.ts | 19 ++ tests/promotion-gate.test.ts | 443 ++++++++++++++++++++++++++++++++ tests/promotion.test.ts | 16 ++ 8 files changed, 738 insertions(+), 2 deletions(-) create mode 100644 src/promotion-gate.ts create mode 100644 tests/promotion-gate.test.ts diff --git a/.github/workflows/promotion.yml b/.github/workflows/promotion.yml index 5f679cd..343295d 100644 --- a/.github/workflows/promotion.yml +++ b/.github/workflows/promotion.yml @@ -56,6 +56,12 @@ jobs: - name: Reconcile configured promotions id: promotion + # A safety net only, on the step rather than the job so a timeout + # still leaves the always() commit step below time to run. Gate + # checks bound themselves (promotionGateRun's deadline), and their + # combined budget is checked against 300 minutes when the config + # loads, so this cap doesn't cut short a long ungated promotion. + timeout-minutes: 330 shell: bash run: | set -euo pipefail diff --git a/README.md b/README.md index 8d2e165..195397f 100644 --- a/README.md +++ b/README.md @@ -439,6 +439,30 @@ its cleaned state after downstream deletion completes. See [sync behavior](docs/learnings/sync-behavior.md#cross-org-promotion-deletions) for the exact lifecycle. +#### Check before promoting (optional) + +Gate an org on a [PR check](#pr-checks-simulations-against-your-branch): +nothing is promoted **out of** it unless the check passes there first. + +```yaml +# promotion.yml +orgs: + example-staging: + check: staging-core # a vapi-checks.yml check whose org (and runOrg) is example-staging +``` + +- Plans print `check would run staging-core in example-staging ( simulations × targets)` + and run nothing. +- On `--apply`, the check runs against `resources/example-staging/` at the + promoted commit, in example-staging, with that org's key from + `VAPI_PROMOTION_TOKENS`, before any file is written to the destination. A + failure, an incomplete run (timeout, billing) or a build error blocks the + transition with the run link; transitions that already applied are still + committed. +- A pass is reused for later transitions out of the same org in the same run, + until something is promoted into it. +- Transitions with no changes skip the check. + #### Rolling Back a Promotion Treat a promotion rollback as a new, auditable Git change: revert the source @@ -588,6 +612,12 @@ Require the **commit status `Vapi Evals`** in branch protection — not the status, so dispatch again after that. Running one named check by hand never changes `Vapi Evals`. +### Gate promotion on a check (optional) + +Multi-org repos can require a check to pass in an org before anything is +promoted out of it: set `orgs..check: ` in `promotion.yml` (see +[Check before promoting](#check-before-promoting-optional)). + ### Dedicated CI org (optional; recommended with `toolMocks: off`) 1. `npm run setup -- my-ci-org --resources none`. diff --git a/promotion.example.yml b/promotion.example.yml index 82bd78a..84165e0 100644 --- a/promotion.example.yml +++ b/promotion.example.yml @@ -8,6 +8,10 @@ orgs: baseUrl: https://api.vapi.ai example-staging: baseUrl: https://api.vapi.ai + # Optional gate: nothing is promoted out of example-staging unless this + # vapi-checks.yml check (whose org and runOrg are example-staging) passes + # there first. Plans print what it would run; --apply runs it. + # check: staging-core bindings: credentials: default: bind diff --git a/src/promote-cmd.ts b/src/promote-cmd.ts index b8a8ad4..a219b17 100644 --- a/src/promote-cmd.ts +++ b/src/promote-cmd.ts @@ -2,9 +2,16 @@ import { execFileSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import type { CheckDefinition } from "./check-config.ts"; import type { OrgConnection } from "./org-connection.ts"; import { childRun, connectionLoad, tokensParse } from "./org-connection.ts"; import type { PromotionConfig, PromotionPipeline } from "./promotion.ts"; +import type { PromotionGateResult } from "./promotion-gate.ts"; +import { + promotionChecksLoad, + promotionGatePlanLine, + promotionGateRun, +} from "./promotion-gate.ts"; import { promotionConfigParse, promotionPlanApply, @@ -42,6 +49,17 @@ export const APPLIED_PATHS_FILE = "tmp/promotion-applied.txt"; export interface PromotionDeps { childRun: typeof orgScriptRun; + checkRun: ( + check: CheckDefinition, + connection: OrgConnection, + ) => Promise; +} + +// Gated orgs' checks, and the orgs whose check already passed in this run. +// A pass stays valid until a transition applies into that org. +interface PromotionGates { + checks: Map; + passed: Set; } function argumentsParse(args: string[]): PromotionArguments { @@ -216,6 +234,7 @@ async function transitionRun( tokens: Map, allowEmptySourceDeletion: boolean, deps: PromotionDeps, + gates: PromotionGates, ): Promise { const run = deps.childRun; if (apply) { @@ -248,7 +267,23 @@ async function transitionRun( for (const change of plan.changes) console.log(` ${change.kind.padEnd(6)} ${change.path}`); if (plan.changes.length === 0) console.log(" no changes"); - if (!apply || plan.changes.length === 0) return false; + if (plan.changes.length === 0) return false; + const check = gates.checks.get(transition.source); + if (check && !apply) + console.log(await promotionGatePlanLine(ROOT_DIR, check)); + if (!apply) return false; + if (check && !gates.passed.has(transition.source)) { + console.log(` check running ${check.name} in ${transition.source}…`); + const result = await deps.checkRun( + check, + orgConnection(config, transition.source, tokens), + ); + if (result.outcome !== "passed") + throw new Error( + `Promotion out of ${transition.source} blocked: check ${check.name} ${result.outcome} (${result.url ?? result.reason})`, + ); + gates.passed.add(transition.source); + } await promotionPlanApply(plan); const changedPaths = plan.changes.map( (change) => `resources/${transition.target}/${change.path}`, @@ -260,18 +295,30 @@ async function transitionRun( ["--force", "--allow-new-files", "--resolve=ours", ...changedPaths], ); appliedPathsRecord(transition.target); + // The target's files just changed, so an earlier pass no longer covers it. + gates.passed.delete(transition.target); return plan.changes.some((change) => change.kind === "delete"); } export async function promotionCommandRun( args = process.argv.slice(2), - deps: PromotionDeps = { childRun: orgScriptRun }, + overrides: Partial = {}, ): Promise { + const deps: PromotionDeps = { + childRun: orgScriptRun, + checkRun: (check, connection) => + promotionGateRun(ROOT_DIR, check, connection), + ...overrides, + }; const parsed = argumentsParse(args); const configPath = resolve(ROOT_DIR, "promotion.yml"); if (!existsSync(configPath)) throw new Error("promotion.yml is required at the repository root"); const config = promotionConfigParse(readFileSync(configPath, "utf8")); + const gates: PromotionGates = { + checks: promotionChecksLoad(ROOT_DIR, config), + passed: new Set(), + }; const tokens = parsed.apply ? tokensParse(TOKENS_ENV) : new Map(); @@ -293,6 +340,7 @@ export async function promotionCommandRun( tokens, deletionAuthorizedSources.has(sourceKey), deps, + gates, ); if (deleted) deletionAuthorizedSources.add( diff --git a/src/promotion-gate.ts b/src/promotion-gate.ts new file mode 100644 index 0000000..4bd2612 --- /dev/null +++ b/src/promotion-gate.ts @@ -0,0 +1,170 @@ +// The promotion check gate: `orgs..check: ` in promotion.yml +// names a vapi-checks.yml check that must pass in that org before any +// transition promotes out of it. The gate runs the same check as the PR +// workflow, built from the source org's files at the promoted commit. + +import type { CheckDefinition } from "./check-config.ts"; +import { CHECKS_CONFIG_FILE, checksConfigLoad } from "./check-config.ts"; +import { checkJobsBuild } from "./check-build.ts"; +import type { CheckOutcome, CheckTargetResult } from "./check-run.ts"; +import { checkRunAll, MAX_CONCURRENT, MIN_START_MS } from "./check-run.ts"; +import type { OrgConnection } from "./org-connection.ts"; +import type { PromotionConfig } from "./promotion.ts"; +import { userAgentGet } from "./user-agent.ts"; + +export interface PromotionGateResult { + outcome: CheckOutcome; + reason: string; + url?: string; +} + +const DEFAULT_BASE_URL = "https://api.vapi.ai"; +// Gate checks must fit well inside the promotion step's timeout, so a check +// that can't finish is rejected when the config loads, not killed mid-run. +export const GATE_BUDGET_MINUTES = 300; +const OUTCOME_RANK: Record = { + passed: 0, + built: 0, + incomplete: 1, + failed: 2, + error: 3, +}; + +// The gated orgs' checks, validated up front so a typo fails before any +// transition applies. +export function promotionChecksLoad( + rootDir: string, + config: PromotionConfig, +): Map { + const gated = Object.entries(config.orgs).filter(([, org]) => org.check); + const checks = new Map(); + if (gated.length === 0) return checks; + const checksConfig = checksConfigLoad(rootDir); + if (!checksConfig) + throw new Error( + `promotion.yml gates ${gated.map(([slug]) => slug).join(", ")} on checks, but there is no ${CHECKS_CONFIG_FILE}`, + ); + // An org that is last in every pipeline is never promoted out of. + const sources = new Set( + Object.values(config.pipelines).flatMap((pipeline) => + pipeline.orgs.slice(0, -1), + ), + ); + let budgetMinutes = 0; + for (const [slug, org] of gated) { + if (!sources.has(slug)) + throw new Error( + `orgs.${slug}.check: nothing is promoted out of ${slug} (it is last in every pipeline), so this check would never run; gate the org before it instead`, + ); + const check = checksConfig.checks[org.check!]; + if (!check) + throw new Error( + `orgs.${slug}.check: no check named ${org.check} in ${CHECKS_CONFIG_FILE}`, + ); + if (check.org !== slug || check.runOrg !== slug) + throw new Error( + `orgs.${slug}.check: check ${check.name} must read and run in ${slug} (it reads ${check.org} and runs in ${check.runOrg})`, + ); + // A gate runs in the real org, never a CI org, so it must not reach real + // systems: no live tools, and no webhooks to the org's own servers. + if (check.toolMocks === "off") + throw new Error( + `orgs.${slug}.check: check ${check.name} sets toolMocks: off, which runs real tools; a gate runs in ${slug} itself, so it must use toolMocks: strict`, + ); + if (!check.stripWebhooks) + throw new Error( + `orgs.${slug}.check: check ${check.name} sets stripWebhooks: false, which sends simulated calls' webhooks to ${slug}'s real servers; a gate must keep the default`, + ); + // The org's token goes only to the host promotion uses for that org. + if (check.baseUrl && check.baseUrl !== org.baseUrl?.replace(/\/+$/, "")) + throw new Error( + `orgs.${slug}.check: check ${check.name} uses ${check.baseUrl}, but promotion.yml uses ${org.baseUrl ?? "the default API"} for ${slug}; set the same baseUrl in both`, + ); + budgetMinutes += gateBudgetMinutes(check); + checks.set(slug, check); + } + if (budgetMinutes > GATE_BUDGET_MINUTES) + throw new Error( + `promotion.yml's gated checks can take up to ${budgetMinutes} minutes in one run, more than the ${GATE_BUDGET_MINUTES} the promotion step allows; lower their timeoutMinutes or targets`, + ); + return checks; +} + +// The longest a gate check can run: targets run MAX_CONCURRENT at a time, +// and each batch gets a full timeoutMinutes. +export function gateBudgetMinutes( + check: Pick & { + targets: readonly unknown[]; + }, +): number { + return ( + Math.ceil(check.targets.length / MAX_CONCURRENT) * check.timeoutMinutes + + MIN_START_MS / 60_000 + ); +} + +export function gateDeadline( + check: Pick & { + targets: readonly unknown[]; + }, + now: number, +): number { + return now + gateBudgetMinutes(check) * 60_000; +} + +// Anything short of every target passing blocks: the worst target wins. +export function gateResultReduce( + results: Array>, +): PromotionGateResult { + const worst = results.reduce((a, b) => + OUTCOME_RANK[b.outcome] > OUTCOME_RANK[a.outcome] ? b : a, + ); + return { outcome: worst.outcome, reason: worst.reason, url: worst.url }; +} + +// The plan-only line: what the gate would run, built offline. +export async function promotionGatePlanLine( + rootDir: string, + check: CheckDefinition, +): Promise { + const jobs = await checkJobsBuild(rootDir, check); + const broken = jobs.find((job) => !job.result.body); + if (broken) + return ` check would run ${check.name} in ${check.org}, but its payload doesn't build: ${broken.result.errors[0]}`; + const simulations = jobs[0]?.result.body?.simulations.length ?? 0; + return ` check would run ${check.name} in ${check.org} (${simulations} simulation${simulations === 1 ? "" : "s"} × ${jobs.length} target${jobs.length === 1 ? "" : "s"})`; +} + +// Run the check live in the source org and reduce it to one result: the +// worst target wins, so anything short of every target passing blocks. +export async function promotionGateRun( + rootDir: string, + check: CheckDefinition, + connection: OrgConnection, +): Promise { + const jobs = await checkJobsBuild(rootDir, check); + const controller = new AbortController(); + const abort = () => controller.abort(); + process.once("SIGINT", abort); + process.once("SIGTERM", abort); + try { + const results = await checkRunAll({ + jobs, + connectionFor: () => ({ + token: connection.token, + baseUrl: connection.baseUrl ?? DEFAULT_BASE_URL, + userAgent: userAgentGet("check"), + }), + deadline: gateDeadline(check, Date.now()), + signal: controller.signal, + }); + for (const result of results) + console.log( + ` check ${result.job.label}: ${result.outcome} — ${result.reason}${result.url ? ` (${result.url})` : ""}`, + ); + return gateResultReduce(results); + } finally { + process.off("SIGINT", abort); + process.off("SIGTERM", abort); + } +} diff --git a/src/promotion.ts b/src/promotion.ts index 7ece0a5..eb13bae 100644 --- a/src/promotion.ts +++ b/src/promotion.ts @@ -29,6 +29,9 @@ export interface PromotionBindings { export interface PromotionOrg { baseUrl?: string; bindings: PromotionBindings; + // A vapi-checks.yml check that must pass in this org before anything is + // promoted out of it. + check?: string; } export interface PromotionPipeline { @@ -142,6 +145,8 @@ export function promotionBindingsParse(value: unknown): PromotionBindings { }; } +const PROMOTION_ORG_KEYS = ["baseUrl", "bindings", "check"]; + export function promotionConfigParse(content: string): PromotionConfig { const raw = object(parseYaml(content), "promotion.yml"); if (raw.version !== 1) throw new Error("promotion.yml version must be 1"); @@ -150,11 +155,25 @@ export function promotionConfigParse(content: string): PromotionConfig { for (const [slug, value] of Object.entries(orgsRaw)) { if (!SLUG_RE.test(slug)) throw new Error(`Invalid org slug: ${slug}`); const org = object(value ?? {}, `org ${slug}`); + // A typo (`checks:`, `Check:`) must not quietly drop a safety gate. + for (const key of Object.keys(org)) + if (!PROMOTION_ORG_KEYS.includes(key)) + throw new Error( + `org ${slug} has unknown key "${key}" (allowed: ${PROMOTION_ORG_KEYS.join(", ")})`, + ); if (org.baseUrl !== undefined && typeof org.baseUrl !== "string") throw new Error(`org ${slug}.baseUrl must be a string`); + if ( + org.check !== undefined && + (typeof org.check !== "string" || !SLUG_RE.test(org.check)) + ) + throw new Error( + `org ${slug}.check must be a check name from vapi-checks.yml`, + ); orgs[slug] = { baseUrl: typeof org.baseUrl === "string" ? org.baseUrl : undefined, bindings: promotionBindingsParse(org.bindings), + ...(typeof org.check === "string" ? { check: org.check } : {}), }; } const pipelinesRaw = object(raw.pipelines, "promotion.yml pipelines"); diff --git a/tests/promotion-gate.test.ts b/tests/promotion-gate.test.ts new file mode 100644 index 0000000..ede1899 --- /dev/null +++ b/tests/promotion-gate.test.ts @@ -0,0 +1,443 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import type { CheckDefinition } from "../src/check-config.ts"; +import type { PromotionGateResult } from "../src/promotion-gate.ts"; +import { + gateBudgetMinutes, + gateDeadline, + gateResultReduce, +} from "../src/promotion-gate.ts"; +import { promotionConfigParse } from "../src/promotion.ts"; + +// promote-cmd.ts binds its root at import: one fixture repo for the file, +// rebuilt per test. +const ROOT = mkdtempSync(join(tmpdir(), "promotion-gate-")); +process.env.VAPI_GITOPS_ROOT = ROOT; +const { promotionCommandRun } = await import("../src/promote-cmd.ts"); + +const CHECK_FILES: Record = { + "assistants/intake.yml": "name: Intake\nmodel:\n provider: openai\n", + "structuredOutputs/ok.yml": "name: ok\nschema:\n type: boolean\n", + "simulations/scenarios/s1.yml": + "name: S1\ninstructions: Hi.\nevaluations:\n - structuredOutputId: ok\n comparator: '='\n value: true\n required: true\n", + "simulations/personalities/calm.yml": "name: Calm\n", + "simulations/tests/t1.yml": "name: T1\npersonalityId: calm\nscenarioId: s1\n", +}; + +function write(path: string, content: string): void { + mkdirSync(dirname(join(ROOT, path)), { recursive: true }); + writeFileSync(join(ROOT, path), content); +} + +interface FixtureArgs { + orgs: Record; // org → check name + pipelines: Record; + files?: Record; // repo-relative + checks?: string; // vapi-checks.yml body after `checks:` +} + +function fixture(args: FixtureArgs): void { + rmSync(ROOT, { recursive: true, force: true }); + mkdirSync(ROOT, { recursive: true }); + const orgs = Object.entries(args.orgs) + .map( + ([org, check]) => ` ${org}:${check ? `\n check: ${check}` : " {}"}`, + ) + .join("\n"); + const pipelines = Object.entries(args.pipelines) + .map( + ([name, list]) => + ` ${name}:\n orgs: [${list.join(", ")}]\n resources: ['**/*']`, + ) + .join("\n"); + write( + "promotion.yml", + `version: 1\norgs:\n${orgs}\npipelines:\n${pipelines}\n`, + ); + if (args.checks !== undefined) + write("vapi-checks.yml", `version: 1\nchecks:\n${args.checks}`); + for (const org of Object.keys(args.orgs)) + write(`.vapi-state.${org}.json`, "{}\n"); + for (const [path, content] of Object.entries(args.files ?? {})) + write(path, content); + write(".gitignore", "tmp/\n.env.*\n"); + execFileSync("git", ["init", "-q", "-b", "main"], { cwd: ROOT }); +} + +const checkFor = (org: string, name = `${org}-core`) => + ` ${name}:\n org: ${org}\n targets: [assistants/intake]\n simulations: [t1]\n`; + +function filesFor(org: string): Record { + return Object.fromEntries( + Object.entries(CHECK_FILES).map(([path, content]) => [ + `resources/${org}/${path}`, + content, + ]), + ); +} + +interface Recorder { + applies: string[]; + checks: string[]; + output: string; +} + +async function promote( + args: string[], + gate: PromotionGateResult | ((check: CheckDefinition) => PromotionGateResult), +): Promise { + const recorder: Recorder = { applies: [], checks: [], output: "" }; + process.env.VAPI_PROMOTION_TOKENS = JSON.stringify({ + a: "t", + b: "t", + c: "t", + d: "t", + }); + const log = console.log; + console.log = (...parts: unknown[]) => { + recorder.output += `${parts.join(" ")}\n`; + }; + try { + await promotionCommandRun(args, { + childRun: (script, org) => { + if (script === "src/apply.ts") recorder.applies.push(org); + }, + checkRun: async (check) => { + recorder.checks.push(`${check.name}@${check.org}`); + return typeof gate === "function" ? gate(check) : gate; + }, + }); + return recorder; + } catch (error) { + return { ...recorder, error: (error as Error).message }; + } finally { + console.log = log; + } +} + +const PASS: PromotionGateResult = { + outcome: "passed", + reason: "1 of 1 simulations passed", + url: "https://run/pass", +}; +const FAIL: PromotionGateResult = { + outcome: "failed", + reason: "1 of 1 simulations failed", + url: "https://run/fail", +}; +const ONE_STEP = ["--pipeline", "release", "--from", "a", "--to", "b"]; + +test("a passing gate lets the transition apply", async () => { + fixture({ + orgs: { a: "a-core", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + checks: checkFor("a"), + }); + const result = await promote([...ONE_STEP, "--apply"], PASS); + assert.deepEqual( + [result.error, result.checks, result.applies], + [undefined, ["a-core@a"], ["b"]], + ); +}); + +test("a failed or incomplete gate blocks, naming the run, and leaves the target untouched", async () => { + const outcomes: Array<[PromotionGateResult, string]> = [ + [ + FAIL, + "Promotion out of a blocked: check a-core failed (https://run/fail)", + ], + [ + { outcome: "incomplete", reason: "timed out after 1200s; run canceled" }, + "Promotion out of a blocked: check a-core incomplete (timed out after 1200s; run canceled)", + ], + ]; + const seen = []; + for (const [gate] of outcomes) { + fixture({ + orgs: { a: "a-core", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + checks: checkFor("a"), + }); + const result = await promote([...ONE_STEP, "--apply"], gate); + seen.push([ + result.error, + result.applies, + existsSync(join(ROOT, "resources/b/assistants/intake.yml")), + ]); + } + assert.deepEqual( + seen, + outcomes.map(([, message]) => [message, [], false]), + ); +}); + +test("a plan-only run says what the gate would run and never runs it", async () => { + fixture({ + orgs: { a: "a-core", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + checks: checkFor("a"), + }); + const result = await promote(ONE_STEP, FAIL); + assert.deepEqual( + [ + result.error, + result.checks, + result.output.includes( + " check would run a-core in a (1 simulation × 1 target)", + ), + ], + [undefined, [], true], + ); +}); + +test("a transition with no changes skips the gate", async () => { + fixture({ + orgs: { a: "a-core", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: { ...filesFor("a"), ...filesFor("b") }, + checks: checkFor("a"), + }); + const result = await promote([...ONE_STEP, "--apply"], FAIL); + assert.deepEqual( + [result.error, result.checks, result.applies], + [undefined, [], []], + ); +}); + +test("gate configuration errors stop the run before anything applies", async () => { + const cases: Array<[FixtureArgs, string]> = [ + [ + { + orgs: { a: "a-core", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + }, + "promotion.yml gates a on checks, but there is no vapi-checks.yml", + ], + [ + { + orgs: { a: "nope", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + checks: checkFor("a"), + }, + "orgs.a.check: no check named nope in vapi-checks.yml", + ], + [ + { + orgs: { a: "b-core", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + checks: checkFor("b"), + }, + "orgs.a.check: check b-core must read and run in a (it reads b and runs in b)", + ], + [ + { + orgs: { a: "a-core", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + checks: `${checkFor("a")} toolMocks: off\n`, + }, + "orgs.a.check: check a-core sets toolMocks: off, which runs real tools; a gate runs in a itself, so it must use toolMocks: strict", + ], + [ + { + orgs: { a: "a-core", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + checks: `${checkFor("a")} stripWebhooks: false\n`, + }, + "orgs.a.check: check a-core sets stripWebhooks: false, which sends simulated calls' webhooks to a's real servers; a gate must keep the default", + ], + [ + { + orgs: { a: "a-core", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + checks: `${checkFor("a")} baseUrl: https://api.eu.vapi.ai\n`, + }, + "orgs.a.check: check a-core uses https://api.eu.vapi.ai, but promotion.yml uses the default API for a; set the same baseUrl in both", + ], + [ + { + // b is last in its only pipeline: nothing is ever promoted out of it. + orgs: { a: undefined, b: "b-core" }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + checks: checkFor("b"), + }, + "orgs.b.check: nothing is promoted out of b (it is last in every pipeline), so this check would never run; gate the org before it instead", + ], + [ + { + // 7 targets at 120 minutes: 3 batches, 365 minutes. + orgs: { a: "a-core", b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + checks: ` a-core:\n org: a\n timeoutMinutes: 120\n targets: [${Array.from({ length: 7 }, (_, i) => `assistants/t${i}`).join(", ")}]\n simulations: [t1]\n`, + }, + "promotion.yml's gated checks can take up to 365 minutes in one run, more than the 300 the promotion step allows; lower their timeoutMinutes or targets", + ], + ]; + const seen = []; + for (const [args] of cases) { + fixture(args); + const result = await promote([...ONE_STEP, "--apply"], PASS); + seen.push([result.error, result.applies]); + } + assert.deepEqual( + seen, + cases.map(([, message]) => [message, []]), + ); +}); + +test("a typo'd org key is rejected, so a misspelled check: can't silently drop the gate", () => { + const errors = ["checks", "Check", "gate"].map((key) => { + try { + promotionConfigParse( + `version: 1\norgs:\n a:\n ${key}: a-core\n b: {}\npipelines:\n release:\n orgs: [a, b]\n resources: ['**/*']\n`, + ); + return "parsed"; + } catch (error) { + return (error as Error).message; + } + }); + assert.deepEqual( + errors, + ["checks", "Check", "gate"].map( + (key) => + `org a has unknown key "${key}" (allowed: baseUrl, bindings, check)`, + ), + ); +}); + +test("each batch of 3 targets gets a full timeout, so a 4th target isn't squeezed", () => { + const targets = (n: number) => Array.from({ length: n }, () => ({})); + assert.deepEqual( + { + three: gateBudgetMinutes({ timeoutMinutes: 20, targets: targets(3) }), + four: gateBudgetMinutes({ timeoutMinutes: 20, targets: targets(4) }), + deadline: gateDeadline( + { timeoutMinutes: 20, targets: targets(4) }, + 1_000, + ), + }, + { three: 25, four: 45, deadline: 1_000 + 45 * 60_000 }, + ); +}); + +test("the gate's result is its worst target: error over failed over incomplete over passed", () => { + const result = (outcome: PromotionGateResult["outcome"]) => ({ + outcome, + reason: outcome, + url: `https://dashboard.vapi.ai/${outcome}`, + }); + assert.deepEqual( + [ + ["passed", "passed"], + ["passed", "incomplete"], + ["incomplete", "failed", "passed"], + ["failed", "error", "passed"], + ].map( + (outcomes) => + gateResultReduce( + outcomes.map((o) => result(o as PromotionGateResult["outcome"])), + ).outcome, + ), + ["passed", "incomplete", "failed", "error"], + ); +}); + +test("a pass is reused for the same org until something applies into it", async () => { + // p1: b→c (gate on b), p2: a→b (changes b), p3: b→d (gate on b again). + fixture({ + orgs: { a: undefined, b: "b-core", c: undefined, d: undefined }, + pipelines: { p1: ["b", "c"], p2: ["a", "b"], p3: ["b", "d"] }, + files: { + ...filesFor("b"), + ...filesFor("a"), + "resources/a/assistants/extra.yml": "name: Extra\n", + }, + checks: checkFor("b"), + }); + const reused = await promote(["--all", "--apply"], PASS); + // Two pipelines out of a gated org with nothing applied into it in between. + fixture({ + orgs: { a: "a-core", b: undefined, c: undefined }, + pipelines: { p1: ["a", "b"], p2: ["a", "c"] }, + files: filesFor("a"), + checks: checkFor("a"), + }); + const cached = await promote(["--all", "--apply"], PASS); + assert.deepEqual( + [ + reused.error, + reused.checks, + reused.applies, + cached.checks, + cached.applies, + ], + [ + undefined, + ["b-core@b", "b-core@b"], + ["c", "b", "d"], + ["a-core@a"], + ["b", "c"], + ], + ); +}); + +test.after(() => rmSync(ROOT, { recursive: true, force: true })); + +// With no `check:` in promotion.yml the gate must be invisible: the same +// plan output as before gates existed, no check run, and vapi-checks.yml +// never read (here it is invalid, so reading it would throw). +const UNGATED_PLAN_OUTPUT = [ + "", + "release: a → b", + " create assistants/intake.yml", + " create simulations/personalities/calm.yml", + " create simulations/scenarios/s1.yml", + " create simulations/tests/t1.yml", + " create structuredOutputs/ok.yml", + "", +].join("\n"); + +test("with no gate configured, promotion is unchanged: same plan output, no check, checks config never read", async () => { + const ungated: FixtureArgs = { + orgs: { a: undefined, b: undefined }, + pipelines: { release: ["a", "b"] }, + files: filesFor("a"), + }; + fixture(ungated); + write("vapi-checks.yml", "version: 999\nnot: [valid\n"); + const plan = await promote(ONE_STEP, FAIL); + fixture(ungated); + write("vapi-checks.yml", "version: 999\nnot: [valid\n"); + const applied = await promote([...ONE_STEP, "--apply"], FAIL); + assert.deepEqual( + [ + plan.error, + plan.output, + plan.checks, + applied.error, + applied.checks, + applied.applies, + ], + [undefined, UNGATED_PLAN_OUTPUT, [], undefined, [], ["b"]], + ); +}); diff --git a/tests/promotion.test.ts b/tests/promotion.test.ts index 08e418c..615efbb 100644 --- a/tests/promotion.test.ts +++ b/tests/promotion.test.ts @@ -449,3 +449,19 @@ test("promote CLI dry run uses the reviewed config without writing target files" await fx.cleanup(); } }); + +test("promotionConfigParse reads an org's optional check gate and rejects a non-slug", () => { + const config = (check: string) => + `version: 1\norgs:\n dev:\n check: ${check}\n prod: {}\npipelines:\n release:\n orgs: [dev, prod]\n resources: ['**/*']\n`; + let error = ""; + try { + promotionConfigParse(config("Not A Slug")); + } catch (caught) { + error = (caught as Error).message; + } + const parsed = promotionConfigParse(config("dev-core")); + assert.deepEqual( + [parsed.orgs.dev?.check, parsed.orgs.prod?.check, error], + ["dev-core", undefined, "org dev.check must be a check name from vapi-checks.yml"], + ); +});