feat(promotion): gate promotion out of an org on a passing check

.github/workflows/promotion.yml

@@ -56,6 +56,12 @@ jobs:
 
       - name: Reconcile configured promotions
         id: promotion
+        # A safety net only, on the step rather than the job so a timeout
+        # still leaves the always() commit step below time to run. Gate
+        # checks bound themselves (promotionGateRun's deadline), and their
+        # combined budget is checked against 300 minutes when the config
+        # loads, so this cap doesn't cut short a long ungated promotion.
+        timeout-minutes: 330
         shell: bash
         run: |
           set -euo pipefail

README.md

@@ -439,6 +439,30 @@ its cleaned state after downstream deletion completes.
 See [sync behavior](docs/learnings/sync-behavior.md#cross-org-promotion-deletions)
 for the exact lifecycle.
 
+#### Check before promoting (optional)
+
+Gate an org on a [PR check](#pr-checks-simulations-against-your-branch):
+nothing is promoted **out of** it unless the check passes there first.
+
+```yaml
+# promotion.yml
+orgs:
+  example-staging:
+    check: staging-core   # a vapi-checks.yml check whose org (and runOrg) is example-staging
+```
+
+- Plans print `check  would run staging-core in example-staging (<n> simulations × <t> targets)`
+  and run nothing.
+- On `--apply`, the check runs against `resources/example-staging/` at the
+  promoted commit, in example-staging, with that org's key from
+  `VAPI_PROMOTION_TOKENS`, before any file is written to the destination. A
+  failure, an incomplete run (timeout, billing) or a build error blocks the
+  transition with the run link; transitions that already applied are still
+  committed.
+- A pass is reused for later transitions out of the same org in the same run,
+  until something is promoted into it.
+- Transitions with no changes skip the check.
+
 #### Rolling Back a Promotion
 
 Treat a promotion rollback as a new, auditable Git change: revert the source
@@ -588,6 +612,12 @@ Require the **commit status `Vapi Evals`** in branch protection — not the
   status, so dispatch again after that. Running one named check by hand
   never changes `Vapi Evals`.
 
+### Gate promotion on a check (optional)
+
+Multi-org repos can require a check to pass in an org before anything is
+promoted out of it: set `orgs.<org>.check: <name>` in `promotion.yml` (see
+[Check before promoting](#check-before-promoting-optional)).
+
 ### Dedicated CI org (optional; recommended with `toolMocks: off`)
 
 1. `npm run setup -- my-ci-org --resources none`.

promotion.example.yml

@@ -8,6 +8,10 @@ orgs:
     baseUrl: https://api.vapi.ai
   example-staging:
     baseUrl: https://api.vapi.ai
+    # Optional gate: nothing is promoted out of example-staging unless this
+    # vapi-checks.yml check (whose org and runOrg are example-staging) passes
+    # there first. Plans print what it would run; --apply runs it.
+    # check: staging-core
     bindings:
       credentials:
         default: bind

src/promote-cmd.ts

@@ -2,9 +2,16 @@ import { execFileSync } from "node:child_process";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
+import type { CheckDefinition } from "./check-config.ts";
 import type { OrgConnection } from "./org-connection.ts";
 import { childRun, connectionLoad, tokensParse } from "./org-connection.ts";
 import type { PromotionConfig, PromotionPipeline } from "./promotion.ts";
+import type { PromotionGateResult } from "./promotion-gate.ts";
+import {
+  promotionChecksLoad,
+  promotionGatePlanLine,
+  promotionGateRun,
+} from "./promotion-gate.ts";
 import {
   promotionConfigParse,
   promotionPlanApply,
@@ -42,6 +49,17 @@ export const APPLIED_PATHS_FILE = "tmp/promotion-applied.txt";
 
 export interface PromotionDeps {
   childRun: typeof orgScriptRun;
+  checkRun: (
+    check: CheckDefinition,
+    connection: OrgConnection,
+  ) => Promise<PromotionGateResult>;
+}
+
+// Gated orgs' checks, and the orgs whose check already passed in this run.
+// A pass stays valid until a transition applies into that org.
+interface PromotionGates {
+  checks: Map<string, CheckDefinition>;
+  passed: Set<string>;
 }
 
 function argumentsParse(args: string[]): PromotionArguments {
@@ -216,6 +234,7 @@ async function transitionRun(
   tokens: Map<string, string>,
   allowEmptySourceDeletion: boolean,
   deps: PromotionDeps,
+  gates: PromotionGates,
 ): Promise<boolean> {
   const run = deps.childRun;
   if (apply) {
@@ -248,7 +267,23 @@ async function transitionRun(
   for (const change of plan.changes)
     console.log(`  ${change.kind.padEnd(6)} ${change.path}`);
   if (plan.changes.length === 0) console.log("  no changes");
-  if (!apply || plan.changes.length === 0) return false;
+  if (plan.changes.length === 0) return false;
+  const check = gates.checks.get(transition.source);
+  if (check && !apply)
+    console.log(await promotionGatePlanLine(ROOT_DIR, check));
+  if (!apply) return false;
+  if (check && !gates.passed.has(transition.source)) {
+    console.log(`  check  running ${check.name} in ${transition.source}…`);
+    const result = await deps.checkRun(
+      check,
+      orgConnection(config, transition.source, tokens),
+    );
+    if (result.outcome !== "passed")
+      throw new Error(
+        `Promotion out of ${transition.source} blocked: check ${check.name} ${result.outcome} (${result.url ?? result.reason})`,
+      );
+    gates.passed.add(transition.source);
+  }
   await promotionPlanApply(plan);
   const changedPaths = plan.changes.map(
     (change) => `resources/${transition.target}/${change.path}`,
@@ -260,18 +295,30 @@ async function transitionRun(
     ["--force", "--allow-new-files", "--resolve=ours", ...changedPaths],
   );
   appliedPathsRecord(transition.target);
+  // The target's files just changed, so an earlier pass no longer covers it.
+  gates.passed.delete(transition.target);
   return plan.changes.some((change) => change.kind === "delete");
 }
 
 export async function promotionCommandRun(
   args = process.argv.slice(2),
-  deps: PromotionDeps = { childRun: orgScriptRun },
+  overrides: Partial<PromotionDeps> = {},
 ): Promise<void> {
+  const deps: PromotionDeps = {
+    childRun: orgScriptRun,
+    checkRun: (check, connection) =>
+      promotionGateRun(ROOT_DIR, check, connection),
+    ...overrides,
+  };
   const parsed = argumentsParse(args);
   const configPath = resolve(ROOT_DIR, "promotion.yml");
   if (!existsSync(configPath))
     throw new Error("promotion.yml is required at the repository root");
   const config = promotionConfigParse(readFileSync(configPath, "utf8"));
+  const gates: PromotionGates = {
+    checks: promotionChecksLoad(ROOT_DIR, config),
+    passed: new Set(),
+  };
   const tokens = parsed.apply
     ? tokensParse(TOKENS_ENV)
     : new Map<string, string>();
@@ -293,6 +340,7 @@ export async function promotionCommandRun(
       tokens,
       deletionAuthorizedSources.has(sourceKey),
       deps,
+      gates,
     );
     if (deleted)
       deletionAuthorizedSources.add(

src/promotion-gate.ts

@@ -0,0 +1,170 @@
+// The promotion check gate: `orgs.<slug>.check: <name>` in promotion.yml
+// names a vapi-checks.yml check that must pass in that org before any
+// transition promotes out of it. The gate runs the same check as the PR
+// workflow, built from the source org's files at the promoted commit.
+
+import type { CheckDefinition } from "./check-config.ts";
+import { CHECKS_CONFIG_FILE, checksConfigLoad } from "./check-config.ts";
+import { checkJobsBuild } from "./check-build.ts";
+import type { CheckOutcome, CheckTargetResult } from "./check-run.ts";
+import { checkRunAll, MAX_CONCURRENT, MIN_START_MS } from "./check-run.ts";
+import type { OrgConnection } from "./org-connection.ts";
+import type { PromotionConfig } from "./promotion.ts";
+import { userAgentGet } from "./user-agent.ts";
+
+export interface PromotionGateResult {
+  outcome: CheckOutcome;
+  reason: string;
+  url?: string;
+}
+
+const DEFAULT_BASE_URL = "https://api.vapi.ai";
+// Gate checks must fit well inside the promotion step's timeout, so a check
+// that can't finish is rejected when the config loads, not killed mid-run.
+export const GATE_BUDGET_MINUTES = 300;
+const OUTCOME_RANK: Record<CheckOutcome, number> = {
+  passed: 0,
+  built: 0,
+  incomplete: 1,
+  failed: 2,
+  error: 3,
+};
+
+// The gated orgs' checks, validated up front so a typo fails before any
+// transition applies.
+export function promotionChecksLoad(
+  rootDir: string,
+  config: PromotionConfig,
+): Map<string, CheckDefinition> {
+  const gated = Object.entries(config.orgs).filter(([, org]) => org.check);
+  const checks = new Map<string, CheckDefinition>();
+  if (gated.length === 0) return checks;
+  const checksConfig = checksConfigLoad(rootDir);
+  if (!checksConfig)
+    throw new Error(
+      `promotion.yml gates ${gated.map(([slug]) => slug).join(", ")} on checks, but there is no ${CHECKS_CONFIG_FILE}`,
+    );
+  // An org that is last in every pipeline is never promoted out of.
+  const sources = new Set(
+    Object.values(config.pipelines).flatMap((pipeline) =>
+      pipeline.orgs.slice(0, -1),
+    ),
+  );
+  let budgetMinutes = 0;
+  for (const [slug, org] of gated) {
+    if (!sources.has(slug))
+      throw new Error(
+        `orgs.${slug}.check: nothing is promoted out of ${slug} (it is last in every pipeline), so this check would never run; gate the org before it instead`,
+      );
+    const check = checksConfig.checks[org.check!];
+    if (!check)
+      throw new Error(
+        `orgs.${slug}.check: no check named ${org.check} in ${CHECKS_CONFIG_FILE}`,
+      );
+    if (check.org !== slug || check.runOrg !== slug)
+      throw new Error(
+        `orgs.${slug}.check: check ${check.name} must read and run in ${slug} (it reads ${check.org} and runs in ${check.runOrg})`,
+      );
+    // A gate runs in the real org, never a CI org, so it must not reach real
+    // systems: no live tools, and no webhooks to the org's own servers.
+    if (check.toolMocks === "off")
+      throw new Error(
+        `orgs.${slug}.check: check ${check.name} sets toolMocks: off, which runs real tools; a gate runs in ${slug} itself, so it must use toolMocks: strict`,
+      );
+    if (!check.stripWebhooks)
+      throw new Error(
+        `orgs.${slug}.check: check ${check.name} sets stripWebhooks: false, which sends simulated calls' webhooks to ${slug}'s real servers; a gate must keep the default`,
+      );
+    // The org's token goes only to the host promotion uses for that org.
+    if (check.baseUrl && check.baseUrl !== org.baseUrl?.replace(/\/+$/, ""))
+      throw new Error(
+        `orgs.${slug}.check: check ${check.name} uses ${check.baseUrl}, but promotion.yml uses ${org.baseUrl ?? "the default API"} for ${slug}; set the same baseUrl in both`,
+      );
+    budgetMinutes += gateBudgetMinutes(check);
+    checks.set(slug, check);
+  }
+  if (budgetMinutes > GATE_BUDGET_MINUTES)
+    throw new Error(
+      `promotion.yml's gated checks can take up to ${budgetMinutes} minutes in one run, more than the ${GATE_BUDGET_MINUTES} the promotion step allows; lower their timeoutMinutes or targets`,
+    );
+  return checks;
+}
+
+// The longest a gate check can run: targets run MAX_CONCURRENT at a time,
+// and each batch gets a full timeoutMinutes.
+export function gateBudgetMinutes(
+  check: Pick<CheckDefinition, "timeoutMinutes"> & {
+    targets: readonly unknown[];
+  },
+): number {
+  return (
+    Math.ceil(check.targets.length / MAX_CONCURRENT) * check.timeoutMinutes +
+    MIN_START_MS / 60_000
+  );
+}
+
+export function gateDeadline(
+  check: Pick<CheckDefinition, "timeoutMinutes"> & {
+    targets: readonly unknown[];
+  },
+  now: number,
+): number {
+  return now + gateBudgetMinutes(check) * 60_000;
+}
+
+// Anything short of every target passing blocks: the worst target wins.
+export function gateResultReduce(
+  results: Array<Pick<CheckTargetResult, "outcome" | "reason" | "url">>,
+): PromotionGateResult {
+  const worst = results.reduce((a, b) =>
+    OUTCOME_RANK[b.outcome] > OUTCOME_RANK[a.outcome] ? b : a,
+  );
+  return { outcome: worst.outcome, reason: worst.reason, url: worst.url };
+}
+
+// The plan-only line: what the gate would run, built offline.
+export async function promotionGatePlanLine(
+  rootDir: string,
+  check: CheckDefinition,
+): Promise<string> {
+  const jobs = await checkJobsBuild(rootDir, check);
+  const broken = jobs.find((job) => !job.result.body);
+  if (broken)
+    return `  check  would run ${check.name} in ${check.org}, but its payload doesn't build: ${broken.result.errors[0]}`;
+  const simulations = jobs[0]?.result.body?.simulations.length ?? 0;
+  return `  check  would run ${check.name} in ${check.org} (${simulations} simulation${simulations === 1 ? "" : "s"} × ${jobs.length} target${jobs.length === 1 ? "" : "s"})`;
+}
+
+// Run the check live in the source org and reduce it to one result: the
+// worst target wins, so anything short of every target passing blocks.
+export async function promotionGateRun(
+  rootDir: string,
+  check: CheckDefinition,
+  connection: OrgConnection,
+): Promise<PromotionGateResult> {
+  const jobs = await checkJobsBuild(rootDir, check);
+  const controller = new AbortController();
+  const abort = () => controller.abort();
+  process.once("SIGINT", abort);
+  process.once("SIGTERM", abort);
+  try {
+    const results = await checkRunAll({
+      jobs,
+      connectionFor: () => ({
+        token: connection.token,
+        baseUrl: connection.baseUrl ?? DEFAULT_BASE_URL,
+        userAgent: userAgentGet("check"),
+      }),
+      deadline: gateDeadline(check, Date.now()),
+      signal: controller.signal,
+    });
+    for (const result of results)
+      console.log(
+        `  check  ${result.job.label}: ${result.outcome} — ${result.reason}${result.url ? ` (${result.url})` : ""}`,
+      );
+    return gateResultReduce(results);
+  } finally {
+    process.off("SIGINT", abort);
+    process.off("SIGTERM", abort);
+  }
+}

src/promotion.ts

@@ -29,6 +29,9 @@ export interface PromotionBindings {
 export interface PromotionOrg {
   baseUrl?: string;
   bindings: PromotionBindings;
+  // A vapi-checks.yml check that must pass in this org before anything is
+  // promoted out of it.
+  check?: string;
 }
 
 export interface PromotionPipeline {
@@ -142,6 +145,8 @@ export function promotionBindingsParse(value: unknown): PromotionBindings {
   };
 }
 
+const PROMOTION_ORG_KEYS = ["baseUrl", "bindings", "check"];
+
 export function promotionConfigParse(content: string): PromotionConfig {
   const raw = object(parseYaml(content), "promotion.yml");
   if (raw.version !== 1) throw new Error("promotion.yml version must be 1");
@@ -150,11 +155,25 @@ export function promotionConfigParse(content: string): PromotionConfig {
   for (const [slug, value] of Object.entries(orgsRaw)) {
     if (!SLUG_RE.test(slug)) throw new Error(`Invalid org slug: ${slug}`);
     const org = object(value ?? {}, `org ${slug}`);
+    // A typo (`checks:`, `Check:`) must not quietly drop a safety gate.
+    for (const key of Object.keys(org))
+      if (!PROMOTION_ORG_KEYS.includes(key))
+        throw new Error(
+          `org ${slug} has unknown key "${key}" (allowed: ${PROMOTION_ORG_KEYS.join(", ")})`,
+        );
     if (org.baseUrl !== undefined && typeof org.baseUrl !== "string")
       throw new Error(`org ${slug}.baseUrl must be a string`);
+    if (
+      org.check !== undefined &&
+      (typeof org.check !== "string" || !SLUG_RE.test(org.check))
+    )
+      throw new Error(
+        `org ${slug}.check must be a check name from vapi-checks.yml`,
+      );
     orgs[slug] = {
       baseUrl: typeof org.baseUrl === "string" ? org.baseUrl : undefined,
       bindings: promotionBindingsParse(org.bindings),
+      ...(typeof org.check === "string" ? { check: org.check } : {}),
     };
   }
   const pipelinesRaw = object(raw.pipelines, "promotion.yml pipelines");