diff --git a/docs/dentalpresence-dogfood-evidence.md b/docs/dentalpresence-dogfood-evidence.md new file mode 100644 index 0000000..4529695 --- /dev/null +++ b/docs/dentalpresence-dogfood-evidence.md @@ -0,0 +1,18 @@ +# DentalPresence evidence collection + +The paired-evidence importer accepts explicitly reviewed selector version/integrity pins, preserves historical versions, and exposes separate version cohorts. Unknown versions, mismatched integrity, local engines, seeded experiments, conflicting identities, incomplete executions and unverified provenance remain excluded. Producer assessment flags are never used to authorize inclusion. + +The matching DentalPresence integration archives paired and seeded JSON evidence inside its private repository's `diffci-evidence` branch, after each experiment and nightly. Workflow artifacts are 90-day recovery copies; the Git archive has no Actions expiry. It stores compressed pair, prediction and receipt bytes with GitHub artifact provenance and recomputes cumulative summaries. Full aggregate test artifacts retain their separate existing collector/consumer and now have a 90-day artifact window; they are not included in this paired archive. + +The dependency-free DentalPresence importer is generated from `scripts/lib/dentalpresence-pairs.ts`: + +```powershell +node scripts/sync-dentalpresence-importer.mjs C:\path\to\DentalPresence.in\scripts\ci-pair-import.mjs +node scripts/sync-dentalpresence-importer.mjs C:\path\to\DentalPresence.in\scripts\ci-pair-import.mjs --check +``` + +Keep the producer package pin and this consumer's allowlist synchronized through reviewed changes. Both the exact version and SHA-512 integrity are required. The archive runs trusted main scripts only and parses downloaded artifacts as bounded data, never executable source. The scheduled collector becomes active after the DentalPresence PR merges; opening a PR alone does not activate it. + +The weekly seeded cohort has three fixed public-content mutations (SEO URL fallback, Organization postal markup, practice Maps place ID). Each requires a passing original targeted control, prospective selection on an identical synthetic committed delta across isolated clones, a full-arm failure, and targeted reproduction of the same failing case. Seeded detection and possible misses are reported separately. They do not establish natural regression recall and cannot contribute to savings. + +On 3 October 2026, a read-only backfill fetched 16 paired artifacts from DentalPresence: seven complete selective passing pairs and nine FULL fallbacks. Six accepted pairs used 0.2.11 and one used 0.3.2. The latter had previously been excluded by the obsolete single-version importer. Three PR artifacts were verified through authenticated GitHub merge-parent metadata because their artifact head SHAs differ from the executed merge SHAs. This is a small internal cohort sharing a founding team; no accepted pair contained a failed case, and natural regression recall remains unestimated. Runtime measurements exclude installation and provider billing; the shadow experiment adds work by running both arms. diff --git a/scripts/lib/dentalpresence-pairs.ts b/scripts/lib/dentalpresence-pairs.ts index 1ae2891..2a9750f 100644 --- a/scripts/lib/dentalpresence-pairs.ts +++ b/scripts/lib/dentalpresence-pairs.ts @@ -9,16 +9,19 @@ const mono = (value: unknown): bigint | null => typeof value === "string" && /^\ export interface PairArtifact { pair: unknown; predictionBytes: Buffer; receiptBytes: Buffer } +// Explicit historical cohorts: upgrading a producer never silently authorizes a new selector. export const DENTALPRESENCE_SELECTOR_PINS: Readonly> = { "0.2.11": "sha512-c6mWfEU7P6k+nroa0LO7/NJyR/Ubzoa36mh3XRoWSOzkZwYIcGC+2QfbM+gHzCWsAqWew4sM0KT3XleSrjqosg==", "0.3.2": "sha512-45tYPyabvMjrhlJCPtEP0UUeIVY567XZl6eGsIXy1LSHADmr9Dj1VSvh7HIAtfea03ZbTwhJ2s8sh/YTSU8Mwg==", }; +export const dentalPresenceSelectorPins = DENTALPRESENCE_SELECTOR_PINS; /** Recompute eligibility from the artifacts, never trust producer assessment booleans. */ function inspectPair(input: PairArtifact) { const invalid = (reason: string) => ({ accepted: false as const, reason }); const pair = input.pair; if (!object(pair) || pair.schema !== "dentalpresence.diffci.pair.v1") return invalid("INVALID_SCHEMA"); + if (pair.seededMutation) return invalid("SEEDED_EXPERIMENT"); if (pair.repository !== "adityankale190895/DentalPresence.in" || typeof pair.workflowRef !== "string" || !/^adityankale190895\/DentalPresence\.in\/\.github\/workflows\/paired-ci-evidence\.yml@refs\//.test(pair.workflowRef) || !/^\d+$/.test(String(pair.runId ?? "")) || !/^[1-9]\d*$/.test(String(pair.runAttempt ?? "")) || @@ -41,9 +44,9 @@ function inspectPair(input: PairArtifact) { identity.headSha !== pair.executedSha || identity.baseSha !== pair.baseSha || prediction.schema !== "diffci.observation.v1" || prediction.status !== "OBSERVED" || prediction.commitRange?.headSha !== pair.executedSha || prediction.commitRange?.baseSha !== pair.baseSha || prediction.nonInterference?.worktreeUnchanged !== true) return invalid("PREDICTION_IDENTITY"); - const selectorVersion = pair.selectorPackage?.version; - if (typeof selectorVersion !== "string" || !Object.hasOwn(DENTALPRESENCE_SELECTOR_PINS, selectorVersion) || - pair.selectorPackage.integrity !== DENTALPRESENCE_SELECTOR_PINS[selectorVersion] || + const version = pair.selectorPackage?.version; + if (typeof version !== "string" || !Object.hasOwn(dentalPresenceSelectorPins, version) || + pair.selectorPackage.integrity !== dentalPresenceSelectorPins[version] || prediction.observer?.version !== pair.selectorPackage.version || identity.selectorVersion !== pair.selectorPackage.version) return invalid("SELECTOR_IDENTITY"); if (pair.mode !== "SELECTIVE" || prediction.result?.mode !== "SELECTIVE") return invalid("FULL_FALLBACK"); const files: unknown = prediction.result.selectedTests; @@ -83,6 +86,7 @@ function inspectPair(input: PairArtifact) { const selectedFailures = new Set(allOutcomes[1].filter((test) => test.status === "FAIL").map((test) => test.testId)); const omittedFailures = failed.filter((test) => !pair.selectedFileIds.includes(test.fileId)).length; return { accepted: true as const, id: `${pair.runId}:${pair.runAttempt}:${pair.executedSha}`, headSha: pair.executedSha as string, + selectorVersion: version, possibleMiss: pair.selected.status === "PASS" && omittedFailures > 0, fullFailingCases: failed.length, correspondingFailingCases: failed.filter((test) => selectedFailures.has(test.testId)).length, omittedFailingCases: omittedFailures, performanceEligible: pair.full.status === "PASS" && pair.selected.status === "PASS", @@ -116,10 +120,21 @@ export function summarizeDentalPresencePairs(inputs: PairArtifact[]) { const fullWallMs = performance.reduce((sum, pair) => sum + pair.fullWallMs, 0); const selectedWallMs = performance.reduce((sum, pair) => sum + pair.selectedWallMs, 0); const analysisWallMs = performance.reduce((sum, pair) => sum + pair.analysisWallMs, 0); + const cohorts = Object.fromEntries([...new Set(pairs.map((pair) => pair.selectorVersion))].sort().map((version) => { + const cohort = pairs.filter((pair) => pair.selectorVersion === version); + const measured = cohort.filter((pair) => pair.performanceEligible); + const full = measured.reduce((sum, pair) => sum + pair.fullWallMs, 0); + const selected = measured.reduce((sum, pair) => sum + pair.selectedWallMs, 0); + const analysis = measured.reduce((sum, pair) => sum + pair.analysisWallMs, 0); + return [version, { acceptedPairs: cohort.length, performancePairs: measured.length, + possibleMissPairs: cohort.filter((pair) => pair.possibleMiss).length, + fullWallMs: full, selectedWallMs: selected, analysisWallMs: analysis, + netRuntimeReduction: full ? 1 - (selected + analysis) / full : null }]; + })); return { schema: "diffci.dentalpresence.paired-summary.v1", inputArtifacts: inputs.length, duplicateDeliveries, acceptedPairs: pairs.length, performancePairs: performance.length, possibleMissPairs: pairs.filter((pair) => pair.possibleMiss).length, fullFailingCases: pairs.reduce((sum, pair) => sum + pair.fullFailingCases, 0), - omittedFailingCases: pairs.reduce((sum, pair) => sum + pair.omittedFailingCases, 0), exclusions, + omittedFailingCases: pairs.reduce((sum, pair) => sum + pair.omittedFailingCases, 0), exclusions, cohorts, fullWallMs, selectedWallMs, analysisWallMs, grossRuntimeReduction: fullWallMs ? 1 - selectedWallMs / fullWallMs : null, netRuntimeReduction: fullWallMs ? 1 - (selectedWallMs + analysisWallMs) / fullWallMs : null, diff --git a/tests/shadow/dentalpresence-pairs.test.ts b/tests/shadow/dentalpresence-pairs.test.ts index f3b8544..6d397f2 100644 --- a/tests/shadow/dentalpresence-pairs.test.ts +++ b/tests/shadow/dentalpresence-pairs.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import { test } from "node:test"; -import { DENTALPRESENCE_SELECTOR_PINS, inspectDentalPresencePair, summarizeDentalPresencePairs } from "../../scripts/lib/dentalpresence-pairs.js"; +import { DENTALPRESENCE_SELECTOR_PINS, dentalPresenceSelectorPins, inspectDentalPresencePair, summarizeDentalPresencePairs } from "../../scripts/lib/dentalpresence-pairs.js"; const hash = (value: string | Buffer) => createHash("sha256").update(value).digest("hex"); function artifact(version = "0.2.11") { @@ -32,14 +32,6 @@ function artifact(version = "0.2.11") { } }; } -test("accepts both qualified releases but rejects mismatched and unknown selector pins", () => { - for (const version of ["0.2.11", "0.3.2"]) assert.equal(inspectDentalPresencePair(artifact(version)).accepted, true); - const mismatched = artifact("0.3.2"); - mismatched.pair.selectorPackage.integrity = DENTALPRESENCE_SELECTOR_PINS["0.2.11"]; - assert.equal(inspectDentalPresencePair(mismatched).accepted, false); - assert.equal(inspectDentalPresencePair(artifact("0.3.3")).accepted, false); -}); - test("recomputes runtime with analysis overhead and deduplicates real pair identity", () => { const input = artifact(); const summary = summarizeDentalPresencePairs([input, input]); @@ -83,3 +75,34 @@ test("a conflicting companion prediction quarantines the pair even if pair.json assert.equal(report.acceptedPairs, 0); assert.equal(report.exclusions.CONFLICTING_PAIR, 1); }); + +test("accepts reviewed selector pins, partitions cohorts, and rejects unknown or mismatched integrity", () => { + const historical = artifact(); const current = artifact(); + current.pair.runId = "101"; + const version = "0.3.2"; + current.pair.selectorPackage = { version, integrity: dentalPresenceSelectorPins[version] }; + const prediction = JSON.parse(current.predictionBytes.toString()); prediction.observer.version = version; + current.predictionBytes = Buffer.from(JSON.stringify(prediction)); + Object.assign(current.pair.prediction, { selectorVersion: version, frozenSha256: hash(current.predictionBytes), finalSha256: hash(current.predictionBytes) }); + current.receiptBytes = Buffer.from(JSON.stringify(current.pair.prediction)); + const report = summarizeDentalPresencePairs([historical, current]); + assert.equal(report.acceptedPairs, 2); + assert.deepEqual(Object.keys(report.cohorts), ["0.2.11", "0.3.2"]); + current.pair.selectorPackage.integrity = historical.pair.selectorPackage.integrity; + assert.deepEqual(inspectDentalPresencePair(current), { accepted: false, reason: "SELECTOR_IDENTITY" }); + current.pair.selectorPackage.version = "999.0.0"; + assert.equal(inspectDentalPresencePair(current).accepted, false); +}); + +test("seeded failures cannot enter natural-change or performance cohorts", () => { + const input = artifact(); Object.assign(input.pair, { seededMutation: { id: "fixture" } }); + assert.deepEqual(inspectDentalPresencePair(input), { accepted: false, reason: "SEEDED_EXPERIMENT" }); +}); + +test("accepts both qualified releases but rejects mismatched and unknown selector pins", () => { + for (const version of ["0.2.11", "0.3.2"]) assert.equal(inspectDentalPresencePair(artifact(version)).accepted, true); + const mismatched = artifact("0.3.2"); + mismatched.pair.selectorPackage.integrity = DENTALPRESENCE_SELECTOR_PINS["0.2.11"]; + assert.equal(inspectDentalPresencePair(mismatched).accepted, false); + assert.equal(inspectDentalPresencePair(artifact("0.3.3")).accepted, false); +});