[TS] Measure paired source coverage and search cost

usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt

@@ -9,7 +9,25 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.StandardCopyOption
 import java.nio.file.StandardOpenOption
+import java.security.MessageDigest
 import kotlin.time.Duration.Companion.milliseconds
+import kotlin.time.TimeSource
+
+@Serializable
+internal data class CallsCoverageUniverseIdentity(
+    val toolRevision: String,
+    val nativeFrontendRevision: String,
+    val sourceManifest: String,
+    val projectIndex: Int,
+    val functionIndex: Int,
+    val sourceFile: String,
+)
+
+@Serializable
+internal data class CallsCoverageUniverseRecord(
+    val identity: CallsCoverageUniverseIdentity,
+    val universe: CallsCoverageUniverse,
+)
 
 @Serializable
 internal data class CallsCoverageCellIdentity(
@@ -24,6 +42,7 @@ internal data class CallsCoverageCellIdentity(
     val budgetMillis: Long,
     val solverLimitMillis: Long,
     val candidateCap: Int,
+    val coverageUniverseSha256: String,
 )
 
 @Serializable
@@ -39,6 +58,7 @@ internal data class CallsCoverageCheckpoint(
     val emittedCandidates: Int,
     val replayedCandidates: Int,
     val coveredStatements: Int,
+    val totalStatements: Int,
     val coveredIfArms: Int,
 )
 
@@ -48,11 +68,47 @@ internal data class CallsCoverageCellResult(
     val search: CallsCoverageSearchResult,
     val replayErrors: Int,
     val completionMismatches: Int,
+    val replayElapsedMillis: Long,
+    val cellElapsedMillis: Long,
     val checkpoints: List<CallsCoverageCheckpoint>,
 )
 
+internal fun runCoverageUniverse(args: List<String>) {
+    require(args.size == 4) { "coverage-universe requires 4 arguments" }
+    val manifestPath = Path.of(args[0]).toRealPath()
+    val manifest = CallsExperimentJson.decodeManifest(Files.readString(manifestPath))
+    val projectIndex = args[1].toInt()
+    val functionIndex = args[2].toInt()
+    val output = Path.of(args[3]).toAbsolutePath().normalize()
+    val project = manifest.projects[projectIndex]
+    val function = project.functions[functionIndex]
+    require(System.getenv("ETS_FRONTEND_SCRIPT") == null) { "ETS_FRONTEND_SCRIPT must be unset" }
+
+    Files.createDirectories(output)
+    require(Files.list(output).use { entries -> entries.findAny().isEmpty }) {
+        "Coverage universe output directory is not empty"
+    }
+    val identity = CallsCoverageUniverseIdentity(
+        toolRevision = CallsBuildIdentity.toolRevision,
+        nativeFrontendRevision = CallsBuildIdentity.nativeFrontendRevision,
+        sourceManifest = manifestPath.toString(),
+        projectIndex = projectIndex,
+        functionIndex = functionIndex,
+        sourceFile = function.sourceFile,
+    )
+    require(Regex("[0-9a-f]{40}").matches(identity.toolRevision)) { "Coverage build must be clean" }
+    val universe = OriginalTypeScriptCoverageReplayer(
+        sourceRoot = manifestPath.parent.resolve(project.sourceRoot).toRealPath(),
+        function = function,
+    ).use { replayer -> replayer.probeUniverse(timeoutMillis = REPLAY_TIMEOUT_MILLIS) }
+    val record = CallsCoverageUniverseRecord(identity = identity, universe = universe)
+    atomicWrite(output.resolve("universe.json"), encode(record))
+    println(encode(record))
+}
+
 internal fun runCoverageCell(args: List<String>) {
-    require(args.size == 9) { "coverage-cell requires 9 arguments" }
+    require(args.size == 10) { "coverage-cell requires 10 arguments" }
+    val cellStarted = TimeSource.Monotonic.markNow()
     val manifestPath = Path.of(args[0]).toRealPath()
     val manifest = CallsExperimentJson.decodeManifest(Files.readString(manifestPath))
     val projectIndex = args[1].toInt()
@@ -63,11 +119,27 @@ internal fun runCoverageCell(args: List<String>) {
     val budgetMillis = args[6].toLong()
     val solverLimitMillis = args[7].toLong()
     val candidateCap = args[8].toInt()
+    val universeBytes = Files.readAllBytes(Path.of(args[9]).toRealPath())
+    val universeRecord = CallsExperimentJson.json.decodeFromString<CallsCoverageUniverseRecord>(
+        universeBytes.toString(Charsets.UTF_8),
+    )
     val project = manifest.projects[projectIndex]
     val function = project.functions[functionIndex]
     require(seed in manifest.seeds) { "Seed is outside the frozen source manifest" }
     require(budgetMillis > 0 && solverLimitMillis in 1..budgetMillis && candidateCap > 0)
     require(System.getenv("ETS_FRONTEND_SCRIPT") == null) { "ETS_FRONTEND_SCRIPT must be unset" }
+    require(universeRecord.identity == CallsCoverageUniverseIdentity(
+        toolRevision = CallsBuildIdentity.toolRevision,
+        nativeFrontendRevision = CallsBuildIdentity.nativeFrontendRevision,
+        sourceManifest = manifestPath.toString(),
+        projectIndex = projectIndex,
+        functionIndex = functionIndex,
+        sourceFile = function.sourceFile,
+    )) { "Coverage universe identity differs" }
+    val universe = universeRecord.universe
+    require(function.sourceFile in universe.sourceFiles && universe.supportedStatementKeys.isNotEmpty()) {
+        "Coverage universe does not include the entry source statements"
+    }
 
     val identity = CallsCoverageCellIdentity(
         schemaVersion = 1,
@@ -81,6 +153,8 @@ internal fun runCoverageCell(args: List<String>) {
         budgetMillis = budgetMillis,
         solverLimitMillis = solverLimitMillis,
         candidateCap = candidateCap,
+        coverageUniverseSha256 = MessageDigest.getInstance("SHA-256").digest(universeBytes)
+            .joinToString("") { byte -> "%02x".format(byte.toInt() and 0xff) },
     )
     require(Regex("[0-9a-f]{40}").matches(identity.toolRevision)) { "Coverage build must be clean" }
 
@@ -152,12 +226,18 @@ internal fun runCoverageCell(args: List<String>) {
             search.candidates[index].emittedAtMillis <= checkpoint
         }
         val replayed = indices.mapNotNull { index -> events[index].replay }
+        val coveredStatements = replayed.flatMap { it.coveredStatementKeys }.toSet()
+        val coveredIfArms = replayed.flatMap { it.coveredIfArmKeys }.toSet()
+        require(universe.supportedStatementKeys.containsAll(coveredStatements)) {
+            "Replayed statements extend beyond the frozen coverage universe"
+        }
         CallsCoverageCheckpoint(
             searchBudgetMillis = checkpoint,
             emittedCandidates = indices.size,
             replayedCandidates = replayed.size,
-            coveredStatements = replayed.flatMap { it.coveredStatementKeys }.toSet().size,
-            coveredIfArms = replayed.flatMap { it.coveredIfArmKeys }.toSet().size,
+            coveredStatements = coveredStatements.size,
+            totalStatements = universe.supportedStatementKeys.size,
+            coveredIfArms = coveredIfArms.size,
         )
     }
     val result = CallsCoverageCellResult(
@@ -168,6 +248,8 @@ internal fun runCoverageCell(args: List<String>) {
             events[index].replay?.completion != null &&
                 events[index].replay?.completion != search.candidates[index].completion
         },
+        replayElapsedMillis = events.sumOf { event -> event.replay?.replayElapsedMillis ?: 0L },
+        cellElapsedMillis = cellStarted.elapsedNow().inWholeMilliseconds,
         checkpoints = checkpoints,
     )
     atomicWrite(output.resolve("result.json"), encode(result))

usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageReplay.kt

@@ -38,6 +38,14 @@ internal data class CallsCoverageReplayResult(
     val replayElapsedMillis: Long,
 )
 
+@Serializable
+internal data class CallsCoverageUniverse(
+    val supportedStatementKeys: Set<String>,
+    val sourceFiles: Set<String>,
+    val diagnostics: List<String>,
+    val probeElapsedMillis: Long,
+)
+
 /** Replays one generated input in a fresh Node.js process and checks its exact transport value. */
 internal class OriginalTypeScriptCoverageReplayer(
     sourceRoot: Path,
@@ -87,6 +95,58 @@ internal class OriginalTypeScriptCoverageReplayer(
         }
     }
 
+    fun probeUniverse(timeoutMillis: Long): CallsCoverageUniverse {
+        require(timeoutMillis > 0)
+
+        val started = TimeSource.Monotonic.markNow()
+        val result = backend.run(
+            property = PropertyDefinition(
+                id = PropertyId("calls.coverage.universe"),
+                inputs = listOf(PropertyInput(name = "__usvmCoverageSentinel", domain = BooleanDomain)),
+                predicate = TypeScriptEntryPoint(module = wrapperName, exportName = "coverageUniverseProbe"),
+            ),
+            configuration = PropertyRunConfiguration(
+                seed = 0,
+                numRuns = 1,
+                timeoutMillis = timeoutMillis,
+                examples = listOf(listOf(JsConcreteValue.Boolean(true))),
+                coverageRequest = PropertyCoverageRequest(
+                    scopes = setOf(CoverageScope.SOURCE_UNDER_TEST),
+                    includePatterns = allowedFiles.sorted(),
+                    includeUnexecutedSources = true,
+                ),
+            ),
+        )
+        val elapsedMillis = started.elapsedNow().inWholeMilliseconds
+        require(result.status == PropertyRunStatus.SUCCESS && result.numRuns == 1) {
+            "Coverage universe probe did not complete once: $result"
+        }
+
+        val coverage = requireNotNull(result.coverage) { "Coverage universe probe produced no c8 artifact" }
+        val statements = linkedSetOf<String>()
+        val sourceFiles = linkedSetOf<String>()
+        coverage.files.forEach { file ->
+            val relative = relativeSourcePath(file.path) ?: return@forEach
+            if (relative !in allowedFiles) return@forEach
+
+            sourceFiles += relative
+            file.statements.forEach { statement ->
+                statements += "$relative:statement:${statement.location.stableKey()}"
+            }
+        }
+        require(sourceFiles == allowedFiles) {
+            "Coverage universe omitted local source files: ${allowedFiles - sourceFiles}"
+        }
+        require(statements.isNotEmpty()) { "Coverage universe has no source statements" }
+
+        return CallsCoverageUniverse(
+            supportedStatementKeys = statements,
+            sourceFiles = sourceFiles,
+            diagnostics = coverage.diagnostics.map { diagnostic -> "${diagnostic.code}: ${diagnostic.message}" },
+            probeElapsedMillis = elapsedMillis,
+        )
+    }
+
     fun replay(inputs: List<JsConcreteValue>, timeoutMillis: Long): CallsCoverageReplayResult {
         require(timeoutMillis > 0)
         require(inputs.size == function.inputs.size)
@@ -246,5 +306,9 @@ private fun coverageWrapper(
 
           return true;
         }
+
+        export function coverageUniverseProbe(_sentinel: boolean): boolean {
+          return true;
+        }
     """.trimIndent() + "\n"
 }

usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt

@@ -22,6 +22,7 @@ import kotlin.time.TimeSource
 @Serializable
 internal data class CallsCoverageCandidate(
     val emittedAtMillis: Long,
+    val emittedAtStep: Long,
     val inputs: List<JsConcreteValue>,
     val newSymbolicStatements: Int,
     val completion: CallsCoverageCompletion,
@@ -49,10 +50,14 @@ internal data class CallsCoverageSearchResult(
     val status: CallsCoverageSearchStatus,
     val candidates: List<CallsCoverageCandidate>,
     val selectedStates: Int,
+    val executedSteps: Long,
+    val stepsWithinBudget: Long,
     val extractionFailures: List<String>,
     val candidateCapReached: Boolean,
     val preparationElapsedMillis: Long,
+    val machineSetupElapsedMillis: Long,
     val searchElapsedMillis: Long,
+    val machineTeardownElapsedMillis: Long,
     val unsupportedCall: Boolean = false,
     val engineFailed: Boolean = false,
     val runtimeLimited: Boolean = false,
@@ -99,10 +104,14 @@ internal class CurrentTsCallsCoverageEngine(
                 status = preparation.status.toCoverageStatus(),
                 candidates = emptyList(),
                 selectedStates = 0,
+                executedSteps = 0,
+                stepsWithinBudget = 0,
                 extractionFailures = emptyList(),
                 candidateCapReached = false,
                 preparationElapsedMillis = preparationElapsedMillis,
+                machineSetupElapsedMillis = 0,
                 searchElapsedMillis = 0,
+                machineTeardownElapsedMillis = 0,
                 diagnostic = "${preparation.reasonCode}: ${preparation.diagnostic}",
             )
         }
@@ -115,13 +124,14 @@ internal class CurrentTsCallsCoverageEngine(
         )
         val candidates = mutableListOf<CallsCoverageCandidate>()
         val extractionFailures = mutableListOf<String>()
-        val searchStarted = TimeSource.Monotonic.markNow()
+        val machineCallStarted = TimeSource.Monotonic.markNow()
         val observer = CompletedCoverageCandidateObserver(
             method = preparation.method,
             inputDomains = request.function.inputs,
             symbolicInputs = symbolicInputs,
             candidateCap = request.candidateCap,
-            searchStarted = searchStarted,
+            budgetMillis = request.budget.inWholeMilliseconds,
+            machineCallStarted = machineCallStarted,
             onCandidate = { candidate ->
                 request.onCandidate(candidate)
                 candidates += candidate
@@ -157,7 +167,10 @@ internal class CurrentTsCallsCoverageEngine(
                 machineObserver = observer,
             ).use { machine -> machine.analyzeWithMetadata(methods = listOf(preparation.method)) }
         }
-        val searchElapsedMillis = searchStarted.elapsedNow().inWholeMilliseconds
+        val machineCallElapsedMillis = machineCallStarted.elapsedNow().inWholeMilliseconds
+        val machineTeardownElapsedMillis = (
+            machineCallElapsedMillis - observer.machineSetupElapsedMillis - observer.searchElapsedMillis
+        ).coerceAtLeast(0)
         val outcome = machineResult.getOrNull()
         val status = when {
             machineResult.isFailure -> CallsCoverageSearchStatus.TOOL_ERROR
@@ -170,10 +183,14 @@ internal class CurrentTsCallsCoverageEngine(
             status = status,
             candidates = candidates,
             selectedStates = observer.selectedStates,
+            executedSteps = observer.executedSteps,
+            stepsWithinBudget = observer.stepsWithinBudget,
             extractionFailures = extractionFailures,
             candidateCapReached = observer.capReached,
             preparationElapsedMillis = preparationElapsedMillis,
-            searchElapsedMillis = searchElapsedMillis,
+            machineSetupElapsedMillis = observer.machineSetupElapsedMillis,
+            searchElapsedMillis = observer.searchElapsedMillis,
+            machineTeardownElapsedMillis = machineTeardownElapsedMillis,
             unsupportedCall = outcome?.unsupportedCall == true,
             engineFailed = outcome?.engineFailed == true,
             runtimeLimited = outcome?.runtimeLimited == true,
@@ -187,20 +204,50 @@ private class CompletedCoverageCandidateObserver(
     private val inputDomains: List<org.usvm.ts.pbt.model.PropertyInput>,
     private val symbolicInputs: CallsSymbolicInputs,
     private val candidateCap: Int,
-    private val searchStarted: TimeSource.Monotonic.ValueTimeMark,
+    private val budgetMillis: Long,
+    private val machineCallStarted: TimeSource.Monotonic.ValueTimeMark,
     private val onCandidate: (CallsCoverageCandidate) -> Unit,
     private val onExtractionFailure: (String) -> Unit,
 ) : UMachineObserver<TsState> {
     private val entryStatements: Set<EtsStmt> = method.cfg.stmts.toSet()
     private val coveredStatements = hashSetOf<EtsStmt>()
     private val inputKeys = hashSetOf<String>()
+    private var machineStarted: TimeSource.Monotonic.ValueTimeMark? = null
 
     var selectedStates: Int = 0
         private set
 
     var capReached: Boolean = false
         private set
 
+    var executedSteps: Long = 0
+        private set
+
+    var stepsWithinBudget: Long = 0
+        private set
+
+    var machineSetupElapsedMillis: Long = 0
+        private set
+
+    var searchElapsedMillis: Long = 0
+        private set
+
+    override fun onMachineStarted() {
+        machineSetupElapsedMillis = machineCallStarted.elapsedNow().inWholeMilliseconds
+        machineStarted = TimeSource.Monotonic.markNow()
+    }
+
+    override fun onMachineStopped() {
+        searchElapsedMillis = elapsedSinceMachineStart()
+    }
+
+    override fun onState(parent: TsState, forks: Sequence<TsState>) {
+        executedSteps++
+        if (elapsedSinceMachineStart() <= budgetMillis) {
+            stepsWithinBudget++
+        }
+    }
+
     override fun onStateTerminated(state: TsState, stateReachable: Boolean) {
         if (!stateReachable) return
 
@@ -231,7 +278,8 @@ private class CompletedCoverageCandidateObserver(
             val key = CallsExperimentJson.json.encodeToUtf8SafeString(inputs)
             if (inputKeys.add(key)) {
                 CallsCoverageCandidate(
-                    emittedAtMillis = searchStarted.elapsedNow().inWholeMilliseconds,
+                    emittedAtMillis = elapsedSinceMachineStart(),
+                    emittedAtStep = executedSteps,
                     inputs = inputs,
                     newSymbolicStatements = newStatements,
                     completion = completion,
@@ -249,6 +297,10 @@ private class CompletedCoverageCandidateObserver(
             coveredStatements += pathStatements
         }
     }
+
+    private fun elapsedSinceMachineStart(): Long = requireNotNull(machineStarted) {
+        "Coverage machine clock has not started"
+    }.elapsedNow().inWholeMilliseconds
 }
 
 private fun CallsSymbolicStatus.toCoverageStatus(): CallsCoverageSearchStatus = when (this) {