diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt index 2a04d11c7f..07bdc0faba 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt @@ -9,7 +9,25 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardCopyOption import java.nio.file.StandardOpenOption +import java.security.MessageDigest import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.TimeSource + +@Serializable +internal data class CallsCoverageUniverseIdentity( + val toolRevision: String, + val nativeFrontendRevision: String, + val sourceManifest: String, + val projectIndex: Int, + val functionIndex: Int, + val sourceFile: String, +) + +@Serializable +internal data class CallsCoverageUniverseRecord( + val identity: CallsCoverageUniverseIdentity, + val universe: CallsCoverageUniverse, +) @Serializable internal data class CallsCoverageCellIdentity( @@ -24,6 +42,7 @@ internal data class CallsCoverageCellIdentity( val budgetMillis: Long, val solverLimitMillis: Long, val candidateCap: Int, + val coverageUniverseSha256: String, ) @Serializable @@ -39,6 +58,7 @@ internal data class CallsCoverageCheckpoint( val emittedCandidates: Int, val replayedCandidates: Int, val coveredStatements: Int, + val totalStatements: Int, val coveredIfArms: Int, ) @@ -48,11 +68,47 @@ internal data class CallsCoverageCellResult( val search: CallsCoverageSearchResult, val replayErrors: Int, val completionMismatches: Int, + val replayElapsedMillis: Long, + val cellElapsedMillis: Long, val checkpoints: List, ) +internal fun runCoverageUniverse(args: List) { + require(args.size == 4) { "coverage-universe requires 4 arguments" } + val manifestPath = Path.of(args[0]).toRealPath() + val manifest = CallsExperimentJson.decodeManifest(Files.readString(manifestPath)) + val projectIndex = args[1].toInt() + val functionIndex = args[2].toInt() + val output = Path.of(args[3]).toAbsolutePath().normalize() + val project = manifest.projects[projectIndex] + val function = project.functions[functionIndex] + require(System.getenv("ETS_FRONTEND_SCRIPT") == null) { "ETS_FRONTEND_SCRIPT must be unset" } + + Files.createDirectories(output) + require(Files.list(output).use { entries -> entries.findAny().isEmpty }) { + "Coverage universe output directory is not empty" + } + val identity = CallsCoverageUniverseIdentity( + toolRevision = CallsBuildIdentity.toolRevision, + nativeFrontendRevision = CallsBuildIdentity.nativeFrontendRevision, + sourceManifest = manifestPath.toString(), + projectIndex = projectIndex, + functionIndex = functionIndex, + sourceFile = function.sourceFile, + ) + require(Regex("[0-9a-f]{40}").matches(identity.toolRevision)) { "Coverage build must be clean" } + val universe = OriginalTypeScriptCoverageReplayer( + sourceRoot = manifestPath.parent.resolve(project.sourceRoot).toRealPath(), + function = function, + ).use { replayer -> replayer.probeUniverse(timeoutMillis = REPLAY_TIMEOUT_MILLIS) } + val record = CallsCoverageUniverseRecord(identity = identity, universe = universe) + atomicWrite(output.resolve("universe.json"), encode(record)) + println(encode(record)) +} + internal fun runCoverageCell(args: List) { - require(args.size == 9) { "coverage-cell requires 9 arguments" } + require(args.size == 10) { "coverage-cell requires 10 arguments" } + val cellStarted = TimeSource.Monotonic.markNow() val manifestPath = Path.of(args[0]).toRealPath() val manifest = CallsExperimentJson.decodeManifest(Files.readString(manifestPath)) val projectIndex = args[1].toInt() @@ -63,11 +119,27 @@ internal fun runCoverageCell(args: List) { val budgetMillis = args[6].toLong() val solverLimitMillis = args[7].toLong() val candidateCap = args[8].toInt() + val universeBytes = Files.readAllBytes(Path.of(args[9]).toRealPath()) + val universeRecord = CallsExperimentJson.json.decodeFromString( + universeBytes.toString(Charsets.UTF_8), + ) val project = manifest.projects[projectIndex] val function = project.functions[functionIndex] require(seed in manifest.seeds) { "Seed is outside the frozen source manifest" } require(budgetMillis > 0 && solverLimitMillis in 1..budgetMillis && candidateCap > 0) require(System.getenv("ETS_FRONTEND_SCRIPT") == null) { "ETS_FRONTEND_SCRIPT must be unset" } + require(universeRecord.identity == CallsCoverageUniverseIdentity( + toolRevision = CallsBuildIdentity.toolRevision, + nativeFrontendRevision = CallsBuildIdentity.nativeFrontendRevision, + sourceManifest = manifestPath.toString(), + projectIndex = projectIndex, + functionIndex = functionIndex, + sourceFile = function.sourceFile, + )) { "Coverage universe identity differs" } + val universe = universeRecord.universe + require(function.sourceFile in universe.sourceFiles && universe.supportedStatementKeys.isNotEmpty()) { + "Coverage universe does not include the entry source statements" + } val identity = CallsCoverageCellIdentity( schemaVersion = 1, @@ -81,6 +153,8 @@ internal fun runCoverageCell(args: List) { budgetMillis = budgetMillis, solverLimitMillis = solverLimitMillis, candidateCap = candidateCap, + coverageUniverseSha256 = MessageDigest.getInstance("SHA-256").digest(universeBytes) + .joinToString("") { byte -> "%02x".format(byte.toInt() and 0xff) }, ) require(Regex("[0-9a-f]{40}").matches(identity.toolRevision)) { "Coverage build must be clean" } @@ -152,12 +226,18 @@ internal fun runCoverageCell(args: List) { search.candidates[index].emittedAtMillis <= checkpoint } val replayed = indices.mapNotNull { index -> events[index].replay } + val coveredStatements = replayed.flatMap { it.coveredStatementKeys }.toSet() + val coveredIfArms = replayed.flatMap { it.coveredIfArmKeys }.toSet() + require(universe.supportedStatementKeys.containsAll(coveredStatements)) { + "Replayed statements extend beyond the frozen coverage universe" + } CallsCoverageCheckpoint( searchBudgetMillis = checkpoint, emittedCandidates = indices.size, replayedCandidates = replayed.size, - coveredStatements = replayed.flatMap { it.coveredStatementKeys }.toSet().size, - coveredIfArms = replayed.flatMap { it.coveredIfArmKeys }.toSet().size, + coveredStatements = coveredStatements.size, + totalStatements = universe.supportedStatementKeys.size, + coveredIfArms = coveredIfArms.size, ) } val result = CallsCoverageCellResult( @@ -168,6 +248,8 @@ internal fun runCoverageCell(args: List) { events[index].replay?.completion != null && events[index].replay?.completion != search.candidates[index].completion }, + replayElapsedMillis = events.sumOf { event -> event.replay?.replayElapsedMillis ?: 0L }, + cellElapsedMillis = cellStarted.elapsedNow().inWholeMilliseconds, checkpoints = checkpoints, ) atomicWrite(output.resolve("result.json"), encode(result)) diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageReplay.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageReplay.kt index 01d5b983bf..78f4f60921 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageReplay.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageReplay.kt @@ -38,6 +38,14 @@ internal data class CallsCoverageReplayResult( val replayElapsedMillis: Long, ) +@Serializable +internal data class CallsCoverageUniverse( + val supportedStatementKeys: Set, + val sourceFiles: Set, + val diagnostics: List, + val probeElapsedMillis: Long, +) + /** Replays one generated input in a fresh Node.js process and checks its exact transport value. */ internal class OriginalTypeScriptCoverageReplayer( sourceRoot: Path, @@ -87,6 +95,58 @@ internal class OriginalTypeScriptCoverageReplayer( } } + fun probeUniverse(timeoutMillis: Long): CallsCoverageUniverse { + require(timeoutMillis > 0) + + val started = TimeSource.Monotonic.markNow() + val result = backend.run( + property = PropertyDefinition( + id = PropertyId("calls.coverage.universe"), + inputs = listOf(PropertyInput(name = "__usvmCoverageSentinel", domain = BooleanDomain)), + predicate = TypeScriptEntryPoint(module = wrapperName, exportName = "coverageUniverseProbe"), + ), + configuration = PropertyRunConfiguration( + seed = 0, + numRuns = 1, + timeoutMillis = timeoutMillis, + examples = listOf(listOf(JsConcreteValue.Boolean(true))), + coverageRequest = PropertyCoverageRequest( + scopes = setOf(CoverageScope.SOURCE_UNDER_TEST), + includePatterns = allowedFiles.sorted(), + includeUnexecutedSources = true, + ), + ), + ) + val elapsedMillis = started.elapsedNow().inWholeMilliseconds + require(result.status == PropertyRunStatus.SUCCESS && result.numRuns == 1) { + "Coverage universe probe did not complete once: $result" + } + + val coverage = requireNotNull(result.coverage) { "Coverage universe probe produced no c8 artifact" } + val statements = linkedSetOf() + val sourceFiles = linkedSetOf() + coverage.files.forEach { file -> + val relative = relativeSourcePath(file.path) ?: return@forEach + if (relative !in allowedFiles) return@forEach + + sourceFiles += relative + file.statements.forEach { statement -> + statements += "$relative:statement:${statement.location.stableKey()}" + } + } + require(sourceFiles == allowedFiles) { + "Coverage universe omitted local source files: ${allowedFiles - sourceFiles}" + } + require(statements.isNotEmpty()) { "Coverage universe has no source statements" } + + return CallsCoverageUniverse( + supportedStatementKeys = statements, + sourceFiles = sourceFiles, + diagnostics = coverage.diagnostics.map { diagnostic -> "${diagnostic.code}: ${diagnostic.message}" }, + probeElapsedMillis = elapsedMillis, + ) + } + fun replay(inputs: List, timeoutMillis: Long): CallsCoverageReplayResult { require(timeoutMillis > 0) require(inputs.size == function.inputs.size) @@ -246,5 +306,9 @@ private fun coverageWrapper( return true; } + + export function coverageUniverseProbe(_sentinel: boolean): boolean { + return true; + } """.trimIndent() + "\n" } diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt index 87a7b710bf..c8eab2c904 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageSearch.kt @@ -22,6 +22,7 @@ import kotlin.time.TimeSource @Serializable internal data class CallsCoverageCandidate( val emittedAtMillis: Long, + val emittedAtStep: Long, val inputs: List, val newSymbolicStatements: Int, val completion: CallsCoverageCompletion, @@ -49,10 +50,14 @@ internal data class CallsCoverageSearchResult( val status: CallsCoverageSearchStatus, val candidates: List, val selectedStates: Int, + val executedSteps: Long, + val stepsWithinBudget: Long, val extractionFailures: List, val candidateCapReached: Boolean, val preparationElapsedMillis: Long, + val machineSetupElapsedMillis: Long, val searchElapsedMillis: Long, + val machineTeardownElapsedMillis: Long, val unsupportedCall: Boolean = false, val engineFailed: Boolean = false, val runtimeLimited: Boolean = false, @@ -99,10 +104,14 @@ internal class CurrentTsCallsCoverageEngine( status = preparation.status.toCoverageStatus(), candidates = emptyList(), selectedStates = 0, + executedSteps = 0, + stepsWithinBudget = 0, extractionFailures = emptyList(), candidateCapReached = false, preparationElapsedMillis = preparationElapsedMillis, + machineSetupElapsedMillis = 0, searchElapsedMillis = 0, + machineTeardownElapsedMillis = 0, diagnostic = "${preparation.reasonCode}: ${preparation.diagnostic}", ) } @@ -115,13 +124,14 @@ internal class CurrentTsCallsCoverageEngine( ) val candidates = mutableListOf() val extractionFailures = mutableListOf() - val searchStarted = TimeSource.Monotonic.markNow() + val machineCallStarted = TimeSource.Monotonic.markNow() val observer = CompletedCoverageCandidateObserver( method = preparation.method, inputDomains = request.function.inputs, symbolicInputs = symbolicInputs, candidateCap = request.candidateCap, - searchStarted = searchStarted, + budgetMillis = request.budget.inWholeMilliseconds, + machineCallStarted = machineCallStarted, onCandidate = { candidate -> request.onCandidate(candidate) candidates += candidate @@ -157,7 +167,10 @@ internal class CurrentTsCallsCoverageEngine( machineObserver = observer, ).use { machine -> machine.analyzeWithMetadata(methods = listOf(preparation.method)) } } - val searchElapsedMillis = searchStarted.elapsedNow().inWholeMilliseconds + val machineCallElapsedMillis = machineCallStarted.elapsedNow().inWholeMilliseconds + val machineTeardownElapsedMillis = ( + machineCallElapsedMillis - observer.machineSetupElapsedMillis - observer.searchElapsedMillis + ).coerceAtLeast(0) val outcome = machineResult.getOrNull() val status = when { machineResult.isFailure -> CallsCoverageSearchStatus.TOOL_ERROR @@ -170,10 +183,14 @@ internal class CurrentTsCallsCoverageEngine( status = status, candidates = candidates, selectedStates = observer.selectedStates, + executedSteps = observer.executedSteps, + stepsWithinBudget = observer.stepsWithinBudget, extractionFailures = extractionFailures, candidateCapReached = observer.capReached, preparationElapsedMillis = preparationElapsedMillis, - searchElapsedMillis = searchElapsedMillis, + machineSetupElapsedMillis = observer.machineSetupElapsedMillis, + searchElapsedMillis = observer.searchElapsedMillis, + machineTeardownElapsedMillis = machineTeardownElapsedMillis, unsupportedCall = outcome?.unsupportedCall == true, engineFailed = outcome?.engineFailed == true, runtimeLimited = outcome?.runtimeLimited == true, @@ -187,13 +204,15 @@ private class CompletedCoverageCandidateObserver( private val inputDomains: List, private val symbolicInputs: CallsSymbolicInputs, private val candidateCap: Int, - private val searchStarted: TimeSource.Monotonic.ValueTimeMark, + private val budgetMillis: Long, + private val machineCallStarted: TimeSource.Monotonic.ValueTimeMark, private val onCandidate: (CallsCoverageCandidate) -> Unit, private val onExtractionFailure: (String) -> Unit, ) : UMachineObserver { private val entryStatements: Set = method.cfg.stmts.toSet() private val coveredStatements = hashSetOf() private val inputKeys = hashSetOf() + private var machineStarted: TimeSource.Monotonic.ValueTimeMark? = null var selectedStates: Int = 0 private set @@ -201,6 +220,34 @@ private class CompletedCoverageCandidateObserver( var capReached: Boolean = false private set + var executedSteps: Long = 0 + private set + + var stepsWithinBudget: Long = 0 + private set + + var machineSetupElapsedMillis: Long = 0 + private set + + var searchElapsedMillis: Long = 0 + private set + + override fun onMachineStarted() { + machineSetupElapsedMillis = machineCallStarted.elapsedNow().inWholeMilliseconds + machineStarted = TimeSource.Monotonic.markNow() + } + + override fun onMachineStopped() { + searchElapsedMillis = elapsedSinceMachineStart() + } + + override fun onState(parent: TsState, forks: Sequence) { + executedSteps++ + if (elapsedSinceMachineStart() <= budgetMillis) { + stepsWithinBudget++ + } + } + override fun onStateTerminated(state: TsState, stateReachable: Boolean) { if (!stateReachable) return @@ -231,7 +278,8 @@ private class CompletedCoverageCandidateObserver( val key = CallsExperimentJson.json.encodeToUtf8SafeString(inputs) if (inputKeys.add(key)) { CallsCoverageCandidate( - emittedAtMillis = searchStarted.elapsedNow().inWholeMilliseconds, + emittedAtMillis = elapsedSinceMachineStart(), + emittedAtStep = executedSteps, inputs = inputs, newSymbolicStatements = newStatements, completion = completion, @@ -249,6 +297,10 @@ private class CompletedCoverageCandidateObserver( coveredStatements += pathStatements } } + + private fun elapsedSinceMachineStart(): Long = requireNotNull(machineStarted) { + "Coverage machine clock has not started" + }.elapsedNow().inWholeMilliseconds } private fun CallsSymbolicStatus.toCoverageStatus(): CallsCoverageSearchStatus = when (this) { diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperimentCli.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperimentCli.kt index 1229d14d8d..9588cf70ef 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperimentCli.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsExperimentCli.kt @@ -10,6 +10,7 @@ fun main(args: Array) { when (args.first()) { "run" -> runExperiment(args.drop(1)) + "coverage-universe" -> runCoverageUniverse(args.drop(1)) "coverage-cell" -> runCoverageCell(args.drop(1)) "replay-witness" -> replayWitness(args.drop(1)) "summarize" -> summarize(args.drop(1)) @@ -89,7 +90,8 @@ private fun summarize(args: List) { private fun usage(): String = """ Usage: calls run - calls coverage-cell + calls coverage-universe + calls coverage-cell calls replay-witness calls summarize """.trimIndent() diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsCoverageExperimentTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsCoverageExperimentTest.kt index 1a545e3288..cef7a7725e 100644 --- a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsCoverageExperimentTest.kt +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsCoverageExperimentTest.kt @@ -42,17 +42,79 @@ class CallsCoverageExperimentTest { assertEquals(search.candidates, emitted) assertEquals(2, search.candidates.map { it.inputs }.distinct().size, search.toString()) assertTrue(search.candidates.all { candidate -> candidate.emittedAtMillis <= search.searchElapsedMillis }) + assertTrue(search.executedSteps > 0) + assertTrue(search.stepsWithinBudget in 1..search.executedSteps) + assertTrue(search.candidates.all { candidate -> candidate.emittedAtStep in 1..search.executedSteps }) + assertTrue(search.machineSetupElapsedMillis >= 0) + assertTrue(search.machineTeardownElapsedMillis >= 0) - val replayed = OriginalTypeScriptCoverageReplayer( + val (universe, replayed) = OriginalTypeScriptCoverageReplayer( sourceRoot = fixture.sourceRoot, function = fixture.function, ).use { replayer -> - search.candidates.map { candidate -> + val universe = replayer.probeUniverse(timeoutMillis = 20_000L) + val replayed = search.candidates.map { candidate -> replayer.replay(inputs = candidate.inputs, timeoutMillis = 20_000L) } + universe to replayed } assertEquals(2, replayed.flatMap { it.coveredIfArmKeys }.toSet().size, replayed.toString()) assertTrue(replayed.all { replay -> replay.coveredStatementKeys.isNotEmpty() }) + assertTrue(replayed.all { replay -> replay.coveredStatementKeys.size < universe.supportedStatementKeys.size }) + assertTrue(universe.supportedStatementKeys.containsAll(replayed.flatMap { it.coveredStatementKeys })) + assertEquals(universe.supportedStatementKeys, replayed.flatMap { it.coveredStatementKeys }.toSet()) + assertTrue(universe.probeElapsedMillis >= 0) + } + + @Test + fun `coverage universe has a denominator without invoking the entry function`() { + val fixture = fixture( + source = """ + export function neverRun(flag: boolean): boolean { + if (flag) throw new Error('The probe must not invoke this function'); + return false; + } + """.trimIndent(), + exportName = "neverRun", + inputs = listOf(PropertyInput(name = "flag", domain = BooleanDomain)), + ) + + val universe = OriginalTypeScriptCoverageReplayer( + sourceRoot = fixture.sourceRoot, + function = fixture.function, + ).use { replayer -> replayer.probeUniverse(timeoutMillis = 20_000L) } + + assertTrue(universe.supportedStatementKeys.isNotEmpty()) + assertTrue(universe.supportedStatementKeys.size >= 4, universe.toString()) + assertEquals(setOf("Fixture.ts"), universe.sourceFiles) + } + + @Test + fun `coverage universe includes the complete local runtime source closure`() { + val fixture = fixture( + source = """ + import { choose } from './Helper'; + export function classify(flag: boolean): number { return choose(flag); } + """.trimIndent(), + exportName = "classify", + inputs = listOf(PropertyInput(name = "flag", domain = BooleanDomain)), + additionalSources = mapOf( + "Helper.ts" to """ + export function choose(flag: boolean): number { + if (flag) return 1; + return 2; + } + """.trimIndent(), + ), + ) + + val universe = OriginalTypeScriptCoverageReplayer( + sourceRoot = fixture.sourceRoot, + function = fixture.function, + ).use { replayer -> replayer.probeUniverse(timeoutMillis = 20_000L) } + + assertEquals(setOf("Fixture.ts", "Helper.ts"), universe.sourceFiles) + assertTrue(universe.supportedStatementKeys.any { key -> key.startsWith("Helper.ts:") }) } @Test @@ -118,9 +180,15 @@ class CallsCoverageExperimentTest { bundledNativeFrontendRevision = "bundled:test", ) - private fun fixture(source: String, exportName: String, inputs: List): CoverageFixture { + private fun fixture( + source: String, + exportName: String, + inputs: List, + additionalSources: Map = emptyMap(), + ): CoverageFixture { val sourceRoot = Files.createDirectory(directory.resolve(exportName)) Files.writeString(sourceRoot.resolve("Fixture.ts"), source) + additionalSources.forEach { (name, contents) -> Files.writeString(sourceRoot.resolve(name), contents) } runGit(sourceRoot, "init") runGit(sourceRoot, "config", "user.name", "USVM Tests") runGit(sourceRoot, "config", "user.email", "usvm@example.test") diff --git a/usvm-ts-fast-check/fast-check-adapter/src/local-source-closure.ts b/usvm-ts-fast-check/fast-check-adapter/src/local-source-closure.ts index 16c84df32f..68978c0414 100644 --- a/usvm-ts-fast-check/fast-check-adapter/src/local-source-closure.ts +++ b/usvm-ts-fast-check/fast-check-adapter/src/local-source-closure.ts @@ -33,6 +33,7 @@ export function inspectLocalSourceClosure(rootPath: string, entryPath: string): const source = ts.createSourceFile(file, readFileSync(file, 'utf8'), ts.ScriptTarget.Latest, true) sources.set(file, source) + const emittedModules = runtimeModuleSpecifiers(source) const edges: string[] = [] dependencies.set(file, edges) const imports: string[] = [] @@ -48,6 +49,8 @@ export function inspectLocalSourceClosure(rootPath: string, entryPath: string): if (!ts.isStringLiteral(statement.moduleSpecifier)) { return reject('IMPORTED_CALLEES_UNSUPPORTED', `Nonliteral module specifier in ${relative}`) } + if (!emittedModules.has(statement.moduleSpecifier.text) + && importedBindingsUsedOnlyAsTypes(statement, source)) continue imports.push(statement.moduleSpecifier.text) ranges.push({ start: statement.getStart(source), end: statement.end }) } else if (ts.isExportDeclaration(statement)) { @@ -55,6 +58,8 @@ export function inspectLocalSourceClosure(rootPath: string, entryPath: string): if (statement.exportClause && ts.isNamedExports(statement.exportClause) && statement.exportClause.elements.length > 0 && statement.exportClause.elements.every(element => element.isTypeOnly)) continue + if (statement.moduleSpecifier && ts.isStringLiteral(statement.moduleSpecifier) + && !emittedModules.has(statement.moduleSpecifier.text)) continue ranges.push({ start: statement.getStart(source), end: statement.end }) if (statement.moduleSpecifier) { if (!ts.isStringLiteral(statement.moduleSpecifier)) { @@ -156,6 +161,51 @@ export function inspectLocalSourceClosure(rootPath: string, entryPath: string): return { files: [...files].sort() } } +function runtimeModuleSpecifiers(source: ts.SourceFile): Set { + const emitted = ts.transpileModule(source.text, { + fileName: source.fileName, + compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ESNext }, + }).outputText + const javascript = ts.createSourceFile(`${source.fileName}.js`, emitted, ts.ScriptTarget.ES2022, true, ts.ScriptKind.JS) + const specifiers = new Set() + for (const statement of javascript.statements) { + if ((ts.isImportDeclaration(statement) || ts.isExportDeclaration(statement)) + && statement.moduleSpecifier && ts.isStringLiteral(statement.moduleSpecifier)) { + specifiers.add(statement.moduleSpecifier.text) + } + } + return specifiers +} + +function importedBindingsUsedOnlyAsTypes(statement: ts.ImportDeclaration, source: ts.SourceFile): boolean { + const clause = statement.importClause + if (!clause) return false + + const names: string[] = [] + if (clause.name) names.push(clause.name.text) + if (clause.namedBindings && ts.isNamedImports(clause.namedBindings)) { + names.push(...clause.namedBindings.elements.map(element => element.name.text)) + } else if (clause.namedBindings && ts.isNamespaceImport(clause.namedBindings)) { + names.push(clause.namedBindings.name.text) + } + if (names.length === 0) return false + + const typeUses = new Set() + const valueUses = new Set() + const visit = (node: ts.Node): void => { + if (ts.isImportDeclaration(node)) return + if (ts.isIdentifier(node) && names.includes(node.text)) { + let ancestor: ts.Node | undefined = node.parent + while (ancestor && ancestor !== source && !ts.isTypeNode(ancestor)) ancestor = ancestor.parent + if (ancestor && ts.isTypeNode(ancestor)) typeUses.add(node.text) + else valueUses.add(node.text) + } + ts.forEachChild(node, visit) + } + visit(source) + return names.every(name => typeUses.has(name) && !valueUses.has(name)) +} + function hasDependencyCycle(dependencies: Map): boolean { const active = new Set() const visited = new Set() diff --git a/usvm-ts-fast-check/fast-check-adapter/test/local-source-closure.test.ts b/usvm-ts-fast-check/fast-check-adapter/test/local-source-closure.test.ts index b8441521cd..bc78137683 100644 --- a/usvm-ts-fast-check/fast-check-adapter/test/local-source-closure.test.ts +++ b/usvm-ts-fast-check/fast-check-adapter/test/local-source-closure.test.ts @@ -34,6 +34,19 @@ test('local closure follows reexports while ignoring unrelated effects and type }) }) +test('local closure follows emitted imports and omits imports used only as types', () => { + workspace({ + 'entry.ts': "import { Nullish } from './types'; import { helper } from './helper'; " + + 'export function entry(value: unknown): value is Nullish { return helper(value); }', + 'types.ts': 'declare const OPAQUE_TAG: unique symbol; export type Nullish = null | undefined;', + 'helper.ts': 'export function helper(value: unknown): boolean { return value == null; }', + }, root => { + assert.deepEqual(inspectLocalSourceClosure(root, path.join(root, 'entry.ts')), { + files: ['entry.ts', 'helper.ts'], + }) + }) +}) + for (const [name, entry, helper, expected] of [ ['external import', "import { x } from 'node:fs';", '', 'IMPORTED_CALLEES_UNSUPPORTED'], ['missing module', "import { helper } from './missing';", '', 'IMPORTED_CALLEES_UNSUPPORTED'], diff --git a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageSession.kt b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageSession.kt index acaf37c560..6ee5c236d4 100644 --- a/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageSession.kt +++ b/usvm-ts-fast-check/src/main/kotlin/org/usvm/ts/pbt/fastcheck/FastCheckCoverageSession.kt @@ -37,6 +37,20 @@ internal class FastCheckCoverageSession private constructor( add("--exclude-after-remap") add("--allowExternal") add("--exclude=__usvm_no_default_excludes__") + if (requireNotNull(request.coverageRequest).includeUnexecutedSources) { + add("--all") + request.sourceRoots.forEach { sourceRoot -> add("--src=$sourceRoot") } + requireNotNull(request.coverageRequest).includePatterns.forEach { pattern -> + val include = Path.of(pattern) + if (include.isAbsolute) { + add("--include=$include") + } else { + request.sourceRoots.forEach { sourceRoot -> + add("--include=${Path.of(sourceRoot).resolve(include)}") + } + } + } + } if (CoverageScope.DEPENDENCIES in requireNotNull(request.coverageRequest).scopes) { add("--exclude-node-modules=false") } diff --git a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt index bbc0ff5db3..1b29ef0765 100644 --- a/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt +++ b/usvm-ts-pbt/src/main/kotlin/org/usvm/ts/pbt/backend/PropertyCoverage.kt @@ -118,6 +118,7 @@ data class PropertyCoverageRequest( val scopes: Set = setOf(CoverageScope.SOURCE_UNDER_TEST), val includePatterns: List = emptyList(), val excludePatterns: List = emptyList(), + val includeUnexecutedSources: Boolean = false, ) { init { require(scopes.isNotEmpty()) { "At least one coverage scope is required" }