From e41f58e6b4c10078fd312741008dd4a5fd74ab07 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Oct 2026 10:59:36 +0100 Subject: [PATCH 1/2] [mypyc] Import setuptools lazily in mypyc.build on Python 3.12+ Importing mypyc.build used to always import setuptools, distutils and mypyc.build_setup, but mypyc_build (type checking and C generation) needs none of them. They are only needed for building extensions, e.g. in mypycify. Importing them is slow, which adds significant fixed overhead to tools that call mypyc_build directly for many small builds, such as Bazel integrations. Move these imports into import_distutils(), called by the functions that need them. On Python versions before 3.12, distutils is still in the stdlib and setuptools must be imported before it to override it, so keep importing everything eagerly there to preserve backward compatibility. In a quick local simulation (interpreted mypy, Python 3.14, warm caches, tiny module, calling mypyc_build like a Bazel driver), this saved about 31-37 ms of a ~190 ms run (16-19%). The number of setuptools-related modules imported went from 102 to 3. --- mypyc/build.py | 58 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/mypyc/build.py b/mypyc/build.py index 8c6eabead17c9..e2faab40aeeeb 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -13,9 +13,14 @@ See the mypycify docs for additional arguments. -mypycify can integrate with either distutils or setuptools, but needs -to know at import-time whether it is using distutils or setuputils. We -hackily decide based on whether setuptools has been imported already. +mypycify uses setuptools if it's installed. On Python versions before 3.12, +it falls back to distutils if setuptools isn't installed. + +setuptools and distutils are only imported when they are needed to build +extensions (such as in mypycify), since importing them is slow and +generating C via mypyc_build doesn't need them. On Python versions before +3.12, they are imported when this module is imported, since setuptools must +be imported before distutils to override it. """ from __future__ import annotations @@ -28,7 +33,6 @@ from collections.abc import Iterable from typing import TYPE_CHECKING, Any, NamedTuple, NoReturn, cast -import mypyc.build_setup # noqa: F401 from mypy.build import BuildSource from mypy.errors import CompileError from mypy.fscache import FileSystemCache @@ -135,12 +139,6 @@ class ModDesc(NamedTuple): ), ] -try: - # Import setuptools so that it monkey-patch overrides distutils - import setuptools -except ImportError: - pass - if TYPE_CHECKING: if sys.version_info >= (3, 12): from setuptools import Extension @@ -152,14 +150,39 @@ class ModDesc(NamedTuple): Extension: TypeAlias = _setuptools_Extension | _distutils_Extension -if sys.version_info >= (3, 12): - # From setuptools' monkeypatch - from distutils import ccompiler, sysconfig # type: ignore[import-not-found] -else: - from distutils import ccompiler, sysconfig + +def import_distutils() -> tuple[Any, Any]: + """Import and return the distutils ccompiler and sysconfig modules. + + This also imports setuptools (if available) first, so that it overrides distutils, + and patches the compiler to support per-file flags. These imports are slow, and + only needed when building C extensions, so this is done lazily (C generation in + mypyc_build doesn't need it). + """ + try: + # Import setuptools so that it monkey-patch overrides distutils + import setuptools # noqa: F401 + except ImportError: + pass + + import mypyc.build_setup # noqa: F401 + + if sys.version_info >= (3, 12): + # From setuptools' monkeypatch + from distutils import ccompiler, sysconfig # type: ignore[import-not-found] + else: + from distutils import ccompiler, sysconfig + return ccompiler, sysconfig + + +if sys.version_info < (3, 12): + # Distutils is still in the stdlib, and setuptools must be imported before it to + # override it, so preserve the old behavior of importing these eagerly. + import_distutils() def get_extension() -> type[Extension]: + import_distutils() # We can work with either setuptools or distutils, and pick setuptools # if it has been imported. use_setuptools = "setuptools" in sys.modules @@ -172,6 +195,8 @@ def get_extension() -> type[Extension]: else: if not use_setuptools: sys.exit("error: setuptools not installed") + import setuptools + extension_class = setuptools.Extension return extension_class @@ -182,6 +207,7 @@ def setup_mypycify_vars() -> None: # There has to be a better approach to this. # The vars can contain ints but we only work with str ones + _, sysconfig = import_distutils() vars = cast(dict[str, str], sysconfig.get_config_vars()) if sys.platform == "darwin": # Disable building 32-bit binaries, since we generate too much code @@ -805,6 +831,7 @@ def get_cflags( List of compiler flags. """ if compiler_type is None: + ccompiler, sysconfig = import_distutils() compiler: Any = ccompiler.new_compiler() sysconfig.customize_compiler(compiler) compiler_type = compiler.compiler_type @@ -952,6 +979,7 @@ def mypycify( also needed if using experimental librt features). These have no backward compatibility guarantees! """ + ccompiler, sysconfig = import_distutils() # Skip redundant inplace .so copies on every build_ext invocation. _patch_setuptools_copy_extensions_to_source() From ab3007184271b93dd6ae9de16ce9bbec92d94a08 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 2 Oct 2026 11:14:15 +0100 Subject: [PATCH 2/2] [mypyc] Fix docstring, librt_cache distutils import, add regression test - The module docstring overstated the distutils fallback: mypycify already requires setuptools (via the copy_extensions_to_source patch). - librt_cache relied on importing mypyc.build to set up distutils, which is no longer true on 3.12+. Use import_distutils() explicitly. - Add a subprocess test checking that C generation via mypyc_build doesn't import setuptools, distutils or mypyc.build_setup. --- mypyc/build.py | 3 +-- mypyc/test/librt_cache.py | 5 ++--- mypyc/test/test_misc.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/mypyc/build.py b/mypyc/build.py index e2faab40aeeeb..ee01858887dd3 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -13,8 +13,7 @@ See the mypycify docs for additional arguments. -mypycify uses setuptools if it's installed. On Python versions before 3.12, -it falls back to distutils if setuptools isn't installed. +mypycify requires setuptools. setuptools and distutils are only imported when they are needed to build extensions (such as in mypycify), since importing them is slow and diff --git a/mypyc/test/librt_cache.py b/mypyc/test/librt_cache.py index bf57b9f54196e..751df36bf1291 100644 --- a/mypyc/test/librt_cache.py +++ b/mypyc/test/librt_cache.py @@ -29,15 +29,14 @@ import filelock -from mypyc.build import LIBRT_MODULES, get_cflags, include_dir +from mypyc.build import LIBRT_MODULES, get_cflags, import_distutils, include_dir from mypyc.common import RUNTIME_C_FILES from mypyc.test.config import PREFIX def _librt_build_hash(experimental: bool, opt_level: str) -> str: """Compute hash for librt build, including sources and build environment.""" - # Import lazily to ensure mypyc.build has ensured that distutils is correctly set up - from distutils import ccompiler + ccompiler, _ = import_distutils() h = hashlib.sha256() # Include experimental flag diff --git a/mypyc/test/test_misc.py b/mypyc/test/test_misc.py index 816875fcc23db..d838d7778925a 100644 --- a/mypyc/test/test_misc.py +++ b/mypyc/test/test_misc.py @@ -1,7 +1,10 @@ from __future__ import annotations import os +import subprocess +import sys import tempfile +import textwrap import unittest from mypyc.build import get_header_deps, resolve_cfile_deps @@ -9,6 +12,7 @@ from mypyc.ir.pprint import format_blocks, generate_names_for_ir from mypyc.irbuild.ll_builder import LowLevelIRBuilder from mypyc.options import CompilerOptions +from mypyc.test.config import PREFIX class TestMisc(unittest.TestCase): @@ -150,3 +154,29 @@ def test_resolve_search_order_matches_preprocessor(self) -> None: assert resolve_cfile_deps( cfile_dir=includer, direct_includes=[(True, "shared.h")], target_dir=target ) == {global_h} + + +class TestLazySetuptoolsImport(unittest.TestCase): + @unittest.skipIf(sys.version_info < (3, 12), "setuptools is imported eagerly before 3.12") + def test_c_generation_does_not_import_setuptools(self) -> None: + # Importing setuptools is slow, and generating C via mypyc_build doesn't need it. + script = textwrap.dedent(""" + import os + import sys + from mypyc.build import mypyc_build + from mypyc.options import CompilerOptions + + mypyc_build(["a.py"], CompilerOptions(target_dir="build"), only_compile_paths=["a.py"]) + assert any(f.endswith(".c") for f in os.listdir("build")), "no C generated" + loaded = {"setuptools", "distutils", "mypyc.build_setup"} & set(sys.modules) + assert not loaded, f"unexpectedly imported: {sorted(loaded)}" + """) + with tempfile.TemporaryDirectory() as tmpdir: + with open(os.path.join(tmpdir, "a.py"), "w") as f: + f.write("def f(x: int) -> int:\n return x + 1\n") + env = os.environ.copy() + env["PYTHONPATH"] = PREFIX + os.pathsep + env.get("PYTHONPATH", "") + result = subprocess.run( + [sys.executable, "-c", script], cwd=tmpdir, env=env, capture_output=True, text=True + ) + assert result.returncode == 0, result.stdout + result.stderr