[mypyc] Import setuptools lazily in mypyc.build on Python 3.12+

mypyc/build.py

@@ -13,9 +13,13 @@
 
 See the mypycify docs for additional arguments.
 
-mypycify can integrate with either distutils or setuptools, but needs
-to know at import-time whether it is using distutils or setuputils. We
-hackily decide based on whether setuptools has been imported already.
+mypycify requires setuptools.
+
+setuptools and distutils are only imported when they are needed to build
+extensions (such as in mypycify), since importing them is slow and
+generating C via mypyc_build doesn't need them. On Python versions before
+3.12, they are imported when this module is imported, since setuptools must
+be imported before distutils to override it.
 """
 
 from __future__ import annotations
@@ -28,7 +32,6 @@
 from collections.abc import Iterable
 from typing import TYPE_CHECKING, Any, NamedTuple, NoReturn, cast
 
-import mypyc.build_setup  # noqa: F401
 from mypy.build import BuildSource
 from mypy.errors import CompileError
 from mypy.fscache import FileSystemCache
@@ -135,12 +138,6 @@ class ModDesc(NamedTuple):
     ),
 ]
 
-try:
-    # Import setuptools so that it monkey-patch overrides distutils
-    import setuptools
-except ImportError:
-    pass
-
 if TYPE_CHECKING:
     if sys.version_info >= (3, 12):
         from setuptools import Extension
@@ -152,14 +149,39 @@ class ModDesc(NamedTuple):
 
         Extension: TypeAlias = _setuptools_Extension | _distutils_Extension
 
-if sys.version_info >= (3, 12):
-    # From setuptools' monkeypatch
-    from distutils import ccompiler, sysconfig  # type: ignore[import-not-found]
-else:
-    from distutils import ccompiler, sysconfig
+
+def import_distutils() -> tuple[Any, Any]:
+    """Import and return the distutils ccompiler and sysconfig modules.
+
+    This also imports setuptools (if available) first, so that it overrides distutils,
+    and patches the compiler to support per-file flags. These imports are slow, and
+    only needed when building C extensions, so this is done lazily (C generation in
+    mypyc_build doesn't need it).
+    """
+    try:
+        # Import setuptools so that it monkey-patch overrides distutils
+        import setuptools  # noqa: F401
+    except ImportError:
+        pass
+
+    import mypyc.build_setup  # noqa: F401
+
+    if sys.version_info >= (3, 12):
+        # From setuptools' monkeypatch
+        from distutils import ccompiler, sysconfig  # type: ignore[import-not-found]
+    else:
+        from distutils import ccompiler, sysconfig
+    return ccompiler, sysconfig
+
+
+if sys.version_info < (3, 12):
+    # Distutils is still in the stdlib, and setuptools must be imported before it to
+    # override it, so preserve the old behavior of importing these eagerly.
+    import_distutils()
 
 
 def get_extension() -> type[Extension]:
+    import_distutils()
     # We can work with either setuptools or distutils, and pick setuptools
     # if it has been imported.
     use_setuptools = "setuptools" in sys.modules
@@ -172,6 +194,8 @@ def get_extension() -> type[Extension]:
     else:
         if not use_setuptools:
             sys.exit("error: setuptools not installed")
+        import setuptools
+
         extension_class = setuptools.Extension
 
     return extension_class
@@ -182,6 +206,7 @@ def setup_mypycify_vars() -> None:
     # There has to be a better approach to this.
 
     # The vars can contain ints but we only work with str ones
+    _, sysconfig = import_distutils()
     vars = cast(dict[str, str], sysconfig.get_config_vars())
     if sys.platform == "darwin":
         # Disable building 32-bit binaries, since we generate too much code
@@ -805,6 +830,7 @@ def get_cflags(
         List of compiler flags.
     """
     if compiler_type is None:
+        ccompiler, sysconfig = import_distutils()
         compiler: Any = ccompiler.new_compiler()
         sysconfig.customize_compiler(compiler)
         compiler_type = compiler.compiler_type
@@ -952,6 +978,7 @@ def mypycify(
                                also needed if using experimental librt features). These
                                have no backward compatibility guarantees!
     """
+    ccompiler, sysconfig = import_distutils()
 
     # Skip redundant inplace .so copies on every build_ext invocation.
     _patch_setuptools_copy_extensions_to_source()

mypyc/test/librt_cache.py

@@ -29,15 +29,14 @@
 
 import filelock
 
-from mypyc.build import LIBRT_MODULES, get_cflags, include_dir
+from mypyc.build import LIBRT_MODULES, get_cflags, import_distutils, include_dir
 from mypyc.common import RUNTIME_C_FILES
 from mypyc.test.config import PREFIX
 
 
 def _librt_build_hash(experimental: bool, opt_level: str) -> str:
     """Compute hash for librt build, including sources and build environment."""
-    # Import lazily to ensure mypyc.build has ensured that distutils is correctly set up
-    from distutils import ccompiler
+    ccompiler, _ = import_distutils()
 
     h = hashlib.sha256()
     # Include experimental flag

mypyc/test/test_misc.py

@@ -1,14 +1,18 @@
 from __future__ import annotations
 
 import os
+import subprocess
+import sys
 import tempfile
+import textwrap
 import unittest
 
 from mypyc.build import get_header_deps, resolve_cfile_deps
 from mypyc.ir.ops import BasicBlock
 from mypyc.ir.pprint import format_blocks, generate_names_for_ir
 from mypyc.irbuild.ll_builder import LowLevelIRBuilder
 from mypyc.options import CompilerOptions
+from mypyc.test.config import PREFIX
 
 
 class TestMisc(unittest.TestCase):
@@ -150,3 +154,29 @@ def test_resolve_search_order_matches_preprocessor(self) -> None:
             assert resolve_cfile_deps(
                 cfile_dir=includer, direct_includes=[(True, "shared.h")], target_dir=target
             ) == {global_h}
+
+
+class TestLazySetuptoolsImport(unittest.TestCase):
+    @unittest.skipIf(sys.version_info < (3, 12), "setuptools is imported eagerly before 3.12")
+    def test_c_generation_does_not_import_setuptools(self) -> None:
+        # Importing setuptools is slow, and generating C via mypyc_build doesn't need it.
+        script = textwrap.dedent("""
+            import os
+            import sys
+            from mypyc.build import mypyc_build
+            from mypyc.options import CompilerOptions
+
+            mypyc_build(["a.py"], CompilerOptions(target_dir="build"), only_compile_paths=["a.py"])
+            assert any(f.endswith(".c") for f in os.listdir("build")), "no C generated"
+            loaded = {"setuptools", "distutils", "mypyc.build_setup"} & set(sys.modules)
+            assert not loaded, f"unexpectedly imported: {sorted(loaded)}"
+            """)
+        with tempfile.TemporaryDirectory() as tmpdir:
+            with open(os.path.join(tmpdir, "a.py"), "w") as f:
+                f.write("def f(x: int) -> int:\n    return x + 1\n")
+            env = os.environ.copy()
+            env["PYTHONPATH"] = PREFIX + os.pathsep + env.get("PYTHONPATH", "")
+            result = subprocess.run(
+                [sys.executable, "-c", script], cwd=tmpdir, env=env, capture_output=True, text=True
+            )
+        assert result.returncode == 0, result.stdout + result.stderr