diff --git a/mypyc/build.py b/mypyc/build.py index 8c6eabead17c..ee01858887dd 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -13,9 +13,13 @@ See the mypycify docs for additional arguments. -mypycify can integrate with either distutils or setuptools, but needs -to know at import-time whether it is using distutils or setuputils. We -hackily decide based on whether setuptools has been imported already. +mypycify requires setuptools. + +setuptools and distutils are only imported when they are needed to build +extensions (such as in mypycify), since importing them is slow and +generating C via mypyc_build doesn't need them. On Python versions before +3.12, they are imported when this module is imported, since setuptools must +be imported before distutils to override it. """ from __future__ import annotations @@ -28,7 +32,6 @@ from collections.abc import Iterable from typing import TYPE_CHECKING, Any, NamedTuple, NoReturn, cast -import mypyc.build_setup # noqa: F401 from mypy.build import BuildSource from mypy.errors import CompileError from mypy.fscache import FileSystemCache @@ -135,12 +138,6 @@ class ModDesc(NamedTuple): ), ] -try: - # Import setuptools so that it monkey-patch overrides distutils - import setuptools -except ImportError: - pass - if TYPE_CHECKING: if sys.version_info >= (3, 12): from setuptools import Extension @@ -152,14 +149,39 @@ class ModDesc(NamedTuple): Extension: TypeAlias = _setuptools_Extension | _distutils_Extension -if sys.version_info >= (3, 12): - # From setuptools' monkeypatch - from distutils import ccompiler, sysconfig # type: ignore[import-not-found] -else: - from distutils import ccompiler, sysconfig + +def import_distutils() -> tuple[Any, Any]: + """Import and return the distutils ccompiler and sysconfig modules. + + This also imports setuptools (if available) first, so that it overrides distutils, + and patches the compiler to support per-file flags. These imports are slow, and + only needed when building C extensions, so this is done lazily (C generation in + mypyc_build doesn't need it). + """ + try: + # Import setuptools so that it monkey-patch overrides distutils + import setuptools # noqa: F401 + except ImportError: + pass + + import mypyc.build_setup # noqa: F401 + + if sys.version_info >= (3, 12): + # From setuptools' monkeypatch + from distutils import ccompiler, sysconfig # type: ignore[import-not-found] + else: + from distutils import ccompiler, sysconfig + return ccompiler, sysconfig + + +if sys.version_info < (3, 12): + # Distutils is still in the stdlib, and setuptools must be imported before it to + # override it, so preserve the old behavior of importing these eagerly. + import_distutils() def get_extension() -> type[Extension]: + import_distutils() # We can work with either setuptools or distutils, and pick setuptools # if it has been imported. use_setuptools = "setuptools" in sys.modules @@ -172,6 +194,8 @@ def get_extension() -> type[Extension]: else: if not use_setuptools: sys.exit("error: setuptools not installed") + import setuptools + extension_class = setuptools.Extension return extension_class @@ -182,6 +206,7 @@ def setup_mypycify_vars() -> None: # There has to be a better approach to this. # The vars can contain ints but we only work with str ones + _, sysconfig = import_distutils() vars = cast(dict[str, str], sysconfig.get_config_vars()) if sys.platform == "darwin": # Disable building 32-bit binaries, since we generate too much code @@ -805,6 +830,7 @@ def get_cflags( List of compiler flags. """ if compiler_type is None: + ccompiler, sysconfig = import_distutils() compiler: Any = ccompiler.new_compiler() sysconfig.customize_compiler(compiler) compiler_type = compiler.compiler_type @@ -952,6 +978,7 @@ def mypycify( also needed if using experimental librt features). These have no backward compatibility guarantees! """ + ccompiler, sysconfig = import_distutils() # Skip redundant inplace .so copies on every build_ext invocation. _patch_setuptools_copy_extensions_to_source() diff --git a/mypyc/test/librt_cache.py b/mypyc/test/librt_cache.py index bf57b9f54196..751df36bf129 100644 --- a/mypyc/test/librt_cache.py +++ b/mypyc/test/librt_cache.py @@ -29,15 +29,14 @@ import filelock -from mypyc.build import LIBRT_MODULES, get_cflags, include_dir +from mypyc.build import LIBRT_MODULES, get_cflags, import_distutils, include_dir from mypyc.common import RUNTIME_C_FILES from mypyc.test.config import PREFIX def _librt_build_hash(experimental: bool, opt_level: str) -> str: """Compute hash for librt build, including sources and build environment.""" - # Import lazily to ensure mypyc.build has ensured that distutils is correctly set up - from distutils import ccompiler + ccompiler, _ = import_distutils() h = hashlib.sha256() # Include experimental flag diff --git a/mypyc/test/test_misc.py b/mypyc/test/test_misc.py index 816875fcc23d..d838d7778925 100644 --- a/mypyc/test/test_misc.py +++ b/mypyc/test/test_misc.py @@ -1,7 +1,10 @@ from __future__ import annotations import os +import subprocess +import sys import tempfile +import textwrap import unittest from mypyc.build import get_header_deps, resolve_cfile_deps @@ -9,6 +12,7 @@ from mypyc.ir.pprint import format_blocks, generate_names_for_ir from mypyc.irbuild.ll_builder import LowLevelIRBuilder from mypyc.options import CompilerOptions +from mypyc.test.config import PREFIX class TestMisc(unittest.TestCase): @@ -150,3 +154,29 @@ def test_resolve_search_order_matches_preprocessor(self) -> None: assert resolve_cfile_deps( cfile_dir=includer, direct_includes=[(True, "shared.h")], target_dir=target ) == {global_h} + + +class TestLazySetuptoolsImport(unittest.TestCase): + @unittest.skipIf(sys.version_info < (3, 12), "setuptools is imported eagerly before 3.12") + def test_c_generation_does_not_import_setuptools(self) -> None: + # Importing setuptools is slow, and generating C via mypyc_build doesn't need it. + script = textwrap.dedent(""" + import os + import sys + from mypyc.build import mypyc_build + from mypyc.options import CompilerOptions + + mypyc_build(["a.py"], CompilerOptions(target_dir="build"), only_compile_paths=["a.py"]) + assert any(f.endswith(".c") for f in os.listdir("build")), "no C generated" + loaded = {"setuptools", "distutils", "mypyc.build_setup"} & set(sys.modules) + assert not loaded, f"unexpectedly imported: {sorted(loaded)}" + """) + with tempfile.TemporaryDirectory() as tmpdir: + with open(os.path.join(tmpdir, "a.py"), "w") as f: + f.write("def f(x: int) -> int:\n return x + 1\n") + env = os.environ.copy() + env["PYTHONPATH"] = PREFIX + os.pathsep + env.get("PYTHONPATH", "") + result = subprocess.run( + [sys.executable, "-c", script], cwd=tmpdir, env=env, capture_output=True, text=True + ) + assert result.returncode == 0, result.stdout + result.stderr