build_env.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349
  1. """Build Environment used for isolation during sdist building"""
  2. from __future__ import annotations
  3. import logging
  4. import os
  5. import pathlib
  6. import site
  7. import sys
  8. import textwrap
  9. from collections import OrderedDict
  10. from collections.abc import Iterable
  11. from types import TracebackType
  12. from typing import TYPE_CHECKING, Protocol
  13. from pip._vendor.packaging.version import Version
  14. from pip import __file__ as pip_location
  15. from pip._internal.cli.spinners import open_spinner
  16. from pip._internal.locations import get_platlib, get_purelib, get_scheme
  17. from pip._internal.metadata import get_default_environment, get_environment
  18. from pip._internal.utils.logging import VERBOSE
  19. from pip._internal.utils.packaging import get_requirement
  20. from pip._internal.utils.subprocess import call_subprocess
  21. from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
  22. if TYPE_CHECKING:
  23. from pip._internal.index.package_finder import PackageFinder
  24. from pip._internal.req.req_install import InstallRequirement
  25. logger = logging.getLogger(__name__)
  26. def _dedup(a: str, b: str) -> tuple[str] | tuple[str, str]:
  27. return (a, b) if a != b else (a,)
  28. class _Prefix:
  29. def __init__(self, path: str) -> None:
  30. self.path = path
  31. self.setup = False
  32. scheme = get_scheme("", prefix=path)
  33. self.bin_dir = scheme.scripts
  34. self.lib_dirs = _dedup(scheme.purelib, scheme.platlib)
  35. def get_runnable_pip() -> str:
  36. """Get a file to pass to a Python executable, to run the currently-running pip.
  37. This is used to run a pip subprocess, for installing requirements into the build
  38. environment.
  39. """
  40. source = pathlib.Path(pip_location).resolve().parent
  41. if not source.is_dir():
  42. # This would happen if someone is using pip from inside a zip file. In that
  43. # case, we can use that directly.
  44. return str(source)
  45. return os.fsdecode(source / "__pip-runner__.py")
  46. def _get_system_sitepackages() -> set[str]:
  47. """Get system site packages
  48. Usually from site.getsitepackages,
  49. but fallback on `get_purelib()/get_platlib()` if unavailable
  50. (e.g. in a virtualenv created by virtualenv<20)
  51. Returns normalized set of strings.
  52. """
  53. if hasattr(site, "getsitepackages"):
  54. system_sites = site.getsitepackages()
  55. else:
  56. # virtualenv < 20 overwrites site.py without getsitepackages
  57. # fallback on get_purelib/get_platlib.
  58. # this is known to miss things, but shouldn't in the cases
  59. # where getsitepackages() has been removed (inside a virtualenv)
  60. system_sites = [get_purelib(), get_platlib()]
  61. return {os.path.normcase(path) for path in system_sites}
  62. class BuildEnvironmentInstaller(Protocol):
  63. """
  64. Interface for installing build dependencies into an isolated build
  65. environment.
  66. """
  67. def install(
  68. self,
  69. requirements: Iterable[str],
  70. prefix: _Prefix,
  71. *,
  72. kind: str,
  73. for_req: InstallRequirement | None,
  74. ) -> None: ...
  75. class SubprocessBuildEnvironmentInstaller:
  76. """
  77. Install build dependencies by calling pip in a subprocess.
  78. """
  79. def __init__(self, finder: PackageFinder) -> None:
  80. self.finder = finder
  81. def install(
  82. self,
  83. requirements: Iterable[str],
  84. prefix: _Prefix,
  85. *,
  86. kind: str,
  87. for_req: InstallRequirement | None,
  88. ) -> None:
  89. finder = self.finder
  90. args: list[str] = [
  91. sys.executable,
  92. get_runnable_pip(),
  93. "install",
  94. "--ignore-installed",
  95. "--no-user",
  96. "--prefix",
  97. prefix.path,
  98. "--no-warn-script-location",
  99. "--disable-pip-version-check",
  100. # As the build environment is ephemeral, it's wasteful to
  101. # pre-compile everything, especially as not every Python
  102. # module will be used/compiled in most cases.
  103. "--no-compile",
  104. # The prefix specified two lines above, thus
  105. # target from config file or env var should be ignored
  106. "--target",
  107. "",
  108. ]
  109. if logger.getEffectiveLevel() <= logging.DEBUG:
  110. args.append("-vv")
  111. elif logger.getEffectiveLevel() <= VERBOSE:
  112. args.append("-v")
  113. for format_control in ("no_binary", "only_binary"):
  114. formats = getattr(finder.format_control, format_control)
  115. args.extend(
  116. (
  117. "--" + format_control.replace("_", "-"),
  118. ",".join(sorted(formats or {":none:"})),
  119. )
  120. )
  121. index_urls = finder.index_urls
  122. if index_urls:
  123. args.extend(["-i", index_urls[0]])
  124. for extra_index in index_urls[1:]:
  125. args.extend(["--extra-index-url", extra_index])
  126. else:
  127. args.append("--no-index")
  128. for link in finder.find_links:
  129. args.extend(["--find-links", link])
  130. if finder.proxy:
  131. args.extend(["--proxy", finder.proxy])
  132. for host in finder.trusted_hosts:
  133. args.extend(["--trusted-host", host])
  134. if finder.custom_cert:
  135. args.extend(["--cert", finder.custom_cert])
  136. if finder.client_cert:
  137. args.extend(["--client-cert", finder.client_cert])
  138. if finder.allow_all_prereleases:
  139. args.append("--pre")
  140. if finder.prefer_binary:
  141. args.append("--prefer-binary")
  142. args.append("--")
  143. args.extend(requirements)
  144. with open_spinner(f"Installing {kind}") as spinner:
  145. call_subprocess(
  146. args,
  147. command_desc=f"pip subprocess to install {kind}",
  148. spinner=spinner,
  149. )
  150. class BuildEnvironment:
  151. """Creates and manages an isolated environment to install build deps"""
  152. def __init__(self, installer: BuildEnvironmentInstaller) -> None:
  153. self.installer = installer
  154. temp_dir = TempDirectory(kind=tempdir_kinds.BUILD_ENV, globally_managed=True)
  155. self._prefixes = OrderedDict(
  156. (name, _Prefix(os.path.join(temp_dir.path, name)))
  157. for name in ("normal", "overlay")
  158. )
  159. self._bin_dirs: list[str] = []
  160. self._lib_dirs: list[str] = []
  161. for prefix in reversed(list(self._prefixes.values())):
  162. self._bin_dirs.append(prefix.bin_dir)
  163. self._lib_dirs.extend(prefix.lib_dirs)
  164. # Customize site to:
  165. # - ensure .pth files are honored
  166. # - prevent access to system site packages
  167. system_sites = _get_system_sitepackages()
  168. self._site_dir = os.path.join(temp_dir.path, "site")
  169. if not os.path.exists(self._site_dir):
  170. os.mkdir(self._site_dir)
  171. with open(
  172. os.path.join(self._site_dir, "sitecustomize.py"), "w", encoding="utf-8"
  173. ) as fp:
  174. fp.write(
  175. textwrap.dedent(
  176. """
  177. import os, site, sys
  178. # First, drop system-sites related paths.
  179. original_sys_path = sys.path[:]
  180. known_paths = set()
  181. for path in {system_sites!r}:
  182. site.addsitedir(path, known_paths=known_paths)
  183. system_paths = set(
  184. os.path.normcase(path)
  185. for path in sys.path[len(original_sys_path):]
  186. )
  187. original_sys_path = [
  188. path for path in original_sys_path
  189. if os.path.normcase(path) not in system_paths
  190. ]
  191. sys.path = original_sys_path
  192. # Second, add lib directories.
  193. # ensuring .pth file are processed.
  194. for path in {lib_dirs!r}:
  195. assert not path in sys.path
  196. site.addsitedir(path)
  197. """
  198. ).format(system_sites=system_sites, lib_dirs=self._lib_dirs)
  199. )
  200. def __enter__(self) -> None:
  201. self._save_env = {
  202. name: os.environ.get(name, None)
  203. for name in ("PATH", "PYTHONNOUSERSITE", "PYTHONPATH")
  204. }
  205. path = self._bin_dirs[:]
  206. old_path = self._save_env["PATH"]
  207. if old_path:
  208. path.extend(old_path.split(os.pathsep))
  209. pythonpath = [self._site_dir]
  210. os.environ.update(
  211. {
  212. "PATH": os.pathsep.join(path),
  213. "PYTHONNOUSERSITE": "1",
  214. "PYTHONPATH": os.pathsep.join(pythonpath),
  215. }
  216. )
  217. def __exit__(
  218. self,
  219. exc_type: type[BaseException] | None,
  220. exc_val: BaseException | None,
  221. exc_tb: TracebackType | None,
  222. ) -> None:
  223. for varname, old_value in self._save_env.items():
  224. if old_value is None:
  225. os.environ.pop(varname, None)
  226. else:
  227. os.environ[varname] = old_value
  228. def check_requirements(
  229. self, reqs: Iterable[str]
  230. ) -> tuple[set[tuple[str, str]], set[str]]:
  231. """Return 2 sets:
  232. - conflicting requirements: set of (installed, wanted) reqs tuples
  233. - missing requirements: set of reqs
  234. """
  235. missing = set()
  236. conflicting = set()
  237. if reqs:
  238. env = (
  239. get_environment(self._lib_dirs)
  240. if hasattr(self, "_lib_dirs")
  241. else get_default_environment()
  242. )
  243. for req_str in reqs:
  244. req = get_requirement(req_str)
  245. # We're explicitly evaluating with an empty extra value, since build
  246. # environments are not provided any mechanism to select specific extras.
  247. if req.marker is not None and not req.marker.evaluate({"extra": ""}):
  248. continue
  249. dist = env.get_distribution(req.name)
  250. if not dist:
  251. missing.add(req_str)
  252. continue
  253. if isinstance(dist.version, Version):
  254. installed_req_str = f"{req.name}=={dist.version}"
  255. else:
  256. installed_req_str = f"{req.name}==={dist.version}"
  257. if not req.specifier.contains(dist.version, prereleases=True):
  258. conflicting.add((installed_req_str, req_str))
  259. # FIXME: Consider direct URL?
  260. return conflicting, missing
  261. def install_requirements(
  262. self,
  263. requirements: Iterable[str],
  264. prefix_as_string: str,
  265. *,
  266. kind: str,
  267. for_req: InstallRequirement | None = None,
  268. ) -> None:
  269. prefix = self._prefixes[prefix_as_string]
  270. assert not prefix.setup
  271. prefix.setup = True
  272. if not requirements:
  273. return
  274. self.installer.install(requirements, prefix, kind=kind, for_req=for_req)
  275. class NoOpBuildEnvironment(BuildEnvironment):
  276. """A no-op drop-in replacement for BuildEnvironment"""
  277. def __init__(self) -> None:
  278. pass
  279. def __enter__(self) -> None:
  280. pass
  281. def __exit__(
  282. self,
  283. exc_type: type[BaseException] | None,
  284. exc_val: BaseException | None,
  285. exc_tb: TracebackType | None,
  286. ) -> None:
  287. pass
  288. def cleanup(self) -> None:
  289. pass
  290. def install_requirements(
  291. self,
  292. requirements: Iterable[str],
  293. prefix_as_string: str,
  294. *,
  295. kind: str,
  296. for_req: InstallRequirement | None = None,
  297. ) -> None:
  298. raise NotImplementedError()