wheel_builder.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334
  1. """Orchestrator for building wheels from InstallRequirements."""
  2. from __future__ import annotations
  3. import logging
  4. import os.path
  5. import re
  6. import shutil
  7. from collections.abc import Iterable
  8. from pip._vendor.packaging.utils import canonicalize_name, canonicalize_version
  9. from pip._vendor.packaging.version import InvalidVersion, Version
  10. from pip._internal.cache import WheelCache
  11. from pip._internal.exceptions import InvalidWheelFilename, UnsupportedWheel
  12. from pip._internal.metadata import FilesystemWheel, get_wheel_distribution
  13. from pip._internal.models.link import Link
  14. from pip._internal.models.wheel import Wheel
  15. from pip._internal.operations.build.wheel import build_wheel_pep517
  16. from pip._internal.operations.build.wheel_editable import build_wheel_editable
  17. from pip._internal.operations.build.wheel_legacy import build_wheel_legacy
  18. from pip._internal.req.req_install import InstallRequirement
  19. from pip._internal.utils.logging import indent_log
  20. from pip._internal.utils.misc import ensure_dir, hash_file
  21. from pip._internal.utils.setuptools_build import make_setuptools_clean_args
  22. from pip._internal.utils.subprocess import call_subprocess
  23. from pip._internal.utils.temp_dir import TempDirectory
  24. from pip._internal.utils.urls import path_to_url
  25. from pip._internal.vcs import vcs
  26. logger = logging.getLogger(__name__)
  27. _egg_info_re = re.compile(r"([a-z0-9_.]+)-([a-z0-9_.!+-]+)", re.IGNORECASE)
  28. BuildResult = tuple[list[InstallRequirement], list[InstallRequirement]]
  29. def _contains_egg_info(s: str) -> bool:
  30. """Determine whether the string looks like an egg_info.
  31. :param s: The string to parse. E.g. foo-2.1
  32. """
  33. return bool(_egg_info_re.search(s))
  34. def _should_build(
  35. req: InstallRequirement,
  36. ) -> bool:
  37. """Return whether an InstallRequirement should be built into a wheel."""
  38. assert not req.constraint
  39. if req.is_wheel:
  40. return False
  41. assert req.source_dir
  42. if req.editable:
  43. # we only build PEP 660 editable requirements
  44. return req.supports_pyproject_editable
  45. return True
  46. def should_build_for_install_command(
  47. req: InstallRequirement,
  48. ) -> bool:
  49. return _should_build(req)
  50. def _should_cache(
  51. req: InstallRequirement,
  52. ) -> bool | None:
  53. """
  54. Return whether a built InstallRequirement can be stored in the persistent
  55. wheel cache, assuming the wheel cache is available, and _should_build()
  56. has determined a wheel needs to be built.
  57. """
  58. if req.editable or not req.source_dir:
  59. # never cache editable requirements
  60. return False
  61. if req.link and req.link.is_vcs:
  62. # VCS checkout. Do not cache
  63. # unless it points to an immutable commit hash.
  64. assert not req.editable
  65. assert req.source_dir
  66. vcs_backend = vcs.get_backend_for_scheme(req.link.scheme)
  67. assert vcs_backend
  68. if vcs_backend.is_immutable_rev_checkout(req.link.url, req.source_dir):
  69. return True
  70. return False
  71. assert req.link
  72. base, ext = req.link.splitext()
  73. if _contains_egg_info(base):
  74. return True
  75. # Otherwise, do not cache.
  76. return False
  77. def _get_cache_dir(
  78. req: InstallRequirement,
  79. wheel_cache: WheelCache,
  80. ) -> str:
  81. """Return the persistent or temporary cache directory where the built
  82. wheel need to be stored.
  83. """
  84. cache_available = bool(wheel_cache.cache_dir)
  85. assert req.link
  86. if cache_available and _should_cache(req):
  87. cache_dir = wheel_cache.get_path_for_link(req.link)
  88. else:
  89. cache_dir = wheel_cache.get_ephem_path_for_link(req.link)
  90. return cache_dir
  91. def _verify_one(req: InstallRequirement, wheel_path: str) -> None:
  92. canonical_name = canonicalize_name(req.name or "")
  93. w = Wheel(os.path.basename(wheel_path))
  94. if canonicalize_name(w.name) != canonical_name:
  95. raise InvalidWheelFilename(
  96. f"Wheel has unexpected file name: expected {canonical_name!r}, "
  97. f"got {w.name!r}",
  98. )
  99. dist = get_wheel_distribution(FilesystemWheel(wheel_path), canonical_name)
  100. dist_verstr = str(dist.version)
  101. if canonicalize_version(dist_verstr) != canonicalize_version(w.version):
  102. raise InvalidWheelFilename(
  103. f"Wheel has unexpected file name: expected {dist_verstr!r}, "
  104. f"got {w.version!r}",
  105. )
  106. metadata_version_value = dist.metadata_version
  107. if metadata_version_value is None:
  108. raise UnsupportedWheel("Missing Metadata-Version")
  109. try:
  110. metadata_version = Version(metadata_version_value)
  111. except InvalidVersion:
  112. msg = f"Invalid Metadata-Version: {metadata_version_value}"
  113. raise UnsupportedWheel(msg)
  114. if metadata_version >= Version("1.2") and not isinstance(dist.version, Version):
  115. raise UnsupportedWheel(
  116. f"Metadata 1.2 mandates PEP 440 version, but {dist_verstr!r} is not"
  117. )
  118. def _build_one(
  119. req: InstallRequirement,
  120. output_dir: str,
  121. verify: bool,
  122. build_options: list[str],
  123. global_options: list[str],
  124. editable: bool,
  125. ) -> str | None:
  126. """Build one wheel.
  127. :return: The filename of the built wheel, or None if the build failed.
  128. """
  129. artifact = "editable" if editable else "wheel"
  130. try:
  131. ensure_dir(output_dir)
  132. except OSError as e:
  133. logger.warning(
  134. "Building %s for %s failed: %s",
  135. artifact,
  136. req.name,
  137. e,
  138. )
  139. return None
  140. # Install build deps into temporary directory (PEP 518)
  141. with req.build_env:
  142. wheel_path = _build_one_inside_env(
  143. req, output_dir, build_options, global_options, editable
  144. )
  145. if wheel_path and verify:
  146. try:
  147. _verify_one(req, wheel_path)
  148. except (InvalidWheelFilename, UnsupportedWheel) as e:
  149. logger.warning("Built %s for %s is invalid: %s", artifact, req.name, e)
  150. return None
  151. return wheel_path
  152. def _build_one_inside_env(
  153. req: InstallRequirement,
  154. output_dir: str,
  155. build_options: list[str],
  156. global_options: list[str],
  157. editable: bool,
  158. ) -> str | None:
  159. with TempDirectory(kind="wheel") as temp_dir:
  160. assert req.name
  161. if req.use_pep517:
  162. assert req.metadata_directory
  163. assert req.pep517_backend
  164. if global_options:
  165. logger.warning(
  166. "Ignoring --global-option when building %s using PEP 517", req.name
  167. )
  168. if build_options:
  169. logger.warning(
  170. "Ignoring --build-option when building %s using PEP 517", req.name
  171. )
  172. if editable:
  173. wheel_path = build_wheel_editable(
  174. name=req.name,
  175. backend=req.pep517_backend,
  176. metadata_directory=req.metadata_directory,
  177. tempd=temp_dir.path,
  178. )
  179. else:
  180. wheel_path = build_wheel_pep517(
  181. name=req.name,
  182. backend=req.pep517_backend,
  183. metadata_directory=req.metadata_directory,
  184. tempd=temp_dir.path,
  185. )
  186. else:
  187. wheel_path = build_wheel_legacy(
  188. name=req.name,
  189. setup_py_path=req.setup_py_path,
  190. source_dir=req.unpacked_source_directory,
  191. global_options=global_options,
  192. build_options=build_options,
  193. tempd=temp_dir.path,
  194. )
  195. if wheel_path is not None:
  196. wheel_name = os.path.basename(wheel_path)
  197. dest_path = os.path.join(output_dir, wheel_name)
  198. try:
  199. wheel_hash, length = hash_file(wheel_path)
  200. shutil.move(wheel_path, dest_path)
  201. logger.info(
  202. "Created wheel for %s: filename=%s size=%d sha256=%s",
  203. req.name,
  204. wheel_name,
  205. length,
  206. wheel_hash.hexdigest(),
  207. )
  208. logger.info("Stored in directory: %s", output_dir)
  209. return dest_path
  210. except Exception as e:
  211. logger.warning(
  212. "Building wheel for %s failed: %s",
  213. req.name,
  214. e,
  215. )
  216. # Ignore return, we can't do anything else useful.
  217. if not req.use_pep517:
  218. _clean_one_legacy(req, global_options)
  219. return None
  220. def _clean_one_legacy(req: InstallRequirement, global_options: list[str]) -> bool:
  221. clean_args = make_setuptools_clean_args(
  222. req.setup_py_path,
  223. global_options=global_options,
  224. )
  225. logger.info("Running setup.py clean for %s", req.name)
  226. try:
  227. call_subprocess(
  228. clean_args, command_desc="python setup.py clean", cwd=req.source_dir
  229. )
  230. return True
  231. except Exception:
  232. logger.error("Failed cleaning build dir for %s", req.name)
  233. return False
  234. def build(
  235. requirements: Iterable[InstallRequirement],
  236. wheel_cache: WheelCache,
  237. verify: bool,
  238. build_options: list[str],
  239. global_options: list[str],
  240. ) -> BuildResult:
  241. """Build wheels.
  242. :return: The list of InstallRequirement that succeeded to build and
  243. the list of InstallRequirement that failed to build.
  244. """
  245. if not requirements:
  246. return [], []
  247. # Build the wheels.
  248. logger.info(
  249. "Building wheels for collected packages: %s",
  250. ", ".join(req.name for req in requirements), # type: ignore
  251. )
  252. with indent_log():
  253. build_successes, build_failures = [], []
  254. for req in requirements:
  255. assert req.name
  256. cache_dir = _get_cache_dir(req, wheel_cache)
  257. wheel_file = _build_one(
  258. req,
  259. cache_dir,
  260. verify,
  261. build_options,
  262. global_options,
  263. req.editable and req.permit_editable_wheels,
  264. )
  265. if wheel_file:
  266. # Record the download origin in the cache
  267. if req.download_info is not None:
  268. # download_info is guaranteed to be set because when we build an
  269. # InstallRequirement it has been through the preparer before, but
  270. # let's be cautious.
  271. wheel_cache.record_download_origin(cache_dir, req.download_info)
  272. # Update the link for this.
  273. req.link = Link(path_to_url(wheel_file))
  274. req.local_file_path = req.link.file_path
  275. assert req.link.is_wheel
  276. build_successes.append(req)
  277. else:
  278. build_failures.append(req)
  279. # notify success/failure
  280. if build_successes:
  281. logger.info(
  282. "Successfully built %s",
  283. " ".join([req.name for req in build_successes]), # type: ignore
  284. )
  285. if build_failures:
  286. logger.info(
  287. "Failed to build %s",
  288. " ".join([req.name for req in build_failures]), # type: ignore
  289. )
  290. # Return a list of requirements that failed to build
  291. return build_successes, build_failures