freeze.py 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259
  1. from __future__ import annotations
  2. import collections
  3. import logging
  4. import os
  5. from collections.abc import Container, Generator, Iterable
  6. from dataclasses import dataclass, field
  7. from typing import NamedTuple
  8. from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
  9. from pip._vendor.packaging.version import InvalidVersion
  10. from pip._internal.exceptions import BadCommand, InstallationError
  11. from pip._internal.metadata import BaseDistribution, get_environment
  12. from pip._internal.req.constructors import (
  13. install_req_from_editable,
  14. install_req_from_line,
  15. )
  16. from pip._internal.req.req_file import COMMENT_RE
  17. from pip._internal.utils.direct_url_helpers import direct_url_as_pep440_direct_reference
  18. logger = logging.getLogger(__name__)
  19. class _EditableInfo(NamedTuple):
  20. requirement: str
  21. comments: list[str]
  22. def freeze(
  23. requirement: list[str] | None = None,
  24. local_only: bool = False,
  25. user_only: bool = False,
  26. paths: list[str] | None = None,
  27. isolated: bool = False,
  28. exclude_editable: bool = False,
  29. skip: Container[str] = (),
  30. ) -> Generator[str, None, None]:
  31. installations: dict[str, FrozenRequirement] = {}
  32. dists = get_environment(paths).iter_installed_distributions(
  33. local_only=local_only,
  34. skip=(),
  35. user_only=user_only,
  36. )
  37. for dist in dists:
  38. req = FrozenRequirement.from_dist(dist)
  39. if exclude_editable and req.editable:
  40. continue
  41. installations[req.canonical_name] = req
  42. if requirement:
  43. # the options that don't get turned into an InstallRequirement
  44. # should only be emitted once, even if the same option is in multiple
  45. # requirements files, so we need to keep track of what has been emitted
  46. # so that we don't emit it again if it's seen again
  47. emitted_options: set[str] = set()
  48. # keep track of which files a requirement is in so that we can
  49. # give an accurate warning if a requirement appears multiple times.
  50. req_files: dict[str, list[str]] = collections.defaultdict(list)
  51. for req_file_path in requirement:
  52. with open(req_file_path) as req_file:
  53. for line in req_file:
  54. if (
  55. not line.strip()
  56. or line.strip().startswith("#")
  57. or line.startswith(
  58. (
  59. "-r",
  60. "--requirement",
  61. "-f",
  62. "--find-links",
  63. "-i",
  64. "--index-url",
  65. "--pre",
  66. "--trusted-host",
  67. "--process-dependency-links",
  68. "--extra-index-url",
  69. "--use-feature",
  70. )
  71. )
  72. ):
  73. line = line.rstrip()
  74. if line not in emitted_options:
  75. emitted_options.add(line)
  76. yield line
  77. continue
  78. if line.startswith(("-e", "--editable")):
  79. if line.startswith("-e"):
  80. line = line[2:].strip()
  81. else:
  82. line = line[len("--editable") :].strip().lstrip("=")
  83. line_req = install_req_from_editable(
  84. line,
  85. isolated=isolated,
  86. )
  87. else:
  88. line_req = install_req_from_line(
  89. COMMENT_RE.sub("", line).strip(),
  90. isolated=isolated,
  91. )
  92. if not line_req.name:
  93. logger.info(
  94. "Skipping line in requirement file [%s] because "
  95. "it's not clear what it would install: %s",
  96. req_file_path,
  97. line.strip(),
  98. )
  99. logger.info(
  100. " (add #egg=PackageName to the URL to avoid"
  101. " this warning)"
  102. )
  103. else:
  104. line_req_canonical_name = canonicalize_name(line_req.name)
  105. if line_req_canonical_name not in installations:
  106. # either it's not installed, or it is installed
  107. # but has been processed already
  108. if not req_files[line_req.name]:
  109. logger.warning(
  110. "Requirement file [%s] contains %s, but "
  111. "package %r is not installed",
  112. req_file_path,
  113. COMMENT_RE.sub("", line).strip(),
  114. line_req.name,
  115. )
  116. else:
  117. req_files[line_req.name].append(req_file_path)
  118. else:
  119. yield str(installations[line_req_canonical_name]).rstrip()
  120. del installations[line_req_canonical_name]
  121. req_files[line_req.name].append(req_file_path)
  122. # Warn about requirements that were included multiple times (in a
  123. # single requirements file or in different requirements files).
  124. for name, files in req_files.items():
  125. if len(files) > 1:
  126. logger.warning(
  127. "Requirement %s included multiple times [%s]",
  128. name,
  129. ", ".join(sorted(set(files))),
  130. )
  131. yield ("## The following requirements were added by pip freeze:")
  132. for installation in sorted(installations.values(), key=lambda x: x.name.lower()):
  133. if installation.canonical_name not in skip:
  134. yield str(installation).rstrip()
  135. def _format_as_name_version(dist: BaseDistribution) -> str:
  136. try:
  137. dist_version = dist.version
  138. except InvalidVersion:
  139. # legacy version
  140. return f"{dist.raw_name}==={dist.raw_version}"
  141. else:
  142. return f"{dist.raw_name}=={dist_version}"
  143. def _get_editable_info(dist: BaseDistribution) -> _EditableInfo:
  144. """
  145. Compute and return values (req, comments) for use in
  146. FrozenRequirement.from_dist().
  147. """
  148. editable_project_location = dist.editable_project_location
  149. assert editable_project_location
  150. location = os.path.normcase(os.path.abspath(editable_project_location))
  151. from pip._internal.vcs import RemoteNotFoundError, RemoteNotValidError, vcs
  152. vcs_backend = vcs.get_backend_for_dir(location)
  153. if vcs_backend is None:
  154. display = _format_as_name_version(dist)
  155. logger.debug(
  156. 'No VCS found for editable requirement "%s" in: %r',
  157. display,
  158. location,
  159. )
  160. return _EditableInfo(
  161. requirement=location,
  162. comments=[f"# Editable install with no version control ({display})"],
  163. )
  164. vcs_name = type(vcs_backend).__name__
  165. try:
  166. req = vcs_backend.get_src_requirement(location, dist.raw_name)
  167. except RemoteNotFoundError:
  168. display = _format_as_name_version(dist)
  169. return _EditableInfo(
  170. requirement=location,
  171. comments=[f"# Editable {vcs_name} install with no remote ({display})"],
  172. )
  173. except RemoteNotValidError as ex:
  174. display = _format_as_name_version(dist)
  175. return _EditableInfo(
  176. requirement=location,
  177. comments=[
  178. f"# Editable {vcs_name} install ({display}) with either a deleted "
  179. f"local remote or invalid URI:",
  180. f"# '{ex.url}'",
  181. ],
  182. )
  183. except BadCommand:
  184. logger.warning(
  185. "cannot determine version of editable source in %s "
  186. "(%s command not found in path)",
  187. location,
  188. vcs_backend.name,
  189. )
  190. return _EditableInfo(requirement=location, comments=[])
  191. except InstallationError as exc:
  192. logger.warning("Error when trying to get requirement for VCS system %s", exc)
  193. else:
  194. return _EditableInfo(requirement=req, comments=[])
  195. logger.warning("Could not determine repository location of %s", location)
  196. return _EditableInfo(
  197. requirement=location,
  198. comments=["## !! Could not determine repository location"],
  199. )
  200. @dataclass(frozen=True)
  201. class FrozenRequirement:
  202. name: str
  203. req: str
  204. editable: bool
  205. comments: Iterable[str] = field(default_factory=tuple)
  206. @property
  207. def canonical_name(self) -> NormalizedName:
  208. return canonicalize_name(self.name)
  209. @classmethod
  210. def from_dist(cls, dist: BaseDistribution) -> FrozenRequirement:
  211. editable = dist.editable
  212. if editable:
  213. req, comments = _get_editable_info(dist)
  214. else:
  215. comments = []
  216. direct_url = dist.direct_url
  217. if direct_url:
  218. # if PEP 610 metadata is present, use it
  219. req = direct_url_as_pep440_direct_reference(direct_url, dist.raw_name)
  220. else:
  221. # name==version requirement
  222. req = _format_as_name_version(dist)
  223. return cls(dist.raw_name, req, editable, comments=comments)
  224. def __str__(self) -> str:
  225. req = self.req
  226. if self.editable:
  227. req = f"-e {req}"
  228. return "\n".join(list(self.comments) + [str(req)]) + "\n"