versioncontrol.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693
  1. """Handles all VCS (version control) support"""
  2. from __future__ import annotations
  3. import logging
  4. import os
  5. import shutil
  6. import sys
  7. import urllib.parse
  8. from collections.abc import Iterable, Iterator, Mapping
  9. from dataclasses import dataclass, field
  10. from typing import (
  11. Any,
  12. Literal,
  13. Optional,
  14. )
  15. from pip._internal.cli.spinners import SpinnerInterface
  16. from pip._internal.exceptions import BadCommand, InstallationError
  17. from pip._internal.utils.misc import (
  18. HiddenText,
  19. ask_path_exists,
  20. backup_dir,
  21. display_path,
  22. hide_url,
  23. hide_value,
  24. is_installable_dir,
  25. rmtree,
  26. )
  27. from pip._internal.utils.subprocess import (
  28. CommandArgs,
  29. call_subprocess,
  30. format_command_args,
  31. make_command,
  32. )
  33. __all__ = ["vcs"]
  34. logger = logging.getLogger(__name__)
  35. AuthInfo = tuple[Optional[str], Optional[str]]
  36. def is_url(name: str) -> bool:
  37. """
  38. Return true if the name looks like a URL.
  39. """
  40. scheme = urllib.parse.urlsplit(name).scheme
  41. if not scheme:
  42. return False
  43. return scheme in ["http", "https", "file", "ftp"] + vcs.all_schemes
  44. def make_vcs_requirement_url(
  45. repo_url: str, rev: str, project_name: str, subdir: str | None = None
  46. ) -> str:
  47. """
  48. Return the URL for a VCS requirement.
  49. Args:
  50. repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+").
  51. project_name: the (unescaped) project name.
  52. """
  53. egg_project_name = project_name.replace("-", "_")
  54. req = f"{repo_url}@{rev}#egg={egg_project_name}"
  55. if subdir:
  56. req += f"&subdirectory={subdir}"
  57. return req
  58. def find_path_to_project_root_from_repo_root(
  59. location: str, repo_root: str
  60. ) -> str | None:
  61. """
  62. Find the the Python project's root by searching up the filesystem from
  63. `location`. Return the path to project root relative to `repo_root`.
  64. Return None if the project root is `repo_root`, or cannot be found.
  65. """
  66. # find project root.
  67. orig_location = location
  68. while not is_installable_dir(location):
  69. last_location = location
  70. location = os.path.dirname(location)
  71. if location == last_location:
  72. # We've traversed up to the root of the filesystem without
  73. # finding a Python project.
  74. logger.warning(
  75. "Could not find a Python project for directory %s (tried all "
  76. "parent directories)",
  77. orig_location,
  78. )
  79. return None
  80. if os.path.samefile(repo_root, location):
  81. return None
  82. return os.path.relpath(location, repo_root)
  83. class RemoteNotFoundError(Exception):
  84. pass
  85. class RemoteNotValidError(Exception):
  86. def __init__(self, url: str):
  87. super().__init__(url)
  88. self.url = url
  89. @dataclass(frozen=True)
  90. class RevOptions:
  91. """
  92. Encapsulates a VCS-specific revision to install, along with any VCS
  93. install options.
  94. Args:
  95. vc_class: a VersionControl subclass.
  96. rev: the name of the revision to install.
  97. extra_args: a list of extra options.
  98. """
  99. vc_class: type[VersionControl]
  100. rev: str | None = None
  101. extra_args: CommandArgs = field(default_factory=list)
  102. branch_name: str | None = None
  103. def __repr__(self) -> str:
  104. return f"<RevOptions {self.vc_class.name}: rev={self.rev!r}>"
  105. @property
  106. def arg_rev(self) -> str | None:
  107. if self.rev is None:
  108. return self.vc_class.default_arg_rev
  109. return self.rev
  110. def to_args(self) -> CommandArgs:
  111. """
  112. Return the VCS-specific command arguments.
  113. """
  114. args: CommandArgs = []
  115. rev = self.arg_rev
  116. if rev is not None:
  117. args += self.vc_class.get_base_rev_args(rev)
  118. args += self.extra_args
  119. return args
  120. def to_display(self) -> str:
  121. if not self.rev:
  122. return ""
  123. return f" (to revision {self.rev})"
  124. def make_new(self, rev: str) -> RevOptions:
  125. """
  126. Make a copy of the current instance, but with a new rev.
  127. Args:
  128. rev: the name of the revision for the new object.
  129. """
  130. return self.vc_class.make_rev_options(rev, extra_args=self.extra_args)
  131. class VcsSupport:
  132. _registry: dict[str, VersionControl] = {}
  133. schemes = ["ssh", "git", "hg", "bzr", "sftp", "svn"]
  134. def __init__(self) -> None:
  135. # Register more schemes with urlparse for various version control
  136. # systems
  137. urllib.parse.uses_netloc.extend(self.schemes)
  138. super().__init__()
  139. def __iter__(self) -> Iterator[str]:
  140. return self._registry.__iter__()
  141. @property
  142. def backends(self) -> list[VersionControl]:
  143. return list(self._registry.values())
  144. @property
  145. def dirnames(self) -> list[str]:
  146. return [backend.dirname for backend in self.backends]
  147. @property
  148. def all_schemes(self) -> list[str]:
  149. schemes: list[str] = []
  150. for backend in self.backends:
  151. schemes.extend(backend.schemes)
  152. return schemes
  153. def register(self, cls: type[VersionControl]) -> None:
  154. if not hasattr(cls, "name"):
  155. logger.warning("Cannot register VCS %s", cls.__name__)
  156. return
  157. if cls.name not in self._registry:
  158. self._registry[cls.name] = cls()
  159. logger.debug("Registered VCS backend: %s", cls.name)
  160. def unregister(self, name: str) -> None:
  161. if name in self._registry:
  162. del self._registry[name]
  163. def get_backend_for_dir(self, location: str) -> VersionControl | None:
  164. """
  165. Return a VersionControl object if a repository of that type is found
  166. at the given directory.
  167. """
  168. vcs_backends = {}
  169. for vcs_backend in self._registry.values():
  170. repo_path = vcs_backend.get_repository_root(location)
  171. if not repo_path:
  172. continue
  173. logger.debug("Determine that %s uses VCS: %s", location, vcs_backend.name)
  174. vcs_backends[repo_path] = vcs_backend
  175. if not vcs_backends:
  176. return None
  177. # Choose the VCS in the inner-most directory. Since all repository
  178. # roots found here would be either `location` or one of its
  179. # parents, the longest path should have the most path components,
  180. # i.e. the backend representing the inner-most repository.
  181. inner_most_repo_path = max(vcs_backends, key=len)
  182. return vcs_backends[inner_most_repo_path]
  183. def get_backend_for_scheme(self, scheme: str) -> VersionControl | None:
  184. """
  185. Return a VersionControl object or None.
  186. """
  187. for vcs_backend in self._registry.values():
  188. if scheme in vcs_backend.schemes:
  189. return vcs_backend
  190. return None
  191. def get_backend(self, name: str) -> VersionControl | None:
  192. """
  193. Return a VersionControl object or None.
  194. """
  195. name = name.lower()
  196. return self._registry.get(name)
  197. vcs = VcsSupport()
  198. class VersionControl:
  199. name = ""
  200. dirname = ""
  201. repo_name = ""
  202. # List of supported schemes for this Version Control
  203. schemes: tuple[str, ...] = ()
  204. # Iterable of environment variable names to pass to call_subprocess().
  205. unset_environ: tuple[str, ...] = ()
  206. default_arg_rev: str | None = None
  207. @classmethod
  208. def should_add_vcs_url_prefix(cls, remote_url: str) -> bool:
  209. """
  210. Return whether the vcs prefix (e.g. "git+") should be added to a
  211. repository's remote url when used in a requirement.
  212. """
  213. return not remote_url.lower().startswith(f"{cls.name}:")
  214. @classmethod
  215. def get_subdirectory(cls, location: str) -> str | None:
  216. """
  217. Return the path to Python project root, relative to the repo root.
  218. Return None if the project root is in the repo root.
  219. """
  220. return None
  221. @classmethod
  222. def get_requirement_revision(cls, repo_dir: str) -> str:
  223. """
  224. Return the revision string that should be used in a requirement.
  225. """
  226. return cls.get_revision(repo_dir)
  227. @classmethod
  228. def get_src_requirement(cls, repo_dir: str, project_name: str) -> str:
  229. """
  230. Return the requirement string to use to redownload the files
  231. currently at the given repository directory.
  232. Args:
  233. project_name: the (unescaped) project name.
  234. The return value has a form similar to the following:
  235. {repository_url}@{revision}#egg={project_name}
  236. """
  237. repo_url = cls.get_remote_url(repo_dir)
  238. if cls.should_add_vcs_url_prefix(repo_url):
  239. repo_url = f"{cls.name}+{repo_url}"
  240. revision = cls.get_requirement_revision(repo_dir)
  241. subdir = cls.get_subdirectory(repo_dir)
  242. req = make_vcs_requirement_url(repo_url, revision, project_name, subdir=subdir)
  243. return req
  244. @staticmethod
  245. def get_base_rev_args(rev: str) -> list[str]:
  246. """
  247. Return the base revision arguments for a vcs command.
  248. Args:
  249. rev: the name of a revision to install. Cannot be None.
  250. """
  251. raise NotImplementedError
  252. def is_immutable_rev_checkout(self, url: str, dest: str) -> bool:
  253. """
  254. Return true if the commit hash checked out at dest matches
  255. the revision in url.
  256. Always return False, if the VCS does not support immutable commit
  257. hashes.
  258. This method does not check if there are local uncommitted changes
  259. in dest after checkout, as pip currently has no use case for that.
  260. """
  261. return False
  262. @classmethod
  263. def make_rev_options(
  264. cls, rev: str | None = None, extra_args: CommandArgs | None = None
  265. ) -> RevOptions:
  266. """
  267. Return a RevOptions object.
  268. Args:
  269. rev: the name of a revision to install.
  270. extra_args: a list of extra options.
  271. """
  272. return RevOptions(cls, rev, extra_args=extra_args or [])
  273. @classmethod
  274. def _is_local_repository(cls, repo: str) -> bool:
  275. """
  276. posix absolute paths start with os.path.sep,
  277. win32 ones start with drive (like c:\\folder)
  278. """
  279. drive, tail = os.path.splitdrive(repo)
  280. return repo.startswith(os.path.sep) or bool(drive)
  281. @classmethod
  282. def get_netloc_and_auth(
  283. cls, netloc: str, scheme: str
  284. ) -> tuple[str, tuple[str | None, str | None]]:
  285. """
  286. Parse the repository URL's netloc, and return the new netloc to use
  287. along with auth information.
  288. Args:
  289. netloc: the original repository URL netloc.
  290. scheme: the repository URL's scheme without the vcs prefix.
  291. This is mainly for the Subversion class to override, so that auth
  292. information can be provided via the --username and --password options
  293. instead of through the URL. For other subclasses like Git without
  294. such an option, auth information must stay in the URL.
  295. Returns: (netloc, (username, password)).
  296. """
  297. return netloc, (None, None)
  298. @classmethod
  299. def get_url_rev_and_auth(cls, url: str) -> tuple[str, str | None, AuthInfo]:
  300. """
  301. Parse the repository URL to use, and return the URL, revision,
  302. and auth info to use.
  303. Returns: (url, rev, (username, password)).
  304. """
  305. scheme, netloc, path, query, frag = urllib.parse.urlsplit(url)
  306. if "+" not in scheme:
  307. raise ValueError(
  308. f"Sorry, {url!r} is a malformed VCS url. "
  309. "The format is <vcs>+<protocol>://<url>, "
  310. "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp"
  311. )
  312. # Remove the vcs prefix.
  313. scheme = scheme.split("+", 1)[1]
  314. netloc, user_pass = cls.get_netloc_and_auth(netloc, scheme)
  315. rev = None
  316. if "@" in path:
  317. path, rev = path.rsplit("@", 1)
  318. if not rev:
  319. raise InstallationError(
  320. f"The URL {url!r} has an empty revision (after @) "
  321. "which is not supported. Include a revision after @ "
  322. "or remove @ from the URL."
  323. )
  324. url = urllib.parse.urlunsplit((scheme, netloc, path, query, ""))
  325. return url, rev, user_pass
  326. @staticmethod
  327. def make_rev_args(username: str | None, password: HiddenText | None) -> CommandArgs:
  328. """
  329. Return the RevOptions "extra arguments" to use in obtain().
  330. """
  331. return []
  332. def get_url_rev_options(self, url: HiddenText) -> tuple[HiddenText, RevOptions]:
  333. """
  334. Return the URL and RevOptions object to use in obtain(),
  335. as a tuple (url, rev_options).
  336. """
  337. secret_url, rev, user_pass = self.get_url_rev_and_auth(url.secret)
  338. username, secret_password = user_pass
  339. password: HiddenText | None = None
  340. if secret_password is not None:
  341. password = hide_value(secret_password)
  342. extra_args = self.make_rev_args(username, password)
  343. rev_options = self.make_rev_options(rev, extra_args=extra_args)
  344. return hide_url(secret_url), rev_options
  345. @staticmethod
  346. def normalize_url(url: str) -> str:
  347. """
  348. Normalize a URL for comparison by unquoting it and removing any
  349. trailing slash.
  350. """
  351. return urllib.parse.unquote(url).rstrip("/")
  352. @classmethod
  353. def compare_urls(cls, url1: str, url2: str) -> bool:
  354. """
  355. Compare two repo URLs for identity, ignoring incidental differences.
  356. """
  357. return cls.normalize_url(url1) == cls.normalize_url(url2)
  358. def fetch_new(
  359. self, dest: str, url: HiddenText, rev_options: RevOptions, verbosity: int
  360. ) -> None:
  361. """
  362. Fetch a revision from a repository, in the case that this is the
  363. first fetch from the repository.
  364. Args:
  365. dest: the directory to fetch the repository to.
  366. rev_options: a RevOptions object.
  367. verbosity: verbosity level.
  368. """
  369. raise NotImplementedError
  370. def switch(
  371. self,
  372. dest: str,
  373. url: HiddenText,
  374. rev_options: RevOptions,
  375. verbosity: int = 0,
  376. ) -> None:
  377. """
  378. Switch the repo at ``dest`` to point to ``URL``.
  379. Args:
  380. rev_options: a RevOptions object.
  381. """
  382. raise NotImplementedError
  383. def update(
  384. self,
  385. dest: str,
  386. url: HiddenText,
  387. rev_options: RevOptions,
  388. verbosity: int = 0,
  389. ) -> None:
  390. """
  391. Update an already-existing repo to the given ``rev_options``.
  392. Args:
  393. rev_options: a RevOptions object.
  394. """
  395. raise NotImplementedError
  396. @classmethod
  397. def is_commit_id_equal(cls, dest: str, name: str | None) -> bool:
  398. """
  399. Return whether the id of the current commit equals the given name.
  400. Args:
  401. dest: the repository directory.
  402. name: a string name.
  403. """
  404. raise NotImplementedError
  405. def obtain(self, dest: str, url: HiddenText, verbosity: int) -> None:
  406. """
  407. Install or update in editable mode the package represented by this
  408. VersionControl object.
  409. :param dest: the repository directory in which to install or update.
  410. :param url: the repository URL starting with a vcs prefix.
  411. :param verbosity: verbosity level.
  412. """
  413. url, rev_options = self.get_url_rev_options(url)
  414. if not os.path.exists(dest):
  415. self.fetch_new(dest, url, rev_options, verbosity=verbosity)
  416. return
  417. rev_display = rev_options.to_display()
  418. if self.is_repository_directory(dest):
  419. existing_url = self.get_remote_url(dest)
  420. if self.compare_urls(existing_url, url.secret):
  421. logger.debug(
  422. "%s in %s exists, and has correct URL (%s)",
  423. self.repo_name.title(),
  424. display_path(dest),
  425. url,
  426. )
  427. if not self.is_commit_id_equal(dest, rev_options.rev):
  428. logger.info(
  429. "Updating %s %s%s",
  430. display_path(dest),
  431. self.repo_name,
  432. rev_display,
  433. )
  434. self.update(dest, url, rev_options, verbosity=verbosity)
  435. else:
  436. logger.info("Skipping because already up-to-date.")
  437. return
  438. logger.warning(
  439. "%s %s in %s exists with URL %s",
  440. self.name,
  441. self.repo_name,
  442. display_path(dest),
  443. existing_url,
  444. )
  445. prompt = ("(s)witch, (i)gnore, (w)ipe, (b)ackup ", ("s", "i", "w", "b"))
  446. else:
  447. logger.warning(
  448. "Directory %s already exists, and is not a %s %s.",
  449. dest,
  450. self.name,
  451. self.repo_name,
  452. )
  453. # https://github.com/python/mypy/issues/1174
  454. prompt = ("(i)gnore, (w)ipe, (b)ackup ", ("i", "w", "b")) # type: ignore
  455. logger.warning(
  456. "The plan is to install the %s repository %s",
  457. self.name,
  458. url,
  459. )
  460. response = ask_path_exists(f"What to do? {prompt[0]}", prompt[1])
  461. if response == "a":
  462. sys.exit(-1)
  463. if response == "w":
  464. logger.warning("Deleting %s", display_path(dest))
  465. rmtree(dest)
  466. self.fetch_new(dest, url, rev_options, verbosity=verbosity)
  467. return
  468. if response == "b":
  469. dest_dir = backup_dir(dest)
  470. logger.warning("Backing up %s to %s", display_path(dest), dest_dir)
  471. shutil.move(dest, dest_dir)
  472. self.fetch_new(dest, url, rev_options, verbosity=verbosity)
  473. return
  474. # Do nothing if the response is "i".
  475. if response == "s":
  476. logger.info(
  477. "Switching %s %s to %s%s",
  478. self.repo_name,
  479. display_path(dest),
  480. url,
  481. rev_display,
  482. )
  483. self.switch(dest, url, rev_options, verbosity=verbosity)
  484. def unpack(self, location: str, url: HiddenText, verbosity: int) -> None:
  485. """
  486. Clean up current location and download the url repository
  487. (and vcs infos) into location
  488. :param url: the repository URL starting with a vcs prefix.
  489. :param verbosity: verbosity level.
  490. """
  491. if os.path.exists(location):
  492. rmtree(location)
  493. self.obtain(location, url=url, verbosity=verbosity)
  494. @classmethod
  495. def get_remote_url(cls, location: str) -> str:
  496. """
  497. Return the url used at location
  498. Raises RemoteNotFoundError if the repository does not have a remote
  499. url configured.
  500. """
  501. raise NotImplementedError
  502. @classmethod
  503. def get_revision(cls, location: str) -> str:
  504. """
  505. Return the current commit id of the files at the given location.
  506. """
  507. raise NotImplementedError
  508. @classmethod
  509. def run_command(
  510. cls,
  511. cmd: list[str] | CommandArgs,
  512. show_stdout: bool = True,
  513. cwd: str | None = None,
  514. on_returncode: Literal["raise", "warn", "ignore"] = "raise",
  515. extra_ok_returncodes: Iterable[int] | None = None,
  516. command_desc: str | None = None,
  517. extra_environ: Mapping[str, Any] | None = None,
  518. spinner: SpinnerInterface | None = None,
  519. log_failed_cmd: bool = True,
  520. stdout_only: bool = False,
  521. ) -> str:
  522. """
  523. Run a VCS subcommand
  524. This is simply a wrapper around call_subprocess that adds the VCS
  525. command name, and checks that the VCS is available
  526. """
  527. cmd = make_command(cls.name, *cmd)
  528. if command_desc is None:
  529. command_desc = format_command_args(cmd)
  530. try:
  531. return call_subprocess(
  532. cmd,
  533. show_stdout,
  534. cwd,
  535. on_returncode=on_returncode,
  536. extra_ok_returncodes=extra_ok_returncodes,
  537. command_desc=command_desc,
  538. extra_environ=extra_environ,
  539. unset_environ=cls.unset_environ,
  540. spinner=spinner,
  541. log_failed_cmd=log_failed_cmd,
  542. stdout_only=stdout_only,
  543. )
  544. except NotADirectoryError:
  545. raise BadCommand(f"Cannot find command {cls.name!r} - invalid PATH")
  546. except FileNotFoundError:
  547. # errno.ENOENT = no such file or directory
  548. # In other words, the VCS executable isn't available
  549. raise BadCommand(
  550. f"Cannot find command {cls.name!r} - do you have "
  551. f"{cls.name!r} installed and in your PATH?"
  552. )
  553. except PermissionError:
  554. # errno.EACCES = Permission denied
  555. # This error occurs, for instance, when the command is installed
  556. # only for another user. So, the current user don't have
  557. # permission to call the other user command.
  558. raise BadCommand(
  559. f"No permission to execute {cls.name!r} - install it "
  560. f"locally, globally (ask admin), or check your PATH. "
  561. f"See possible solutions at "
  562. f"https://pip.pypa.io/en/latest/reference/pip_freeze/"
  563. f"#fixing-permission-denied."
  564. )
  565. @classmethod
  566. def is_repository_directory(cls, path: str) -> bool:
  567. """
  568. Return whether a directory path is a repository directory.
  569. """
  570. logger.debug("Checking in %s for %s (%s)...", path, cls.dirname, cls.name)
  571. return os.path.exists(os.path.join(path, cls.dirname))
  572. @classmethod
  573. def get_repository_root(cls, location: str) -> str | None:
  574. """
  575. Return the "root" (top-level) directory controlled by the vcs,
  576. or `None` if the directory is not in any.
  577. It is meant to be overridden to implement smarter detection
  578. mechanisms for specific vcs.
  579. This can do more than is_repository_directory() alone. For
  580. example, the Git override checks that Git is actually available.
  581. """
  582. if cls.is_repository_directory(location):
  583. return location
  584. return None