subprocess.py 8.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248
  1. from __future__ import annotations
  2. import logging
  3. import os
  4. import shlex
  5. import subprocess
  6. from collections.abc import Iterable, Mapping
  7. from typing import Any, Callable, Literal, Union
  8. from pip._vendor.rich.markup import escape
  9. from pip._internal.cli.spinners import SpinnerInterface, open_spinner
  10. from pip._internal.exceptions import InstallationSubprocessError
  11. from pip._internal.utils.logging import VERBOSE, subprocess_logger
  12. from pip._internal.utils.misc import HiddenText
  13. CommandArgs = list[Union[str, HiddenText]]
  14. def make_command(*args: str | HiddenText | CommandArgs) -> CommandArgs:
  15. """
  16. Create a CommandArgs object.
  17. """
  18. command_args: CommandArgs = []
  19. for arg in args:
  20. # Check for list instead of CommandArgs since CommandArgs is
  21. # only known during type-checking.
  22. if isinstance(arg, list):
  23. command_args.extend(arg)
  24. else:
  25. # Otherwise, arg is str or HiddenText.
  26. command_args.append(arg)
  27. return command_args
  28. def format_command_args(args: list[str] | CommandArgs) -> str:
  29. """
  30. Format command arguments for display.
  31. """
  32. # For HiddenText arguments, display the redacted form by calling str().
  33. # Also, we don't apply str() to arguments that aren't HiddenText since
  34. # this can trigger a UnicodeDecodeError in Python 2 if the argument
  35. # has type unicode and includes a non-ascii character. (The type
  36. # checker doesn't ensure the annotations are correct in all cases.)
  37. return " ".join(
  38. shlex.quote(str(arg)) if isinstance(arg, HiddenText) else shlex.quote(arg)
  39. for arg in args
  40. )
  41. def reveal_command_args(args: list[str] | CommandArgs) -> list[str]:
  42. """
  43. Return the arguments in their raw, unredacted form.
  44. """
  45. return [arg.secret if isinstance(arg, HiddenText) else arg for arg in args]
  46. def call_subprocess(
  47. cmd: list[str] | CommandArgs,
  48. show_stdout: bool = False,
  49. cwd: str | None = None,
  50. on_returncode: Literal["raise", "warn", "ignore"] = "raise",
  51. extra_ok_returncodes: Iterable[int] | None = None,
  52. extra_environ: Mapping[str, Any] | None = None,
  53. unset_environ: Iterable[str] | None = None,
  54. spinner: SpinnerInterface | None = None,
  55. log_failed_cmd: bool | None = True,
  56. stdout_only: bool | None = False,
  57. *,
  58. command_desc: str,
  59. ) -> str:
  60. """
  61. Args:
  62. show_stdout: if true, use INFO to log the subprocess's stderr and
  63. stdout streams. Otherwise, use DEBUG. Defaults to False.
  64. extra_ok_returncodes: an iterable of integer return codes that are
  65. acceptable, in addition to 0. Defaults to None, which means [].
  66. unset_environ: an iterable of environment variable names to unset
  67. prior to calling subprocess.Popen().
  68. log_failed_cmd: if false, failed commands are not logged, only raised.
  69. stdout_only: if true, return only stdout, else return both. When true,
  70. logging of both stdout and stderr occurs when the subprocess has
  71. terminated, else logging occurs as subprocess output is produced.
  72. """
  73. if extra_ok_returncodes is None:
  74. extra_ok_returncodes = []
  75. if unset_environ is None:
  76. unset_environ = []
  77. # Most places in pip use show_stdout=False. What this means is--
  78. #
  79. # - We connect the child's output (combined stderr and stdout) to a
  80. # single pipe, which we read.
  81. # - We log this output to stderr at DEBUG level as it is received.
  82. # - If DEBUG logging isn't enabled (e.g. if --verbose logging wasn't
  83. # requested), then we show a spinner so the user can still see the
  84. # subprocess is in progress.
  85. # - If the subprocess exits with an error, we log the output to stderr
  86. # at ERROR level if it hasn't already been displayed to the console
  87. # (e.g. if --verbose logging wasn't enabled). This way we don't log
  88. # the output to the console twice.
  89. #
  90. # If show_stdout=True, then the above is still done, but with DEBUG
  91. # replaced by INFO.
  92. if show_stdout:
  93. # Then log the subprocess output at INFO level.
  94. log_subprocess: Callable[..., None] = subprocess_logger.info
  95. used_level = logging.INFO
  96. else:
  97. # Then log the subprocess output using VERBOSE. This also ensures
  98. # it will be logged to the log file (aka user_log), if enabled.
  99. log_subprocess = subprocess_logger.verbose
  100. used_level = VERBOSE
  101. # Whether the subprocess will be visible in the console.
  102. showing_subprocess = subprocess_logger.getEffectiveLevel() <= used_level
  103. # Only use the spinner if we're not showing the subprocess output
  104. # and we have a spinner.
  105. use_spinner = not showing_subprocess and spinner is not None
  106. log_subprocess("Running command %s", command_desc)
  107. env = os.environ.copy()
  108. if extra_environ:
  109. env.update(extra_environ)
  110. for name in unset_environ:
  111. env.pop(name, None)
  112. try:
  113. proc = subprocess.Popen(
  114. # Convert HiddenText objects to the underlying str.
  115. reveal_command_args(cmd),
  116. stdin=subprocess.PIPE,
  117. stdout=subprocess.PIPE,
  118. stderr=subprocess.STDOUT if not stdout_only else subprocess.PIPE,
  119. cwd=cwd,
  120. env=env,
  121. errors="backslashreplace",
  122. )
  123. except Exception as exc:
  124. if log_failed_cmd:
  125. subprocess_logger.critical(
  126. "Error %s while executing command %s",
  127. exc,
  128. command_desc,
  129. )
  130. raise
  131. all_output = []
  132. if not stdout_only:
  133. assert proc.stdout
  134. assert proc.stdin
  135. proc.stdin.close()
  136. # In this mode, stdout and stderr are in the same pipe.
  137. while True:
  138. line: str = proc.stdout.readline()
  139. if not line:
  140. break
  141. line = line.rstrip()
  142. all_output.append(line + "\n")
  143. # Show the line immediately.
  144. log_subprocess(line)
  145. # Update the spinner.
  146. if use_spinner:
  147. assert spinner
  148. spinner.spin()
  149. try:
  150. proc.wait()
  151. finally:
  152. if proc.stdout:
  153. proc.stdout.close()
  154. output = "".join(all_output)
  155. else:
  156. # In this mode, stdout and stderr are in different pipes.
  157. # We must use communicate() which is the only safe way to read both.
  158. out, err = proc.communicate()
  159. # log line by line to preserve pip log indenting
  160. for out_line in out.splitlines():
  161. log_subprocess(out_line)
  162. all_output.append(out)
  163. for err_line in err.splitlines():
  164. log_subprocess(err_line)
  165. all_output.append(err)
  166. output = out
  167. proc_had_error = proc.returncode and proc.returncode not in extra_ok_returncodes
  168. if use_spinner:
  169. assert spinner
  170. if proc_had_error:
  171. spinner.finish("error")
  172. else:
  173. spinner.finish("done")
  174. if proc_had_error:
  175. if on_returncode == "raise":
  176. error = InstallationSubprocessError(
  177. command_description=command_desc,
  178. exit_code=proc.returncode,
  179. output_lines=all_output if not showing_subprocess else None,
  180. )
  181. if log_failed_cmd:
  182. subprocess_logger.error("%s", error, extra={"rich": True})
  183. subprocess_logger.verbose(
  184. "[bold magenta]full command[/]: [blue]%s[/]",
  185. escape(format_command_args(cmd)),
  186. extra={"markup": True},
  187. )
  188. subprocess_logger.verbose(
  189. "[bold magenta]cwd[/]: %s",
  190. escape(cwd or "[inherit]"),
  191. extra={"markup": True},
  192. )
  193. raise error
  194. elif on_returncode == "warn":
  195. subprocess_logger.warning(
  196. 'Command "%s" had error code %s in %s',
  197. command_desc,
  198. proc.returncode,
  199. cwd,
  200. )
  201. elif on_returncode == "ignore":
  202. pass
  203. else:
  204. raise ValueError(f"Invalid value: on_returncode={on_returncode!r}")
  205. return output
  206. def runner_with_spinner_message(message: str) -> Callable[..., None]:
  207. """Provide a subprocess_runner that shows a spinner message.
  208. Intended for use with for BuildBackendHookCaller. Thus, the runner has
  209. an API that matches what's expected by BuildBackendHookCaller.subprocess_runner.
  210. """
  211. def runner(
  212. cmd: list[str],
  213. cwd: str | None = None,
  214. extra_environ: Mapping[str, Any] | None = None,
  215. ) -> None:
  216. with open_spinner(message) as spinner:
  217. call_subprocess(
  218. cmd,
  219. command_desc=message,
  220. cwd=cwd,
  221. extra_environ=extra_environ,
  222. spinner=spinner,
  223. )
  224. return runner