configuration.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397
  1. """Configuration management setup
  2. Some terminology:
  3. - name
  4. As written in config files.
  5. - value
  6. Value associated with a name
  7. - key
  8. Name combined with it's section (section.name)
  9. - variant
  10. A single word describing where the configuration key-value pair came from
  11. """
  12. from __future__ import annotations
  13. import configparser
  14. import locale
  15. import os
  16. import sys
  17. from collections.abc import Iterable
  18. from typing import Any, NewType
  19. from pip._internal.exceptions import (
  20. ConfigurationError,
  21. ConfigurationFileCouldNotBeLoaded,
  22. )
  23. from pip._internal.utils import appdirs
  24. from pip._internal.utils.compat import WINDOWS
  25. from pip._internal.utils.logging import getLogger
  26. from pip._internal.utils.misc import ensure_dir, enum
  27. RawConfigParser = configparser.RawConfigParser # Shorthand
  28. Kind = NewType("Kind", str)
  29. CONFIG_BASENAME = "pip.ini" if WINDOWS else "pip.conf"
  30. ENV_NAMES_IGNORED = "version", "help"
  31. # The kinds of configurations there are.
  32. kinds = enum(
  33. USER="user", # User Specific
  34. GLOBAL="global", # System Wide
  35. SITE="site", # [Virtual] Environment Specific
  36. ENV="env", # from PIP_CONFIG_FILE
  37. ENV_VAR="env-var", # from Environment Variables
  38. )
  39. OVERRIDE_ORDER = kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR
  40. VALID_LOAD_ONLY = kinds.USER, kinds.GLOBAL, kinds.SITE
  41. logger = getLogger(__name__)
  42. # NOTE: Maybe use the optionx attribute to normalize keynames.
  43. def _normalize_name(name: str) -> str:
  44. """Make a name consistent regardless of source (environment or file)"""
  45. name = name.lower().replace("_", "-")
  46. if name.startswith("--"):
  47. name = name[2:] # only prefer long opts
  48. return name
  49. def _disassemble_key(name: str) -> list[str]:
  50. if "." not in name:
  51. error_message = (
  52. "Key does not contain dot separated section and key. "
  53. f"Perhaps you wanted to use 'global.{name}' instead?"
  54. )
  55. raise ConfigurationError(error_message)
  56. return name.split(".", 1)
  57. def get_configuration_files() -> dict[Kind, list[str]]:
  58. global_config_files = [
  59. os.path.join(path, CONFIG_BASENAME) for path in appdirs.site_config_dirs("pip")
  60. ]
  61. site_config_file = os.path.join(sys.prefix, CONFIG_BASENAME)
  62. legacy_config_file = os.path.join(
  63. os.path.expanduser("~"),
  64. "pip" if WINDOWS else ".pip",
  65. CONFIG_BASENAME,
  66. )
  67. new_config_file = os.path.join(appdirs.user_config_dir("pip"), CONFIG_BASENAME)
  68. return {
  69. kinds.GLOBAL: global_config_files,
  70. kinds.SITE: [site_config_file],
  71. kinds.USER: [legacy_config_file, new_config_file],
  72. }
  73. class Configuration:
  74. """Handles management of configuration.
  75. Provides an interface to accessing and managing configuration files.
  76. This class converts provides an API that takes "section.key-name" style
  77. keys and stores the value associated with it as "key-name" under the
  78. section "section".
  79. This allows for a clean interface wherein the both the section and the
  80. key-name are preserved in an easy to manage form in the configuration files
  81. and the data stored is also nice.
  82. """
  83. def __init__(self, isolated: bool, load_only: Kind | None = None) -> None:
  84. super().__init__()
  85. if load_only is not None and load_only not in VALID_LOAD_ONLY:
  86. raise ConfigurationError(
  87. "Got invalid value for load_only - should be one of {}".format(
  88. ", ".join(map(repr, VALID_LOAD_ONLY))
  89. )
  90. )
  91. self.isolated = isolated
  92. self.load_only = load_only
  93. # Because we keep track of where we got the data from
  94. self._parsers: dict[Kind, list[tuple[str, RawConfigParser]]] = {
  95. variant: [] for variant in OVERRIDE_ORDER
  96. }
  97. self._config: dict[Kind, dict[str, dict[str, Any]]] = {
  98. variant: {} for variant in OVERRIDE_ORDER
  99. }
  100. self._modified_parsers: list[tuple[str, RawConfigParser]] = []
  101. def load(self) -> None:
  102. """Loads configuration from configuration files and environment"""
  103. self._load_config_files()
  104. if not self.isolated:
  105. self._load_environment_vars()
  106. def get_file_to_edit(self) -> str | None:
  107. """Returns the file with highest priority in configuration"""
  108. assert self.load_only is not None, "Need to be specified a file to be editing"
  109. try:
  110. return self._get_parser_to_modify()[0]
  111. except IndexError:
  112. return None
  113. def items(self) -> Iterable[tuple[str, Any]]:
  114. """Returns key-value pairs like dict.items() representing the loaded
  115. configuration
  116. """
  117. return self._dictionary.items()
  118. def get_value(self, key: str) -> Any:
  119. """Get a value from the configuration."""
  120. orig_key = key
  121. key = _normalize_name(key)
  122. try:
  123. clean_config: dict[str, Any] = {}
  124. for file_values in self._dictionary.values():
  125. clean_config.update(file_values)
  126. return clean_config[key]
  127. except KeyError:
  128. # disassembling triggers a more useful error message than simply
  129. # "No such key" in the case that the key isn't in the form command.option
  130. _disassemble_key(key)
  131. raise ConfigurationError(f"No such key - {orig_key}")
  132. def set_value(self, key: str, value: Any) -> None:
  133. """Modify a value in the configuration."""
  134. key = _normalize_name(key)
  135. self._ensure_have_load_only()
  136. assert self.load_only
  137. fname, parser = self._get_parser_to_modify()
  138. if parser is not None:
  139. section, name = _disassemble_key(key)
  140. # Modify the parser and the configuration
  141. if not parser.has_section(section):
  142. parser.add_section(section)
  143. parser.set(section, name, value)
  144. self._config[self.load_only].setdefault(fname, {})
  145. self._config[self.load_only][fname][key] = value
  146. self._mark_as_modified(fname, parser)
  147. def unset_value(self, key: str) -> None:
  148. """Unset a value in the configuration."""
  149. orig_key = key
  150. key = _normalize_name(key)
  151. self._ensure_have_load_only()
  152. assert self.load_only
  153. fname, parser = self._get_parser_to_modify()
  154. if (
  155. key not in self._config[self.load_only][fname]
  156. and key not in self._config[self.load_only]
  157. ):
  158. raise ConfigurationError(f"No such key - {orig_key}")
  159. if parser is not None:
  160. section, name = _disassemble_key(key)
  161. if not (
  162. parser.has_section(section) and parser.remove_option(section, name)
  163. ):
  164. # The option was not removed.
  165. raise ConfigurationError(
  166. "Fatal Internal error [id=1]. Please report as a bug."
  167. )
  168. # The section may be empty after the option was removed.
  169. if not parser.items(section):
  170. parser.remove_section(section)
  171. self._mark_as_modified(fname, parser)
  172. try:
  173. del self._config[self.load_only][fname][key]
  174. except KeyError:
  175. del self._config[self.load_only][key]
  176. def save(self) -> None:
  177. """Save the current in-memory state."""
  178. self._ensure_have_load_only()
  179. for fname, parser in self._modified_parsers:
  180. logger.info("Writing to %s", fname)
  181. # Ensure directory exists.
  182. ensure_dir(os.path.dirname(fname))
  183. # Ensure directory's permission(need to be writeable)
  184. try:
  185. with open(fname, "w") as f:
  186. parser.write(f)
  187. except OSError as error:
  188. raise ConfigurationError(
  189. f"An error occurred while writing to the configuration file "
  190. f"{fname}: {error}"
  191. )
  192. #
  193. # Private routines
  194. #
  195. def _ensure_have_load_only(self) -> None:
  196. if self.load_only is None:
  197. raise ConfigurationError("Needed a specific file to be modifying.")
  198. logger.debug("Will be working with %s variant only", self.load_only)
  199. @property
  200. def _dictionary(self) -> dict[str, dict[str, Any]]:
  201. """A dictionary representing the loaded configuration."""
  202. # NOTE: Dictionaries are not populated if not loaded. So, conditionals
  203. # are not needed here.
  204. retval = {}
  205. for variant in OVERRIDE_ORDER:
  206. retval.update(self._config[variant])
  207. return retval
  208. def _load_config_files(self) -> None:
  209. """Loads configuration from configuration files"""
  210. config_files = dict(self.iter_config_files())
  211. if config_files[kinds.ENV][0:1] == [os.devnull]:
  212. logger.debug(
  213. "Skipping loading configuration files due to "
  214. "environment's PIP_CONFIG_FILE being os.devnull"
  215. )
  216. return
  217. for variant, files in config_files.items():
  218. for fname in files:
  219. # If there's specific variant set in `load_only`, load only
  220. # that variant, not the others.
  221. if self.load_only is not None and variant != self.load_only:
  222. logger.debug("Skipping file '%s' (variant: %s)", fname, variant)
  223. continue
  224. parser = self._load_file(variant, fname)
  225. # Keeping track of the parsers used
  226. self._parsers[variant].append((fname, parser))
  227. def _load_file(self, variant: Kind, fname: str) -> RawConfigParser:
  228. logger.verbose("For variant '%s', will try loading '%s'", variant, fname)
  229. parser = self._construct_parser(fname)
  230. for section in parser.sections():
  231. items = parser.items(section)
  232. self._config[variant].setdefault(fname, {})
  233. self._config[variant][fname].update(self._normalized_keys(section, items))
  234. return parser
  235. def _construct_parser(self, fname: str) -> RawConfigParser:
  236. parser = configparser.RawConfigParser()
  237. # If there is no such file, don't bother reading it but create the
  238. # parser anyway, to hold the data.
  239. # Doing this is useful when modifying and saving files, where we don't
  240. # need to construct a parser.
  241. if os.path.exists(fname):
  242. locale_encoding = locale.getpreferredencoding(False)
  243. try:
  244. parser.read(fname, encoding=locale_encoding)
  245. except UnicodeDecodeError:
  246. # See https://github.com/pypa/pip/issues/4963
  247. raise ConfigurationFileCouldNotBeLoaded(
  248. reason=f"contains invalid {locale_encoding} characters",
  249. fname=fname,
  250. )
  251. except configparser.Error as error:
  252. # See https://github.com/pypa/pip/issues/4893
  253. raise ConfigurationFileCouldNotBeLoaded(error=error)
  254. return parser
  255. def _load_environment_vars(self) -> None:
  256. """Loads configuration from environment variables"""
  257. self._config[kinds.ENV_VAR].setdefault(":env:", {})
  258. self._config[kinds.ENV_VAR][":env:"].update(
  259. self._normalized_keys(":env:", self.get_environ_vars())
  260. )
  261. def _normalized_keys(
  262. self, section: str, items: Iterable[tuple[str, Any]]
  263. ) -> dict[str, Any]:
  264. """Normalizes items to construct a dictionary with normalized keys.
  265. This routine is where the names become keys and are made the same
  266. regardless of source - configuration files or environment.
  267. """
  268. normalized = {}
  269. for name, val in items:
  270. key = section + "." + _normalize_name(name)
  271. normalized[key] = val
  272. return normalized
  273. def get_environ_vars(self) -> Iterable[tuple[str, str]]:
  274. """Returns a generator with all environmental vars with prefix PIP_"""
  275. for key, val in os.environ.items():
  276. if key.startswith("PIP_"):
  277. name = key[4:].lower()
  278. if name not in ENV_NAMES_IGNORED:
  279. yield name, val
  280. # XXX: This is patched in the tests.
  281. def iter_config_files(self) -> Iterable[tuple[Kind, list[str]]]:
  282. """Yields variant and configuration files associated with it.
  283. This should be treated like items of a dictionary. The order
  284. here doesn't affect what gets overridden. That is controlled
  285. by OVERRIDE_ORDER. However this does control the order they are
  286. displayed to the user. It's probably most ergonomic to display
  287. things in the same order as OVERRIDE_ORDER
  288. """
  289. # SMELL: Move the conditions out of this function
  290. env_config_file = os.environ.get("PIP_CONFIG_FILE", None)
  291. config_files = get_configuration_files()
  292. yield kinds.GLOBAL, config_files[kinds.GLOBAL]
  293. # per-user config is not loaded when env_config_file exists
  294. should_load_user_config = not self.isolated and not (
  295. env_config_file and os.path.exists(env_config_file)
  296. )
  297. if should_load_user_config:
  298. # The legacy config file is overridden by the new config file
  299. yield kinds.USER, config_files[kinds.USER]
  300. # virtualenv config
  301. yield kinds.SITE, config_files[kinds.SITE]
  302. if env_config_file is not None:
  303. yield kinds.ENV, [env_config_file]
  304. else:
  305. yield kinds.ENV, []
  306. def get_values_in_config(self, variant: Kind) -> dict[str, Any]:
  307. """Get values present in a config file"""
  308. return self._config[variant]
  309. def _get_parser_to_modify(self) -> tuple[str, RawConfigParser]:
  310. # Determine which parser to modify
  311. assert self.load_only
  312. parsers = self._parsers[self.load_only]
  313. if not parsers:
  314. # This should not happen if everything works correctly.
  315. raise ConfigurationError(
  316. "Fatal Internal error [id=2]. Please report as a bug."
  317. )
  318. # Use the highest priority parser.
  319. return parsers[-1]
  320. # XXX: This is patched in the tests.
  321. def _mark_as_modified(self, fname: str, parser: RawConfigParser) -> None:
  322. file_parser_tuple = (fname, parser)
  323. if file_parser_tuple not in self._modified_parsers:
  324. self._modified_parsers.append(file_parser_tuple)
  325. def __repr__(self) -> str:
  326. return f"{self.__class__.__name__}({self._dictionary!r})"