introspection.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587
  1. """High-level introspection utilities, used to inspect type annotations."""
  2. from __future__ import annotations
  3. import sys
  4. import types
  5. from collections.abc import Generator
  6. from dataclasses import InitVar
  7. from enum import Enum, IntEnum, auto
  8. from typing import Any, Literal, NamedTuple, cast
  9. from typing_extensions import TypeAlias, assert_never, get_args, get_origin
  10. from . import typing_objects
  11. __all__ = (
  12. 'AnnotationSource',
  13. 'ForbiddenQualifier',
  14. 'InspectedAnnotation',
  15. 'Qualifier',
  16. 'get_literal_values',
  17. 'inspect_annotation',
  18. 'is_union_origin',
  19. )
  20. if sys.version_info >= (3, 14) or sys.version_info < (3, 10):
  21. def is_union_origin(obj: Any, /) -> bool:
  22. """Return whether the provided origin is the union form.
  23. ```pycon
  24. >>> is_union_origin(typing.Union)
  25. True
  26. >>> is_union_origin(get_origin(int | str))
  27. True
  28. >>> is_union_origin(types.UnionType)
  29. True
  30. ```
  31. !!! note
  32. Since Python 3.14, both `Union[<t1>, <t2>, ...]` and `<t1> | <t2> | ...` forms create instances
  33. of the same [`typing.Union`][] class. As such, it is recommended to not use this function
  34. anymore (provided that you only support Python 3.14 or greater), and instead use the
  35. [`typing_objects.is_union()`][typing_inspection.typing_objects.is_union] function directly:
  36. ```python
  37. from typing import Union, get_origin
  38. from typing_inspection import typing_objects
  39. typ = int | str # Or Union[int, str]
  40. origin = get_origin(typ)
  41. if typing_objects.is_union(origin):
  42. ...
  43. ```
  44. """
  45. return typing_objects.is_union(obj)
  46. else:
  47. def is_union_origin(obj: Any, /) -> bool:
  48. """Return whether the provided origin is the union form.
  49. ```pycon
  50. >>> is_union_origin(typing.Union)
  51. True
  52. >>> is_union_origin(get_origin(int | str))
  53. True
  54. >>> is_union_origin(types.UnionType)
  55. True
  56. ```
  57. !!! note
  58. Since Python 3.14, both `Union[<t1>, <t2>, ...]` and `<t1> | <t2> | ...` forms create instances
  59. of the same [`typing.Union`][] class. As such, it is recommended to not use this function
  60. anymore (provided that you only support Python 3.14 or greater), and instead use the
  61. [`typing_objects.is_union()`][typing_inspection.typing_objects.is_union] function directly:
  62. ```python
  63. from typing import Union, get_origin
  64. from typing_inspection import typing_objects
  65. typ = int | str # Or Union[int, str]
  66. origin = get_origin(typ)
  67. if typing_objects.is_union(origin):
  68. ...
  69. ```
  70. """
  71. return typing_objects.is_union(obj) or obj is types.UnionType
  72. def _literal_type_check(value: Any, /) -> None:
  73. """Type check the provided literal value against the legal parameters."""
  74. if (
  75. not isinstance(value, (int, bytes, str, bool, Enum, typing_objects.NoneType))
  76. and value is not typing_objects.NoneType
  77. ):
  78. raise TypeError(f'{value} is not a valid literal value, must be one of: int, bytes, str, Enum, None.')
  79. def get_literal_values(
  80. annotation: Any,
  81. /,
  82. *,
  83. type_check: bool = False,
  84. unpack_type_aliases: Literal['skip', 'lenient', 'eager'] = 'eager',
  85. ) -> Generator[Any]:
  86. """Yield the values contained in the provided [`Literal`][typing.Literal] [special form][].
  87. Args:
  88. annotation: The [`Literal`][typing.Literal] [special form][] to unpack.
  89. type_check: Whether to check if the literal values are [legal parameters][literal-legal-parameters].
  90. Raises a [`TypeError`][] otherwise.
  91. unpack_type_aliases: What to do when encountering [PEP 695](https://peps.python.org/pep-0695/)
  92. [type aliases][type-aliases]. Can be one of:
  93. - `'skip'`: Do not try to parse type aliases. Note that this can lead to incorrect results:
  94. ```pycon
  95. >>> type MyAlias = Literal[1, 2]
  96. >>> list(get_literal_values(Literal[MyAlias, 3], unpack_type_aliases="skip"))
  97. [MyAlias, 3]
  98. ```
  99. - `'lenient'`: Try to parse type aliases, and fallback to `'skip'` if the type alias can't be inspected
  100. (because of an undefined forward reference).
  101. - `'eager'`: Parse type aliases and raise any encountered [`NameError`][] exceptions (the default):
  102. ```pycon
  103. >>> type MyAlias = Literal[1, 2]
  104. >>> list(get_literal_values(Literal[MyAlias, 3], unpack_type_aliases="eager"))
  105. [1, 2, 3]
  106. ```
  107. Note:
  108. While `None` is [equivalent to][none] `type(None)`, the runtime implementation of [`Literal`][typing.Literal]
  109. does not de-duplicate them. This function makes sure this de-duplication is applied:
  110. ```pycon
  111. >>> list(get_literal_values(Literal[NoneType, None]))
  112. [None]
  113. ```
  114. Example:
  115. ```pycon
  116. >>> type Ints = Literal[1, 2]
  117. >>> list(get_literal_values(Literal[1, Ints], unpack_type_alias="skip"))
  118. ["a", Ints]
  119. >>> list(get_literal_values(Literal[1, Ints]))
  120. [1, 2]
  121. >>> list(get_literal_values(Literal[1.0], type_check=True))
  122. Traceback (most recent call last):
  123. ...
  124. TypeError: 1.0 is not a valid literal value, must be one of: int, bytes, str, Enum, None.
  125. ```
  126. """
  127. # `literal` is guaranteed to be a `Literal[...]` special form, so use
  128. # `__args__` directly instead of calling `get_args()`.
  129. if unpack_type_aliases == 'skip':
  130. _has_none = False
  131. # `Literal` parameters are already deduplicated, no need to do it ourselves.
  132. # (we only check for `None` and `NoneType`, which should be considered as duplicates).
  133. for arg in annotation.__args__:
  134. if type_check:
  135. _literal_type_check(arg)
  136. if arg is None or arg is typing_objects.NoneType:
  137. if not _has_none:
  138. yield None
  139. _has_none = True
  140. else:
  141. yield arg
  142. else:
  143. # We'll need to manually deduplicate parameters, see the `Literal` implementation in `typing`.
  144. values_and_type: list[tuple[Any, type[Any]]] = []
  145. for arg in annotation.__args__:
  146. # Note: we could also check for generic aliases with a type alias as an origin.
  147. # However, it is very unlikely that this happens as type variables can't appear in
  148. # `Literal` forms, so the only valid (but unnecessary) use case would be something like:
  149. # `type Test[T] = Literal['a']` (and then use `Test[SomeType]`).
  150. if typing_objects.is_typealiastype(arg):
  151. try:
  152. alias_value = arg.__value__
  153. except NameError:
  154. if unpack_type_aliases == 'eager':
  155. raise
  156. # unpack_type_aliases == "lenient":
  157. if type_check:
  158. _literal_type_check(arg)
  159. values_and_type.append((arg, type(arg)))
  160. else:
  161. sub_args = get_literal_values(
  162. alias_value, type_check=type_check, unpack_type_aliases=unpack_type_aliases
  163. )
  164. values_and_type.extend((a, type(a)) for a in sub_args) # pyright: ignore[reportUnknownArgumentType]
  165. else:
  166. if type_check:
  167. _literal_type_check(arg)
  168. if arg is typing_objects.NoneType:
  169. values_and_type.append((None, typing_objects.NoneType))
  170. else:
  171. values_and_type.append((arg, type(arg))) # pyright: ignore[reportUnknownArgumentType]
  172. try:
  173. dct = dict.fromkeys(values_and_type)
  174. except TypeError:
  175. # Unhashable parameters, the Python implementation allows them
  176. yield from (p for p, _ in values_and_type)
  177. else:
  178. yield from (p for p, _ in dct)
  179. Qualifier: TypeAlias = Literal['required', 'not_required', 'read_only', 'class_var', 'init_var', 'final']
  180. """A [type qualifier][]."""
  181. _all_qualifiers: set[Qualifier] = set(get_args(Qualifier))
  182. # TODO at some point, we could switch to an enum flag, so that multiple sources
  183. # can be combined. However, is there a need for this?
  184. class AnnotationSource(IntEnum):
  185. # TODO if/when https://peps.python.org/pep-0767/ is accepted, add 'read_only'
  186. # to CLASS and NAMED_TUPLE (even though for named tuples it is redundant).
  187. """The source of an annotation, e.g. a class or a function.
  188. Depending on the source, different [type qualifiers][type qualifier] may be (dis)allowed.
  189. """
  190. ASSIGNMENT_OR_VARIABLE = auto()
  191. """An annotation used in an assignment or variable annotation:
  192. ```python
  193. x: Final[int] = 1
  194. y: Final[str]
  195. ```
  196. **Allowed type qualifiers:** [`Final`][typing.Final].
  197. """
  198. CLASS = auto()
  199. """An annotation used in the body of a class:
  200. ```python
  201. class Test:
  202. x: Final[int] = 1
  203. y: ClassVar[str]
  204. ```
  205. **Allowed type qualifiers:** [`ClassVar`][typing.ClassVar], [`Final`][typing.Final].
  206. """
  207. DATACLASS = auto()
  208. """An annotation used in the body of a dataclass:
  209. ```python
  210. @dataclass
  211. class Test:
  212. x: Final[int] = 1
  213. y: InitVar[str] = 'test'
  214. ```
  215. **Allowed type qualifiers:** [`ClassVar`][typing.ClassVar], [`Final`][typing.Final], [`InitVar`][dataclasses.InitVar].
  216. """ # noqa: E501
  217. TYPED_DICT = auto()
  218. """An annotation used in the body of a [`TypedDict`][typing.TypedDict]:
  219. ```python
  220. class TD(TypedDict):
  221. x: Required[ReadOnly[int]]
  222. y: ReadOnly[NotRequired[str]]
  223. ```
  224. **Allowed type qualifiers:** [`ReadOnly`][typing.ReadOnly], [`Required`][typing.Required],
  225. [`NotRequired`][typing.NotRequired].
  226. """
  227. NAMED_TUPLE = auto()
  228. """An annotation used in the body of a [`NamedTuple`][typing.NamedTuple].
  229. ```python
  230. class NT(NamedTuple):
  231. x: int
  232. y: str
  233. ```
  234. **Allowed type qualifiers:** none.
  235. """
  236. FUNCTION = auto()
  237. """An annotation used in a function, either for a parameter or the return value.
  238. ```python
  239. def func(a: int) -> str:
  240. ...
  241. ```
  242. **Allowed type qualifiers:** none.
  243. """
  244. ANY = auto()
  245. """An annotation that might come from any source.
  246. **Allowed type qualifiers:** all.
  247. """
  248. BARE = auto()
  249. """An annotation that is inspected as is.
  250. **Allowed type qualifiers:** none.
  251. """
  252. @property
  253. def allowed_qualifiers(self) -> set[Qualifier]:
  254. """The allowed [type qualifiers][type qualifier] for this annotation source."""
  255. # TODO use a match statement when Python 3.9 support is dropped.
  256. if self is AnnotationSource.ASSIGNMENT_OR_VARIABLE:
  257. return {'final'}
  258. elif self is AnnotationSource.CLASS:
  259. return {'final', 'class_var'}
  260. elif self is AnnotationSource.DATACLASS:
  261. return {'final', 'class_var', 'init_var'}
  262. elif self is AnnotationSource.TYPED_DICT:
  263. return {'required', 'not_required', 'read_only'}
  264. elif self in (AnnotationSource.NAMED_TUPLE, AnnotationSource.FUNCTION, AnnotationSource.BARE):
  265. return set()
  266. elif self is AnnotationSource.ANY:
  267. return _all_qualifiers
  268. else: # pragma: no cover
  269. assert_never(self)
  270. class ForbiddenQualifier(Exception):
  271. """The provided [type qualifier][] is forbidden."""
  272. qualifier: Qualifier
  273. """The forbidden qualifier."""
  274. def __init__(self, qualifier: Qualifier, /) -> None:
  275. self.qualifier = qualifier
  276. class _UnknownTypeEnum(Enum):
  277. UNKNOWN = auto()
  278. def __str__(self) -> str:
  279. return 'UNKNOWN'
  280. def __repr__(self) -> str:
  281. return '<UNKNOWN>'
  282. UNKNOWN = _UnknownTypeEnum.UNKNOWN
  283. """A sentinel value used when no [type expression][] is present."""
  284. _UnkownType: TypeAlias = Literal[_UnknownTypeEnum.UNKNOWN]
  285. """The type of the [`UNKNOWN`][typing_inspection.introspection.UNKNOWN] sentinel value."""
  286. class InspectedAnnotation(NamedTuple):
  287. """The result of the inspected annotation."""
  288. type: Any | _UnkownType
  289. """The final [type expression][], with [type qualifiers][type qualifier] and annotated metadata stripped.
  290. If no type expression is available, the [`UNKNOWN`][typing_inspection.introspection.UNKNOWN] sentinel
  291. value is used instead. This is the case when a [type qualifier][] is used with no type annotation:
  292. ```python
  293. ID: Final = 1
  294. class C:
  295. x: ClassVar = 'test'
  296. ```
  297. """
  298. qualifiers: set[Qualifier]
  299. """The [type qualifiers][type qualifier] present on the annotation."""
  300. metadata: list[Any]
  301. """The annotated metadata."""
  302. def inspect_annotation( # noqa: PLR0915
  303. annotation: Any,
  304. /,
  305. *,
  306. annotation_source: AnnotationSource,
  307. unpack_type_aliases: Literal['skip', 'lenient', 'eager'] = 'skip',
  308. ) -> InspectedAnnotation:
  309. """Inspect an [annotation expression][], extracting any [type qualifier][] and metadata.
  310. An [annotation expression][] is a [type expression][] optionally surrounded by one or more
  311. [type qualifiers][type qualifier] or by [`Annotated`][typing.Annotated]. This function will:
  312. - Unwrap the type expression, keeping track of the type qualifiers.
  313. - Unwrap [`Annotated`][typing.Annotated] forms, keeping track of the annotated metadata.
  314. Args:
  315. annotation: The annotation expression to be inspected.
  316. annotation_source: The source of the annotation. Depending on the source (e.g. a class), different type
  317. qualifiers may be (dis)allowed. To allow any type qualifier, use
  318. [`AnnotationSource.ANY`][typing_inspection.introspection.AnnotationSource.ANY].
  319. unpack_type_aliases: What to do when encountering [PEP 695](https://peps.python.org/pep-0695/)
  320. [type aliases][type-aliases]. Can be one of:
  321. - `'skip'`: Do not try to parse type aliases (the default):
  322. ```pycon
  323. >>> type MyInt = Annotated[int, 'meta']
  324. >>> inspect_annotation(MyInt, annotation_source=AnnotationSource.BARE, unpack_type_aliases='skip')
  325. InspectedAnnotation(type=MyInt, qualifiers={}, metadata=[])
  326. ```
  327. - `'lenient'`: Try to parse type aliases, and fallback to `'skip'` if the type alias
  328. can't be inspected (because of an undefined forward reference):
  329. ```pycon
  330. >>> type MyInt = Annotated[Undefined, 'meta']
  331. >>> inspect_annotation(MyInt, annotation_source=AnnotationSource.BARE, unpack_type_aliases='lenient')
  332. InspectedAnnotation(type=MyInt, qualifiers={}, metadata=[])
  333. >>> Undefined = int
  334. >>> inspect_annotation(MyInt, annotation_source=AnnotationSource.BARE, unpack_type_aliases='lenient')
  335. InspectedAnnotation(type=int, qualifiers={}, metadata=['meta'])
  336. ```
  337. - `'eager'`: Parse type aliases and raise any encountered [`NameError`][] exceptions.
  338. Returns:
  339. The result of the inspected annotation, where the type expression, used qualifiers and metadata is stored.
  340. Example:
  341. ```pycon
  342. >>> inspect_annotation(
  343. ... Final[Annotated[ClassVar[Annotated[int, 'meta_1']], 'meta_2']],
  344. ... annotation_source=AnnotationSource.CLASS,
  345. ... )
  346. ...
  347. InspectedAnnotation(type=int, qualifiers={'class_var', 'final'}, metadata=['meta_1', 'meta_2'])
  348. ```
  349. """
  350. allowed_qualifiers = annotation_source.allowed_qualifiers
  351. qualifiers: set[Qualifier] = set()
  352. metadata: list[Any] = []
  353. while True:
  354. annotation, _meta = _unpack_annotated(annotation, unpack_type_aliases=unpack_type_aliases)
  355. if _meta:
  356. metadata = _meta + metadata
  357. continue
  358. origin = get_origin(annotation)
  359. if origin is not None:
  360. if typing_objects.is_classvar(origin):
  361. if 'class_var' not in allowed_qualifiers:
  362. raise ForbiddenQualifier('class_var')
  363. qualifiers.add('class_var')
  364. annotation = annotation.__args__[0]
  365. elif typing_objects.is_final(origin):
  366. if 'final' not in allowed_qualifiers:
  367. raise ForbiddenQualifier('final')
  368. qualifiers.add('final')
  369. annotation = annotation.__args__[0]
  370. elif typing_objects.is_required(origin):
  371. if 'required' not in allowed_qualifiers:
  372. raise ForbiddenQualifier('required')
  373. qualifiers.add('required')
  374. annotation = annotation.__args__[0]
  375. elif typing_objects.is_notrequired(origin):
  376. if 'not_required' not in allowed_qualifiers:
  377. raise ForbiddenQualifier('not_required')
  378. qualifiers.add('not_required')
  379. annotation = annotation.__args__[0]
  380. elif typing_objects.is_readonly(origin):
  381. if 'read_only' not in allowed_qualifiers:
  382. raise ForbiddenQualifier('not_required')
  383. qualifiers.add('read_only')
  384. annotation = annotation.__args__[0]
  385. else:
  386. # origin is not None but not a type qualifier nor `Annotated` (e.g. `list[int]`):
  387. break
  388. elif isinstance(annotation, InitVar):
  389. if 'init_var' not in allowed_qualifiers:
  390. raise ForbiddenQualifier('init_var')
  391. qualifiers.add('init_var')
  392. annotation = cast(Any, annotation.type)
  393. else:
  394. break
  395. # `Final`, `ClassVar` and `InitVar` are type qualifiers allowed to be used as a bare annotation:
  396. if typing_objects.is_final(annotation):
  397. if 'final' not in allowed_qualifiers:
  398. raise ForbiddenQualifier('final')
  399. qualifiers.add('final')
  400. annotation = UNKNOWN
  401. elif typing_objects.is_classvar(annotation):
  402. if 'class_var' not in allowed_qualifiers:
  403. raise ForbiddenQualifier('class_var')
  404. qualifiers.add('class_var')
  405. annotation = UNKNOWN
  406. elif annotation is InitVar:
  407. if 'init_var' not in allowed_qualifiers:
  408. raise ForbiddenQualifier('init_var')
  409. qualifiers.add('init_var')
  410. annotation = UNKNOWN
  411. return InspectedAnnotation(annotation, qualifiers, metadata)
  412. def _unpack_annotated_inner(
  413. annotation: Any, unpack_type_aliases: Literal['lenient', 'eager'], check_annotated: bool
  414. ) -> tuple[Any, list[Any]]:
  415. origin = get_origin(annotation)
  416. if check_annotated and typing_objects.is_annotated(origin):
  417. annotated_type = annotation.__origin__
  418. metadata = list(annotation.__metadata__)
  419. # The annotated type might be a PEP 695 type alias, so we need to recursively
  420. # unpack it. Because Python already flattens `Annotated[Annotated[<type>, ...], ...]` forms,
  421. # we can skip the `is_annotated()` check in the next call:
  422. annotated_type, sub_meta = _unpack_annotated_inner(
  423. annotated_type, unpack_type_aliases=unpack_type_aliases, check_annotated=False
  424. )
  425. metadata = sub_meta + metadata
  426. return annotated_type, metadata
  427. elif typing_objects.is_typealiastype(annotation):
  428. try:
  429. value = annotation.__value__
  430. except NameError:
  431. if unpack_type_aliases == 'eager':
  432. raise
  433. else:
  434. typ, metadata = _unpack_annotated_inner(
  435. value, unpack_type_aliases=unpack_type_aliases, check_annotated=True
  436. )
  437. if metadata:
  438. # Having metadata means the type alias' `__value__` was an `Annotated` form
  439. # (or, recursively, a type alias to an `Annotated` form). It is important to check
  440. # for this, as we don't want to unpack other type aliases (e.g. `type MyInt = int`).
  441. return typ, metadata
  442. return annotation, []
  443. elif typing_objects.is_typealiastype(origin):
  444. # When parameterized, PEP 695 type aliases become generic aliases
  445. # (e.g. with `type MyList[T] = Annotated[list[T], ...]`, `MyList[int]`
  446. # is a generic alias).
  447. try:
  448. value = origin.__value__
  449. except NameError:
  450. if unpack_type_aliases == 'eager':
  451. raise
  452. else:
  453. # While Python already handles type variable replacement for simple `Annotated` forms,
  454. # we need to manually apply the same logic for PEP 695 type aliases:
  455. # - With `MyList = Annotated[list[T], ...]`, `MyList[int] == Annotated[list[int], ...]`
  456. # - With `type MyList[T] = Annotated[list[T], ...]`, `MyList[int].__value__ == Annotated[list[T], ...]`.
  457. try:
  458. # To do so, we emulate the parameterization of the value with the arguments:
  459. # with `type MyList[T] = Annotated[list[T], ...]`, to emulate `MyList[int]`,
  460. # we do `Annotated[list[T], ...][int]` (which gives `Annotated[list[T], ...]`):
  461. value = value[annotation.__args__]
  462. except TypeError:
  463. # Might happen if the type alias is parameterized, but its value doesn't have any
  464. # type variables, e.g. `type MyInt[T] = int`.
  465. pass
  466. typ, metadata = _unpack_annotated_inner(
  467. value, unpack_type_aliases=unpack_type_aliases, check_annotated=True
  468. )
  469. if metadata:
  470. return typ, metadata
  471. return annotation, []
  472. return annotation, []
  473. # This could eventually be made public:
  474. def _unpack_annotated(
  475. annotation: Any, /, *, unpack_type_aliases: Literal['skip', 'lenient', 'eager'] = 'eager'
  476. ) -> tuple[Any, list[Any]]:
  477. if unpack_type_aliases == 'skip':
  478. if typing_objects.is_annotated(get_origin(annotation)):
  479. return annotation.__origin__, list(annotation.__metadata__)
  480. else:
  481. return annotation, []
  482. return _unpack_annotated_inner(annotation, unpack_type_aliases=unpack_type_aliases, check_annotated=True)