typing.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614
  1. import functools
  2. import operator
  3. import sys
  4. import typing
  5. from collections.abc import Callable
  6. from os import PathLike
  7. from typing import ( # type: ignore
  8. TYPE_CHECKING,
  9. AbstractSet,
  10. Any,
  11. Callable as TypingCallable,
  12. ClassVar,
  13. Dict,
  14. ForwardRef,
  15. Generator,
  16. Iterable,
  17. List,
  18. Mapping,
  19. NewType,
  20. Optional,
  21. Sequence,
  22. Set,
  23. Tuple,
  24. Type,
  25. TypeVar,
  26. Union,
  27. _eval_type,
  28. cast,
  29. get_type_hints,
  30. )
  31. from typing_extensions import (
  32. Annotated,
  33. Final,
  34. Literal,
  35. NotRequired as TypedDictNotRequired,
  36. Required as TypedDictRequired,
  37. )
  38. try:
  39. from typing import _TypingBase as typing_base # type: ignore
  40. except ImportError:
  41. from typing import _Final as typing_base # type: ignore
  42. try:
  43. from typing import GenericAlias as TypingGenericAlias # type: ignore
  44. except ImportError:
  45. # python < 3.9 does not have GenericAlias (list[int], tuple[str, ...] and so on)
  46. TypingGenericAlias = ()
  47. try:
  48. from types import UnionType as TypesUnionType # type: ignore
  49. except ImportError:
  50. # python < 3.10 does not have UnionType (str | int, byte | bool and so on)
  51. TypesUnionType = ()
  52. if sys.version_info < (3, 9):
  53. def evaluate_forwardref(type_: ForwardRef, globalns: Any, localns: Any) -> Any:
  54. return type_._evaluate(globalns, localns)
  55. elif sys.version_info < (3, 12, 4):
  56. def evaluate_forwardref(type_: ForwardRef, globalns: Any, localns: Any) -> Any:
  57. # Even though it is the right signature for python 3.9, mypy complains with
  58. # `error: Too many arguments for "_evaluate" of "ForwardRef"` hence the cast...
  59. # Python 3.13/3.12.4+ made `recursive_guard` a kwarg, so name it explicitly to avoid:
  60. # TypeError: ForwardRef._evaluate() missing 1 required keyword-only argument: 'recursive_guard'
  61. return cast(Any, type_)._evaluate(globalns, localns, recursive_guard=set())
  62. else:
  63. def evaluate_forwardref(type_: ForwardRef, globalns: Any, localns: Any) -> Any:
  64. # Pydantic 1.x will not support PEP 695 syntax, but provide `type_params` to avoid
  65. # warnings:
  66. return cast(Any, type_)._evaluate(globalns, localns, type_params=(), recursive_guard=set())
  67. if sys.version_info < (3, 9):
  68. # Ensure we always get all the whole `Annotated` hint, not just the annotated type.
  69. # For 3.7 to 3.8, `get_type_hints` doesn't recognize `typing_extensions.Annotated`,
  70. # so it already returns the full annotation
  71. get_all_type_hints = get_type_hints
  72. else:
  73. def get_all_type_hints(obj: Any, globalns: Any = None, localns: Any = None) -> Any:
  74. return get_type_hints(obj, globalns, localns, include_extras=True)
  75. _T = TypeVar('_T')
  76. AnyCallable = TypingCallable[..., Any]
  77. NoArgAnyCallable = TypingCallable[[], Any]
  78. # workaround for https://github.com/python/mypy/issues/9496
  79. AnyArgTCallable = TypingCallable[..., _T]
  80. # Annotated[...] is implemented by returning an instance of one of these classes, depending on
  81. # python/typing_extensions version.
  82. AnnotatedTypeNames = {'AnnotatedMeta', '_AnnotatedAlias'}
  83. LITERAL_TYPES: Set[Any] = {Literal}
  84. if hasattr(typing, 'Literal'):
  85. LITERAL_TYPES.add(typing.Literal)
  86. if sys.version_info < (3, 8):
  87. def get_origin(t: Type[Any]) -> Optional[Type[Any]]:
  88. if type(t).__name__ in AnnotatedTypeNames:
  89. # weirdly this is a runtime requirement, as well as for mypy
  90. return cast(Type[Any], Annotated)
  91. return getattr(t, '__origin__', None)
  92. else:
  93. from typing import get_origin as _typing_get_origin
  94. def get_origin(tp: Type[Any]) -> Optional[Type[Any]]:
  95. """
  96. We can't directly use `typing.get_origin` since we need a fallback to support
  97. custom generic classes like `ConstrainedList`
  98. It should be useless once https://github.com/cython/cython/issues/3537 is
  99. solved and https://github.com/pydantic/pydantic/pull/1753 is merged.
  100. """
  101. if type(tp).__name__ in AnnotatedTypeNames:
  102. return cast(Type[Any], Annotated) # mypy complains about _SpecialForm
  103. return _typing_get_origin(tp) or getattr(tp, '__origin__', None)
  104. if sys.version_info < (3, 8):
  105. from typing import _GenericAlias
  106. def get_args(t: Type[Any]) -> Tuple[Any, ...]:
  107. """Compatibility version of get_args for python 3.7.
  108. Mostly compatible with the python 3.8 `typing` module version
  109. and able to handle almost all use cases.
  110. """
  111. if type(t).__name__ in AnnotatedTypeNames:
  112. return t.__args__ + t.__metadata__
  113. if isinstance(t, _GenericAlias):
  114. res = t.__args__
  115. if t.__origin__ is Callable and res and res[0] is not Ellipsis:
  116. res = (list(res[:-1]), res[-1])
  117. return res
  118. return getattr(t, '__args__', ())
  119. else:
  120. from typing import get_args as _typing_get_args
  121. def _generic_get_args(tp: Type[Any]) -> Tuple[Any, ...]:
  122. """
  123. In python 3.9, `typing.Dict`, `typing.List`, ...
  124. do have an empty `__args__` by default (instead of the generic ~T for example).
  125. In order to still support `Dict` for example and consider it as `Dict[Any, Any]`,
  126. we retrieve the `_nparams` value that tells us how many parameters it needs.
  127. """
  128. if hasattr(tp, '_nparams'):
  129. return (Any,) * tp._nparams
  130. # Special case for `tuple[()]`, which used to return ((),) with `typing.Tuple`
  131. # in python 3.10- but now returns () for `tuple` and `Tuple`.
  132. # This will probably be clarified in pydantic v2
  133. try:
  134. if tp == Tuple[()] or sys.version_info >= (3, 9) and tp == tuple[()]: # type: ignore[misc]
  135. return ((),)
  136. # there is a TypeError when compiled with cython
  137. except TypeError: # pragma: no cover
  138. pass
  139. return ()
  140. def get_args(tp: Type[Any]) -> Tuple[Any, ...]:
  141. """Get type arguments with all substitutions performed.
  142. For unions, basic simplifications used by Union constructor are performed.
  143. Examples::
  144. get_args(Dict[str, int]) == (str, int)
  145. get_args(int) == ()
  146. get_args(Union[int, Union[T, int], str][int]) == (int, str)
  147. get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int])
  148. get_args(Callable[[], T][int]) == ([], int)
  149. """
  150. if type(tp).__name__ in AnnotatedTypeNames:
  151. return tp.__args__ + tp.__metadata__
  152. # the fallback is needed for the same reasons as `get_origin` (see above)
  153. return _typing_get_args(tp) or getattr(tp, '__args__', ()) or _generic_get_args(tp)
  154. if sys.version_info < (3, 9):
  155. def convert_generics(tp: Type[Any]) -> Type[Any]:
  156. """Python 3.9 and older only supports generics from `typing` module.
  157. They convert strings to ForwardRef automatically.
  158. Examples::
  159. typing.List['Hero'] == typing.List[ForwardRef('Hero')]
  160. """
  161. return tp
  162. else:
  163. def convert_generics(tp: Type[Any]) -> Type[Any]:
  164. """
  165. Recursively searches for `str` type hints and replaces them with ForwardRef.
  166. Examples::
  167. convert_generics(list['Hero']) == list[ForwardRef('Hero')]
  168. convert_generics(dict['Hero', 'Team']) == dict[ForwardRef('Hero'), ForwardRef('Team')]
  169. convert_generics(typing.Dict['Hero', 'Team']) == typing.Dict[ForwardRef('Hero'), ForwardRef('Team')]
  170. convert_generics(list[str | 'Hero'] | int) == list[str | ForwardRef('Hero')] | int
  171. """
  172. origin = get_origin(tp)
  173. if not origin or not hasattr(tp, '__args__'):
  174. return tp
  175. args = get_args(tp)
  176. # typing.Annotated needs special treatment
  177. if origin is Annotated:
  178. return Annotated[(convert_generics(args[0]), *args[1:])] # type: ignore
  179. # recursively replace `str` instances inside of `GenericAlias` with `ForwardRef(arg)`
  180. converted = tuple(
  181. ForwardRef(arg) if isinstance(arg, str) and isinstance(tp, TypingGenericAlias) else convert_generics(arg)
  182. for arg in args
  183. )
  184. if converted == args:
  185. return tp
  186. elif isinstance(tp, TypingGenericAlias):
  187. return TypingGenericAlias(origin, converted)
  188. elif isinstance(tp, TypesUnionType):
  189. # recreate types.UnionType (PEP604, Python >= 3.10)
  190. return functools.reduce(operator.or_, converted) # type: ignore
  191. else:
  192. try:
  193. setattr(tp, '__args__', converted)
  194. except AttributeError:
  195. pass
  196. return tp
  197. if sys.version_info < (3, 10):
  198. def is_union(tp: Optional[Type[Any]]) -> bool:
  199. return tp is Union
  200. WithArgsTypes = (TypingGenericAlias,)
  201. else:
  202. import types
  203. import typing
  204. def is_union(tp: Optional[Type[Any]]) -> bool:
  205. return tp is Union or tp is types.UnionType # noqa: E721
  206. WithArgsTypes = (typing._GenericAlias, types.GenericAlias, types.UnionType)
  207. StrPath = Union[str, PathLike]
  208. if TYPE_CHECKING:
  209. from pydantic.v1.fields import ModelField
  210. TupleGenerator = Generator[Tuple[str, Any], None, None]
  211. DictStrAny = Dict[str, Any]
  212. DictAny = Dict[Any, Any]
  213. SetStr = Set[str]
  214. ListStr = List[str]
  215. IntStr = Union[int, str]
  216. AbstractSetIntStr = AbstractSet[IntStr]
  217. DictIntStrAny = Dict[IntStr, Any]
  218. MappingIntStrAny = Mapping[IntStr, Any]
  219. CallableGenerator = Generator[AnyCallable, None, None]
  220. ReprArgs = Sequence[Tuple[Optional[str], Any]]
  221. MYPY = False
  222. if MYPY:
  223. AnyClassMethod = classmethod[Any]
  224. else:
  225. # classmethod[TargetType, CallableParamSpecType, CallableReturnType]
  226. AnyClassMethod = classmethod[Any, Any, Any]
  227. __all__ = (
  228. 'AnyCallable',
  229. 'NoArgAnyCallable',
  230. 'NoneType',
  231. 'is_none_type',
  232. 'display_as_type',
  233. 'resolve_annotations',
  234. 'is_callable_type',
  235. 'is_literal_type',
  236. 'all_literal_values',
  237. 'is_namedtuple',
  238. 'is_typeddict',
  239. 'is_typeddict_special',
  240. 'is_new_type',
  241. 'new_type_supertype',
  242. 'is_classvar',
  243. 'is_finalvar',
  244. 'update_field_forward_refs',
  245. 'update_model_forward_refs',
  246. 'TupleGenerator',
  247. 'DictStrAny',
  248. 'DictAny',
  249. 'SetStr',
  250. 'ListStr',
  251. 'IntStr',
  252. 'AbstractSetIntStr',
  253. 'DictIntStrAny',
  254. 'CallableGenerator',
  255. 'ReprArgs',
  256. 'AnyClassMethod',
  257. 'CallableGenerator',
  258. 'WithArgsTypes',
  259. 'get_args',
  260. 'get_origin',
  261. 'get_sub_types',
  262. 'typing_base',
  263. 'get_all_type_hints',
  264. 'is_union',
  265. 'StrPath',
  266. 'MappingIntStrAny',
  267. )
  268. NoneType = None.__class__
  269. NONE_TYPES: Tuple[Any, Any, Any] = (None, NoneType, Literal[None])
  270. if sys.version_info < (3, 8):
  271. # Even though this implementation is slower, we need it for python 3.7:
  272. # In python 3.7 "Literal" is not a builtin type and uses a different
  273. # mechanism.
  274. # for this reason `Literal[None] is Literal[None]` evaluates to `False`,
  275. # breaking the faster implementation used for the other python versions.
  276. def is_none_type(type_: Any) -> bool:
  277. return type_ in NONE_TYPES
  278. elif sys.version_info[:2] == (3, 8):
  279. def is_none_type(type_: Any) -> bool:
  280. for none_type in NONE_TYPES:
  281. if type_ is none_type:
  282. return True
  283. # With python 3.8, specifically 3.8.10, Literal "is" check sare very flakey
  284. # can change on very subtle changes like use of types in other modules,
  285. # hopefully this check avoids that issue.
  286. if is_literal_type(type_): # pragma: no cover
  287. return all_literal_values(type_) == (None,)
  288. return False
  289. else:
  290. def is_none_type(type_: Any) -> bool:
  291. return type_ in NONE_TYPES
  292. def display_as_type(v: Type[Any]) -> str:
  293. if not isinstance(v, typing_base) and not isinstance(v, WithArgsTypes) and not isinstance(v, type):
  294. v = v.__class__
  295. if is_union(get_origin(v)):
  296. return f'Union[{", ".join(map(display_as_type, get_args(v)))}]'
  297. if isinstance(v, WithArgsTypes):
  298. # Generic alias are constructs like `list[int]`
  299. return str(v).replace('typing.', '')
  300. try:
  301. return v.__name__
  302. except AttributeError:
  303. # happens with typing objects
  304. return str(v).replace('typing.', '')
  305. def resolve_annotations(raw_annotations: Dict[str, Type[Any]], module_name: Optional[str]) -> Dict[str, Type[Any]]:
  306. """
  307. Partially taken from typing.get_type_hints.
  308. Resolve string or ForwardRef annotations into type objects if possible.
  309. """
  310. base_globals: Optional[Dict[str, Any]] = None
  311. if module_name:
  312. try:
  313. module = sys.modules[module_name]
  314. except KeyError:
  315. # happens occasionally, see https://github.com/pydantic/pydantic/issues/2363
  316. pass
  317. else:
  318. base_globals = module.__dict__
  319. annotations = {}
  320. for name, value in raw_annotations.items():
  321. if isinstance(value, str):
  322. if (3, 10) > sys.version_info >= (3, 9, 8) or sys.version_info >= (3, 10, 1):
  323. value = ForwardRef(value, is_argument=False, is_class=True)
  324. else:
  325. value = ForwardRef(value, is_argument=False)
  326. try:
  327. if sys.version_info >= (3, 13):
  328. value = _eval_type(value, base_globals, None, type_params=())
  329. else:
  330. value = _eval_type(value, base_globals, None)
  331. except NameError:
  332. # this is ok, it can be fixed with update_forward_refs
  333. pass
  334. annotations[name] = value
  335. return annotations
  336. def is_callable_type(type_: Type[Any]) -> bool:
  337. return type_ is Callable or get_origin(type_) is Callable
  338. def is_literal_type(type_: Type[Any]) -> bool:
  339. return Literal is not None and get_origin(type_) in LITERAL_TYPES
  340. def literal_values(type_: Type[Any]) -> Tuple[Any, ...]:
  341. return get_args(type_)
  342. def all_literal_values(type_: Type[Any]) -> Tuple[Any, ...]:
  343. """
  344. This method is used to retrieve all Literal values as
  345. Literal can be used recursively (see https://www.python.org/dev/peps/pep-0586)
  346. e.g. `Literal[Literal[Literal[1, 2, 3], "foo"], 5, None]`
  347. """
  348. if not is_literal_type(type_):
  349. return (type_,)
  350. values = literal_values(type_)
  351. return tuple(x for value in values for x in all_literal_values(value))
  352. def is_namedtuple(type_: Type[Any]) -> bool:
  353. """
  354. Check if a given class is a named tuple.
  355. It can be either a `typing.NamedTuple` or `collections.namedtuple`
  356. """
  357. from pydantic.v1.utils import lenient_issubclass
  358. return lenient_issubclass(type_, tuple) and hasattr(type_, '_fields')
  359. def is_typeddict(type_: Type[Any]) -> bool:
  360. """
  361. Check if a given class is a typed dict (from `typing` or `typing_extensions`)
  362. In 3.10, there will be a public method (https://docs.python.org/3.10/library/typing.html#typing.is_typeddict)
  363. """
  364. from pydantic.v1.utils import lenient_issubclass
  365. return lenient_issubclass(type_, dict) and hasattr(type_, '__total__')
  366. def _check_typeddict_special(type_: Any) -> bool:
  367. return type_ is TypedDictRequired or type_ is TypedDictNotRequired
  368. def is_typeddict_special(type_: Any) -> bool:
  369. """
  370. Check if type is a TypedDict special form (Required or NotRequired).
  371. """
  372. return _check_typeddict_special(type_) or _check_typeddict_special(get_origin(type_))
  373. test_type = NewType('test_type', str)
  374. def is_new_type(type_: Type[Any]) -> bool:
  375. """
  376. Check whether type_ was created using typing.NewType
  377. """
  378. return isinstance(type_, test_type.__class__) and hasattr(type_, '__supertype__') # type: ignore
  379. def new_type_supertype(type_: Type[Any]) -> Type[Any]:
  380. while hasattr(type_, '__supertype__'):
  381. type_ = type_.__supertype__
  382. return type_
  383. def _check_classvar(v: Optional[Type[Any]]) -> bool:
  384. if v is None:
  385. return False
  386. return v.__class__ == ClassVar.__class__ and getattr(v, '_name', None) == 'ClassVar'
  387. def _check_finalvar(v: Optional[Type[Any]]) -> bool:
  388. """
  389. Check if a given type is a `typing.Final` type.
  390. """
  391. if v is None:
  392. return False
  393. return v.__class__ == Final.__class__ and (sys.version_info < (3, 8) or getattr(v, '_name', None) == 'Final')
  394. def is_classvar(ann_type: Type[Any]) -> bool:
  395. if _check_classvar(ann_type) or _check_classvar(get_origin(ann_type)):
  396. return True
  397. # this is an ugly workaround for class vars that contain forward references and are therefore themselves
  398. # forward references, see #3679
  399. if ann_type.__class__ == ForwardRef and ann_type.__forward_arg__.startswith('ClassVar['):
  400. return True
  401. return False
  402. def is_finalvar(ann_type: Type[Any]) -> bool:
  403. return _check_finalvar(ann_type) or _check_finalvar(get_origin(ann_type))
  404. def update_field_forward_refs(field: 'ModelField', globalns: Any, localns: Any) -> None:
  405. """
  406. Try to update ForwardRefs on fields based on this ModelField, globalns and localns.
  407. """
  408. prepare = False
  409. if field.type_.__class__ == ForwardRef:
  410. prepare = True
  411. field.type_ = evaluate_forwardref(field.type_, globalns, localns or None)
  412. if field.outer_type_.__class__ == ForwardRef:
  413. prepare = True
  414. field.outer_type_ = evaluate_forwardref(field.outer_type_, globalns, localns or None)
  415. if prepare:
  416. field.prepare()
  417. if field.sub_fields:
  418. for sub_f in field.sub_fields:
  419. update_field_forward_refs(sub_f, globalns=globalns, localns=localns)
  420. if field.discriminator_key is not None:
  421. field.prepare_discriminated_union_sub_fields()
  422. def update_model_forward_refs(
  423. model: Type[Any],
  424. fields: Iterable['ModelField'],
  425. json_encoders: Dict[Union[Type[Any], str, ForwardRef], AnyCallable],
  426. localns: 'DictStrAny',
  427. exc_to_suppress: Tuple[Type[BaseException], ...] = (),
  428. ) -> None:
  429. """
  430. Try to update model fields ForwardRefs based on model and localns.
  431. """
  432. if model.__module__ in sys.modules:
  433. globalns = sys.modules[model.__module__].__dict__.copy()
  434. else:
  435. globalns = {}
  436. globalns.setdefault(model.__name__, model)
  437. for f in fields:
  438. try:
  439. update_field_forward_refs(f, globalns=globalns, localns=localns)
  440. except exc_to_suppress:
  441. pass
  442. for key in set(json_encoders.keys()):
  443. if isinstance(key, str):
  444. fr: ForwardRef = ForwardRef(key)
  445. elif isinstance(key, ForwardRef):
  446. fr = key
  447. else:
  448. continue
  449. try:
  450. new_key = evaluate_forwardref(fr, globalns, localns or None)
  451. except exc_to_suppress: # pragma: no cover
  452. continue
  453. json_encoders[new_key] = json_encoders.pop(key)
  454. def get_class(type_: Type[Any]) -> Union[None, bool, Type[Any]]:
  455. """
  456. Tries to get the class of a Type[T] annotation. Returns True if Type is used
  457. without brackets. Otherwise returns None.
  458. """
  459. if type_ is type:
  460. return True
  461. if get_origin(type_) is None:
  462. return None
  463. args = get_args(type_)
  464. if not args or not isinstance(args[0], type):
  465. return True
  466. else:
  467. return args[0]
  468. def get_sub_types(tp: Any) -> List[Any]:
  469. """
  470. Return all the types that are allowed by type `tp`
  471. `tp` can be a `Union` of allowed types or an `Annotated` type
  472. """
  473. origin = get_origin(tp)
  474. if origin is Annotated:
  475. return get_sub_types(get_args(tp)[0])
  476. elif is_union(origin):
  477. return [x for t in get_args(tp) for x in get_sub_types(t)]
  478. else:
  479. return [tp]