default.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406
  1. """
  2. Custom transports, with nicely configured defaults.
  3. The following additional keyword arguments are currently supported by httpcore...
  4. * uds: str
  5. * local_address: str
  6. * retries: int
  7. Example usages...
  8. # Disable HTTP/2 on a single specific domain.
  9. mounts = {
  10. "all://": httpx.HTTPTransport(http2=True),
  11. "all://*example.org": httpx.HTTPTransport()
  12. }
  13. # Using advanced httpcore configuration, with connection retries.
  14. transport = httpx.HTTPTransport(retries=1)
  15. client = httpx.Client(transport=transport)
  16. # Using advanced httpcore configuration, with unix domain sockets.
  17. transport = httpx.HTTPTransport(uds="socket.uds")
  18. client = httpx.Client(transport=transport)
  19. """
  20. from __future__ import annotations
  21. import contextlib
  22. import typing
  23. from types import TracebackType
  24. if typing.TYPE_CHECKING:
  25. import ssl # pragma: no cover
  26. import httpx # pragma: no cover
  27. from .._config import DEFAULT_LIMITS, Limits, Proxy, create_ssl_context
  28. from .._exceptions import (
  29. ConnectError,
  30. ConnectTimeout,
  31. LocalProtocolError,
  32. NetworkError,
  33. PoolTimeout,
  34. ProtocolError,
  35. ProxyError,
  36. ReadError,
  37. ReadTimeout,
  38. RemoteProtocolError,
  39. TimeoutException,
  40. UnsupportedProtocol,
  41. WriteError,
  42. WriteTimeout,
  43. )
  44. from .._models import Request, Response
  45. from .._types import AsyncByteStream, CertTypes, ProxyTypes, SyncByteStream
  46. from .._urls import URL
  47. from .base import AsyncBaseTransport, BaseTransport
  48. T = typing.TypeVar("T", bound="HTTPTransport")
  49. A = typing.TypeVar("A", bound="AsyncHTTPTransport")
  50. SOCKET_OPTION = typing.Union[
  51. typing.Tuple[int, int, int],
  52. typing.Tuple[int, int, typing.Union[bytes, bytearray]],
  53. typing.Tuple[int, int, None, int],
  54. ]
  55. __all__ = ["AsyncHTTPTransport", "HTTPTransport"]
  56. HTTPCORE_EXC_MAP: dict[type[Exception], type[httpx.HTTPError]] = {}
  57. def _load_httpcore_exceptions() -> dict[type[Exception], type[httpx.HTTPError]]:
  58. import httpcore
  59. return {
  60. httpcore.TimeoutException: TimeoutException,
  61. httpcore.ConnectTimeout: ConnectTimeout,
  62. httpcore.ReadTimeout: ReadTimeout,
  63. httpcore.WriteTimeout: WriteTimeout,
  64. httpcore.PoolTimeout: PoolTimeout,
  65. httpcore.NetworkError: NetworkError,
  66. httpcore.ConnectError: ConnectError,
  67. httpcore.ReadError: ReadError,
  68. httpcore.WriteError: WriteError,
  69. httpcore.ProxyError: ProxyError,
  70. httpcore.UnsupportedProtocol: UnsupportedProtocol,
  71. httpcore.ProtocolError: ProtocolError,
  72. httpcore.LocalProtocolError: LocalProtocolError,
  73. httpcore.RemoteProtocolError: RemoteProtocolError,
  74. }
  75. @contextlib.contextmanager
  76. def map_httpcore_exceptions() -> typing.Iterator[None]:
  77. global HTTPCORE_EXC_MAP
  78. if len(HTTPCORE_EXC_MAP) == 0:
  79. HTTPCORE_EXC_MAP = _load_httpcore_exceptions()
  80. try:
  81. yield
  82. except Exception as exc:
  83. mapped_exc = None
  84. for from_exc, to_exc in HTTPCORE_EXC_MAP.items():
  85. if not isinstance(exc, from_exc):
  86. continue
  87. # We want to map to the most specific exception we can find.
  88. # Eg if `exc` is an `httpcore.ReadTimeout`, we want to map to
  89. # `httpx.ReadTimeout`, not just `httpx.TimeoutException`.
  90. if mapped_exc is None or issubclass(to_exc, mapped_exc):
  91. mapped_exc = to_exc
  92. if mapped_exc is None: # pragma: no cover
  93. raise
  94. message = str(exc)
  95. raise mapped_exc(message) from exc
  96. class ResponseStream(SyncByteStream):
  97. def __init__(self, httpcore_stream: typing.Iterable[bytes]) -> None:
  98. self._httpcore_stream = httpcore_stream
  99. def __iter__(self) -> typing.Iterator[bytes]:
  100. with map_httpcore_exceptions():
  101. for part in self._httpcore_stream:
  102. yield part
  103. def close(self) -> None:
  104. if hasattr(self._httpcore_stream, "close"):
  105. self._httpcore_stream.close()
  106. class HTTPTransport(BaseTransport):
  107. def __init__(
  108. self,
  109. verify: ssl.SSLContext | str | bool = True,
  110. cert: CertTypes | None = None,
  111. trust_env: bool = True,
  112. http1: bool = True,
  113. http2: bool = False,
  114. limits: Limits = DEFAULT_LIMITS,
  115. proxy: ProxyTypes | None = None,
  116. uds: str | None = None,
  117. local_address: str | None = None,
  118. retries: int = 0,
  119. socket_options: typing.Iterable[SOCKET_OPTION] | None = None,
  120. ) -> None:
  121. import httpcore
  122. proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy
  123. ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env)
  124. if proxy is None:
  125. self._pool = httpcore.ConnectionPool(
  126. ssl_context=ssl_context,
  127. max_connections=limits.max_connections,
  128. max_keepalive_connections=limits.max_keepalive_connections,
  129. keepalive_expiry=limits.keepalive_expiry,
  130. http1=http1,
  131. http2=http2,
  132. uds=uds,
  133. local_address=local_address,
  134. retries=retries,
  135. socket_options=socket_options,
  136. )
  137. elif proxy.url.scheme in ("http", "https"):
  138. self._pool = httpcore.HTTPProxy(
  139. proxy_url=httpcore.URL(
  140. scheme=proxy.url.raw_scheme,
  141. host=proxy.url.raw_host,
  142. port=proxy.url.port,
  143. target=proxy.url.raw_path,
  144. ),
  145. proxy_auth=proxy.raw_auth,
  146. proxy_headers=proxy.headers.raw,
  147. ssl_context=ssl_context,
  148. proxy_ssl_context=proxy.ssl_context,
  149. max_connections=limits.max_connections,
  150. max_keepalive_connections=limits.max_keepalive_connections,
  151. keepalive_expiry=limits.keepalive_expiry,
  152. http1=http1,
  153. http2=http2,
  154. socket_options=socket_options,
  155. )
  156. elif proxy.url.scheme in ("socks5", "socks5h"):
  157. try:
  158. import socksio # noqa
  159. except ImportError: # pragma: no cover
  160. raise ImportError(
  161. "Using SOCKS proxy, but the 'socksio' package is not installed. "
  162. "Make sure to install httpx using `pip install httpx[socks]`."
  163. ) from None
  164. self._pool = httpcore.SOCKSProxy(
  165. proxy_url=httpcore.URL(
  166. scheme=proxy.url.raw_scheme,
  167. host=proxy.url.raw_host,
  168. port=proxy.url.port,
  169. target=proxy.url.raw_path,
  170. ),
  171. proxy_auth=proxy.raw_auth,
  172. ssl_context=ssl_context,
  173. max_connections=limits.max_connections,
  174. max_keepalive_connections=limits.max_keepalive_connections,
  175. keepalive_expiry=limits.keepalive_expiry,
  176. http1=http1,
  177. http2=http2,
  178. )
  179. else: # pragma: no cover
  180. raise ValueError(
  181. "Proxy protocol must be either 'http', 'https', 'socks5', or 'socks5h',"
  182. f" but got {proxy.url.scheme!r}."
  183. )
  184. def __enter__(self: T) -> T: # Use generics for subclass support.
  185. self._pool.__enter__()
  186. return self
  187. def __exit__(
  188. self,
  189. exc_type: type[BaseException] | None = None,
  190. exc_value: BaseException | None = None,
  191. traceback: TracebackType | None = None,
  192. ) -> None:
  193. with map_httpcore_exceptions():
  194. self._pool.__exit__(exc_type, exc_value, traceback)
  195. def handle_request(
  196. self,
  197. request: Request,
  198. ) -> Response:
  199. assert isinstance(request.stream, SyncByteStream)
  200. import httpcore
  201. req = httpcore.Request(
  202. method=request.method,
  203. url=httpcore.URL(
  204. scheme=request.url.raw_scheme,
  205. host=request.url.raw_host,
  206. port=request.url.port,
  207. target=request.url.raw_path,
  208. ),
  209. headers=request.headers.raw,
  210. content=request.stream,
  211. extensions=request.extensions,
  212. )
  213. with map_httpcore_exceptions():
  214. resp = self._pool.handle_request(req)
  215. assert isinstance(resp.stream, typing.Iterable)
  216. return Response(
  217. status_code=resp.status,
  218. headers=resp.headers,
  219. stream=ResponseStream(resp.stream),
  220. extensions=resp.extensions,
  221. )
  222. def close(self) -> None:
  223. self._pool.close()
  224. class AsyncResponseStream(AsyncByteStream):
  225. def __init__(self, httpcore_stream: typing.AsyncIterable[bytes]) -> None:
  226. self._httpcore_stream = httpcore_stream
  227. async def __aiter__(self) -> typing.AsyncIterator[bytes]:
  228. with map_httpcore_exceptions():
  229. async for part in self._httpcore_stream:
  230. yield part
  231. async def aclose(self) -> None:
  232. if hasattr(self._httpcore_stream, "aclose"):
  233. await self._httpcore_stream.aclose()
  234. class AsyncHTTPTransport(AsyncBaseTransport):
  235. def __init__(
  236. self,
  237. verify: ssl.SSLContext | str | bool = True,
  238. cert: CertTypes | None = None,
  239. trust_env: bool = True,
  240. http1: bool = True,
  241. http2: bool = False,
  242. limits: Limits = DEFAULT_LIMITS,
  243. proxy: ProxyTypes | None = None,
  244. uds: str | None = None,
  245. local_address: str | None = None,
  246. retries: int = 0,
  247. socket_options: typing.Iterable[SOCKET_OPTION] | None = None,
  248. ) -> None:
  249. import httpcore
  250. proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy
  251. ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env)
  252. if proxy is None:
  253. self._pool = httpcore.AsyncConnectionPool(
  254. ssl_context=ssl_context,
  255. max_connections=limits.max_connections,
  256. max_keepalive_connections=limits.max_keepalive_connections,
  257. keepalive_expiry=limits.keepalive_expiry,
  258. http1=http1,
  259. http2=http2,
  260. uds=uds,
  261. local_address=local_address,
  262. retries=retries,
  263. socket_options=socket_options,
  264. )
  265. elif proxy.url.scheme in ("http", "https"):
  266. self._pool = httpcore.AsyncHTTPProxy(
  267. proxy_url=httpcore.URL(
  268. scheme=proxy.url.raw_scheme,
  269. host=proxy.url.raw_host,
  270. port=proxy.url.port,
  271. target=proxy.url.raw_path,
  272. ),
  273. proxy_auth=proxy.raw_auth,
  274. proxy_headers=proxy.headers.raw,
  275. proxy_ssl_context=proxy.ssl_context,
  276. ssl_context=ssl_context,
  277. max_connections=limits.max_connections,
  278. max_keepalive_connections=limits.max_keepalive_connections,
  279. keepalive_expiry=limits.keepalive_expiry,
  280. http1=http1,
  281. http2=http2,
  282. socket_options=socket_options,
  283. )
  284. elif proxy.url.scheme in ("socks5", "socks5h"):
  285. try:
  286. import socksio # noqa
  287. except ImportError: # pragma: no cover
  288. raise ImportError(
  289. "Using SOCKS proxy, but the 'socksio' package is not installed. "
  290. "Make sure to install httpx using `pip install httpx[socks]`."
  291. ) from None
  292. self._pool = httpcore.AsyncSOCKSProxy(
  293. proxy_url=httpcore.URL(
  294. scheme=proxy.url.raw_scheme,
  295. host=proxy.url.raw_host,
  296. port=proxy.url.port,
  297. target=proxy.url.raw_path,
  298. ),
  299. proxy_auth=proxy.raw_auth,
  300. ssl_context=ssl_context,
  301. max_connections=limits.max_connections,
  302. max_keepalive_connections=limits.max_keepalive_connections,
  303. keepalive_expiry=limits.keepalive_expiry,
  304. http1=http1,
  305. http2=http2,
  306. )
  307. else: # pragma: no cover
  308. raise ValueError(
  309. "Proxy protocol must be either 'http', 'https', 'socks5', or 'socks5h',"
  310. " but got {proxy.url.scheme!r}."
  311. )
  312. async def __aenter__(self: A) -> A: # Use generics for subclass support.
  313. await self._pool.__aenter__()
  314. return self
  315. async def __aexit__(
  316. self,
  317. exc_type: type[BaseException] | None = None,
  318. exc_value: BaseException | None = None,
  319. traceback: TracebackType | None = None,
  320. ) -> None:
  321. with map_httpcore_exceptions():
  322. await self._pool.__aexit__(exc_type, exc_value, traceback)
  323. async def handle_async_request(
  324. self,
  325. request: Request,
  326. ) -> Response:
  327. assert isinstance(request.stream, AsyncByteStream)
  328. import httpcore
  329. req = httpcore.Request(
  330. method=request.method,
  331. url=httpcore.URL(
  332. scheme=request.url.raw_scheme,
  333. host=request.url.raw_host,
  334. port=request.url.port,
  335. target=request.url.raw_path,
  336. ),
  337. headers=request.headers.raw,
  338. content=request.stream,
  339. extensions=request.extensions,
  340. )
  341. with map_httpcore_exceptions():
  342. resp = await self._pool.handle_async_request(req)
  343. assert isinstance(resp.stream, typing.AsyncIterable)
  344. return Response(
  345. status_code=resp.status,
  346. headers=resp.headers,
  347. stream=AsyncResponseStream(resp.stream),
  348. extensions=resp.extensions,
  349. )
  350. async def aclose(self) -> None:
  351. await self._pool.aclose()