Update to httpcore 0.10 (#1126)

* Keep HTTPError as a base class for .request() and .raise_for_status()

* Updates for httpcore 0.10

* Update httpx/_exceptions.py

Co-authored-by: Stephen Brown II <Stephen.Brown2@gmail.com>

* Use httpcore.SimpleByteStream/httpcore.IteratorByteStream

* Use httpcore.PlainByteStream

* Merge master

* Update to httpcore 0.10.x

Co-authored-by: Stephen Brown II <Stephen.Brown2@gmail.com>

Author: lovelydinosaur

httpx/__init__.py

@@ -12,13 +12,15 @@
     HTTPError,
     HTTPStatusError,
     InvalidURL,
+    LocalProtocolError,
     NetworkError,
     NotRedirectResponse,
     PoolTimeout,
     ProtocolError,
     ProxyError,
     ReadError,
     ReadTimeout,
+    RemoteProtocolError,
     RequestBodyUnavailable,
     RequestError,
     RequestNotRead,
@@ -29,6 +31,7 @@
     TimeoutException,
     TooManyRedirects,
     TransportError,
+    UnsupportedProtocol,
     WriteError,
     WriteTimeout,
 )
@@ -72,6 +75,9 @@
     "HTTPError",
     "HTTPStatusError",
     "InvalidURL",
+    "UnsupportedProtocol",
+    "LocalProtocolError",
+    "RemoteProtocolError",
     "NetworkError",
     "NotRedirectResponse",
     "PoolTimeout",

httpx/_client.py

@@ -19,7 +19,6 @@
 from ._content_streams import ContentStream
 from ._exceptions import (
     HTTPCORE_EXC_MAP,
-    InvalidURL,
     RequestBodyUnavailable,
     TooManyRedirects,
     map_exceptions,
@@ -44,7 +43,6 @@
 from ._utils import (
     NetRCInfo,
     URLPattern,
-    enforce_http_url,
     get_environment_proxies,
     get_logger,
     same_origin,
@@ -344,11 +342,6 @@ def _redirect_url(self, request: Request, response: Response) -> URL:
 
         url = URL(location)
 
-        # Check that we can handle the scheme
-        if url.scheme and url.scheme not in ("http", "https"):
-            message = f'Scheme "{url.scheme}" not supported.'
-            raise InvalidURL(message, request=request)
-
         # Handle malformed 'Location' headers that are "absolute" form, have no host.
         # See: https://github.com/encode/httpx/issues/771
         if url.scheme and not url.host:
@@ -540,8 +533,8 @@ def _init_transport(
 
         return httpcore.SyncConnectionPool(
             ssl_context=ssl_context,
-            max_keepalive=limits.max_keepalive,
             max_connections=limits.max_connections,
+            max_keepalive_connections=limits.max_keepalive_connections,
             keepalive_expiry=KEEPALIVE_EXPIRY,
             http2=http2,
         )
@@ -562,20 +555,17 @@ def _init_proxy_transport(
             proxy_headers=proxy.headers.raw,
             proxy_mode=proxy.mode,
             ssl_context=ssl_context,
-            max_keepalive=limits.max_keepalive,
             max_connections=limits.max_connections,
+            max_keepalive_connections=limits.max_keepalive_connections,
             keepalive_expiry=KEEPALIVE_EXPIRY,
             http2=http2,
         )
 
-    def _transport_for_url(self, request: Request) -> httpcore.SyncHTTPTransport:
+    def _transport_for_url(self, url: URL) -> httpcore.SyncHTTPTransport:
         """
         Returns the transport instance that should be used for a given URL.
         This will either be the standard connection pool, or a proxy.
         """
-        url = request.url
-        enforce_http_url(request)
-
         for pattern, transport in self._proxies.items():
             if pattern.matches(url):
                 return self._transport if transport is None else transport
@@ -620,10 +610,6 @@ def send(
         allow_redirects: bool = True,
         timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET,
     ) -> Response:
-        if request.url.scheme not in ("http", "https"):
-            message = 'URL scheme must be "http" or "https".'
-            raise InvalidURL(message, request=request)
-
         timeout = self.timeout if isinstance(timeout, UnsetType) else Timeout(timeout)
 
         auth = self._build_auth(request, auth)
@@ -714,7 +700,7 @@ def _send_single_request(self, request: Request, timeout: Timeout) -> Response:
         """
         Sends a single request, without handling any redirections.
         """
-        transport = self._transport_for_url(request)
+        transport = self._transport_for_url(request.url)
 
         with map_exceptions(HTTPCORE_EXC_MAP, request=request):
             (
@@ -1072,8 +1058,8 @@ def _init_transport(
 
         return httpcore.AsyncConnectionPool(
             ssl_context=ssl_context,
-            max_keepalive=limits.max_keepalive,
             max_connections=limits.max_connections,
+            max_keepalive_connections=limits.max_keepalive_connections,
             keepalive_expiry=KEEPALIVE_EXPIRY,
             http2=http2,
         )
@@ -1094,20 +1080,17 @@ def _init_proxy_transport(
             proxy_headers=proxy.headers.raw,
             proxy_mode=proxy.mode,
             ssl_context=ssl_context,
-            max_keepalive=limits.max_keepalive,
             max_connections=limits.max_connections,
+            max_keepalive_connections=limits.max_keepalive_connections,
             keepalive_expiry=KEEPALIVE_EXPIRY,
             http2=http2,
         )
 
-    def _transport_for_url(self, request: Request) -> httpcore.AsyncHTTPTransport:
+    def _transport_for_url(self, url: URL) -> httpcore.AsyncHTTPTransport:
         """
         Returns the transport instance that should be used for a given URL.
         This will either be the standard connection pool, or a proxy.
         """
-        url = request.url
-        enforce_http_url(request)
-
         for pattern, transport in self._proxies.items():
             if pattern.matches(url):
                 return self._transport if transport is None else transport
@@ -1245,7 +1228,7 @@ async def _send_single_request(
         """
         Sends a single request, without handling any redirections.
         """
-        transport = self._transport_for_url(request)
+        transport = self._transport_for_url(request.url)
 
         with map_exceptions(HTTPCORE_EXC_MAP, request=request):
             (

httpx/_config.py

@@ -323,42 +323,53 @@ class Limits:
 
     **Parameters:**
 
-    * **max_keepalive** - Allow the connection pool to maintain keep-alive connections
-                       below this point.
     * **max_connections** - The maximum number of concurrent connections that may be
-                       established.
+            established.
+    * **max_keepalive_connections** - Allow the connection pool to maintain
+            keep-alive connections below this point. Should be less than or equal
+            to `max_connections`.
     """
 
     def __init__(
-        self, *, max_keepalive: int = None, max_connections: int = None,
+        self,
+        *,
+        max_connections: int = None,
+        max_keepalive_connections: int = None,
+        # Deprecated parameter naming, in favour of more explicit version:
+        max_keepalive: int = None,
     ):
-        self.max_keepalive = max_keepalive
+        if max_keepalive is not None:
+            warnings.warn(
+                "'max_keepalive' is deprecated. Use 'max_keepalive_connections'.",
+                DeprecationWarning,
+            )
+            max_keepalive_connections = max_keepalive
+
         self.max_connections = max_connections
+        self.max_keepalive_connections = max_keepalive_connections
 
     def __eq__(self, other: typing.Any) -> bool:
         return (
             isinstance(other, self.__class__)
-            and self.max_keepalive == other.max_keepalive
             and self.max_connections == other.max_connections
+            and self.max_keepalive_connections == other.max_keepalive_connections
         )
 
     def __repr__(self) -> str:
         class_name = self.__class__.__name__
         return (
-            f"{class_name}(max_keepalive={self.max_keepalive}, "
-            f"max_connections={self.max_connections})"
+            f"{class_name}(max_connections={self.max_connections}, "
+            f"max_keepalive_connections={self.max_keepalive_connections})"
         )
 
 
 class PoolLimits(Limits):
-    def __init__(
-        self, *, max_keepalive: int = None, max_connections: int = None,
-    ) -> None:
+    def __init__(self, **kwargs: typing.Any) -> None:
         warn_deprecated(
             "httpx.PoolLimits(...) is deprecated and will raise errors in the future. "
             "Use httpx.Limits(...) instead."
         )
-        super().__init__(max_keepalive=max_keepalive, max_connections=max_connections)
+        super().__init__(**kwargs)
 
 
 class Proxy:

httpx/_exceptions.py

@@ -15,11 +15,13 @@
         · WriteError
         · CloseError
       - ProtocolError
+        · LocalProtocolError
+        · RemoteProtocolError
       - ProxyError
+      - UnsupportedProtocol
     + DecodingError
     + TooManyRedirects
     + RequestBodyUnavailable
-    + InvalidURL
   x HTTPStatusError
 * NotRedirectResponse
 * CookieConflict
@@ -153,9 +155,35 @@ class ProxyError(TransportError):
     """
 
 
+class UnsupportedProtocol(TransportError):
+    """
+    Attempted to make a request to an unsupported protocol.
+
+    For example issuing a request to `ftp://www.example.com`.
+    """
+
+
 class ProtocolError(TransportError):
     """
-    A protocol was violated by the server.
+    The protocol was violated.
+    """
+
+
+class LocalProtocolError(ProtocolError):
+    """
+    A protocol was violated by the client.
+
+    For example if the user instantiated a `Request` instance explicitly,
+    failed to include the mandatory `Host:` header, and then issued it directly
+    using `client.send()`.
+    """
+
+
+class RemoteProtocolError(ProtocolError):
+    """
+    The protocol was violated by the server.
+
+    For exaample, returning malformed HTTP.
     """
 
 
@@ -181,12 +209,6 @@ class RequestBodyUnavailable(RequestError):
     """
 
 
-class InvalidURL(RequestError):
-    """
-    URL was missing a hostname, or was not one of HTTP/HTTPS.
-    """
-
-
 # Client errors
 
 
@@ -297,6 +319,14 @@ def __init__(self) -> None:
         super().__init__(message)
 
 
+# The `InvalidURL` class is no longer required. It was being used to enforce only
+# 'http'/'https' URLs being requested, but is now treated instead at the
+# transport layer using `UnsupportedProtocol()`.`
+
+# We are currently still exposing this class, but it will be removed in 1.0.
+InvalidURL = UnsupportedProtocol
+
+
 @contextlib.contextmanager
 def map_exceptions(
     mapping: typing.Mapping[typing.Type[Exception], typing.Type[Exception]],
@@ -335,5 +365,8 @@ def map_exceptions(
     httpcore.WriteError: WriteError,
     httpcore.CloseError: CloseError,
     httpcore.ProxyError: ProxyError,
+    httpcore.UnsupportedProtocol: UnsupportedProtocol,
     httpcore.ProtocolError: ProtocolError,
+    httpcore.LocalProtocolError: LocalProtocolError,
+    httpcore.RemoteProtocolError: RemoteProtocolError,
 }

httpx/_transports/asgi.py

@@ -1,13 +1,4 @@
-from typing import (
-    TYPE_CHECKING,
-    AsyncIterator,
-    Callable,
-    Dict,
-    List,
-    Optional,
-    Tuple,
-    Union,
-)
+from typing import TYPE_CHECKING, Callable, List, Mapping, Optional, Tuple, Union
 
 import httpcore
 import sniffio
@@ -31,10 +22,6 @@ def create_event() -> "Event":
         return asyncio.Event()
 
 
-async def async_byte_iterator(bytestring: bytes) -> AsyncIterator[bytes]:
-    yield bytestring
-
-
 class ASGITransport(httpcore.AsyncHTTPTransport):
     """
     A custom AsyncTransport that handles sending requests directly to an ASGI app.
@@ -86,14 +73,10 @@ async def request(
         url: Tuple[bytes, bytes, Optional[int], bytes],
         headers: List[Tuple[bytes, bytes]] = None,
         stream: httpcore.AsyncByteStream = None,
-        timeout: Dict[str, Optional[float]] = None,
+        timeout: Mapping[str, Optional[float]] = None,
     ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], httpcore.AsyncByteStream]:
         headers = [] if headers is None else headers
-        stream = (
-            httpcore.AsyncByteStream(async_byte_iterator(b""))
-            if stream is None
-            else stream
-        )
+        stream = httpcore.PlainByteStream(content=b"") if stream is None else stream
 
         # ASGI scope.
         scheme, host, port, full_path = url
@@ -170,8 +153,6 @@ async def send(message: dict) -> None:
         assert status_code is not None
         assert response_headers is not None
 
-        response_body = b"".join(body_parts)
-
-        stream = httpcore.AsyncByteStream(async_byte_iterator(response_body))
+        stream = httpcore.PlainByteStream(content=b"".join(body_parts))
 
         return (b"HTTP/1.1", status_code, b"", response_headers, stream)

httpx/_transports/urllib3.py

@@ -1,5 +1,5 @@
 import socket
-from typing import Dict, Iterator, List, Optional, Tuple
+from typing import Iterator, List, Mapping, Optional, Tuple
 
 import httpcore
 
@@ -45,7 +45,7 @@ def request(
         url: Tuple[bytes, bytes, Optional[int], bytes],
         headers: List[Tuple[bytes, bytes]] = None,
         stream: httpcore.SyncByteStream = None,
-        timeout: Dict[str, Optional[float]] = None,
+        timeout: Mapping[str, Optional[float]] = None,
     ) -> Tuple[bytes, int, bytes, List[Tuple[bytes, bytes]], httpcore.SyncByteStream]:
         headers = [] if headers is None else headers
         stream = ByteStream(b"") if stream is None else stream