Merge branch 'master' into feat/trio-backend

Author: sethmlarson

README.md

@@ -56,7 +56,7 @@ Plus all the standard features of `requests`...
 * Keep-Alive & Connection Pooling
 * Sessions with Cookie Persistence
 * Browser-style SSL Verification
-* Basic/Digest Authentication *(Digest is still TODO)*
+* Basic/Digest Authentication
 * Elegant Key/Value Cookies
 * Automatic Decompression
 * Automatic Content Decoding

docs/async.md

@@ -15,23 +15,30 @@ async client for sending outgoing HTTP requests.
 To make asynchronous requests, you'll need an `AsyncClient`.
 
 ```python
->>> client = httpx.AsyncClient()
->>> r = await client.get('https://www.example.com/')
+>>> async with httpx.AsyncClient() as client:
+>>>     r = await client.get('https://www.example.com/')
+>>> r
+<Response [200 OK]>
 ```
 
+!!! tip
+    Use [IPython](https://ipython.readthedocs.io/en/stable/) to try this code interactively, as it supports executing `async`/`await` expressions in the console.
+
+!!! note
+    The `async with` syntax ensures that all active connections are closed on exit.
+
+    It is safe to access response content (e.g. `r.text`) both inside and outside the `async with` block, unless you are using response streaming. In that case, you should `.read()`, `.stream()`, or `.close()` the response *inside* the `async with` block.
+
 ## API Differences
 
 If you're using streaming responses then there are a few bits of API that
 use async methods:
 
 ```python
->>> client = httpx.AsyncClient()
->>> r = await client.get('https://www.example.com/', stream=True)
->>> try:
+>>> async with httpx.AsyncClient() as client:
+>>>     r = await client.get('https://www.example.com/', stream=True)
 >>>     async for chunk in r.stream():
 >>>         ...
->>> finally:
->>>     await r.close()
 ```
 
 The async response methods are:
@@ -41,15 +48,15 @@ The async response methods are:
 * `.raw()`
 * `.close()`
 
-If you're making parallel requests, then you'll also need to use an async API:
+If you're making [parallel requests](/parallel/), then you'll also need to use an async API:
 
 ```python
->>> client = httpx.AsyncClient()
->>> async with client.parallel() as parallel:
->>>     pending_one = parallel.get('https://example.com/1')
->>>     pending_two = parallel.get('https://example.com/2')
->>>     response_one = await pending_one.get_response()
->>>     response_two = await pending_two.get_response()
+>>> async with httpx.AsyncClient() as client:
+>>>     async with client.parallel() as parallel:
+>>>         pending_one = parallel.get('https://example.com/1')
+>>>         pending_two = parallel.get('https://example.com/2')
+>>>         response_one = await pending_one.get_response()
+>>>         response_two = await pending_two.get_response()
 ```
 
 The async parallel methods are:

docs/quickstart.md

@@ -379,3 +379,25 @@ value to be more or less strict:
 ```python
 >>> httpx.get('https://github.com/', timeout=0.001)
 ```
+
+## Authentication
+
+HTTPX supports Basic and Digest HTTP authentication.
+
+To provide Basic authentication credentials, pass a 2-tuple of
+plaintext `str` or `bytes` objects as the `auth` argument to the request
+functions:
+
+```python
+>>> httpx.get("https://example.com", auth=("my_user", "password123"))
+```
+
+To provide credentials for Digest authentication you'll need to instantiate
+a `DigestAuth` object with the plaintext username and password as arguments.
+This object can be then passed as the `auth` argument to the request methods
+as above:
+
+```python
+>>> auth = httpx.DigestAuth("my_user", "password123")
+>>> httpx.get("https://example.com", auth=auth)
+<Response [200 OK]>

httpx/__init__.py

@@ -5,7 +5,7 @@
 from .concurrency.base import (
     BaseBackgroundManager,
     BasePoolSemaphore,
-    BaseStream,
+    BaseTCPStream,
     ConcurrencyBackend,
 )
 from .config import (
@@ -22,6 +22,7 @@
 from .dispatch.base import AsyncDispatcher, Dispatcher
 from .dispatch.connection import HTTPConnection
 from .dispatch.connection_pool import ConnectionPool
+from .dispatch.proxy_http import HTTPProxy, HTTPProxyMode
 from .exceptions import (
     ConnectTimeout,
     CookieConflict,
@@ -30,6 +31,7 @@
     NotRedirectResponse,
     PoolTimeout,
     ProtocolError,
+    ProxyError,
     ReadTimeout,
     RedirectBodyUnavailable,
     RedirectLoop,
@@ -90,6 +92,8 @@
     "BasePoolSemaphore",
     "BaseBackgroundManager",
     "ConnectionPool",
+    "HTTPProxy",
+    "HTTPProxyMode",
     "ConnectTimeout",
     "CookieConflict",
     "DecodingError",
@@ -103,11 +107,12 @@
     "ResponseClosed",
     "ResponseNotRead",
     "StreamConsumed",
+    "ProxyError",
     "Timeout",
     "TooManyRedirects",
     "WriteTimeout",
     "AsyncDispatcher",
-    "BaseStream",
+    "BaseTCPStream",
     "ConcurrencyBackend",
     "Dispatcher",
     "URL",

httpx/api.py

@@ -27,7 +27,6 @@ def request(
     auth: AuthTypes = None,
     timeout: TimeoutTypes = None,
     allow_redirects: bool = True,
-    # proxies
     cert: CertTypes = None,
     verify: VerifyTypes = True,
     stream: bool = False,

httpx/client.py

@@ -334,7 +334,7 @@ async def head(
         cookies: CookieTypes = None,
         stream: bool = False,
         auth: AuthTypes = None,
-        allow_redirects: bool = False,  #  Note: Differs to usual default.
+        allow_redirects: bool = False,  # NOTE: Differs to usual default.
         cert: CertTypes = None,
         verify: VerifyTypes = None,
         timeout: TimeoutTypes = None,
@@ -784,7 +784,7 @@ def head(
         cookies: CookieTypes = None,
         stream: bool = False,
         auth: AuthTypes = None,
-        allow_redirects: bool = False,  #  Note: Differs to usual default.
+        allow_redirects: bool = False,  # NOTE: Differs to usual default.
         cert: CertTypes = None,
         verify: VerifyTypes = None,
         timeout: TimeoutTypes = None,

httpx/concurrency/asyncio.py

@@ -1,13 +1,3 @@
-"""
-The `Stream` class here provides a lightweight layer over
-`asyncio.StreamReader` and `asyncio.StreamWriter`.
-
-Similarly `PoolSemaphore` is a lightweight layer over `BoundedSemaphore`.
-
-These classes help encapsulate the timeout logic, make it easier to unit-test
-protocols, and help keep the rest of the package more `async`/`await`
-based, and less strictly `asyncio`-specific.
-"""
 import asyncio
 import functools
 import ssl
@@ -21,7 +11,7 @@
     BaseEvent,
     BasePoolSemaphore,
     BaseQueue,
-    BaseStream,
+    BaseTCPStream,
     ConcurrencyBackend,
     TimeoutFlag,
 )
@@ -50,7 +40,7 @@ def _fixed_write(self, data: bytes) -> None:  # type: ignore
     MonkeyPatch.write = _fixed_write
 
 
-class Stream(BaseStream):
+class TCPStream(BaseTCPStream):
     def __init__(
         self,
         stream_reader: asyncio.StreamReader,
@@ -190,13 +180,13 @@ def loop(self) -> asyncio.AbstractEventLoop:
                 self._loop = asyncio.new_event_loop()
         return self._loop
 
-    async def connect(
+    async def open_tcp_stream(
         self,
         hostname: str,
         port: int,
         ssl_context: typing.Optional[ssl.SSLContext],
         timeout: TimeoutConfig,
-    ) -> BaseStream:
+    ) -> BaseTCPStream:
         try:
             stream_reader, stream_writer = await asyncio.wait_for(  # type: ignore
                 asyncio.open_connection(hostname, port, ssl=ssl_context),
@@ -205,25 +195,25 @@ async def connect(
         except asyncio.TimeoutError:
             raise ConnectTimeout()
 
-        return Stream(
+        return TCPStream(
             stream_reader=stream_reader, stream_writer=stream_writer, timeout=timeout
         )
 
     async def start_tls(
         self,
-        stream: BaseStream,
+        stream: BaseTCPStream,
         hostname: str,
         ssl_context: ssl.SSLContext,
         timeout: TimeoutConfig,
-    ) -> BaseStream:
+    ) -> BaseTCPStream:
 
         loop = self.loop
         if not hasattr(loop, "start_tls"):  # pragma: no cover
             raise NotImplementedError(
                 "asyncio.AbstractEventLoop.start_tls() is only available in Python 3.7+"
             )
 
-        assert isinstance(stream, Stream)
+        assert isinstance(stream, TCPStream)
 
         stream_reader = asyncio.StreamReader()
         protocol = asyncio.StreamReaderProtocol(stream_reader)

httpx/concurrency/base.py

@@ -37,9 +37,9 @@ def set_write_timeouts(self) -> None:
         self.raise_on_write_timeout = True
 
 
-class BaseStream:
+class BaseTCPStream:
     """
-    A stream with read/write operations. Abstracts away any asyncio-specific
+    A TCP stream with read/write operations. Abstracts away any asyncio-specific
     interfaces into a more generic base class, that we can use with alternate
     backends, or for stand-alone test cases.
     """
@@ -110,22 +110,22 @@ def release(self) -> None:
 
 
 class ConcurrencyBackend:
-    async def connect(
+    async def open_tcp_stream(
         self,
         hostname: str,
         port: int,
         ssl_context: typing.Optional[ssl.SSLContext],
         timeout: TimeoutConfig,
-    ) -> BaseStream:
+    ) -> BaseTCPStream:
         raise NotImplementedError()  # pragma: no cover
 
     async def start_tls(
         self,
-        stream: BaseStream,
+        stream: BaseTCPStream,
         hostname: str,
         ssl_context: ssl.SSLContext,
         timeout: TimeoutConfig,
-    ) -> BaseStream:
+    ) -> BaseTCPStream:
         raise NotImplementedError()  # pragma: no cover
 
     def get_semaphore(self, limits: PoolLimits) -> BasePoolSemaphore:

httpx/dispatch/connection.py

@@ -85,7 +85,7 @@ async def connect(
             on_release = functools.partial(self.release_func, self)
 
         logger.debug(f"start_connect host={host!r} port={port!r} timeout={timeout!r}")
-        stream = await self.backend.connect(host, port, ssl_context, timeout)
+        stream = await self.backend.open_tcp_stream(host, port, ssl_context, timeout)
         http_version = stream.get_http_version()
         logger.debug(f"connected http_version={http_version!r}")
 

httpx/dispatch/connection_pool.py

@@ -128,14 +128,8 @@ async def send(
         return response
 
     async def acquire_connection(self, origin: Origin) -> HTTPConnection:
-        logger.debug(f"acquire_connection origin={origin!r}")
-        connection = self.active_connections.pop_by_origin(origin, http2_only=True)
-        if connection is None:
-            connection = self.keepalive_connections.pop_by_origin(origin)
-
-        if connection is not None and connection.is_connection_dropped():
-            self.max_connections.release()
-            connection = None
+        logger.debug("acquire_connection origin={origin!r}")
+        connection = self.pop_connection(origin)
 
         if connection is None:
             await self.max_connections.acquire()
@@ -179,3 +173,14 @@ async def close(self) -> None:
         self.keepalive_connections.clear()
         for connection in connections:
             await connection.close()
+
+    def pop_connection(self, origin: Origin) -> typing.Optional[HTTPConnection]:
+        connection = self.active_connections.pop_by_origin(origin, http2_only=True)
+        if connection is None:
+            connection = self.keepalive_connections.pop_by_origin(origin)
+
+        if connection is not None and connection.is_connection_dropped():
+            self.max_connections.release()
+            connection = None
+
+        return connection