[upd] upgrade httpx 0.18.2

adjust searx.network module to the new internal API
see encode/httpx#1522

Author: dalf

requirements.txt

@@ -7,10 +7,10 @@ lxml==4.6.3
 pygments==2.9.0
 python-dateutil==2.8.2
 pyyaml==5.4.1
-httpx[http2]==0.17.1
+httpx[http2]==0.18.2
 Brotli==1.0.9
 uvloop==0.16.0; python_version >= '3.7'
 uvloop==0.14.0; python_version < '3.7'
-httpx-socks[asyncio]==0.3.1
+httpx-socks[asyncio]==0.4.1
 langdetect==1.0.9
 setproctitle==1.2.2

searx/network/client.py

@@ -30,15 +30,18 @@
 LOOP = None
 SSLCONTEXTS = {}
 TRANSPORT_KWARGS = {
-    'backend': 'asyncio',
+    # use anyio :
+    # * https://github.com/encode/httpcore/issues/344
+    # * https://github.com/encode/httpx/discussions/1511
+    'backend': 'anyio',
     'trust_env': False,
 }
 
 
 # pylint: disable=protected-access
 async def close_connections_for_url(
-        connection_pool: httpcore.AsyncConnectionPool,
-        url: httpcore._utils.URL ):
+    connection_pool: httpcore.AsyncConnectionPool, url: httpcore._utils.URL
+):
 
     origin = httpcore._utils.url_to_origin(url)
     logger.debug('Drop connections for %r', origin)
@@ -63,8 +66,10 @@ def get_sslcontexts(proxy_url=None, cert=None, verify=True, trust_env=True, http
 class AsyncHTTPTransportNoHttp(httpcore.AsyncHTTPTransport):
     """Block HTTP request"""
 
-    async def arequest(self, method, url, headers=None, stream=None, ext=None):
-        raise httpcore.UnsupportedProtocol("HTTP protocol is disabled")
+    async def handle_async_request(
+        self, method, url, headers=None, stream=None, extensions=None
+    ):
+        raise httpcore.UnsupportedProtocol('HTTP protocol is disabled')
 
 
 class AsyncProxyTransportFixed(AsyncProxyTransport):
@@ -79,59 +84,63 @@ class AsyncProxyTransportFixed(AsyncProxyTransport):
     * self._response_closed(self, connection)
 
     Note: AsyncProxyTransport inherit from AsyncConnectionPool
-
-    Note: the API is going to change on httpx 0.18.0
-    see https://github.com/encode/httpx/pull/1522
     """
 
-    async def arequest(self, method, url, headers=None, stream=None, ext=None):
+    async def handle_async_request(
+        self, method, url, headers=None, stream=None, extensions=None
+    ):
         retry = 2
         while retry > 0:
             retry -= 1
             try:
-                return await super().arequest(method, url, headers, stream, ext)
+                return await super().handle_async_request(
+                    method, url, headers=headers, stream=stream, extensions=extensions
+                )
             except (ProxyConnectionError, ProxyTimeoutError, ProxyError) as e:
-                raise httpcore.ProxyError(e)
+                raise httpx.ProxyError(e)
             except OSError as e:
                 # socket.gaierror when DNS resolution fails
-                raise httpcore.NetworkError(e)
+                raise httpx.NetworkError(e)
             except httpcore.RemoteProtocolError as e:
                 # in case of httpcore.RemoteProtocolError: Server disconnected
                 await close_connections_for_url(self, url)
                 logger.warning('httpcore.RemoteProtocolError: retry', exc_info=e)
                 # retry
             except (httpcore.NetworkError, httpcore.ProtocolError) as e:
+                # workaround before anyio was the default backend:
                 # httpcore.WriteError on HTTP/2 connection leaves a new opened stream
                 # then each new request creates a new stream and raise the same WriteError
+                # most probably not necessary: https://github.com/encode/httpcore/issues/357
                 await close_connections_for_url(self, url)
                 raise e
 
 
 class AsyncHTTPTransportFixed(httpx.AsyncHTTPTransport):
     """Fix httpx.AsyncHTTPTransport"""
 
-    async def arequest(self, method, url, headers=None, stream=None, ext=None):
+    async def handle_async_request(
+        self, method, url, headers=None, stream=None, extensions=None
+    ):
         retry = 2
         while retry > 0:
             retry -= 1
             try:
-                return await super().arequest(method, url, headers, stream, ext)
+                return await super().handle_async_request(
+                    method, url, headers=headers, stream=stream, extensions=extensions
+                )
             except OSError as e:
                 # socket.gaierror when DNS resolution fails
                 raise httpcore.ConnectError(e)
-            except httpcore.CloseError as e:
-                # httpcore.CloseError: [Errno 104] Connection reset by peer
-                # raised by _keepalive_sweep()
-                #   from https://github.com/encode/httpcore/blob/4b662b5c42378a61e54d673b4c949420102379f5/httpcore/_backends/asyncio.py#L198  # pylint: disable=line-too-long
-                await close_connections_for_url(self._pool, url)
-                logger.warning('httpcore.CloseError: retry', exc_info=e)
-                # retry
             except httpcore.RemoteProtocolError as e:
                 # in case of httpcore.RemoteProtocolError: Server disconnected
                 await close_connections_for_url(self._pool, url)
                 logger.warning('httpcore.RemoteProtocolError: retry', exc_info=e)
                 # retry
             except (httpcore.ProtocolError, httpcore.NetworkError) as e:
+                # workaround before anyio was the default backend:
+                # httpcore.WriteError on HTTP/2 connection leaves a new opened stream
+                # then each new request creates a new stream and raise the same WriteError
+                # most probably not necessary: https://github.com/encode/httpcore/issues/357
                 await close_connections_for_url(self._pool, url)
                 raise e
 
@@ -206,7 +215,7 @@ def new_client(
         if not enable_http and (pattern == 'http' or pattern.startswith('http://')):
             continue
         if (proxy_url.startswith('socks4://')
-           or proxy_url.startswith('socks5://')
+            or proxy_url.startswith('socks5://')
             or proxy_url.startswith('socks5h://')
         ):
             mounts[pattern] = get_transport_for_socks_proxy(