dependency resolution fails for %pip install aioboto3==14.1.0

[tubaman]

%pip install aioboto3==14.1.0 fails with:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[2], line 1
----> 1 await __import__("piplite").install(**{'requirements': ['aioboto3==14.1.0']})

File /lib/python3.12/site-packages/piplite/piplite.py:121, in _install(requirements, keep_going, deps, credentials, pre, index_urls, verbose)
    119 """Invoke micropip.install with a patch to get data from local indexes"""
    120 with patch("micropip.package_index.query_package", _query_package):
--> 121     return await micropip.install(
    122         requirements=requirements,
    123         keep_going=keep_going,
    124         deps=deps,
    125         credentials=credentials,
    126         pre=pre,
    127         index_urls=index_urls,
    128         verbose=verbose,
    129     )

File /lib/python3.12/site-packages/micropip/package_manager.py:133, in PackageManager.install(self, requirements, keep_going, deps, credentials, pre, index_urls, verbose)
    130 if index_urls is None:
    131     index_urls = self.index_urls
--> 133 return await install(
    134     requirements,
    135     index_urls,
    136     keep_going,
    137     deps,
    138     credentials,
    139     pre,
    140     verbose=verbose,
    141 )

File /lib/python3.12/site-packages/micropip/install.py:53, in install(requirements, index_urls, keep_going, deps, credentials, pre, verbose)
     41 wheel_base = Path(getsitepackages()[0])
     43 transaction = Transaction(
     44     ctx=ctx,  # type: ignore[arg-type]
     45     ctx_extras=[],
   (...)
     51     index_urls=index_urls,
     52 )
---> 53 await transaction.gather_requirements(requirements)
     55 if transaction.failed:
     56     failed_requirements = ", ".join([f"'{req}'" for req in transaction.failed])

File /lib/python3.12/site-packages/micropip/transaction.py:55, in Transaction.gather_requirements(self, requirements)
     52 for requirement in requirements:
     53     requirement_promises.append(self.add_requirement(requirement))
---> 55 await asyncio.gather(*requirement_promises)

File /lib/python3.12/site-packages/micropip/transaction.py:62, in Transaction.add_requirement(self, req)
     59     return await self.add_requirement_inner(req)
     61 if not urlparse(req).path.endswith(".whl"):
---> 62     return await self.add_requirement_inner(Requirement(req))
     64 # custom download location
     65 wheel = WheelInfo.from_url(req)

File /lib/python3.12/site-packages/micropip/transaction.py:152, in Transaction.add_requirement_inner(self, req)
    149         logger.debug("Transaction: package found in lock file: %r", req)
    150         return
--> 152     await self._add_requirement_from_package_index(req)
    153 else:
    154     try:

File /lib/python3.12/site-packages/micropip/transaction.py:214, in Transaction._add_requirement_from_package_index(self, req)
    211 if satisfied:
    212     logger.info("Requirement already satisfied: %s (%s)", req, ver)
--> 214 await self.add_wheel(wheel, req.extras, specifier=str(req.specifier))

File /lib/python3.12/site-packages/micropip/transaction.py:262, in Transaction.add_wheel(self, wheel, extras, specifier)
    257     except OSError:
    258         # If something goes wrong while downloading the metadata,
    259         # we have to wait for the wheel to be downloaded.
    260         await wheel_download_task
--> 262     await asyncio.gather(
    263         self.gather_requirements(wheel.requires(extras)),
    264         wheel_download_task,
    265     )
    267 # Case 2) If metadata file is not available,
    268 #         we have to wait for the wheel to be downloaded.
    269 else:
    270     await wheel_download_task

File /lib/python3.12/site-packages/micropip/transaction.py:55, in Transaction.gather_requirements(self, requirements)
     52 for requirement in requirements:
     53     requirement_promises.append(self.add_requirement(requirement))
---> 55 await asyncio.gather(*requirement_promises)

File /lib/python3.12/site-packages/micropip/transaction.py:59, in Transaction.add_requirement(self, req)
     57 async def add_requirement(self, req: str | Requirement) -> None:
     58     if isinstance(req, Requirement):
---> 59         return await self.add_requirement_inner(req)
     61     if not urlparse(req).path.endswith(".whl"):
     62         return await self.add_requirement_inner(Requirement(req))

File /lib/python3.12/site-packages/micropip/transaction.py:152, in Transaction.add_requirement_inner(self, req)
    149         logger.debug("Transaction: package found in lock file: %r", req)
    150         return
--> 152     await self._add_requirement_from_package_index(req)
    153 else:
    154     try:

File /lib/python3.12/site-packages/micropip/transaction.py:214, in Transaction._add_requirement_from_package_index(self, req)
    211 if satisfied:
    212     logger.info("Requirement already satisfied: %s (%s)", req, ver)
--> 214 await self.add_wheel(wheel, req.extras, specifier=str(req.specifier))

File /lib/python3.12/site-packages/micropip/transaction.py:262, in Transaction.add_wheel(self, wheel, extras, specifier)
    257     except OSError:
    258         # If something goes wrong while downloading the metadata,
    259         # we have to wait for the wheel to be downloaded.
    260         await wheel_download_task
--> 262     await asyncio.gather(
    263         self.gather_requirements(wheel.requires(extras)),
    264         wheel_download_task,
    265     )
    267 # Case 2) If metadata file is not available,
    268 #         we have to wait for the wheel to be downloaded.
    269 else:
    270     await wheel_download_task

File /lib/python3.12/site-packages/micropip/transaction.py:55, in Transaction.gather_requirements(self, requirements)
     52 for requirement in requirements:
     53     requirement_promises.append(self.add_requirement(requirement))
---> 55 await asyncio.gather(*requirement_promises)

File /lib/python3.12/site-packages/micropip/transaction.py:59, in Transaction.add_requirement(self, req)
     57 async def add_requirement(self, req: str | Requirement) -> None:
     58     if isinstance(req, Requirement):
---> 59         return await self.add_requirement_inner(req)
     61     if not urlparse(req).path.endswith(".whl"):
     62         return await self.add_requirement_inner(Requirement(req))

    [... skipping similar frames: Transaction._add_requirement_from_package_index at line 214 (1 times), Transaction.add_requirement at line 59 (1 times), Transaction.add_requirement_inner at line 152 (1 times), Transaction.add_wheel at line 262 (1 times), Transaction.gather_requirements at line 55 (1 times)]

File /lib/python3.12/site-packages/micropip/transaction.py:152, in Transaction.add_requirement_inner(self, req)
    149         logger.debug("Transaction: package found in lock file: %r", req)
    150         return
--> 152     await self._add_requirement_from_package_index(req)
    153 else:
    154     try:

File /lib/python3.12/site-packages/micropip/transaction.py:214, in Transaction._add_requirement_from_package_index(self, req)
    211 if satisfied:
    212     logger.info("Requirement already satisfied: %s (%s)", req, ver)
--> 214 await self.add_wheel(wheel, req.extras, specifier=str(req.specifier))

File /lib/python3.12/site-packages/micropip/transaction.py:262, in Transaction.add_wheel(self, wheel, extras, specifier)
    257     except OSError:
    258         # If something goes wrong while downloading the metadata,
    259         # we have to wait for the wheel to be downloaded.
    260         await wheel_download_task
--> 262     await asyncio.gather(
    263         self.gather_requirements(wheel.requires(extras)),
    264         wheel_download_task,
    265     )
    267 # Case 2) If metadata file is not available,
    268 #         we have to wait for the wheel to be downloaded.
    269 else:
    270     await wheel_download_task

File /lib/python3.12/site-packages/micropip/transaction.py:55, in Transaction.gather_requirements(self, requirements)
     52 for requirement in requirements:
     53     requirement_promises.append(self.add_requirement(requirement))
---> 55 await asyncio.gather(*requirement_promises)

File /lib/python3.12/site-packages/micropip/transaction.py:59, in Transaction.add_requirement(self, req)
     57 async def add_requirement(self, req: str | Requirement) -> None:
     58     if isinstance(req, Requirement):
---> 59         return await self.add_requirement_inner(req)
     61     if not urlparse(req).path.endswith(".whl"):
     62         return await self.add_requirement_inner(Requirement(req))

File /lib/python3.12/site-packages/micropip/transaction.py:141, in Transaction.add_requirement_inner(self, req)
    138 # Is some version of this package is already installed?
    139 req.name = canonicalize_name(req.name)
--> 141 satisfied, ver = self.check_version_satisfied(req)
    142 if satisfied:
    143     logger.info("Requirement already satisfied: %s (%s)", req, ver)

File /lib/python3.12/site-packages/micropip/transaction.py:86, in Transaction.check_version_satisfied(self, req)
     82 if req.specifier.contains(ver, prereleases=True):
     83     # installed version matches, nothing to do
     84     return True, ver
---> 86 raise ValueError(
     87     f"Requested '{req}', " f"but {req.name}=={ver} is already installed"
     88 )

ValueError: Requested 'botocore<2.0a.0,>=1.37.4', but botocore==1.37.1 is already installed

pip install aioboto3==14.1.0 succeeds using regular pip at a Linux commandline:

Collecting aioboto3==14.1.0
  Using cached aioboto3-14.1.0-py3-none-any.whl.metadata (8.8 kB)
Collecting aiobotocore==2.21.1 (from aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached aiobotocore-2.21.1-py3-none-any.whl.metadata (24 kB)
Collecting aiofiles>=23.2.1 (from aioboto3==14.1.0)
  Using cached aiofiles-24.1.0-py3-none-any.whl.metadata (10 kB)
Collecting aiohttp<4.0.0,>=3.9.2 (from aiobotocore==2.21.1->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached aiohttp-3.11.16-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.7 kB)
Collecting aioitertools<1.0.0,>=0.5.1 (from aiobotocore==2.21.1->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached aioitertools-0.12.0-py3-none-any.whl.metadata (3.8 kB)
Collecting botocore<1.37.2,>=1.37.0 (from aiobotocore==2.21.1->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached botocore-1.37.1-py3-none-any.whl.metadata (5.7 kB)
Collecting python-dateutil<3.0.0,>=2.1 (from aiobotocore==2.21.1->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl.metadata (8.4 kB)
Collecting jmespath<2.0.0,>=0.7.1 (from aiobotocore==2.21.1->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached jmespath-1.0.1-py3-none-any.whl.metadata (7.6 kB)
Collecting multidict<7.0.0,>=6.0.0 (from aiobotocore==2.21.1->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Downloading multidict-6.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.1 kB)
Collecting wrapt<2.0.0,>=1.10.10 (from aiobotocore==2.21.1->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached wrapt-1.17.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.4 kB)
Collecting boto3<1.37.2,>=1.37.0 (from aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached boto3-1.37.1-py3-none-any.whl.metadata (6.6 kB)
Collecting aiohappyeyeballs>=2.3.0 (from aiohttp<4.0.0,>=3.9.2->aiobotocore==2.21.1->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached aiohappyeyeballs-2.6.1-py3-none-any.whl.metadata (5.9 kB)
Collecting aiosignal>=1.1.2 (from aiohttp<4.0.0,>=3.9.2->aiobotocore==2.21.1->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached aiosignal-1.3.2-py2.py3-none-any.whl.metadata (3.8 kB)
Collecting attrs>=17.3.0 (from aiohttp<4.0.0,>=3.9.2->aiobotocore==2.21.1->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached attrs-25.3.0-py3-none-any.whl.metadata (10 kB)
Collecting frozenlist>=1.1.1 (from aiohttp<4.0.0,>=3.9.2->aiobotocore==2.21.1->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached frozenlist-1.5.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (13 kB)
Collecting propcache>=0.2.0 (from aiohttp<4.0.0,>=3.9.2->aiobotocore==2.21.1->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached propcache-0.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (10 kB)
Collecting yarl<2.0,>=1.17.0 (from aiohttp<4.0.0,>=3.9.2->aiobotocore==2.21.1->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached yarl-1.18.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (69 kB)
Collecting s3transfer<0.12.0,>=0.11.0 (from boto3<1.37.2,>=1.37.0->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached s3transfer-0.11.4-py3-none-any.whl.metadata (1.7 kB)
Collecting urllib3!=2.2.0,<3,>=1.25.4 (from botocore<1.37.2,>=1.37.0->aiobotocore==2.21.1->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached urllib3-2.3.0-py3-none-any.whl.metadata (6.5 kB)
Collecting six>=1.5 (from python-dateutil<3.0.0,>=2.1->aiobotocore==2.21.1->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached six-1.17.0-py2.py3-none-any.whl.metadata (1.7 kB)
INFO: pip is looking at multiple versions of s3transfer to determine which version is compatible with other requirements. This could take a while.
Collecting s3transfer<0.12.0,>=0.11.0 (from boto3<1.37.2,>=1.37.0->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached s3transfer-0.11.3-py3-none-any.whl.metadata (1.7 kB)
Collecting idna>=2.0 (from yarl<2.0,>=1.17.0->aiohttp<4.0.0,>=3.9.2->aiobotocore==2.21.1->aiobotocore[boto3]==2.21.1->aioboto3==14.1.0)
  Using cached idna-3.10-py3-none-any.whl.metadata (10 kB)
Using cached aioboto3-14.1.0-py3-none-any.whl (35 kB)
Using cached aiobotocore-2.21.1-py3-none-any.whl (78 kB)
Using cached aiofiles-24.1.0-py3-none-any.whl (15 kB)
Using cached aiohttp-3.11.16-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)
Using cached aioitertools-0.12.0-py3-none-any.whl (24 kB)
Using cached boto3-1.37.1-py3-none-any.whl (139 kB)
Using cached botocore-1.37.1-py3-none-any.whl (13.4 MB)
Using cached jmespath-1.0.1-py3-none-any.whl (20 kB)
Downloading multidict-6.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (249 kB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 249.7/249.7 kB 4.0 MB/s eta 0:00:00
Using cached python_dateutil-2.9.0.post0-py2.py3-none-any.whl (229 kB)
Using cached wrapt-1.17.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (89 kB)
Using cached aiohappyeyeballs-2.6.1-py3-none-any.whl (15 kB)
Using cached aiosignal-1.3.2-py2.py3-none-any.whl (7.6 kB)
Using cached attrs-25.3.0-py3-none-any.whl (63 kB)
Using cached frozenlist-1.5.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (283 kB)
Using cached propcache-0.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (245 kB)
Using cached s3transfer-0.11.3-py3-none-any.whl (84 kB)
Using cached six-1.17.0-py2.py3-none-any.whl (11 kB)
Using cached urllib3-2.3.0-py3-none-any.whl (128 kB)
Using cached yarl-1.18.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (336 kB)
Using cached idna-3.10-py3-none-any.whl (70 kB)
Installing collected packages: wrapt, urllib3, six, propcache, multidict, jmespath, idna, frozenlist, attrs, aioitertools, aiohappyeyeballs, aiofiles, yarl, python-dateutil, aiosignal, botocore, aiohttp, s3transfer, aiobotocore, boto3, aioboto3
Successfully installed aioboto3-14.1.0 aiobotocore-2.21.1 aiofiles-24.1.0 aiohappyeyeballs-2.6.1 aiohttp-3.11.16 aioitertools-0.12.0 aiosignal-1.3.2 attrs-25.3.0 boto3-1.37.1 botocore-1.37.1 frozenlist-1.5.0 idna-3.10 jmespath-1.0.1 multidict-6.3.2 propcache-0.3.1 python-dateutil-2.9.0.post0 s3transfer-0.11.3 six-1.17.0 urllib3-2.3.0 wrapt-1.17.2 yarl-1.18.3

With micropip, instead of selecting the s3transfer nested dependency that would work properly with other dependencies(0.11.3), micropip selects the latest(0.11.4) which requires a later version of botocore than is supported by other dependencies. Here's a workaround:

%pip install botocore==1.37.1
%pip install s3transfer==0.11.3
%pip install aioboto3==14.1.0

[tubaman]

Note, the same problem exists if I call using micropip directly:

import micropip
await micropip.install(["aioboto3==14.1.0"])

[agriyakhetarpal]

Thanks for the report, @tubaman! micropip's dependency resolver is quite naïve, so there are cases where this can happen. Here's a bug report with a similar problem: #113. The previous blocker on implementing a better resolver was that a lot of wheels would be downloaded if their metadata were to be extracted for the resolution, hogging up bandwidth. Given that dependency metadata exists separately from the actual wheels nowadays and we have support to parse that from #139, maybe it is time for us to consider a better resolver (see #93).