Fix test infrastructure for Python-version-excluded providers

Dockerfile

@@ -1114,15 +1114,35 @@ from __future__ import annotations
 
 import os
 import sys
+import zipfile
+from email.parser import HeaderParser
 from pathlib import Path
 
+from packaging.specifiers import InvalidSpecifier, SpecifierSet
 from packaging.utils import (
     InvalidSdistFilename,
     InvalidWheelFilename,
     parse_sdist_filename,
     parse_wheel_filename,
 )
 
+_CURRENT_PYTHON = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
+
+
+def _compatible_with_current_python(wheel_path: str) -> bool:
+    """Return False if the wheel's Requires-Python excludes the running interpreter."""
+    try:
+        with zipfile.ZipFile(wheel_path) as zf:
+            for name in zf.namelist():
+                if name.endswith(".dist-info/METADATA"):
+                    requires = HeaderParser().parsestr(zf.read(name).decode("utf-8")).get("Requires-Python")
+                    if requires:
+                        return _CURRENT_PYTHON in SpecifierSet(requires)
+                    return True
+    except (zipfile.BadZipFile, InvalidSpecifier, KeyError) as exc:
+        print(f"Warning: could not check Requires-Python for {wheel_path}: {exc}", file=sys.stderr)
+    return True
+
 
 def print_package_specs(extras: str = "") -> None:
     for package_path in sys.argv[1:]:
@@ -1134,6 +1154,12 @@ def print_package_specs(extras: str = "") -> None:
             except InvalidSdistFilename:
                 print(f"Could not parse package name from {package_path}", file=sys.stderr)
                 continue
+        if package_path.endswith(".whl") and not _compatible_with_current_python(package_path):
+            print(
+                f"Skipping {package} (Requires-Python not satisfied by {_CURRENT_PYTHON})",
+                file=sys.stderr,
+            )
+            continue
         print(f"{package}{extras} @ file://{package_path}")
 
 

dev/breeze/src/airflow_breeze/commands/testing_commands.py

@@ -249,10 +249,9 @@ def _run_test(
     pytest_args.extend(extra_pytest_args)
     # Skip "FOLDER" in case "--ignore=FOLDER" is passed as an argument
     # Which might be the case if we are ignoring some providers during compatibility checks
-    pytest_args_before_skip = pytest_args
     pytest_args = [arg for arg in pytest_args if f"--ignore={arg}" not in pytest_args]
-    # Double check: If no test is leftover we can skip running the test
-    if pytest_args_before_skip != pytest_args and pytest_args[0].startswith("--"):
+    # If no test directory is left (all positional args were excluded/ignored), skip
+    if pytest_args and pytest_args[0].startswith("--"):
         return 0, f"Skipped test, no tests needed: {shell_params.test_type}"
     run_cmd.extend(pytest_args)
     try:

dev/breeze/src/airflow_breeze/utils/packages.py

@@ -705,7 +705,7 @@ def get_provider_jinja_context(
     requires_python_version: str = f">={DEFAULT_PYTHON_MAJOR_MINOR_VERSION}"
     # Most providers require the same python versions, but some may have exclusions
     for excluded_python_version in provider_details.excluded_python_versions:
-        requires_python_version += f",!={excluded_python_version}"
+        requires_python_version += f",!={excluded_python_version}.*"
 
     context: dict[str, Any] = {
         "PROVIDER_ID": provider_details.provider_id,

docker-tests/tests/docker_tests/test_prod_image.py

@@ -21,6 +21,7 @@
 from importlib.util import find_spec
 
 import pytest
+import yaml
 from python_on_whales import DockerException
 
 from docker_tests.constants import AIRFLOW_ROOT_PATH
@@ -57,6 +58,39 @@
 testing_slim_image = os.environ.get("TEST_SLIM_IMAGE", str(False)).lower() in ("true", "1", "yes")
 
 
+def _get_provider_python_exclusions() -> dict[str, list[str]]:
+    """Return mapping of provider_id -> list of excluded Python minor versions from provider.yaml."""
+    exclusions: dict[str, list[str]] = {}
+    providers_root = AIRFLOW_ROOT_PATH / "providers"
+    for line in PROD_IMAGE_PROVIDERS_FILE_PATH.read_text().splitlines():
+        provider_id = line.split(">=")[0].strip()
+        if not provider_id or provider_id.startswith("#"):
+            continue
+        provider_yaml_path = providers_root / provider_id.replace(".", "/") / "provider.yaml"
+        if not provider_yaml_path.exists():
+            continue
+        with open(provider_yaml_path) as f:
+            data = yaml.safe_load(f)
+        excluded = data.get("excluded-python-versions", [])
+        if excluded:
+            exclusions[provider_id] = [str(v) for v in excluded]
+    return exclusions
+
+
+PROVIDER_PYTHON_EXCLUSIONS = _get_provider_python_exclusions()
+
+
+def _get_python_minor_version(default_docker_image: str) -> str:
+    """Get Python minor version (e.g. '3.14') from the Docker image."""
+    python_version = run_bash_in_docker("python --version", image=default_docker_image)
+    return ".".join(python_version.strip().split()[1].split(".")[:2])
+
+
+def _get_excluded_provider_ids(python_minor: str) -> set[str]:
+    """Return set of provider IDs excluded for the given Python minor version."""
+    return {pid for pid, versions in PROVIDER_PYTHON_EXCLUSIONS.items() if python_minor in versions}
+
+
 class TestCommands:
     def test_without_command(self, default_docker_image):
         """Checking the image without a command. It should return non-zero exit code."""
@@ -91,7 +125,10 @@ def test_required_providers_are_installed(self, default_docker_image):
         if testing_slim_image:
             packages_to_install = set(SLIM_IMAGE_PROVIDERS)
         else:
-            packages_to_install = set(REGULAR_IMAGE_PROVIDERS)
+            python_minor = _get_python_minor_version(default_docker_image)
+            excluded_ids = _get_excluded_provider_ids(python_minor)
+            excluded_packages = {f"apache-airflow-providers-{pid.replace('.', '-')}" for pid in excluded_ids}
+            packages_to_install = set(REGULAR_IMAGE_PROVIDERS) - excluded_packages
         assert len(packages_to_install) != 0
         output = run_bash_in_docker(
             "airflow providers list --output json",
@@ -197,15 +234,19 @@ def test_pip_dependencies_conflict(self, default_docker_image):
     def test_check_dependencies_imports(
         self, package_name: str, import_names: list[str], default_docker_image: str
     ):
+        python_minor = _get_python_minor_version(default_docker_image)
+        excluded_ids = _get_excluded_provider_ids(python_minor)
+        # Skip individual provider test cases if the provider is excluded for this Python version
+        if package_name in excluded_ids:
+            pytest.skip(f"Provider {package_name} is excluded for Python {python_minor}")
         if package_name == "providers":
-            python_version = run_bash_in_docker(
-                "python --version",
-                image=default_docker_image,
-            )
-            if python_version.startswith("Python 3.13"):
-                if "airflow.providers.fab" in import_names:
-                    import_names.remove("airflow.providers.fab")
-        run_python_in_docker(f"import {','.join(import_names)}", image=default_docker_image)
+            excluded_imports = {f"airflow.providers.{pid}" for pid in excluded_ids}
+            import_names = [name for name in import_names if name not in excluded_imports]
+            # FAB provider has import issues on Python 3.13
+            if python_minor == "3.13":
+                import_names = [name for name in import_names if name != "airflow.providers.fab"]
+        if import_names:
+            run_python_in_docker(f"import {','.join(import_names)}", image=default_docker_image)
 
     def test_there_is_no_opt_airflow_airflow_folder(self, default_docker_image):
         output = run_bash_in_docker(

scripts/docker/get_distribution_specs.py

@@ -19,15 +19,35 @@
 
 import os
 import sys
+import zipfile
+from email.parser import HeaderParser
 from pathlib import Path
 
+from packaging.specifiers import InvalidSpecifier, SpecifierSet
 from packaging.utils import (
     InvalidSdistFilename,
     InvalidWheelFilename,
     parse_sdist_filename,
     parse_wheel_filename,
 )
 
+_CURRENT_PYTHON = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
+
+
+def _compatible_with_current_python(wheel_path: str) -> bool:
+    """Return False if the wheel's Requires-Python excludes the running interpreter."""
+    try:
+        with zipfile.ZipFile(wheel_path) as zf:
+            for name in zf.namelist():
+                if name.endswith(".dist-info/METADATA"):
+                    requires = HeaderParser().parsestr(zf.read(name).decode("utf-8")).get("Requires-Python")
+                    if requires:
+                        return _CURRENT_PYTHON in SpecifierSet(requires)
+                    return True
+    except (zipfile.BadZipFile, InvalidSpecifier, KeyError) as exc:
+        print(f"Warning: could not check Requires-Python for {wheel_path}: {exc}", file=sys.stderr)
+    return True
+
 
 def print_package_specs(extras: str = "") -> None:
     for package_path in sys.argv[1:]:
@@ -39,6 +59,12 @@ def print_package_specs(extras: str = "") -> None:
             except InvalidSdistFilename:
                 print(f"Could not parse package name from {package_path}", file=sys.stderr)
                 continue
+        if package_path.endswith(".whl") and not _compatible_with_current_python(package_path):
+            print(
+                f"Skipping {package} (Requires-Python not satisfied by {_CURRENT_PYTHON})",
+                file=sys.stderr,
+            )
+            continue
         print(f"{package}{extras} @ file://{package_path}")
 
 

scripts/tests/docker/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

scripts/tests/docker/test_get_distribution_specs.py

@@ -0,0 +1,152 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import subprocess
+import sys
+import zipfile
+from pathlib import Path
+
+import pytest
+
+SCRIPT_PATH = Path(__file__).resolve().parents[2] / "docker" / "get_distribution_specs.py"
+
+CURRENT_PYTHON_MAJOR_MINOR = f"{sys.version_info.major}.{sys.version_info.minor}"
+
+
+def _make_wheel(directory: Path, name: str, version: str, requires_python: str | None = None) -> Path:
+    """Create a minimal .whl (zip) with METADATA."""
+    safe_name = name.replace("-", "_")
+    wheel_path = directory / f"{safe_name}-{version}-py3-none-any.whl"
+    dist_info = f"{safe_name}-{version}.dist-info"
+    metadata_lines = [
+        "Metadata-Version: 2.1",
+        f"Name: {name}",
+        f"Version: {version}",
+    ]
+    if requires_python is not None:
+        metadata_lines.append(f"Requires-Python: {requires_python}")
+    with zipfile.ZipFile(wheel_path, "w") as zf:
+        zf.writestr(f"{dist_info}/METADATA", "\n".join(metadata_lines))
+        zf.writestr(f"{dist_info}/RECORD", "")
+    return wheel_path
+
+
+def _run_script(*wheel_paths: Path, env: dict[str, str] | None = None) -> subprocess.CompletedProcess:
+    return subprocess.run(
+        [sys.executable, str(SCRIPT_PATH), *(str(p) for p in wheel_paths)],
+        capture_output=True,
+        text=True,
+        check=False,
+        env=env,
+    )
+
+
+class TestRequiresPythonFiltering:
+    def test_wheel_without_requires_python_is_included(self, tmp_path):
+        whl = _make_wheel(tmp_path, "some-package", "1.0.0")
+        result = _run_script(whl)
+        assert result.returncode == 0
+        assert f"some-package @ file://{whl}" in result.stdout
+
+    def test_wheel_matching_current_python_is_included(self, tmp_path):
+        whl = _make_wheel(tmp_path, "some-package", "1.0.0", requires_python=">=3.10")
+        result = _run_script(whl)
+        assert result.returncode == 0
+        assert f"some-package @ file://{whl}" in result.stdout
+
+    def test_wheel_excluding_current_python_is_skipped(self, tmp_path):
+        whl = _make_wheel(
+            tmp_path,
+            "excluded-package",
+            "2.0.0",
+            requires_python=f">=3.10,!={CURRENT_PYTHON_MAJOR_MINOR}.*",
+        )
+        result = _run_script(whl)
+        assert result.returncode == 0
+        assert result.stdout == ""
+        assert "Skipping" in result.stderr
+        assert "excluded-package" in result.stderr
+
+    def test_corrupt_wheel_is_included_with_warning(self, tmp_path):
+        bad_whl = tmp_path / "bad_package-1.0.0-py3-none-any.whl"
+        bad_whl.write_bytes(b"not a zip")
+        result = _run_script(bad_whl)
+        assert result.returncode == 0
+        assert f"bad-package @ file://{bad_whl}" in result.stdout
+        assert "Warning" in result.stderr
+
+
+class TestMixedInputs:
+    def test_compatible_and_incompatible_together(self, tmp_path):
+        good_whl = _make_wheel(
+            tmp_path, "apache-airflow-providers-standard", "1.0.0", requires_python=">=3.10"
+        )
+        bad_whl = _make_wheel(
+            tmp_path,
+            "apache-airflow-providers-google",
+            "21.0.0",
+            requires_python=f">=3.10,!={CURRENT_PYTHON_MAJOR_MINOR}.*",
+        )
+        result = _run_script(good_whl, bad_whl)
+        assert result.returncode == 0
+        assert "apache-airflow-providers-standard" in result.stdout
+        assert "apache-airflow-providers-google" not in result.stdout
+        assert "Skipping" in result.stderr
+
+    def test_sdist_is_not_filtered(self, tmp_path):
+        """Sdists cannot be inspected for Requires-Python without building, so they pass through."""
+        import tarfile
+
+        sdist = tmp_path / "apache_airflow_providers_google-21.0.0.tar.gz"
+        with tarfile.open(sdist, "w:gz"):
+            pass
+        result = _run_script(sdist)
+        assert result.returncode == 0
+        assert "apache-airflow-providers-google" in result.stdout
+
+
+class TestExtrasEnvVar:
+    def test_extras_appended_to_spec(self, tmp_path):
+        import os
+
+        whl = _make_wheel(tmp_path, "apache-airflow", "3.0.0", requires_python=">=3.10")
+        env = {**os.environ, "EXTRAS": "[celery,google]"}
+        result = _run_script(whl, env=env)
+        assert result.returncode == 0
+        assert f"apache-airflow[celery,google] @ file://{whl}" in result.stdout
+
+
+@pytest.mark.parametrize(
+    ("requires_python", "should_include"),
+    [
+        pytest.param(">=3.10", True, id="lower-bound-satisfied"),
+        pytest.param(f"!={CURRENT_PYTHON_MAJOR_MINOR}.*", False, id="wildcard-minor-excluded"),
+        pytest.param(f">=3.10,!={CURRENT_PYTHON_MAJOR_MINOR}.*", False, id="range-with-exclusion"),
+        pytest.param("<3.10", False, id="upper-bound-below-current"),
+        pytest.param(None, True, id="no-requires-python"),
+    ],
+)
+def test_requires_python_specifiers(tmp_path, requires_python, should_include):
+    whl = _make_wheel(tmp_path, "test-package", "1.0.0", requires_python=requires_python)
+    result = _run_script(whl)
+    assert result.returncode == 0
+    if should_include:
+        assert f"test-package @ file://{whl}" in result.stdout
+    else:
+        assert result.stdout == ""
+        assert "Skipping" in result.stderr