Skip to content
43 changes: 36 additions & 7 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -1540,6 +1540,15 @@ function install_from_sources() {
if [[ ${VIRTUAL_ENV=} != "" ]]; then
extra_sync_flags="--active"
fi
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
echo
echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew; skipping the locked third-party installation.${COLOR_RESET}"
echo
return
fi
extra_sync_flags="${extra_sync_flags} --no-install-workspace"
fi
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then
set +x
Expand All @@ -1560,19 +1569,35 @@ function install_from_sources() {
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python
else
set +x
echo
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
echo
# Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving.
# --no-binary-package is needed in order to avoid libxml and xmlsec using different version of
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
# shellcheck disable=SC2206 # extra_sync_flags holds several flags
local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec
--no-python-downloads --no-managed-python)
set +x
# Without the workspace sources a lock that does not match the manifests cannot be re-resolved;
# the installation that adds the workspace packages does that later.
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && ! "${frozen_sync[@]}" --dry-run; then
echo
echo "${COLOR_YELLOW}uv.lock does not match the dependency manifests; skipping the locked third-party installation.${COLOR_RESET}"
echo
return
fi
echo
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
echo
set -x
if ! uv sync --all-packages --frozen --group ci-image \
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python; then
if ! "${frozen_sync[@]}"; then
set +x
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
echo
echo "${COLOR_RED}Failing because the locked third-party installation failed although uv.lock matches the dependency manifests.${COLOR_RESET}"
echo
exit 1
fi
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
echo
echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}"
Expand Down Expand Up @@ -1662,6 +1687,10 @@ function install_airflow_when_building_images() {
install_from_external_spec
fi
set +x
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
# Packaging tools and 'pip check' are handled by the installation that adds the workspace packages
return
fi
common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand Down
71 changes: 63 additions & 8 deletions Dockerfile.ci
Original file line number Diff line number Diff line change
Expand Up @@ -1233,6 +1233,15 @@ function install_from_sources() {
if [[ ${VIRTUAL_ENV=} != "" ]]; then
extra_sync_flags="--active"
fi
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
echo
echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew; skipping the locked third-party installation.${COLOR_RESET}"
echo
return
fi
extra_sync_flags="${extra_sync_flags} --no-install-workspace"
fi
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then
set +x
Expand All @@ -1253,19 +1262,35 @@ function install_from_sources() {
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python
else
set +x
echo
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
echo
# Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving.
# --no-binary-package is needed in order to avoid libxml and xmlsec using different version of
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
# shellcheck disable=SC2206 # extra_sync_flags holds several flags
local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec
--no-python-downloads --no-managed-python)
set +x
# Without the workspace sources a lock that does not match the manifests cannot be re-resolved;
# the installation that adds the workspace packages does that later.
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && ! "${frozen_sync[@]}" --dry-run; then
echo
echo "${COLOR_YELLOW}uv.lock does not match the dependency manifests; skipping the locked third-party installation.${COLOR_RESET}"
echo
return
fi
echo
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
echo
set -x
if ! uv sync --all-packages --frozen --group ci-image \
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python; then
if ! "${frozen_sync[@]}"; then
set +x
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
echo
echo "${COLOR_RED}Failing because the locked third-party installation failed although uv.lock matches the dependency manifests.${COLOR_RESET}"
echo
exit 1
fi
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
echo
echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}"
Expand Down Expand Up @@ -1355,6 +1380,10 @@ function install_airflow_when_building_images() {
install_from_external_spec
fi
set +x
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
# Packaging tools and 'pip check' are handled by the installation that adds the workspace packages
return
fi
common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand Down Expand Up @@ -1992,6 +2021,22 @@ fi
exec /bin/bash "${@}"
EOF

# Extract manifests separately so COPY below depends on their contents, not all source files.
FROM ${BASE_IMAGE} as dependency-manifests
WORKDIR /source
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"]
RUN --mount=type=bind,target=/source \
manifest_list="$(mktemp)" && \
trap 'rm -f "${manifest_list}"' EXIT && \
find . -name pyproject.toml -type f -print0 > "${manifest_list}" && \
mkdir -p /dependency-manifests && \
while IFS= read -r -d '' manifest; do \
relative_path="${manifest#./}"; \
mkdir -p "/dependency-manifests/$(dirname "${relative_path}")" || exit 1; \
cp "${manifest}" "/dependency-manifests/${relative_path}" || exit 1; \
done < "${manifest_list}" && \
if [ -f uv.lock ]; then cp uv.lock /dependency-manifests/ || exit 1; fi

FROM ${BASE_IMAGE} as main

# Nolog bash flag is currently ignored - but you can replace it with other flags (for example
Expand Down Expand Up @@ -2233,13 +2278,23 @@ RUN bash /scripts/docker/install_packaging_tools.sh

COPY --from=scripts install_airflow_when_building_images.sh /scripts/docker/

COPY --from=dependency-manifests /dependency-manifests/ ${AIRFLOW_SOURCES}/
ARG UPGRADE_RANDOM_INDICATOR_STRING=""
# Only preinstall locked third-party dependencies, with the same uv sync as the full installation.
# A uv.lock that does not match the manifests only skips this warm-up: the full sync after COPY still
# installs workspace packages from the current checkout and retains its resolution fallback. Any other
# failure would fail the full sync as well, so it fails the build here.
RUN --mount=type=cache,id=ci-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/root/.cache/ \
if [[ -z "${UPGRADE_RANDOM_INDICATOR_STRING}" ]]; then \
INSTALL_LOCKED_THIRD_PARTY_ONLY="true" bash /scripts/docker/install_airflow_when_building_images.sh; \
fi

# We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not
# copying over stuff that is accidentally generated or that we do not need (such as egg-info)
# if you want to add something that is missing and you expect to see it in the image you can
# add it with ! in .dockerignore next to the airflow, test etc. directories there
COPY . ${AIRFLOW_SOURCES}/

ARG UPGRADE_RANDOM_INDICATOR_STRING=""
ARG VERSION_SUFFIX=""

ENV UPGRADE_RANDOM_INDICATOR_STRING=${UPGRADE_RANDOM_INDICATOR_STRING} \
Expand Down
48 changes: 41 additions & 7 deletions scripts/docker/install_airflow_when_building_images.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@
# UPGRADE_RANDOM_INDICATOR_STRING - if set with random value determines that dependency upgrade should be done
# with highest resolution. It is random in order to trigger cache
# invalidation when building images
# INSTALL_LOCKED_THIRD_PARTY_ONLY - if "true", only the third-party dependencies locked in uv.lock are installed
# from sources (--no-install-workspace), so only the pyproject.toml files
# and uv.lock are needed. When a frozen dry run shows that uv.lock does not
# match them, or for upgrade builds, nothing is installed; any other
# failure fails. The steps that need the workspace packages are skipped.
#
# shellcheck shell=bash disable=SC2086
# shellcheck source=scripts/docker/common.sh
Expand All @@ -41,6 +46,15 @@ function install_from_sources() {
if [[ ${VIRTUAL_ENV=} != "" ]]; then
extra_sync_flags="--active"
fi
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
echo
echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew; skipping the locked third-party installation.${COLOR_RESET}"
echo
return
fi
extra_sync_flags="${extra_sync_flags} --no-install-workspace"
Comment thread
zozo123 marked this conversation as resolved.
Resolved
fi
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then
set +x
Expand All @@ -61,19 +75,35 @@ function install_from_sources() {
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python
else
set +x
echo
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
echo
# Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving.
# --no-binary-package is needed in order to avoid libxml and xmlsec using different version of
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
# shellcheck disable=SC2206 # extra_sync_flags holds several flags
local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec
--no-python-downloads --no-managed-python)
set +x
# Without the workspace sources a lock that does not match the manifests cannot be re-resolved;
# the installation that adds the workspace packages does that later.
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && ! "${frozen_sync[@]}" --dry-run; then
echo
echo "${COLOR_YELLOW}uv.lock does not match the dependency manifests; skipping the locked third-party installation.${COLOR_RESET}"
echo
return
fi
echo
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
echo
set -x
if ! uv sync --all-packages --frozen --group ci-image \
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python; then
if ! "${frozen_sync[@]}"; then
set +x
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
echo
echo "${COLOR_RED}Failing because the locked third-party installation failed although uv.lock matches the dependency manifests.${COLOR_RESET}"
echo
exit 1
fi
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
echo
echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}"
Expand Down Expand Up @@ -163,6 +193,10 @@ function install_airflow_when_building_images() {
install_from_external_spec
fi
set +x
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
# Packaging tools and 'pip check' are handled by the installation that adds the workspace packages
return
fi
common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand Down
Loading