Build source and documentation in GitHub Actions (#71)

sethmlarson · ezio-melotti · ambv · web-flow · commit c1fceb722283 · 2024-03-19T22:36:53.000+01:00
Co-authored-by: Ezio Melotti &lt;ezio.melotti@gmail.com&gt;
Co-authored-by: Łukasz Langa &lt;lukasz@langa.pl&gt;
diff --git a/.github/workflows/source-and-docs-release.yml b/.github/workflows/source-and-docs-release.yml
@@ -0,0 +1,100 @@
+on:
+  workflow_dispatch:
+    inputs:
+      git_remote:
+        type: choice
+        description: "Git remote to checkout"
+        options:
+          - python
+          - Yhg1s
+          - pablogsal
+          - ambv
+      git_commit:
+        type: string
+        description: "Git commit to target for the release. Must use the full commit SHA, not the short ID"
+      cpython_release:
+        type: string
+        description: "CPython release number (ie '3.11.5', note without the 'v' prefix)"
+
+name: "Build Python source and docs artifacts"
+
+jobs:
+  source-and-docs:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: "Checkout python/release-tools"
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: "Checkout ${{ inputs.git_remote }}/cpython"
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          repository: "${{ inputs.git_remote }}/cpython"
+          ref: "v${{ inputs.cpython_release }}"
+          path: "cpython"
+
+      - name: "Verify CPython commit matches tag"
+        run: |
+          if [[ "${{ inputs.git_commit }}" != "$(cd cpython && git rev-parse HEAD)" ]]; then
+            echo "expected git commit ('${{ inputs.git_commit }}') didn't match tagged commit ('$(git rev-parse HEAD)')"
+            exit 1
+          fi
+
+      - name: "Setup Python"
+        uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
+        with:
+          python-version: 3.11
+
+      - name: "Install source dependencies"
+        run: |
+          python -m pip install --no-deps \
+            -r requirements.txt
+
+      - name: "Install docs dependencies"
+        # Docs aren't built for alpha or beta releases.
+        if: ${{ !(contains(inputs.cpython_release, 'a') || contains(inputs.cpython_release, 'b')) }}
+        run: |
+          python -m pip install \
+            -r cpython/Doc/requirements.txt
+
+          sudo apt-get update
+          sudo apt-get install --yes --no-install-recommends \
+            latexmk texlive-xetex xindy texinfo texlive-latex-base \
+            texlive-fonts-recommended texlive-fonts-extra \
+            texlive-full
+
+      - name: "Build Python release artifacts"
+        run: |
+          cd cpython
+          python ../release.py --export ${{ inputs.cpython_release }}
+
+      - name: "Test Python source tarballs"
+        run: |
+          mkdir -p ./tmp/installation/
+          cp cpython/${{ inputs.cpython_release }}/src/Python-${{ inputs.cpython_release }}.tgz ./tmp/
+          cd tmp/
+          tar xvf Python-${{ inputs.cpython_release }}.tgz
+          cd Python-${{ inputs.cpython_release }}
+
+          ./configure --prefix=$(realpath '../installation/')
+          make -j
+          make install -j
+
+          cd ../installation
+          ./bin/python3 -m test -uall
+
+      - name: "Upload the source artifacts"
+        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
+        with:
+          name: source
+          path: |
+            cpython/${{ inputs.cpython_release }}/src
+
+      - name: "Upload the docs artifacts"
+        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
+        # Conditionally run this step if there is a 'docs/' directory.
+        # Docs aren't built for alpha or beta releases.
+        if: ${{ hashFiles('cpython/${{ inputs.cpython_release }}/docs') != '' }}
+        with:
+          name: docs
+          path: |
+            cpython/${{ inputs.cpython_release }}/docs
diff --git a/release.py b/release.py
@@ -277,19 +277,6 @@ def tarball(source, clamp_mtime):
     print('  %s  %8s  %s' % (
         checksum_xz.hexdigest(), int(os.path.getsize(xz)), xz))
 
-    print('Signing tarballs with GPG')
-    uid = os.environ.get("GPG_KEY_FOR_RELEASE")
-    if not uid:
-        print('List of available private keys:')
-        run_cmd(['gpg -K | grep -A 1 "^sec"'], shell=True)
-        uid = input('Please enter key ID to use for signing: ')
-    run_cmd(['gpg', '-bas', '-u', uid, tgz])
-    run_cmd(['gpg', '-bas', '-u', uid, xz])
-
-    print('Signing tarballs with Sigstore')
-    run_cmd(['python3', '-m', 'sigstore', 'sign',
-             '--oidc-disable-ambient-providers', tgz, xz], shell=False)
-
 
 def export(tag, silent=False):
     make_dist(tag.text)
diff --git a/run_release.py b/run_release.py
@@ -297,7 +297,6 @@ def check_tool(db: DbfilenameShelf, tool: str) -> None:
 
 
 check_git = functools.partial(check_tool, tool="git")
-check_latexmk = functools.partial(check_tool, tool="latexmk")
 check_make = functools.partial(check_tool, tool="make")
 check_blurb = functools.partial(check_tool, tool="blurb")
 check_autoconf = functools.partial(check_tool, tool="autoconf")
@@ -474,39 +473,63 @@ def create_tag(db: DbfilenameShelf) -> None:
     )
 
 
-def build_release_artifacts(db: DbfilenameShelf) -> None:
-    with cd(db["git_repo"]):
-        release_mod.export(db["release"])
-
-
-def test_release_artifacts(db: DbfilenameShelf) -> None:
-    with tempfile.TemporaryDirectory() as tmppath:
-        the_dir = pathlib.Path(tmppath)
-        the_dir.mkdir(exist_ok=True)
-        filename = f"Python-{db['release']}"
-        tarball = f"Python-{db['release']}.tgz"
-        shutil.copy2(
-            db["git_repo"] / str(db["release"]) / "src" / tarball,
-            the_dir / tarball,
-        )
-        subprocess.check_call(["tar", "xvf", tarball], cwd=the_dir)
-        subprocess.check_call(
-            ["./configure", "--prefix", str(the_dir / "installation")],
-            cwd=the_dir / filename,
-        )
-        subprocess.check_call(["make", "-j"], cwd=the_dir / filename)
-        subprocess.check_call(["make", "install", "-j"], cwd=the_dir / filename)
-        process = subprocess.run(
-            ["./bin/python3", "-m", "test", "-uall"],
-            cwd=str(the_dir / "installation"),
-            text=True,
-        )
+def wait_for_source_and_docs_artifacts(db: DbfilenameShelf) -> None:
+    # Determine if we need to wait for docs or only source artifacts.
+    release_tag = db["release"]
+    should_wait_for_docs = release_tag.is_final or release_tag.is_release_candiate
 
-    if process.returncode == 0:
-        return
+    # Create the directory so it's easier to place the artifacts there.
+    release_path = pathlib.Path(db["git_repo"] / str(release_tag))
+    release_path.mkdir(parents=True, exist_ok=True)
 
-    if not ask_question("Some test_failed! Do you want to continue?"):
-        raise ReleaseException("Test failed!")
+    # Build the list of filepaths we're expecting.
+    wait_for_paths = [
+        release_path / "src" / f"Python-{release_tag}.tgz",
+        release_path / "src" / f"Python-{release_tag}.tar.xz"
+    ]
+    if should_wait_for_docs:
+        docs_path = release_path / "docs"
+        wait_for_paths.extend([
+            docs_path / f"python-{release_tag}-docs.epub",
+            docs_path / f"python-{release_tag}-docs-html.tar.bz2",
+            docs_path / f"python-{release_tag}-docs-html.zip",
+            docs_path / f"python-{release_tag}-docs-pdf-a4.tar.bz2",
+            docs_path / f"python-{release_tag}-docs-pdf-a4.zip",
+            docs_path / f"python-{release_tag}-docs-pdf-letter.tar.bz2",
+            docs_path / f"python-{release_tag}-docs-pdf-letter.zip",
+            docs_path / f"python-{release_tag}-docs-texinfo.tar.bz2",
+            docs_path / f"python-{release_tag}-docs-texinfo.zip",
+            docs_path / f"python-{release_tag}-docs-text.tar.bz2",
+            docs_path / f"python-{release_tag}-docs-text.zip",
+        ])
+
+    print(f"Waiting for source{' and docs' if should_wait_for_docs else ''} artifacts to be built")
+    print(f"Artifacts should be placed at '{release_path}':")
+    for path in wait_for_paths:
+        print(f"- '{os.path.relpath(path, release_path)}'")
+
+    while not all(path.exists() for path in wait_for_paths):
+        time.sleep(1)
+
+
+def sign_source_artifacts(db: DbfilenameShelf) -> None:
+    print('Signing tarballs with GPG')
+    uid = os.environ.get("GPG_KEY_FOR_RELEASE")
+    if not uid:
+        print('List of available private keys:')
+        subprocess.check_call('gpg -K | grep -A 1 "^sec"', shell=True)
+        uid = input('Please enter key ID to use for signing: ')
+
+    tarballs_path = pathlib.Path(db["git_repo"] / str(db["release"]) / "src")
+    tgz = str(tarballs_path / ("Python-%s.tgz" % db["release"]))
+    xz = str(tarballs_path / ("Python-%s.tar.xz" % db["release"]))
+
+    subprocess.check_call(['gpg', '-bas', '-u', uid, tgz])
+    subprocess.check_call(['gpg', '-bas', '-u', uid, xz])
+
+    print('Signing tarballs with Sigstore')
+    subprocess.check_call(['python3', '-m', 'sigstore', 'sign',
+                           '--oidc-disable-ambient-providers', tgz, xz])
 
 
 def build_sbom_artifacts(db):
@@ -698,9 +721,55 @@ def execute_command(command):
     execute_command(f"find {destination} -type f -exec chmod 664 {{}} \\;")
 
 
+def start_build_of_source_and_docs(db: DbfilenameShelf) -> None:
+    # Get the git commit SHA for the tag
+    commit_sha = subprocess.check_output(
+        ["git", "rev-list", "-n", "1", db["release"].gitname],
+        cwd=db["git_repo"]
+    ).decode().strip()
+
+    # Get the owner of the GitHub repo (first path segment in a 'github.com' remote URL)
+    # This works for both 'https' and 'ssh' style remote URLs.
+    origin_remote_url = subprocess.check_output(
+        ["git", "ls-remote", "--get-url", "origin"],
+        cwd=db["git_repo"]
+    ).decode().strip()
+    match = re.match(r"github\.com/([^/]+)/", origin_remote_url)
+    if not match:
+        raise ReleaseException(f"Could not parse GitHub owner from 'origin' remote URL: {origin_remote_url}")
+    origin_remote_github_owner = match.group(1)
+
+    # We ask for human verification at this point since this commit SHA is 'locked in'
+    print()
+    print(f"Go to https://github.com/{origin_remote_github_owner}/cpython/commit/{commit_sha}")
+    print("- Ensure that there is no warning that the commit does not belong to this repository.")
+    print("- Ensure that the commit diff does not contain any unexpected changes.")
+    print("- For the next step, ensure the commit SHA matches the one you verified on GitHub in this step.")
+    print()
+    if not ask_question(
+        "Have you verified the release commit hasn't been tampered with on GitHub?"
+    ):
+        raise ReleaseException("Commit must be visually reviewed before starting build")
+
+    # After visually confirming the release manager can start the build process
+    # with the known good commit SHA.
+    print()
+    print("Go to https://github.com/python/release-tools/actions/workflows/source-and-docs-release.yml")
+    print("Select 'Run workflow' and enter the following values:")
+    print(f"- Git remote to checkout: {origin_remote_github_owner}")
+    print(f"- Git commit to target for the release: {commit_sha}")
+    print(f"- CPython release number: {db['release']}")
+    print()
+
+    if not ask_question(
+        "Have you started the source and docs build?"
+    ):
+        raise ReleaseException("Source and docs build must be started")
+
+
 def send_email_to_platform_release_managers(db: DbfilenameShelf) -> None:
     if not ask_question(
-        "Have you notified the platform release managers about the availability of artifacts?"
+        "Have you notified the platform release managers about the availability of the commit SHA and tag?"
     ):
         raise ReleaseException("Platform release managers muy be notified")
 
@@ -1040,9 +1109,9 @@ def _api_key(api_key):
     )
     args = parser.parse_args()
     auth_key = args.auth_key or os.getenv("AUTH_INFO")
+    assert isinstance(auth_key, str), "We need an AUTH_INFO env var or --auth-key"
     tasks = [
         Task(check_git, "Checking git is available"),
-        Task(check_latexmk, "Checking latexmk is available"),
         Task(check_make, "Checking make is available"),
         Task(check_blurb, "Checking blurb is available"),
         Task(check_docker, "Checking docker is available"),
@@ -1061,18 +1130,19 @@ def _api_key(api_key):
         Task(bump_version, "Bump version"),
         Task(check_cpython_repo_is_clean, "Checking git repository is clean"),
         Task(create_tag, "Create tag"),
-        Task(build_release_artifacts, "Building release artifacts"),
-        Task(test_release_artifacts, "Test release artifacts"),
+        Task(push_to_local_fork, "Push new tags and branches to private fork"),
+        Task(start_build_of_source_and_docs, "Start the builds for source and docs artifacts"),
+        Task(
+            send_email_to_platform_release_managers,
+            "Platform release managers have been notified of the commit SHA",
+        ),
+        Task(wait_for_source_and_docs_artifacts, "Wait for source and docs artifacts to build"),
         Task(build_sbom_artifacts, "Building SBOM artifacts"),
+        Task(sign_source_artifacts, "Sign source artifacts"),
         Task(upload_files_to_server, "Upload files to the PSF server"),
         Task(place_files_in_download_folder, "Place files in the download folder"),
         Task(upload_docs_to_the_docs_server, "Upload docs to the PSF docs server"),
         Task(unpack_docs_in_the_docs_server, "Place docs files in the docs folder"),
-        Task(push_to_local_fork, "Push new tags and branches to private fork"),
-        Task(
-            send_email_to_platform_release_managers,
-            "Platform release managers have been notified of the release artifacts",
-        ),
         Task(wait_util_all_files_are_in_folder, "Wait until all files are ready"),
         Task(create_release_object_in_db, "The django release object has been created"),
         Task(post_release_merge, "Merge the tag into the release branch"),
