Treat PR builder setup more like regular builder setup

Author: zware

master/custom/__init__.py

@@ -2,4 +2,5 @@
 # The Git branch is called "main", but we give it a different name in buildbot.
 # See git_branches in master/master.cfg.
 MAIN_BRANCH_NAME = "3.x"
+PR_BRANCH_PLACEHOLDER = "PR"
 JUNIT_FILENAME = "test-results.xml"

master/custom/factories.py

@@ -9,7 +9,7 @@
 
 from buildbot.plugins import util
 
-from . import (MAIN_BRANCH_VERSION, MAIN_BRANCH_NAME,
+from . import (MAIN_BRANCH_VERSION, MAIN_BRANCH_NAME, PR_BRANCH_PLACEHOLDER,
                JUNIT_FILENAME)
 from .steps import (
     Test,
@@ -151,7 +151,7 @@ def setup(self, parallel_processes, branch, test_with_PTY=False, **kwargs):
             env=self.test_environ,
             **oot_kwargs
         ))
-        if branch not in ("3",) and not has_option("-R", self.testFlags):
+        if branch not in {PR_BRANCH_PLACEHOLDER} and not has_option("-R", self.testFlags):
             filename = JUNIT_FILENAME
             if self.build_out_of_tree:
                 filename = os.path.join(out_of_tree_dir, filename)
@@ -206,7 +206,7 @@ class UnixInstalledBuild(BaseBuild):
     def setup(self, parallel_processes, branch, test_with_PTY=False, **kwargs):
         if branch == MAIN_BRANCH_NAME:
             branch = MAIN_BRANCH_VERSION
-        elif branch == "custom":
+        elif branch == PR_BRANCH_PLACEHOLDER:
             branch = "3"
         installed_python = f"./target/bin/python{branch}"
         self.addStep(
@@ -643,7 +643,7 @@ def setup(self, parallel_processes, branch, **kwargs):
             command=test_command,
             timeout=step_timeout(self.test_timeout),
         ))
-        if branch not in ("3",) and not has_option("-R", self.testFlags):
+        if branch not in {PR_BRANCH_PLACEHOLDER} and not has_option("-R", self.testFlags):
             self.addStep(UploadTestResults(branch))
         self.addStep(Clean(command=clean_command))
 
@@ -859,7 +859,7 @@ def setup(self, parallel_processes, branch, test_with_PTY=False, **kwargs):
                 env=self.test_environ,
                 workdir=oot_host_path,
             ))
-            if branch not in ("3",) and not has_option("-R", self.testFlags):
+            if branch not in {PR_BRANCH_PLACEHOLDER} and not has_option("-R", self.testFlags):
                 filename = os.path.join(oot_host_path, JUNIT_FILENAME)
                 self.addStep(UploadTestResults(branch, filename=filename))
         self.addStep(
@@ -998,7 +998,7 @@ def setup(self, parallel_processes, branch, test_with_PTY=False, **kwargs):
                 workdir=host_path,
             )
         )
-        if branch not in ("3",) and not has_option("-R", self.testFlags):
+        if branch not in {PR_BRANCH_PLACEHOLDER} and not has_option("-R", self.testFlags):
             filename = os.path.join(host_path, JUNIT_FILENAME)
             self.addStep(UploadTestResults(branch, filename=filename))
 

master/master.cfg

@@ -31,7 +31,7 @@ for k in list(sys.modules):
     if k.split(".")[0] in ["custom"]:
         sys.modules.pop(k)
 
-from custom import MAIN_BRANCH_NAME  # noqa: E402
+from custom import MAIN_BRANCH_NAME, PR_BRANCH_PLACEHOLDER  # noqa: E402
 from custom.auth import set_up_authorization  # noqa: E402
 from custom.email_formatter import MESSAGE_FORMATTER  # noqa: E402
 from custom.pr_reporter import GitHubPullRequestReporter  # noqa: E402
@@ -41,7 +41,7 @@ from custom.pr_testing import (  # noqa: E402
     should_pr_be_tested,
 )
 from custom.settings import Settings  # noqa: E402
-from custom.steps import Git, GitHub  # noqa: E402
+from custom.steps import GitHub  # noqa: E402
 from custom.workers import get_workers  # noqa: E402
 from custom.schedulers import GitHubPrScheduler # noqa: E402
 from custom.release_dashboard import get_release_status_app    # noqa: E402
@@ -136,9 +136,10 @@ git_branches = [
     (git_url, "3.12", "3.12"),
     (git_url, "3.11", "3.11"),
     (git_url, "3.10", "3.10"),
+    (git_url, PR_BRANCH_PLACEHOLDER, "HEAD"),
 ]
 
-# common Git() and GitHub() keyword arguments
+# common Git/GitHub step keyword arguments
 GIT_KWDS = {
     "timeout": 3600,
     # "git clean -fdx": remove all files which are not tracked by Git,
@@ -233,6 +234,7 @@ mail_status_builders = []
 
 for git_url, branchname, git_branch in git_branches:
     buildernames = []
+    stablebuildernames = []
     refleakbuildernames = []
     for name, worker, buildfactory, stability, tier in BUILDERS:
         if any(
@@ -248,7 +250,7 @@ for git_url, branchname, git_branch in git_branches:
             continue
 
         buildername = name + " " + branchname
-        source = Git(repourl=git_url, branch=git_branch, **GIT_KWDS)
+        source = GitHub(repourl=git_url, branch=git_branch, **GIT_KWDS)
         f = buildfactory(
             source,
             parallel_processes=worker.parallel_processes,
@@ -283,9 +285,7 @@ for git_url, branchname, git_branch in git_branches:
         # (all these lists are the same now, but we might need to
         # diverge gain later.)
         if stability == STABLE:
-            mail_status_builders.append(buildername)
-            github_status_builders.append(buildername)
-            release_status_builders.append(buildername)
+            stablebuildernames.append(buildername)
 
         builder = util.BuilderConfig(
             name=buildername,
@@ -309,90 +309,44 @@ for git_url, branchname, git_branch in git_branches:
 
         c["builders"].append(builder)
 
-    c["schedulers"].append(
-        schedulers.SingleBranchScheduler(
-            name=branchname,
-            change_filter=util.ChangeFilter(branch=git_branch),
-            treeStableTimer=30,  # seconds
-            builderNames=buildernames,
-            fileIsImportant=is_important_change,
+    if branchname == PR_BRANCH_PLACEHOLDER:
+        all_pull_request_builders = buildernames
+        c["schedulers"].append(
+            GitHubPrScheduler(
+                name="pull-request-scheduler",
+                change_filter=util.ChangeFilter(filter_fn=should_pr_be_tested),
+                treeStableTimer=30,  # seconds
+                builderNames=buildernames,
+                stable_builder_names=set(stablebuildernames),
+            )
         )
-    )
-    if refleakbuildernames:
+    else:
+        mail_status_builders.extend(stablebuildernames)
+        github_status_builders.extend(stablebuildernames)
+        release_status_builders.extend(stablebuildernames)
         c["schedulers"].append(
             schedulers.SingleBranchScheduler(
-                name=branchname + "-refleak",
+                name=branchname,
                 change_filter=util.ChangeFilter(branch=git_branch),
-                # Wait this many seconds for no commits before starting a build
-                # NB: During extremely busy times, this can cause the builders
-                # to never actually fire.  The current expectation is that it
-                # won't ever actually be that busy, but we need to keep an eye
-                # on that.
-                treeStableTimer=1 * 60 * 60,  # h * m * s
-                builderNames=refleakbuildernames,
+                treeStableTimer=30,  # seconds
+                builderNames=buildernames,
+                fileIsImportant=is_important_change,
             )
         )
-
-
-# Set up Pull Request builders
-
-stable_pull_request_builders = []
-all_pull_request_builders = []
-
-for name, worker, buildfactory, stability, tier in BUILDERS:
-    buildername = f"{name} PR"
-
-    source = GitHub(repourl=git_url, **GIT_KWDS)
-
-    f = buildfactory(
-        source,
-        parallel_processes=worker.parallel_processes,
-        # Use the same downstream branch names as the "custom"
-        # builder (check what the factories are doing with this
-        # parameter for more info).
-        branch="3",
-        **worker.extra_factory_args,
-    )
-
-    tags = ["PullRequest", stability, *getattr(f, "tags", [])]
-    if tier:
-        tags.append(tier)
-
-    # Don't use the WASI buildbot meant for 3.11 and 3.12 on PRs;
-    # it's tier 3 only and getting it to work on PRs against `main`
-    # is too much work.
-    if "wasi" in tags and "nondebug" in tags:
-        continue
-
-    all_pull_request_builders.append(buildername)
-    if stability == STABLE:
-        stable_pull_request_builders.append(buildername)
-
-    builder = util.BuilderConfig(
-        name=buildername,
-        workernames=[worker.name],
-        builddir="%s.%s%s" % ("pull_request", worker.name, getattr(f, "buildersuffix", "")),
-        factory=f,
-        tags=tags,
-        locks=[cpulock.access("counting")],
-    )
-
-    # This worker runs pyperformance at 12am. If a build is scheduled between
-    # 10pm and 2am, it will be delayed at 2am.
-    if worker.name == "diegorusso-aarch64-bigmem":
-        builder.canStartBuild = no_builds_between("22:00", "2:00")
-
-    c["builders"].append(builder)
-
-c["schedulers"].append(
-    GitHubPrScheduler(
-        name="pull-request-scheduler",
-        change_filter=util.ChangeFilter(filter_fn=should_pr_be_tested),
-        treeStableTimer=30,  # seconds
-        builderNames=all_pull_request_builders,
-        stable_builder_names=set(stable_pull_request_builders),
-    )
-)
+        if refleakbuildernames:
+            c["schedulers"].append(
+                schedulers.SingleBranchScheduler(
+                    name=branchname + "-refleak",
+                    change_filter=util.ChangeFilter(branch=git_branch),
+                    # Wait this many seconds for no commits before starting a build
+                    # NB: During extremely busy times, this can cause the builders
+                    # to never actually fire.  The current expectation is that it
+                    # won't ever actually be that busy, but we need to keep an eye
+                    # on that.
+                    treeStableTimer=1 * 60 * 60,  # h * m * s
+                    builderNames=refleakbuildernames,
+                )
+            )
 
 
 # Set up aditional schedulers