Rollup merge of #152768 - ZuseZ4:autodiff-in-ci-for-all-os, r=Kobzol

Enable autodiff in ci for all major os

*[View all comments](https://triagebot.infra.rust-lang.org/gh-comments/rust-lang/rust/pull/152768)*

Follow-up attempt to #140064 after moving autodiff to dlopen.
It covers Linux (x86_64+aarch64), MacOS (aarch64), Windows (mingw-llvm aarch64+x86_64)
The extra build time for Enzyme are 180.27s on our slowest runner (aarch64-apple).

The follow-up distribution via rustup probably still needs a small fix, see #151063 (comment)

Placing the downloaded libEnzyme artifact on my local linux under `~/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/lib` enables my nightly compiler to run autodiff.

r? @Kobzol

closes: #140064
closes: #151243
closes: #151063

Author: JonathanBrouwer

src/bootstrap/src/core/build_steps/llvm.rs

@@ -23,7 +23,7 @@ use crate::core::config::{Config, TargetSelection};
 use crate::utils::build_stamp::{BuildStamp, generate_smart_stamp_hash};
 use crate::utils::exec::command;
 use crate::utils::helpers::{
-    self, exe, get_clang_cl_resource_dir, t, unhashed_basename, up_to_date,
+    self, exe, get_clang_cl_resource_dir, libdir, t, unhashed_basename, up_to_date,
 };
 use crate::{CLang, GitRepo, Kind, trace};
 
@@ -561,7 +561,6 @@ impl Step for Llvm {
             }
         };
 
-        // FIXME(ZuseZ4): Do we need that for Enzyme too?
         // When building LLVM with LLVM_LINK_LLVM_DYLIB for macOS, an unversioned
         // libLLVM.dylib will be built. However, llvm-config will still look
         // for a versioned path like libLLVM-14.dylib. Manually create a symbolic
@@ -1167,7 +1166,7 @@ impl Step for Enzyme {
         let llvm_version_major = llvm::get_llvm_version_major(builder, &host_llvm_config);
         let lib_ext = std::env::consts::DLL_EXTENSION;
         let libenzyme = format!("libEnzyme-{llvm_version_major}");
-        let build_dir = out_dir.join("lib");
+        let build_dir = out_dir.join(libdir(target));
         let dylib = build_dir.join(&libenzyme).with_extension(lib_ext);
 
         trace!("checking build stamp to see if we need to rebuild enzyme artifacts");
@@ -1205,7 +1204,16 @@ impl Step for Enzyme {
         // hard to spot more relevant issues.
         let mut cflags = CcFlags::default();
         cflags.push_all("-Wno-deprecated");
-        configure_cmake(builder, target, &mut cfg, true, LdFlags::default(), cflags, &[]);
+
+        // Logic copied from `configure_llvm`
+        // ThinLTO is only available when building with LLVM, enabling LLD is required.
+        // Apple's linker ld64 supports ThinLTO out of the box though, so don't use LLD on Darwin.
+        let mut ldflags = LdFlags::default();
+        if builder.config.llvm_thin_lto && !target.contains("apple") {
+            ldflags.push_all("-fuse-ld=lld");
+        }
+
+        configure_cmake(builder, target, &mut cfg, true, ldflags, cflags, &[]);
 
         // Re-use the same flags as llvm to control the level of debug information
         // generated by Enzyme.

src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile

@@ -97,7 +97,7 @@ ENV RUST_CONFIGURE_ARGS \
 
 ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \
       ./build/$HOSTS/stage1-tools-bin/opt-dist linux-ci --  python3 ../x.py dist \
-      --host $HOSTS --target $HOSTS --include-default-paths build-manifest bootstrap
+      --host $HOSTS --target $HOSTS --include-default-paths build-manifest bootstrap enzyme
 
 ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=clang
 ENV LIBCURL_NO_PKG_CONFIG 1

src/ci/docker/host-x86_64/dist-x86_64-linux/dist.sh

@@ -9,6 +9,7 @@ python3 ../x.py build --set rust.debug=true opt-dist
     --include-default-paths \
     build-manifest \
     bootstrap \
+    enzyme \
     rustc_codegen_gcc
 
 # Use GCC for building GCC components, as it seems to behave badly when built with Clang

src/ci/github-actions/jobs.yml

@@ -504,7 +504,7 @@ auto:
   - name: dist-aarch64-apple
     env:
       SCRIPT: >-
-        ./x.py dist bootstrap
+        ./x.py dist bootstrap enzyme
         --include-default-paths
         --host=aarch64-apple-darwin
         --target=aarch64-apple-darwin
@@ -513,6 +513,7 @@ auto:
         --enable-sanitizers
         --enable-profiler
         --set rust.jemalloc
+        --set llvm.link-shared=true
         --set rust.lto=thin
         --set rust.codegen-units=1
       # Aarch64 tooling only needs to support macOS 11.0 and up as nothing else
@@ -706,7 +707,7 @@ auto:
   # i686 has no dedicated job, build it here because this job is fast
   - name: dist-aarch64-llvm-mingw
     env:
-      SCRIPT: python x.py dist bootstrap --include-default-paths
+      SCRIPT: python x.py dist bootstrap enzyme --include-default-paths
       RUST_CONFIGURE_ARGS: >-
         --build=aarch64-pc-windows-gnullvm
         --target=aarch64-pc-windows-gnullvm,i686-pc-windows-gnullvm
@@ -720,7 +721,7 @@ auto:
 
   - name: dist-x86_64-llvm-mingw
     env:
-      SCRIPT: python x.py dist bootstrap --include-default-paths
+      SCRIPT: python x.py dist bootstrap enzyme --include-default-paths
       RUST_CONFIGURE_ARGS: >-
         --build=x86_64-pc-windows-gnullvm
         --enable-full-tools