Fix preprocessor cache mode with distributed builds

...and disable the workaround that disabled the former in case of
the latter.
Under the following conditions:
- Build with preprocessor cache mode enabled
- Distributed build
- Preprocessor cache hit
- Main cache miss
the compile that was run as "fallback" after the cache miss did
not have preprocessed sources passed to it since skipping the
preprocessing step is the whole point of preprocessor cache mode.
But local preprocessing is what enables distributed compilation.
To fix this, when the problem occurs, recursively re-invoke
get_cached_or_compile() with ForceRecache(*) so that the
preprocessor cache hit is (mostly) ignored, which makes
get_cached_or_compile() invoke the local preprocessing codepath,
which yields code suitable for distributed compilation.

(*) which also prevents further recursion

Author: ahartmetz

src/compiler/c.rs

@@ -401,8 +401,8 @@ where
         }
 
         let use_preprocessor_cache_mode = {
-            let can_use_preprocessor_cache_mode = !may_dist
-                && preprocessor_cache_mode_config.use_preprocessor_cache_mode
+            let can_use_preprocessor_cache_mode =
+                preprocessor_cache_mode_config.use_preprocessor_cache_mode
                 && !too_hard_for_preprocessor_cache_mode;
 
             let mut use_preprocessor_cache_mode = can_use_preprocessor_cache_mode;
@@ -429,7 +429,6 @@ where
             use_preprocessor_cache_mode
         };
 
-        // Disable preprocessor cache when doing distributed compilation
         let mut preprocessor_key = if use_preprocessor_cache_mode {
             preprocessor_cache_entry_hash_key(
                 &self.executable_digest,
@@ -492,7 +491,7 @@ where
                                 compilation: Box::new(CCompilation {
                                     parsed_args: self.parsed_args.to_owned(),
                                     #[cfg(feature = "dist-client")]
-                                    preprocessed_input: vec![],
+                                    preprocessed_input: PREPROCESSING_SKIPPED_COMPILE_POISON.to_vec(),
                                     executable: self.executable.to_owned(),
                                     compiler: self.compiler.to_owned(),
                                     cwd: cwd.to_owned(),
@@ -1143,6 +1142,12 @@ fn include_is_too_new(
     false
 }
 
+// Used as "preprocessed code" when no preprocessing took place to immediately produce an error (which
+// should never happen / be caught earlier though!). Previously, an empty u8 vector was used, which is
+// unfortunately a valid C/C++ program and caused errors that only surfaced when linking.
+#[cfg(feature = "dist-client")]
+const PREPROCESSING_SKIPPED_COMPILE_POISON: &[u8] = b"([{SCCACHE -*-* INVALID_C_CPP_CODE([{\"";
+
 impl<T: CommandCreatorSync, I: CCompilerImpl> Compilation<T> for CCompilation<I> {
     fn generate_compile_commands(
         &self,
@@ -1194,6 +1199,11 @@ impl<T: CommandCreatorSync, I: CCompilerImpl> Compilation<T> for CCompilation<I>
         Ok((inputs_packager, toolchain_packager, outputs_rewriter))
     }
 
+    #[cfg(feature = "dist-client")]
+    fn is_preprocessed_for_distribution(&self) -> bool {
+        self.preprocessed_input != PREPROCESSING_SKIPPED_COMPILE_POISON
+    }
+
     fn outputs<'a>(&'a self) -> Box<dyn Iterator<Item = FileObjectSource> + 'a> {
         Box::new(
             self.parsed_args

src/compiler/compiler.rs

@@ -567,6 +567,29 @@ where
                 // Cache miss, so compile it.
                 let start = Instant::now();
 
+                #[cfg(feature = "dist-client")]
+                if may_dist
+                    && !compilation.is_preprocessed_for_distribution()
+                    && cache_control == CacheControl::Default
+                {
+                    // This compilation only had enough information to find and use a cache entry (or to
+                    // run a local compile, which doesn't need locally preprocessed code).
+                    // For distributed compilation, the local preprocessing step still needs to be done.
+                    return self
+                        .get_cached_or_compile(
+                            service,
+                            dist_client,
+                            creator,
+                            storage,
+                            arguments,
+                            cwd,
+                            env_vars,
+                            CacheControl::ForceRecache,
+                            pool,
+                        )
+                        .await;
+                }
+
                 let (cacheable, dist_type, compiler_result) = dist_or_local_compile(
                     service,
                     dist_client,
@@ -932,6 +955,11 @@ where
         _path_transformer: dist::PathTransformer,
     ) -> Result<DistPackagers>;
 
+    #[cfg(feature = "dist-client")]
+    fn is_preprocessed_for_distribution(&self) -> bool {
+        true
+    }
+
     /// Returns an iterator over the results of this compilation.
     ///
     /// Each item is a descriptive (and unique) name of the output paired with