Copy fuzzer script logs to correct buckets (#5099)

Handler for `store_fuzzer_run_results` in postprocess (#3406) was not
adding the fuzzer script logs to the correct structured buckets, as
expected by the fuzzer page on the UI. This PR tries to fix it by
copying the data from the blob store to the expected buckets during
postprocess.

b/361867455

Author: ViniciustCosta

src/clusterfuzz/_internal/bot/tasks/utasks/fuzz_task.py

@@ -784,18 +784,33 @@ def preprocess_store_fuzzer_run_results(fuzz_task_input):
   fuzz_task_input.sample_testcase_upload_key = blobs.generate_new_blob_name()
   fuzz_task_input.sample_testcase_upload_url = blobs.get_signed_upload_url(
       fuzz_task_input.sample_testcase_upload_key)
-  script_log_upload_key = blobs.generate_new_blob_name()
+  fuzz_task_input.script_log_upload_key = blobs.generate_new_blob_name()
   fuzz_task_input.script_log_upload_url = blobs.get_signed_upload_url(
-      script_log_upload_key)
+      fuzz_task_input.script_log_upload_key)
 
 
 def postprocess_store_fuzzer_run_results(output):
   """Postprocess store_fuzzer_run_results."""
   if environment.is_engine_fuzzer_job(output.uworker_input.job_type):
     return
+  uworker_input = output.uworker_input
+
+  # Copy fuzzer logs to the structured bucket.
+  if uworker_input.fuzz_task_input.script_log_upload_key:
+    logs_blob_bucket = blobs.get_gcs_path(
+        uworker_input.fuzz_task_input.script_log_upload_key)
+    fuzzer_logs_bucket = fuzzer_logs.get_logs_gcs_path(
+        fuzzer_name=output.fuzz_task_output.fully_qualified_fuzzer_name)
+    try:
+      if not storage.copy_blob(logs_blob_bucket, fuzzer_logs_bucket):
+        raise FuzzTaskError('Copying fuzzer logs returned false.')
+    except:
+      logs.warning(
+          f'Failed copying fuzzer logs from blobs {logs_blob_bucket} to the'
+          f'structured bucket {fuzzer_logs_bucket}.')
+
   if not output.fuzz_task_output.fuzzer_run_results:
     return
-  uworker_input = output.uworker_input
   fuzzer = data_types.Fuzzer.query(
       data_types.Fuzzer.name == output.uworker_input.fuzzer_name).get()
   if not fuzzer:

src/clusterfuzz/_internal/metrics/fuzzer_logs.py

@@ -128,7 +128,8 @@ def upload_to_logs(bucket_name,
   """
   if signed_upload_url:
     if storage.upload_signed_url(contents, signed_upload_url):
-      logs.info('Uploaded file to logs bucket.')
+      logs.info(
+          'Uploaded file to logs bucket.', signed_upload_url=signed_upload_url)
     else:
       logs.error(
           'Failed to write file to logs bucket.',

src/clusterfuzz/_internal/protos/uworker_msg.proto

@@ -54,6 +54,7 @@ message FuzzTaskInput {
   repeated BlobUploadUrl crash_upload_urls = 7;
   repeated google.protobuf.Any trials = 8;
   optional string web_tests_url = 9;
+  optional string script_log_upload_key = 10;
 }
 
 message DataBundleCorpus {

src/clusterfuzz/_internal/protos/uworker_msg_pb2.py

@@ -252,6 +252,7 @@ class FuzzTaskInput(google.protobuf.message.Message):
     CRASH_UPLOAD_URLS_FIELD_NUMBER: builtins.int
     TRIALS_FIELD_NUMBER: builtins.int
     WEB_TESTS_URL_FIELD_NUMBER: builtins.int
+    SCRIPT_LOG_UPLOAD_KEY_FIELD_NUMBER: builtins.int
     sample_testcase_upload_key: builtins.str
     sample_testcase_upload_url: builtins.str
     script_log_upload_url: builtins.str
@@ -267,6 +268,7 @@ class FuzzTaskInput(google.protobuf.message.Message):
     @property
     def trials(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[google.protobuf.any_pb2.Any]: ...
     web_tests_url: builtins.str
+    script_log_upload_key: builtins.str
     def __init__(
         self,
         *,
@@ -279,9 +281,10 @@ class FuzzTaskInput(google.protobuf.message.Message):
         crash_upload_urls: collections.abc.Iterable[global___BlobUploadUrl] | None = ...,
         trials: collections.abc.Iterable[google.protobuf.any_pb2.Any] | None = ...,
         web_tests_url: builtins.str | None = ...,
+        script_log_upload_key: builtins.str | None = ...,
     ) -> None: ...
-    def HasField(self, field_name: typing_extensions.Literal["_corpus", b"_corpus", "_fuzz_target", b"_fuzz_target", "_sample_testcase_upload_key", b"_sample_testcase_upload_key", "_sample_testcase_upload_url", b"_sample_testcase_upload_url", "_script_log_upload_url", b"_script_log_upload_url", "_web_tests_url", b"_web_tests_url", "corpus", b"corpus", "fuzz_target", b"fuzz_target", "sample_testcase_upload_key", b"sample_testcase_upload_key", "sample_testcase_upload_url", b"sample_testcase_upload_url", "script_log_upload_url", b"script_log_upload_url", "web_tests_url", b"web_tests_url"]) -> builtins.bool: ...
-    def ClearField(self, field_name: typing_extensions.Literal["_corpus", b"_corpus", "_fuzz_target", b"_fuzz_target", "_sample_testcase_upload_key", b"_sample_testcase_upload_key", "_sample_testcase_upload_url", b"_sample_testcase_upload_url", "_script_log_upload_url", b"_script_log_upload_url", "_web_tests_url", b"_web_tests_url", "corpus", b"corpus", "crash_upload_urls", b"crash_upload_urls", "fuzz_target", b"fuzz_target", "global_blacklisted_functions", b"global_blacklisted_functions", "sample_testcase_upload_key", b"sample_testcase_upload_key", "sample_testcase_upload_url", b"sample_testcase_upload_url", "script_log_upload_url", b"script_log_upload_url", "trials", b"trials", "web_tests_url", b"web_tests_url"]) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["_corpus", b"_corpus", "_fuzz_target", b"_fuzz_target", "_sample_testcase_upload_key", b"_sample_testcase_upload_key", "_sample_testcase_upload_url", b"_sample_testcase_upload_url", "_script_log_upload_key", b"_script_log_upload_key", "_script_log_upload_url", b"_script_log_upload_url", "_web_tests_url", b"_web_tests_url", "corpus", b"corpus", "fuzz_target", b"fuzz_target", "sample_testcase_upload_key", b"sample_testcase_upload_key", "sample_testcase_upload_url", b"sample_testcase_upload_url", "script_log_upload_key", b"script_log_upload_key", "script_log_upload_url", b"script_log_upload_url", "web_tests_url", b"web_tests_url"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["_corpus", b"_corpus", "_fuzz_target", b"_fuzz_target", "_sample_testcase_upload_key", b"_sample_testcase_upload_key", "_sample_testcase_upload_url", b"_sample_testcase_upload_url", "_script_log_upload_key", b"_script_log_upload_key", "_script_log_upload_url", b"_script_log_upload_url", "_web_tests_url", b"_web_tests_url", "corpus", b"corpus", "crash_upload_urls", b"crash_upload_urls", "fuzz_target", b"fuzz_target", "global_blacklisted_functions", b"global_blacklisted_functions", "sample_testcase_upload_key", b"sample_testcase_upload_key", "sample_testcase_upload_url", b"sample_testcase_upload_url", "script_log_upload_key", b"script_log_upload_key", "script_log_upload_url", b"script_log_upload_url", "trials", b"trials", "web_tests_url", b"web_tests_url"]) -> None: ...
     @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_corpus", b"_corpus"]) -> typing_extensions.Literal["corpus"] | None: ...
     @typing.overload
@@ -291,6 +294,8 @@ class FuzzTaskInput(google.protobuf.message.Message):
     @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_sample_testcase_upload_url", b"_sample_testcase_upload_url"]) -> typing_extensions.Literal["sample_testcase_upload_url"] | None: ...
     @typing.overload
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["_script_log_upload_key", b"_script_log_upload_key"]) -> typing_extensions.Literal["script_log_upload_key"] | None: ...
+    @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_script_log_upload_url", b"_script_log_upload_url"]) -> typing_extensions.Literal["script_log_upload_url"] | None: ...
     @typing.overload
     def WhichOneof(self, oneof_group: typing_extensions.Literal["_web_tests_url", b"_web_tests_url"]) -> typing_extensions.Literal["web_tests_url"] | None: ...

src/clusterfuzz/_internal/protos/uworker_msg_pb2.pyi

@@ -1505,15 +1505,18 @@ class PreprocessStoreFuzzerRunResultsTest(unittest.TestCase):
   """Tests for preprocess_store_fuzzer_run_results."""
 
   SIGNED_URL = 'https://signed'
+  BLOB_KEY = 'blob_key'
 
   def setUp(self):
     helpers.patch(self, [
         'clusterfuzz._internal.google_cloud_utils.storage._sign_url',
         'clusterfuzz._internal.google_cloud_utils.blobs.get_signed_upload_url',
+        'clusterfuzz._internal.google_cloud_utils.blobs.generate_new_blob_name'
     ])
     self.mock._sign_url.side_effect = (
         lambda remote_path, method, minutes: remote_path)
     self.mock.get_signed_upload_url.return_value = self.SIGNED_URL
+    self.mock.generate_new_blob_name.return_value = self.BLOB_KEY
     helpers.patch_environ(self)
     os.environ['JOB_NAME'] = 'linux_chrome_asan'
 
@@ -1522,8 +1525,9 @@ def test_preprocess_store_fuzzer_run_results(self):
     fuzz_task.preprocess_store_fuzzer_run_results(fuzz_task_input)
     self.assertEqual(fuzz_task_input.sample_testcase_upload_url,
                      self.SIGNED_URL)
-
+    self.assertEqual(fuzz_task_input.sample_testcase_upload_key, self.BLOB_KEY)
     self.assertEqual(fuzz_task_input.script_log_upload_url, self.SIGNED_URL)
+    self.assertEqual(fuzz_task_input.script_log_upload_key, self.BLOB_KEY)
 
 
 @test_utils.with_cloud_emulators('datastore')