[GCP] Add ingest pipeline for dataproc (#4344)

Author: endorama

packages/gcp/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.11.10-beta.6"
+  changes:
+    - description: Add ingest pipeline for dataproc.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/4344
 - version: "2.11.10-beta.5"
   changes:
     - description: Add GCP loadbalancing ingest pipeline

packages/gcp/data_stream/dataproc/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,100 @@
+---
+description: Pipeline for parsing GCP Dataproc metrics.
+processors:
+  - rename:
+      field: gcp.metrics.batch.spark.executors.count
+      target_field: gcp.dataproc.batch.spark.executors.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.hdfs.datanodes.count
+      target_field: gcp.dataproc.cluster.hdfs.datanodes.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.hdfs.storage_capacity.value
+      target_field: gcp.dataproc.cluster.hdfs.storage_capacity.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.hdfs.storage_utilization.value
+      target_field: gcp.dataproc.cluster.hdfs.storage_utilization.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.hdfs.unhealthy_blocks.count
+      target_field: gcp.dataproc.cluster.hdfs.unhealthy_blocks.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.job.failed.count
+      target_field: gcp.dataproc.cluster.job.failed.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.job.running.count
+      target_field: gcp.dataproc.cluster.job.running.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.job.submitted.count
+      target_field: gcp.dataproc.cluster.job.submitted.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.operation.failed.count
+      target_field: gcp.dataproc.cluster.operation.failed.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.operation.running.count
+      target_field: gcp.dataproc.cluster.operation.running.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.operation.submitted.count
+      target_field: gcp.dataproc.cluster.operation.submitted.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.yarn.allocated_memory_percentage.value
+      target_field: gcp.dataproc.cluster.yarn.allocated_memory_percentage.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.yarn.apps.count
+      target_field: gcp.dataproc.cluster.yarn.apps.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.yarn.containers.count
+      target_field: gcp.dataproc.cluster.yarn.containers.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.yarn.memory_size.value
+      target_field: gcp.dataproc.cluster.yarn.memory_size.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.yarn.nodemanagers.count
+      target_field: gcp.dataproc.cluster.yarn.nodemanagers.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.yarn.pending_memory_size.value
+      target_field: gcp.dataproc.cluster.yarn.pending_memory_size.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.yarn.virtual_cores.count
+      target_field: gcp.dataproc.cluster.yarn.virtual_cores.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.job.completion_time.value
+      target_field: gcp.dataproc.cluster.job.completion_time.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.job.duration.value
+      target_field: gcp.dataproc.cluster.job.duration.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.operation.completion_time.value
+      target_field: gcp.dataproc.cluster.operation.completion_time.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.cluster.operation.duration.value
+      target_field: gcp.dataproc.cluster.operation.duration.value
+      ignore_missing: true
+  - remove:
+      ignore_missing: true
+      field:
+        - gcp.metrics
+    
+on_failure:
+  - set:
+      field: error.message
+      value: '{{ _ingest.on_failure_message }}'

packages/gcp/manifest.yml

@@ -1,6 +1,6 @@
 name: gcp
 title: Google Cloud Platform
-version: "2.11.10-beta.5"
+version: "2.11.10-beta.6"
 release: ga
 description: Collect logs from Google Cloud Platform with Elastic Agent.
 type: integration