[google_workspace] Add support for Rules datastream and dashboard for all datastreams

packages/google_workspace/_dev/build/docs/README.md

@@ -11,6 +11,7 @@ It is compatible with a subset of applications under the [Google Reports API v1]
 | [SAML](https://developers.google.com/admin-sdk/reports/v1/appendix/activity/saml) [help](https://support.google.com/a/answer/7007375?hl=en&ref_topic=9027054) | View users’ successful and failed sign-ins to SAML applications. |
 | [User Accounts](https://developers.google.com/admin-sdk/reports/v1/appendix/activity/user-accounts) [help](https://support.google.com/a/answer/9022875?hl=en&ref_topic=9027054) | Audit actions carried out by users on their own accounts including password changes, account recovery details and 2-Step Verification enrollment. |
 | [Login](https://developers.google.com/admin-sdk/reports/v1/appendix/activity/login) [help](https://support.google.com/a/answer/4580120?hl=en&ref_topic=9027054) | Track user sign-in activity to your domain. |
+| [Rules](https://developers.google.com/admin-sdk/reports/v1/appendix/activity/rules) [help](https://support.google.com/a/answer/9656783?hl=en&ref_topic=9027054) | View a record of actions to review your user’s attempts to share sensitive data. |
 | [Admin](https://developers.google.com/admin-sdk/reports/v1/appendix/activity/admin-application-settings) [help](https://support.google.com/a/answer/4579579?hl=en&ref_topic=9027054) | View administrator activity performed within the Google Admin console. |
 | [Drive](https://developers.google.com/admin-sdk/reports/v1/appendix/activity/drive) [help](https://support.google.com/a/answer/4579696?hl=en&ref_topic=9027054) | Record user activity within Google Drive including content creation in such as Google Docs, as well as content created elsewhere that your users upload to Drive such as PDFs and Microsoft Word files. |
 | [Groups](https://developers.google.com/admin-sdk/reports/v1/appendix/activity/groups) [help](https://support.google.com/a/answer/6270454?hl=en&ref_topic=9027054) | Track changes to groups, group memberships and group messages. |
@@ -148,6 +149,14 @@ This is the `login` dataset.
 
 {{fields "login"}}
 
+### Rules
+
+This is the `rules` dataset.
+
+{{event "rules"}}
+
+{{fields "rules"}}
+
 ### Admin
 
 This is the `admin` dataset.

packages/google_workspace/_dev/deploy/docker/config.yml

@@ -262,3 +262,19 @@ rules:
         body: |
           {"kind": "reports#auditActivities","items": [{"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"events":[{"name":"2sv_disable","type":"2sv_change"},{"name":"2sv_enroll","type":"2sv_change"},{"name":"password_edit","type":"password_change"},{"name":"recovery_email_edit","type":"recovery_info_change"},{"name":"recovery_phone_edit","type":"recovery_info_change"},{"name":"recovery_secret_qa_edit","type":"recovery_info_change"},
           {"name":"titanium_enroll","type":"titanium_change"},{"name":"titanium_unenroll","type":"titanium_change"}],"id":{"applicationName":"user_accounts","customerId":"1","time":"{{.request.vars.startTime}}","uniqueQualifier":1},"ipAddress":"98.235.162.24","kind":"admin#reports#activity","ownerDomain":"elastic.com"}]}
+  - path: /admin/reports/v1/activity/users/all/applications/rules
+    methods: [GET]
+    query_params:
+      startTime: "{startTime:.*}"
+    request_headers:
+      Accept:
+        - "application/json"
+      Authorization:
+        - "Bearer 1/fFAGRNJru1FTz70BzhT3Zg"
+    responses:
+      - status_code: 200
+        headers:
+          Content-Type:
+            - "application/json"
+        body: |
+          {"kind": "reports#auditActivities","items": [{"kind":"admin#reports#activity","id":{"time":"2020-10-02T15:00:00Z","uniqueQualifier":1,"applicationName":"rules","customerId":"1"},"actor":{"callerType":"USER","email":"foo@bar.com","profileId":1},"ownerDomain":"elastic.com","ipAddress":"67.43.156.13","events":[{"type":"rule_match_type","name":"rule_match","parameters":[{"name":"has_alert","boolValue":"true"},{"name":"actor_ip_address","value":"127.0.0.0"},{"name":"resource_recipients_omitted_count","intValue":"1234"},{"name":"rule_name","multiValue":["managers"]},{"name":"rule_id","multiIntValue":["12"]}]}]}]}

packages/google_workspace/_dev/deploy/docker/docker-compose.yml

@@ -16,7 +16,7 @@ services:
         awk -v var="$$(sed -E ':a;N;$$!ba;s/\r{0,1}\n/\\\\n/g' pkcs8.key)" '{sub(/the-key/,var)}1' /credentials.json > /config/credentials.json;
         sleep 1000
   google_workspace:
-    image: docker.elastic.co/observability/stream:v0.5.0
+    image: docker.elastic.co/observability/stream:v0.8.0
     hostname: google_workspace
     ports:
       - 8080

packages/google_workspace/changelog.yml

@@ -1,4 +1,15 @@
 # newer versions go on top
+- version: "2.1.0"
+  changes:
+    - description: Add New Rules Data Stream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/4588
+    - description: Add Missing Dashboards.
+      type: enhancement
+      link: https://github.com/elastic/integrations/issues/3102
+    - description: Improve ECS Utilization.
+      type: enhancement
+      link: https://github.com/elastic/integrations/issues/4317
 - version: "2.0.0"
   changes:
     - description: Add a new alert data stream and fix the request query parameter inconsistent between intervals.

packages/google_workspace/data_stream/admin/_dev/test/pipeline/test-admin-application.log-expected.json

@@ -12,6 +12,7 @@
                     "configuration"
                 ],
                 "id": "1",
+                "kind": "event",
                 "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"APPLICATION_SETTINGS\",\"name\":\"CHANGE_APPLICATION_SETTING\",\"parameters\":[{\"name\":\"APPLICATION_EDITION\",\"value\":\"basic\"},{\"name\":\"APPLICATION_NAME\",\"value\":\"drive\"},{\"name\":\"GROUP_EMAIL\",\"value\":\"group@example.com\"},{\"name\":\"NEW_VALUE\",\"value\":\"new\"},{\"name\":\"OLD_VALUE\",\"value\":\"old\"},{\"name\":\"ORG_UNIT_NAME\",\"value\":\"org\"},{\"name\":\"SETTING_NAME\",\"value\":\"setting\"}]}}",
                 "provider": "admin",
                 "type": [
@@ -88,6 +89,7 @@
             ],
             "user": {
                 "domain": "bar.com",
+                "email": "foo@bar.com",
                 "id": "1",
                 "name": "foo",
                 "target": {
@@ -109,6 +111,7 @@
                     "iam"
                 ],
                 "id": "1",
+                "kind": "event",
                 "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"APPLICATION_SETTINGS\",\"name\":\"CREATE_APPLICATION_SETTING\",\"parameters\":[{\"name\":\"APPLICATION_EDITION\",\"value\":\"basic\"},{\"name\":\"APPLICATION_NAME\",\"value\":\"drive\"},{\"name\":\"GROUP_EMAIL\",\"value\":\"group@example.com\"},{\"name\":\"NEW_VALUE\",\"value\":\"new\"},{\"name\":\"ORG_UNIT_NAME\",\"value\":\"org\"},{\"name\":\"SETTING_NAME\",\"value\":\"setting\"}]}}",
                 "provider": "admin",
                 "type": [
@@ -185,6 +188,7 @@
             ],
             "user": {
                 "domain": "bar.com",
+                "email": "foo@bar.com",
                 "id": "1",
                 "name": "foo",
                 "target": {
@@ -207,6 +211,7 @@
                     "configuration"
                 ],
                 "id": "1",
+                "kind": "event",
                 "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"APPLICATION_SETTINGS\",\"name\":\"DELETE_APPLICATION_SETTING\",\"parameters\":[{\"name\":\"APPLICATION_EDITION\",\"value\":\"basic\"},{\"name\":\"APPLICATION_NAME\",\"value\":\"drive\"},{\"name\":\"GROUP_EMAIL\",\"value\":\"group@example.com\"},{\"name\":\"OLD_VALUE\",\"value\":\"old\"},{\"name\":\"ORG_UNIT_NAME\",\"value\":\"org\"},{\"name\":\"SETTING_NAME\",\"value\":\"setting\"}]}}",
                 "provider": "admin",
                 "type": [
@@ -282,6 +287,7 @@
             ],
             "user": {
                 "domain": "bar.com",
+                "email": "foo@bar.com",
                 "id": "1",
                 "name": "foo",
                 "target": {
@@ -303,6 +309,7 @@
                     "iam"
                 ],
                 "id": "1",
+                "kind": "event",
                 "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"APPLICATION_SETTINGS\",\"name\":\"REORDER_GROUP_BASED_POLICIES_EVENT\",\"parameters\":[{\"name\":\"APPLICATION_NAME\",\"value\":\"drive\"},{\"name\":\"GROUP_PRIORITIES\",\"multiValue\":[\"a\",\"b\"]},{\"name\":\"SETTING_NAME\",\"value\":\"setting\"}]}}",
                 "provider": "admin",
                 "type": [
@@ -373,6 +380,7 @@
             ],
             "user": {
                 "domain": "bar.com",
+                "email": "foo@bar.com",
                 "id": "1",
                 "name": "foo"
             }
@@ -389,6 +397,7 @@
                     "configuration"
                 ],
                 "id": "1",
+                "kind": "event",
                 "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"APPLICATION_SETTINGS\",\"name\":\"GPLUS_PREMIUM_FEATURES\",\"parameters\":[{\"name\":\"DOMAIN_NAME\",\"value\":\"example.com\"},{\"name\":\"NEW_VALUE\",\"value\":\"new\"}]}}",
                 "provider": "admin",
                 "type": [
@@ -450,6 +459,7 @@
             ],
             "user": {
                 "domain": "bar.com",
+                "email": "foo@bar.com",
                 "id": "1",
                 "name": "foo"
             }
@@ -465,6 +475,7 @@
                     "iam"
                 ],
                 "id": "1",
+                "kind": "event",
                 "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"APPLICATION_SETTINGS\",\"name\":\"CREATE_MANAGED_CONFIGURATION\",\"parameters\":[{\"name\":\"MANAGED_CONFIGURATION_NAME\",\"value\":\"a\"},{\"name\":\"MOBILE_APP_PACKAGE_ID\",\"value\":\"1234\"}]}}",
                 "provider": "admin",
                 "type": [
@@ -526,6 +537,7 @@
             ],
             "user": {
                 "domain": "bar.com",
+                "email": "foo@bar.com",
                 "id": "1",
                 "name": "foo"
             }
@@ -541,6 +553,7 @@
                     "iam"
                 ],
                 "id": "1",
+                "kind": "event",
                 "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"APPLICATION_SETTINGS\",\"name\":\"DELETE_MANAGED_CONFIGURATION\",\"parameters\":[{\"name\":\"MANAGED_CONFIGURATION_NAME\",\"value\":\"a\"},{\"name\":\"MOBILE_APP_PACKAGE_ID\",\"value\":\"1234\"}]}}",
                 "provider": "admin",
                 "type": [
@@ -602,6 +615,7 @@
             ],
             "user": {
                 "domain": "bar.com",
+                "email": "foo@bar.com",
                 "id": "1",
                 "name": "foo"
             }
@@ -618,6 +632,7 @@
                     "configuration"
                 ],
                 "id": "1",
+                "kind": "event",
                 "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"APPLICATION_SETTINGS\",\"name\":\"UPDATE_MANAGED_CONFIGURATION\",\"parameters\":[{\"name\":\"MANAGED_CONFIGURATION_NAME\",\"value\":\"a\"},{\"name\":\"MOBILE_APP_PACKAGE_ID\",\"value\":\"1234\"}]}}",
                 "provider": "admin",
                 "type": [
@@ -679,6 +694,7 @@
             ],
             "user": {
                 "domain": "bar.com",
+                "email": "foo@bar.com",
                 "id": "1",
                 "name": "foo"
             }
@@ -695,6 +711,7 @@
                     "configuration"
                 ],
                 "id": "1",
+                "kind": "event",
                 "original": "{\"kind\":\"admin#reports#activity\",\"id\":{\"time\":\"2020-10-02T15:00:00Z\",\"uniqueQualifier\":1,\"applicationName\":\"admin\",\"customerId\":\"1\"},\"actor\":{\"callerType\":\"USER\",\"email\":\"foo@bar.com\",\"profileId\":1},\"ownerDomain\":\"elastic.com\",\"ipAddress\":\"67.43.156.13\",\"events\":{\"type\":\"APPLICATION_SETTINGS\",\"name\":\"FLASHLIGHT_EDU_NON_FEATURED_SERVICES_SELECTED\",\"parameters\":[{\"name\":\"FLASHLIGHT_EDU_NON_FEATURED_SERVICES_SELECTION\",\"value\":\"FLASHLIGHT_EDU_SELECTION_MANUAL\"}]}}",
                 "provider": "admin",
                 "type": [
@@ -753,6 +770,7 @@
             ],
             "user": {
                 "domain": "bar.com",
+                "email": "foo@bar.com",
                 "id": "1",
                 "name": "foo"
             }