Splunk2HDF Updates

apps/frontend/src/components/global/upload_tabs/splunk/AuthStep.vue

@@ -5,6 +5,7 @@
         v-model="username"
         label="Username"
         for="username_field"
+        hint="admin"
         data-cy="splunkusername"
       />
       <v-text-field
@@ -18,13 +19,14 @@
         v-model="hostname"
         label="Hostname"
         for="hostname_field"
+        hint="https://yourdomain.com:8089"
         data-cy="splunkhostname"
       />
     </v-form>
     <v-row class="mx-1">
       <v-btn
         color="primary"
-        class="my-4 mt-0"
+        class="my-4 mt-4"
         data-cy="splunkLoginButton"
         @click="login"
       >

apps/frontend/src/components/global/upload_tabs/splunk/FileList.vue

@@ -86,14 +86,15 @@ export default class FileList extends Vue {
 
   loadResults() {
     const files = this.selectedExecutions.map((execution) => {
-      return getExecution(this.splunkClient, execution.guid).then((result) =>
+      return getExecution(this.splunkClient, execution.guid).then((result) => {
+        console.log(result);
         InspecIntakeModule.loadText({
           text: JSON.stringify(result),
           filename: _.get(result, 'meta.filename')
         }).catch((err) => {
           SnackbarModule.failure(String(err));
-        })
-      );
+        });
+      });
     });
     this.$emit('got-files', files);
   }

apps/frontend/src/utilities/splunk_util.ts

@@ -137,10 +137,20 @@ function consolidateFilePayloads(
     (ctrl) => ctrl.meta.profile_sha256
   );
   for (const profile of profileEvents) {
+    profile.controls = [];
     // Get the corresponding controls, and put them into the profile
     const sha = profile.meta.profile_sha256;
     const corrControls = shaGroupedControls[sha] || [];
-    profile.controls.push(...corrControls);
+    profile.controls.push(
+      ...replaceKeyValueDescriptions(
+        corrControls as unknown as (ExecJSON.Control &
+          GenericPayloadWithMetaData & {
+            descriptions?:
+              | {[key: string]: string}
+              | ExecJSON.ControlDescription[];
+          })[]
+      )
+    );
   }
 
   return exec as unknown as ExecJSON.Execution;
@@ -179,7 +189,7 @@ export async function createSearch(
   // We basically can't, and really shouldn't, do typescript here. Output is changes depending on the job called
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
 ): Promise<JobID> {
-  const fullQuery = `search=search index="hdf" | ${searchString || ''}`;
+  const fullQuery = `search=search index="*" | ${searchString || ''}`;
   return apiClient({
     method: 'POST',
     url: `${splunkClient.host}/services/search/jobs?output_mode=json`,

libs/hdf-converters/package.json

@@ -33,6 +33,7 @@
     "inspecjs": "^2.6.10",
     "lodash": "^4.17.21",
     "moment": "^2.29.1",
+    "winston": "^3.6.0",
     "xml2js": "^0.4.23"
   },
   "devDependencies": {

libs/hdf-converters/src/converters-from-hdf/splunk/Schemas.md

@@ -153,6 +153,7 @@ index="<<YOUR INDEX>>" | stats list(meta.filename) list(meta.filetype) list(tags
             "compliance": "?"
         }
     ],
+    controls: [], // An empty array, for backwards compatibility with versions of Heimdall that assume this is already defined
     // "Inputs" in modern Inspec parlance, Attributes are the parameters specified at runtime for the profile.
     "attributes": [
         {

libs/hdf-converters/src/converters-from-hdf/splunk/reverse-splunk-mapper.ts

@@ -1,22 +1,28 @@
 import axios, {AxiosResponse} from 'axios';
+import https from 'https';
 import {
   ContextualizedControl,
   ContextualizedEvaluation,
   ContextualizedProfile,
   contextualizeEvaluation,
   ExecJSON
 } from 'inspecjs';
+import winston from 'winston';
 import {MappedTransform} from '../../base-converter';
+import {createWinstonLogger} from '../../utils/global';
 import {FromAnyBaseConverter} from '../reverse-any-base-converter';
 import {ILookupPathFH} from '../reverse-base-converter';
 import {SplunkControl} from './splunk-control-types';
 import {SplunkProfile} from './splunk-profile-types';
 import {SplunkReport} from './splunk-report-types';
 
 export const HDF_SPLUNK_SCHEMA = '1.0';
+export const MAPPER_NAME = 'HDF2Splunk';
+
 export type SplunkConfig = {
   host: string;
   port: number;
+  insecure?: boolean;
   token: string;
   protocol: string;
   index?: string;
@@ -28,6 +34,9 @@ export type SplunkData = {
   reports: SplunkReport[];
 };
 
+let logger: winston.Logger | undefined = undefined;
+let httpsAgent = new https.Agent();
+
 export function createGUID(length: number) {
   let result = '';
   const characters =
@@ -62,6 +71,7 @@ export function postDataToSplunkHEC(
           index: config.index
         },
         {
+          httpsAgent: httpsAgent,
           headers: {
             Authorization: `Splunk ${config.token}`
           }
@@ -76,6 +86,7 @@ export function postDataToSplunkHEC(
           event: data
         },
         {
+          httpsAgent: httpsAgent,
           headers: {
             Authorization: `Splunk ${config.token}`
           }
@@ -98,6 +109,7 @@ export function createReportMapping(
       hdf_splunk_schema: HDF_SPLUNK_SCHEMA,
       filetype: 'evaluation'
     },
+    profiles: [],
     platform: execution.data.platform,
     statistics: execution.data.statistics,
     version: execution.data.version
@@ -155,7 +167,7 @@ export function createProfileMapping(
       profile_sha256: {
         path: 'data.sha256'
       },
-      subtype: 'header'
+      subtype: 'profile'
     },
     summary: {
       path: 'data.summary'
@@ -187,6 +199,7 @@ export function createProfileMapping(
     title: {
       path: 'data.title'
     },
+    controls: [],
     parent_profile: {
       path: 'data.depends[0].name'
     },
@@ -231,9 +244,11 @@ export function createControlMapping(
       path: 'data.descriptions',
       transformer: (data: {label: string; data: string}[]) => {
         const descObjects: Record<string, string> = {};
-        data.forEach((item) => {
-          descObjects[item['label']] = item['data'];
-        });
+        if (Array.isArray(data)) {
+          data.forEach((item) => {
+            descObjects[item['label']] = item['data'];
+          });
+        }
         return descObjects;
       }
     },
@@ -250,9 +265,18 @@ export class FromHDFExecutionToSplunkExecutionMapper extends FromAnyBaseConverte
   constructor(
     evaluation: ContextualizedEvaluation,
     filename: string,
-    guid: string
+    guid: string,
+    logService?: winston.Logger,
+    loggingLevel?: string
   ) {
     super(evaluation);
+    if (logService) {
+      logger = logService;
+    } else {
+      logger = createWinstonLogger(MAPPER_NAME, loggingLevel || 'debug');
+    }
+    logger.debug('Got Execution: ' + filename);
+    logger.debug('Using GUID: ' + guid);
     this.setMappings(createReportMapping(evaluation, filename, guid));
   }
 
@@ -305,6 +329,11 @@ export class FromHDFToSplunkMapper extends FromAnyBaseConverter {
       profiles: [],
       reports: []
     };
+    if (config.insecure) {
+      httpsAgent = new https.Agent({
+        rejectUnauthorized: false
+      });
+    }
     const guid = createGUID(30);
     splunkData.reports.push(
       new FromHDFExecutionToSplunkExecutionMapper(
@@ -313,6 +342,7 @@ export class FromHDFToSplunkMapper extends FromAnyBaseConverter {
         guid
       ).toSplunkExecution() as SplunkReport
     );
+    logger?.debug(`Converted execution: ${JSON.stringify(splunkData.reports)}`);
     this.data.contains.forEach((profile: ContextualizedProfile) => {
       splunkData.profiles.push(
         new FromHDFProfileToSplunkProfileMapper(
@@ -335,10 +365,19 @@ export class FromHDFToSplunkMapper extends FromAnyBaseConverter {
     });
 
     const uploads: Promise<AxiosResponse>[] = [];
+    logger?.info(
+      'Data converted, uploading to Splunk with config: ' +
+        JSON.stringify(config)
+    );
     uploads.push(...postDataToSplunkHEC(splunkData.reports, config));
     uploads.push(...postDataToSplunkHEC(splunkData.profiles, config));
     uploads.push(...postDataToSplunkHEC(splunkData.controls, config));
 
-    return Promise.all(uploads).then(() => guid);
+    return Promise.all(uploads).then(() => {
+      logger?.info(
+        `Uploaded into splunk successfully, to find this data search for: meta.guid="${guid}"`
+      );
+      return guid;
+    });
   }
 }

libs/hdf-converters/src/converters-from-hdf/splunk/splunk-profile-types.ts

@@ -4,6 +4,7 @@ export type SplunkProfile = {
   meta: Meta;
   summary: string;
   sha256: string;
+  controls: any[];
   supports: any[] | ILookupPathFH;
   name: string;
   copyright: string;

libs/hdf-converters/src/converters-from-hdf/splunk/splunk-report-types.ts

@@ -3,6 +3,7 @@ import {ExecJSON} from 'inspecjs';
 export type SplunkReport = {
   meta: Meta;
   statistics?: ExecJSON.Statistics;
+  profiles: any[];
   platform: Platform;
   version: string;
 };

libs/hdf-converters/src/utils/global.ts

@@ -0,0 +1,19 @@
+import winston from 'winston';
+
+export function createWinstonLogger(
+  mapperName: string,
+  level = 'debug'
+): winston.Logger {
+  return winston.createLogger({
+    transports: [new winston.transports.Console()],
+    level: level,
+    format: winston.format.combine(
+      winston.format.timestamp({
+        format: 'MMM-DD-YYYY HH:mm:ss Z'
+      }),
+      winston.format.printf(
+        (info) => `[${[info.timestamp]}] ${mapperName} ${info.message}`
+      )
+    )
+  });
+}

yarn.lock

@@ -20558,7 +20558,7 @@ winston-transport@^4.5.0:
     readable-stream "^3.6.0"
     triple-beam "^1.3.0"
 
-winston@^3.3.3:
+winston@^3.3.3, winston@^3.6.0:
   version "3.6.0"
   resolved "https://registry.yarnpkg.com/winston/-/winston-3.6.0.tgz#be32587a099a292b88c49fac6fa529d478d93fb6"
   integrity sha512-9j8T75p+bcN6D00sF/zjFVmPp+t8KMPB1MzbbzYjeN9VWxdsYnTB40TkbNUEXAmILEfChMvAMgidlX64OG3p6w==