RFC: Define preProcessors API

.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "./node_modules/gts/"
+}

.prettierrc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  ...require('gts/.prettierrc.json')
+}

api/EXPLORATION.md

@@ -186,3 +186,14 @@ $ node --loader=presm
   },
 }
 ```
+## Implementation of Loader Workflow for On-the-Fly Running
+
+For all files:
+1. Load all `resourceProviders`, `preProcessors`, and `postProcessors`
+2. Expose a custom `resolve` hook that likely comes from a `resourceProvider`
+3. Expose a dummy `getFormat` hook to eliminate formatting problems for unsupported formats:
+  - NOTE: Supported formats are `builtin`, `commonjs`, `dynamic`, `json`, `module`, `wasm`
+4. Get the source (`string`) of the file via a `resourceProvider` that matches its URL (e.g. `file:`) - stop iterating `resourceProviders` when found
+5. For all `preProcessors`, input and output `source` 
+6. For all `postProcessors`, input and output `source`
+7. Return `source` in `object` for node to run

examples/loaders/postprocessor-consolelog.mjs

@@ -0,0 +1,13 @@
+export const sourceFormatTypes = ['module'];
+
+export function getPostProcessor(options = {}) {
+  return {
+    async process(source) {
+      return {
+        source:
+          source +
+          "\nconsole.log('This line was added by a post processor!!');",
+      };
+    },
+  };
+}

examples/loaders/preprocessor-yaml.mjs

@@ -0,0 +1,16 @@
+import yaml from 'yaml';
+
+export const sourceExtensionTypes = ['.yaml'];
+
+export const outputExtensionTypes = ['.json'];
+
+export function getPreProcessor(options = {}) {
+  return {
+    async process(source) {
+      const yamlSource = yaml.parse(source);
+      return {
+        source: JSON.stringify(yamlSource),
+      };
+    },
+  };
+}

examples/loaders/resourceprovider-dummy-fs.mjs

@@ -0,0 +1,31 @@
+import * as fs from 'fs';
+
+export let prefixes = ['file:'];
+
+export let suffixes = [];
+
+// Following was adapted from the example in the Node.js docs
+// https://nodejs.org/api/esm.html#esm_code_resolve_code_hook
+export async function resolve(specifier, context, defaultResolve) {
+  console.log(
+    `### Resolving resource in dummy resourceProvider for ${specifier}\n`
+  );
+
+  return defaultResolve(specifier, context, defaultResolve);
+}
+
+export function getResourceProvider() {
+  return {
+    async getResource(url) {
+      console.log(
+        `### Getting resource in dummy resourceProvider for ${url}\n`
+      );
+
+      return new Promise((resolve, reject) => {
+        fs.readFile(new URL(url), 'utf8', (err, data) => {
+          resolve(data);
+        });
+      });
+    },
+  };
+}

examples/loaders/test.mjs

@@ -0,0 +1,6 @@
+import yamlFile from './yamlExample.yaml';
+
+console.log('\nHello from test.mjs');
+
+console.log('YAML Contents as JSON');
+console.log(yamlFile);

examples/loaders/yamlExample.yaml

@@ -0,0 +1,12 @@
+# Some example YAML data
+- John Doe:
+    job: SWE
+    skills:
+      - python
+      - java
+- Jane Doe:
+    job: SWE
+    skills:
+      - java
+      - python
+      - php

loaderconfig.mjs

@@ -0,0 +1,21 @@
+export default {
+  outputPrefix: './dist',
+  resourceProviders: [
+    {
+      type: '../examples/loaders/resourceprovider-dummy-fs.mjs',
+      base: './src',
+    },
+  ],
+  preProcessors: [
+    {
+      name: '../examples/loaders/preprocessor-yaml.mjs',
+      options: {},
+    },
+  ],
+  postProcessors: [
+    {
+      name: '../examples/loaders/postprocessor-consolelog.mjs',
+      options: {},
+    },
+  ],
+};

examples/calc/package.json → package.json

@@ -3,20 +3,19 @@
   "version": "0.0.0",
   "description": "Pre-processing for ES modules in node",
   "devDependencies": {
-    "@k-foss/ts-esnode": "^1.6.0",
     "@types/node": "^14.0.12",
     "@types/tape": "^4.13.0",
     "@types/yargs": "^15.0.5",
     "@typescript-eslint/eslint-plugin": "^3.2.0",
     "@typescript-eslint/parser": "^3.2.0",
     "eslint": "^7.2.0",
     "gts": "^2.0.2",
+    "prettier": "^2.0.5",
     "tape": "^5.0.1",
-    "ts-node": "^8.10.2",
     "typescript": "^3.9.5",
+    "yaml": "^1.10.0",
     "yargs": "^15.3.1"
   },
-  "type": "module",
   "engines": {
     "node": ">=12.2.0"
   },

src/loader.mjs

@@ -0,0 +1,74 @@
+import config from '../loaderconfig.mjs';
+
+// console.log("Config File:");
+// console.log(config);
+
+// Load all resourceProviders, preProcessors, and postProcessors as specified in config file
+
+let resourceProviders = await Promise.all(
+  config.resourceProviders.map((resourceProvider, i) =>
+    import(resourceProvider.type)
+  )
+);
+
+let preProcessors = await Promise.all(
+  config.preProcessors.map((preProcessor, i) => import(preProcessor.name))
+);
+
+let postProcessors = await Promise.all(
+  config.postProcessors.map((postProcessor, i) => import(postProcessor.name))
+);
+
+export let resolve = resourceProviders[0].resolve;
+
+// Dummy getFormat, effectively eliminating this step
+export async function getFormat(url, context, defaultGetFormat) {
+  if (url.endsWith('.yaml') || url.endsWith('.yml')) {
+    return {
+      format: 'json',
+    };
+  } else {
+    return {
+      format: 'module',
+    };
+  }
+}
+
+// This getSource hook executes chained resourceProviders, preProcessors, and postProcessors
+export async function getSource(url, context, defaultGetSource) {
+  const {format} = context;
+
+  let source;
+
+  // Get source using any resouce preovider that accepts this type of URL ("file:")
+  for (const resourceProvider of resourceProviders) {
+    if (resourceProvider.prefixes.some(prefix => url.startsWith(prefix))) {
+      let resourceProviderInstance = resourceProvider.getResourceProvider();
+      source = await resourceProviderInstance.getResource(url);
+      break;
+    }
+  }
+
+  // Redefine source for every preProcessor that exists
+  for (const preProcessor of preProcessors) {
+    if (preProcessor.sourceExtensionTypes.some(ext => url.endsWith(ext))) {
+      let preProcessorInstance = preProcessor.getPreProcessor();
+      source = (await preProcessorInstance.process(source)).source;
+    }
+  }
+
+  // Redefine source for every postProcessor that exists
+  for (const postProcessor of postProcessors) {
+    if (postProcessor.sourceFormatTypes.includes(format)) {
+      let postProcessorInstance = postProcessor.getPostProcessor();
+      source = (await postProcessorInstance.process(source)).source;
+    }
+  }
+
+  return {
+    source: source,
+  };
+
+  // Defer to Node.js for all other URLs.
+  // return defaultGetSource(url, context, defaultGetSource);
+}

tsconfig.json

@@ -2,9 +2,7 @@
   "extends": "./node_modules/gts/tsconfig-google.json",
   "compilerOptions": {
     "rootDir": ".",
-    "outDir": "build",
-    "module": "esnext",
-    "allowSyntheticDefaultImports": true
+    "outDir": "build"
   },
   "include": [
     "src/**/*.ts",