diff --git a/.eslintrc b/.eslintrc deleted file mode 100644 index 1f8ba58..0000000 --- a/.eslintrc +++ /dev/null @@ -1,16 +0,0 @@ -{ - "root": true, - "parser": "@typescript-eslint/parser", - "plugins": [ - "@typescript-eslint" - ], - "extends": [ - "eslint:recommended", - "plugin:@typescript-eslint/eslint-recommended", - "plugin:@typescript-eslint/recommended", - "./node_modules/gts/" - ], - "rules": { - "prettier/prettier": 0 - } -} \ No newline at end of file diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000..b2eaa06 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,3 @@ +{ + "extends": "gts" +} diff --git a/.prettierrc.js b/.prettierrc.js new file mode 100644 index 0000000..c5166c2 --- /dev/null +++ b/.prettierrc.js @@ -0,0 +1,3 @@ +module.exports = { + ...require('gts/.prettierrc.json'), +}; diff --git a/api/API.md b/api/API.md index 3754eb5..9d6a8da 100644 --- a/api/API.md +++ b/api/API.md @@ -6,4 +6,49 @@ ## `preProcessors` +Main role: convert from one format to another format. + +### Requirements of writing a preProcessor itself: +- Expose sourceExtensionTypes (from) and outputExtensionTypes (to) extension types for file types it can handle and output + - Both sourceExtensionTypes and outputExtensionTypes are likely arrays + - E.g. a TypeScript preProcessor might convert from [.ts , .tsx] to [.mjs, .cjs] + - NOTE: TBD a uniform formal way to expose these +- Create an appropriate interface to configure this preProcessor's behavior in project config + - See example below ↓ ↓ +- NOTE: Could also define defaults options + - For a TypeScript preProcessor, a default might be to use the project's existing tsconfig.json + +- Expect `source` as a param coming from a valid `resourceProvider` OR *IMPORTANT* from another chained `prePrecessor` + - NOTE: I believe `source` should be of type `string` + - [Reasoning for this](https://github.com/googleinterns/presm/pull/3#pullrequestreview-436898909) + - [Example `preProcessor` template](https://github.com/googleinterns/presm/blob/api-preProcessors/api/preProcessor-template.js) + +- Return converted source file(s) either: + - In the form of a `string` that `node` will be able to run + - As a saved file complete with a valid extension of `outputExtensionTypes` + - NOTE: Until Node.js `getFormat` hook is updated as discussed in [this PR](https://github.com/nodejs/node/pull/34144), all `preProcessors` are expected to return a module with a default export for all formats + - E.g. a YAML loader must return its converted `JSON` object as the default export in an module + - See current `examples/loaders/preprocessor-yaml.mjs` loader in for a working example + +- Behavior Note: I do not believe a preProcessor should be filtering files based on these extensions, I believe that to be the responsibility of the caller for this loader + +### Requirements of including and configuring a preProcessor - in project config: +- Specify loader name +- Specify options (optional) + - NOTE: this interface should be defined by the preProcessor but all configuration options should go under options + - Example: + ```js + "preProcessors": [ + { + "name": "@presm/typescript", + "options": { + "tsconfigFile": "./tsconfig.json", + }, + }, + { + "name": "@presm/json", + }, + ], + ``` + ## `postProcessors` diff --git a/api/EXPLORATION.md b/api/EXPLORATION.md index 0a87096..8b3416c 100644 --- a/api/EXPLORATION.md +++ b/api/EXPLORATION.md @@ -186,3 +186,16 @@ $ node --loader=presm }, } ``` +## Implementation of Loader Workflow for On-the-Fly Running + +For all files: +1. Load all `resourceProviders`, `preProcessors`, and `postProcessors` +2. Expose a custom `resolve` hook that likely comes from a `resourceProvider` +3. Expose a `getFormat` hook that recognizes the format of eventual `source` to be returned: + - NOTE: Ideally, format resolving should be done later in the tool to be most effective + - Therefore I will be submitting a PR to `Node.js` to change this bevavior slightly + - FYI: Supported formats are `builtin`, `commonjs`, `dynamic`, `json`, `module`, `wasm` +4. Get the source (`string`) of the file via a `resourceProvider` that matches its URL (e.g. `file:`) - stop iterating `resourceProviders` when found +5. For all `preProcessors`, input and output `source` for appropriate file extensions/formats +6. For all `postProcessors`, input and output `source` for appropriate file extensions/formats +7. Return `source` in an `object` for node to run \ No newline at end of file diff --git a/api/preProcessor-template.ts b/api/preProcessor-template.ts new file mode 100644 index 0000000..053a166 --- /dev/null +++ b/api/preProcessor-template.ts @@ -0,0 +1,37 @@ +import thing from 'module'; + +export let sourceExtensionTypes: string[] = ['.ts', '.tsx']; + +export let outputExtensionTypes: string[] = ['.mjs', '.cjs']; + +// Example of defining "options" for a TypeScript loader +export interface TypeScriptOptions { + tsconfigFile?: string; +} + +interface ProcessedFile { + source: string; + extension?: string; +} + +interface preProcessorInstance { + process: Function; +} + +export function getPreprocessor( + options: TypeScriptOptions = {} +): preProcessorInstance { + // Create instances and state variables + // E.g. a CompilerHost instance to compile TypeScript + + // Example of returning a transpiled source using options provided by project config + return { + process(source: string): ProcessedFile { + return { + source: thing.transpileSource(source, options), + extension: outputExtensionTypes[0], // Example of choosing + // a valid extension from possible extensions} + }; + }, + }; +} diff --git a/examples/loaders/postprocessor-consolelog.mjs b/examples/loaders/postprocessor-consolelog.mjs new file mode 100644 index 0000000..6e3aa44 --- /dev/null +++ b/examples/loaders/postprocessor-consolelog.mjs @@ -0,0 +1,13 @@ +export const sourceFormatTypes = ['module']; + +export function getPostProcessor(options = {}) { + return { + async process(source) { + return { + source: + source + + "\nconsole.log('This line was added by a post processor!!');", + }; + }, + }; +} diff --git a/examples/loaders/preprocessor-yaml.mjs b/examples/loaders/preprocessor-yaml.mjs new file mode 100644 index 0000000..34076f1 --- /dev/null +++ b/examples/loaders/preprocessor-yaml.mjs @@ -0,0 +1,18 @@ +import {moduleWrapper} from '../../src/utils.mjs'; + +import yaml from 'yaml'; + +export const sourceExtensionTypes = ['.yaml', '.yml']; + +export const outputExtensionTypes = ['.json']; + +export function getPreProcessor(options = {}) { + return { + async process(source) { + const yamlSource = yaml.parse(source); + return { + source: moduleWrapper(JSON.stringify(yamlSource)), + }; + }, + }; +} diff --git a/examples/loaders/resourceprovider-dummy-fs.mjs b/examples/loaders/resourceprovider-dummy-fs.mjs new file mode 100644 index 0000000..6e02de2 --- /dev/null +++ b/examples/loaders/resourceprovider-dummy-fs.mjs @@ -0,0 +1,27 @@ +import {promises as fs} from 'fs'; + +export let prefixes = ['file:']; + +export let suffixes = []; + +// Following was adapted from the example in the Node.js docs +// https://nodejs.org/api/esm.html#esm_code_resolve_code_hook +export async function resolve(specifier, context, defaultResolve) { + console.log( + `### Resolving resource in dummy resourceProvider for ${specifier}\n` + ); + + return defaultResolve(specifier, context, defaultResolve); +} + +export function getResourceProvider() { + return { + async getResource(url) { + console.log( + `### Getting resource in dummy resourceProvider for ${url}\n` + ); + + return fs.readFile(new URL(url), 'utf8'); + }, + }; +} diff --git a/examples/loaders/test.mjs b/examples/loaders/test.mjs new file mode 100644 index 0000000..9a262f2 --- /dev/null +++ b/examples/loaders/test.mjs @@ -0,0 +1,6 @@ +import yamlFile from './yamlExample.yaml'; + +console.log('\nHello from test.mjs'); + +console.log('YAML Contents as JSON'); +console.log(yamlFile); diff --git a/examples/loaders/yamlExample.yaml b/examples/loaders/yamlExample.yaml new file mode 100644 index 0000000..c1a6590 --- /dev/null +++ b/examples/loaders/yamlExample.yaml @@ -0,0 +1,12 @@ +# Some example YAML data +- John Doe: + job: SWE + skills: + - python + - java +- Jane Doe: + job: SWE + skills: + - java + - python + - php diff --git a/loaderconfig.mjs b/loaderconfig.mjs new file mode 100644 index 0000000..7477d73 --- /dev/null +++ b/loaderconfig.mjs @@ -0,0 +1,21 @@ +export default { + outputPrefix: './dist', + resourceProviders: [ + { + type: '../examples/loaders/resourceprovider-dummy-fs.mjs', + base: './src', + }, + ], + preProcessors: [ + { + name: '../examples/loaders/preprocessor-yaml.mjs', + options: {}, + }, + ], + postProcessors: [ + { + name: '../examples/loaders/postprocessor-consolelog.mjs', + options: {}, + }, + ], +}; diff --git a/examples/calc/package-lock.json b/package-lock.json similarity index 97% rename from examples/calc/package-lock.json rename to package-lock.json index 0654ac2..4eaf387 100644 --- a/examples/calc/package-lock.json +++ b/package-lock.json @@ -67,12 +67,6 @@ } } }, - "@k-foss/ts-esnode": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@k-foss/ts-esnode/-/ts-esnode-1.6.0.tgz", - "integrity": "sha512-oB01IvV7Huu9DR14b4MyT74VktkzXMVQ0GucJ2jDErQ/aELpZQX90Qew0w0m46Ns7SoWVop8aD4uHNJtFAaFmg==", - "dev": true - }, "@sindresorhus/is": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", @@ -306,12 +300,6 @@ "color-convert": "^2.0.1" } }, - "arg": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", - "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", - "dev": true - }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -407,12 +395,6 @@ "concat-map": "0.0.1" } }, - "buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", - "dev": true - }, "cacheable-request": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", @@ -688,12 +670,6 @@ "integrity": "sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM=", "dev": true }, - "diff": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", - "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", - "dev": true - }, "doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -1998,12 +1974,6 @@ } } }, - "make-error": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", - "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", - "dev": true - }, "map-obj": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-4.1.0.tgz", @@ -2691,22 +2661,6 @@ } } }, - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true - }, - "source-map-support": { - "version": "0.5.19", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", - "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", - "dev": true, - "requires": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, "spdx-correct": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.1.tgz", @@ -2964,19 +2918,6 @@ "integrity": "sha512-C4+gOpvmxaSMKuEf9Qc134F1ZuOHVXKRbtEflf4NTtuuJDEIJ9p5PXsalL8SkeRw+qit1Mo+yuvMPAKwWg/1hA==", "dev": true }, - "ts-node": { - "version": "8.10.2", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-8.10.2.tgz", - "integrity": "sha512-ISJJGgkIpDdBhWVu3jufsWpK3Rzo7bdiIXJjQc0ynKxVOVcg2oIrf2H2cejminGrptVc6q6/uynAHNCuWGbpVA==", - "dev": true, - "requires": { - "arg": "^4.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "source-map-support": "^0.5.17", - "yn": "3.1.1" - } - }, "tslib": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.13.0.tgz", @@ -3232,6 +3173,12 @@ "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==", "dev": true }, + "yaml": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.0.tgz", + "integrity": "sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg==", + "dev": true + }, "yargs": { "version": "15.3.1", "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.3.1.tgz", @@ -3260,12 +3207,6 @@ "camelcase": "^5.0.0", "decamelize": "^1.2.0" } - }, - "yn": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", - "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", - "dev": true } } } diff --git a/examples/calc/package.json b/package.json similarity index 91% rename from examples/calc/package.json rename to package.json index 93367e2..5c30926 100644 --- a/examples/calc/package.json +++ b/package.json @@ -3,7 +3,6 @@ "version": "0.0.0", "description": "Pre-processing for ES modules in node", "devDependencies": { - "@k-foss/ts-esnode": "^1.6.0", "@types/node": "^14.0.12", "@types/tape": "^4.13.0", "@types/yargs": "^15.0.5", @@ -11,12 +10,12 @@ "@typescript-eslint/parser": "^3.2.0", "eslint": "^7.2.0", "gts": "^2.0.2", + "prettier": "^2.0.5", "tape": "^5.0.1", - "ts-node": "^8.10.2", "typescript": "^3.9.5", + "yaml": "^1.10.0", "yargs": "^15.3.1" }, - "type": "module", "engines": { "node": ">=12.2.0" }, diff --git a/src/loader.mjs b/src/loader.mjs new file mode 100644 index 0000000..62a9558 --- /dev/null +++ b/src/loader.mjs @@ -0,0 +1,75 @@ +import config from '../loaderconfig.mjs'; +import {isWrappedModule} from './utils.mjs'; + +// Load all resourceProviders, preProcessors, and postProcessors as specified in config file + +let resourceProviders = await Promise.all( + config.resourceProviders.map((resourceProvider, i) => + import(resourceProvider.type) + ) +); + +let preProcessors = await Promise.all( + config.preProcessors.map((preProcessor, i) => import(preProcessor.name)) +); + +let postProcessors = await Promise.all( + config.postProcessors.map((postProcessor, i) => import(postProcessor.name)) +); + +export let resolve = resourceProviders[0].resolve; + +// Dummy getFormat, effectively eliminating this step +export async function getFormat(url, context, defaultGetFormat) { + return { + format: 'module', + }; +} + +// This getSource hook executes chained resourceProviders, preProcessors, and postProcessors +export async function getSource(url, context, defaultGetSource) { + const {format} = context; + + let source; + let sourceIsWrappedModule = false; + + // Get source using any resouce preovider that accepts this type of URL ("file:") + for (const resourceProvider of resourceProviders) { + if (resourceProvider.prefixes.some(prefix => url.startsWith(prefix))) { + let resourceProviderInstance = resourceProvider.getResourceProvider(); + source = await resourceProviderInstance.getResource(url); + break; + } + } + + // Redefine source for every preProcessor that exists + for (const preProcessor of preProcessors) { + if (preProcessor.sourceExtensionTypes.some(ext => url.endsWith(ext))) { + let preProcessorInstance = preProcessor.getPreProcessor(); + source = (await preProcessorInstance.process(source)).source; + sourceIsWrappedModule = + sourceIsWrappedModule || + isWrappedModule(preProcessor.outputExtensionTypes) + ? true + : false; + } + } + + // Redefine source for every postProcessor that exists + for (const postProcessor of postProcessors) { + if ( + postProcessor.sourceFormatTypes.includes(format) && + !sourceIsWrappedModule + ) { + let postProcessorInstance = postProcessor.getPostProcessor(); + source = (await postProcessorInstance.process(source)).source; + } + } + + return { + source: source, + }; + + // Defer to Node.js for all other URLs. + // return defaultGetSource(url, context, defaultGetSource); +} diff --git a/src/utils.mjs b/src/utils.mjs new file mode 100644 index 0000000..6df1d34 --- /dev/null +++ b/src/utils.mjs @@ -0,0 +1,11 @@ +export function moduleWrapper(source) { + return `export default ${source}`; +} + +export function isWrappedModule(extensions) { + if (extensions.includes('.mjs') || extensions.includes('.mjs')) { + return false; + } else { + return true; + } +} diff --git a/tsconfig.json b/tsconfig.json index 5a1bcd3..d1646f0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,9 +2,7 @@ "extends": "./node_modules/gts/tsconfig-google.json", "compilerOptions": { "rootDir": ".", - "outDir": "build", - "module": "esnext", - "allowSyntheticDefaultImports": true + "outDir": "build" }, "include": [ "src/**/*.ts",