Modernize typescript (#1529)

### Changelog

TypeScript packages now properly export both ESM + CJS modules.

Types and constants are now exported directly from `@mcap/core`:

```typescript
// Before
import type { McapTypes } from "@mcap/core";
const MetadataIndex = { McapTypes };

// After
import type { MetadataIndex } from "@mcap/core";
```

Nested imports are no longer supported:

```typescript
// Before
import { Metadata } from "@mcap/core/src/types";

// After
import type { Metadata } from "@mcap/core";
```

### Docs

None

### Description

- Upgrade TypeScript
- Upgrade `@foxglove/tsconfig` and fix all issues
- Replace `ts-node` with `tsx`
- Maintain both esm and cjs exports for backwards compatibility
- Semver minor bump all 4 public packages

Author: amacneil

.github/workflows/ci.yml

@@ -233,7 +233,7 @@ jobs:
           registry-url: https://registry.npmjs.org
       - run: yarn install --immutable
       - run: yarn dedupe --check
-      - run: yarn prettier:check
+      - run: yarn fmt:check
       - run: yarn workspace @mcap/core lint:ci
       - run: yarn workspace @mcap/core build
       - run: yarn workspace @mcap/support lint:ci

.gitignore

@@ -11,6 +11,6 @@ site/
 .pnp.*
 .yarn/*
 *.tgz
-
+*.tsbuildinfo
 *.db3-shm
 *.db3-wal

package.json

@@ -12,13 +12,15 @@
       "website"
     ]
   },
+  "type": "module",
   "scripts": {
-    "prettier": "prettier --write .",
-    "prettier:check": "prettier --check .",
+    "fmt": "prettier --write .",
+    "fmt:check": "prettier --check .",
     "docs:swift:start": "swift package --disable-sandbox preview-documentation --target MCAP",
     "typedoc": "yarn typescript:build && typedoc --out __docs__/typescript --options typescript/typedoc.json",
     "start": "yarn workspace website start",
     "spellcheck": "cspell --relative '**'",
+    "tsc:all": "tsc --build --incremental --verbose $(git ls-files -- 'tsconfig*.json' '**/tsconfig*.json')",
     "typescript:test": "yarn jest --config typescript/jest.config.json",
     "typescript:build": "yarn workspace @mcap/core build && yarn workspace @mcap/support build && yarn workspace @mcap/nodejs build && yarn workspace @mcap/browser build",
     "typescript:clean": "yarn workspace @mcap/core build --clean && yarn workspace @mcap/support build --clean && yarn workspace @mcap/nodejs build --clean && yarn workspace @mcap/browser build --clean",
@@ -27,13 +29,12 @@
   },
   "packageManager": "yarn@4.12.0",
   "devDependencies": {
-    "@types/node": "18.13.0",
+    "@types/node": "20.19.33",
     "cspell": "8.0.0",
     "jest": "29.7.0",
     "prettier": "3.1.0",
     "ts-jest": "29.1.1",
-    "ts-node": "10.9.1",
     "typedoc": "0.25.3",
-    "typescript": "5.2.2"
+    "typescript": "5.9.3"
   }
 }

tests/conformance/.eslintrc.js tests/conformance/.eslintrc.cjstests/conformance/.eslintrc.js renamed to tests/conformance/.eslintrc.cjs

@@ -1,23 +1,24 @@
 {
   "name": "@foxglove/mcap-conformance",
-  "version": "0.0.0",
   "private": true,
+  "type": "module",
   "scripts": {
-    "build": "tsc -b tsconfig.json tsconfig.cjs.json",
-    "lint:ci": "eslint --report-unused-disable-directives .",
-    "lint": "eslint --report-unused-disable-directives --fix .",
-    "generate-inputs": "ts-node --files --project tsconfig.cjs.json scripts/generate-inputs",
-    "run-tests": "tsc -b tsconfig.json tsconfig.cjs.json && ts-node --files --project tsconfig.cjs.json scripts/run-tests"
+    "build": "tsc -b",
+    "lint:ci": "tsc -b && eslint --report-unused-disable-directives .",
+    "lint": "tsc -b && eslint --report-unused-disable-directives --fix .",
+    "generate-inputs": "tsc -b && tsx scripts/generate-inputs.ts",
+    "run-tests": "tsc -b && tsx scripts/run-tests/index.ts"
   },
   "devDependencies": {
     "@foxglove/crc": "^0.0.3",
     "@foxglove/eslint-plugin": "1.0.1",
-    "@foxglove/tsconfig": "1.1.0",
+    "@foxglove/tsconfig": "3.2.0",
     "@mcap/core": "workspace:*",
-    "@mcap/support": "workspace:*",
+    "@mcap/nodejs": "workspace:*",
     "@types/diff": "^5.0.2",
     "@types/js-yaml": "^4.0.5",
-    "@types/node": "18.13.0",
+    "@types/lodash-es": "4.17.12",
+    "@types/node": "20.19.33",
     "@typescript-eslint/eslint-plugin": "6.11.0",
     "@typescript-eslint/parser": "6.11.0",
     "colors": "1.4.0",
@@ -35,10 +36,9 @@
     "json-stringify-pretty-compact": "4.0.0",
     "kaitai-struct": "0.10.0",
     "kaitai-struct-compiler": "0.10.0",
-    "lodash": "4.17.21",
+    "lodash-es": "4.17.23",
     "prettier": "3.1.0",
-    "ts-node": "10.9.1",
-    "tsconfig-paths": "^4.1.2",
-    "typescript": "5.2.2"
+    "tsx": "4.21.0",
+    "typescript": "5.9.3"
   }
 }

tests/conformance/package.json

@@ -1,19 +1,17 @@
 import { crc32, crc32Init, crc32Update, crc32Final } from "@foxglove/crc";
-import { McapTypes, McapConstants, McapRecordBuilder, McapChunkBuilder } from "@mcap/core";
+import { McapRecordBuilder, McapChunkBuilder, Opcode } from "@mcap/core";
+import type { AttachmentIndex, ChunkIndex, MetadataIndex } from "@mcap/core";
 import { program } from "commander";
-import fs from "fs/promises";
-import path from "path";
-import generateTestVariants from "variants/generateTestVariants";
+import fs from "node:fs/promises";
+import path from "node:path";
 
-import { collect } from "../util/collect";
-import listDirRecursive from "../util/listDirRecursive";
-import { TestDataRecord, TestFeatures } from "../variants/types";
+import { collect } from "../util/collect.ts";
+import listDirRecursive from "../util/listDirRecursive.ts";
+import generateTestVariants from "../variants/generateTestVariants.ts";
+import { TestFeatures } from "../variants/types.ts";
+import type { TestDataRecord, TestFeature } from "../variants/types.ts";
 
-type MetadataIndex = McapTypes.MetadataIndex;
-type ChunkIndex = McapTypes.ChunkIndex;
-type AttachmentIndex = McapTypes.AttachmentIndex;
-
-function generateFile(features: Set<TestFeatures>, records: TestDataRecord[]) {
+function generateFile(features: Set<TestFeature>, records: TestDataRecord[]) {
   const builder = new McapRecordBuilder({
     padRecords: features.has(TestFeatures.AddExtraDataToRecords),
   });
@@ -204,42 +202,42 @@ function generateFile(features: Set<TestFeatures>, records: TestDataRecord[]) {
     summaryOffsetStart = BigInt(builder.length);
     if (repeatedSchemasLength !== 0n) {
       builder.writeSummaryOffset({
-        groupOpcode: McapConstants.Opcode.SCHEMA,
+        groupOpcode: Opcode.SCHEMA,
         groupStart: repeatedSchemasStart,
         groupLength: repeatedSchemasLength,
       });
     }
     if (repeatedChannelInfosLength !== 0n) {
       builder.writeSummaryOffset({
-        groupOpcode: McapConstants.Opcode.CHANNEL,
+        groupOpcode: Opcode.CHANNEL,
         groupStart: repeatedChannelInfosStart,
         groupLength: repeatedChannelInfosLength,
       });
     }
     if (statisticsLength !== 0n) {
       builder.writeSummaryOffset({
-        groupOpcode: McapConstants.Opcode.STATISTICS,
+        groupOpcode: Opcode.STATISTICS,
         groupStart: statisticsStart,
         groupLength: statisticsLength,
       });
     }
     if (metadataIndexLength !== 0n) {
       builder.writeSummaryOffset({
-        groupOpcode: McapConstants.Opcode.METADATA_INDEX,
+        groupOpcode: Opcode.METADATA_INDEX,
         groupStart: metadataIndexStart,
         groupLength: metadataIndexLength,
       });
     }
     if (attachmentIndexLength !== 0n) {
       builder.writeSummaryOffset({
-        groupOpcode: McapConstants.Opcode.ATTACHMENT_INDEX,
+        groupOpcode: Opcode.ATTACHMENT_INDEX,
         groupStart: attachmentIndexStart,
         groupLength: attachmentIndexLength,
       });
     }
     if (chunkIndexLength !== 0n) {
       builder.writeSummaryOffset({
-        groupOpcode: McapConstants.Opcode.CHUNK_INDEX,
+        groupOpcode: Opcode.CHUNK_INDEX,
         groupStart: chunkIndexStart,
         groupLength: chunkIndexLength,
       });
@@ -255,7 +253,7 @@ function generateFile(features: Set<TestFeatures>, records: TestDataRecord[]) {
   );
   summaryCrc = crc32Update(summaryCrc, summaryData);
   const tempBuffer = new DataView(new ArrayBuffer(1 + 8 + 8 + 8));
-  tempBuffer.setUint8(0, McapConstants.Opcode.FOOTER);
+  tempBuffer.setUint8(0, Opcode.FOOTER);
   tempBuffer.setBigUint64(1, 8n + 8n + 4n, true);
   tempBuffer.setBigUint64(1 + 8, hasSummary ? summaryStart : 0n, true);
   tempBuffer.setBigUint64(1 + 8 + 8, summaryOffsetStart, true);

tests/conformance/scripts/generate-inputs.ts

@@ -1,27 +1,28 @@
-import { McapStreamReader, McapTypes } from "@mcap/core";
+import { McapStreamReader } from "@mcap/core";
+import type { TypedMcapRecord } from "@mcap/core";
 import colors from "colors";
 import { program } from "commander";
 import * as Diff from "diff";
-import fs from "fs/promises";
 import stableStringify from "json-stable-stringify";
-import { chunk } from "lodash";
-import path from "path";
+import { chunk } from "lodash-es";
+import fs from "node:fs/promises";
+import path from "node:path";
 
-import runners from "./runners";
 import {
   IndexedReadTestRunner,
   StreamedReadTestRunner,
   WriteTestRunner,
-} from "./runners/TestRunner";
-import { toSerializableMcapRecord } from "./toSerializableMcapRecord";
-import {
+} from "./runners/TestRunner.ts";
+import runners from "./runners/index.ts";
+import { toSerializableMcapRecord } from "./toSerializableMcapRecord.ts";
+import type {
   IndexedReadTestResult,
   SerializableMcapRecord,
   StreamedReadTestResult,
   TestCase,
-} from "./types";
-import { splitMcapRecords } from "../../util/splitMcapRecords";
-import generateTestVariants from "../../variants/generateTestVariants";
+} from "./types.ts";
+import { splitMcapRecords } from "../../util/splitMcapRecords.ts";
+import generateTestVariants from "../../variants/generateTestVariants.ts";
 
 type TestOptions = {
   dataDir: string;
@@ -142,7 +143,9 @@ async function runWriterTest(
       hadError = true;
       continue;
     }
-    const expectedParts = splitMcapRecords(expectedOutput).map(spaceHexString).join("\n");
+    const expectedParts = splitMcapRecords(new Uint8Array(expectedOutput))
+      .map(spaceHexString)
+      .join("\n");
     const outputParts = splitMcapRecords(output).map(spaceHexString).join("\n");
     if (!expectedOutput.equals(output)) {
       console.error(colors.red("fail       "), path.basename(testCasePath));
@@ -152,7 +155,7 @@ async function runWriterTest(
         ) as TestCase;
         const reader = new McapStreamReader({ validateCrcs: true });
         reader.append(output);
-        const records: McapTypes.TypedMcapRecord[] = [];
+        const records: TypedMcapRecord[] = [];
         for (let record; (record = reader.nextRecord()); ) {
           records.push(record);
         }

tests/conformance/scripts/run-tests/index.ts

@@ -1,17 +1,18 @@
-import { exec } from "child_process";
-import { join } from "path";
-import { promisify } from "util";
-import { TestFeatures, TestVariant } from "variants/types";
+import { exec } from "node:child_process";
+import { join } from "node:path";
+import { promisify } from "node:util";
 
-import { IndexedReadTestRunner } from "./TestRunner";
-import { IndexedReadTestResult } from "../types";
+import { IndexedReadTestRunner } from "./TestRunner.ts";
+import { TestFeatures } from "../../../variants/types.ts";
+import type { TestVariant } from "../../../variants/types.ts";
+import type { IndexedReadTestResult } from "../types.ts";
 
 export default class CppIndexedReaderTestRunner extends IndexedReadTestRunner {
   readonly name = "cpp-indexed-reader";
 
   async runReadTest(filePath: string): Promise<IndexedReadTestResult> {
     const { stdout } = await promisify(exec)(`./indexed-reader-conformance ${filePath}`, {
-      cwd: join(__dirname, "../../../../../cpp/test/build/Debug/bin"),
+      cwd: join(import.meta.dirname, "../../../../../cpp/test/build/Debug/bin"),
     });
     return JSON.parse(stdout) as IndexedReadTestResult;
   }

tests/conformance/scripts/run-tests/runners/CppIndexedReaderTestRunner.ts

@@ -1,17 +1,17 @@
-import { exec } from "child_process";
-import { join } from "path";
-import { promisify } from "util";
-import { TestVariant } from "variants/types";
+import { exec } from "node:child_process";
+import { join } from "node:path";
+import { promisify } from "node:util";
 
-import { StreamedReadTestRunner } from "./TestRunner";
-import { StreamedReadTestResult } from "../types";
+import { StreamedReadTestRunner } from "./TestRunner.ts";
+import type { TestVariant } from "../../../variants/types.ts";
+import type { StreamedReadTestResult } from "../types.ts";
 
 export default class CppStreamedReaderTestRunner extends StreamedReadTestRunner {
   readonly name = "cpp-streamed-reader";
 
   async runReadTest(filePath: string): Promise<StreamedReadTestResult> {
     const { stdout } = await promisify(exec)(`./streamed-reader-conformance ${filePath}`, {
-      cwd: join(__dirname, "../../../../../cpp/test/build/Debug/bin"),
+      cwd: join(import.meta.dirname, "../../../../../cpp/test/build/Debug/bin"),
     });
     return JSON.parse(stdout) as StreamedReadTestResult;
   }

tests/conformance/scripts/run-tests/runners/CppStreamedReaderTestRunner.ts

@@ -1,16 +1,17 @@
-import { exec } from "child_process";
-import { join } from "path";
-import { promisify } from "util";
-import { TestFeatures, TestVariant } from "variants/types";
+import { exec } from "node:child_process";
+import { join } from "node:path";
+import { promisify } from "node:util";
 
-import { WriteTestRunner } from "./TestRunner";
+import { WriteTestRunner } from "./TestRunner.ts";
+import { TestFeatures } from "../../../variants/types.ts";
+import type { TestVariant } from "../../../variants/types.ts";
 
 export default class CppStreamedWriterTestRunner extends WriteTestRunner {
   name = "cpp-streamed-writer";
 
   async runWriteTest(filePath: string): Promise<Uint8Array> {
     const { stdout } = await promisify(exec)(`./streamed-writer-conformance ${filePath}`, {
-      cwd: join(__dirname, "../../../../../cpp/test/build/Debug/bin"),
+      cwd: join(import.meta.dirname, "../../../../../cpp/test/build/Debug/bin"),
       encoding: undefined,
     });
     return stdout as unknown as Uint8Array;