fix: gateway conformance improvements (#85)

* feat: resolve IPFS_NS_MAP with @multiformats/dns

* fix: various fixes to gateway conformance

* fix: gateway-conformance expects 301 to path

* fix: gateway conformance subdomain handling

* fix: more testgatewaysubdomain passing tests

* fix: more testgatewaysubdomain passing tests

* fix: some redirect and url parsing

* test: make sure test constructs proper unixfs data

* fix: more gwc improvements for testgatewaysubdomains

* test: some adjustments of default tests enabled

* test: adjust total success expectation

* test: update latest sucess rates

* test: update 'all' test success rate and remove reverse proxy

* fix: pass datastore and blockstore to helia instance

* chore: lint fix and header types

* deps: update deps

* chore: modify log prefixes slightly

* chore: cleanup and re-enable tests not timing out

* fix: pull out fetch->nodejs header logic

* chore: apply suggestions from code review

* chore: lint fix after gh pr suggestions

* chore: apply suggestions from code review

Co-authored-by: Daniel Norman <1992255+2color@users.noreply.github.com>

* chore: apply suggestions from code review

Co-authored-by: Daniel Norman <1992255+2color@users.noreply.github.com>

* test: add handle-redirects tests

* chore: fix build after applying pr suggestions

---------

Co-authored-by: Daniel Norman <1992255+2color@users.noreply.github.com>

Author: SgtPooki

package.json

@@ -122,7 +122,7 @@
     "docs:no-publish": "aegir docs --publish false"
   },
   "devDependencies": {
-    "aegir": "^42.2.5",
+    "aegir": "^42.2.11",
     "npm-run-all": "^4.1.5"
   },
   "type": "module",

packages/gateway-conformance/.aegir.js

@@ -1,5 +1,7 @@
 // @ts-check
 import getPort from 'aegir/get-port'
+import { logger } from '@libp2p/logger'
+const log = logger('aegir')
 
 /** @type {import('aegir').PartialOptions} */
 export default {
@@ -12,50 +14,46 @@ export default {
 
       const { createKuboNode } = await import('./dist/src/fixtures/create-kubo.js')
       const KUBO_PORT = await getPort(3440)
+      const SERVER_PORT = await getPort(3441)
+      // The Kubo gateway will be passed to the VerifiedFetch config
       const { node: controller, gatewayUrl, repoPath } = await createKuboNode(KUBO_PORT)
       await controller.start()
       const { loadKuboFixtures } = await import('./dist/src/fixtures/kubo-mgmt.js')
       const IPFS_NS_MAP = await loadKuboFixtures(repoPath)
       const kuboGateway = gatewayUrl
 
-      const { startBasicServer } = await import('./dist/src/fixtures/basic-server.js')
-      const SERVER_PORT = await getPort(3441)
-      const stopBasicServer = await startBasicServer({
+      const { startVerifiedFetchGateway } = await import('./dist/src/fixtures/basic-server.js')
+      const stopBasicServer = await startVerifiedFetchGateway({
         serverPort: SERVER_PORT,
-        kuboGateway
-      })
-
-      const { startReverseProxy } = await import('./dist/src/fixtures/reverse-proxy.js')
-      const PROXY_PORT = await getPort(3442)
-      const stopReverseProxy = await startReverseProxy({
-        backendPort: SERVER_PORT,
-        targetHost: 'localhost',
-        proxyPort: PROXY_PORT
+        kuboGateway,
+        IPFS_NS_MAP
+      }).catch((err) => {
+        log.error(err)
       })
 
       const CONFORMANCE_HOST = 'localhost'
 
       return {
         controller,
-        stopReverseProxy,
         stopBasicServer,
         env: {
           IPFS_NS_MAP,
           CONFORMANCE_HOST,
           KUBO_PORT: `${KUBO_PORT}`,
-          PROXY_PORT: `${PROXY_PORT}`,
           SERVER_PORT: `${SERVER_PORT}`,
           KUBO_GATEWAY: kuboGateway
         }
       }
     },
     after: async (options, beforeResult) => {
       // @ts-expect-error - broken aegir types
-      await beforeResult.stopReverseProxy()
+      await beforeResult.controller.stop()
+      log('controller stopped')
+
       // @ts-expect-error - broken aegir types
       await beforeResult.stopBasicServer()
-      // @ts-expect-error - broken aegir types
-      await beforeResult.controller.stop()
+      log('basic server stopped')
+
     }
   }
 }

packages/gateway-conformance/package.json

@@ -52,17 +52,29 @@
     "test": "aegir test -t node"
   },
   "dependencies": {
+    "@helia/block-brokers": "^3.0.1",
+    "@helia/http": "^1.0.8",
     "@helia/interface": "^4.3.0",
+    "@helia/routers": "^1.1.0",
     "@helia/verified-fetch": "1.4.2",
-    "@libp2p/logger": "^4.0.11",
+    "@libp2p/kad-dht": "^12.0.17",
+    "@libp2p/logger": "^4.0.13",
+    "@libp2p/peer-id": "^4.1.2",
+    "@multiformats/dns": "^1.0.6",
     "@sgtpooki/file-type": "^1.0.1",
-    "aegir": "^42.2.5",
-    "execa": "^8.0.1",
+    "aegir": "^42.2.11",
+    "blockstore-core": "^4.4.1",
+    "datastore-core": "^9.2.9",
+    "execa": "^9.1.0",
     "fast-glob": "^3.3.2",
+    "interface-blockstore": "^5.2.10",
+    "interface-datastore": "^8.2.11",
     "ipfsd-ctl": "^14.1.0",
-    "kubo": "^0.27.0",
+    "ipns": "^9.1.0",
+    "kubo": "^0.28.0",
     "kubo-rpc-client": "^4.1.1",
-    "undici": "^6.15.0"
+    "uint8arrays": "^5.1.0",
+    "undici": "^6.18.1"
   },
   "browser": {
     "./dist/src/fixtures/create-kubo.js": "./dist/src/fixtures/create-kubo.browser.js",