refactor: prefix all subgraph entities with subgraph_

.changeset/tame-turkeys-design.md

@@ -0,0 +1,5 @@
+---
+"ensindexer": minor
+---
+
+BREAKING: Removed support for PgLite: DATABASE_URL is now required and must be a valid PostgresQL Connection String.

.github/workflows/test_ci.yml

@@ -66,6 +66,21 @@ jobs:
   integrity-check:
     name: "Integrity Check"
     runs-on: blacksmith-4vcpu-ubuntu-2204
+    services:
+      # run postgres alongside this job
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_DB: postgres
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: password
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
@@ -79,7 +94,7 @@ jobs:
       # ensure the app healthcheck is live. If the command does not
       # print the log with the healthcheck message within that time, the step
       # will exit with a failure.
-      # This runtime check uses a pglite database that only lives in the CI
+      # This runtime check uses an ephemeral postgres database that only lives in the CI
       # environment. It will be discarded after the CI run. The app will not
       # check anything beyond the healthcheck as its job is to ensure the app
       # starts successfully only. With the configured RPCs there is likely to
@@ -92,18 +107,18 @@ jobs:
           # Public RPC URLs are used as fallbacks for repository forks
           # that don't have the relevant secrets configured.
           NAMESPACE: mainnet
+          DATABASE_URL: postgresql://postgres:password@localhost:5432/postgres
+          DATABASE_SCHEMA: public
           PLUGINS: subgraph,basenames,lineanames,threedns,protocol-acceleration,referrals,tokenscope
           RPC_URL_1: ${{ secrets.MAINNET_RPC_URL || 'https://eth.drpc.org' }}
           RPC_URL_10: ${{ secrets.OPTIMISM_RPC_URL || 'https://optimism.drpc.org' }}
           RPC_URL_8453: ${{ secrets.BASE_RPC_URL || 'https://base.drpc.org' }}
           RPC_URL_59144: ${{ secrets.LINEA_RPC_URL || 'https://linea.drpc.org' }}
           RPC_URL_42161: ${{ secrets.ARBITRUM_RPC_URL || 'https://arbitrum.drpc.org' }}
           RPC_URL_534352: ${{ secrets.SCROLL_RPC_URL || 'https://scroll.drpc.org' }}
-          HEALTH_CHECK_TIMEOUT: 60
           ENSRAINBOW_URL: https://api.ensrainbow.io
           ENSNODE_PUBLIC_URL: http://localhost:42069
           ENSINDEXER_URL: http://localhost:42069
-          # Label Set Configuration (REQUIRED)
-          LABEL_SET_ID: ens-test-env
-          LABEL_SET_VERSION: 0
+          # healthcheck script env variables
+          HEALTH_CHECK_TIMEOUT: 60
         run: ./.github/scripts/run_ensindexer_healthcheck.sh

apps/ensindexer/package.json

@@ -47,6 +47,7 @@
     "date-fns": "catalog:",
     "deepmerge-ts": "^7.1.5",
     "dns-packet": "^5.6.1",
+    "drizzle-orm": "catalog:",
     "hono": "catalog:",
     "pg-connection-string": "^2.9.0",
     "ponder": "catalog:",

apps/ensindexer/src/api/index.ts

@@ -4,11 +4,12 @@ import { db, publicClients } from "ponder:api";
 import schema from "ponder:schema";
 import { Hono } from "hono";
 import { cors } from "hono/cors";
+import { createDocumentationMiddleware } from "ponder-enrich-gql-docs-middleware";
 
 import { sdk } from "@/api/lib/tracing/instrumentation";
 import config from "@/config";
-import { makeApiDocumentationMiddleware } from "@/lib/api-documentation";
-import { filterSchemaExtensions } from "@/lib/filter-schema-extensions";
+import { makeSubgraphApiDocumentation } from "@/lib/api-documentation";
+import { filterSchemaByPrefix } from "@/lib/filter-schema-by-prefix";
 import { fixContentLengthMiddleware } from "@/lib/fix-content-length-middleware";
 import {
   fetchEnsRainbowVersion,
@@ -18,14 +19,19 @@ import {
   makePonderMetadataProvider,
 } from "@/lib/ponder-metadata-provider";
 import { ponderMetadata } from "@ensnode/ponder-metadata";
-import {
-  buildGraphQLSchema as buildSubgraphGraphQLSchema,
-  graphql as subgraphGraphQL,
-} from "@ensnode/ponder-subgraph";
+import { buildGraphQLSchema, subgraphGraphQLMiddleware } from "@ensnode/ponder-subgraph";
 
 import ensNodeApi from "@/api/handlers/ensnode-api";
-
-const schemaWithoutExtensions = filterSchemaExtensions(schema);
+import { makeDrizzle } from "@/api/lib/handlers/drizzle";
+
+// generate a subgraph-specific subset of the schema
+const subgraphSchema = filterSchemaByPrefix("subgraph_", schema);
+// and a drizzle db object that accesses it
+const subgaphDrizzle = makeDrizzle({
+  schema: subgraphSchema,
+  databaseUrl: config.databaseUrl,
+  databaseSchema: config.databaseSchemaName,
+});
 
 const app = new Hono();
 
@@ -79,43 +85,50 @@ app.get(
 // use ENSNode HTTP API at /api
 app.route("/api", ensNodeApi);
 
-// use our custom graphql middleware at /subgraph with description injection
-app.use("/subgraph", fixContentLengthMiddleware);
-app.use("/subgraph", makeApiDocumentationMiddleware("/subgraph"));
+// at /subgraph
 app.use(
   "/subgraph",
-  subgraphGraphQL({
-    db,
-    graphqlSchema: buildSubgraphGraphQLSchema({
-      schema: schemaWithoutExtensions,
-      // provide the schema with ponder's internal metadata to power _meta
+  // hotfix content length after documentation injection
+  fixContentLengthMiddleware,
+  // inject api documentation into graphql introspection requests
+  createDocumentationMiddleware(makeSubgraphApiDocumentation(), { path: "/subgraph" }),
+  // use our custom graphql middleware
+  subgraphGraphQLMiddleware({
+    drizzle: subgaphDrizzle,
+    graphqlSchema: buildGraphQLSchema({
+      schema: subgraphSchema,
+      // provide PonderMetadataProvider to power `_meta` field
       metadataProvider: makePonderMetadataProvider({ db, publicClients }),
       // describes the polymorphic (interface) relationships in the schema
       polymorphicConfig: {
         types: {
           DomainEvent: [
-            schema.transfer,
-            schema.newOwner,
-            schema.newResolver,
-            schema.newTTL,
-            schema.wrappedTransfer,
-            schema.nameWrapped,
-            schema.nameUnwrapped,
-            schema.fusesSet,
-            schema.expiryExtended,
+            subgraphSchema.transfer,
+            subgraphSchema.newOwner,
+            subgraphSchema.newResolver,
+            subgraphSchema.newTTL,
+            subgraphSchema.wrappedTransfer,
+            subgraphSchema.nameWrapped,
+            subgraphSchema.nameUnwrapped,
+            subgraphSchema.fusesSet,
+            subgraphSchema.expiryExtended,
+          ],
+          RegistrationEvent: [
+            subgraphSchema.nameRegistered,
+            subgraphSchema.nameRenewed,
+            subgraphSchema.nameTransferred,
           ],
-          RegistrationEvent: [schema.nameRegistered, schema.nameRenewed, schema.nameTransferred],
           ResolverEvent: [
-            schema.addrChanged,
-            schema.multicoinAddrChanged,
-            schema.nameChanged,
-            schema.abiChanged,
-            schema.pubkeyChanged,
-            schema.textChanged,
-            schema.contenthashChanged,
-            schema.interfaceChanged,
-            schema.authorisationChanged,
-            schema.versionChanged,
+            subgraphSchema.addrChanged,
+            subgraphSchema.multicoinAddrChanged,
+            subgraphSchema.nameChanged,
+            subgraphSchema.abiChanged,
+            subgraphSchema.pubkeyChanged,
+            subgraphSchema.textChanged,
+            subgraphSchema.contenthashChanged,
+            subgraphSchema.interfaceChanged,
+            subgraphSchema.authorisationChanged,
+            subgraphSchema.versionChanged,
           ],
         },
         fields: {

apps/ensindexer/src/api/lib/handlers/drizzle.ts

@@ -0,0 +1,36 @@
+import { Table, isTable } from "drizzle-orm";
+import { drizzle } from "drizzle-orm/node-postgres";
+import { isPgEnum } from "drizzle-orm/pg-core";
+
+type Schema = { [name: string]: unknown };
+
+// https://github.com/ponder-sh/ponder/blob/f7f6444ab8d1a870fe6492023941091df7b7cddf/packages/client/src/index.ts#L226C1-L239C3
+const setDatabaseSchema = <T extends Schema>(schema: T, schemaName: string) => {
+  for (const table of Object.values(schema)) {
+    if (isTable(table)) {
+      // @ts-ignore
+      table[Table.Symbol.Schema] = schemaName;
+    } else if (isPgEnum(table)) {
+      // @ts-ignore
+      table.schema = schemaName;
+    }
+  }
+};
+
+/**
+ * Makes a Drizzle DB object.
+ */
+export const makeDrizzle = <SCHEMA extends Schema>({
+  schema,
+  databaseUrl,
+  databaseSchema,
+}: {
+  schema: SCHEMA;
+  databaseUrl: string;
+  databaseSchema: string;
+}) => {
+  // monkeypatch schema onto tables
+  setDatabaseSchema(schema, databaseSchema);
+
+  return drizzle(databaseUrl, { schema, casing: "snake_case" });
+};

apps/ensindexer/src/api/lib/protocol-acceleration/find-resolver.ts

@@ -187,10 +187,10 @@ async function findResolverWithIndex(
       // 3. for each node, find its associated resolver (only in the specified registry)
       const nodeResolverRelations = await withSpanAsync(
         tracer,
-        "ext_nodeResolverRelation.findMany",
+        "nodeResolverRelation.findMany",
         {},
         async () => {
-          const records = await db.query.ext_nodeResolverRelation.findMany({
+          const records = await db.query.nodeResolverRelation.findMany({
             where: (nrr, { inArray, and, eq }) =>
               and(
                 eq(nrr.chainId, registry.chainId), // exclusively for the requested registry

apps/ensindexer/src/api/lib/protocol-acceleration/get-primary-name-from-index.ts

@@ -17,10 +17,10 @@ export async function getENSIP19ReverseNameRecordFromIndex(
   // retrieve from index
   const records = await withSpanAsync(
     tracer,
-    "ext_reverseNameRecord.findMany",
+    "reverseNameRecord.findMany",
     { address, coinType: coinTypeReverseLabel(coinType) },
     () =>
-      db.query.ext_reverseNameRecord.findMany({
+      db.query.reverseNameRecord.findMany({
         where: (t, { and, inArray, eq }) =>
           and(
             // address = address

apps/ensindexer/src/api/lib/protocol-acceleration/get-records-from-index.ts

@@ -27,25 +27,20 @@ export async function getRecordsFromIndex<SELECTION extends ResolverRecordsSelec
   selection: SELECTION;
 }): Promise<IndexedResolverRecords | null> {
   // fetch the Resolver Records from index
-  const resolverRecords = await withSpanAsync(
-    tracer,
-    "ext_resolverRecords.findFirst",
-    {},
-    async () => {
-      const records = await db.query.ext_resolverRecords.findFirst({
-        where: (resolver, { and, eq }) =>
-          and(
-            eq(resolver.chainId, chainId),
-            eq(resolver.resolver, resolverAddress),
-            eq(resolver.node, node),
-          ),
-        columns: { name: true },
-        with: { addressRecords: true, textRecords: true },
-      });
+  const resolverRecords = await withSpanAsync(tracer, "resolverRecords.findFirst", {}, async () => {
+    const records = await db.query.resolverRecords.findFirst({
+      where: (resolver, { and, eq }) =>
+        and(
+          eq(resolver.chainId, chainId),
+          eq(resolver.resolver, resolverAddress),
+          eq(resolver.node, node),
+        ),
+      columns: { name: true },
+      with: { addressRecords: true, textRecords: true },
+    });
 
-      return records as IndexedResolverRecords | undefined;
-    },
-  );
+    return records as IndexedResolverRecords | undefined;
+  });
 
   if (!resolverRecords) return null;
 

apps/ensindexer/src/config/config.schema.ts

@@ -145,23 +145,20 @@ const RpcConfigsSchema = z
     return rpcConfigs;
   });
 
-const DatabaseUrlSchema = z.union(
-  [
-    z.string().refine((url) => {
-      try {
-        if (!url.startsWith("postgresql://") && !url.startsWith("postgres://")) {
-          return false;
-        }
-        const config = parseConnectionString(url);
-        return !!(config.host && config.port && config.database);
-      } catch {
+const DatabaseUrlSchema = z.string().refine(
+  (url) => {
+    try {
+      if (!url.startsWith("postgresql://") && !url.startsWith("postgres://")) {
         return false;
       }
-    }),
-    z.undefined(),
-  ],
+      const config = parseConnectionString(url);
+      return !!(config.host && config.port && config.database);
+    } catch {
+      return false;
+    }
+  },
   {
-    message:
+    error:
       "Invalid PostgreSQL connection string. Expected format: postgresql://username:password@host:port/database",
   },
 );

apps/ensindexer/src/config/types.ts

@@ -149,13 +149,12 @@ export interface ENSIndexerConfig {
   indexedChainIds: Set<ChainId>;
 
   /**
-   * The database connection string for the indexer, if present. When undefined
-   * ponder will default to using an in-memory database (pglite).
+   * The database connection string for the indexer.
    *
    * Invariants:
-   * - If defined, the URL must be a valid PostgreSQL connection string
+   * - The URL must be a valid PostgreSQL connection string
    */
-  databaseUrl: string | undefined;
+  databaseUrl: string;
 
   /**
    * The "primary" ENSIndexer service URL