Ensure valid ENSRainbow connection before indexing starts

apps/ensindexer/src/lib/ensrainbow/singleton.ts

@@ -1,15 +1,20 @@
 import config from "@/config";
 
+import { secondsToMilliseconds } from "date-fns";
 import pRetry from "p-retry";
 
+import type { Duration } from "@ensnode/ensnode-sdk";
 import { EnsRainbowApiClient } from "@ensnode/ensrainbow-sdk";
 
+import { logger } from "@/lib/logger";
+
 const { ensRainbowUrl, labelSet } = config;
 
 if (ensRainbowUrl.href === EnsRainbowApiClient.defaultOptions().endpointUrl.href) {
-  console.warn(
-    `Using default public ENSRainbow server which may cause increased network latency. For production, use your own ENSRainbow server that runs on the same network as the ENSIndexer server.`,
-  );
+  logger.warn({
+    msg: `Using default public ENSRainbow server which may cause increased network latency`,
+    advice: `For production, use your own ENSRainbow server that runs on the same network as the ENSIndexer server.`,
+  });
 }
 
 /**
@@ -35,38 +40,58 @@ let waitForEnsRainbowToBeReadyPromise: Promise<void> | undefined;
  * Blocks execution until the ENSRainbow instance is ready to serve requests.
  *
  * Note: It may take 30+ minutes for the ENSRainbow instance to become ready in
- * a cold start scenario. We use retries for the ENSRainbow health check with
- * an exponential backoff strategy to handle this.
+ * a cold start scenario. We use retries with a fixed interval between attempts
+ * for the ENSRainbow health check to allow for ample time for ENSRainbow to
+ * become ready.
  *
  * @throws When ENSRainbow fails to become ready after all configured retry attempts.
  *         This error will trigger termination of the ENSIndexer process.
  */
 export function waitForEnsRainbowToBeReady(): Promise<void> {
-  if (waitForEnsRainbowToBeReadyPromise) {
-    return waitForEnsRainbowToBeReadyPromise;
-  }
+  if (waitForEnsRainbowToBeReadyPromise) return waitForEnsRainbowToBeReadyPromise;
+
+  logger.info({
+    msg: `Waiting for ENSRainbow instance to be ready`,
+    ensRainbowInstance: ensRainbowUrl.href,
+  });
 
-  console.log(`Waiting for ENSRainbow instance to be ready at '${ensRainbowUrl}'...`);
+  const retryInterval: Duration = 5;
+  const retryIntervalMs = secondsToMilliseconds(retryInterval);
+  const retriesPerMinute = 60 / retryInterval;
 
   waitForEnsRainbowToBeReadyPromise = pRetry(async () => ensRainbowClient.health(), {
-    retries: 12, // This allows for a total of over 1 hour of retries with the exponential backoff strategy.
-    // 1 + 2 + 4 + ... + 2048 = 2^12 - 1 = 4,095s ≈ 1h 8m
+    retries: retriesPerMinute * 60, // This allows for a total of over 1 hour of retries with `retryInterval` between attempts.
+    minTimeout: retryIntervalMs,
+    maxTimeout: retryIntervalMs,
     onFailedAttempt: ({ error, attemptNumber, retriesLeft }) => {
-      console.warn(
-        `Attempt ${attemptNumber} failed for the ENSRainbow health check at '${ensRainbowUrl}' (${error.message}). ${retriesLeft} retries left.`,
-      );
+      // Log once every minute to avoid excessive logging during ENSRainbow cold start,
+      // while still providing visibility into the retry process.
+      if (attemptNumber % 12 === 0) {
+        logger.warn({
+          msg: `ENSRainbow health check failed`,
+          attempt: attemptNumber,
+          retriesLeft,
+          error: retriesLeft === 0 ? error : undefined,
+          ensRainbowInstance: ensRainbowUrl.href,
+          advice: `This might be due to ENSRainbow having a cold start, which can take 30+ minutes.`,
+        });
+      }
     },
   })
-    .then(() => console.log(`ENSRainbow instance is ready at '${ensRainbowUrl}'.`))
+    .then(() => {
+      logger.info({
+        msg: `ENSRainbow instance is ready`,
+        ensRainbowInstance: ensRainbowUrl.href,
+      });
+    })
     .catch((error) => {
-      const errorMessage = error instanceof Error ? error.message : "Unknown error";
-
-      console.error(`ENSRainbow health check failed after multiple attempts: ${errorMessage}`);
-
-      // Throw the error to terminate the ENSIndexer process due to the failed health check of a critical dependency
-      throw new Error(errorMessage, {
-        cause: error instanceof Error ? error : undefined,
+      logger.error({
+        msg: `ENSRainbow health check failed after multiple attempts`,
+        error,
+        ensRainbowInstance: ensRainbowUrl.href,
       });
+
+      throw error;
     });
 
   return waitForEnsRainbowToBeReadyPromise;

apps/ensindexer/src/lib/graphnode-helpers.ts

@@ -4,6 +4,7 @@ import type { LabelHash, LiteralLabel } from "@ensnode/ensnode-sdk";
 import { type EnsRainbow, ErrorCode, isHealError } from "@ensnode/ensrainbow-sdk";
 
 import { ensRainbowClient } from "@/lib/ensrainbow/singleton";
+import { logger } from "@/lib/logger";
 
 /**
  * Attempt to heal a labelHash to its original label.

apps/ensindexer/src/lib/indexing-engines/ponder.test.ts

@@ -651,9 +651,7 @@ describe("addOnchainEventListener", () => {
         });
         expect.fail("Should have thrown");
       } catch (error) {
-        expect(error).toBeInstanceOf(Error);
-        expect((error as Error).message).toBe("Original connection error");
-        expect((error as Error).cause).toBe(originalError);
+        expect(error).toBe(originalError);
       }
     });
   });

apps/ensindexer/src/lib/indexing-engines/ponder.ts

@@ -109,9 +109,10 @@ const EventTypeIds = {
   Setup: "Setup",
 
   /**
-   * Onchain event
+   * Onchain log event
    *
-   * Driven by an onchain event emitted by an indexed contract.
+   * This is Ponder's `LogEvent`, driven by an onchain log event emitted by
+   * an indexed contract.
    */
   Onchain: "Onchain",
 } as const;
@@ -129,107 +130,86 @@ function buildEventTypeId(eventName: EventNames): EventTypeId {
   }
 }
 
-let preparedIndexingSetup = false;
-
 /**
  * Prepare for executing the "setup" event handlers.
  *
- * This function is guaranteed to be executed just once before
- * the first "setup" event handler is executed.
- * This is to ensure that we affect the "hot path" of
- * indexing as little as possible, since this function is
- * called as part of preconditions for every "setup" event.
+ * During Ponder startup, the "setup" event handlers are executed:
+ * - After Ponder completed database migrations for ENSIndexer Schema in ENSDb.
+ * - Before Ponder starts processing any onchain events for indexed chains.
  *
- * Note that this functions should not have any long-running operations.
- * That would delay the population of Ponder Indexing Metrics for
- * all indexed chains. Ponder Indexing Metrics are populated only after
- * all setup handlers have completed. ENSIndexer relies on
- * Ponder Indexing Metrics being immediately available on startup to
- * build and store the current Indexing Status snapshot in ENSDb.
+ * This function is useful to make sure ENSDb is ready for writes, for example,
+ * by ensuring all required Postgres extensions are installed, etc.
  */
-async function prepareIndexingSetup(): Promise<void> {
-  if (preparedIndexingSetup) {
-    return;
-  }
-
-  preparedIndexingSetup = true;
-
-  // Currently, we don't have any indexing setup preconditions.
+async function initializeIndexingSetup(): Promise<void> {
+  /**
+   * Setup event handlers should not have any *long-running* preconditions. This is because
+   * Ponder populates the indexing metrics for all indexed chains only after all setup handlers have run.
+   * ENSIndexer relies on these indexing metrics being immediately available on startup to build and
+   * store the current Indexing Status in ENSDb.
+   */
 }
 
-let preparedIndexingActivation = false;
-
 /**
  * Prepare for executing the "onchain" event handlers.
  *
- * This function is guaranteed to be executed just once before
- * the first "onchain" event handler is executed.
- * This is to ensure that we affect the "hot path" of
- * indexing as little as possible, since this function is
- * called as part of preconditions for every "onchain" event.
+ * During Ponder startup, the "onchain" event handlers are executed
+ * after all "setup" event handlers have completed.
  *
- * @throws If valid ENSRainbow connection could not be established after
- *         multiple attempts.
+ * This function is useful to make sure any long-running preconditions for
+ * onchain event handlers are met, for example, waiting for
+ * the ENSRainbow instance to be ready before processing any onchain events
+ * that require data from ENSRainbow.
  *
  * @example A single blocking precondition
  * ```ts
- * await ensureValidEnsRainbowConnection();
+ * await waitForEnsRainbowToBeReady();
  * ```
  *
- * @example Multiple concurrent blocking preconditions
+ * @example Multiple blocking preconditions
  * ```ts
  * await Promise.all([
- *   ensureValidEnsRainbowConnection(),
+ *   waitForEnsRainbowToBeReady(),
  *   waitForAnotherPrecondition(),
  * ]);
  * ```
- *
- * @example Multiple sequential blocking preconditions
- * ```ts
- * await ensureValidEnsRainbowConnection();
- * await waitForAnotherPrecondition();
- * ```
  */
-async function prepareIndexingActivation() {
-  if (preparedIndexingActivation) {
-    return;
-  }
-
-  preparedIndexingActivation = true;
-
-  try {
-    await ensureValidEnsRainbowConnection();
-  } catch (error) {
-    const errorMessage = error instanceof Error ? error.message : "Unknown error";
-
-    console.error(
-      `[Ponder Indexing Engine]: Failed to establish a valid connection to ENSRainbow: ${errorMessage}`,
-    );
-
-    // Throw the error to terminate the ENSIndexer process due to failed connection to critical dependency
-    throw new Error(errorMessage, {
-      cause: error,
-    });
-  }
+async function initializeIndexingActivation(): Promise<void> {
+  await ensureValidEnsRainbowConnection();
 }
 
+let indexingSetupPromise: Promise<void> | null = null;
+let indexingActivationPromise: Promise<void> | null = null;
+
 /**
  * Execute any necessary preconditions before running an event handler
  * for a given event type.
  *
  * Some event handlers may have preconditions that need to be met before
  * they can run.
+ *
+ * This function is idempotent and will only execute its logic once, even if
+ * called multiple times. This is to ensure that we affect the "hot path" of
+ * indexing as little as possible, since this function is called for every
+ * "onchain" event.
  */
-async function eventHandlerPreconditions<EventName extends EventNames>(
-  eventName: EventName,
-): Promise<void> {
-  const eventType = buildEventTypeId(eventName);
-
+async function eventHandlerPreconditions(eventType: EventTypeId): Promise<void> {
   switch (eventType) {
-    case EventTypeIds.Setup:
-      return prepareIndexingSetup();
+    case EventTypeIds.Setup: {
+      // Initialize the indexing setup just once
+      if (indexingSetupPromise === null) {
+        indexingSetupPromise = initializeIndexingSetup();
+      }
+
+      return indexingSetupPromise;
+    }
+
     case EventTypeIds.Onchain: {
-      return prepareIndexingActivation();
+      // Initialize the indexing activation just once
+      if (indexingActivationPromise === null) {
+        indexingActivationPromise = initializeIndexingActivation();
+      }
+
+      return indexingActivationPromise;
     }
   }
 }
@@ -249,12 +229,10 @@ export function addOnchainEventListener<EventName extends EventNames>(
   eventName: EventName,
   eventHandler: (args: IndexingEngineEventHandlerArgs<EventName>) => Promise<void> | void,
 ) {
-  return ponder.on(eventName, async ({ context, event }) => {
-    await eventHandlerPreconditions(eventName);
+  const eventType = buildEventTypeId(eventName);
 
-    await eventHandler({
-      context: buildIndexingEngineContext(context),
-      event,
-    });
+  return ponder.on(eventName, async ({ context, event }) => {
+    await eventHandlerPreconditions(eventType);
+    await eventHandler({ context: buildIndexingEngineContext(context), event });
   });
 }

apps/ensindexer/src/lib/indexing-engines/preconditions/valid-ensrainbow-connection.ts

@@ -133,7 +133,7 @@ export async function ensureValidEnsRainbowConnection(): Promise<void> {
    */
   if (!storedConfig) {
     console.log(
-      'No stored ENSRainbow Public Config found in ENSDb. Validating the omnichain indexing status is "Unstarted"...',
+      "No stored ENSRainbow Public Config found in ENSDb. Validating the omnichain indexing status is 'Unstarted'...",
     );
     /**
      * Fetch the indexing status snapshot with retries, to handle potential

packages/ensdb-sdk/src/client/ensdb-reader.ts

@@ -151,9 +151,7 @@ export class EnsDbReader<
       key: EnsNodeMetadataKeys.EnsIndexerPublicConfig,
     });
 
-    if (!record) {
-      return undefined;
-    }
+    if (!record) return undefined;
 
     return deserializeEnsIndexerPublicConfig(record);
   }
@@ -168,10 +166,6 @@ export class EnsDbReader<
       key: EnsNodeMetadataKeys.EnsRainbowPublicConfig,
     });
 
-    if (!record) {
-      return undefined;
-    }
-
     return record;
   }
 
@@ -187,9 +181,7 @@ export class EnsDbReader<
       },
     );
 
-    if (!record) {
-      return undefined;
-    }
+    if (!record) return undefined;
 
     return deserializeCrossChainIndexingStatusSnapshot(record);
   }
@@ -215,9 +207,7 @@ export class EnsDbReader<
         ),
       );
 
-    if (result.length === 0) {
-      return undefined;
-    }
+    if (result.length === 0) return undefined;
 
     if (result.length === 1 && result[0]) {
       return result[0].value as EnsNodeMetadataType["value"];