feat(qstash): add centralized client with DLQ and retry configuration

Add QStash infrastructure with standardized retry policy enforcement
and dead letter queue for failed job recovery.

- Create centralized QStash client with ADR-0048 retry configuration
- Add DLQ with atomic LREM, payload sanitization, and alerting
- Include error stack traces in DLQ entries for debugging
- Add signing key fallback warning for operational clarity
- Integrate with notify-collaborators job handler

Author: BjornMelin

src/app/api/jobs/notify-collaborators/route.ts

@@ -15,20 +15,50 @@ import {
 import { getServerEnvVar, getServerEnvVarWithFallback } from "@/lib/env/server";
 import { tryReserveKey } from "@/lib/idempotency/redis";
 import { sendCollaboratorNotifications } from "@/lib/notifications/collaborators";
+import { pushToDLQ } from "@/lib/qstash/dlq";
 import { withTelemetrySpan } from "@/lib/telemetry/span";
 
 /**
  * Creates a QStash receiver for webhook signature verification.
  *
- * @return QStash receiver instance or null if not configured.
+ * Logs a warning if QSTASH_NEXT_SIGNING_KEY is not set, as this
+ * could cause issues during key rotation.
+ *
+ * @return QStash receiver instance.
  */
 function getQstashReceiver(): Receiver {
   const current = getServerEnvVar("QSTASH_CURRENT_SIGNING_KEY") as string;
-  const next = getServerEnvVarWithFallback(
-    "QSTASH_NEXT_SIGNING_KEY",
-    current
-  ) as string;
-  return new Receiver({ currentSigningKey: current, nextSigningKey: next });
+  const next = getServerEnvVarWithFallback("QSTASH_NEXT_SIGNING_KEY", "");
+
+  if (!next) {
+    // Log warning about fallback to help operators during key rotation
+    console.warn(
+      "[QStash Worker] QSTASH_NEXT_SIGNING_KEY not configured. " +
+        "Using current key for both. This is normal for regular operation " +
+        "but may cause request failures during key rotation if not addressed. " +
+        "See: https://upstash.com/docs/qstash/howto/signature-validation"
+    );
+  }
+
+  return new Receiver({
+    currentSigningKey: current,
+    nextSigningKey: next || current,
+  });
+}
+
+/** Max retries configured for QStash (per ADR-0048) */
+const MAX_RETRIES = 5;
+
+/**
+ * Extract retry attempt information from QStash headers.
+ *
+ * @param req - Incoming request
+ * @return Object with current attempt and max retries
+ */
+function getRetryInfo(req: Request): { attempt: number; maxRetries: number } {
+  const retried = Number(req.headers.get("Upstash-Retried")) || 0;
+  const maxRetries = Number(req.headers.get("Upstash-Max-Retries")) || MAX_RETRIES;
+  return { attempt: retried + 1, maxRetries };
 }
 
 /**
@@ -42,6 +72,13 @@ export async function POST(req: Request) {
     "jobs.notify-collaborators",
     { attributes: { route: "/api/jobs/notify-collaborators" } },
     async (span) => {
+      const { attempt, maxRetries } = getRetryInfo(req);
+      span.setAttribute("qstash.attempt", attempt);
+      span.setAttribute("qstash.max_retries", maxRetries);
+
+      // Store parsed job data for DLQ on failure
+      let jobPayload: unknown = null;
+
       try {
         let receiver: Receiver;
         try {
@@ -82,6 +119,7 @@ export async function POST(req: Request) {
         }
 
         const json = (await req.json()) as unknown;
+        jobPayload = json; // Store for DLQ
         const validation = validateSchema(notifyJobSchema, json);
         if ("error" in validation) {
           return validation.error;
@@ -102,6 +140,23 @@ export async function POST(req: Request) {
         return NextResponse.json({ ok: true, ...result });
       } catch (error) {
         span.recordException(error as Error);
+
+        // Check if this is the final retry attempt
+        const isFinalAttempt = attempt >= maxRetries;
+        span.setAttribute("qstash.final_attempt", isFinalAttempt);
+
+        if (isFinalAttempt) {
+          // Push to DLQ on final failure per ADR-0048
+          const dlqEntryId = await pushToDLQ(
+            "notify-collaborators",
+            jobPayload,
+            error,
+            attempt
+          );
+          span.setAttribute("qstash.dlq", true);
+          span.setAttribute("qstash.dlq_entry_id", dlqEntryId ?? "unavailable");
+        }
+
         return errorResponse({
           err: error,
           error: "internal",