Harden home connector observability and snapshot handling

packages/home-connector/Dockerfile

@@ -24,4 +24,4 @@ ENV SENTRY_TRACES_SAMPLE_RATE=1.0
 
 EXPOSE 4040
 
-CMD ["node", "index.ts"]
+CMD ["node", "--import", "./src/sentry-init.ts", "index.ts"]

packages/home-connector/index.ts

@@ -1,4 +1,9 @@
 import 'dotenv/config'
+import { initializeHomeConnectorSentry } from './src/sentry.ts'
+
+// `--import ./src/sentry-init.ts` can run before `dotenv/config`, so initialize
+// again after env-file loading to pick up `.env`-only DSNs.
+initializeHomeConnectorSentry()
 
 if (process.env.MOCKS === 'true') {
 	await import('./mocks/index.ts')

packages/home-connector/server/index.ts

@@ -2,11 +2,99 @@
 import { createRequestListener } from '@remix-run/node-fetch-server'
 import { createHomeConnectorRouter } from '../app/router.ts'
 import {
+	closeHomeConnectorSentry,
 	captureHomeConnectorException,
 	flushHomeConnectorSentry,
 } from '../src/sentry.ts'
 import { startHomeConnectorApp } from '../src/index.ts'
 
+const signalExitCodeByName = {
+	SIGINT: 130,
+	SIGTERM: 143,
+} as const
+
+function installGracefulShutdownHandlers(input: {
+	server: http.Server
+	connector: Awaited<ReturnType<typeof startHomeConnectorApp>>
+}) {
+	let shutdownPromise: Promise<void> | null = null
+
+	async function closeServerWithWatchdog() {
+		await new Promise<void>((resolve) => {
+			const watchdog = setTimeout(() => {
+				input.server.closeAllConnections()
+				resolve()
+			}, 5_000)
+			input.server.close(() => {
+				clearTimeout(watchdog)
+				resolve()
+			})
+		})
+	}
+
+	function shutdown(reason: string) {
+		if (shutdownPromise) {
+			return shutdownPromise
+		}
+
+		shutdownPromise = (async () => {
+			console.info(`Shutting down home connector reason=${reason}`)
+			input.connector.workerConnector.stop()
+			await closeServerWithWatchdog()
+			await closeHomeConnectorSentry()
+		})()
+
+		return shutdownPromise
+	}
+
+	for (const signal of ['SIGINT', 'SIGTERM'] as const) {
+		process.once(signal, () => {
+			// For clean termination, close the client so it stops accepting events
+			// before the process exits.
+			void shutdown(`signal:${signal}`).finally(() => {
+				process.exit(signalExitCodeByName[signal])
+			})
+		})
+	}
+
+	process.once('uncaughtException', (error) => {
+		captureHomeConnectorException(error, {
+			tags: {
+				area: 'process',
+				process_event: 'uncaughtException',
+			},
+		})
+		// On fatal process paths, flush buffered events but avoid relying on a full
+		// async shutdown from an undefined runtime state.
+		void flushHomeConnectorSentry().finally(() => {
+			process.exit(1)
+		})
+	})
+
+	process.once('unhandledRejection', (reason, promise) => {
+		captureHomeConnectorException(reason, {
+			tags: {
+				area: 'process',
+				process_event: 'unhandledRejection',
+			},
+			extra: {
+				...(typeof reason === 'string' ||
+				typeof reason === 'number' ||
+				typeof reason === 'boolean'
+					? { reason: String(reason) }
+					: {}),
+				reasonType: typeof reason,
+				...(reason instanceof Error ? { reasonName: reason.name } : {}),
+			},
+		})
+		// On fatal process paths, flush buffered events but avoid relying on a full
+		// async shutdown from an undefined runtime state.
+		void flushHomeConnectorSentry().finally(() => {
+			process.exit(1)
+		})
+	})
+}
+
 async function main() {
 	const connector = await startHomeConnectorApp()
 	const router = createHomeConnectorRouter(
@@ -51,6 +139,11 @@
 			`home-connector listening on http://localhost:${connector.config.port}`,
 		)
 	})
+
+	installGracefulShutdownHandlers({
+		server,
+		connector,
+	})
 }
 
 try {

packages/home-connector/src/sentry.node.test.ts

@@ -1,8 +1,24 @@
-import { expect, test } from 'vitest'
-import {
+import { expect, test, vi } from 'vitest'
+
+const sentryMock = vi.hoisted(() => ({
+	addBreadcrumb: vi.fn(),
+	close: vi.fn(),
+	flush: vi.fn(),
+	init: vi.fn(),
+	isEnabled: vi.fn(() => false),
+	setContext: vi.fn(),
+	setTag: vi.fn(),
+}))
+
+vi.mock('@sentry/node', () => sentryMock)
+
+const {
 	buildHomeConnectorSentryOptions,
+	addHomeConnectorSentryBreadcrumb,
+	closeHomeConnectorSentry,
+	flushHomeConnectorSentry,
 	initializeHomeConnectorSentry,
-} from './sentry.ts'
+} = await import('./sentry.ts')
 
 function createTemporaryEnv(values: Record<string, string | undefined>) {
 	const previousValues = Object.fromEntries(
@@ -73,6 +89,7 @@ test('buildHomeConnectorSentryOptions falls back to defaults for invalid sample
 })
 
 test('initializeHomeConnectorSentry skips initialization without a DSN', () => {
+	sentryMock.isEnabled.mockReturnValue(false)
 	using _env = createTemporaryEnv({
 		SENTRY_DSN: undefined,
 		SENTRY_ENVIRONMENT: undefined,
@@ -82,3 +99,29 @@ test('initializeHomeConnectorSentry skips initialization without a DSN', () => {
 
 	expect(() => initializeHomeConnectorSentry()).not.toThrow()
 })
+
+test('flushHomeConnectorSentry returns true when Sentry is disabled', async () => {
+	sentryMock.isEnabled.mockReturnValue(false)
+	sentryMock.flush.mockReset()
+	await expect(flushHomeConnectorSentry()).resolves.toBe(true)
+	expect(sentryMock.flush).not.toHaveBeenCalled()
+})
+
+test('closeHomeConnectorSentry returns true when Sentry is disabled', async () => {
+	sentryMock.isEnabled.mockReturnValue(false)
+	sentryMock.close.mockReset()
+	await expect(closeHomeConnectorSentry()).resolves.toBe(true)
+	expect(sentryMock.close).not.toHaveBeenCalled()
+})
+
+test('addHomeConnectorSentryBreadcrumb is a no-op when Sentry is disabled', () => {
+	sentryMock.isEnabled.mockReturnValue(false)
+	sentryMock.addBreadcrumb.mockReset()
+
+	addHomeConnectorSentryBreadcrumb({
+		message: 'Opening home connector websocket.',
+		category: 'websocket.lifecycle',
+	})
+
+	expect(sentryMock.addBreadcrumb).not.toHaveBeenCalled()
+})

packages/home-connector/src/sentry.ts

@@ -112,10 +112,40 @@ export function captureHomeConnectorMessage(
 	})
 }
 
+export function addHomeConnectorSentryBreadcrumb(input: {
+	message: string
+	category: string
+	level?: 'info' | 'warning' | 'error'
+	data?: Record<string, unknown>
+}) {
+	if (!Sentry.isEnabled()) {
+		return
+	}
+
+	Sentry.addBreadcrumb({
+		type: 'default',
+		category: input.category,
+		message: input.message,
+		level: input.level ?? 'info',
+		data: {
+			service: 'home-connector',
+			...(input.data ?? {}),
+		},
+	})
+}
+
 export async function flushHomeConnectorSentry(timeout = 2_000) {
 	if (!Sentry.isEnabled()) {
 		return true
 	}
 
 	return Sentry.flush(timeout)
 }
+
+export async function closeHomeConnectorSentry(timeout = 2_000) {
+	if (!Sentry.isEnabled()) {
+		return true
+	}
+
+	return Sentry.close(timeout)
+}