fix(BA-4330): Fix OTel aiohttp instrumentation ordering for trace propagation

Move instrument_aiohttp_server/client() from service_discovery_ctx to
server_main() before the app is frozen. The instrumentor patches the
Application class, but since root_app is already instantiated by that
point, we must manually inject the OTel server middleware into the
existing app's middleware list. This ensures incoming W3C traceparent
headers are extracted and cross-service traces are correlated.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: hhoikoo

changes/8694.feature.md

@@ -1 +1 @@
-Enable OpenTelemetry distributed tracing in Manager by activating the global TracerProvider, instrumenting aiohttp server/client for W3C Trace Context propagation, and adding OTel spans to GraphQL resolver middleware.
+Enable OpenTelemetry distributed tracing in Manager by activating the global TracerProvider and instrumenting aiohttp server/client for W3C Trace Context propagation.

src/ai/backend/logging/otel.py

@@ -69,12 +69,10 @@ def apply_otel_tracer(spec: OpenTelemetrySpec) -> None:
 
 
 def instrument_aiohttp_server() -> None:
-    # TODO: Apply after the setup procedure is decoupled from aiohttp
     AioHttpServerInstrumentor().instrument()
     logging.info("OpenTelemetry tracing for aiohttp server initialized successfully.")
 
 
 def instrument_aiohttp_client() -> None:
-    # TODO: Apply after the setup procedure is decoupled from aiohttp
     AioHttpClientInstrumentor().instrument()
     logging.info("OpenTelemetry tracing for aiohttp client initialized successfully.")

src/ai/backend/manager/cli/__main__.py

@@ -265,17 +265,11 @@ def clear_history(cli_ctx: CLIContext, retention: str, vacuum_full: bool) -> Non
     from more_itertools import chunked
 
     from ai.backend.common.validators import TimeDuration
-    from ai.backend.manager.models.agent import AgentRow
     from ai.backend.manager.models.error_logs import error_logs
     from ai.backend.manager.models.kernel import kernels
     from ai.backend.manager.models.session import SessionRow
     from ai.backend.manager.models.utils import connect_database, vacuum_db
 
-    # AgentRow must be imported to register it with SQLAlchemy's ORM mapper.
-    # KernelRow (via SessionRow) has a relationship to AgentRow using a string reference,
-    # which requires AgentRow to be in the mapper registry.
-    _ = AgentRow
-
     from .context import redis_ctx
 
     log = _get_logger()

src/ai/backend/manager/server.py

@@ -41,6 +41,9 @@
 import uvloop
 from aiohttp import web
 from aiohttp.typedefs import Handler, Middleware
+from opentelemetry.instrumentation.aiohttp_server import (
+    middleware as otel_server_middleware,
+)
 from setproctitle import setproctitle
 from zmq.auth.certs import load_certificate
 
@@ -835,8 +838,6 @@ async def service_discovery_ctx(root_ctx: RootContext) -> AsyncIterator[None]:
             service_instance_name=meta.display_name,
         )
         BraceStyleAdapter.apply_otel(otel_spec)
-        instrument_aiohttp_server()
-        instrument_aiohttp_client()
     try:
         yield
     finally:
@@ -1787,6 +1788,15 @@ async def webapp_ctx(root_app: web.Application) -> AsyncGenerator[None]:
         jwt_config = root_ctx.config_provider.config.jwt.to_jwt_config()
         root_ctx.jwt_validator = JWTValidator(jwt_config)
 
+        # TODO: Remove manual middleware injection once the manager startup is
+        # decoupled from the aiohttp Application lifecycle. Currently root_app is
+        # instantiated before OTel config is available, so instrument_aiohttp_server()
+        # (which patches the class via setattr) cannot take effect automatically.
+        if root_ctx.config_provider.config.otel.enabled:
+            instrument_aiohttp_server()
+            instrument_aiohttp_client()
+            root_app.middlewares.insert(0, otel_server_middleware)
+
         # Plugin webapps should be loaded before runner.setup() because root_app is frozen upon on_startup event.
         await manager_init_stack.enter_async_context(webapp_plugin_ctx(root_app))
         await manager_init_stack.enter_async_context(webapp_ctx(root_app))