diff --git a/mkdocs/docs/concepts/gateways.md b/mkdocs/docs/concepts/gateways.md
index c84c19af76..9239f915c4 100644
--- a/mkdocs/docs/concepts/gateways.md
+++ b/mkdocs/docs/concepts/gateways.md
@@ -247,6 +247,40 @@ $ dstack gateway list
Replicated gateways do not support automatic certificate issuance via Let's Encrypt (`certificate: { type: lets-encrypt }`). For TLS termination, use an external load balancer or one of the other [certificate types](#certificate).
+### OpenTelemetry
+
+The gateway can export its traces, logs, and metrics via OTLP over HTTP.
+
+
+
+```yaml
+type: gateway
+name: example-gateway
+
+backend: aws
+region: eu-west-1
+
+domain: example.com
+
+otel:
+ endpoint: https://otlp.example.com:4318
+ headers:
+ Authorization: Bearer ${{ secrets.otlp_token }}
+ traces: true
+ logs: true
+ metrics: true
+```
+
+
+
+!!! warning "Use secrets for credentials"
+ The `otel` configuration is visible to everyone who can view the gateway, including members of
+ projects the gateway is [exported](exports.md) to. Store credentials, such as authorization
+ headers, in [secrets](secrets.md) and reference them as `${{ secrets. }}`. Only the
+ references are visible, not the secret values.
+
+Telemetry is reported with `service.name` set to `dstack-gateway` and resource attributes identifying the gateway and its replica.
+
!!! info "Reference"
For all gateway configuration options, refer to the [reference](../reference/dstack.yml/gateway.md).
diff --git a/mkdocs/docs/concepts/secrets.md b/mkdocs/docs/concepts/secrets.md
index 0528e10c09..91127a68f0 100644
--- a/mkdocs/docs/concepts/secrets.md
+++ b/mkdocs/docs/concepts/secrets.md
@@ -84,7 +84,7 @@ OK
## Use secrets
-You can use the `${{ secrets. }}` syntax to reference secrets in run configurations. Currently, secrets interpolation is supported in `env` and `registry_auth` properties.
+You can use the `${{ secrets. }}` syntax to reference secrets in run configurations. Currently, secrets interpolation is supported in `env` and `registry_auth` properties. Gateway configurations also support secrets in [`otel`](gateways.md#opentelemetry) `endpoint` and `headers`.
### `env`
diff --git a/mkdocs/docs/guides/server-deployment.md b/mkdocs/docs/guides/server-deployment.md
index bd26538acb..7e0cfca24a 100644
--- a/mkdocs/docs/guides/server-deployment.md
+++ b/mkdocs/docs/guides/server-deployment.md
@@ -459,6 +459,10 @@ $ DSTACK_OTEL_TRACES_ENABLED=1 \
`rate()` queries may be empty. Set e.g. `OTEL_METRIC_EXPORT_INTERVAL=30000`,
and make sure the datasource's `timeInterval` reflects the cadence you choose.
+#### Gateways
+
+Gateways can export their own telemetry. It is configured per gateway, see [gateway OpenTelemetry](../concepts/gateways.md#opentelemetry).
+
## Encryption
By default, `dstack` stores data in plaintext. To enforce encryption, you
diff --git a/mkdocs/docs/reference/dstack.yml/gateway.md b/mkdocs/docs/reference/dstack.yml/gateway.md
index d52fac9e0a..ae44170246 100644
--- a/mkdocs/docs/reference/dstack.yml/gateway.md
+++ b/mkdocs/docs/reference/dstack.yml/gateway.md
@@ -47,3 +47,9 @@ Set to `null` to disable certificates (e.g. for [private gateways](../../concept
show_root_heading: false
type:
required: true
+
+### `otel`
+
+#SCHEMA# dstack._internal.core.models.gateways.GatewayOTelConfig
+ overrides:
+ show_root_heading: false
diff --git a/pyproject.toml b/pyproject.toml
index 3912c45065..4f0cf85baa 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -226,6 +226,13 @@ gateway = [
# Gateway authorization needs the pool cancellation and recovery fixes in httpx2 (#4338).
"httpx2>=2.13.1",
"jinja2",
+ # Always installed so that OpenTelemetry can be enabled without reinstalling the gateway.
+ "opentelemetry-sdk>=1.39.0",
+ "opentelemetry-exporter-otlp-proto-http>=1.39.0",
+ "opentelemetry-instrumentation-fastapi>=0.51b0",
+ "opentelemetry-instrumentation-httpx>=0.65b0",
+ "opentelemetry-instrumentation-system-metrics>=0.51b0",
+ "opentelemetry-instrumentation-logging>=0.61b0",
]
server = [
"fastapi",
@@ -311,8 +318,8 @@ fluentbit = [
"dstack[server]",
]
otel = [
- "opentelemetry-sdk>=1.30.0",
- "opentelemetry-exporter-otlp-proto-http>=1.30.0",
+ "opentelemetry-sdk>=1.39.0",
+ "opentelemetry-exporter-otlp-proto-http>=1.39.0",
"opentelemetry-exporter-prometheus>=0.51b0",
"opentelemetry-instrumentation-fastapi>=0.51b0",
"opentelemetry-instrumentation-sqlalchemy>=0.51b0",
diff --git a/src/dstack/_internal/core/compatibility/gateways.py b/src/dstack/_internal/core/compatibility/gateways.py
index 63798b4f79..aab17b08a3 100644
--- a/src/dstack/_internal/core/compatibility/gateways.py
+++ b/src/dstack/_internal/core/compatibility/gateways.py
@@ -71,5 +71,7 @@ def _get_gateway_configuration_excludes(
configuration_excludes["default"] = True
if configuration.load_balancer is None:
configuration_excludes["load_balancer"] = True
+ if configuration.otel is None:
+ configuration_excludes["otel"] = True
return configuration_excludes
diff --git a/src/dstack/_internal/core/models/gateways.py b/src/dstack/_internal/core/models/gateways.py
index bff16d3e57..0eb7067b21 100644
--- a/src/dstack/_internal/core/models/gateways.py
+++ b/src/dstack/_internal/core/models/gateways.py
@@ -90,6 +90,53 @@ class ALBGatewayLoadBalancer(CoreModel):
AnyGatewayLoadBalancer = Union[ALBGatewayLoadBalancer]
+class GatewayOTelConfig(CoreModel):
+ endpoint: Annotated[
+ str,
+ Field(
+ description=(
+ "The OTLP over HTTP endpoint, e.g. `https://otlp.example.com:4318`."
+ " Signal paths such as `/v1/traces` are appended to it."
+ " Can reference secrets, e.g. `${{ secrets.otlp_endpoint }}`"
+ ),
+ ),
+ ]
+ headers: Annotated[
+ Optional[Dict[str, str]],
+ Field(
+ description=(
+ "Headers sent with OTLP requests, e.g. for authorization."
+ " Values can reference secrets, e.g. `Bearer ${{ secrets.otlp_token }}`"
+ ),
+ ),
+ ] = None
+ traces: Annotated[bool, Field(description="Whether to export traces")] = False
+ logs: Annotated[bool, Field(description="Whether to export logs")] = False
+ metrics: Annotated[bool, Field(description="Whether to export metrics")] = False
+ traces_sample_rate: Annotated[
+ float,
+ Field(
+ description=(
+ "The head sampling rate for traces."
+ " The default assumes sampling is done downstream, e.g. in an OTel Collector"
+ ),
+ ge=0,
+ le=1,
+ ),
+ ] = 1.0
+ traces_auth_sample_rate: Annotated[
+ float,
+ Field(
+ description=(
+ "The head sampling rate for traces of authorization subrequests,"
+ " which Nginx makes for every request to services with authorization enabled"
+ ),
+ ge=0,
+ le=1,
+ ),
+ ] = 0.01
+
+
class GatewayConfiguration(CoreModel):
type: Literal["gateway"] = "gateway"
name: Annotated[Optional[str], Field(description="The gateway name")] = None
@@ -166,6 +213,10 @@ class GatewayConfiguration(CoreModel):
)
),
] = None
+ otel: Annotated[
+ Optional[GatewayOTelConfig],
+ Field(description=("The OpenTelemetry export configuration of the gateway app")),
+ ] = None
_validate_tags = field_validator("tags", mode="before")(tags_validator)
diff --git a/src/dstack/_internal/proxy/gateway/app.py b/src/dstack/_internal/proxy/gateway/app.py
index 6627e9e3f3..72946b15da 100644
--- a/src/dstack/_internal/proxy/gateway/app.py
+++ b/src/dstack/_internal/proxy/gateway/app.py
@@ -7,6 +7,7 @@
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
+from dstack._internal.proxy.gateway import otel
from dstack._internal.proxy.gateway.auth import GatewayProxyAuthProvider
from dstack._internal.proxy.gateway.const import (
DSTACK_DIR_ON_GATEWAY,
@@ -43,6 +44,7 @@ async def lifespan(app: FastAPI):
repo = await get_gateway_proxy_repo(await injector.get_repo().__anext__())
nginx = injector.get_nginx()
service_conn_pool = await injector.get_service_connection_pool()
+ otel.apply_settings((await repo.get_config()).otel_settings)
await run_async(nginx.write_global_conf)
await apply_all(repo, nginx, service_conn_pool)
diff --git a/src/dstack/_internal/proxy/gateway/main.py b/src/dstack/_internal/proxy/gateway/main.py
index 134f34f538..76681a8349 100644
--- a/src/dstack/_internal/proxy/gateway/main.py
+++ b/src/dstack/_internal/proxy/gateway/main.py
@@ -1,5 +1,6 @@
import logging
+from dstack._internal.proxy.gateway import otel
from dstack._internal.proxy.gateway.app import make_app
@@ -15,3 +16,5 @@ def configure_logging(level: int = logging.INFO):
configure_logging(logging.DEBUG)
app = make_app()
+# Not in make_app, since global instrumentation would leak between apps created in tests
+otel.instrument(app)
diff --git a/src/dstack/_internal/proxy/gateway/models.py b/src/dstack/_internal/proxy/gateway/models.py
index 80ce0b8abe..e1d4746922 100644
--- a/src/dstack/_internal/proxy/gateway/models.py
+++ b/src/dstack/_internal/proxy/gateway/models.py
@@ -1,6 +1,6 @@
"""Things stored in GatewayProxyRepo in addition to those from BaseProxyRepo."""
-from typing import Optional
+from typing import Optional, Union
from pydantic import AnyHttpUrl
@@ -21,5 +21,21 @@ class ACMESettings(ImmutableModel):
eab_hmac_key: Optional[str] = None
+class OTelSettings(ImmutableModel):
+ traces_enabled: bool
+ logs_enabled: bool
+ metrics_enabled: bool
+ endpoint: str
+ headers: dict[str, str]
+ traces_sample_rate: float
+ traces_auth_sample_rate: float
+ resource_attributes: dict[str, Union[str, int]]
+
+ @property
+ def any_enabled(self) -> bool:
+ return self.traces_enabled or self.logs_enabled or self.metrics_enabled
+
+
class GlobalProxyConfig(ImmutableModel):
acme_settings: ACMESettings = ACMESettings()
+ otel_settings: Optional[OTelSettings] = None
diff --git a/src/dstack/_internal/proxy/gateway/otel.py b/src/dstack/_internal/proxy/gateway/otel.py
new file mode 100644
index 0000000000..88b0fb8d58
--- /dev/null
+++ b/src/dstack/_internal/proxy/gateway/otel.py
@@ -0,0 +1,104 @@
+import logging
+from typing import Literal, Optional
+
+from fastapi import FastAPI
+from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.httpx import HTTPX2ClientInstrumentor, HTTPXClientInstrumentor
+from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace.sampling import ParentBased
+
+from dstack._internal import settings as core_settings
+from dstack._internal.proxy.gateway.models import OTelSettings
+from dstack._internal.utils import otel as otel_utils
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+_AUTH_SPAN_NAME_PREFIX = "GET /api/auth/"
+
+_applied_settings: Optional[OTelSettings] = None
+
+
+def instrument(app: FastAPI) -> None:
+ """Must be called before the app starts serving. No-op until `apply_settings` is called."""
+ FastAPIInstrumentor.instrument_app(app)
+ HTTPXClientInstrumentor().instrument()
+ HTTPX2ClientInstrumentor().instrument()
+ SystemMetricsInstrumentor(config=otel_utils.PROCESS_METRICS_CONFIG).instrument()
+
+
+def apply_settings(settings: Optional[OTelSettings]) -> bool:
+ """
+ Returns True if the app must be restarted to apply `settings`,
+ since OpenTelemetry providers can only be set once per process.
+ """
+ global _applied_settings
+ if settings is not None and not settings.any_enabled:
+ settings = None
+ if settings == _applied_settings:
+ return False
+ if _applied_settings is not None:
+ logger.info("OpenTelemetry settings changed, the gateway app needs a restart")
+ return True
+ assert settings is not None
+ # Set before configuring, since a failed configuration cannot be retried in this process
+ _applied_settings = settings
+ try:
+ _configure(settings)
+ except Exception:
+ logger.exception("Failed to configure OpenTelemetry")
+ return False
+
+
+def _configure(settings: OTelSettings) -> None:
+ resource = _get_resource(settings)
+ if settings.traces_enabled:
+ otel_utils.configure_tracing(
+ resource,
+ ParentBased(root=_build_root_sampler(settings)),
+ OTLPSpanExporter(
+ endpoint=_get_signal_endpoint(settings, "traces"), headers=settings.headers
+ ),
+ )
+ if settings.metrics_enabled:
+ exporter = OTLPMetricExporter(
+ endpoint=_get_signal_endpoint(settings, "metrics"), headers=settings.headers
+ )
+ otel_utils.configure_metrics(resource, [PeriodicExportingMetricReader(exporter)])
+ if settings.logs_enabled:
+ otel_utils.configure_log_export(
+ resource,
+ OTLPLogExporter(
+ endpoint=_get_signal_endpoint(settings, "logs"), headers=settings.headers
+ ),
+ # The app logs at DEBUG level to journald, which would be too noisy to export
+ level=logging.INFO,
+ )
+
+
+def _build_root_sampler(settings: OTelSettings) -> otel_utils.RootSpanNameSampler:
+ return otel_utils.RootSpanNameSampler(
+ default_rate=settings.traces_sample_rate,
+ rates_by_name_prefix={_AUTH_SPAN_NAME_PREFIX: settings.traces_auth_sample_rate},
+ )
+
+
+def _get_signal_endpoint(
+ settings: OTelSettings, signal: Literal["traces", "metrics", "logs"]
+) -> str:
+ return f"{settings.endpoint.rstrip('/')}/v1/{signal}"
+
+
+def _get_resource(settings: OTelSettings) -> Resource:
+ return Resource.create(
+ {
+ **settings.resource_attributes,
+ "service.name": "dstack-gateway",
+ "service.version": core_settings.DSTACK_VERSION or "dev",
+ }
+ )
diff --git a/src/dstack/_internal/proxy/gateway/resources/nginx/entrypoint.jinja2 b/src/dstack/_internal/proxy/gateway/resources/nginx/entrypoint.jinja2
index ce0160f558..fd4029c946 100644
--- a/src/dstack/_internal/proxy/gateway/resources/nginx/entrypoint.jinja2
+++ b/src/dstack/_internal/proxy/gateway/resources/nginx/entrypoint.jinja2
@@ -4,6 +4,10 @@ server {
proxy_pass http://localhost:{{ proxy_port }}/api/models/{{ project_name }}/;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
+ # Don't let public clients control the gateway app's tracing
+ proxy_set_header traceparent "";
+ proxy_set_header tracestate "";
+ proxy_set_header baggage "";
proxy_read_timeout {{ read_timeout }}s;
}
listen 80;
diff --git a/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 b/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2
index e3eced5f6d..002c8c3687 100644
--- a/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2
+++ b/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2
@@ -109,6 +109,10 @@ server {
proxy_set_header Content-Length "";
proxy_set_header X-Original-URI $request_uri;
proxy_set_header Authorization $http_authorization;
+ # Don't let public clients control the gateway app's tracing
+ proxy_set_header traceparent "";
+ proxy_set_header tracestate "";
+ proxy_set_header baggage "";
}
{% endif %}
diff --git a/src/dstack/_internal/proxy/gateway/routers/config.py b/src/dstack/_internal/proxy/gateway/routers/config.py
index 24dac6a52f..25b5862c4f 100644
--- a/src/dstack/_internal/proxy/gateway/routers/config.py
+++ b/src/dstack/_internal/proxy/gateway/routers/config.py
@@ -1,7 +1,10 @@
+import os
+import signal
from typing import Annotated
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, BackgroundTasks, Depends
+from dstack._internal.proxy.gateway import otel
from dstack._internal.proxy.gateway.deps import get_gateway_proxy_repo
from dstack._internal.proxy.gateway.models import ACMESettings, GlobalProxyConfig
from dstack._internal.proxy.gateway.repo.repo import GatewayProxyRepo
@@ -15,6 +18,7 @@
async def update_global_config(
body: ConfigRequest,
repo: Annotated[GatewayProxyRepo, Depends(get_gateway_proxy_repo)],
+ background_tasks: BackgroundTasks,
) -> OkResponse:
await repo.set_config(
GlobalProxyConfig(
@@ -23,6 +27,10 @@ async def update_global_config(
eab_kid=body.acme_eab_kid,
eab_hmac_key=body.acme_eab_hmac_key,
),
+ otel_settings=body.otel_settings,
),
)
+ if otel.apply_settings(body.otel_settings):
+ # Saved settings are applied on startup. systemd restarts the app (`Restart=always`)
+ background_tasks.add_task(os.kill, os.getpid(), signal.SIGTERM)
return OkResponse()
diff --git a/src/dstack/_internal/proxy/gateway/schemas/config.py b/src/dstack/_internal/proxy/gateway/schemas/config.py
index 09cfc2101f..85b67c32f9 100644
--- a/src/dstack/_internal/proxy/gateway/schemas/config.py
+++ b/src/dstack/_internal/proxy/gateway/schemas/config.py
@@ -2,8 +2,11 @@
from pydantic import AnyHttpUrl, BaseModel
+from dstack._internal.proxy.gateway.models import OTelSettings
+
class ConfigRequest(BaseModel):
acme_server: Optional[AnyHttpUrl] = None
acme_eab_kid: Optional[str] = None
acme_eab_hmac_key: Optional[str] = None
+ otel_settings: Optional[OTelSettings] = None
diff --git a/src/dstack/_internal/proxy/gateway/testing/common.py b/src/dstack/_internal/proxy/gateway/testing/common.py
index 2718ec6bae..1feb374c0e 100644
--- a/src/dstack/_internal/proxy/gateway/testing/common.py
+++ b/src/dstack/_internal/proxy/gateway/testing/common.py
@@ -1,7 +1,9 @@
from dataclasses import dataclass
-from typing import Union
+from typing import Any, Union
from unittest.mock import AsyncMock, MagicMock
+from dstack._internal.proxy.gateway.models import OTelSettings
+
AnyMock = Union[MagicMock, AsyncMock]
@@ -11,3 +13,17 @@ class Mocks:
run_certbot: AnyMock
open_conn: AnyMock
close_conn: AnyMock
+
+
+def make_otel_settings(**kwargs: Any) -> OTelSettings:
+ defaults: dict[str, Any] = {
+ "traces_enabled": False,
+ "logs_enabled": False,
+ "metrics_enabled": False,
+ "endpoint": "https://otlp.example.com",
+ "headers": {},
+ "traces_sample_rate": 1.0,
+ "traces_auth_sample_rate": 0.01,
+ "resource_attributes": {},
+ }
+ return OTelSettings(**{**defaults, **kwargs})
diff --git a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py
index 63da7292ef..95460f6c23 100644
--- a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py
+++ b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py
@@ -295,6 +295,7 @@ async def _load_gateway_replica(
replica_fields: list[InstrumentedAttribute[Any]],
gateway_fields: list[InstrumentedAttribute[Any]],
load_backends: bool = False,
+ load_gateway_project: bool = False,
load_gateway_backend_type: bool = False,
) -> Optional[GatewayReplicaModel]:
def build_gateway_options(
@@ -302,10 +303,11 @@ def build_gateway_options(
) -> list[ExecutableOption]:
gateway_load = joinedload(gateway_attr).load_only(*gateway_fields)
options: list[ExecutableOption] = [gateway_load]
- if load_backends:
- options.append(
- gateway_load.joinedload(GatewayModel.project).selectinload(ProjectModel.backends)
- )
+ if load_gateway_project or load_backends:
+ project_load = gateway_load.joinedload(GatewayModel.project)
+ if load_backends:
+ project_load = project_load.selectinload(ProjectModel.backends)
+ options.append(project_load)
if load_gateway_backend_type:
options.append(
gateway_load.joinedload(GatewayModel.backend).load_only(BackendModel.type)
@@ -515,6 +517,7 @@ async def _process_provisioning_item(item: GatewayReplicaPipelineItem):
GatewayReplicaModel.instance_id,
GatewayReplicaModel.backend_id,
GatewayReplicaModel.configuration,
+ GatewayReplicaModel.region,
],
gateway_fields=_GATEWAY_FIELDS_MIN
+ [
@@ -525,6 +528,7 @@ async def _process_provisioning_item(item: GatewayReplicaPipelineItem):
GatewayModel.backend_data,
],
load_backends=True,
+ load_gateway_project=True,
load_gateway_backend_type=True,
)
if replica_model is None:
@@ -539,7 +543,7 @@ async def _process_provisioning_item(item: GatewayReplicaPipelineItem):
if _is_legacy_aws_acm_gateway_with_pending_migration(gateway_model):
await _commit_update(item, replica_model, update_map={})
return
- error = await _connect_and_configure_gateway_replica(replica_model)
+ error = await _connect_and_configure_gateway_replica(gateway_model, replica_model)
if error is not None:
await _commit_update(
item,
@@ -604,6 +608,7 @@ async def _register_replica_with_load_balancer(
async def _connect_and_configure_gateway_replica(
+ gateway_model: GatewayModel,
gateway_replica: GatewayReplicaModel,
) -> Optional[str]:
"""Returns an error message on failure, None on success."""
@@ -618,7 +623,11 @@ async def _connect_and_configure_gateway_replica(
if connection is None:
return "Failed to connect to gateway replica"
try:
- await gateways_services.configure_gateway_replica(connection)
+ async with get_session_ctx() as session:
+ otel_settings = await gateways_services.resolve_gateway_replica_otel_settings(
+ session, gateway_model, gateway_replica
+ )
+ await gateways_services.configure_gateway_replica(connection, otel_settings)
except Exception:
logger.exception("%s: failed to configure gateway replica", fmt(gateway_replica))
return "Failed to configure gateway replica"
@@ -634,6 +643,7 @@ async def _process_running_item(item: GatewayReplicaPipelineItem):
GatewayReplicaModel.scale_in,
GatewayReplicaModel.ip_address,
GatewayReplicaModel.ssh_private_key,
+ GatewayReplicaModel.region,
],
gateway_fields=_GATEWAY_FIELDS_MIN
+ [
@@ -641,6 +651,7 @@ async def _process_running_item(item: GatewayReplicaPipelineItem):
GatewayModel.region,
GatewayModel.wildcard_domain,
],
+ load_gateway_project=True,
load_gateway_backend_type=True,
)
if replica_model is None:
@@ -712,8 +723,12 @@ async def _process_running_item(item: GatewayReplicaPipelineItem):
# lost its global config along with its services, and certbot would fall back to
# its default ACME server, so resend the config first.
try:
+ async with get_session_ctx() as session:
+ otel_settings = await gateways_services.resolve_gateway_replica_otel_settings(
+ session, gateway_model, replica_model
+ )
async with connection.client() as client:
- await client.submit_gateway_config()
+ await client.submit_gateway_config(otel_settings)
except Exception as e:
logger.warning("%s: failed to submit gateway config: %r", fmt(replica_model), e)
await _commit_update(item, replica_model, update_map={})
diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py
index 678649426d..18650bb00b 100644
--- a/src/dstack/_internal/server/services/gateways/__init__.py
+++ b/src/dstack/_internal/server/services/gateways/__init__.py
@@ -2,10 +2,9 @@
import datetime
import itertools
import uuid
-from collections.abc import AsyncGenerator
+from collections.abc import AsyncGenerator, Mapping
from contextlib import asynccontextmanager
from datetime import timedelta
-from functools import partial
from typing import List, Optional, Sequence
import httpx
@@ -53,6 +52,7 @@
from dstack._internal.core.services.diff import ModelDiff, format_diff_fields_for_event
from dstack._internal.core.services.gateways import diff_gateway_configurations
from dstack._internal.proxy.gateway.const import SERVICE_SCALING_WINDOWS
+from dstack._internal.proxy.gateway.models import OTelSettings
from dstack._internal.proxy.gateway.schemas.stats import PerWindowStats, Stat
from dstack._internal.server import settings
from dstack._internal.server.db import get_db, is_db_postgres, is_db_sqlite
@@ -72,6 +72,12 @@
)
from dstack._internal.server.services.gateways.connection import GatewayConnection
from dstack._internal.server.services.gateways.pool import gateway_connections_pool
+from dstack._internal.server.services.gateways.telemetry import (
+ InvalidGatewayOTelConfigError,
+ get_gateway_replica_otel_settings,
+ is_gateway_otel_enabled,
+ resolve_gateway_otel_config,
+)
from dstack._internal.server.services.locking import (
advisory_lock_ctx,
get_locker,
@@ -79,6 +85,7 @@
)
from dstack._internal.server.services.pipelines import PipelineHinterProtocol
from dstack._internal.server.services.plugins import apply_plugin_policies
+from dstack._internal.server.services.secrets import get_project_secrets_mapping
from dstack._internal.server.utils.common import gather_map_async
from dstack._internal.utils import crypto
from dstack._internal.utils.common import (
@@ -252,7 +259,7 @@ async def create_gateway(
spec=GatewaySpec(configuration=configuration),
)
effective_configuration = spec.configuration
- _validate_gateway_configuration(effective_configuration)
+ await _validate_gateway_configuration(session, project, effective_configuration)
configuration = effective_configuration
# Leasing the backend also checks that it can be initialized, not just that it's configured.
@@ -746,13 +753,46 @@ def _merge_per_window_stats(stats_per_gateway_replica: list[PerWindowStats]) ->
async def init_gateways(session: AsyncSession):
res = await session.execute(
- select(GatewayReplicaModel).where(
+ select(GatewayReplicaModel)
+ .where(
GatewayReplicaModel.status == GatewayReplicaStatus.RUNNING,
GatewayReplicaModel.active == True,
GatewayReplicaModel.deleted == False,
)
+ .options(
+ *(
+ joinedload(gateway_attr).options(
+ joinedload(GatewayModel.project),
+ joinedload(GatewayModel.backend).load_only(BackendModel.type),
+ )
+ for gateway_attr in (
+ GatewayReplicaModel.gateway,
+ GatewayReplicaModel.legacy_gateway,
+ )
+ )
+ )
)
- gateway_replicas = res.scalars().all()
+ gateway_replicas = res.unique().scalars().all()
+ otel_settings: dict[str, Optional[OTelSettings]] = {}
+ secrets_cache: dict[uuid.UUID, Mapping[str, str]] = {}
+ for gateway_replica in gateway_replicas:
+ if gateway_replica.ip_address is None:
+ continue
+ try:
+ replica_otel_settings = await resolve_gateway_replica_otel_settings(
+ session,
+ gateway_replica.gateway or gateway_replica.legacy_gateway,
+ gateway_replica,
+ secrets_cache=secrets_cache,
+ )
+ except Exception:
+ # Don't let one gateway prevent initializing the others
+ logger.exception(
+ "Failed to resolve OpenTelemetry settings for gateway replica %s",
+ gateway_replica.ip_address,
+ )
+ replica_otel_settings = None
+ otel_settings[gateway_replica.ip_address] = replica_otel_settings
if len(gateway_replicas) > 0:
logger.info(
@@ -798,9 +838,9 @@ async def init_gateways(session: AsyncSession):
gateway_replica.app_updated_at = get_current_datetime()
for connection, error in await gather_map_async(
- await gateway_connections_pool.all(),
+ [c for c in await gateway_connections_pool.all() if c.ip_address in otel_settings],
# Need several attempts to handle short gateway replica downtime after update
- partial(configure_gateway_replica, attempts=7),
+ lambda c: configure_gateway_replica(c, otel_settings[c.ip_address], attempts=7),
return_exceptions=True,
):
if isinstance(error, Exception):
@@ -848,6 +888,7 @@ def _recently_updated(gateway_replica_model: GatewayReplicaModel) -> bool:
async def configure_gateway_replica(
connection: GatewayConnection,
+ otel_settings: Optional[OTelSettings],
attempts: int = GATEWAY_CONFIGURE_ATTEMPTS,
) -> None:
"""
@@ -860,7 +901,7 @@ async def configure_gateway_replica(
for attempt in range(attempts - 1):
try:
async with connection.client() as client:
- await client.submit_gateway_config()
+ await client.submit_gateway_config(otel_settings)
break
except httpx.RequestError as e:
logger.debug(
@@ -873,11 +914,35 @@ async def configure_gateway_replica(
await asyncio.sleep(GATEWAY_CONFIGURE_DELAY)
else:
async with connection.client() as client:
- await client.submit_gateway_config()
+ await client.submit_gateway_config(otel_settings)
logger.info("Gateway replica %s configured", connection.ip_address)
+async def resolve_gateway_replica_otel_settings(
+ session: AsyncSession,
+ gateway_model: Optional[GatewayModel],
+ replica_model: GatewayReplicaModel,
+ secrets_cache: Optional[dict[uuid.UUID, Mapping[str, str]]] = None,
+) -> Optional[OTelSettings]:
+ """
+ `secrets_cache` maps project IDs to project secrets. When resolving settings
+ for many replicas, pass the same dict to fetch each project's secrets only once.
+ """
+ if gateway_model is None:
+ return None
+ configuration = get_gateway_configuration(gateway_model)
+ if configuration.otel is None or not is_gateway_otel_enabled(configuration.otel):
+ return None
+ if secrets_cache is None:
+ secrets_cache = {}
+ secrets = secrets_cache.get(gateway_model.project_id)
+ if secrets is None:
+ secrets = await get_project_secrets_mapping(session=session, project=gateway_model.project)
+ secrets_cache[gateway_model.project_id] = secrets
+ return get_gateway_replica_otel_settings(gateway_model, configuration, replica_model, secrets)
+
+
def get_gateway_replica_models(gateway_model: GatewayModel) -> List[GatewayReplicaModel]:
replicas = list(gateway_model.gateway_replicas)
if gateway_model.gateway_replica is not None: # pre-0.20.25 gateway
@@ -1010,7 +1075,7 @@ async def get_plan(
project=project.name,
spec=spec,
)
- _validate_gateway_configuration(effective_spec.configuration)
+ await _validate_gateway_configuration(session, project, effective_spec.configuration)
action = ApplyAction.CREATE
current_gateway: Optional[Gateway] = None
@@ -1066,7 +1131,7 @@ async def apply_plan(
spec=plan.spec,
)
new_configuration = spec.configuration
- _validate_gateway_configuration(new_configuration)
+ await _validate_gateway_configuration(session, project, new_configuration)
if new_configuration.name is None:
return await create_gateway(
@@ -1163,7 +1228,9 @@ def _can_update_gateway_in_place(conf_diff: ModelDiff) -> bool:
return all(field in _CONF_UPDATABLE_FIELDS for field in conf_diff)
-def _validate_gateway_configuration(configuration: GatewayConfiguration):
+async def _validate_gateway_configuration(
+ session: AsyncSession, project: ProjectModel, configuration: GatewayConfiguration
+):
check_backend_type_available(configuration.backend)
if configuration.backend not in BACKENDS_WITH_GATEWAY_SUPPORT:
raise ServerClientError(
@@ -1258,3 +1325,10 @@ def _validate_gateway_configuration(configuration: GatewayConfiguration):
" or `certificate: { type: gcp-cm, name: }` (GCP Certificate Manager)"
)
raise ServerClientError(err)
+
+ if configuration.otel is not None:
+ secrets = await get_project_secrets_mapping(session=session, project=project)
+ try:
+ resolve_gateway_otel_config(configuration.otel, secrets)
+ except InvalidGatewayOTelConfigError as e:
+ raise ServerClientError(f"Invalid gateway otel configuration: {e}")
diff --git a/src/dstack/_internal/server/services/gateways/client.py b/src/dstack/_internal/server/services/gateways/client.py
index 0c9defecca..b3a803ca05 100644
--- a/src/dstack/_internal/server/services/gateways/client.py
+++ b/src/dstack/_internal/server/services/gateways/client.py
@@ -11,6 +11,7 @@
from dstack._internal.core.models.configurations import RateLimit, ServiceConfiguration
from dstack._internal.core.models.instances import SSHConnectionParams
from dstack._internal.core.models.runs import JobSpec, JobSubmission, get_service_port
+from dstack._internal.proxy.gateway.models import OTelSettings
from dstack._internal.proxy.gateway.schemas.services import ServiceListItem, ServiceListResponse
from dstack._internal.proxy.gateway.schemas.stats import ServiceStats
from dstack._internal.server import settings
@@ -187,13 +188,16 @@ async def list_services(self) -> list[ServiceListItem]:
self.is_server_ready = True
return resp_parsed.services
- async def submit_gateway_config(self) -> None:
+ async def submit_gateway_config(self, otel_settings: Optional[OTelSettings]) -> None:
resp = await self._client.post(
self._url("/api/config"),
json={
"acme_server": settings.ACME_SERVER,
"acme_eab_kid": settings.ACME_EAB_KID,
"acme_eab_hmac_key": settings.ACME_EAB_HMAC_KEY,
+ "otel_settings": (
+ otel_settings.model_dump(mode="json") if otel_settings is not None else None
+ ),
},
)
if resp.status_code == 400:
diff --git a/src/dstack/_internal/server/services/gateways/telemetry.py b/src/dstack/_internal/server/services/gateways/telemetry.py
new file mode 100644
index 0000000000..19ef253641
--- /dev/null
+++ b/src/dstack/_internal/server/services/gateways/telemetry.py
@@ -0,0 +1,112 @@
+import re
+from collections.abc import Mapping
+from typing import Optional, Union
+from urllib.parse import urlsplit
+
+from dstack._internal.core.models.gateways import GatewayConfiguration, GatewayOTelConfig
+from dstack._internal.proxy.gateway.models import OTelSettings
+from dstack._internal.server import settings
+from dstack._internal.server.models import GatewayModel, GatewayReplicaModel
+from dstack._internal.server.services.logging import fmt
+from dstack._internal.utils.interpolator import InterpolatorError, VariablesInterpolator
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def get_gateway_replica_otel_settings(
+ gateway_model: GatewayModel,
+ gateway_configuration: GatewayConfiguration,
+ replica_model: GatewayReplicaModel,
+ secrets: Mapping[str, str],
+) -> Optional[OTelSettings]:
+ otel = gateway_configuration.otel
+ if otel is None or not is_gateway_otel_enabled(otel):
+ return None
+ try:
+ endpoint, headers = resolve_gateway_otel_config(otel, secrets)
+ except InvalidGatewayOTelConfigError as e:
+ # E.g., a referenced secret was deleted or changed to an invalid value
+ logger.warning("%s: cannot enable OpenTelemetry: %s", fmt(replica_model), e)
+ return None
+ return OTelSettings(
+ traces_enabled=otel.traces,
+ logs_enabled=otel.logs,
+ metrics_enabled=otel.metrics,
+ endpoint=endpoint,
+ headers=headers,
+ traces_sample_rate=otel.traces_sample_rate,
+ traces_auth_sample_rate=otel.traces_auth_sample_rate,
+ resource_attributes=_get_resource_attributes(gateway_model, replica_model),
+ )
+
+
+def is_gateway_otel_enabled(otel: GatewayOTelConfig) -> bool:
+ return otel.traces or otel.logs or otel.metrics
+
+
+class InvalidGatewayOTelConfigError(ValueError):
+ """The message is shown to users and must not contain secret values."""
+
+
+def resolve_gateway_otel_config(
+ otel: GatewayOTelConfig, secrets: Mapping[str, str]
+) -> tuple[str, dict[str, str]]:
+ """Returns the endpoint and headers with secrets interpolated."""
+ interpolate = VariablesInterpolator({"secrets": secrets}).interpolate_or_error
+ try:
+ endpoint = interpolate(otel.endpoint)
+ headers = {name: interpolate(value) for name, value in (otel.headers or {}).items()}
+ except InterpolatorError as e:
+ raise InvalidGatewayOTelConfigError(str(e)) from None
+ _validate_endpoint(endpoint)
+ for name, value in headers.items():
+ _validate_header(name, value)
+ return endpoint, headers
+
+
+def _validate_endpoint(endpoint: str) -> None:
+ # Checked explicitly, since urlsplit silently strips tabs and line breaks
+ if any(c.isspace() or not c.isprintable() for c in endpoint):
+ raise InvalidGatewayOTelConfigError(
+ "endpoint must not contain whitespace or control characters"
+ )
+ try:
+ url = urlsplit(endpoint)
+ url.port # Raises on invalid ports
+ except ValueError:
+ raise InvalidGatewayOTelConfigError("endpoint must be a valid URL") from None
+ if url.scheme not in ("http", "https") or not url.hostname:
+ raise InvalidGatewayOTelConfigError("endpoint must be an http:// or https:// URL")
+ # Signal paths are appended to the endpoint and would end up in the query or fragment
+ if "?" in endpoint or "#" in endpoint:
+ raise InvalidGatewayOTelConfigError("endpoint must not contain a query or fragment")
+
+
+# RFC 9110 token. Use with fullmatch
+_HEADER_NAME_RE = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
+
+
+def _validate_header(name: str, value: str) -> None:
+ if not _HEADER_NAME_RE.fullmatch(name):
+ raise InvalidGatewayOTelConfigError(f"invalid header name {name!r}")
+ if not (value.isprintable() and value == value.strip() and all(ord(c) < 256 for c in value)):
+ raise InvalidGatewayOTelConfigError(f"header {name!r} has an invalid value")
+
+
+def _get_resource_attributes(
+ gateway_model: GatewayModel, replica_model: GatewayReplicaModel
+) -> dict[str, Union[str, int]]:
+ attributes: dict[str, Union[str, int]] = {
+ "service.instance.id": str(replica_model.id),
+ "dstack.gateway.replica.num": replica_model.replica_num,
+ "deployment.environment.name": settings.SERVER_ENVIRONMENT,
+ "dstack.project.name": gateway_model.project.name,
+ "dstack.gateway.name": gateway_model.name,
+ "dstack.backend": gateway_model.backend.type.value,
+ }
+ if replica_model.ip_address is not None:
+ attributes["host.name"] = replica_model.ip_address
+ if replica_model.region is not None:
+ attributes["cloud.region"] = replica_model.region
+ return attributes
diff --git a/src/dstack/_internal/server/utils/otel/utils.py b/src/dstack/_internal/server/utils/otel/utils.py
index ed4f943542..3f7f018187 100644
--- a/src/dstack/_internal/server/utils/otel/utils.py
+++ b/src/dstack/_internal/server/utils/otel/utils.py
@@ -1,37 +1,19 @@
-import logging
import re
-from typing import List, Optional, Sequence
+from typing import List, Optional
from fastapi import FastAPI
-from opentelemetry import metrics, trace
-from opentelemetry._logs import set_logger_provider
-from opentelemetry.context import Context
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.prometheus import PrometheusMetricReader
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
-from opentelemetry.instrumentation.logging.handler import LoggingHandler
from opentelemetry.instrumentation.requests import RequestsInstrumentor
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor
-from opentelemetry.sdk._logs import LoggerProvider
-from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
-from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import MetricReader, PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor
-from opentelemetry.sdk.trace.sampling import (
- ParentBased,
- Sampler,
- SamplingResult,
- TraceIdRatioBased,
-)
-from opentelemetry.trace import Link, SpanKind
-from opentelemetry.trace.span import TraceState
-from opentelemetry.util.types import Attributes
+from opentelemetry.sdk.trace.sampling import ParentBased
from sqlalchemy import event
from sqlalchemy.engine import Engine
from sqlalchemy.ext.asyncio import AsyncEngine
@@ -39,11 +21,9 @@
from dstack._internal import settings as core_settings
from dstack._internal.server import settings
from dstack._internal.server.identity import SERVER_REPLICA_ID
-from dstack._internal.server.utils.common import is_background_task_name
+from dstack._internal.server.utils.common import PIPELINE_TASKS_PREFIX, SCHEDULED_TASKS_PREFIX
from dstack._internal.server.utils.logging import AsyncioCancelledErrorFilter
-from dstack._internal.utils.logging import get_logger
-
-logger = get_logger(__name__)
+from dstack._internal.utils import otel as otel_utils
def configure(app: FastAPI, engine: AsyncEngine) -> None:
@@ -60,18 +40,20 @@ def configure(app: FastAPI, engine: AsyncEngine) -> None:
def _configure_tracing() -> None:
- provider = TracerProvider(
- resource=_get_resource(),
- sampler=ParentBased(
- root=_RootSpanNameSampler(
- default_rate=settings.OTEL_TRACES_SAMPLE_RATE,
- background_rate=settings.OTEL_TRACES_BACKGROUND_SAMPLE_RATE,
- )
- ),
+ otel_utils.configure_tracing(
+ _get_resource(), ParentBased(root=_build_root_sampler()), OTLPSpanExporter()
+ )
+
+
+def _build_root_sampler() -> otel_utils.RootSpanNameSampler:
+ background_rate = settings.OTEL_TRACES_BACKGROUND_SAMPLE_RATE
+ return otel_utils.RootSpanNameSampler(
+ default_rate=settings.OTEL_TRACES_SAMPLE_RATE,
+ rates_by_name_prefix={
+ SCHEDULED_TASKS_PREFIX: background_rate,
+ PIPELINE_TASKS_PREFIX: background_rate,
+ },
)
- provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
- trace.set_tracer_provider(provider)
- logger.info("OpenTelemetry tracing enabled")
def _configure_metrics() -> None:
@@ -88,17 +70,13 @@ def _configure_metrics() -> None:
f"Unknown exporter {exporter!r} in DSTACK_OTEL_METRICS_EXPORTERS."
" Supported exporters: prometheus, otlp"
)
- provider = MeterProvider(resource=_get_resource(), metric_readers=readers)
- metrics.set_meter_provider(provider)
- logger.info("OpenTelemetry metrics enabled")
+ otel_utils.configure_metrics(_get_resource(), readers)
def _configure_log_export() -> None:
- logger_provider = LoggerProvider(resource=_get_resource())
- logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
- set_logger_provider(logger_provider)
- logging.getLogger().addHandler(_build_log_handler(logger_provider))
- logger.info("OpenTelemetry log export enabled")
+ otel_utils.configure_log_export(
+ _get_resource(), OTLPLogExporter(), filters=[AsyncioCancelledErrorFilter()]
+ )
def _instrument(app: FastAPI, engine: AsyncEngine) -> None:
@@ -111,7 +89,7 @@ def _instrument(app: FastAPI, engine: AsyncEngine) -> None:
# attribute, so the SDK would accumulate state for an unbounded number of series.
RequestsInstrumentor().instrument(excluded_urls=r"http\+unix://")
if settings.OTEL_METRICS_ENABLED:
- SystemMetricsInstrumentor(config=_PROCESS_METRICS_CONFIG).instrument()
+ SystemMetricsInstrumentor(config=otel_utils.PROCESS_METRICS_CONFIG).instrument()
def _get_metrics_exporters() -> List[str]:
@@ -124,22 +102,6 @@ def _get_metrics_exporters() -> List[str]:
return ["otlp"]
-# Process-level metrics only. Host-level (`system.*`) metrics are left
-# to node exporters. `process.runtime.*` metrics are deprecated duplicates.
-_PROCESS_METRICS_CONFIG = {
- "process.cpu.time": ["user", "system"],
- "process.cpu.utilization": ["user", "system"],
- "process.memory.usage": None,
- "process.memory.virtual": None,
- "process.thread.count": None,
- "process.open_file_descriptor.count": None,
- "process.context_switches": ["involuntary", "voluntary"],
- "cpython.gc.collections": None,
- "cpython.gc.collected_objects": None,
- "cpython.gc.uncollectable_objects": None,
-}
-
-
def _get_resource() -> Resource:
return Resource.create(
{
@@ -153,24 +115,6 @@ def _get_resource() -> Resource:
)
-def _build_log_handler(logger_provider: LoggerProvider) -> logging.Handler:
- handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider)
- handler.addFilter(AsyncioCancelledErrorFilter())
- handler.addFilter(_OTelSDKLogFilter())
- return handler
-
-
-class _OTelSDKLogFilter(logging.Filter):
- """Drops log records emitted by the OTel SDK itself.
-
- Exporting them would be a feedback loop: an OTLP export failure is logged,
- the record is queued for the same failing exporter, and so on indefinitely.
- """
-
- def filter(self, record: logging.LogRecord) -> bool:
- return not (record.name + ".").startswith("opentelemetry.")
-
-
def _register_db_span_renaming(engine: Engine) -> None:
"""Renames DB spans from ` ` to ` `.
@@ -205,34 +149,3 @@ def _get_db_span_name(statement: str) -> Optional[str]:
if match is None:
return operation
return f"{operation} {match.group(1)}"
-
-
-class _RootSpanNameSampler(Sampler):
- """Samples background task traces and other (HTTP) traces at different rates."""
-
- def __init__(self, default_rate: float, background_rate: float):
- self._default_sampler = TraceIdRatioBased(default_rate)
- self._background_sampler = TraceIdRatioBased(background_rate)
-
- def should_sample(
- self,
- parent_context: Optional[Context],
- trace_id: int,
- name: str,
- kind: Optional[SpanKind] = None,
- attributes: Attributes = None,
- links: Optional[Sequence[Link]] = None,
- trace_state: Optional[TraceState] = None,
- ) -> SamplingResult:
- sampler = self._default_sampler
- if is_background_task_name(name):
- sampler = self._background_sampler
- return sampler.should_sample(
- parent_context, trace_id, name, kind, attributes, links, trace_state
- )
-
- def get_description(self) -> str:
- return (
- f"RootSpanNameSampler(default={self._default_sampler.rate},"
- f"background={self._background_sampler.rate})"
- )
diff --git a/src/dstack/_internal/utils/otel.py b/src/dstack/_internal/utils/otel.py
new file mode 100644
index 0000000000..e575b0f6c9
--- /dev/null
+++ b/src/dstack/_internal/utils/otel.py
@@ -0,0 +1,123 @@
+import logging
+from typing import Mapping, Optional, Sequence
+
+from opentelemetry import metrics, trace
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.context import Context
+from opentelemetry.instrumentation.logging.handler import LoggingHandler
+from opentelemetry.sdk._logs import LoggerProvider
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor, LogRecordExporter
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import MetricReader
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter
+from opentelemetry.sdk.trace.sampling import Sampler, SamplingResult, TraceIdRatioBased
+from opentelemetry.trace import Link, SpanKind
+from opentelemetry.trace.span import TraceState
+from opentelemetry.util.types import Attributes
+
+from dstack._internal.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+# Process-level metrics only. Host-level (`system.*`) metrics are left
+# to node exporters. `process.runtime.*` metrics are deprecated duplicates.
+PROCESS_METRICS_CONFIG = {
+ "process.cpu.time": ["user", "system"],
+ "process.cpu.utilization": ["user", "system"],
+ "process.memory.usage": None,
+ "process.memory.virtual": None,
+ "process.thread.count": None,
+ "process.open_file_descriptor.count": None,
+ "process.context_switches": ["involuntary", "voluntary"],
+ "cpython.gc.collections": None,
+ "cpython.gc.collected_objects": None,
+ "cpython.gc.uncollectable_objects": None,
+}
+
+
+def configure_tracing(resource: Resource, sampler: Sampler, exporter: SpanExporter) -> None:
+ provider = TracerProvider(resource=resource, sampler=sampler)
+ provider.add_span_processor(BatchSpanProcessor(exporter))
+ trace.set_tracer_provider(provider)
+ logger.info("OpenTelemetry tracing enabled")
+
+
+def configure_metrics(resource: Resource, readers: Sequence[MetricReader]) -> None:
+ provider = MeterProvider(resource=resource, metric_readers=readers)
+ metrics.set_meter_provider(provider)
+ logger.info("OpenTelemetry metrics enabled")
+
+
+def configure_log_export(
+ resource: Resource,
+ exporter: LogRecordExporter,
+ level: int = logging.NOTSET,
+ filters: Sequence[logging.Filter] = (),
+) -> None:
+ logger_provider = LoggerProvider(resource=resource)
+ logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
+ set_logger_provider(logger_provider)
+ logging.getLogger().addHandler(build_log_handler(logger_provider, level, filters))
+ logger.info("OpenTelemetry log export enabled")
+
+
+def build_log_handler(
+ logger_provider: LoggerProvider,
+ level: int = logging.NOTSET,
+ filters: Sequence[logging.Filter] = (),
+) -> logging.Handler:
+ handler = LoggingHandler(level=level, logger_provider=logger_provider)
+ for log_filter in filters:
+ handler.addFilter(log_filter)
+ handler.addFilter(_OTelSDKLogFilter())
+ return handler
+
+
+class _OTelSDKLogFilter(logging.Filter):
+ """Drops log records emitted by the OTel SDK itself.
+
+ Exporting them would be a feedback loop: an OTLP export failure is logged,
+ the record is queued for the same failing exporter, and so on indefinitely.
+ """
+
+ def filter(self, record: logging.LogRecord) -> bool:
+ return not (record.name + ".").startswith("opentelemetry.")
+
+
+class RootSpanNameSampler(Sampler):
+ """Samples traces at different rates depending on the root span name prefix."""
+
+ def __init__(self, default_rate: float, rates_by_name_prefix: Mapping[str, float]):
+ self._default_sampler = TraceIdRatioBased(default_rate)
+ self._samplers_by_name_prefix = {
+ prefix: TraceIdRatioBased(rate) for prefix, rate in rates_by_name_prefix.items()
+ }
+
+ def should_sample(
+ self,
+ parent_context: Optional[Context],
+ trace_id: int,
+ name: str,
+ kind: Optional[SpanKind] = None,
+ attributes: Attributes = None,
+ links: Optional[Sequence[Link]] = None,
+ trace_state: Optional[TraceState] = None,
+ ) -> SamplingResult:
+ sampler = self._default_sampler
+ for prefix, prefix_sampler in self._samplers_by_name_prefix.items():
+ if name.startswith(prefix):
+ sampler = prefix_sampler
+ break
+ return sampler.should_sample(
+ parent_context, trace_id, name, kind, attributes, links, trace_state
+ )
+
+ def get_description(self) -> str:
+ rates = [f"default={self._default_sampler.rate}"]
+ rates.extend(
+ f"{prefix!r}={sampler.rate}"
+ for prefix, sampler in self._samplers_by_name_prefix.items()
+ )
+ return f"RootSpanNameSampler({','.join(rates)})"
diff --git a/src/tests/_internal/proxy/gateway/routers/test_config.py b/src/tests/_internal/proxy/gateway/routers/test_config.py
new file mode 100644
index 0000000000..e247d36b44
--- /dev/null
+++ b/src/tests/_internal/proxy/gateway/routers/test_config.py
@@ -0,0 +1,66 @@
+import signal
+from unittest.mock import Mock
+
+import httpx
+import pytest
+
+from dstack._internal.proxy.gateway.app import make_app
+from dstack._internal.proxy.gateway.models import OTelSettings
+from dstack._internal.proxy.gateway.repo.repo import GatewayProxyRepo
+from dstack._internal.proxy.gateway.routers import config
+
+
+def make_client(repo: GatewayProxyRepo) -> httpx.AsyncClient:
+ app = make_app(repo)
+ return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test/")
+
+
+@pytest.mark.asyncio
+class TestUpdateGlobalConfig:
+ @pytest.mark.parametrize("restart_required", [False, True])
+ async def test_saves_and_applies_otel_settings(
+ self, monkeypatch: pytest.MonkeyPatch, restart_required: bool
+ ) -> None:
+ apply_settings = Mock(return_value=restart_required)
+ kill = Mock()
+ monkeypatch.setattr(config.otel, "apply_settings", apply_settings)
+ monkeypatch.setattr(config.os, "kill", kill)
+ repo = GatewayProxyRepo()
+ otel_settings = {
+ "traces_enabled": True,
+ "logs_enabled": False,
+ "metrics_enabled": False,
+ "endpoint": "https://otel.example.com:4318",
+ "headers": {"api-key": "secret"},
+ "traces_sample_rate": 1.0,
+ "traces_auth_sample_rate": 0.01,
+ "resource_attributes": {"service.instance.id": "replica-id"},
+ }
+
+ async with make_client(repo) as client:
+ resp = await client.post("/api/config", json={"otel_settings": otel_settings})
+
+ assert resp.status_code == 200
+ expected = OTelSettings(**otel_settings)
+ assert (await repo.get_config()).otel_settings == expected
+ apply_settings.assert_called_once_with(expected)
+ if restart_required:
+ kill.assert_called_once()
+ assert kill.call_args.args[1] == signal.SIGTERM
+ else:
+ kill.assert_not_called()
+
+ async def test_accepts_config_without_otel_settings(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ # Older servers do not send otel_settings
+ apply_settings = Mock(return_value=False)
+ monkeypatch.setattr(config.otel, "apply_settings", apply_settings)
+ repo = GatewayProxyRepo()
+
+ async with make_client(repo) as client:
+ resp = await client.post("/api/config", json={"acme_server": None})
+
+ assert resp.status_code == 200
+ assert (await repo.get_config()).otel_settings is None
+ apply_settings.assert_called_once_with(None)
diff --git a/src/tests/_internal/proxy/gateway/routers/test_registry.py b/src/tests/_internal/proxy/gateway/routers/test_registry.py
index a9b265d9fc..7572aa13fb 100644
--- a/src/tests/_internal/proxy/gateway/routers/test_registry.py
+++ b/src/tests/_internal/proxy/gateway/routers/test_registry.py
@@ -156,6 +156,9 @@ async def test_register_with_auth(self, tmp_path: Path, system_mocks: Mocks) ->
conf = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text()
assert "auth_request /_dstack_auth;" in conf
assert "proxy_pass http://localhost:8000/api/auth/test-proj;" in conf
+ assert 'proxy_set_header traceparent "";' in conf
+ assert 'proxy_set_header tracestate "";' in conf
+ assert 'proxy_set_header baggage "";' in conf
async def test_register_same_name_error(self, tmp_path: Path, system_mocks: Mocks) -> None:
client = make_client(tmp_path)
@@ -618,6 +621,9 @@ async def test_register(self, tmp_path: Path, system_mocks: Mocks) -> None:
assert resp.json() == {"status": "ok"}
conf = (tmp_path / "sites-enabled" / "443-gateway.gtw.test.conf").read_text()
assert "proxy_pass http://localhost:8000/api/models/test-proj/;" in conf
+ assert 'proxy_set_header traceparent "";' in conf
+ assert 'proxy_set_header tracestate "";' in conf
+ assert 'proxy_set_header baggage "";' in conf
assert "listen 80;" in conf
assert "listen 443" not in conf
assert system_mocks.reload_nginx.call_count == 1
diff --git a/src/tests/_internal/proxy/gateway/test_app.py b/src/tests/_internal/proxy/gateway/test_app.py
index 335be84fa1..56a13be9aa 100644
--- a/src/tests/_internal/proxy/gateway/test_app.py
+++ b/src/tests/_internal/proxy/gateway/test_app.py
@@ -1,12 +1,14 @@
from pathlib import Path
+from unittest.mock import Mock
import pytest
+from dstack._internal.proxy.gateway import app as app_module
from dstack._internal.proxy.gateway.app import lifespan, make_app
-from dstack._internal.proxy.gateway.models import ModelEntrypoint
+from dstack._internal.proxy.gateway.models import GlobalProxyConfig, ModelEntrypoint
from dstack._internal.proxy.gateway.repo.repo import GatewayProxyRepo
from dstack._internal.proxy.gateway.services.nginx import Nginx
-from dstack._internal.proxy.gateway.testing.common import Mocks
+from dstack._internal.proxy.gateway.testing.common import Mocks, make_otel_settings
from dstack._internal.proxy.lib.testing.common import make_project, make_service
@@ -32,3 +34,19 @@ async def test_lifespan(tmp_path: Path, system_mocks: Mocks) -> None:
assert system_mocks.open_conn.call_count == 1
assert system_mocks.close_conn.call_count == 0
assert system_mocks.close_conn.call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_lifespan_applies_saved_otel_settings(
+ tmp_path: Path, system_mocks: Mocks, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ apply_settings = Mock(return_value=False)
+ monkeypatch.setattr(app_module.otel, "apply_settings", apply_settings)
+ repo = GatewayProxyRepo()
+ otel_settings = make_otel_settings(traces_enabled=True)
+ await repo.set_config(GlobalProxyConfig(otel_settings=otel_settings))
+ (tmp_path / "nginx" / "sites-enabled").mkdir(parents=True)
+ app = make_app(repo=repo, nginx=Nginx(nginx_dir=tmp_path / "nginx"))
+
+ async with lifespan(app):
+ apply_settings.assert_called_once_with(otel_settings)
diff --git a/src/tests/_internal/proxy/gateway/test_otel.py b/src/tests/_internal/proxy/gateway/test_otel.py
new file mode 100644
index 0000000000..6d667916ea
--- /dev/null
+++ b/src/tests/_internal/proxy/gateway/test_otel.py
@@ -0,0 +1,100 @@
+from typing import Optional
+from unittest.mock import Mock
+
+import pytest
+from opentelemetry.sdk.trace.sampling import Decision
+
+from dstack._internal.proxy.gateway import otel
+from dstack._internal.proxy.gateway.models import OTelSettings
+from dstack._internal.proxy.gateway.testing.common import make_otel_settings
+
+
+class TestApplySettings:
+ @pytest.fixture
+ def configure(self, monkeypatch: pytest.MonkeyPatch) -> Mock:
+ # Stubbed, as the global providers can only be set once per process
+ configure = Mock()
+ monkeypatch.setattr(otel, "_configure", configure)
+ monkeypatch.setattr(otel, "_applied_settings", None)
+ return configure
+
+ def test_does_not_raise_or_retry_if_configuration_fails(self, configure: Mock):
+ configure.side_effect = ValueError("bad settings")
+ settings = make_otel_settings(traces_enabled=True)
+
+ assert otel.apply_settings(settings) is False
+ assert otel.apply_settings(settings) is False
+ assert otel.apply_settings(make_otel_settings(logs_enabled=True)) is True
+
+ configure.assert_called_once_with(settings)
+
+ def test_keeps_disabled_without_settings(self, configure: Mock):
+ assert otel.apply_settings(None) is False
+ assert otel.apply_settings(make_otel_settings()) is False
+ configure.assert_not_called()
+
+ def test_applies_first_settings_immediately(self, configure: Mock):
+ settings = make_otel_settings(traces_enabled=True)
+
+ assert otel.apply_settings(settings) is False
+ assert otel.apply_settings(settings) is False
+
+ configure.assert_called_once_with(settings)
+
+ @pytest.mark.parametrize(
+ "new_settings",
+ [
+ pytest.param(make_otel_settings(traces_enabled=True, logs_enabled=True), id="changed"),
+ pytest.param(None, id="disabled"),
+ ],
+ )
+ def test_requires_restart_to_change_applied_settings(
+ self, configure: Mock, new_settings: Optional[OTelSettings]
+ ):
+ otel.apply_settings(make_otel_settings(traces_enabled=True))
+
+ assert otel.apply_settings(new_settings) is True
+
+ configure.assert_called_once()
+
+
+class TestGetSignalEndpoint:
+ @pytest.mark.parametrize(
+ ("endpoint", "expected"),
+ [
+ ("https://otel.example.com:4318", "https://otel.example.com:4318/v1/traces"),
+ ("https://example.com/otlp/", "https://example.com/otlp/v1/traces"),
+ ],
+ )
+ def test_appends_signal_path(self, endpoint: str, expected: str):
+ settings = make_otel_settings(endpoint=endpoint)
+ assert otel._get_signal_endpoint(settings, "traces") == expected
+
+
+class TestBuildRootSampler:
+ def test_samples_auth_subrequests_at_auth_rate(self):
+ sampler = otel._build_root_sampler(
+ make_otel_settings(traces_sample_rate=1.0, traces_auth_sample_rate=0.0)
+ )
+ registry_result = sampler.should_sample(
+ None, trace_id=123, name="POST /api/registry/{project_name}/services/register"
+ )
+ auth_result = sampler.should_sample(
+ None, trace_id=123, name="GET /api/auth/{project_name}"
+ )
+ assert registry_result.decision == Decision.RECORD_AND_SAMPLE
+ assert auth_result.decision == Decision.DROP
+
+
+class TestGetResource:
+ def test_adds_service_to_replica_attributes(self):
+ settings = make_otel_settings(
+ resource_attributes={"service.instance.id": "replica-id", "dstack.gateway.name": "gw"}
+ )
+
+ attributes = otel._get_resource(settings).attributes
+
+ assert attributes["service.name"] == "dstack-gateway"
+ assert "service.version" in attributes
+ assert attributes["service.instance.id"] == "replica-id"
+ assert attributes["dstack.gateway.name"] == "gw"
diff --git a/src/tests/_internal/server/routers/test_gateways.py b/src/tests/_internal/server/routers/test_gateways.py
index d5980f5f96..37872ea2e5 100644
--- a/src/tests/_internal/server/routers/test_gateways.py
+++ b/src/tests/_internal/server/routers/test_gateways.py
@@ -6,9 +6,10 @@
from sqlalchemy.ext.asyncio import AsyncSession
from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.core.models.common import ApplyAction
+from dstack._internal.core.models.common import ApplyAction, EntityReference
from dstack._internal.core.models.gateways import GatewayStatus
from dstack._internal.core.models.users import GlobalRole, ProjectRole
+from dstack._internal.server.services.gateways import get_project_gateway_model_by_reference
from dstack._internal.server.services.projects import add_project_member
from dstack._internal.server.testing.common import (
clear_events,
@@ -17,6 +18,7 @@
create_gateway,
create_gateway_replica,
create_project,
+ create_secret,
create_user,
get_auth_headers,
list_events,
@@ -112,6 +114,7 @@ async def test_list(
"load_balancer": None,
"certificate": {"type": "lets-encrypt"},
"tags": None,
+ "otel": None,
"replicas": None,
},
}
@@ -199,6 +202,7 @@ async def test_get(
"load_balancer": None,
"certificate": {"type": "lets-encrypt"},
"tags": None,
+ "otel": None,
"replicas": None,
},
}
@@ -541,6 +545,7 @@ async def test_create_gateway(self, test_db, session: AsyncSession, client: Asyn
"load_balancer": None,
"certificate": {"type": "lets-encrypt"},
"tags": None,
+ "otel": None,
"replicas": None,
},
}
@@ -632,6 +637,7 @@ async def test_create_gateway_without_name(
"load_balancer": None,
"certificate": {"type": "lets-encrypt"},
"tags": None,
+ "otel": None,
"replicas": None,
},
}
@@ -974,6 +980,7 @@ async def test_set_default_gateway(
"load_balancer": None,
"certificate": {"type": "lets-encrypt"},
"tags": None,
+ "otel": None,
"replicas": None,
},
}
@@ -1371,6 +1378,7 @@ async def test_set_wildcard_domain(
"load_balancer": None,
"certificate": {"type": "lets-encrypt"},
"tags": None,
+ "otel": None,
"replicas": None,
},
}
@@ -3037,3 +3045,130 @@ async def test_force_apply_default_false_unsets_default(
await session.refresh(project)
assert project.default_gateway_id is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
+class TestGatewayOTel:
+ async def test_creates_gateway_with_secret_references(
+ self, test_db, session: AsyncSession, client: AsyncClient
+ ):
+ user = await create_user(session, global_role=GlobalRole.USER)
+ project = await create_project(session)
+ await add_project_member(
+ session=session, project=project, user=user, project_role=ProjectRole.ADMIN
+ )
+ await create_secret(session=session, project=project, name="token", value="s3cret")
+ await create_backend(session, project.id, backend_type=BackendType.AWS)
+ otel = {
+ "endpoint": "https://otlp.example.com",
+ "headers": {"Authorization": "Bearer ${{ secrets.token }}"},
+ "traces": True,
+ }
+
+ response = await client.post(
+ f"/api/project/{project.name}/gateways/apply",
+ json={
+ "plan": {
+ "spec": {
+ "configuration": {
+ "type": "gateway",
+ "name": "my-gateway",
+ "backend": "aws",
+ "region": "us-east-1",
+ "otel": otel,
+ }
+ },
+ "current_resource": None,
+ },
+ "force": False,
+ },
+ headers=get_auth_headers(user.token),
+ )
+
+ assert response.status_code == 200
+ assert response.json()["configuration"]["otel"]["headers"] == otel["headers"]
+ assert "s3cret" not in response.text
+ gateway = await get_project_gateway_model_by_reference(
+ session, project, EntityReference(name="my-gateway", project=None)
+ )
+ assert gateway is not None
+ assert "secrets.token" in (gateway.configuration or "")
+ assert "s3cret" not in (gateway.configuration or "")
+ events = await list_events(session)
+ assert all("s3cret" not in e.message for e in events)
+
+ async def test_cannot_update_otel_in_place(
+ self, test_db, session: AsyncSession, client: AsyncClient
+ ):
+ user = await create_user(session, global_role=GlobalRole.USER)
+ project = await create_project(session)
+ await add_project_member(
+ session=session, project=project, user=user, project_role=ProjectRole.ADMIN
+ )
+ backend = await create_backend(session, project.id, backend_type=BackendType.AWS)
+ gateway = await create_gateway(
+ session=session,
+ project_id=project.id,
+ backend_id=backend.id,
+ name="my-gateway",
+ region="us-east-1",
+ )
+ await create_gateway_replica(session=session, backend=backend, gateway_id=gateway.id)
+ get_response = await client.post(
+ f"/api/project/{project.name}/gateways/get",
+ json={"name": "my-gateway"},
+ headers=get_auth_headers(user.token),
+ )
+
+ response = await client.post(
+ f"/api/project/{project.name}/gateways/apply",
+ json={
+ "plan": {
+ "spec": {
+ "configuration": {
+ "type": "gateway",
+ "name": "my-gateway",
+ "backend": "aws",
+ "region": "us-east-1",
+ "otel": {"endpoint": "https://otlp.example.com", "traces": True},
+ }
+ },
+ "current_resource": get_response.json(),
+ },
+ "force": False,
+ },
+ headers=get_auth_headers(user.token),
+ )
+
+ assert response.status_code == 400
+ assert "cannot be updated in-place" in response.json()["detail"][0]["msg"]
+
+ async def test_rejects_missing_secret(
+ self, test_db, session: AsyncSession, client: AsyncClient
+ ):
+ user = await create_user(session, global_role=GlobalRole.USER)
+ project = await create_project(session)
+ await add_project_member(
+ session=session, project=project, user=user, project_role=ProjectRole.ADMIN
+ )
+ await create_backend(session, project.id, backend_type=BackendType.AWS)
+
+ response = await client.post(
+ f"/api/project/{project.name}/gateways/get_plan",
+ json={
+ "spec": {
+ "configuration": {
+ "type": "gateway",
+ "name": "my-gateway",
+ "backend": "aws",
+ "region": "us-east-1",
+ "otel": {"endpoint": "${{ secrets.missing }}", "traces": True},
+ }
+ }
+ },
+ headers=get_auth_headers(user.token),
+ )
+
+ assert response.status_code == 400
+ assert "secrets.missing" in response.text
diff --git a/src/tests/_internal/server/services/gateways/test_gateways.py b/src/tests/_internal/server/services/gateways/test_gateways.py
index 78cecf9036..6284d69bff 100644
--- a/src/tests/_internal/server/services/gateways/test_gateways.py
+++ b/src/tests/_internal/server/services/gateways/test_gateways.py
@@ -6,11 +6,15 @@
from dstack._internal import settings as core_settings
from dstack._internal.core.backends.base.compute import get_dstack_gateway_install_command
+from dstack._internal.core.errors import ServerClientError
+from dstack._internal.core.models.backends.base import BackendType
+from dstack._internal.core.models.gateways import GatewayConfiguration, GatewayOTelConfig
from dstack._internal.proxy.gateway.const import SERVICE_SCALING_WINDOWS
from dstack._internal.proxy.gateway.schemas.stats import Stat
from dstack._internal.server.services import gateways
from dstack._internal.server.services.gateways import (
_merge_per_window_stats,
+ _validate_gateway_configuration,
get_gateway_replica_models,
)
from dstack._internal.server.testing.common import (
@@ -18,6 +22,8 @@
create_gateway,
create_gateway_replica,
create_project,
+ create_secret,
+ create_user,
)
@@ -147,3 +153,91 @@ async def test_runs_install_command_on_existing_gateway_replicas(
)
configure.assert_awaited_once()
assert replica.app_updated_at == (now if updated else previously_updated_at)
+
+ async def test_configures_replicas_with_otel_settings(
+ self, session: AsyncSession, monkeypatch: pytest.MonkeyPatch
+ ):
+ connection = Mock(ip_address="1.1.1.1")
+ monkeypatch.setattr(gateways.settings, "SKIP_GATEWAY_UPDATE", True)
+ monkeypatch.setattr(
+ gateways.gateway_connections_pool, "get_or_add", AsyncMock(return_value=connection)
+ )
+ monkeypatch.setattr(
+ gateways.gateway_connections_pool, "all", AsyncMock(return_value=[connection])
+ )
+ configure = AsyncMock()
+ monkeypatch.setattr(gateways, "configure_gateway_replica", configure)
+ project = await create_project(session=session, name="main")
+ backend = await create_backend(session=session, project_id=project.id)
+ gateway = await create_gateway(
+ session=session, project_id=project.id, backend_id=backend.id, name="my-gateway"
+ )
+ configuration = gateways.get_gateway_configuration(gateway)
+ configuration.otel = GatewayOTelConfig(
+ endpoint="https://otlp",
+ headers={"Authorization": "Bearer ${{ secrets.token }}"},
+ traces=True,
+ )
+ gateway.configuration = configuration.model_dump_json()
+ await create_secret(session=session, project=project, name="token", value="s3cret")
+ replica = await create_gateway_replica(
+ session=session, backend=backend, gateway_id=gateway.id, replica_num=1
+ )
+ await session.commit()
+ # Relationships must be loaded by init_gateways, not come from the identity map
+ session.expunge_all()
+
+ await gateways.init_gateways(session)
+
+ configure.assert_awaited_once()
+ assert configure.await_args is not None
+ assert configure.await_args.args[0] is connection
+ otel_settings = configure.await_args.args[1]
+ assert otel_settings.traces_enabled
+ assert otel_settings.headers == {"Authorization": "Bearer s3cret"}
+ attributes = otel_settings.resource_attributes
+ assert attributes["service.instance.id"] == str(replica.id)
+ assert attributes["dstack.project.name"] == "main"
+ assert attributes["dstack.gateway.name"] == "my-gateway"
+ assert attributes["dstack.backend"] == backend.type.value
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True)
+class TestValidateGatewayConfiguration:
+ @pytest.mark.parametrize(
+ ("endpoint", "error"),
+ [
+ pytest.param("${{ secrets.missing }}", "secrets.missing", id="missing-secret"),
+ pytest.param("${{ secrets.token }}", "endpoint must be", id="invalid-secret-value"),
+ ],
+ )
+ async def test_rejects_invalid_otel_without_revealing_secrets(
+ self, test_db, session: AsyncSession, endpoint: str, error: str
+ ):
+ project = await create_project(session=session)
+ await create_secret(session=session, project=project, name="token", value="s3cret")
+ configuration = _make_otel_configuration(
+ GatewayOTelConfig(endpoint=endpoint, headers={"A": "${{ secrets.token }}"})
+ )
+
+ with pytest.raises(ServerClientError, match=error) as exc_info:
+ await _validate_gateway_configuration(session, project, configuration)
+
+ assert "s3cret" not in str(exc_info.value)
+
+ async def test_does_not_use_other_projects_secrets(self, test_db, session: AsyncSession):
+ user = await create_user(session=session)
+ project = await create_project(session=session, owner=user, name="owner")
+ other_project = await create_project(session=session, owner=user, name="other")
+ await create_secret(session=session, project=other_project, name="token", value="x")
+ configuration = _make_otel_configuration(
+ GatewayOTelConfig(endpoint="https://otlp", headers={"A": "${{ secrets.token }}"})
+ )
+
+ with pytest.raises(ServerClientError, match="secrets.token"):
+ await _validate_gateway_configuration(session, project, configuration)
+
+
+def _make_otel_configuration(otel: GatewayOTelConfig) -> GatewayConfiguration:
+ return GatewayConfiguration(backend=BackendType.AWS, region="us-east-1", otel=otel)
diff --git a/src/tests/_internal/server/services/gateways/test_telemetry.py b/src/tests/_internal/server/services/gateways/test_telemetry.py
new file mode 100644
index 0000000000..760d69a4b3
--- /dev/null
+++ b/src/tests/_internal/server/services/gateways/test_telemetry.py
@@ -0,0 +1,175 @@
+import logging
+import uuid
+from typing import Optional
+
+import pytest
+
+from dstack._internal.core.models.backends.base import BackendType
+from dstack._internal.core.models.gateways import GatewayConfiguration, GatewayOTelConfig
+from dstack._internal.proxy.gateway.models import OTelSettings
+from dstack._internal.server import settings
+from dstack._internal.server.models import (
+ BackendModel,
+ GatewayModel,
+ GatewayReplicaModel,
+ ProjectModel,
+)
+from dstack._internal.server.services.gateways.telemetry import (
+ InvalidGatewayOTelConfigError,
+ get_gateway_replica_otel_settings,
+ resolve_gateway_otel_config,
+)
+
+
+class TestGetGatewayReplicaOTelSettings:
+ @pytest.mark.parametrize(
+ "otel",
+ [
+ pytest.param(None, id="no-otel"),
+ pytest.param(GatewayOTelConfig(endpoint="https://otlp"), id="no-signals"),
+ ],
+ )
+ def test_none_if_disabled(self, otel: Optional[GatewayOTelConfig]):
+ gateway_model, replica_model = _make_models()
+
+ assert (
+ get_gateway_replica_otel_settings(
+ gateway_model, _make_configuration(otel), replica_model, secrets={}
+ )
+ is None
+ )
+
+ def test_resolves_secrets_and_identifies_replica(self, monkeypatch: pytest.MonkeyPatch):
+ monkeypatch.setattr(settings, "SERVER_ENVIRONMENT", "production")
+ gateway_model, replica_model = _make_models()
+ otel = GatewayOTelConfig(
+ endpoint="${{ secrets.otlp_endpoint }}",
+ headers={"Authorization": "Bearer ${{ secrets.otlp_token }}", "X-Team": "ml"},
+ traces=True,
+ metrics=True,
+ traces_sample_rate=0.5,
+ )
+
+ otel_settings = get_gateway_replica_otel_settings(
+ gateway_model,
+ _make_configuration(otel),
+ replica_model,
+ secrets={"otlp_endpoint": "https://otlp.example.com", "otlp_token": "s3cret"},
+ )
+
+ assert otel_settings == OTelSettings(
+ traces_enabled=True,
+ logs_enabled=False,
+ metrics_enabled=True,
+ endpoint="https://otlp.example.com",
+ headers={"Authorization": "Bearer s3cret", "X-Team": "ml"},
+ traces_sample_rate=0.5,
+ traces_auth_sample_rate=0.01,
+ resource_attributes={
+ "service.instance.id": str(replica_model.id),
+ "dstack.gateway.replica.num": 1,
+ "deployment.environment.name": "production",
+ "dstack.project.name": "main",
+ "dstack.gateway.name": "my-gateway",
+ "dstack.backend": "aws",
+ "host.name": "203.0.113.5",
+ "cloud.region": "us-east-1",
+ },
+ )
+
+ @pytest.mark.parametrize(
+ ("endpoint", "headers"),
+ [
+ pytest.param(
+ "https://otlp", {"Authorization": "${{ secrets.deleted }}"}, id="missing"
+ ),
+ pytest.param("${{ secrets.value }}", {}, id="invalid-endpoint"),
+ pytest.param(
+ "https://otlp", {"Authorization": "${{ secrets.value }}"}, id="invalid-header"
+ ),
+ ],
+ )
+ def test_none_if_secrets_unusable(
+ self, caplog: pytest.LogCaptureFixture, endpoint: str, headers: dict[str, str]
+ ):
+ gateway_model, replica_model = _make_models()
+ otel = GatewayOTelConfig(endpoint=endpoint, headers=headers, traces=True)
+
+ with caplog.at_level(logging.WARNING):
+ otel_settings = get_gateway_replica_otel_settings(
+ gateway_model,
+ _make_configuration(otel),
+ replica_model,
+ secrets={"value": "s3cret\n"},
+ )
+
+ assert otel_settings is None
+ assert "gateway-replica(" in caplog.text
+ assert "cannot enable OpenTelemetry" in caplog.text
+ assert "s3cret" not in caplog.text
+
+
+class TestResolveGatewayOTelConfig:
+ @pytest.mark.parametrize(
+ "endpoint",
+ ["https://otlp.example.com:4318", "http://10.0.0.1/otlp"],
+ )
+ def test_accepts_valid_endpoints(self, endpoint: str):
+ assert resolve_gateway_otel_config(GatewayOTelConfig(endpoint=endpoint), {}) == (
+ endpoint,
+ {},
+ )
+
+ @pytest.mark.parametrize(
+ "endpoint",
+ [
+ "otlp.example.com",
+ "ftp://otlp.example.com",
+ "https://",
+ "https://otlp.example.com:99999",
+ "https://otlp.example.com\n",
+ " https://otlp.example.com",
+ "https://otlp.example.com:4318?tenant=x",
+ "https://otlp.example.com/otlp?",
+ "https://otlp.example.com#x",
+ ],
+ )
+ def test_rejects_invalid_endpoints(self, endpoint: str):
+ with pytest.raises(InvalidGatewayOTelConfigError, match="endpoint"):
+ resolve_gateway_otel_config(GatewayOTelConfig(endpoint=endpoint), {})
+
+ @pytest.mark.parametrize(
+ ("name", "value"),
+ [
+ pytest.param("Authorization", "Bearer a\nb", id="line-break"),
+ pytest.param("Authorization", " Bearer a", id="leading-whitespace"),
+ pytest.param("Authorization", "Bearer a ", id="trailing-whitespace"),
+ pytest.param("Authorization", "Bearer \u2192", id="non-latin-1"),
+ pytest.param("Bad Name", "a", id="invalid-name"),
+ ],
+ )
+ def test_rejects_invalid_headers_without_revealing_values(self, name: str, value: str):
+ otel = GatewayOTelConfig(endpoint="https://otlp", headers={name: value})
+ with pytest.raises(InvalidGatewayOTelConfigError) as exc_info:
+ resolve_gateway_otel_config(otel, {})
+ assert "Bearer" not in str(exc_info.value)
+
+
+def _make_configuration(otel: Optional[GatewayOTelConfig]) -> GatewayConfiguration:
+ return GatewayConfiguration(backend=BackendType.AWS, region="us-east-1", otel=otel)
+
+
+def _make_models() -> tuple[GatewayModel, GatewayReplicaModel]:
+ gateway_model = GatewayModel(
+ name="my-gateway",
+ project=ProjectModel(name="main"),
+ backend=BackendModel(type=BackendType.AWS),
+ )
+ replica_model = GatewayReplicaModel(
+ id=uuid.uuid4(),
+ name="my-gateway-1",
+ replica_num=1,
+ ip_address="203.0.113.5",
+ region="us-east-1",
+ )
+ return gateway_model, replica_model
diff --git a/src/tests/_internal/server/utils/test_tracing.py b/src/tests/_internal/server/utils/test_tracing.py
index 6e2bdd2b53..00a4b13fc5 100644
--- a/src/tests/_internal/server/utils/test_tracing.py
+++ b/src/tests/_internal/server/utils/test_tracing.py
@@ -1,6 +1,3 @@
-import asyncio
-import logging
-
import pytest
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
@@ -10,10 +7,9 @@
from dstack._internal.server.utils import tracing
from dstack._internal.server.utils.otel.utils import (
- _build_log_handler,
+ _build_root_sampler,
_get_db_span_name,
_get_metrics_exporters,
- _RootSpanNameSampler,
)
_span_exporter = InMemorySpanExporter()
@@ -99,55 +95,6 @@ async def failing_task():
assert counts == {"success": 2, "error": 1}
-class TestBuildLogHandler:
- def test_exports_records_with_trace_context(self, span_exporter: InMemorySpanExporter):
- from opentelemetry.sdk._logs import LoggerProvider
- from opentelemetry.sdk._logs.export import (
- InMemoryLogRecordExporter,
- SimpleLogRecordProcessor,
- )
-
- log_exporter = InMemoryLogRecordExporter()
- logger_provider = LoggerProvider()
- logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter))
- logger = logging.getLogger("test_otel_logs")
- logger.addHandler(_build_log_handler(logger_provider))
-
- tracer = trace.get_tracer(__name__)
- with tracer.start_as_current_span("op") as span:
- logger.warning("something happened")
- try:
- logger.error("cancelled", exc_info=asyncio.CancelledError())
- except Exception:
- pass
-
- records = [d.log_record for d in log_exporter.get_finished_logs()]
- assert len(records) == 1
- assert records[0].body == "something happened"
- assert records[0].severity_text == "WARN"
- assert records[0].trace_id == span.get_span_context().trace_id
-
- def test_does_not_export_otel_sdk_records(self):
- from opentelemetry.sdk._logs import LoggerProvider
- from opentelemetry.sdk._logs.export import (
- InMemoryLogRecordExporter,
- SimpleLogRecordProcessor,
- )
-
- log_exporter = InMemoryLogRecordExporter()
- logger_provider = LoggerProvider()
- logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter))
- handler = _build_log_handler(logger_provider)
-
- # An OTLP export failure logged by the SDK must not be re-exported —
- # that would be a feedback loop through the same failing exporter
- sdk_logger = logging.getLogger("opentelemetry.exporter.otlp.proto.http._log_exporter")
- sdk_logger.addHandler(handler)
- sdk_logger.error("Failed to export logs batch")
-
- assert log_exporter.get_finished_logs() == ()
-
-
class TestGetMetricsExporters:
@pytest.mark.parametrize(
("exporters", "expected"),
@@ -182,12 +129,20 @@ def test_returns_expected(self, statement, expected):
assert _get_db_span_name(statement) == expected
-class TestRootSpanNameSampler:
- def test_samples_at_rate_by_name(self):
- sampler = _RootSpanNameSampler(default_rate=1.0, background_rate=0.0)
+class TestBuildRootSampler:
+ def test_samples_background_tasks_at_background_rate(self, monkeypatch):
+ from dstack._internal.server import settings
+
+ monkeypatch.setattr(settings, "OTEL_TRACES_SAMPLE_RATE", 1.0)
+ monkeypatch.setattr(settings, "OTEL_TRACES_BACKGROUND_SAMPLE_RATE", 0.0)
+ sampler = _build_root_sampler()
http_result = sampler.should_sample(None, trace_id=123, name="GET /api/project")
- background_result = sampler.should_sample(
+ pipeline_result = sampler.should_sample(
None, trace_id=123, name="pipeline_tasks.VolumeWorker.process"
)
+ scheduled_result = sampler.should_sample(
+ None, trace_id=123, name="scheduled_tasks.process_fleets"
+ )
assert http_result.decision == Decision.RECORD_AND_SAMPLE
- assert background_result.decision == Decision.DROP
+ assert pipeline_result.decision == Decision.DROP
+ assert scheduled_result.decision == Decision.DROP
diff --git a/src/tests/_internal/utils/test_otel.py b/src/tests/_internal/utils/test_otel.py
new file mode 100644
index 0000000000..c3c892d85d
--- /dev/null
+++ b/src/tests/_internal/utils/test_otel.py
@@ -0,0 +1,60 @@
+import asyncio
+import logging
+
+import pytest
+from opentelemetry.sdk._logs import LoggerProvider
+from opentelemetry.sdk._logs.export import InMemoryLogRecordExporter, SimpleLogRecordProcessor
+from opentelemetry.sdk.trace import TracerProvider
+
+from dstack._internal.server.utils.logging import AsyncioCancelledErrorFilter
+from dstack._internal.utils.otel import build_log_handler
+
+
+@pytest.fixture
+def log_exporter() -> InMemoryLogRecordExporter:
+ return InMemoryLogRecordExporter()
+
+
+@pytest.fixture
+def logger_provider(log_exporter: InMemoryLogRecordExporter) -> LoggerProvider:
+ logger_provider = LoggerProvider()
+ logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter))
+ return logger_provider
+
+
+class TestBuildLogHandler:
+ def test_exports_records_with_trace_context(
+ self, log_exporter: InMemoryLogRecordExporter, logger_provider: LoggerProvider
+ ):
+ logger = logging.getLogger("test_otel_logs")
+ logger.addHandler(
+ build_log_handler(logger_provider, filters=[AsyncioCancelledErrorFilter()])
+ )
+
+ # A local provider, as the global one can only be set once per process
+ tracer = TracerProvider().get_tracer(__name__)
+ with tracer.start_as_current_span("op") as span:
+ logger.warning("something happened")
+ try:
+ logger.error("cancelled", exc_info=asyncio.CancelledError())
+ except Exception:
+ pass
+
+ records = [d.log_record for d in log_exporter.get_finished_logs()]
+ assert len(records) == 1
+ assert records[0].body == "something happened"
+ assert records[0].severity_text == "WARN"
+ assert records[0].trace_id == span.get_span_context().trace_id
+
+ def test_does_not_export_otel_sdk_records(
+ self, log_exporter: InMemoryLogRecordExporter, logger_provider: LoggerProvider
+ ):
+ handler = build_log_handler(logger_provider)
+
+ # An OTLP export failure logged by the SDK must not be re-exported —
+ # that would be a feedback loop through the same failing exporter
+ sdk_logger = logging.getLogger("opentelemetry.exporter.otlp.proto.http._log_exporter")
+ sdk_logger.addHandler(handler)
+ sdk_logger.error("Failed to export logs batch")
+
+ assert log_exporter.get_finished_logs() == ()