diff --git a/mkdocs/docs/concepts/gateways.md b/mkdocs/docs/concepts/gateways.md index c84c19af76..9239f915c4 100644 --- a/mkdocs/docs/concepts/gateways.md +++ b/mkdocs/docs/concepts/gateways.md @@ -247,6 +247,40 @@ $ dstack gateway list Replicated gateways do not support automatic certificate issuance via Let's Encrypt (`certificate: { type: lets-encrypt }`). For TLS termination, use an external load balancer or one of the other [certificate types](#certificate). +### OpenTelemetry + +The gateway can export its traces, logs, and metrics via OTLP over HTTP. + +
+ +```yaml +type: gateway +name: example-gateway + +backend: aws +region: eu-west-1 + +domain: example.com + +otel: + endpoint: https://otlp.example.com:4318 + headers: + Authorization: Bearer ${{ secrets.otlp_token }} + traces: true + logs: true + metrics: true +``` + +
+ +!!! warning "Use secrets for credentials" + The `otel` configuration is visible to everyone who can view the gateway, including members of + projects the gateway is [exported](exports.md) to. Store credentials, such as authorization + headers, in [secrets](secrets.md) and reference them as `${{ secrets. }}`. Only the + references are visible, not the secret values. + +Telemetry is reported with `service.name` set to `dstack-gateway` and resource attributes identifying the gateway and its replica. + !!! info "Reference" For all gateway configuration options, refer to the [reference](../reference/dstack.yml/gateway.md). diff --git a/mkdocs/docs/concepts/secrets.md b/mkdocs/docs/concepts/secrets.md index 0528e10c09..91127a68f0 100644 --- a/mkdocs/docs/concepts/secrets.md +++ b/mkdocs/docs/concepts/secrets.md @@ -84,7 +84,7 @@ OK ## Use secrets -You can use the `${{ secrets. }}` syntax to reference secrets in run configurations. Currently, secrets interpolation is supported in `env` and `registry_auth` properties. +You can use the `${{ secrets. }}` syntax to reference secrets in run configurations. Currently, secrets interpolation is supported in `env` and `registry_auth` properties. Gateway configurations also support secrets in [`otel`](gateways.md#opentelemetry) `endpoint` and `headers`. ### `env` diff --git a/mkdocs/docs/guides/server-deployment.md b/mkdocs/docs/guides/server-deployment.md index bd26538acb..7e0cfca24a 100644 --- a/mkdocs/docs/guides/server-deployment.md +++ b/mkdocs/docs/guides/server-deployment.md @@ -459,6 +459,10 @@ $ DSTACK_OTEL_TRACES_ENABLED=1 \ `rate()` queries may be empty. Set e.g. `OTEL_METRIC_EXPORT_INTERVAL=30000`, and make sure the datasource's `timeInterval` reflects the cadence you choose. +#### Gateways + +Gateways can export their own telemetry. It is configured per gateway, see [gateway OpenTelemetry](../concepts/gateways.md#opentelemetry). + ## Encryption By default, `dstack` stores data in plaintext. To enforce encryption, you diff --git a/mkdocs/docs/reference/dstack.yml/gateway.md b/mkdocs/docs/reference/dstack.yml/gateway.md index d52fac9e0a..ae44170246 100644 --- a/mkdocs/docs/reference/dstack.yml/gateway.md +++ b/mkdocs/docs/reference/dstack.yml/gateway.md @@ -47,3 +47,9 @@ Set to `null` to disable certificates (e.g. for [private gateways](../../concept show_root_heading: false type: required: true + +### `otel` + +#SCHEMA# dstack._internal.core.models.gateways.GatewayOTelConfig + overrides: + show_root_heading: false diff --git a/pyproject.toml b/pyproject.toml index 3912c45065..4f0cf85baa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -226,6 +226,13 @@ gateway = [ # Gateway authorization needs the pool cancellation and recovery fixes in httpx2 (#4338). "httpx2>=2.13.1", "jinja2", + # Always installed so that OpenTelemetry can be enabled without reinstalling the gateway. + "opentelemetry-sdk>=1.39.0", + "opentelemetry-exporter-otlp-proto-http>=1.39.0", + "opentelemetry-instrumentation-fastapi>=0.51b0", + "opentelemetry-instrumentation-httpx>=0.65b0", + "opentelemetry-instrumentation-system-metrics>=0.51b0", + "opentelemetry-instrumentation-logging>=0.61b0", ] server = [ "fastapi", @@ -311,8 +318,8 @@ fluentbit = [ "dstack[server]", ] otel = [ - "opentelemetry-sdk>=1.30.0", - "opentelemetry-exporter-otlp-proto-http>=1.30.0", + "opentelemetry-sdk>=1.39.0", + "opentelemetry-exporter-otlp-proto-http>=1.39.0", "opentelemetry-exporter-prometheus>=0.51b0", "opentelemetry-instrumentation-fastapi>=0.51b0", "opentelemetry-instrumentation-sqlalchemy>=0.51b0", diff --git a/src/dstack/_internal/core/compatibility/gateways.py b/src/dstack/_internal/core/compatibility/gateways.py index 63798b4f79..aab17b08a3 100644 --- a/src/dstack/_internal/core/compatibility/gateways.py +++ b/src/dstack/_internal/core/compatibility/gateways.py @@ -71,5 +71,7 @@ def _get_gateway_configuration_excludes( configuration_excludes["default"] = True if configuration.load_balancer is None: configuration_excludes["load_balancer"] = True + if configuration.otel is None: + configuration_excludes["otel"] = True return configuration_excludes diff --git a/src/dstack/_internal/core/models/gateways.py b/src/dstack/_internal/core/models/gateways.py index bff16d3e57..0eb7067b21 100644 --- a/src/dstack/_internal/core/models/gateways.py +++ b/src/dstack/_internal/core/models/gateways.py @@ -90,6 +90,53 @@ class ALBGatewayLoadBalancer(CoreModel): AnyGatewayLoadBalancer = Union[ALBGatewayLoadBalancer] +class GatewayOTelConfig(CoreModel): + endpoint: Annotated[ + str, + Field( + description=( + "The OTLP over HTTP endpoint, e.g. `https://otlp.example.com:4318`." + " Signal paths such as `/v1/traces` are appended to it." + " Can reference secrets, e.g. `${{ secrets.otlp_endpoint }}`" + ), + ), + ] + headers: Annotated[ + Optional[Dict[str, str]], + Field( + description=( + "Headers sent with OTLP requests, e.g. for authorization." + " Values can reference secrets, e.g. `Bearer ${{ secrets.otlp_token }}`" + ), + ), + ] = None + traces: Annotated[bool, Field(description="Whether to export traces")] = False + logs: Annotated[bool, Field(description="Whether to export logs")] = False + metrics: Annotated[bool, Field(description="Whether to export metrics")] = False + traces_sample_rate: Annotated[ + float, + Field( + description=( + "The head sampling rate for traces." + " The default assumes sampling is done downstream, e.g. in an OTel Collector" + ), + ge=0, + le=1, + ), + ] = 1.0 + traces_auth_sample_rate: Annotated[ + float, + Field( + description=( + "The head sampling rate for traces of authorization subrequests," + " which Nginx makes for every request to services with authorization enabled" + ), + ge=0, + le=1, + ), + ] = 0.01 + + class GatewayConfiguration(CoreModel): type: Literal["gateway"] = "gateway" name: Annotated[Optional[str], Field(description="The gateway name")] = None @@ -166,6 +213,10 @@ class GatewayConfiguration(CoreModel): ) ), ] = None + otel: Annotated[ + Optional[GatewayOTelConfig], + Field(description=("The OpenTelemetry export configuration of the gateway app")), + ] = None _validate_tags = field_validator("tags", mode="before")(tags_validator) diff --git a/src/dstack/_internal/proxy/gateway/app.py b/src/dstack/_internal/proxy/gateway/app.py index 6627e9e3f3..72946b15da 100644 --- a/src/dstack/_internal/proxy/gateway/app.py +++ b/src/dstack/_internal/proxy/gateway/app.py @@ -7,6 +7,7 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from dstack._internal.proxy.gateway import otel from dstack._internal.proxy.gateway.auth import GatewayProxyAuthProvider from dstack._internal.proxy.gateway.const import ( DSTACK_DIR_ON_GATEWAY, @@ -43,6 +44,7 @@ async def lifespan(app: FastAPI): repo = await get_gateway_proxy_repo(await injector.get_repo().__anext__()) nginx = injector.get_nginx() service_conn_pool = await injector.get_service_connection_pool() + otel.apply_settings((await repo.get_config()).otel_settings) await run_async(nginx.write_global_conf) await apply_all(repo, nginx, service_conn_pool) diff --git a/src/dstack/_internal/proxy/gateway/main.py b/src/dstack/_internal/proxy/gateway/main.py index 134f34f538..76681a8349 100644 --- a/src/dstack/_internal/proxy/gateway/main.py +++ b/src/dstack/_internal/proxy/gateway/main.py @@ -1,5 +1,6 @@ import logging +from dstack._internal.proxy.gateway import otel from dstack._internal.proxy.gateway.app import make_app @@ -15,3 +16,5 @@ def configure_logging(level: int = logging.INFO): configure_logging(logging.DEBUG) app = make_app() +# Not in make_app, since global instrumentation would leak between apps created in tests +otel.instrument(app) diff --git a/src/dstack/_internal/proxy/gateway/models.py b/src/dstack/_internal/proxy/gateway/models.py index 80ce0b8abe..e1d4746922 100644 --- a/src/dstack/_internal/proxy/gateway/models.py +++ b/src/dstack/_internal/proxy/gateway/models.py @@ -1,6 +1,6 @@ """Things stored in GatewayProxyRepo in addition to those from BaseProxyRepo.""" -from typing import Optional +from typing import Optional, Union from pydantic import AnyHttpUrl @@ -21,5 +21,21 @@ class ACMESettings(ImmutableModel): eab_hmac_key: Optional[str] = None +class OTelSettings(ImmutableModel): + traces_enabled: bool + logs_enabled: bool + metrics_enabled: bool + endpoint: str + headers: dict[str, str] + traces_sample_rate: float + traces_auth_sample_rate: float + resource_attributes: dict[str, Union[str, int]] + + @property + def any_enabled(self) -> bool: + return self.traces_enabled or self.logs_enabled or self.metrics_enabled + + class GlobalProxyConfig(ImmutableModel): acme_settings: ACMESettings = ACMESettings() + otel_settings: Optional[OTelSettings] = None diff --git a/src/dstack/_internal/proxy/gateway/otel.py b/src/dstack/_internal/proxy/gateway/otel.py new file mode 100644 index 0000000000..88b0fb8d58 --- /dev/null +++ b/src/dstack/_internal/proxy/gateway/otel.py @@ -0,0 +1,104 @@ +import logging +from typing import Literal, Optional + +from fastapi import FastAPI +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.instrumentation.httpx import HTTPX2ClientInstrumentor, HTTPXClientInstrumentor +from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace.sampling import ParentBased + +from dstack._internal import settings as core_settings +from dstack._internal.proxy.gateway.models import OTelSettings +from dstack._internal.utils import otel as otel_utils +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) + +_AUTH_SPAN_NAME_PREFIX = "GET /api/auth/" + +_applied_settings: Optional[OTelSettings] = None + + +def instrument(app: FastAPI) -> None: + """Must be called before the app starts serving. No-op until `apply_settings` is called.""" + FastAPIInstrumentor.instrument_app(app) + HTTPXClientInstrumentor().instrument() + HTTPX2ClientInstrumentor().instrument() + SystemMetricsInstrumentor(config=otel_utils.PROCESS_METRICS_CONFIG).instrument() + + +def apply_settings(settings: Optional[OTelSettings]) -> bool: + """ + Returns True if the app must be restarted to apply `settings`, + since OpenTelemetry providers can only be set once per process. + """ + global _applied_settings + if settings is not None and not settings.any_enabled: + settings = None + if settings == _applied_settings: + return False + if _applied_settings is not None: + logger.info("OpenTelemetry settings changed, the gateway app needs a restart") + return True + assert settings is not None + # Set before configuring, since a failed configuration cannot be retried in this process + _applied_settings = settings + try: + _configure(settings) + except Exception: + logger.exception("Failed to configure OpenTelemetry") + return False + + +def _configure(settings: OTelSettings) -> None: + resource = _get_resource(settings) + if settings.traces_enabled: + otel_utils.configure_tracing( + resource, + ParentBased(root=_build_root_sampler(settings)), + OTLPSpanExporter( + endpoint=_get_signal_endpoint(settings, "traces"), headers=settings.headers + ), + ) + if settings.metrics_enabled: + exporter = OTLPMetricExporter( + endpoint=_get_signal_endpoint(settings, "metrics"), headers=settings.headers + ) + otel_utils.configure_metrics(resource, [PeriodicExportingMetricReader(exporter)]) + if settings.logs_enabled: + otel_utils.configure_log_export( + resource, + OTLPLogExporter( + endpoint=_get_signal_endpoint(settings, "logs"), headers=settings.headers + ), + # The app logs at DEBUG level to journald, which would be too noisy to export + level=logging.INFO, + ) + + +def _build_root_sampler(settings: OTelSettings) -> otel_utils.RootSpanNameSampler: + return otel_utils.RootSpanNameSampler( + default_rate=settings.traces_sample_rate, + rates_by_name_prefix={_AUTH_SPAN_NAME_PREFIX: settings.traces_auth_sample_rate}, + ) + + +def _get_signal_endpoint( + settings: OTelSettings, signal: Literal["traces", "metrics", "logs"] +) -> str: + return f"{settings.endpoint.rstrip('/')}/v1/{signal}" + + +def _get_resource(settings: OTelSettings) -> Resource: + return Resource.create( + { + **settings.resource_attributes, + "service.name": "dstack-gateway", + "service.version": core_settings.DSTACK_VERSION or "dev", + } + ) diff --git a/src/dstack/_internal/proxy/gateway/resources/nginx/entrypoint.jinja2 b/src/dstack/_internal/proxy/gateway/resources/nginx/entrypoint.jinja2 index ce0160f558..fd4029c946 100644 --- a/src/dstack/_internal/proxy/gateway/resources/nginx/entrypoint.jinja2 +++ b/src/dstack/_internal/proxy/gateway/resources/nginx/entrypoint.jinja2 @@ -4,6 +4,10 @@ server { proxy_pass http://localhost:{{ proxy_port }}/api/models/{{ project_name }}/; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $host; + # Don't let public clients control the gateway app's tracing + proxy_set_header traceparent ""; + proxy_set_header tracestate ""; + proxy_set_header baggage ""; proxy_read_timeout {{ read_timeout }}s; } listen 80; diff --git a/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 b/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 index e3eced5f6d..002c8c3687 100644 --- a/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 +++ b/src/dstack/_internal/proxy/gateway/resources/nginx/service.jinja2 @@ -109,6 +109,10 @@ server { proxy_set_header Content-Length ""; proxy_set_header X-Original-URI $request_uri; proxy_set_header Authorization $http_authorization; + # Don't let public clients control the gateway app's tracing + proxy_set_header traceparent ""; + proxy_set_header tracestate ""; + proxy_set_header baggage ""; } {% endif %} diff --git a/src/dstack/_internal/proxy/gateway/routers/config.py b/src/dstack/_internal/proxy/gateway/routers/config.py index 24dac6a52f..25b5862c4f 100644 --- a/src/dstack/_internal/proxy/gateway/routers/config.py +++ b/src/dstack/_internal/proxy/gateway/routers/config.py @@ -1,7 +1,10 @@ +import os +import signal from typing import Annotated -from fastapi import APIRouter, Depends +from fastapi import APIRouter, BackgroundTasks, Depends +from dstack._internal.proxy.gateway import otel from dstack._internal.proxy.gateway.deps import get_gateway_proxy_repo from dstack._internal.proxy.gateway.models import ACMESettings, GlobalProxyConfig from dstack._internal.proxy.gateway.repo.repo import GatewayProxyRepo @@ -15,6 +18,7 @@ async def update_global_config( body: ConfigRequest, repo: Annotated[GatewayProxyRepo, Depends(get_gateway_proxy_repo)], + background_tasks: BackgroundTasks, ) -> OkResponse: await repo.set_config( GlobalProxyConfig( @@ -23,6 +27,10 @@ async def update_global_config( eab_kid=body.acme_eab_kid, eab_hmac_key=body.acme_eab_hmac_key, ), + otel_settings=body.otel_settings, ), ) + if otel.apply_settings(body.otel_settings): + # Saved settings are applied on startup. systemd restarts the app (`Restart=always`) + background_tasks.add_task(os.kill, os.getpid(), signal.SIGTERM) return OkResponse() diff --git a/src/dstack/_internal/proxy/gateway/schemas/config.py b/src/dstack/_internal/proxy/gateway/schemas/config.py index 09cfc2101f..85b67c32f9 100644 --- a/src/dstack/_internal/proxy/gateway/schemas/config.py +++ b/src/dstack/_internal/proxy/gateway/schemas/config.py @@ -2,8 +2,11 @@ from pydantic import AnyHttpUrl, BaseModel +from dstack._internal.proxy.gateway.models import OTelSettings + class ConfigRequest(BaseModel): acme_server: Optional[AnyHttpUrl] = None acme_eab_kid: Optional[str] = None acme_eab_hmac_key: Optional[str] = None + otel_settings: Optional[OTelSettings] = None diff --git a/src/dstack/_internal/proxy/gateway/testing/common.py b/src/dstack/_internal/proxy/gateway/testing/common.py index 2718ec6bae..1feb374c0e 100644 --- a/src/dstack/_internal/proxy/gateway/testing/common.py +++ b/src/dstack/_internal/proxy/gateway/testing/common.py @@ -1,7 +1,9 @@ from dataclasses import dataclass -from typing import Union +from typing import Any, Union from unittest.mock import AsyncMock, MagicMock +from dstack._internal.proxy.gateway.models import OTelSettings + AnyMock = Union[MagicMock, AsyncMock] @@ -11,3 +13,17 @@ class Mocks: run_certbot: AnyMock open_conn: AnyMock close_conn: AnyMock + + +def make_otel_settings(**kwargs: Any) -> OTelSettings: + defaults: dict[str, Any] = { + "traces_enabled": False, + "logs_enabled": False, + "metrics_enabled": False, + "endpoint": "https://otlp.example.com", + "headers": {}, + "traces_sample_rate": 1.0, + "traces_auth_sample_rate": 0.01, + "resource_attributes": {}, + } + return OTelSettings(**{**defaults, **kwargs}) diff --git a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py index 63da7292ef..95460f6c23 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/gateway_replicas.py @@ -295,6 +295,7 @@ async def _load_gateway_replica( replica_fields: list[InstrumentedAttribute[Any]], gateway_fields: list[InstrumentedAttribute[Any]], load_backends: bool = False, + load_gateway_project: bool = False, load_gateway_backend_type: bool = False, ) -> Optional[GatewayReplicaModel]: def build_gateway_options( @@ -302,10 +303,11 @@ def build_gateway_options( ) -> list[ExecutableOption]: gateway_load = joinedload(gateway_attr).load_only(*gateway_fields) options: list[ExecutableOption] = [gateway_load] - if load_backends: - options.append( - gateway_load.joinedload(GatewayModel.project).selectinload(ProjectModel.backends) - ) + if load_gateway_project or load_backends: + project_load = gateway_load.joinedload(GatewayModel.project) + if load_backends: + project_load = project_load.selectinload(ProjectModel.backends) + options.append(project_load) if load_gateway_backend_type: options.append( gateway_load.joinedload(GatewayModel.backend).load_only(BackendModel.type) @@ -515,6 +517,7 @@ async def _process_provisioning_item(item: GatewayReplicaPipelineItem): GatewayReplicaModel.instance_id, GatewayReplicaModel.backend_id, GatewayReplicaModel.configuration, + GatewayReplicaModel.region, ], gateway_fields=_GATEWAY_FIELDS_MIN + [ @@ -525,6 +528,7 @@ async def _process_provisioning_item(item: GatewayReplicaPipelineItem): GatewayModel.backend_data, ], load_backends=True, + load_gateway_project=True, load_gateway_backend_type=True, ) if replica_model is None: @@ -539,7 +543,7 @@ async def _process_provisioning_item(item: GatewayReplicaPipelineItem): if _is_legacy_aws_acm_gateway_with_pending_migration(gateway_model): await _commit_update(item, replica_model, update_map={}) return - error = await _connect_and_configure_gateway_replica(replica_model) + error = await _connect_and_configure_gateway_replica(gateway_model, replica_model) if error is not None: await _commit_update( item, @@ -604,6 +608,7 @@ async def _register_replica_with_load_balancer( async def _connect_and_configure_gateway_replica( + gateway_model: GatewayModel, gateway_replica: GatewayReplicaModel, ) -> Optional[str]: """Returns an error message on failure, None on success.""" @@ -618,7 +623,11 @@ async def _connect_and_configure_gateway_replica( if connection is None: return "Failed to connect to gateway replica" try: - await gateways_services.configure_gateway_replica(connection) + async with get_session_ctx() as session: + otel_settings = await gateways_services.resolve_gateway_replica_otel_settings( + session, gateway_model, gateway_replica + ) + await gateways_services.configure_gateway_replica(connection, otel_settings) except Exception: logger.exception("%s: failed to configure gateway replica", fmt(gateway_replica)) return "Failed to configure gateway replica" @@ -634,6 +643,7 @@ async def _process_running_item(item: GatewayReplicaPipelineItem): GatewayReplicaModel.scale_in, GatewayReplicaModel.ip_address, GatewayReplicaModel.ssh_private_key, + GatewayReplicaModel.region, ], gateway_fields=_GATEWAY_FIELDS_MIN + [ @@ -641,6 +651,7 @@ async def _process_running_item(item: GatewayReplicaPipelineItem): GatewayModel.region, GatewayModel.wildcard_domain, ], + load_gateway_project=True, load_gateway_backend_type=True, ) if replica_model is None: @@ -712,8 +723,12 @@ async def _process_running_item(item: GatewayReplicaPipelineItem): # lost its global config along with its services, and certbot would fall back to # its default ACME server, so resend the config first. try: + async with get_session_ctx() as session: + otel_settings = await gateways_services.resolve_gateway_replica_otel_settings( + session, gateway_model, replica_model + ) async with connection.client() as client: - await client.submit_gateway_config() + await client.submit_gateway_config(otel_settings) except Exception as e: logger.warning("%s: failed to submit gateway config: %r", fmt(replica_model), e) await _commit_update(item, replica_model, update_map={}) diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index 678649426d..18650bb00b 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -2,10 +2,9 @@ import datetime import itertools import uuid -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, Mapping from contextlib import asynccontextmanager from datetime import timedelta -from functools import partial from typing import List, Optional, Sequence import httpx @@ -53,6 +52,7 @@ from dstack._internal.core.services.diff import ModelDiff, format_diff_fields_for_event from dstack._internal.core.services.gateways import diff_gateway_configurations from dstack._internal.proxy.gateway.const import SERVICE_SCALING_WINDOWS +from dstack._internal.proxy.gateway.models import OTelSettings from dstack._internal.proxy.gateway.schemas.stats import PerWindowStats, Stat from dstack._internal.server import settings from dstack._internal.server.db import get_db, is_db_postgres, is_db_sqlite @@ -72,6 +72,12 @@ ) from dstack._internal.server.services.gateways.connection import GatewayConnection from dstack._internal.server.services.gateways.pool import gateway_connections_pool +from dstack._internal.server.services.gateways.telemetry import ( + InvalidGatewayOTelConfigError, + get_gateway_replica_otel_settings, + is_gateway_otel_enabled, + resolve_gateway_otel_config, +) from dstack._internal.server.services.locking import ( advisory_lock_ctx, get_locker, @@ -79,6 +85,7 @@ ) from dstack._internal.server.services.pipelines import PipelineHinterProtocol from dstack._internal.server.services.plugins import apply_plugin_policies +from dstack._internal.server.services.secrets import get_project_secrets_mapping from dstack._internal.server.utils.common import gather_map_async from dstack._internal.utils import crypto from dstack._internal.utils.common import ( @@ -252,7 +259,7 @@ async def create_gateway( spec=GatewaySpec(configuration=configuration), ) effective_configuration = spec.configuration - _validate_gateway_configuration(effective_configuration) + await _validate_gateway_configuration(session, project, effective_configuration) configuration = effective_configuration # Leasing the backend also checks that it can be initialized, not just that it's configured. @@ -746,13 +753,46 @@ def _merge_per_window_stats(stats_per_gateway_replica: list[PerWindowStats]) -> async def init_gateways(session: AsyncSession): res = await session.execute( - select(GatewayReplicaModel).where( + select(GatewayReplicaModel) + .where( GatewayReplicaModel.status == GatewayReplicaStatus.RUNNING, GatewayReplicaModel.active == True, GatewayReplicaModel.deleted == False, ) + .options( + *( + joinedload(gateway_attr).options( + joinedload(GatewayModel.project), + joinedload(GatewayModel.backend).load_only(BackendModel.type), + ) + for gateway_attr in ( + GatewayReplicaModel.gateway, + GatewayReplicaModel.legacy_gateway, + ) + ) + ) ) - gateway_replicas = res.scalars().all() + gateway_replicas = res.unique().scalars().all() + otel_settings: dict[str, Optional[OTelSettings]] = {} + secrets_cache: dict[uuid.UUID, Mapping[str, str]] = {} + for gateway_replica in gateway_replicas: + if gateway_replica.ip_address is None: + continue + try: + replica_otel_settings = await resolve_gateway_replica_otel_settings( + session, + gateway_replica.gateway or gateway_replica.legacy_gateway, + gateway_replica, + secrets_cache=secrets_cache, + ) + except Exception: + # Don't let one gateway prevent initializing the others + logger.exception( + "Failed to resolve OpenTelemetry settings for gateway replica %s", + gateway_replica.ip_address, + ) + replica_otel_settings = None + otel_settings[gateway_replica.ip_address] = replica_otel_settings if len(gateway_replicas) > 0: logger.info( @@ -798,9 +838,9 @@ async def init_gateways(session: AsyncSession): gateway_replica.app_updated_at = get_current_datetime() for connection, error in await gather_map_async( - await gateway_connections_pool.all(), + [c for c in await gateway_connections_pool.all() if c.ip_address in otel_settings], # Need several attempts to handle short gateway replica downtime after update - partial(configure_gateway_replica, attempts=7), + lambda c: configure_gateway_replica(c, otel_settings[c.ip_address], attempts=7), return_exceptions=True, ): if isinstance(error, Exception): @@ -848,6 +888,7 @@ def _recently_updated(gateway_replica_model: GatewayReplicaModel) -> bool: async def configure_gateway_replica( connection: GatewayConnection, + otel_settings: Optional[OTelSettings], attempts: int = GATEWAY_CONFIGURE_ATTEMPTS, ) -> None: """ @@ -860,7 +901,7 @@ async def configure_gateway_replica( for attempt in range(attempts - 1): try: async with connection.client() as client: - await client.submit_gateway_config() + await client.submit_gateway_config(otel_settings) break except httpx.RequestError as e: logger.debug( @@ -873,11 +914,35 @@ async def configure_gateway_replica( await asyncio.sleep(GATEWAY_CONFIGURE_DELAY) else: async with connection.client() as client: - await client.submit_gateway_config() + await client.submit_gateway_config(otel_settings) logger.info("Gateway replica %s configured", connection.ip_address) +async def resolve_gateway_replica_otel_settings( + session: AsyncSession, + gateway_model: Optional[GatewayModel], + replica_model: GatewayReplicaModel, + secrets_cache: Optional[dict[uuid.UUID, Mapping[str, str]]] = None, +) -> Optional[OTelSettings]: + """ + `secrets_cache` maps project IDs to project secrets. When resolving settings + for many replicas, pass the same dict to fetch each project's secrets only once. + """ + if gateway_model is None: + return None + configuration = get_gateway_configuration(gateway_model) + if configuration.otel is None or not is_gateway_otel_enabled(configuration.otel): + return None + if secrets_cache is None: + secrets_cache = {} + secrets = secrets_cache.get(gateway_model.project_id) + if secrets is None: + secrets = await get_project_secrets_mapping(session=session, project=gateway_model.project) + secrets_cache[gateway_model.project_id] = secrets + return get_gateway_replica_otel_settings(gateway_model, configuration, replica_model, secrets) + + def get_gateway_replica_models(gateway_model: GatewayModel) -> List[GatewayReplicaModel]: replicas = list(gateway_model.gateway_replicas) if gateway_model.gateway_replica is not None: # pre-0.20.25 gateway @@ -1010,7 +1075,7 @@ async def get_plan( project=project.name, spec=spec, ) - _validate_gateway_configuration(effective_spec.configuration) + await _validate_gateway_configuration(session, project, effective_spec.configuration) action = ApplyAction.CREATE current_gateway: Optional[Gateway] = None @@ -1066,7 +1131,7 @@ async def apply_plan( spec=plan.spec, ) new_configuration = spec.configuration - _validate_gateway_configuration(new_configuration) + await _validate_gateway_configuration(session, project, new_configuration) if new_configuration.name is None: return await create_gateway( @@ -1163,7 +1228,9 @@ def _can_update_gateway_in_place(conf_diff: ModelDiff) -> bool: return all(field in _CONF_UPDATABLE_FIELDS for field in conf_diff) -def _validate_gateway_configuration(configuration: GatewayConfiguration): +async def _validate_gateway_configuration( + session: AsyncSession, project: ProjectModel, configuration: GatewayConfiguration +): check_backend_type_available(configuration.backend) if configuration.backend not in BACKENDS_WITH_GATEWAY_SUPPORT: raise ServerClientError( @@ -1258,3 +1325,10 @@ def _validate_gateway_configuration(configuration: GatewayConfiguration): " or `certificate: { type: gcp-cm, name: }` (GCP Certificate Manager)" ) raise ServerClientError(err) + + if configuration.otel is not None: + secrets = await get_project_secrets_mapping(session=session, project=project) + try: + resolve_gateway_otel_config(configuration.otel, secrets) + except InvalidGatewayOTelConfigError as e: + raise ServerClientError(f"Invalid gateway otel configuration: {e}") diff --git a/src/dstack/_internal/server/services/gateways/client.py b/src/dstack/_internal/server/services/gateways/client.py index 0c9defecca..b3a803ca05 100644 --- a/src/dstack/_internal/server/services/gateways/client.py +++ b/src/dstack/_internal/server/services/gateways/client.py @@ -11,6 +11,7 @@ from dstack._internal.core.models.configurations import RateLimit, ServiceConfiguration from dstack._internal.core.models.instances import SSHConnectionParams from dstack._internal.core.models.runs import JobSpec, JobSubmission, get_service_port +from dstack._internal.proxy.gateway.models import OTelSettings from dstack._internal.proxy.gateway.schemas.services import ServiceListItem, ServiceListResponse from dstack._internal.proxy.gateway.schemas.stats import ServiceStats from dstack._internal.server import settings @@ -187,13 +188,16 @@ async def list_services(self) -> list[ServiceListItem]: self.is_server_ready = True return resp_parsed.services - async def submit_gateway_config(self) -> None: + async def submit_gateway_config(self, otel_settings: Optional[OTelSettings]) -> None: resp = await self._client.post( self._url("/api/config"), json={ "acme_server": settings.ACME_SERVER, "acme_eab_kid": settings.ACME_EAB_KID, "acme_eab_hmac_key": settings.ACME_EAB_HMAC_KEY, + "otel_settings": ( + otel_settings.model_dump(mode="json") if otel_settings is not None else None + ), }, ) if resp.status_code == 400: diff --git a/src/dstack/_internal/server/services/gateways/telemetry.py b/src/dstack/_internal/server/services/gateways/telemetry.py new file mode 100644 index 0000000000..19ef253641 --- /dev/null +++ b/src/dstack/_internal/server/services/gateways/telemetry.py @@ -0,0 +1,112 @@ +import re +from collections.abc import Mapping +from typing import Optional, Union +from urllib.parse import urlsplit + +from dstack._internal.core.models.gateways import GatewayConfiguration, GatewayOTelConfig +from dstack._internal.proxy.gateway.models import OTelSettings +from dstack._internal.server import settings +from dstack._internal.server.models import GatewayModel, GatewayReplicaModel +from dstack._internal.server.services.logging import fmt +from dstack._internal.utils.interpolator import InterpolatorError, VariablesInterpolator +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) + + +def get_gateway_replica_otel_settings( + gateway_model: GatewayModel, + gateway_configuration: GatewayConfiguration, + replica_model: GatewayReplicaModel, + secrets: Mapping[str, str], +) -> Optional[OTelSettings]: + otel = gateway_configuration.otel + if otel is None or not is_gateway_otel_enabled(otel): + return None + try: + endpoint, headers = resolve_gateway_otel_config(otel, secrets) + except InvalidGatewayOTelConfigError as e: + # E.g., a referenced secret was deleted or changed to an invalid value + logger.warning("%s: cannot enable OpenTelemetry: %s", fmt(replica_model), e) + return None + return OTelSettings( + traces_enabled=otel.traces, + logs_enabled=otel.logs, + metrics_enabled=otel.metrics, + endpoint=endpoint, + headers=headers, + traces_sample_rate=otel.traces_sample_rate, + traces_auth_sample_rate=otel.traces_auth_sample_rate, + resource_attributes=_get_resource_attributes(gateway_model, replica_model), + ) + + +def is_gateway_otel_enabled(otel: GatewayOTelConfig) -> bool: + return otel.traces or otel.logs or otel.metrics + + +class InvalidGatewayOTelConfigError(ValueError): + """The message is shown to users and must not contain secret values.""" + + +def resolve_gateway_otel_config( + otel: GatewayOTelConfig, secrets: Mapping[str, str] +) -> tuple[str, dict[str, str]]: + """Returns the endpoint and headers with secrets interpolated.""" + interpolate = VariablesInterpolator({"secrets": secrets}).interpolate_or_error + try: + endpoint = interpolate(otel.endpoint) + headers = {name: interpolate(value) for name, value in (otel.headers or {}).items()} + except InterpolatorError as e: + raise InvalidGatewayOTelConfigError(str(e)) from None + _validate_endpoint(endpoint) + for name, value in headers.items(): + _validate_header(name, value) + return endpoint, headers + + +def _validate_endpoint(endpoint: str) -> None: + # Checked explicitly, since urlsplit silently strips tabs and line breaks + if any(c.isspace() or not c.isprintable() for c in endpoint): + raise InvalidGatewayOTelConfigError( + "endpoint must not contain whitespace or control characters" + ) + try: + url = urlsplit(endpoint) + url.port # Raises on invalid ports + except ValueError: + raise InvalidGatewayOTelConfigError("endpoint must be a valid URL") from None + if url.scheme not in ("http", "https") or not url.hostname: + raise InvalidGatewayOTelConfigError("endpoint must be an http:// or https:// URL") + # Signal paths are appended to the endpoint and would end up in the query or fragment + if "?" in endpoint or "#" in endpoint: + raise InvalidGatewayOTelConfigError("endpoint must not contain a query or fragment") + + +# RFC 9110 token. Use with fullmatch +_HEADER_NAME_RE = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+") + + +def _validate_header(name: str, value: str) -> None: + if not _HEADER_NAME_RE.fullmatch(name): + raise InvalidGatewayOTelConfigError(f"invalid header name {name!r}") + if not (value.isprintable() and value == value.strip() and all(ord(c) < 256 for c in value)): + raise InvalidGatewayOTelConfigError(f"header {name!r} has an invalid value") + + +def _get_resource_attributes( + gateway_model: GatewayModel, replica_model: GatewayReplicaModel +) -> dict[str, Union[str, int]]: + attributes: dict[str, Union[str, int]] = { + "service.instance.id": str(replica_model.id), + "dstack.gateway.replica.num": replica_model.replica_num, + "deployment.environment.name": settings.SERVER_ENVIRONMENT, + "dstack.project.name": gateway_model.project.name, + "dstack.gateway.name": gateway_model.name, + "dstack.backend": gateway_model.backend.type.value, + } + if replica_model.ip_address is not None: + attributes["host.name"] = replica_model.ip_address + if replica_model.region is not None: + attributes["cloud.region"] = replica_model.region + return attributes diff --git a/src/dstack/_internal/server/utils/otel/utils.py b/src/dstack/_internal/server/utils/otel/utils.py index ed4f943542..3f7f018187 100644 --- a/src/dstack/_internal/server/utils/otel/utils.py +++ b/src/dstack/_internal/server/utils/otel/utils.py @@ -1,37 +1,19 @@ -import logging import re -from typing import List, Optional, Sequence +from typing import List, Optional from fastapi import FastAPI -from opentelemetry import metrics, trace -from opentelemetry._logs import set_logger_provider -from opentelemetry.context import Context from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.exporter.prometheus import PrometheusMetricReader from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor -from opentelemetry.instrumentation.logging.handler import LoggingHandler from opentelemetry.instrumentation.requests import RequestsInstrumentor from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor -from opentelemetry.sdk._logs import LoggerProvider -from opentelemetry.sdk._logs.export import BatchLogRecordProcessor -from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import MetricReader, PeriodicExportingMetricReader from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.sdk.trace.sampling import ( - ParentBased, - Sampler, - SamplingResult, - TraceIdRatioBased, -) -from opentelemetry.trace import Link, SpanKind -from opentelemetry.trace.span import TraceState -from opentelemetry.util.types import Attributes +from opentelemetry.sdk.trace.sampling import ParentBased from sqlalchemy import event from sqlalchemy.engine import Engine from sqlalchemy.ext.asyncio import AsyncEngine @@ -39,11 +21,9 @@ from dstack._internal import settings as core_settings from dstack._internal.server import settings from dstack._internal.server.identity import SERVER_REPLICA_ID -from dstack._internal.server.utils.common import is_background_task_name +from dstack._internal.server.utils.common import PIPELINE_TASKS_PREFIX, SCHEDULED_TASKS_PREFIX from dstack._internal.server.utils.logging import AsyncioCancelledErrorFilter -from dstack._internal.utils.logging import get_logger - -logger = get_logger(__name__) +from dstack._internal.utils import otel as otel_utils def configure(app: FastAPI, engine: AsyncEngine) -> None: @@ -60,18 +40,20 @@ def configure(app: FastAPI, engine: AsyncEngine) -> None: def _configure_tracing() -> None: - provider = TracerProvider( - resource=_get_resource(), - sampler=ParentBased( - root=_RootSpanNameSampler( - default_rate=settings.OTEL_TRACES_SAMPLE_RATE, - background_rate=settings.OTEL_TRACES_BACKGROUND_SAMPLE_RATE, - ) - ), + otel_utils.configure_tracing( + _get_resource(), ParentBased(root=_build_root_sampler()), OTLPSpanExporter() + ) + + +def _build_root_sampler() -> otel_utils.RootSpanNameSampler: + background_rate = settings.OTEL_TRACES_BACKGROUND_SAMPLE_RATE + return otel_utils.RootSpanNameSampler( + default_rate=settings.OTEL_TRACES_SAMPLE_RATE, + rates_by_name_prefix={ + SCHEDULED_TASKS_PREFIX: background_rate, + PIPELINE_TASKS_PREFIX: background_rate, + }, ) - provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) - trace.set_tracer_provider(provider) - logger.info("OpenTelemetry tracing enabled") def _configure_metrics() -> None: @@ -88,17 +70,13 @@ def _configure_metrics() -> None: f"Unknown exporter {exporter!r} in DSTACK_OTEL_METRICS_EXPORTERS." " Supported exporters: prometheus, otlp" ) - provider = MeterProvider(resource=_get_resource(), metric_readers=readers) - metrics.set_meter_provider(provider) - logger.info("OpenTelemetry metrics enabled") + otel_utils.configure_metrics(_get_resource(), readers) def _configure_log_export() -> None: - logger_provider = LoggerProvider(resource=_get_resource()) - logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter())) - set_logger_provider(logger_provider) - logging.getLogger().addHandler(_build_log_handler(logger_provider)) - logger.info("OpenTelemetry log export enabled") + otel_utils.configure_log_export( + _get_resource(), OTLPLogExporter(), filters=[AsyncioCancelledErrorFilter()] + ) def _instrument(app: FastAPI, engine: AsyncEngine) -> None: @@ -111,7 +89,7 @@ def _instrument(app: FastAPI, engine: AsyncEngine) -> None: # attribute, so the SDK would accumulate state for an unbounded number of series. RequestsInstrumentor().instrument(excluded_urls=r"http\+unix://") if settings.OTEL_METRICS_ENABLED: - SystemMetricsInstrumentor(config=_PROCESS_METRICS_CONFIG).instrument() + SystemMetricsInstrumentor(config=otel_utils.PROCESS_METRICS_CONFIG).instrument() def _get_metrics_exporters() -> List[str]: @@ -124,22 +102,6 @@ def _get_metrics_exporters() -> List[str]: return ["otlp"] -# Process-level metrics only. Host-level (`system.*`) metrics are left -# to node exporters. `process.runtime.*` metrics are deprecated duplicates. -_PROCESS_METRICS_CONFIG = { - "process.cpu.time": ["user", "system"], - "process.cpu.utilization": ["user", "system"], - "process.memory.usage": None, - "process.memory.virtual": None, - "process.thread.count": None, - "process.open_file_descriptor.count": None, - "process.context_switches": ["involuntary", "voluntary"], - "cpython.gc.collections": None, - "cpython.gc.collected_objects": None, - "cpython.gc.uncollectable_objects": None, -} - - def _get_resource() -> Resource: return Resource.create( { @@ -153,24 +115,6 @@ def _get_resource() -> Resource: ) -def _build_log_handler(logger_provider: LoggerProvider) -> logging.Handler: - handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider) - handler.addFilter(AsyncioCancelledErrorFilter()) - handler.addFilter(_OTelSDKLogFilter()) - return handler - - -class _OTelSDKLogFilter(logging.Filter): - """Drops log records emitted by the OTel SDK itself. - - Exporting them would be a feedback loop: an OTLP export failure is logged, - the record is queued for the same failing exporter, and so on indefinitely. - """ - - def filter(self, record: logging.LogRecord) -> bool: - return not (record.name + ".").startswith("opentelemetry.") - - def _register_db_span_renaming(engine: Engine) -> None: """Renames DB spans from ` ` to ` `. @@ -205,34 +149,3 @@ def _get_db_span_name(statement: str) -> Optional[str]: if match is None: return operation return f"{operation} {match.group(1)}" - - -class _RootSpanNameSampler(Sampler): - """Samples background task traces and other (HTTP) traces at different rates.""" - - def __init__(self, default_rate: float, background_rate: float): - self._default_sampler = TraceIdRatioBased(default_rate) - self._background_sampler = TraceIdRatioBased(background_rate) - - def should_sample( - self, - parent_context: Optional[Context], - trace_id: int, - name: str, - kind: Optional[SpanKind] = None, - attributes: Attributes = None, - links: Optional[Sequence[Link]] = None, - trace_state: Optional[TraceState] = None, - ) -> SamplingResult: - sampler = self._default_sampler - if is_background_task_name(name): - sampler = self._background_sampler - return sampler.should_sample( - parent_context, trace_id, name, kind, attributes, links, trace_state - ) - - def get_description(self) -> str: - return ( - f"RootSpanNameSampler(default={self._default_sampler.rate}," - f"background={self._background_sampler.rate})" - ) diff --git a/src/dstack/_internal/utils/otel.py b/src/dstack/_internal/utils/otel.py new file mode 100644 index 0000000000..e575b0f6c9 --- /dev/null +++ b/src/dstack/_internal/utils/otel.py @@ -0,0 +1,123 @@ +import logging +from typing import Mapping, Optional, Sequence + +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.context import Context +from opentelemetry.instrumentation.logging.handler import LoggingHandler +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor, LogRecordExporter +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import MetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter +from opentelemetry.sdk.trace.sampling import Sampler, SamplingResult, TraceIdRatioBased +from opentelemetry.trace import Link, SpanKind +from opentelemetry.trace.span import TraceState +from opentelemetry.util.types import Attributes + +from dstack._internal.utils.logging import get_logger + +logger = get_logger(__name__) + +# Process-level metrics only. Host-level (`system.*`) metrics are left +# to node exporters. `process.runtime.*` metrics are deprecated duplicates. +PROCESS_METRICS_CONFIG = { + "process.cpu.time": ["user", "system"], + "process.cpu.utilization": ["user", "system"], + "process.memory.usage": None, + "process.memory.virtual": None, + "process.thread.count": None, + "process.open_file_descriptor.count": None, + "process.context_switches": ["involuntary", "voluntary"], + "cpython.gc.collections": None, + "cpython.gc.collected_objects": None, + "cpython.gc.uncollectable_objects": None, +} + + +def configure_tracing(resource: Resource, sampler: Sampler, exporter: SpanExporter) -> None: + provider = TracerProvider(resource=resource, sampler=sampler) + provider.add_span_processor(BatchSpanProcessor(exporter)) + trace.set_tracer_provider(provider) + logger.info("OpenTelemetry tracing enabled") + + +def configure_metrics(resource: Resource, readers: Sequence[MetricReader]) -> None: + provider = MeterProvider(resource=resource, metric_readers=readers) + metrics.set_meter_provider(provider) + logger.info("OpenTelemetry metrics enabled") + + +def configure_log_export( + resource: Resource, + exporter: LogRecordExporter, + level: int = logging.NOTSET, + filters: Sequence[logging.Filter] = (), +) -> None: + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter)) + set_logger_provider(logger_provider) + logging.getLogger().addHandler(build_log_handler(logger_provider, level, filters)) + logger.info("OpenTelemetry log export enabled") + + +def build_log_handler( + logger_provider: LoggerProvider, + level: int = logging.NOTSET, + filters: Sequence[logging.Filter] = (), +) -> logging.Handler: + handler = LoggingHandler(level=level, logger_provider=logger_provider) + for log_filter in filters: + handler.addFilter(log_filter) + handler.addFilter(_OTelSDKLogFilter()) + return handler + + +class _OTelSDKLogFilter(logging.Filter): + """Drops log records emitted by the OTel SDK itself. + + Exporting them would be a feedback loop: an OTLP export failure is logged, + the record is queued for the same failing exporter, and so on indefinitely. + """ + + def filter(self, record: logging.LogRecord) -> bool: + return not (record.name + ".").startswith("opentelemetry.") + + +class RootSpanNameSampler(Sampler): + """Samples traces at different rates depending on the root span name prefix.""" + + def __init__(self, default_rate: float, rates_by_name_prefix: Mapping[str, float]): + self._default_sampler = TraceIdRatioBased(default_rate) + self._samplers_by_name_prefix = { + prefix: TraceIdRatioBased(rate) for prefix, rate in rates_by_name_prefix.items() + } + + def should_sample( + self, + parent_context: Optional[Context], + trace_id: int, + name: str, + kind: Optional[SpanKind] = None, + attributes: Attributes = None, + links: Optional[Sequence[Link]] = None, + trace_state: Optional[TraceState] = None, + ) -> SamplingResult: + sampler = self._default_sampler + for prefix, prefix_sampler in self._samplers_by_name_prefix.items(): + if name.startswith(prefix): + sampler = prefix_sampler + break + return sampler.should_sample( + parent_context, trace_id, name, kind, attributes, links, trace_state + ) + + def get_description(self) -> str: + rates = [f"default={self._default_sampler.rate}"] + rates.extend( + f"{prefix!r}={sampler.rate}" + for prefix, sampler in self._samplers_by_name_prefix.items() + ) + return f"RootSpanNameSampler({','.join(rates)})" diff --git a/src/tests/_internal/proxy/gateway/routers/test_config.py b/src/tests/_internal/proxy/gateway/routers/test_config.py new file mode 100644 index 0000000000..e247d36b44 --- /dev/null +++ b/src/tests/_internal/proxy/gateway/routers/test_config.py @@ -0,0 +1,66 @@ +import signal +from unittest.mock import Mock + +import httpx +import pytest + +from dstack._internal.proxy.gateway.app import make_app +from dstack._internal.proxy.gateway.models import OTelSettings +from dstack._internal.proxy.gateway.repo.repo import GatewayProxyRepo +from dstack._internal.proxy.gateway.routers import config + + +def make_client(repo: GatewayProxyRepo) -> httpx.AsyncClient: + app = make_app(repo) + return httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test/") + + +@pytest.mark.asyncio +class TestUpdateGlobalConfig: + @pytest.mark.parametrize("restart_required", [False, True]) + async def test_saves_and_applies_otel_settings( + self, monkeypatch: pytest.MonkeyPatch, restart_required: bool + ) -> None: + apply_settings = Mock(return_value=restart_required) + kill = Mock() + monkeypatch.setattr(config.otel, "apply_settings", apply_settings) + monkeypatch.setattr(config.os, "kill", kill) + repo = GatewayProxyRepo() + otel_settings = { + "traces_enabled": True, + "logs_enabled": False, + "metrics_enabled": False, + "endpoint": "https://otel.example.com:4318", + "headers": {"api-key": "secret"}, + "traces_sample_rate": 1.0, + "traces_auth_sample_rate": 0.01, + "resource_attributes": {"service.instance.id": "replica-id"}, + } + + async with make_client(repo) as client: + resp = await client.post("/api/config", json={"otel_settings": otel_settings}) + + assert resp.status_code == 200 + expected = OTelSettings(**otel_settings) + assert (await repo.get_config()).otel_settings == expected + apply_settings.assert_called_once_with(expected) + if restart_required: + kill.assert_called_once() + assert kill.call_args.args[1] == signal.SIGTERM + else: + kill.assert_not_called() + + async def test_accepts_config_without_otel_settings( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # Older servers do not send otel_settings + apply_settings = Mock(return_value=False) + monkeypatch.setattr(config.otel, "apply_settings", apply_settings) + repo = GatewayProxyRepo() + + async with make_client(repo) as client: + resp = await client.post("/api/config", json={"acme_server": None}) + + assert resp.status_code == 200 + assert (await repo.get_config()).otel_settings is None + apply_settings.assert_called_once_with(None) diff --git a/src/tests/_internal/proxy/gateway/routers/test_registry.py b/src/tests/_internal/proxy/gateway/routers/test_registry.py index a9b265d9fc..7572aa13fb 100644 --- a/src/tests/_internal/proxy/gateway/routers/test_registry.py +++ b/src/tests/_internal/proxy/gateway/routers/test_registry.py @@ -156,6 +156,9 @@ async def test_register_with_auth(self, tmp_path: Path, system_mocks: Mocks) -> conf = (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").read_text() assert "auth_request /_dstack_auth;" in conf assert "proxy_pass http://localhost:8000/api/auth/test-proj;" in conf + assert 'proxy_set_header traceparent "";' in conf + assert 'proxy_set_header tracestate "";' in conf + assert 'proxy_set_header baggage "";' in conf async def test_register_same_name_error(self, tmp_path: Path, system_mocks: Mocks) -> None: client = make_client(tmp_path) @@ -618,6 +621,9 @@ async def test_register(self, tmp_path: Path, system_mocks: Mocks) -> None: assert resp.json() == {"status": "ok"} conf = (tmp_path / "sites-enabled" / "443-gateway.gtw.test.conf").read_text() assert "proxy_pass http://localhost:8000/api/models/test-proj/;" in conf + assert 'proxy_set_header traceparent "";' in conf + assert 'proxy_set_header tracestate "";' in conf + assert 'proxy_set_header baggage "";' in conf assert "listen 80;" in conf assert "listen 443" not in conf assert system_mocks.reload_nginx.call_count == 1 diff --git a/src/tests/_internal/proxy/gateway/test_app.py b/src/tests/_internal/proxy/gateway/test_app.py index 335be84fa1..56a13be9aa 100644 --- a/src/tests/_internal/proxy/gateway/test_app.py +++ b/src/tests/_internal/proxy/gateway/test_app.py @@ -1,12 +1,14 @@ from pathlib import Path +from unittest.mock import Mock import pytest +from dstack._internal.proxy.gateway import app as app_module from dstack._internal.proxy.gateway.app import lifespan, make_app -from dstack._internal.proxy.gateway.models import ModelEntrypoint +from dstack._internal.proxy.gateway.models import GlobalProxyConfig, ModelEntrypoint from dstack._internal.proxy.gateway.repo.repo import GatewayProxyRepo from dstack._internal.proxy.gateway.services.nginx import Nginx -from dstack._internal.proxy.gateway.testing.common import Mocks +from dstack._internal.proxy.gateway.testing.common import Mocks, make_otel_settings from dstack._internal.proxy.lib.testing.common import make_project, make_service @@ -32,3 +34,19 @@ async def test_lifespan(tmp_path: Path, system_mocks: Mocks) -> None: assert system_mocks.open_conn.call_count == 1 assert system_mocks.close_conn.call_count == 0 assert system_mocks.close_conn.call_count == 1 + + +@pytest.mark.asyncio +async def test_lifespan_applies_saved_otel_settings( + tmp_path: Path, system_mocks: Mocks, monkeypatch: pytest.MonkeyPatch +) -> None: + apply_settings = Mock(return_value=False) + monkeypatch.setattr(app_module.otel, "apply_settings", apply_settings) + repo = GatewayProxyRepo() + otel_settings = make_otel_settings(traces_enabled=True) + await repo.set_config(GlobalProxyConfig(otel_settings=otel_settings)) + (tmp_path / "nginx" / "sites-enabled").mkdir(parents=True) + app = make_app(repo=repo, nginx=Nginx(nginx_dir=tmp_path / "nginx")) + + async with lifespan(app): + apply_settings.assert_called_once_with(otel_settings) diff --git a/src/tests/_internal/proxy/gateway/test_otel.py b/src/tests/_internal/proxy/gateway/test_otel.py new file mode 100644 index 0000000000..6d667916ea --- /dev/null +++ b/src/tests/_internal/proxy/gateway/test_otel.py @@ -0,0 +1,100 @@ +from typing import Optional +from unittest.mock import Mock + +import pytest +from opentelemetry.sdk.trace.sampling import Decision + +from dstack._internal.proxy.gateway import otel +from dstack._internal.proxy.gateway.models import OTelSettings +from dstack._internal.proxy.gateway.testing.common import make_otel_settings + + +class TestApplySettings: + @pytest.fixture + def configure(self, monkeypatch: pytest.MonkeyPatch) -> Mock: + # Stubbed, as the global providers can only be set once per process + configure = Mock() + monkeypatch.setattr(otel, "_configure", configure) + monkeypatch.setattr(otel, "_applied_settings", None) + return configure + + def test_does_not_raise_or_retry_if_configuration_fails(self, configure: Mock): + configure.side_effect = ValueError("bad settings") + settings = make_otel_settings(traces_enabled=True) + + assert otel.apply_settings(settings) is False + assert otel.apply_settings(settings) is False + assert otel.apply_settings(make_otel_settings(logs_enabled=True)) is True + + configure.assert_called_once_with(settings) + + def test_keeps_disabled_without_settings(self, configure: Mock): + assert otel.apply_settings(None) is False + assert otel.apply_settings(make_otel_settings()) is False + configure.assert_not_called() + + def test_applies_first_settings_immediately(self, configure: Mock): + settings = make_otel_settings(traces_enabled=True) + + assert otel.apply_settings(settings) is False + assert otel.apply_settings(settings) is False + + configure.assert_called_once_with(settings) + + @pytest.mark.parametrize( + "new_settings", + [ + pytest.param(make_otel_settings(traces_enabled=True, logs_enabled=True), id="changed"), + pytest.param(None, id="disabled"), + ], + ) + def test_requires_restart_to_change_applied_settings( + self, configure: Mock, new_settings: Optional[OTelSettings] + ): + otel.apply_settings(make_otel_settings(traces_enabled=True)) + + assert otel.apply_settings(new_settings) is True + + configure.assert_called_once() + + +class TestGetSignalEndpoint: + @pytest.mark.parametrize( + ("endpoint", "expected"), + [ + ("https://otel.example.com:4318", "https://otel.example.com:4318/v1/traces"), + ("https://example.com/otlp/", "https://example.com/otlp/v1/traces"), + ], + ) + def test_appends_signal_path(self, endpoint: str, expected: str): + settings = make_otel_settings(endpoint=endpoint) + assert otel._get_signal_endpoint(settings, "traces") == expected + + +class TestBuildRootSampler: + def test_samples_auth_subrequests_at_auth_rate(self): + sampler = otel._build_root_sampler( + make_otel_settings(traces_sample_rate=1.0, traces_auth_sample_rate=0.0) + ) + registry_result = sampler.should_sample( + None, trace_id=123, name="POST /api/registry/{project_name}/services/register" + ) + auth_result = sampler.should_sample( + None, trace_id=123, name="GET /api/auth/{project_name}" + ) + assert registry_result.decision == Decision.RECORD_AND_SAMPLE + assert auth_result.decision == Decision.DROP + + +class TestGetResource: + def test_adds_service_to_replica_attributes(self): + settings = make_otel_settings( + resource_attributes={"service.instance.id": "replica-id", "dstack.gateway.name": "gw"} + ) + + attributes = otel._get_resource(settings).attributes + + assert attributes["service.name"] == "dstack-gateway" + assert "service.version" in attributes + assert attributes["service.instance.id"] == "replica-id" + assert attributes["dstack.gateway.name"] == "gw" diff --git a/src/tests/_internal/server/routers/test_gateways.py b/src/tests/_internal/server/routers/test_gateways.py index d5980f5f96..37872ea2e5 100644 --- a/src/tests/_internal/server/routers/test_gateways.py +++ b/src/tests/_internal/server/routers/test_gateways.py @@ -6,9 +6,10 @@ from sqlalchemy.ext.asyncio import AsyncSession from dstack._internal.core.models.backends.base import BackendType -from dstack._internal.core.models.common import ApplyAction +from dstack._internal.core.models.common import ApplyAction, EntityReference from dstack._internal.core.models.gateways import GatewayStatus from dstack._internal.core.models.users import GlobalRole, ProjectRole +from dstack._internal.server.services.gateways import get_project_gateway_model_by_reference from dstack._internal.server.services.projects import add_project_member from dstack._internal.server.testing.common import ( clear_events, @@ -17,6 +18,7 @@ create_gateway, create_gateway_replica, create_project, + create_secret, create_user, get_auth_headers, list_events, @@ -112,6 +114,7 @@ async def test_list( "load_balancer": None, "certificate": {"type": "lets-encrypt"}, "tags": None, + "otel": None, "replicas": None, }, } @@ -199,6 +202,7 @@ async def test_get( "load_balancer": None, "certificate": {"type": "lets-encrypt"}, "tags": None, + "otel": None, "replicas": None, }, } @@ -541,6 +545,7 @@ async def test_create_gateway(self, test_db, session: AsyncSession, client: Asyn "load_balancer": None, "certificate": {"type": "lets-encrypt"}, "tags": None, + "otel": None, "replicas": None, }, } @@ -632,6 +637,7 @@ async def test_create_gateway_without_name( "load_balancer": None, "certificate": {"type": "lets-encrypt"}, "tags": None, + "otel": None, "replicas": None, }, } @@ -974,6 +980,7 @@ async def test_set_default_gateway( "load_balancer": None, "certificate": {"type": "lets-encrypt"}, "tags": None, + "otel": None, "replicas": None, }, } @@ -1371,6 +1378,7 @@ async def test_set_wildcard_domain( "load_balancer": None, "certificate": {"type": "lets-encrypt"}, "tags": None, + "otel": None, "replicas": None, }, } @@ -3037,3 +3045,130 @@ async def test_force_apply_default_false_unsets_default( await session.refresh(project) assert project.default_gateway_id is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) +class TestGatewayOTel: + async def test_creates_gateway_with_secret_references( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.ADMIN + ) + await create_secret(session=session, project=project, name="token", value="s3cret") + await create_backend(session, project.id, backend_type=BackendType.AWS) + otel = { + "endpoint": "https://otlp.example.com", + "headers": {"Authorization": "Bearer ${{ secrets.token }}"}, + "traces": True, + } + + response = await client.post( + f"/api/project/{project.name}/gateways/apply", + json={ + "plan": { + "spec": { + "configuration": { + "type": "gateway", + "name": "my-gateway", + "backend": "aws", + "region": "us-east-1", + "otel": otel, + } + }, + "current_resource": None, + }, + "force": False, + }, + headers=get_auth_headers(user.token), + ) + + assert response.status_code == 200 + assert response.json()["configuration"]["otel"]["headers"] == otel["headers"] + assert "s3cret" not in response.text + gateway = await get_project_gateway_model_by_reference( + session, project, EntityReference(name="my-gateway", project=None) + ) + assert gateway is not None + assert "secrets.token" in (gateway.configuration or "") + assert "s3cret" not in (gateway.configuration or "") + events = await list_events(session) + assert all("s3cret" not in e.message for e in events) + + async def test_cannot_update_otel_in_place( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.ADMIN + ) + backend = await create_backend(session, project.id, backend_type=BackendType.AWS) + gateway = await create_gateway( + session=session, + project_id=project.id, + backend_id=backend.id, + name="my-gateway", + region="us-east-1", + ) + await create_gateway_replica(session=session, backend=backend, gateway_id=gateway.id) + get_response = await client.post( + f"/api/project/{project.name}/gateways/get", + json={"name": "my-gateway"}, + headers=get_auth_headers(user.token), + ) + + response = await client.post( + f"/api/project/{project.name}/gateways/apply", + json={ + "plan": { + "spec": { + "configuration": { + "type": "gateway", + "name": "my-gateway", + "backend": "aws", + "region": "us-east-1", + "otel": {"endpoint": "https://otlp.example.com", "traces": True}, + } + }, + "current_resource": get_response.json(), + }, + "force": False, + }, + headers=get_auth_headers(user.token), + ) + + assert response.status_code == 400 + assert "cannot be updated in-place" in response.json()["detail"][0]["msg"] + + async def test_rejects_missing_secret( + self, test_db, session: AsyncSession, client: AsyncClient + ): + user = await create_user(session, global_role=GlobalRole.USER) + project = await create_project(session) + await add_project_member( + session=session, project=project, user=user, project_role=ProjectRole.ADMIN + ) + await create_backend(session, project.id, backend_type=BackendType.AWS) + + response = await client.post( + f"/api/project/{project.name}/gateways/get_plan", + json={ + "spec": { + "configuration": { + "type": "gateway", + "name": "my-gateway", + "backend": "aws", + "region": "us-east-1", + "otel": {"endpoint": "${{ secrets.missing }}", "traces": True}, + } + } + }, + headers=get_auth_headers(user.token), + ) + + assert response.status_code == 400 + assert "secrets.missing" in response.text diff --git a/src/tests/_internal/server/services/gateways/test_gateways.py b/src/tests/_internal/server/services/gateways/test_gateways.py index 78cecf9036..6284d69bff 100644 --- a/src/tests/_internal/server/services/gateways/test_gateways.py +++ b/src/tests/_internal/server/services/gateways/test_gateways.py @@ -6,11 +6,15 @@ from dstack._internal import settings as core_settings from dstack._internal.core.backends.base.compute import get_dstack_gateway_install_command +from dstack._internal.core.errors import ServerClientError +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.core.models.gateways import GatewayConfiguration, GatewayOTelConfig from dstack._internal.proxy.gateway.const import SERVICE_SCALING_WINDOWS from dstack._internal.proxy.gateway.schemas.stats import Stat from dstack._internal.server.services import gateways from dstack._internal.server.services.gateways import ( _merge_per_window_stats, + _validate_gateway_configuration, get_gateway_replica_models, ) from dstack._internal.server.testing.common import ( @@ -18,6 +22,8 @@ create_gateway, create_gateway_replica, create_project, + create_secret, + create_user, ) @@ -147,3 +153,91 @@ async def test_runs_install_command_on_existing_gateway_replicas( ) configure.assert_awaited_once() assert replica.app_updated_at == (now if updated else previously_updated_at) + + async def test_configures_replicas_with_otel_settings( + self, session: AsyncSession, monkeypatch: pytest.MonkeyPatch + ): + connection = Mock(ip_address="1.1.1.1") + monkeypatch.setattr(gateways.settings, "SKIP_GATEWAY_UPDATE", True) + monkeypatch.setattr( + gateways.gateway_connections_pool, "get_or_add", AsyncMock(return_value=connection) + ) + monkeypatch.setattr( + gateways.gateway_connections_pool, "all", AsyncMock(return_value=[connection]) + ) + configure = AsyncMock() + monkeypatch.setattr(gateways, "configure_gateway_replica", configure) + project = await create_project(session=session, name="main") + backend = await create_backend(session=session, project_id=project.id) + gateway = await create_gateway( + session=session, project_id=project.id, backend_id=backend.id, name="my-gateway" + ) + configuration = gateways.get_gateway_configuration(gateway) + configuration.otel = GatewayOTelConfig( + endpoint="https://otlp", + headers={"Authorization": "Bearer ${{ secrets.token }}"}, + traces=True, + ) + gateway.configuration = configuration.model_dump_json() + await create_secret(session=session, project=project, name="token", value="s3cret") + replica = await create_gateway_replica( + session=session, backend=backend, gateway_id=gateway.id, replica_num=1 + ) + await session.commit() + # Relationships must be loaded by init_gateways, not come from the identity map + session.expunge_all() + + await gateways.init_gateways(session) + + configure.assert_awaited_once() + assert configure.await_args is not None + assert configure.await_args.args[0] is connection + otel_settings = configure.await_args.args[1] + assert otel_settings.traces_enabled + assert otel_settings.headers == {"Authorization": "Bearer s3cret"} + attributes = otel_settings.resource_attributes + assert attributes["service.instance.id"] == str(replica.id) + assert attributes["dstack.project.name"] == "main" + assert attributes["dstack.gateway.name"] == "my-gateway" + assert attributes["dstack.backend"] == backend.type.value + + +@pytest.mark.asyncio +@pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) +class TestValidateGatewayConfiguration: + @pytest.mark.parametrize( + ("endpoint", "error"), + [ + pytest.param("${{ secrets.missing }}", "secrets.missing", id="missing-secret"), + pytest.param("${{ secrets.token }}", "endpoint must be", id="invalid-secret-value"), + ], + ) + async def test_rejects_invalid_otel_without_revealing_secrets( + self, test_db, session: AsyncSession, endpoint: str, error: str + ): + project = await create_project(session=session) + await create_secret(session=session, project=project, name="token", value="s3cret") + configuration = _make_otel_configuration( + GatewayOTelConfig(endpoint=endpoint, headers={"A": "${{ secrets.token }}"}) + ) + + with pytest.raises(ServerClientError, match=error) as exc_info: + await _validate_gateway_configuration(session, project, configuration) + + assert "s3cret" not in str(exc_info.value) + + async def test_does_not_use_other_projects_secrets(self, test_db, session: AsyncSession): + user = await create_user(session=session) + project = await create_project(session=session, owner=user, name="owner") + other_project = await create_project(session=session, owner=user, name="other") + await create_secret(session=session, project=other_project, name="token", value="x") + configuration = _make_otel_configuration( + GatewayOTelConfig(endpoint="https://otlp", headers={"A": "${{ secrets.token }}"}) + ) + + with pytest.raises(ServerClientError, match="secrets.token"): + await _validate_gateway_configuration(session, project, configuration) + + +def _make_otel_configuration(otel: GatewayOTelConfig) -> GatewayConfiguration: + return GatewayConfiguration(backend=BackendType.AWS, region="us-east-1", otel=otel) diff --git a/src/tests/_internal/server/services/gateways/test_telemetry.py b/src/tests/_internal/server/services/gateways/test_telemetry.py new file mode 100644 index 0000000000..760d69a4b3 --- /dev/null +++ b/src/tests/_internal/server/services/gateways/test_telemetry.py @@ -0,0 +1,175 @@ +import logging +import uuid +from typing import Optional + +import pytest + +from dstack._internal.core.models.backends.base import BackendType +from dstack._internal.core.models.gateways import GatewayConfiguration, GatewayOTelConfig +from dstack._internal.proxy.gateway.models import OTelSettings +from dstack._internal.server import settings +from dstack._internal.server.models import ( + BackendModel, + GatewayModel, + GatewayReplicaModel, + ProjectModel, +) +from dstack._internal.server.services.gateways.telemetry import ( + InvalidGatewayOTelConfigError, + get_gateway_replica_otel_settings, + resolve_gateway_otel_config, +) + + +class TestGetGatewayReplicaOTelSettings: + @pytest.mark.parametrize( + "otel", + [ + pytest.param(None, id="no-otel"), + pytest.param(GatewayOTelConfig(endpoint="https://otlp"), id="no-signals"), + ], + ) + def test_none_if_disabled(self, otel: Optional[GatewayOTelConfig]): + gateway_model, replica_model = _make_models() + + assert ( + get_gateway_replica_otel_settings( + gateway_model, _make_configuration(otel), replica_model, secrets={} + ) + is None + ) + + def test_resolves_secrets_and_identifies_replica(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(settings, "SERVER_ENVIRONMENT", "production") + gateway_model, replica_model = _make_models() + otel = GatewayOTelConfig( + endpoint="${{ secrets.otlp_endpoint }}", + headers={"Authorization": "Bearer ${{ secrets.otlp_token }}", "X-Team": "ml"}, + traces=True, + metrics=True, + traces_sample_rate=0.5, + ) + + otel_settings = get_gateway_replica_otel_settings( + gateway_model, + _make_configuration(otel), + replica_model, + secrets={"otlp_endpoint": "https://otlp.example.com", "otlp_token": "s3cret"}, + ) + + assert otel_settings == OTelSettings( + traces_enabled=True, + logs_enabled=False, + metrics_enabled=True, + endpoint="https://otlp.example.com", + headers={"Authorization": "Bearer s3cret", "X-Team": "ml"}, + traces_sample_rate=0.5, + traces_auth_sample_rate=0.01, + resource_attributes={ + "service.instance.id": str(replica_model.id), + "dstack.gateway.replica.num": 1, + "deployment.environment.name": "production", + "dstack.project.name": "main", + "dstack.gateway.name": "my-gateway", + "dstack.backend": "aws", + "host.name": "203.0.113.5", + "cloud.region": "us-east-1", + }, + ) + + @pytest.mark.parametrize( + ("endpoint", "headers"), + [ + pytest.param( + "https://otlp", {"Authorization": "${{ secrets.deleted }}"}, id="missing" + ), + pytest.param("${{ secrets.value }}", {}, id="invalid-endpoint"), + pytest.param( + "https://otlp", {"Authorization": "${{ secrets.value }}"}, id="invalid-header" + ), + ], + ) + def test_none_if_secrets_unusable( + self, caplog: pytest.LogCaptureFixture, endpoint: str, headers: dict[str, str] + ): + gateway_model, replica_model = _make_models() + otel = GatewayOTelConfig(endpoint=endpoint, headers=headers, traces=True) + + with caplog.at_level(logging.WARNING): + otel_settings = get_gateway_replica_otel_settings( + gateway_model, + _make_configuration(otel), + replica_model, + secrets={"value": "s3cret\n"}, + ) + + assert otel_settings is None + assert "gateway-replica(" in caplog.text + assert "cannot enable OpenTelemetry" in caplog.text + assert "s3cret" not in caplog.text + + +class TestResolveGatewayOTelConfig: + @pytest.mark.parametrize( + "endpoint", + ["https://otlp.example.com:4318", "http://10.0.0.1/otlp"], + ) + def test_accepts_valid_endpoints(self, endpoint: str): + assert resolve_gateway_otel_config(GatewayOTelConfig(endpoint=endpoint), {}) == ( + endpoint, + {}, + ) + + @pytest.mark.parametrize( + "endpoint", + [ + "otlp.example.com", + "ftp://otlp.example.com", + "https://", + "https://otlp.example.com:99999", + "https://otlp.example.com\n", + " https://otlp.example.com", + "https://otlp.example.com:4318?tenant=x", + "https://otlp.example.com/otlp?", + "https://otlp.example.com#x", + ], + ) + def test_rejects_invalid_endpoints(self, endpoint: str): + with pytest.raises(InvalidGatewayOTelConfigError, match="endpoint"): + resolve_gateway_otel_config(GatewayOTelConfig(endpoint=endpoint), {}) + + @pytest.mark.parametrize( + ("name", "value"), + [ + pytest.param("Authorization", "Bearer a\nb", id="line-break"), + pytest.param("Authorization", " Bearer a", id="leading-whitespace"), + pytest.param("Authorization", "Bearer a ", id="trailing-whitespace"), + pytest.param("Authorization", "Bearer \u2192", id="non-latin-1"), + pytest.param("Bad Name", "a", id="invalid-name"), + ], + ) + def test_rejects_invalid_headers_without_revealing_values(self, name: str, value: str): + otel = GatewayOTelConfig(endpoint="https://otlp", headers={name: value}) + with pytest.raises(InvalidGatewayOTelConfigError) as exc_info: + resolve_gateway_otel_config(otel, {}) + assert "Bearer" not in str(exc_info.value) + + +def _make_configuration(otel: Optional[GatewayOTelConfig]) -> GatewayConfiguration: + return GatewayConfiguration(backend=BackendType.AWS, region="us-east-1", otel=otel) + + +def _make_models() -> tuple[GatewayModel, GatewayReplicaModel]: + gateway_model = GatewayModel( + name="my-gateway", + project=ProjectModel(name="main"), + backend=BackendModel(type=BackendType.AWS), + ) + replica_model = GatewayReplicaModel( + id=uuid.uuid4(), + name="my-gateway-1", + replica_num=1, + ip_address="203.0.113.5", + region="us-east-1", + ) + return gateway_model, replica_model diff --git a/src/tests/_internal/server/utils/test_tracing.py b/src/tests/_internal/server/utils/test_tracing.py index 6e2bdd2b53..00a4b13fc5 100644 --- a/src/tests/_internal/server/utils/test_tracing.py +++ b/src/tests/_internal/server/utils/test_tracing.py @@ -1,6 +1,3 @@ -import asyncio -import logging - import pytest from opentelemetry import trace from opentelemetry.sdk.trace import TracerProvider @@ -10,10 +7,9 @@ from dstack._internal.server.utils import tracing from dstack._internal.server.utils.otel.utils import ( - _build_log_handler, + _build_root_sampler, _get_db_span_name, _get_metrics_exporters, - _RootSpanNameSampler, ) _span_exporter = InMemorySpanExporter() @@ -99,55 +95,6 @@ async def failing_task(): assert counts == {"success": 2, "error": 1} -class TestBuildLogHandler: - def test_exports_records_with_trace_context(self, span_exporter: InMemorySpanExporter): - from opentelemetry.sdk._logs import LoggerProvider - from opentelemetry.sdk._logs.export import ( - InMemoryLogRecordExporter, - SimpleLogRecordProcessor, - ) - - log_exporter = InMemoryLogRecordExporter() - logger_provider = LoggerProvider() - logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) - logger = logging.getLogger("test_otel_logs") - logger.addHandler(_build_log_handler(logger_provider)) - - tracer = trace.get_tracer(__name__) - with tracer.start_as_current_span("op") as span: - logger.warning("something happened") - try: - logger.error("cancelled", exc_info=asyncio.CancelledError()) - except Exception: - pass - - records = [d.log_record for d in log_exporter.get_finished_logs()] - assert len(records) == 1 - assert records[0].body == "something happened" - assert records[0].severity_text == "WARN" - assert records[0].trace_id == span.get_span_context().trace_id - - def test_does_not_export_otel_sdk_records(self): - from opentelemetry.sdk._logs import LoggerProvider - from opentelemetry.sdk._logs.export import ( - InMemoryLogRecordExporter, - SimpleLogRecordProcessor, - ) - - log_exporter = InMemoryLogRecordExporter() - logger_provider = LoggerProvider() - logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) - handler = _build_log_handler(logger_provider) - - # An OTLP export failure logged by the SDK must not be re-exported — - # that would be a feedback loop through the same failing exporter - sdk_logger = logging.getLogger("opentelemetry.exporter.otlp.proto.http._log_exporter") - sdk_logger.addHandler(handler) - sdk_logger.error("Failed to export logs batch") - - assert log_exporter.get_finished_logs() == () - - class TestGetMetricsExporters: @pytest.mark.parametrize( ("exporters", "expected"), @@ -182,12 +129,20 @@ def test_returns_expected(self, statement, expected): assert _get_db_span_name(statement) == expected -class TestRootSpanNameSampler: - def test_samples_at_rate_by_name(self): - sampler = _RootSpanNameSampler(default_rate=1.0, background_rate=0.0) +class TestBuildRootSampler: + def test_samples_background_tasks_at_background_rate(self, monkeypatch): + from dstack._internal.server import settings + + monkeypatch.setattr(settings, "OTEL_TRACES_SAMPLE_RATE", 1.0) + monkeypatch.setattr(settings, "OTEL_TRACES_BACKGROUND_SAMPLE_RATE", 0.0) + sampler = _build_root_sampler() http_result = sampler.should_sample(None, trace_id=123, name="GET /api/project") - background_result = sampler.should_sample( + pipeline_result = sampler.should_sample( None, trace_id=123, name="pipeline_tasks.VolumeWorker.process" ) + scheduled_result = sampler.should_sample( + None, trace_id=123, name="scheduled_tasks.process_fleets" + ) assert http_result.decision == Decision.RECORD_AND_SAMPLE - assert background_result.decision == Decision.DROP + assert pipeline_result.decision == Decision.DROP + assert scheduled_result.decision == Decision.DROP diff --git a/src/tests/_internal/utils/test_otel.py b/src/tests/_internal/utils/test_otel.py new file mode 100644 index 0000000000..c3c892d85d --- /dev/null +++ b/src/tests/_internal/utils/test_otel.py @@ -0,0 +1,60 @@ +import asyncio +import logging + +import pytest +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import InMemoryLogRecordExporter, SimpleLogRecordProcessor +from opentelemetry.sdk.trace import TracerProvider + +from dstack._internal.server.utils.logging import AsyncioCancelledErrorFilter +from dstack._internal.utils.otel import build_log_handler + + +@pytest.fixture +def log_exporter() -> InMemoryLogRecordExporter: + return InMemoryLogRecordExporter() + + +@pytest.fixture +def logger_provider(log_exporter: InMemoryLogRecordExporter) -> LoggerProvider: + logger_provider = LoggerProvider() + logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) + return logger_provider + + +class TestBuildLogHandler: + def test_exports_records_with_trace_context( + self, log_exporter: InMemoryLogRecordExporter, logger_provider: LoggerProvider + ): + logger = logging.getLogger("test_otel_logs") + logger.addHandler( + build_log_handler(logger_provider, filters=[AsyncioCancelledErrorFilter()]) + ) + + # A local provider, as the global one can only be set once per process + tracer = TracerProvider().get_tracer(__name__) + with tracer.start_as_current_span("op") as span: + logger.warning("something happened") + try: + logger.error("cancelled", exc_info=asyncio.CancelledError()) + except Exception: + pass + + records = [d.log_record for d in log_exporter.get_finished_logs()] + assert len(records) == 1 + assert records[0].body == "something happened" + assert records[0].severity_text == "WARN" + assert records[0].trace_id == span.get_span_context().trace_id + + def test_does_not_export_otel_sdk_records( + self, log_exporter: InMemoryLogRecordExporter, logger_provider: LoggerProvider + ): + handler = build_log_handler(logger_provider) + + # An OTLP export failure logged by the SDK must not be re-exported — + # that would be a feedback loop through the same failing exporter + sdk_logger = logging.getLogger("opentelemetry.exporter.otlp.proto.http._log_exporter") + sdk_logger.addHandler(handler) + sdk_logger.error("Failed to export logs batch") + + assert log_exporter.get_finished_logs() == ()