Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions mkdocs/docs/concepts/gateways.md
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,40 @@ $ dstack gateway list

Replicated gateways do not support automatic certificate issuance via Let's Encrypt (`certificate: { type: lets-encrypt }`). For TLS termination, use an external load balancer or one of the other [certificate types](#certificate).

### OpenTelemetry

The gateway can export its traces, logs, and metrics via OTLP over HTTP.

<div editor-title="gateway.dstack.yml">

```yaml
type: gateway
name: example-gateway

backend: aws
region: eu-west-1

domain: example.com

otel:
endpoint: https://otlp.example.com:4318
headers:
Authorization: Bearer ${{ secrets.otlp_token }}
traces: true
logs: true
metrics: true
```

</div>

!!! warning "Use secrets for credentials"
The `otel` configuration is visible to everyone who can view the gateway, including members of
projects the gateway is [exported](exports.md) to. Store credentials, such as authorization
headers, in [secrets](secrets.md) and reference them as `${{ secrets.<name> }}`. Only the
references are visible, not the secret values.

Telemetry is reported with `service.name` set to `dstack-gateway` and resource attributes identifying the gateway and its replica.

!!! info "Reference"
For all gateway configuration options, refer to the [reference](../reference/dstack.yml/gateway.md).

Expand Down
2 changes: 1 addition & 1 deletion mkdocs/docs/concepts/secrets.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ OK

## Use secrets

You can use the `${{ secrets.<secret_name> }}` syntax to reference secrets in run configurations. Currently, secrets interpolation is supported in `env` and `registry_auth` properties.
You can use the `${{ secrets.<secret_name> }}` syntax to reference secrets in run configurations. Currently, secrets interpolation is supported in `env` and `registry_auth` properties. Gateway configurations also support secrets in [`otel`](gateways.md#opentelemetry) `endpoint` and `headers`.

### `env`

Expand Down
4 changes: 4 additions & 0 deletions mkdocs/docs/guides/server-deployment.md
Original file line number Diff line number Diff line change
Expand Up @@ -459,6 +459,10 @@ $ DSTACK_OTEL_TRACES_ENABLED=1 \
`rate()` queries may be empty. Set e.g. `OTEL_METRIC_EXPORT_INTERVAL=30000`,
and make sure the datasource's `timeInterval` reflects the cadence you choose.

#### Gateways

Gateways can export their own telemetry. It is configured per gateway, see [gateway OpenTelemetry](../concepts/gateways.md#opentelemetry).

## Encryption

By default, `dstack` stores data in plaintext. To enforce encryption, you
Expand Down
6 changes: 6 additions & 0 deletions mkdocs/docs/reference/dstack.yml/gateway.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,9 @@ Set to `null` to disable certificates (e.g. for [private gateways](../../concept
show_root_heading: false
type:
required: true

### `otel`

#SCHEMA# dstack._internal.core.models.gateways.GatewayOTelConfig
overrides:
show_root_heading: false
11 changes: 9 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,13 @@ gateway = [
# Gateway authorization needs the pool cancellation and recovery fixes in httpx2 (#4338).
"httpx2>=2.13.1",
"jinja2",
# Always installed so that OpenTelemetry can be enabled without reinstalling the gateway.
"opentelemetry-sdk>=1.39.0",
"opentelemetry-exporter-otlp-proto-http>=1.39.0",
"opentelemetry-instrumentation-fastapi>=0.51b0",
"opentelemetry-instrumentation-httpx>=0.65b0",
"opentelemetry-instrumentation-system-metrics>=0.51b0",
"opentelemetry-instrumentation-logging>=0.61b0",
]
server = [
"fastapi",
Expand Down Expand Up @@ -311,8 +318,8 @@ fluentbit = [
"dstack[server]",
]
otel = [
"opentelemetry-sdk>=1.30.0",
"opentelemetry-exporter-otlp-proto-http>=1.30.0",
"opentelemetry-sdk>=1.39.0",
"opentelemetry-exporter-otlp-proto-http>=1.39.0",
"opentelemetry-exporter-prometheus>=0.51b0",
"opentelemetry-instrumentation-fastapi>=0.51b0",
"opentelemetry-instrumentation-sqlalchemy>=0.51b0",
Expand Down
2 changes: 2 additions & 0 deletions src/dstack/_internal/core/compatibility/gateways.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,5 +71,7 @@ def _get_gateway_configuration_excludes(
configuration_excludes["default"] = True
if configuration.load_balancer is None:
configuration_excludes["load_balancer"] = True
if configuration.otel is None:
configuration_excludes["otel"] = True

return configuration_excludes
51 changes: 51 additions & 0 deletions src/dstack/_internal/core/models/gateways.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,53 @@ class ALBGatewayLoadBalancer(CoreModel):
AnyGatewayLoadBalancer = Union[ALBGatewayLoadBalancer]


class GatewayOTelConfig(CoreModel):
endpoint: Annotated[
str,
Field(
description=(
"The OTLP over HTTP endpoint, e.g. `https://otlp.example.com:4318`."
" Signal paths such as `/v1/traces` are appended to it."
" Can reference secrets, e.g. `${{ secrets.otlp_endpoint }}`"
),
),
]
headers: Annotated[
Optional[Dict[str, str]],
Field(
description=(
"Headers sent with OTLP requests, e.g. for authorization."
" Values can reference secrets, e.g. `Bearer ${{ secrets.otlp_token }}`"
),
),
] = None
traces: Annotated[bool, Field(description="Whether to export traces")] = False
logs: Annotated[bool, Field(description="Whether to export logs")] = False
metrics: Annotated[bool, Field(description="Whether to export metrics")] = False
traces_sample_rate: Annotated[
float,
Field(
description=(
"The head sampling rate for traces."
" The default assumes sampling is done downstream, e.g. in an OTel Collector"
),
ge=0,
le=1,
),
] = 1.0
traces_auth_sample_rate: Annotated[
float,
Field(
description=(
"The head sampling rate for traces of authorization subrequests,"
" which Nginx makes for every request to services with authorization enabled"
),
ge=0,
le=1,
),
] = 0.01


class GatewayConfiguration(CoreModel):
type: Literal["gateway"] = "gateway"
name: Annotated[Optional[str], Field(description="The gateway name")] = None
Expand Down Expand Up @@ -166,6 +213,10 @@ class GatewayConfiguration(CoreModel):
)
),
] = None
otel: Annotated[
Optional[GatewayOTelConfig],
Field(description=("The OpenTelemetry export configuration of the gateway app")),
] = None

_validate_tags = field_validator("tags", mode="before")(tags_validator)

Expand Down
2 changes: 2 additions & 0 deletions src/dstack/_internal/proxy/gateway/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware

from dstack._internal.proxy.gateway import otel
from dstack._internal.proxy.gateway.auth import GatewayProxyAuthProvider
from dstack._internal.proxy.gateway.const import (
DSTACK_DIR_ON_GATEWAY,
Expand Down Expand Up @@ -43,6 +44,7 @@ async def lifespan(app: FastAPI):
repo = await get_gateway_proxy_repo(await injector.get_repo().__anext__())
nginx = injector.get_nginx()
service_conn_pool = await injector.get_service_connection_pool()
otel.apply_settings((await repo.get_config()).otel_settings)
await run_async(nginx.write_global_conf)
await apply_all(repo, nginx, service_conn_pool)

Expand Down
3 changes: 3 additions & 0 deletions src/dstack/_internal/proxy/gateway/main.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import logging

from dstack._internal.proxy.gateway import otel
from dstack._internal.proxy.gateway.app import make_app


Expand All @@ -15,3 +16,5 @@ def configure_logging(level: int = logging.INFO):

configure_logging(logging.DEBUG)
app = make_app()
# Not in make_app, since global instrumentation would leak between apps created in tests
otel.instrument(app)
18 changes: 17 additions & 1 deletion src/dstack/_internal/proxy/gateway/models.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""Things stored in GatewayProxyRepo in addition to those from BaseProxyRepo."""

from typing import Optional
from typing import Optional, Union

from pydantic import AnyHttpUrl

Expand All @@ -21,5 +21,21 @@ class ACMESettings(ImmutableModel):
eab_hmac_key: Optional[str] = None


class OTelSettings(ImmutableModel):
traces_enabled: bool
logs_enabled: bool
metrics_enabled: bool
endpoint: str
headers: dict[str, str]
traces_sample_rate: float
traces_auth_sample_rate: float
resource_attributes: dict[str, Union[str, int]]

@property
def any_enabled(self) -> bool:
return self.traces_enabled or self.logs_enabled or self.metrics_enabled


class GlobalProxyConfig(ImmutableModel):
acme_settings: ACMESettings = ACMESettings()
otel_settings: Optional[OTelSettings] = None
104 changes: 104 additions & 0 deletions src/dstack/_internal/proxy/gateway/otel.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
import logging
from typing import Literal, Optional

from fastapi import FastAPI
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from opentelemetry.instrumentation.httpx import HTTPX2ClientInstrumentor, HTTPXClientInstrumentor
from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace.sampling import ParentBased

from dstack._internal import settings as core_settings
from dstack._internal.proxy.gateway.models import OTelSettings
from dstack._internal.utils import otel as otel_utils
from dstack._internal.utils.logging import get_logger

logger = get_logger(__name__)

_AUTH_SPAN_NAME_PREFIX = "GET /api/auth/"

_applied_settings: Optional[OTelSettings] = None


def instrument(app: FastAPI) -> None:
"""Must be called before the app starts serving. No-op until `apply_settings` is called."""
FastAPIInstrumentor.instrument_app(app)
HTTPXClientInstrumentor().instrument()
HTTPX2ClientInstrumentor().instrument()
SystemMetricsInstrumentor(config=otel_utils.PROCESS_METRICS_CONFIG).instrument()


def apply_settings(settings: Optional[OTelSettings]) -> bool:
"""
Returns True if the app must be restarted to apply `settings`,
since OpenTelemetry providers can only be set once per process.
"""
global _applied_settings
if settings is not None and not settings.any_enabled:
settings = None
if settings == _applied_settings:
return False
if _applied_settings is not None:
logger.info("OpenTelemetry settings changed, the gateway app needs a restart")
return True
assert settings is not None
# Set before configuring, since a failed configuration cannot be retried in this process
_applied_settings = settings
try:
_configure(settings)
except Exception:
logger.exception("Failed to configure OpenTelemetry")
return False


def _configure(settings: OTelSettings) -> None:
resource = _get_resource(settings)
if settings.traces_enabled:
otel_utils.configure_tracing(
resource,
ParentBased(root=_build_root_sampler(settings)),
OTLPSpanExporter(
endpoint=_get_signal_endpoint(settings, "traces"), headers=settings.headers
),
)
if settings.metrics_enabled:
exporter = OTLPMetricExporter(
endpoint=_get_signal_endpoint(settings, "metrics"), headers=settings.headers
)
otel_utils.configure_metrics(resource, [PeriodicExportingMetricReader(exporter)])
if settings.logs_enabled:
otel_utils.configure_log_export(
resource,
OTLPLogExporter(
endpoint=_get_signal_endpoint(settings, "logs"), headers=settings.headers
),
# The app logs at DEBUG level to journald, which would be too noisy to export
level=logging.INFO,
)


def _build_root_sampler(settings: OTelSettings) -> otel_utils.RootSpanNameSampler:
return otel_utils.RootSpanNameSampler(
default_rate=settings.traces_sample_rate,
rates_by_name_prefix={_AUTH_SPAN_NAME_PREFIX: settings.traces_auth_sample_rate},
)


def _get_signal_endpoint(
settings: OTelSettings, signal: Literal["traces", "metrics", "logs"]
) -> str:
return f"{settings.endpoint.rstrip('/')}/v1/{signal}"


def _get_resource(settings: OTelSettings) -> Resource:
return Resource.create(
{
**settings.resource_attributes,
"service.name": "dstack-gateway",
"service.version": core_settings.DSTACK_VERSION or "dev",
}
)
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ server {
proxy_pass http://localhost:{{ proxy_port }}/api/models/{{ project_name }}/;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
# Don't let public clients control the gateway app's tracing
proxy_set_header traceparent "";
proxy_set_header tracestate "";
proxy_set_header baggage "";
proxy_read_timeout {{ read_timeout }}s;
}
listen 80;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,10 @@ server {
proxy_set_header Content-Length "";
proxy_set_header X-Original-URI $request_uri;
proxy_set_header Authorization $http_authorization;
# Don't let public clients control the gateway app's tracing
proxy_set_header traceparent "";
proxy_set_header tracestate "";
proxy_set_header baggage "";
}
{% endif %}

Expand Down
10 changes: 9 additions & 1 deletion src/dstack/_internal/proxy/gateway/routers/config.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
import os
import signal
from typing import Annotated

from fastapi import APIRouter, Depends
from fastapi import APIRouter, BackgroundTasks, Depends

from dstack._internal.proxy.gateway import otel
from dstack._internal.proxy.gateway.deps import get_gateway_proxy_repo
from dstack._internal.proxy.gateway.models import ACMESettings, GlobalProxyConfig
from dstack._internal.proxy.gateway.repo.repo import GatewayProxyRepo
Expand All @@ -15,6 +18,7 @@
async def update_global_config(
body: ConfigRequest,
repo: Annotated[GatewayProxyRepo, Depends(get_gateway_proxy_repo)],
background_tasks: BackgroundTasks,
) -> OkResponse:
await repo.set_config(
GlobalProxyConfig(
Expand All @@ -23,6 +27,10 @@ async def update_global_config(
eab_kid=body.acme_eab_kid,
eab_hmac_key=body.acme_eab_hmac_key,
),
otel_settings=body.otel_settings,
),
)
if otel.apply_settings(body.otel_settings):
# Saved settings are applied on startup. systemd restarts the app (`Restart=always`)
background_tasks.add_task(os.kill, os.getpid(), signal.SIGTERM)
return OkResponse()
3 changes: 3 additions & 0 deletions src/dstack/_internal/proxy/gateway/schemas/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,11 @@

from pydantic import AnyHttpUrl, BaseModel

from dstack._internal.proxy.gateway.models import OTelSettings


class ConfigRequest(BaseModel):
acme_server: Optional[AnyHttpUrl] = None
acme_eab_kid: Optional[str] = None
acme_eab_hmac_key: Optional[str] = None
otel_settings: Optional[OTelSettings] = None
Loading
Loading