flyteplugins-otel 2.6.1__py3-none-any.whl

flyteplugins/otel/__init__.py

@@ -0,0 +1,49 @@
+"""OpenTelemetry tracing for Flyte.
+
+Every task becomes a span, every `flyte.trace` step becomes a child span inside it, and
+spans created by your own code or by any instrumentation library nest underneath without
+wiring. Export goes wherever OTLP goes.
+
+Usage:
+
+    import flyte
+    from flyteplugins.otel import init
+
+    # At module scope, not inside a task: the task span opens before the task body runs.
+    init(service_name="my-service")
+
+    env = flyte.TaskEnvironment(name="my_env")
+
+    @env.task
+    async def main(n: int) -> int:
+        ...
+
+With no arguments `init` reads OTEL_EXPORTER_OTLP_ENDPOINT and OTEL_EXPORTER_OTLP_HEADERS,
+so pointing it at a vendor is a matter of setting those usually from a `flyte.Secret`. If
+you already configure OpenTelemetry yourself, pass `tracer_provider` and your setup is
+adopted unchanged.
+
+Trace context travels in Flyte's `custom_context` as a W3C carrier in both directions: a
+run submitted inside a caller's span joins that trace, and a child task nests under the task
+that spawned it even though it runs in another pod.
+
+Two things are specific to Flyte being durable. When no trace context arrives from outside,
+the trace id is derived from the run identity, so the several processes that make up a
+crashed-and-resumed run all record into one trace with no coordination. And steps that a
+resumed run served from its durable log, which never execute and so would otherwise be
+missing, are recorded as spans marked `flyte.replayed`.
+"""
+
+from ._ids import format_trace_id, trace_id_for_run
+from ._observer import OtelObserver, RunScopedIdGenerator
+from ._setup import get_tracer, init, shutdown
+
+__all__ = [
+    "OtelObserver",
+    "RunScopedIdGenerator",
+    "format_trace_id",
+    "get_tracer",
+    "init",
+    "shutdown",
+    "trace_id_for_run",
+]

flyteplugins/otel/_ids.py

@@ -0,0 +1,59 @@
+"""Deriving a stable OpenTelemetry trace id from a Flyte run.
+
+A durable run can span several containers: it crashes, resumes, retries, and each of those
+is a fresh process with a fresh OpenTelemetry SDK. Left alone, every one of them mints its
+own random trace id and the backend ends up holding several unrelated traces for what the
+user thinks of as a single agent run.
+
+Deriving the trace id from the run identifier fixes that without any coordination. Every
+process computes the same 16 bytes from values it already has, so spans recorded before a
+crash and spans recorded after the resume land in the same trace even though neither
+process ever spoke to the other.
+
+Only the trace id is derived. Span ids stay random, which keeps each attempt a distinct
+subtree under the shared trace rather than a set of colliding ids, and means a resumed run
+shows its earlier attempts alongside the one that finally succeeded.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from flyte.models import ActionID
+
+__all__ = ["format_trace_id", "trace_id_for_run"]
+
+# Namespace prefix so these digests cannot collide with any other use of the same inputs.
+_NAMESPACE = b"flyte.otel.run.v1"
+
+# W3C trace ids are 16 bytes and must not be all zero.
+_TRACE_ID_BYTES = 16
+
+
+def trace_id_for_run(action: "ActionID") -> int:
+    """The trace id shared by every span in this run, across attempts and containers.
+
+    Derived from the fully qualified run identity rather than the run name alone, so two
+    runs that happen to share a name in different projects or domains stay distinct.
+    """
+    parts = (
+        action.org or "",
+        action.project or "",
+        action.domain or "",
+        action.run_name or action.name,
+    )
+    digest = hashlib.blake2b(
+        b"\x00".join(part.encode("utf-8") for part in parts),
+        digest_size=_TRACE_ID_BYTES,
+        person=_NAMESPACE[:16],
+    ).digest()
+
+    # A zero trace id is invalid per the spec and would be silently dropped by exporters.
+    return int.from_bytes(digest, "big") or 1
+
+
+def format_trace_id(trace_id: int) -> str:
+    """Render a trace id the way backends display it: 32 lowercase hex characters."""
+    return format(trace_id, "032x")

flyteplugins/otel/_observer.py

@@ -0,0 +1,259 @@
+"""Turning Flyte's task and trace callbacks into OpenTelemetry spans."""
+
+from __future__ import annotations
+
+import contextlib
+import contextvars
+from collections.abc import Generator, Mapping
+from typing import TYPE_CHECKING, Any, Optional
+
+import flyte
+from flyte._logging import logger
+from opentelemetry import trace as trace_api
+from opentelemetry.context import Context
+from opentelemetry.propagate import extract, inject
+from opentelemetry.sdk.trace.id_generator import IdGenerator, RandomIdGenerator
+from opentelemetry.trace import Span, SpanKind, Status, StatusCode
+
+from ._ids import trace_id_for_run
+
+if TYPE_CHECKING:
+    from flyte._observe import Recorder, StepInfo, TaskInfo
+    from flyte.models import ActionID
+
+__all__ = ["OtelObserver", "RunScopedIdGenerator"]
+
+# Set for the instant a root span is being created, so the id generator below can hand back
+# the run's derived trace id instead of a random one. A ContextVar rather than a plain
+# attribute because tasks may start spans from several coroutines on the same thread.
+_pinned_trace_id: contextvars.ContextVar[Optional[int]] = contextvars.ContextVar("flyte_pinned_trace_id", default=None)
+
+
+class RunScopedIdGenerator(IdGenerator):
+    """Hands back the run's derived trace id when one has been pinned.
+
+    OpenTelemetry gives no way to ask for a specific trace id when starting a span; the
+    tracer always asks its provider's id generator. Pinning the value around the one call
+    that needs it is the supported way in.
+
+    Everything else is delegated, so wrapping a provider that was configured elsewhere keeps
+    whatever id generation it already had.
+    """
+
+    def __init__(self, delegate: IdGenerator | None = None):
+        self._delegate = delegate or RandomIdGenerator()
+
+    def generate_span_id(self) -> int:
+        return self._delegate.generate_span_id()
+
+    def generate_trace_id(self) -> int:
+        pinned = _pinned_trace_id.get()
+        return pinned if pinned is not None else self._delegate.generate_trace_id()
+
+
+def _action_attributes(action: "ActionID") -> dict[str, Any]:
+    """The Flyte identifiers that let a span be traced back to the run that produced it.
+
+    These are also what a Grafana data link queries on to jump from a span back into the
+    Flyte UI, so the names here are effectively public API.
+    """
+    attributes: dict[str, Any] = {"flyte.action_name": action.name}
+    if action.run_name:
+        attributes["flyte.run_name"] = action.run_name
+    if action.project:
+        attributes["flyte.project"] = action.project
+    if action.domain:
+        attributes["flyte.domain"] = action.domain
+    if action.org:
+        attributes["flyte.org"] = action.org
+    return attributes
+
+
+def _inbound_context(carrier: Mapping[str, str] | None) -> Context | None:
+    """Read a parent span context out of a Flyte custom_context carrier.
+
+    This is the same W3C carrier the documented `custom_context` propagation pattern uses,
+    so a run submitted inside a caller's span joins that caller's trace rather than starting
+    its own. Returns None when the carrier holds nothing usable, which is the ordinary case
+    for a run kicked off without any surrounding trace.
+    """
+    if not carrier:
+        return None
+    try:
+        context = extract(dict(carrier))
+    except Exception:
+        logger.debug("Could not extract a trace context from custom_context", exc_info=True)
+        return None
+    span_context = trace_api.get_current_span(context).get_span_context()
+    return context if span_context.is_valid else None
+
+
+@contextlib.contextmanager
+def _propagate_to_sub_actions() -> Generator[None, None, None]:
+    """Publish the active span into custom_context for the duration of the block.
+
+    Best effort: propagation failing is not a reason to fail the task, and outside a task
+    context `flyte.custom_context` is itself a no-op.
+    """
+    carrier: dict[str, str] = {}
+    try:
+        inject(carrier)
+    except Exception:
+        logger.debug("Could not inject a trace context for sub-actions", exc_info=True)
+
+    if not carrier:
+        yield
+        return
+
+    # Driven by hand rather than with `with`, so that only entering and leaving the context
+    # are guarded. Wrapping the body too would swallow the task's own exceptions.
+    manager = None
+    try:
+        manager = flyte.custom_context(**carrier)
+        manager.__enter__()
+    except Exception:
+        logger.debug("Could not set custom_context for sub-actions", exc_info=True)
+        manager = None
+
+    try:
+        yield
+    finally:
+        if manager is not None:
+            try:
+                manager.__exit__(None, None, None)
+            except Exception:
+                logger.debug("Could not restore custom_context after the task", exc_info=True)
+
+
+def _finish(span: Span, recorder: "Recorder") -> None:
+    """Stamp the outcome on a span before it ends.
+
+    `flyte.trace` swallows the user's exception so it can be written to the durable log,
+    so the error arrives on the recorder rather than propagating through the span's own
+    exception handling.
+    """
+    error = recorder.error
+    if error is None:
+        span.set_status(Status(StatusCode.OK))
+        return
+    span.record_exception(error)
+    span.set_status(Status(StatusCode.ERROR, f"{type(error).__name__}: {error}"))
+
+
+class OtelObserver:
+    """A `flyte._observe.Observer` that records spans.
+
+    Task spans are roots pinned to the run's derived trace id. Every attempt of a task
+    therefore starts its own subtree and because the trace id is shared those subtrees all
+    collect into the one trace: a resumed run reads as the attempt that crashed followed by
+    the attempt that finished.
+    """
+
+    def __init__(self, tracer: trace_api.Tracer):
+        self._tracer = tracer
+
+    @contextlib.contextmanager
+    def task_span(self, info: "TaskInfo", recorder: "Recorder") -> Generator[None, None, None]:
+        span = self._start_task_span(info)
+
+        with trace_api.use_span(span, end_on_exit=True, record_exception=False, set_status_on_exception=False):
+            # Hand this span down as the parent for anything spawned inside the task. Flyte
+            # copies custom_context into every sub-action's inputs, so a child task running in
+            # another pod picks the parent up with no wiring of our own, and because inputs are
+            # durable it survives a resume too.
+            with _propagate_to_sub_actions():
+                try:
+                    yield
+                finally:
+                    _finish(span, recorder)
+
+    def _start_task_span(self, info: "TaskInfo") -> Span:
+        """Start the task span under an inbound trace when there is one.
+
+        A `traceparent` in the run's context means something outside Flyte already started
+        this trace: the caller who submitted the run or the parent task that spawned this
+        one. Nesting under it is what joins the two halves, and it keeps working across a
+        resume because the carrier travels with the action's persisted inputs.
+
+        With nothing inbound this is a root span, and the trace id is derived from the run so
+        that separate attempts still converge on one trace.
+        """
+        attributes = {**_action_attributes(info.action), "flyte.task_name": info.name}
+        parent = _inbound_context(info.custom_context)
+
+        if parent is not None:
+            return self._tracer.start_span(
+                info.name,
+                context=parent,
+                kind=SpanKind.INTERNAL,
+                attributes=attributes,
+                record_exception=False,
+                set_status_on_exception=False,
+            )
+
+        token = _pinned_trace_id.set(trace_id_for_run(info.action))
+        try:
+            # An empty Context makes this a root span, which is what forces the tracer to
+            # mint a trace id at all, and therefore to consult the pinned value.
+            return self._tracer.start_span(
+                info.name,
+                context=Context(),
+                kind=SpanKind.INTERNAL,
+                attributes=attributes,
+                record_exception=False,
+                set_status_on_exception=False,
+            )
+        finally:
+            _pinned_trace_id.reset(token)
+
+    @contextlib.contextmanager
+    def step_span(self, info: "StepInfo", recorder: "Recorder") -> Generator[None, None, None]:
+        attributes = {
+            **_action_attributes(info.action),
+            "flyte.step_name": info.name,
+            "flyte.replayed": info.replayed,
+        }
+        if info.task_action is not None:
+            attributes["flyte.task_action_name"] = info.task_action.name
+
+        # A replayed step opens and closes this block without doing any work, so it lands as
+        # a near-instant span. That is the honest shape: the step did not run here, it was
+        # read back from the durable log, and flyte.replayed says so.
+        with self._tracer.start_as_current_span(
+            info.name,
+            kind=SpanKind.INTERNAL,
+            attributes=attributes,
+            record_exception=False,
+            set_status_on_exception=False,
+        ) as span:
+            try:
+                yield
+            finally:
+                _finish(span, recorder)
+
+
+def make_id_generator(existing: Optional[IdGenerator] = None) -> IdGenerator:
+    """Return an id generator that honours pinned trace ids, wrapping one if given."""
+    if isinstance(existing, RunScopedIdGenerator):
+        return existing
+    return RunScopedIdGenerator(existing)
+
+
+def adopt_provider(tracer_provider: object) -> bool:
+    """Make an already configured TracerProvider able to honour run derived trace ids.
+
+    Someone who set up their own provider should not have to give it up to get durable traces.
+    Swapping in a wrapping id generator leaves their sampler, resource and exporters exactly as they were.
+
+    Returns whether the swap happened. A provider without an `id_generator` (the API's
+    no-op provider, or a stub) is left alone and simply gets random trace ids.
+    """
+    existing = getattr(tracer_provider, "id_generator", None)
+    if existing is None:
+        return False
+    try:
+        tracer_provider.id_generator = make_id_generator(existing)  # type: ignore[attr-defined]
+    except Exception:
+        logger.debug("Could not install a run scoped id generator on the supplied provider", exc_info=True)
+        return False
+    return True

flyteplugins/otel/_setup.py

@@ -0,0 +1,337 @@
+"""Wiring the observer up to an exporter.
+
+OTLP is the default because it is what most backends document but nothing here requires it.
+Any `SpanExporter` works, several can run side by side, and a provider you configured
+yourself is adopted whole.
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+from collections.abc import Sequence
+from typing import Any, Mapping, Optional, Union
+
+import flyte._observe as observe
+from flyte._logging import logger
+from opentelemetry import trace as trace_api
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, SimpleSpanProcessor, SpanExporter
+
+from ._observer import OtelObserver, adopt_provider, make_id_generator
+
+__all__ = ["get_tracer", "init", "shutdown"]
+
+_INSTRUMENTATION_NAME = "flyteplugins-otel"
+
+# Process-global, not context-local, and deliberately so: init installs an OpenTelemetry
+# TracerProvider (a process global itself) and appends to flyte._observe's module-level
+# observer list. Context-local state here would let shutdown miss an observer that is still
+# firing for every task. The lock makes the check-then-set in init atomic, matching how
+# flyte's own _ControllerState guards the same shape.
+_lock = threading.Lock()
+_state: dict[str, Any] = {"provider": None, "observer": None, "tracer": None}
+
+
+def _normalize_headers(headers: Union[Mapping[str, str], str, None]) -> Optional[dict[str, str]]:
+    """Accept either a mapping or the comma separated form used by the OTEL_ env vars."""
+    if headers is None or isinstance(headers, dict):
+        return dict(headers) if headers else None
+    if isinstance(headers, Mapping):
+        return dict(headers)
+    parsed: dict[str, str] = {}
+    for pair in str(headers).split(","):
+        if not pair.strip():
+            continue
+        key, _, value = pair.partition("=")
+        parsed[key.strip()] = value.strip()
+    return parsed or None
+
+
+def _traces_endpoint(endpoint: Optional[str]) -> Optional[str]:
+    """Point an OTLP gateway base URL at its traces path.
+
+    Backends hand out a base URL (Grafana Cloud gives you the OTLP gateway), while the HTTP
+    exporter wants the signal specific path when the endpoint is passed explicitly. Doing
+    the append here means the value copied out of a vendor console works as-is.
+    """
+    if not endpoint:
+        return None
+    trimmed = endpoint.rstrip("/")
+    if trimmed.endswith("/v1/traces"):
+        return trimmed
+    return f"{trimmed}/v1/traces"
+
+
+def _resolve_protocol(protocol: Optional[str]) -> str:
+    """Pick the OTLP wire protocol, honouring OpenTelemetry's own environment variable.
+
+    The spec spells these "http/protobuf" and "grpc"; the short forms are accepted because
+    that is what people type. Ignoring OTEL_EXPORTER_OTLP_PROTOCOL would mean a backend
+    configured entirely through the standard variables silently got the wrong transport.
+    """
+    raw = (
+        protocol
+        or os.environ.get("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL")
+        or os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL")
+    )
+    value = (raw or "http/protobuf").strip().lower()
+    if value == "grpc":
+        return "grpc"
+    if value in ("http", "http/protobuf", "httpprotobuf", "http/proto"):
+        return "http/protobuf"
+    raise ValueError(f"Unsupported OTLP protocol {raw!r}. Use 'http/protobuf' or 'grpc'.")
+
+
+def _otlp_exporter(
+    *,
+    protocol: str,
+    endpoint: Optional[str],
+    headers: Union[Mapping[str, str], str, None],
+) -> SpanExporter:
+    """Build the OTLP exporter for the chosen transport.
+
+    gRPC lives in a separate distribution, so its absence is reported as something to install
+    rather than an ImportError from deep inside the SDK. Note the endpoint differs by
+    transport: gRPC takes the base endpoint, HTTP wants the signal specific path.
+    """
+    normalized_headers = _normalize_headers(headers)
+    if protocol == "grpc":
+        try:
+            from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter as GrpcExporter
+        except ImportError as e:
+            raise ImportError(
+                "OTLP over gRPC needs the opentelemetry-exporter-otlp-proto-grpc package. "
+                'Install it with `pip install "flyteplugins-otel[grpc]"`, or use protocol="http/protobuf".'
+            ) from e
+        return GrpcExporter(endpoint=endpoint or None, headers=normalized_headers)
+
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter as HttpExporter
+
+    return HttpExporter(endpoint=_traces_endpoint(endpoint), headers=normalized_headers)
+
+
+def _otlp_endpoint_configured(endpoint: Optional[str]) -> bool:
+    """Whether anyone actually said where traces should go.
+
+    Without this the OTLP exporter silently falls back to the spec default of
+    http://localhost:4318, and every batch then produces several lines of connection-refused
+    retries. That is noise rather than information, and it is the normal state of the driver
+    process, which imports the module to submit a run and has no reason to export anything.
+    """
+    return bool(
+        endpoint
+        or os.environ.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
+        or os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")
+    )
+
+
+def _as_exporters(exporter: Union[SpanExporter, Sequence[SpanExporter], None]) -> list[SpanExporter]:
+    """Accept one exporter or several. A single SpanExporter is not itself a sequence."""
+    if exporter is None:
+        return []
+    if isinstance(exporter, SpanExporter):
+        return [exporter]
+    return list(exporter)
+
+
+def _warn_if_task_already_started() -> None:
+    """Say something when init lands too late to record the task's own span.
+
+    The task span opens before the task body runs, so an observer registered from inside the
+    body has already missed it. The symptom is a trace holding the trace steps but no task
+    span to hang them from, which is hard to work out from the output alone.
+    """
+    try:
+        from flyte._context import internal_ctx
+
+        # Not `flyte.ctx() is not None`: outside a task that returns a falsy NullTaskContext
+        # rather than None, so the identity check would report every call as being in a task.
+        in_task = internal_ctx().is_task_context()
+    except Exception:  # pragma: no cover - context lookup should not break init
+        return
+
+    if in_task:
+        logger.warning(
+            "flyteplugins-otel was initialized from inside a running task, so that task's own span "
+            "has already started and will not be recorded. Call init() at module scope instead, so "
+            "the observer is registered before any task begins."
+        )
+
+
+def init(
+    *,
+    service_name: Optional[str] = None,
+    endpoint: Optional[str] = None,
+    headers: Union[Mapping[str, str], str, None] = None,
+    protocol: Optional[str] = None,
+    resource_attributes: Optional[Mapping[str, Any]] = None,
+    exporter: Union[SpanExporter, Sequence[SpanExporter], None] = None,
+    tracer_provider: Optional[trace_api.TracerProvider] = None,
+    disable_batch: bool = False,
+    set_global: bool = True,
+) -> OtelObserver:
+    """Start recording Flyte tasks and trace steps as OpenTelemetry spans.
+
+    Call this once at module scope, not inside a task. The task span opens before the task
+    body runs, so initializing from within the body means that task's own span has already
+    been missed. Module scope runs during import, which happens before any task starts.
+
+    With no arguments it reads the standard OTEL_EXPORTER_OTLP_ENDPOINT and
+    OTEL_EXPORTER_OTLP_HEADERS variables, which is the shape most vendors document.
+
+    If you already configure OpenTelemetry yourself, pass `tracer_provider` and none of the
+    exporter arguments. The provider is adopted as it stands, with its sampler, resource, and
+    exporters untouched; only its id generator is wrapped, so run derived trace ids keep
+    working without you giving up your own setup.
+
+    Args:
+        service_name: Value for service.name. Defaults to OTEL_SERVICE_NAME, then "flyte".
+            Not used when adopting a provider, which carries its own resource.
+        endpoint: OTLP endpoint. On http/protobuf a base gateway URL is fine and the traces
+            path is added; on grpc the base endpoint is used as given.
+        headers: Export headers, as a mapping or the "k=v,k2=v2" form.
+        protocol: OTLP transport, "http/protobuf" or "grpc". Defaults to
+            OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, then OTEL_EXPORTER_OTLP_PROTOCOL, then
+            http/protobuf. gRPC needs the [grpc] extra.
+        resource_attributes: Extra resource attributes to attach to every span.
+        exporter: Export through this instead of building an OTLP exporter, or pass several
+            to fan out — a ConsoleSpanExporter alongside a real backend, say. Any
+            `SpanExporter` works; nothing here requires OTLP.
+        tracer_provider: Adopt a provider you configured yourself instead of building one.
+            Cannot be combined with the exporter building arguments.
+        disable_batch: Export each span as it ends. Slower, but nothing is lost if the
+            process dies, which matters when the thing being demonstrated is a crash.
+        set_global: Install the provider as the global one, so other instrumentation shares
+            it. Not used when adopting a provider, which is assumed to be installed already.
+
+    Returns:
+        The registered observer, which can be passed to `shutdown`.
+
+    Raises:
+        ValueError: If tracer_provider is combined with the exporter building arguments.
+    """
+    with _lock:
+        if _state["observer"] is not None:
+            logger.debug("flyteplugins-otel already initialized; returning the existing observer")
+            return _state["observer"]
+
+        return _init_locked(
+            service_name=service_name,
+            endpoint=endpoint,
+            headers=headers,
+            protocol=protocol,
+            resource_attributes=resource_attributes,
+            exporter=exporter,
+            tracer_provider=tracer_provider,
+            disable_batch=disable_batch,
+            set_global=set_global,
+        )
+
+
+def _init_locked(
+    *,
+    service_name: Optional[str],
+    endpoint: Optional[str],
+    headers: Union[Mapping[str, str], str, None],
+    protocol: Optional[str],
+    resource_attributes: Optional[Mapping[str, Any]],
+    exporter: Union[SpanExporter, Sequence[SpanExporter], None],
+    tracer_provider: Optional[trace_api.TracerProvider],
+    disable_batch: bool,
+    set_global: bool,
+) -> OtelObserver:
+    """The body of `init` run with the lock already held."""
+    _warn_if_task_already_started()
+
+    if tracer_provider is not None:
+        conflicting = {
+            "endpoint": endpoint,
+            "headers": headers,
+            "exporter": exporter,
+            "protocol": protocol,
+            "resource_attributes": resource_attributes,
+        }
+        supplied = sorted(name for name, value in conflicting.items() if value is not None)
+        if supplied:
+            raise ValueError(
+                f"tracer_provider cannot be combined with {', '.join(supplied)}: the supplied provider "
+                f"already owns its exporters and resource. Configure them on the provider instead."
+            )
+        return _attach(tracer_provider, owns_provider=False)
+
+    attributes: dict[str, Any] = {
+        "service.name": service_name or os.environ.get("OTEL_SERVICE_NAME") or "flyte",
+    }
+    if resource_attributes:
+        attributes.update(resource_attributes)
+
+    provider = TracerProvider(resource=Resource.create(attributes), id_generator=make_id_generator())
+
+    exporters = _as_exporters(exporter)
+    if not exporters:
+        if _otlp_endpoint_configured(endpoint):
+            exporters = [_otlp_exporter(protocol=_resolve_protocol(protocol), endpoint=endpoint, headers=headers)]
+        else:
+            # Say it once, here, rather than let it surface as a retry storm from the
+            # exporter. Spans are still recorded, so nesting and context propagation behave
+            # exactly as they would; they are simply dropped instead of shipped.
+            logger.warning(
+                "flyteplugins-otel: no OTLP endpoint configured, so spans are recorded but not exported. "
+                "Set OTEL_EXPORTER_OTLP_ENDPOINT, pass endpoint=, or pass an exporter such as "
+                "ConsoleSpanExporter. If you are running a local collector, point the variable at it "
+                "explicitly (http://localhost:4318) rather than relying on the default."
+            )
+
+    # One processor per exporter, so several can run side by side.
+    for each in exporters:
+        provider.add_span_processor(SimpleSpanProcessor(each) if disable_batch else BatchSpanProcessor(each))
+
+    if set_global:
+        trace_api.set_tracer_provider(provider)
+
+    return _attach(provider, owns_provider=True)
+
+
+def _attach(provider: trace_api.TracerProvider, *, owns_provider: bool) -> OtelObserver:
+    """Register an observer against a provider, remembering whether we may shut it down."""
+    if not owns_provider and not adopt_provider(provider):
+        logger.debug(
+            "The supplied tracer provider has no id generator to wrap. Spans are still recorded, but trace "
+            "ids will not be derived from the run, so separate attempts of a run will not share a trace."
+        )
+
+    tracer = provider.get_tracer(_INSTRUMENTATION_NAME)
+    observer = OtelObserver(tracer)
+    observe.register_observer(observer)
+
+    # Only a provider we built is ours to shut down; adopting one must not flush or close it.
+    _state.update({"provider": provider if owns_provider else None, "observer": observer, "tracer": tracer})
+    return observer
+
+
+def get_tracer() -> Optional[trace_api.Tracer]:
+    """The tracer `init` built for handing to another instrumentation library.
+
+    Libraries that create their own provider still nest correctly since parenting comes from
+    the active context rather than the provider. Sharing the tracer just keeps everything on
+    one export pipeline.
+    """
+    return _state["tracer"]
+
+
+def shutdown() -> None:
+    """Unregister the observer and flush pending spans."""
+    with _lock:
+        _shutdown_locked()
+
+
+def _shutdown_locked() -> None:
+    observer = _state["observer"]
+    if observer is not None:
+        observe.unregister_observer(observer)
+    provider = _state["provider"]
+    if provider is not None:
+        provider.shutdown()
+    _state.update({"provider": None, "observer": None, "tracer": None})

flyteplugins/otel/grafana.py

@@ -0,0 +1,110 @@
+"""Links from a Flyte action into Grafana.
+
+Flyte renders a task's `links` in its UI, so this is the jump from a run to the telemetry
+it produced. These are pure URL builders with no Grafana dependency, and they work off the
+`flyte.*` span attributes this plugin already stamps.
+
+The trace link queries rather than addressing a trace by id. That matters: a query on
+`flyte.run_name` finds a run's spans whatever their trace ids turn out to be, so it works
+before the run-derived trace id is involved, and keeps working after. Addressing by id would
+depend on the derivation and break the moment anything upstream propagated a trace context.
+
+Only the Tempo link lives here. A link to a run's conversation in Grafana Agent Observability
+belongs with `flyteplugins-agento11y`, since it is that package's identity binding that
+makes a run addressable by conversation id at all.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from typing import Dict, Optional
+from urllib.parse import urlencode
+
+from flyte import Link
+
+__all__ = ["GrafanaTrace"]
+
+# Flyte hands links placeholders like {{.runName}} at serialization time and substitutes them
+# by string replacement on the finished URI. Percent-encoding turns "{{" into "%7B%7B", which
+# that replacement never matches, so the placeholder would reach Grafana verbatim and the
+# query would search for a run literally named "{{.runName}}".
+_ENCODED_TEMPLATE = re.compile(r"%7B%7B([.\w]+)%7D%7D")
+
+
+def _encode_preserving_templates(params: Dict[str, object]) -> str:
+    """urlencode, then put Flyte's template placeholders back in literal form."""
+    return _ENCODED_TEMPLATE.sub(r"{{\1}}", urlencode(params))
+
+
+def _explore_url(*, host: str, datasource_uid: str, query: str, time_from: str, time_to: str) -> str:
+    """Build a Grafana Explore deep link running a TraceQL query.
+
+    Explore encodes its state as a JSON `panes` parameter. The time range has to be part of
+    it because Explore otherwise defaults to the last hour, and a link to yesterday's run
+    would open on an empty pane and read as broken.
+    """
+    panes = {
+        "flyte": {
+            "datasource": datasource_uid,
+            "queries": [
+                {
+                    "refId": "A",
+                    "datasource": {"type": "tempo", "uid": datasource_uid},
+                    "queryType": "traceql",
+                    "query": query,
+                }
+            ],
+            "range": {"from": time_from, "to": time_to},
+        }
+    }
+    params = _encode_preserving_templates(
+        {"schemaVersion": 1, "panes": json.dumps(panes, separators=(",", ":")), "orgId": 1}
+    )
+    return f"{host.rstrip('/')}/explore?{params}"
+
+
+@dataclass
+class GrafanaTrace(Link):
+    """A link to this run's spans in Grafana Tempo.
+
+    Args:
+        host: Stack URL, e.g. `https://myorg.grafana.net`.
+        datasource_uid: UID of the Tempo datasource. Per-stack and not guessable, so there is
+            no useful default; find it under Connections, Data sources in Grafana.
+        name: Label shown in the Flyte UI.
+        lookback: Start of the Explore time range, in Grafana's relative syntax. The link
+            protocol gives no timestamps, so the window is relative and deliberately wide.
+        action_scoped: Narrow the query to the single action rather than the whole run.
+    """
+
+    host: str
+    datasource_uid: str
+    name: str = "Grafana trace"
+    icon_uri: Optional[str] = ""
+    lookback: str = "now-7d"
+    action_scoped: bool = False
+
+    def get_link(
+        self,
+        run_name: str,
+        project: str,
+        domain: str,
+        context: Dict[str, str],
+        parent_action_name: str,
+        action_name: str,
+        pod_name: str,
+        **kwargs,
+    ) -> str:
+        selectors = [f'.flyte.run_name="{run_name}"']
+        if self.action_scoped and action_name:
+            selectors.append(f'.flyte.action_name="{action_name}"')
+        query = "{" + " && ".join(selectors) + "}"
+        return _explore_url(
+            host=self.host,
+            datasource_uid=self.datasource_uid,
+            query=query,
+            time_from=self.lookback,
+            time_to="now",
+        )

flyteplugins_otel-2.6.1.dist-info/METADATA

@@ -0,0 +1,234 @@
+Metadata-Version: 2.4
+Name: flyteplugins-otel
+Version: 2.6.1
+Summary: OpenTelemetry tracing for Flyte, with traces that survive durable resume
+Author-email: Samhita Alla <samhita@union.ai>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: opentelemetry-api>=1.20.0
+Requires-Dist: opentelemetry-sdk>=1.20.0
+Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.20.0
+Requires-Dist: flyte
+Provides-Extra: grpc
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.20.0; extra == "grpc"
+
+# flyteplugins-otel
+
+OpenTelemetry tracing for Flyte.
+
+Every task becomes a span, every `flyte.trace` step becomes a child span inside it, and spans
+created by your own code or by any instrumentation library nest underneath without wiring.
+Export goes wherever OTLP goes.
+
+None of this is specific to agents or to LLM work. It is ordinary distributed tracing for
+ordinary Flyte tasks with some additions for the things Flyte does that a normal
+OpenTelemetry setup has no way to model.
+
+## Install and use
+
+```python
+import flyte
+from flyteplugins.otel import init
+
+# At module scope, not inside a task. The task span opens before the task body runs, so
+# initializing from within the body means that task's own span has already been missed.
+init(service_name="my-service")
+
+env = flyte.TaskEnvironment(name="my_env")
+
+@env.task
+async def main(n: int) -> int:
+    ...
+```
+
+With no arguments `init` reads the standard `OTEL_EXPORTER_OTLP_ENDPOINT` and
+`OTEL_EXPORTER_OTLP_HEADERS` variables, which is the shape most vendors document. Supply them
+as a `flyte.Secret` rather than hardcoding them.
+
+A base gateway URL is fine if you pass the endpoint directly; the traces path is appended:
+
+```python
+init(
+    service_name="my-service",
+    endpoint="https://otlp-gateway-prod-us-east-0.grafana.net/otlp",
+    headers={"Authorization": "Basic <base64>"},
+)
+```
+
+### Exporters
+
+OTLP is the default but not a requirement. Any `SpanExporter` works and several can run side by
+side — a console exporter alongside a real backend, say:
+
+```python
+init(exporter=[ConsoleSpanExporter(), JaegerExporter(...)])
+```
+
+For OTLP, both transports are supported. The protocol follows
+`OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, then `OTEL_EXPORTER_OTLP_PROTOCOL`, then
+`http/protobuf`, or pass it directly. gRPC ships separately:
+
+```bash
+pip install "flyteplugins-otel[grpc]"
+```
+
+```python
+init(endpoint="http://collector:4317", protocol="grpc")
+```
+
+If you already configure OpenTelemetry yourself, hand over the provider instead and nothing
+about your setup changes:
+
+```python
+init(tracer_provider=my_provider)
+```
+
+## Examples
+
+| Example                    | What it covers                                          |
+| -------------------------- | ------------------------------------------------------- |
+| `basic_tracing.py`         | The smallest setup, printing spans to the console       |
+| `custom_spans.py`          | Your own spans inside a task, nesting automatically     |
+| `nested_tasks.py`          | Tasks calling tasks, across pods, in one trace          |
+| `existing_provider.py`     | Adopting a TracerProvider you already configured        |
+| `propagate_from_caller.py` | Joining a trace that started outside Flyte              |
+| `http_instrumentation.py`  | Third party auto-instrumentation, here httpx            |
+| `grafana_cloud.py`         | Exporting to a real backend, with secrets               |
+| `durable_trace.py`         | A crash and its resume, as a single trace               |
+
+## Trace context, in and out
+
+Flyte propagates a key-value `custom_context` through a run and into every sub-action. This
+plugin uses it as a W3C carrier in both directions.
+
+Inbound: if `custom_context` holds a `traceparent`, the task span starts under it rather than
+becoming a root. So a run submitted from inside a caller's span joins the caller's trace:
+
+```python
+with tracer.start_as_current_span("incoming_request"):
+    carrier = {}
+    inject(carrier)
+    run = flyte.with_runcontext(custom_context=carrier).run(main, ...)
+```
+
+Outbound: once the task span is open, the plugin publishes it back into `custom_context`, so
+a child task running in another pod nests under the task that spawned it with nothing passed
+by hand. Because `custom_context` travels in the action's persisted inputs, this survives a
+resume too.
+
+Which means the manual `extract` inside each task is no longer needed just to get spans
+parented correctly. Reach for it when you want to start your own spans under the incoming
+context; the plugin's own spans are already there.
+
+## What durability adds
+
+A run that crashes and resumes is several processes over time. Each starts a fresh
+OpenTelemetry SDK that has no idea the earlier ones existed, and two things go wrong.
+
+Every attempt mints its own random trace id, so one agent run arrives at the backend as
+several unrelated traces. And every step the resumed run replayed out of its durable log is
+missing entirely, because replayed steps never execute, so nothing instruments them. The
+trace ends up with holes in it exactly where durability did its job.
+
+So when there is no inbound trace context, the trace id is derived from the run identity
+rather than generated. Every process computes the same value from information it already has,
+and all of them record into one trace with no coordination. Span ids stay random, so each
+attempt is a distinct subtree: a resumed run reads as the attempt that crashed followed by
+the attempt that finished.
+
+Replayed steps are recorded as spans marked `flyte.replayed`. They have no meaningful
+duration, because no work happened, but they are present, so the trace is complete.
+
+For a live demo, `disable_batch=True` exports each span as it ends. It is slower, but nothing
+is lost when the process dies, which matters when the thing being demonstrated is a crash.
+
+## Linking back from Grafana
+
+`flyteplugins.otel.grafana` builds links from a Flyte action into Grafana, rendered in the
+Flyte UI. They are plain URL builders with no Grafana dependency.
+
+```python
+from flyteplugins.otel.grafana import GrafanaTrace
+
+@env.task(links=(GrafanaTrace(host="https://myorg.grafana.net", datasource_uid="<tempo-uid>"),))
+async def my_task() -> str:
+    ...
+```
+
+The trace link queries on `flyte.run_name` rather than addressing a trace by id, so it finds
+a run's spans whatever their trace ids turn out to be — including runs whose trace context
+came from outside. It embeds a time range because Explore otherwise defaults to the last
+hour, and a link to an older run would open on an empty pane.
+
+The datasource UID is per-stack and not guessable; find it under Connections, Data sources.
+
+Flyte hands links placeholders such as `{{.runName}}` when the task is serialized and swaps
+them for real values on the finished URI, so the link is built once and works for every run.
+That substitution is a plain string replacement, which means the placeholders have to survive
+URL encoding intact — the link builders keep them literal for exactly this reason.
+
+Only the Tempo link lives here. A link to a run's conversation in Grafana Agent
+Observability ships with `flyteplugins-agento11y`, since it is that package's identity
+binding that makes a run addressable by conversation id at all.
+
+## Span attributes
+
+Every span carries the identifiers needed to get back to the run that produced it. These are
+effectively public API, since a Grafana data link queries on them to jump from a span into
+the Flyte UI.
+
+| Attribute                                    | On         | Meaning                                           |
+| -------------------------------------------- | ---------- | ------------------------------------------------- |
+| `flyte.run_name`                             | all        | The run, and what the trace id is derived from    |
+| `flyte.action_name`                          | all        | The action that produced the span                 |
+| `flyte.project`, `flyte.domain`, `flyte.org` | all        | Where the run lives                               |
+| `flyte.task_name`                            | task spans | The task being executed                           |
+| `flyte.step_name`                            | step spans | The traced function                               |
+| `flyte.task_action_name`                     | step spans | The task that owns the step                       |
+| `flyte.replayed`                             | step spans | Whether this step was served from the durable log |
+
+## Using it alongside other instrumentation
+
+Libraries that emit their own spans — the OpenTelemetry instrumentation packages, agent
+observability SDKs, vendor SDKs — need no extra wiring. Parenting comes from the active
+context rather than from the tracer provider, and the task span is active for the whole task
+body, so their spans land inside it.
+
+Calling `init` before the other library keeps everything on one export pipeline.
+`get_tracer()` returns the tracer this plugin built for libraries that accept one.
+
+## Flyte's own control-plane calls appear as spans
+
+Once tracing is on you will see `POST` client spans for Flyte's calls to the control plane —
+`Enqueue`, `CreateRun`, `UploadInputs` and so on — alongside your own.
+
+They do not come from this plugin. Flyte's transport is `pyqwest`, whose `HTTPTransport`
+takes `enable_otel: bool = True` and falls back to the global tracer provider when it is not
+given one. `init(set_global=True)`, the default, installs that provider, so the transport
+starts recording through it.
+
+Mostly this is useful: inside a task the spans nest under the task span, so you can see how
+much of a task's wall clock went on talking to the control plane, and the 401-then-200 pairs
+show the auth retry. Two things to be aware of. The volume scales with sub-action count, so a
+wide fan-out produces a lot of them. And calls made outside a task span, during submission,
+arrive as their own root traces rather than joining the run's trace.
+
+There is no switch for this in the plugin, since the transport is Flyte's rather than ours.
+`init(set_global=False)` keeps the provider out of the global slot, which stops the transport
+finding it, at the cost of other instrumentation not finding it either.
+
+## Limitations
+
+With no OTLP endpoint configured — no `OTEL_EXPORTER_OTLP_ENDPOINT`, no `endpoint=`, no
+explicit exporter — spans are recorded but not exported, and `init` says so once. This is the
+normal state of the process that submits a run, since it imports your module and therefore
+runs `init` without needing to export anything. If you run a local collector, point the
+variable at it explicitly rather than relying on the OTLP default of `localhost:4318`.
+
+Replayed spans have no duration. The original timing is written to the control plane but does
+not come back over the channel a resumed run reads from, so recovering it needs a backend
+change. What you get is the step's presence, identity, and outcome.
+
+Trace context rides in `custom_context`, which is a flat string map that Flyte propagates
+wholesale. The `traceparent` key is therefore visible to task code and will be overwritten if
+something else writes that key.

flyteplugins_otel-2.6.1.dist-info/RECORD

@@ -0,0 +1,9 @@
+flyteplugins/otel/__init__.py,sha256=TtOndV9edavTbBokh1RQEnTKTt7TJ1605jwCb-jlVlQ,1808
+flyteplugins/otel/_ids.py,sha256=SDxUBeGTObz9Vaugna3u3t3ncPMwgHEh7r1TjIfYveU,2264
+flyteplugins/otel/_observer.py,sha256=85XYgYUqiWG3mp-j2E1t7hNVr34Fth0OP0MIMiYP9Qg,10468
+flyteplugins/otel/_setup.py,sha256=NXdqEImn7PQn0gMXFqsphkeqvjuhE7TGCBekZcmlJJc,14697
+flyteplugins/otel/grafana.py,sha256=2LEemHdyVliUhjinspMFT0syuyfkY3DdIAEBFcgAZz8,4217
+flyteplugins_otel-2.6.1.dist-info/METADATA,sha256=zo7ZFyqYSMyPvbnxiCXbt-CWVNR6q5d5Qe54vdUhuVc,11206
+flyteplugins_otel-2.6.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+flyteplugins_otel-2.6.1.dist-info/top_level.txt,sha256=cgd779rPu9EsvdtuYgUxNHHgElaQvPn74KhB5XSeMBE,13
+flyteplugins_otel-2.6.1.dist-info/RECORD,,

flyteplugins_otel-2.6.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

flyteplugins_otel-2.6.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+flyteplugins