dagster-cloud 1.12.10__py3-none-any.whl → 1.12.12__py3-none-any.whl

dagster_cloud/opentelemetry/metrics/meter.py

@@ -1,140 +0,0 @@
-from collections.abc import Generator, Iterable, Sequence
-from typing import Callable, Optional, Union, cast
-
-# (Gauge is only exposed as a private class)
-from opentelemetry.metrics import (
-    CallbackOptions,
-    Counter,
-    Histogram,
-    Instrument,
-    ObservableCounter,
-    ObservableGauge,
-    ObservableUpDownCounter,
-    Observation,
-    UpDownCounter,
-)
-from opentelemetry.metrics._internal import Meter as SDKMeter
-
-InstrumentKeyType = tuple[str, str, str]
-ObservableCallback = Callable[[CallbackOptions], Iterable[Observation]]
-ObservableGenerator = Generator[Iterable[Observation], CallbackOptions, None]
-ObserverType = Union[ObservableCallback, ObservableGenerator]
-
-
-class Meter:
-    def __init__(self, meter: SDKMeter):
-        self._meter = meter
-        self._instruments: dict[InstrumentKeyType, Instrument] = {}
-
-    @property
-    def name(self) -> str:
-        return self._meter.name
-
-    @property
-    def version(self) -> Optional[str]:
-        return self._meter.version
-
-    @property
-    def schema_url(self) -> Optional[str]:
-        return self._meter.schema_url
-
-    def get_counter(self, name: str, description: str = "", unit: str = "") -> Counter:
-        key = (name, description, unit)
-        instrument = self._instruments.get(key)
-        if not instrument:
-            instrument = self._meter.create_counter(
-                name=name,
-                description=description,
-                unit=unit,
-            )
-            self._instruments[key] = instrument
-
-        return cast("Counter", instrument)
-
-    def get_up_down_counter(
-        self, name: str, description: str = "", unit: str = ""
-    ) -> UpDownCounter:
-        key = (name, description, unit)
-        instrument = self._instruments.get(key)
-        if not instrument:
-            instrument = self._meter.create_up_down_counter(
-                name=name,
-                description=description,
-                unit=unit,
-            )
-            self._instruments[key] = instrument
-
-        return cast("UpDownCounter", instrument)
-
-    def get_histogram(self, name: str, description: str = "", unit: str = "") -> Histogram:
-        key = (name, description, unit)
-        instrument = self._instruments.get(key)
-        if not instrument:
-            instrument = self._meter.create_histogram(
-                name=name,
-                description=description,
-                unit=unit,
-            )
-            self._instruments[key] = instrument
-
-        return cast("Histogram", instrument)
-
-    def get_observable_counter(
-        self,
-        name: str,
-        description: str = "",
-        unit: str = "",
-        observers: Optional[Sequence[ObserverType]] = None,
-    ) -> ObservableCounter:
-        key = (name, description, unit)
-        instrument = self._instruments.get(key)
-        if not instrument:
-            instrument = self._meter.create_observable_counter(
-                name=name,
-                description=description,
-                unit=unit,
-                callbacks=observers,
-            )
-            self._instruments[key] = instrument
-
-        return cast("ObservableCounter", instrument)
-
-    def get_observable_up_down_counter(
-        self,
-        name: str,
-        description: str = "",
-        unit: str = "",
-        observers: Optional[Sequence[ObserverType]] = None,
-    ) -> ObservableUpDownCounter:
-        key = (name, description, unit)
-        instrument = self._instruments.get(key)
-        if not instrument:
-            instrument = self._meter.create_observable_up_down_counter(
-                name=name,
-                description=description,
-                unit=unit,
-                callbacks=observers,
-            )
-            self._instruments[key] = instrument
-
-        return cast("ObservableUpDownCounter", instrument)
-
-    def get_observable_gauge(
-        self,
-        name: str,
-        description: str = "",
-        unit: str = "",
-        observers: Optional[Sequence[ObserverType]] = None,
-    ) -> ObservableGauge:
-        key = (name, description, unit)
-        instrument = self._instruments.get(key)
-        if not instrument:
-            instrument = self._meter.create_observable_gauge(
-                name=name,
-                description=description,
-                unit=unit,
-                callbacks=observers,
-            )
-            self._instruments[key] = instrument
-
-        return cast("ObservableGauge", instrument)

dagster_cloud/opentelemetry/observers/dagster_exception_handler.py

@@ -1,40 +0,0 @@
-from dagster import DagsterError
-from dagster._utils.error import SerializableErrorInfo
-from grpc import Call as GrpcCall
-
-
-def extract_dagster_error_attributes(error: DagsterError) -> dict[str, str]:
-    """Extracts attributes from a DagsterError that can be used for logging or metrics."""
-    error_attributes = {
-        "exception": type(error).__name__,
-    }
-
-    # DagsterError subtypes often have a field that contains a SerializableErrorInfo object
-    # which identify the cause of the error. This field names vary between error types and is not
-    # always present, but if they are they often identify the root cause of the error.
-    for field in dir(error):
-        if field.startswith("__"):
-            # Skip magic methods and fields
-            continue
-
-        attr = getattr(error, field)
-        if type(attr) is list and all(isinstance(item, SerializableErrorInfo) for item in attr):
-            serializable_error_infos: list[SerializableErrorInfo] = getattr(error, field)
-            for serializable_error_info in serializable_error_infos:
-                if serializable_error_info.cause and serializable_error_info.cause.cls_name:
-                    error_attributes["cause"] = serializable_error_info.cause.cls_name
-                    return error_attributes
-
-    # If not obtained from SerializableErrorInfo, use __cause__ to determine the error attributes
-    if hasattr(error, "__cause__") and error.__cause__:
-        error_cause = error.__cause__
-        cause_name = type(error_cause).__name__
-        if cause_name:
-            error_attributes["cause"] = cause_name
-
-        if isinstance(error_cause, GrpcCall):
-            grpc_call: GrpcCall = error_cause
-            code = grpc_call.code()
-            error_attributes["code"] = code.name
-
-    return error_attributes

dagster_cloud/opentelemetry/observers/execution_observer.py

@@ -1,178 +0,0 @@
-from abc import ABC, abstractmethod
-from contextlib import contextmanager
-from typing import Callable, Optional
-
-from dagster import DagsterError
-from dagster._time import get_current_timestamp
-
-from dagster_cloud.opentelemetry.controller import OpenTelemetryController
-from dagster_cloud.opentelemetry.observers.dagster_exception_handler import (
-    extract_dagster_error_attributes,
-)
-
-ResultEvaluatorCallback = Callable[..., str]
-
-
-class ExecutionObserverInstruments:
-    """OpenTelemetry instruments used to record generic code block executions."""
-
-    def __init__(
-        self, opentelemetry: OpenTelemetryController, metric_base_name: str, description: str
-    ):
-        self._meter = opentelemetry.get_meter(
-            name=metric_base_name,
-            attributes={
-                "description": description,
-            },
-        )
-
-        self._executions = self._meter.get_counter(
-            name=metric_base_name,
-            description=f"Number of {description}",
-        )
-
-        self._completions = self._meter.get_counter(
-            name=f"{metric_base_name}.completions",
-            description=f"Number of {description} completions by status",
-        )
-
-        self._exceptions = self._meter.get_counter(
-            name=f"{metric_base_name}.exceptions",
-            description=f"Number of exceptions caught in {description}",
-        )
-
-        self._duration = self._meter.get_histogram(
-            name=f"{metric_base_name}.duration",
-            description=f"Duration of {description}",
-            unit="seconds",
-        )
-
-    @property
-    def meter(self):
-        """Get the OpenTelemetry meter. This is useful for adding custom metrics."""
-        return self._meter
-
-    @property
-    def executions(self):
-        """Get the OpenTelemetry counter for the number of executions of the observed code block."""
-        return self._executions
-
-    @property
-    def completions(self):
-        """Get the OpenTelemetry counter for the number of completions of the observed code block."""
-        return self._completions
-
-    @property
-    def exceptions(self):
-        """Get the OpenTelemetry counter for the number of exceptions caught in the observed code block."""
-        return self._exceptions
-
-    @property
-    def duration(self):
-        """Get the OpenTelemetry gauge for the duration of the observed code block."""
-        return self._duration
-
-
-class ExecutionResultObserver(ABC):
-    @abstractmethod
-    def evaluate_result(self, *args, **kwargs):
-        raise NotImplementedError("Subclasses must implement this method")
-
-
-class _NoopExecutionResultObserver(ExecutionResultObserver):
-    """A no-op observer that does not record any metrics."""
-
-    def evaluate_result(self, *args, **kwargs):
-        pass
-
-
-class _DagsterExecutionResultObserver(ExecutionResultObserver):
-    def __init__(
-        self,
-        instruments: ExecutionObserverInstruments,
-        attributes: dict[str, str],
-        result_evaluator_callback: Optional[ResultEvaluatorCallback] = None,
-    ):
-        self._result_evaluator_callback = result_evaluator_callback
-        self._instruments = instruments
-        self._attributes = attributes or {}
-        self._status = None
-
-    @property
-    def meter(self):
-        """Get the OpenTelemetry meter. This is useful for adding custom metrics."""
-        return self._instruments.meter
-
-    def evaluate_result(self, *args, **kwargs):
-        """Evaluate the result of the observed code block.
-
-        If a result evaluator callback is present, it will be called with the arguments passed to
-        this method, as well as the attributes and instruments passed to the observer.
-
-        If no result evaluator callback is present, the status will be set to "success".
-        """
-        if self._result_evaluator_callback:
-            self._status = self._result_evaluator_callback(
-                *args, attributes=self._attributes, instruments=self._instruments, **kwargs
-            )
-        else:
-            self._status = "success"
-
-    def set_status(self, status: str):
-        self._status = status
-
-    def get_status(self) -> Optional[str]:
-        return self._status
-
-
-@contextmanager
-def observe_execution(
-    opentelemetry: Optional[OpenTelemetryController],
-    event_key: str,
-    short_description: str,
-    result_evaluator_callback: Optional[ResultEvaluatorCallback] = None,
-    attributes: Optional[dict[str, str]] = None,
-):
-    """This context manager is used to observe the execution of a code block.
-
-    It records the number of executions, completions, exceptions, and the duration of the code block.
-    An optional evaluator callback can be provided to evaluate the result of the code block,
-    setting a completion status and also optionally implementing other side effects such as
-    recording additional metrics or logging based on the results.
-    """
-    attributes = attributes or {}
-    if opentelemetry and opentelemetry.metrics_enabled:
-        instruments = ExecutionObserverInstruments(
-            opentelemetry=opentelemetry,
-            metric_base_name=event_key,
-            description=short_description,
-        )
-        instruments.executions.add(1, attributes)
-
-        start_time = get_current_timestamp()
-        observer = _DagsterExecutionResultObserver(
-            instruments=instruments,
-            attributes=attributes,
-            result_evaluator_callback=result_evaluator_callback,
-        )
-
-        try:
-            yield observer
-        except DagsterError as e:
-            err_attributes = extract_dagster_error_attributes(e)
-            observer.set_status("exception")
-            instruments.exceptions.add(1, attributes={**attributes, **err_attributes})
-            raise
-        except Exception as e:
-            err_attributes = {"exception": type(e).__name__, **attributes}
-            observer.set_status("exception")
-            instruments.exceptions.add(1, err_attributes)
-            raise
-        finally:
-            instruments.completions.add(
-                1, attributes={**attributes, "status": observer.get_status() or "unknown"}
-            )
-            instruments.duration.record(get_current_timestamp() - start_time, attributes)
-
-    else:
-        yield _NoopExecutionResultObserver()