dagster-cloud 1.12.10__py3-none-any.whl → 1.12.12__py3-none-any.whl

dagster_cloud/opentelemetry/controller.py

@@ -1,319 +0,0 @@
-import logging
-from collections.abc import Mapping
-from threading import Lock
-from typing import TYPE_CHECKING, Any, Optional
-
-import dagster_cloud.opentelemetry.factories.logs as logs_factory
-import dagster_cloud.opentelemetry.factories.metrics as metrics_factory
-from dagster_cloud.opentelemetry.metrics.meter import Meter
-
-if TYPE_CHECKING:
-    from opentelemetry._logs import LoggerProvider as APILoggerProvider
-    from opentelemetry.metrics import MeterProvider as APIMeterProvider
-    from opentelemetry.sdk._logs import LogRecordProcessor
-    from opentelemetry.sdk._logs.export import LogExporter
-    from opentelemetry.sdk.metrics.export import MetricExporter, MetricReader
-
-
-DEFAULT_SHUTDOWN_TIMEOUT_MILLIS = 5_000
-
-
-class OpenTelemetryController:
-    """The OpenTelemetryController manages the initialization and shutdown of OpenTelemetry components.
-
-    Provided with a configuration, the OpenTelemetryController will initialize the logging and metrics components.
-    The python logging will be instrumented with OpenTelemetry and captured and exported.
-
-    The get_meter() method allow to obtain a meter than can create metric instruments.
-    See: https://opentelemetry.io/docs/languages/python/instrumentation/#metrics
-
-    Note: As some OpenTelemetry pipeline components run their own thread, it's important to shut down properly.
-
-    Examples:
-        .. code-block:: python
-            import logging
-            import sys
-            import time
-
-            from dagster_cloud.opentelemetry.controller import OpenTelemetryController
-
-            otel_config = {
-                "enabled": True,
-                "logging": {
-                    "enabled": True,
-                    "exporter": {
-                        "type": "ConsoleLogExporter",
-                    },
-                },
-                "metrics": {
-                    "enabled": True,
-                    "reader": {
-                        "type": "PeriodicExportingMetricReader",
-                        "params": {
-                            # this is the interval at which metrics are captured and exported
-                            "export_interval_millis": 800,
-                        },
-                    },
-                    "exporter": {
-                        "type": "ConsoleMetricExporter",
-                    },
-                },
-            }
-
-
-            def main():
-                logging.basicConfig(level=logging.INFO, stream=sys.stdout)
-                logger = logging.getLogger("my_logger")
-
-                logger.info("OpenTelemetry is not started. This log will not be captured by OpenTelemetry.")
-                opentelemetry = OpenTelemetryController(
-                    instance_id="my_instance",
-                    version="0.0.1",
-                    config=otel_config,
-                    logger=logger,  # this and its children will be instrumented with OpenTelemetry
-                )
-                logger.info("OpenTelemetry started. This log will be exported by OpenTelemetry.")
-                meter = opentelemetry.get_meter(name="item_processor", attributes={"environment": "dev"})
-                # the counter should be captured as a metric
-                counter = meter.create_counter(
-                    name="items_processed", description="The number of items processed.", unit="unit"
-                )
-
-                # simulate doing some work, and emitting metrics that will be exported by OpenTelemetry
-                counter.add(1)
-                time.sleep(1)
-                counter.add(1)
-                time.sleep(1)
-
-                opentelemetry.dispose()
-                logger.info("OpenTelemetry stopped. This log will not be captured by OpenTelemetry.")
-
-
-            if __name__ == "__main__":
-                main()
-    """
-
-    def __init__(
-        self,
-        instance_id: str,
-        version: str,
-        config: Optional[Mapping[str, Any]] = None,
-        logger: Optional[logging.Logger] = None,
-    ):
-        """Initialize the telemetry instance.
-        params:
-            instance_id: str: The unique identifier of the service instance.
-            version: str: The version of the service.
-            config: dict: The configuration for the telemetry components.
-            logger: Logger: The logger to use for telemetry logs. Defaults to the root logger.
-        """
-        self._shutdown = False
-
-        self._log_exporter: Optional[LogExporter] = None
-        self._logger_provider: APILoggerProvider = logs_factory.build_noop_logger_provider()
-        self._log_record_processor: Optional[LogRecordProcessor] = None
-        self._logging_handler: Optional[logging.Handler] = None
-
-        self._metric_exporter: Optional[MetricExporter] = None
-        self._metric_reader: Optional[MetricReader] = None
-        self._meter_provider: APIMeterProvider = metrics_factory.build_noop_meter_provider()
-        self._meters: dict[str, Meter] = {}
-
-        self._config = config or {}
-
-        service_name = self._config["service_name"]
-
-        self._logger = logger or logging.getLogger()
-        self._resource_attributes = {
-            "service.name": service_name,
-            "service.version": version,
-            "service.instance.id": instance_id,
-        }
-        self._lock = Lock()
-        self._initialize()
-
-    def _initialize(self):
-        """Initialize the OpenTelemetry components."""
-        # an initialized controller can be used in code without having to check if it's
-        # enabled as the components will default to NoOp implementations if disabled
-        if not self.enabled:
-            self._logger.debug(
-                "OpenTelemetry is disabled by configuration. Telemetry will not be captured."
-            )
-            return
-
-        self._initialize_logging()
-        self._initialize_metrics()
-        # TODO - self._initialize_tracing()
-
-    @property
-    def enabled(self):
-        """Check if the OpenTelemetry instance is enabled. If not enabled,
-        all the components will default to NoOp implementations.
-        """
-        if not self._config:
-            return False
-
-        if not self._config.get("enabled", False):
-            return False
-
-        # If no specific opentelemetry components are enabled, then we can effectively disable opentelemetry
-        return any(
-            [
-                self._config.get("metrics", {}).get("enabled", False),
-                self._config.get("logging", {}).get("enabled", False),
-                self._config.get("tracing", {}).get("enabled", False),
-            ]
-        )
-
-    @property
-    def active(self) -> bool:
-        """Check if the OpenTelemetry instance is active.
-
-        Returns: bool: True if the instance is active, otherwise False.
-        """
-        return not self._shutdown
-
-    @property
-    def logging_enabled(self) -> bool:
-        """Check if the logging component is enabled."""
-        return self.enabled and self._config.get("logging", {}).get("enabled", False)
-
-    @property
-    def metrics_enabled(self) -> bool:
-        """Check if the metrics component is enabled."""
-        return self.enabled and self._config.get("metrics", {}).get("enabled", False)
-
-    @property
-    def tracing_enabled(self) -> bool:
-        """Check if the tracing component is enabled."""
-        return self.enabled and self._config.get("tracing", {}).get("enabled", False)
-
-    def dispose(self):
-        """Shutdown the OpenTelemetry components."""
-        with self._lock:
-            if self._shutdown:
-                self._logger.warning("OpenTelemetry: Already shutdown, skipping.")
-                return
-
-            # TODO - self._shutdown_tracing()
-            self._shutdown_metrics()
-            self._shutdown_logging()
-            self._shutdown = True
-
-    def get_meter(
-        self,
-        name: str,
-        version: Optional[str] = None,
-        schema_url: Optional[str] = None,
-        attributes: Optional[Mapping[str, Any]] = None,
-    ) -> Meter:
-        """Get or create a meter.
-
-        params:
-            name: str: The name of the meter.
-            version: str: The version of the meter.
-            schema_url: str: The schema URL of the meter.
-            attributes: dict: The attributes of the meter.
-
-        Returns: Meter: The meter instance.
-        """
-        if self._shutdown:
-            raise RuntimeError("OpenTelemetry: Cannot get meter, already shutdown.")
-
-        with self._lock:
-            # guarding with a lock because concurrent calls to get_meter
-            # could attempt to create multiple identical meters
-            if name not in self._meters:
-                meter = self._meter_provider.get_meter(
-                    name=name,
-                    version=version,
-                    schema_url=schema_url,
-                    attributes=attributes,
-                )
-                self._meters[name] = Meter(meter)
-            return self._meters[name]
-
-    def _initialize_logging(self):
-        logging_config = self._config.get("logging", {})
-        if not logging_config.get("enabled"):
-            self._logger.info(
-                "OpenTelemetry: Logging instrumentation is disabled by configuration."
-            )
-            return
-
-        self._log_exporter = logs_factory.build_log_exporter(logging_config.get("exporter"))  # pyright: ignore[reportArgumentType]
-        self._log_record_processor = logs_factory.build_log_record_processor(
-            self._log_exporter, logging_config.get("processor", {})
-        )
-        self._logger_provider = logs_factory.build_logger_provider(
-            self._log_record_processor, self._resource_attributes
-        )
-        self._logging_handler = logs_factory.build_logging_handler(
-            self._logger_provider,  # pyright: ignore[reportArgumentType]
-            logging_config.get("handler", {}),
-        )
-        self._logger.addHandler(self._logging_handler)
-
-    def _initialize_metrics(self):
-        metrics_config = self._config.get("metrics", {})
-        if not metrics_config.get("enabled"):
-            self._logger.info("OpenTelemetry: Metric instrumentation is disabled by configuration.")
-            return
-
-        self._metric_exporter = metrics_factory.build_metric_exporter(
-            metrics_config.get("exporter", {})
-        )
-
-        self._metric_reader = metrics_factory.build_metric_reader(
-            self._metric_exporter, metrics_config.get("reader", {})
-        )
-        self._meter_provider = metrics_factory.build_meter_provider(
-            [self._metric_reader], self._resource_attributes
-        )
-
-    def _initialize_tracing(self):
-        tracing_config = self._config.get("tracing", {})
-        if not tracing_config.get("enabled"):
-            self._logger.info(
-                "OpenTelemetry: Tracing instrumentation is disabled by configuration."
-            )
-            return
-        raise NotImplementedError("Tracing instrumentation is not yet implemented.")
-
-    def _shutdown_logging(self):
-        if self._logger and self._logging_handler:
-            self._logging_handler.flush()
-            self._logger.removeHandler(self._logging_handler)
-
-        # The LoggerProvider will shut down the record processor, which in turns handles the exporter.
-        if self._logger_provider:
-            # The NoOpLoggerProvider does not have a force_flush or shutdown method.
-            if _validate_object_has_method(self._logger_provider, "force_flush"):
-                self._logger_provider.force_flush(timeout_millis=DEFAULT_SHUTDOWN_TIMEOUT_MILLIS)  # type: ignore
-            if _validate_object_has_method(self._logger_provider, "shutdown"):
-                self._logger_provider.shutdown()  # pyright: ignore[reportAttributeAccessIssue]
-
-    def _shutdown_metrics(self):
-        # The meter provider will shut down the reader and the PeriodicExportingMetricReader will handle the exporter.
-        # Note: the InMemoryMetricReader will not shut down the exporter, it should only be used for testing and
-        # does not accept an exporter and one should not exist.
-        if self._meter_provider:
-            # Duck typing: The base interface and the NoOpMeterProvider do not have a force_flush or shutdown method.
-            if _validate_object_has_method(self._meter_provider, "force_flush"):
-                self._meter_provider.force_flush(timeout_millis=DEFAULT_SHUTDOWN_TIMEOUT_MILLIS)  # type: ignore
-            if _validate_object_has_method(self._meter_provider, "shutdown"):
-                self._meter_provider.shutdown()  # type: ignore
-        self._meters = {}
-
-    def _shutdown_tracing(self):
-        #  TODO - Tracing instrumentation is not yet implemented.
-        pass
-
-
-def _validate_object_has_method(obj, method_name) -> bool:
-    """Check if an object has a callable method.
-
-    Some OpenTelemetry components do not have some methods, so we need to check if they exist before calling them.
-    """
-    return hasattr(obj, method_name) and callable(getattr(obj, method_name))

dagster_cloud/opentelemetry/enum.py

@@ -1,58 +0,0 @@
-from enum import Enum
-
-
-class CompressionEnum(Enum):
-    GZIP = "gzip"
-    DEFLATE = "deflate"
-
-
-class LoggingExporterEnum(Enum):
-    ConsoleLogExporter = "ConsoleLogExporter"
-    HttpProtoOTLPExporter = "HttpProtoOTLPExporter"
-    GrpcProtoOTLPExporter = "GrpcProtoOTLPExporter"
-
-
-class LogRecordProcessorEnum(Enum):
-    BatchLogRecordProcessor = "BatchLogRecordProcessor"
-    SimpleLogRecordProcessor = "SimpleLogRecordProcessor"
-
-
-class MetricsExporterEnum(Enum):
-    ConsoleMetricExporter = "ConsoleMetricExporter"
-    HttpProtoOTLPExporter = "HttpProtoOTLPExporter"
-    GrpcProtoOTLPExporter = "GrpcProtoOTLPExporter"
-
-
-class MetricsReaderEnum(Enum):
-    PeriodicExportingMetricReader = "PeriodicExportingMetricReader"
-    InMemoryMetricReader = "InMemoryMetricReader"
-
-
-class TracingExporterEnum(Enum):
-    HttpProtoOTLPExporter = "HttpProtoOTLPExporter"
-    GrpcProtoOTLPExporter = "GrpcProtoOTLPExporter"
-
-
-class MetricsInstrumentTypesEnum(Enum):
-    Counter = "Counter"
-    UpDownCounter = "UpDownCounter"
-    Histogram = "Histogram"
-    Gauge = "Gauge"
-    ObservableCounter = "ObservableCounter"
-    ObservableUpDownCounter = "ObservableUpDownCounter"
-    ObservableGauge = "ObservableGauge"
-
-
-class AggregationTemporalityEnum(Enum):
-    Cumulative = "Cumulative"
-    Delta = "Delta"
-    Unspecified = "Unspecified"
-
-
-class ViewAggregationEnum(Enum):
-    DefaultAggregation = "DefaultAggregation"
-    DropAggregation = "DropAggregation"
-    ExplicitBucketHistogramAggregation = "ExplicitBucketHistogramAggregation"
-    ExponentialHistogramAggregation = "ExponentialHistogramAggregation"
-    LastValueAggregation = "LastValueAggregation"
-    SumAggregation = "SumAggregation"

dagster_cloud/opentelemetry/factories/__init__.py

@@ -1 +0,0 @@
-

dagster_cloud/opentelemetry/factories/logs.py

@@ -1,113 +0,0 @@
-from typing import Optional
-
-from opentelemetry._logs import LoggerProvider as APILoggerProvider
-from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler, LogRecordProcessor
-from opentelemetry.sdk._logs.export import LogExporter
-from opentelemetry.sdk.resources import Resource
-
-from dagster_cloud.opentelemetry.enum import LoggingExporterEnum, LogRecordProcessorEnum
-
-
-def build_log_exporter(exporter_config: dict) -> LogExporter:
-    """Build an OpenTelemetry log exporter.
-    params:
-        exporter_config: dict: The exporter configuration.
-
-    Returns:
-        LogExporter: The log exporter instance.
-    """
-    exporter_type = exporter_config.get("type", LoggingExporterEnum.ConsoleLogExporter.value)
-    exporter_params = exporter_config.get("params", {})
-
-    if exporter_type == LoggingExporterEnum.HttpProtoOTLPExporter.value:
-        from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
-
-        return OTLPLogExporter(**exporter_params)
-    elif exporter_type == LoggingExporterEnum.GrpcProtoOTLPExporter.value:
-        from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
-
-        return OTLPLogExporter(**exporter_params)
-    elif exporter_type == LoggingExporterEnum.ConsoleLogExporter.value:
-        from opentelemetry.sdk._logs.export import ConsoleLogExporter
-
-        return ConsoleLogExporter(**exporter_params)
-
-    raise ValueError(f"Invalid exporter type: {exporter_type}")
-
-
-def build_log_record_processor(exporter: LogExporter, processor_config: dict) -> LogRecordProcessor:
-    """Build an OpenTelemetry log record processor.
-    params:
-        processor_config: dict: The processor configuration.
-
-    Returns:
-        LogRecordProcessor: The log record processor instance.
-    """
-    processor_type = processor_config.get(
-        "type", LogRecordProcessorEnum.SimpleLogRecordProcessor.value
-    )
-    processor_params = processor_config.get("params", {})
-
-    if processor_type == LogRecordProcessorEnum.BatchLogRecordProcessor.value:
-        from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
-
-        return BatchLogRecordProcessor(exporter, **processor_params)
-    elif processor_type == LogRecordProcessorEnum.SimpleLogRecordProcessor.value:
-        from opentelemetry.sdk._logs.export import SimpleLogRecordProcessor
-
-        return SimpleLogRecordProcessor(exporter)
-
-    raise ValueError(f"Invalid log record processor type: {processor_type}")
-
-
-def build_logger_provider(
-    processor: LogRecordProcessor,
-    resource_attributes: Optional[dict[str, str]] = None,
-) -> APILoggerProvider:
-    """Build an OpenTelemetry logger provider.
-    params:
-        processor: LogRecordProcessor: The log record processor instance.
-        resource: Resource (optional): The resource instance.
-
-    Returns:
-        LoggerProvider: The logger provider instance.
-    """
-    from opentelemetry.sdk._logs import LoggerProvider
-
-    resource = (
-        Resource.create(attributes=resource_attributes)
-        if resource_attributes
-        else Resource.create({})
-    )
-
-    logger_provider = LoggerProvider(
-        resource=resource,
-        shutdown_on_exit=False,
-    )
-    logger_provider.add_log_record_processor(processor)
-    return logger_provider
-
-
-def build_logging_handler(
-    logger_provider: LoggerProvider, handler_config: Optional[dict] = None
-) -> LoggingHandler:
-    """Build an OpenTelemetry logging handler.
-    Params:
-        logger_provider (LoggerProvider): The logger provider instance.
-        handler_config (Optional[dict]): The handler configuration.
-
-    Returns:
-        LoggingHandler: The logging handler instance.
-    """
-    from opentelemetry.sdk._logs import LoggingHandler
-
-    handler_config = handler_config or {}
-
-    return LoggingHandler(logger_provider=logger_provider, **handler_config)
-
-
-def build_noop_logger_provider() -> APILoggerProvider:
-    """Build a no-op OpenTelemetry meter provider."""
-    from opentelemetry._logs import NoOpLoggerProvider
-
-    return NoOpLoggerProvider()

dagster_cloud/opentelemetry/factories/metrics.py

@@ -1,121 +0,0 @@
-from typing import Optional
-
-from opentelemetry.metrics import MeterProvider as APIMeterProvider
-from opentelemetry.sdk.metrics.export import MetricExporter, MetricReader
-
-from dagster_cloud.opentelemetry.config.exporter import metrics_instrument_types_enum
-from dagster_cloud.opentelemetry.enum import MetricsExporterEnum, MetricsReaderEnum
-
-
-def _validate_dict_keys_as_metric_instruments(src: dict, dict_name: str) -> None:
-    """Validate that all keys in a dictionary are valid metric instrument types."""
-    for key in src:
-        if key not in metrics_instrument_types_enum.enum_values:
-            raise ValueError(
-                f"OpenTelemetry configuration error - Invalid instrument type '{key}' in {dict_name}."
-            )
-
-
-def build_metric_exporter(exporter_config: dict) -> MetricExporter:
-    """Build an OpenTelemetry metric exporter.
-    params:
-        exporter: dict: The exporter configuration.
-          - type: str: The exporter type.
-          - params: dict: The exporter parameters (varies).
-
-    Returns:
-        MetricExporter: The metric exporter instance.
-    """
-    exporter_type = exporter_config.get("type", MetricsExporterEnum.ConsoleMetricExporter.value)
-    exporter_params = exporter_config.get("params", {})
-
-    if "preferred_temporality" in exporter_params:
-        _validate_dict_keys_as_metric_instruments(
-            exporter_params["preferred_temporality"], "preferred_temporality"
-        )
-
-    if "preferred_aggregation" in exporter_params:
-        _validate_dict_keys_as_metric_instruments(
-            exporter_params["preferred_aggregation"], "preferred_aggregation"
-        )
-
-    if exporter_type == MetricsExporterEnum.HttpProtoOTLPExporter.value:
-        from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
-
-        return OTLPMetricExporter(**exporter_params)
-    elif exporter_type == MetricsExporterEnum.GrpcProtoOTLPExporter.value:
-        from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
-
-        return OTLPMetricExporter(**exporter_params)
-    elif exporter_type == MetricsExporterEnum.ConsoleMetricExporter.value:
-        from opentelemetry.sdk.metrics.export import ConsoleMetricExporter
-
-        return ConsoleMetricExporter(**exporter_params)
-
-    raise ValueError(f"Invalid exporter type: {exporter_type}")
-
-
-def build_metric_reader(
-    exporter: Optional[MetricExporter] = None, reader_config: Optional[dict] = None
-) -> MetricReader:
-    """Build an OpenTelemetry metric reader.
-    params:
-        reader: dict: The reader configuration.
-
-    Returns:
-        MetricReader: The metric reader instance.
-    """
-    reader_config = reader_config or {}
-    reader_type = reader_config.get("type", MetricsReaderEnum.PeriodicExportingMetricReader.value)
-    reader_params = reader_config.get("params", {})
-
-    if reader_type == MetricsReaderEnum.PeriodicExportingMetricReader.value:
-        from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
-
-        if exporter is None:
-            raise ValueError("Exporter is required for PeriodicExportingMetricReader.")
-        return PeriodicExportingMetricReader(exporter=exporter, **reader_params)
-    elif reader_type == MetricsReaderEnum.InMemoryMetricReader.value:
-        from opentelemetry.sdk.metrics.export import InMemoryMetricReader
-
-        return InMemoryMetricReader(**reader_params)
-
-    raise ValueError(f"Invalid metric reader type: {reader_type}")
-
-
-def build_meter_provider(
-    metric_readers: list[MetricReader],
-    resource_attributes: Optional[dict[str, str]] = None,
-) -> APIMeterProvider:
-    """Build an OpenTelemetry meter provider.
-    params:
-        metric_reader: MetricReader: The metric reader instance.
-        resource: Resource (optional): The resource instance.
-
-    Returns:
-        MeterProvider: The meter provider instance.
-    """
-    from opentelemetry.sdk.metrics import MeterProvider
-    from opentelemetry.sdk.resources import Resource
-
-    resource = (
-        Resource.create(attributes=resource_attributes)
-        if resource_attributes
-        else Resource.create({})
-    )
-
-    return MeterProvider(
-        metric_readers=metric_readers,
-        resource=resource,
-        shutdown_on_exit=False,
-        # TODO - implement support for views?
-        # https://opentelemetry-python.readthedocs.io/en/latest/_modules/opentelemetry/sdk/metrics/_internal/view.html#View
-        # views=views,
-    )
-
-
-def build_noop_meter_provider() -> APIMeterProvider:
-    """Build a no-op OpenTelemetry meter provider."""
-    from opentelemetry.metrics import NoOpMeterProvider
-
-    return NoOpMeterProvider()