dkist-processing-core 5.2.1__tar.gz → 6.0.0__tar.gz

{dkist_processing_core-5.2.1 → dkist_processing_core-6.0.0}/CHANGELOG.rst

@@ -1,3 +1,12 @@
+v6.0.0 (2025-09-26)
+===================
+
+Misc
+----
+
+- Swap out Elastic APM for OpenTelemetry tracing and metrics. (`#56 <https://bitbucket.org/dkistdc/dkist-processing-core/pull-requests/56>`__)
+
+
 v5.2.1 (2025-09-08)
 ===================
 

{dkist_processing_core-5.2.1 → dkist_processing_core-6.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-core
-Version: 5.2.1
+Version: 6.0.0
 Summary: Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -14,12 +14,11 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.12
 Description-Content-Type: text/x-rst
 Requires-Dist: apache-airflow[celery,postgres]==2.11.0
-Requires-Dist: elastic-apm<7.0.0
 Requires-Dist: requests>=2.23
-Requires-Dist: talus<2.0,>=1.1.0
+Requires-Dist: talus<2.0,>=1.3.4
 Requires-Dist: pendulum
 Requires-Dist: nbformat>=5.9.2
-Requires-Dist: dkist-service-configuration<3.0,>=2.0.2
+Requires-Dist: dkist-service-configuration<5.0,>=4.1.7
 Requires-Dist: pydantic>2.0
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
@@ -128,53 +127,47 @@ Environment Variables
 ---------------------
 
 .. list-table::
-   :widths: 10 70 10 10
+   :widths: 10 90
    :header-rows: 1
 
    * - Variable
-     - Description
-     - Type
-     - Default
-   * - BUILD_VERSION
-     - Build/Export pipelines only.  This is the value that will be appended to all artifacts and represents their unique version
-     - STR
-     - dev
+     - Field Info
+   * - LOGURU_LEVEL
+     - annotation=str required=False default='INFO' alias_priority=2 validation_alias='LOGURU_LEVEL' description='Log level for the application'
    * - MESH_CONFIG
-     - Provides the dkistdc cloud mesh configuration.  Specifically the location of the message broker
-     - JSON
-     - ``{}``
+     - annotation=dict[str, MeshService] required=False default_factory=dict alias_priority=2 validation_alias='MESH_CONFIG' description='Service mesh configuration' examples=[{'upstream_service_name': {'mesh_address': 'localhost', 'mesh_port': 6742}}]
+   * - RETRY_CONFIG
+     - annotation=RetryConfig required=False default_factory=RetryConfig description='Retry configuration for the service'
+   * - OTEL_SERVICE_NAME
+     - annotation=str required=False default='unknown-service-name' alias_priority=2 validation_alias='OTEL_SERVICE_NAME' description='Service name for OpenTelemetry'
+   * - DKIST_SERVICE_VERSION
+     - annotation=str required=False default='unknown-service-version' alias_priority=2 validation_alias='DKIST_SERVICE_VERSION' description='Service version for OpenTelemetry'
+   * - NOMAD_ALLOC_ID
+     - annotation=str required=False default='unknown-allocation-id' alias_priority=2 validation_alias='NOMAD_ALLOC_ID' description='Nomad allocation ID for OpenTelemetry'
+   * - OTEL_EXPORTER_OTLP_TRACES_INSECURE
+     - annotation=bool required=False default=True description='Use insecure connection for OTLP traces'
+   * - OTEL_EXPORTER_OTLP_METRICS_INSECURE
+     - annotation=bool required=False default=True description='Use insecure connection for OTLP metrics'
+   * - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='OTLP traces endpoint. Overrides mesh configuration' examples=['localhost:4317']
+   * - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='OTLP metrics endpoint. Overrides mesh configuration' examples=['localhost:4317']
+   * - OTEL_PYTHON_DISABLED_INSTRUMENTATIONS
+     - annotation=list[str] required=False default_factory=list description='List of instrumentations to disable. https://opentelemetry.io/docs/zero-code/python/configuration/' examples=[['pika', 'requests']]
+   * - OTEL_PYTHON_FASTAPI_EXCLUDED_URLS
+     - annotation=str required=False default='health' description='Comma separated list of URLs to exclude from OpenTelemetry instrumentation in FastAPI.' examples=['client/.*/info,healthcheck']
+   * - SYSTEM_METRIC_INSTRUMENTATION_CONFIG
+     - annotation=Union[dict[str, bool], NoneType] required=False default=None description='Configuration for system metric instrumentation. https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/system_metrics/system_metrics.html' examples=[{'system.memory.usage': ['used', 'free', 'cached'], 'system.cpu.time': ['idle', 'user', 'system', 'irq'], 'system.network.io': ['transmit', 'receive'], 'process.runtime.memory': ['rss', 'vms'], 'process.runtime.cpu.time': ['user', 'system'], 'process.runtime.context_switches': ['involuntary', 'voluntary']}]
    * - ISB_USERNAME
-     - Message broker user name
-     - STR
-     - guest
+     - annotation=str required=False default='guest' description='Username for the interservice-bus.'
    * - ISB_PASSWORD
-     - Message broker password
-     - STR
-     - guest
+     - annotation=str required=False default='guest' description='Password for the interservice-bus.'
    * - ISB_EXCHANGE
-     - Message Broker Exchange name for publishing messages
-     - STR
-     - master.direct.x
+     - annotation=str required=False default='master.direct.x' description='Exchange for the interservice-bus.'
    * - ISB_QUEUE_TYPE
-     - Message Broker queue type for transporting messages
-     - STR
-     - classic
-   * - ELASTIC_APM_SERVICE_NAME
-     - Service Name used by Elastic Application Performance Monitoring
-     - STR
-     -
-   * - ELASTIC_APM_OTHER_OPTIONS
-     - Dictionary of configuration for the Elastic Application Performance Monitoring client
-     - STR
-     - ``{}``
-   * - ELASTIC_APM_ENABLED
-     - Flag to disable/enable Elastic Application Performance Monitoring client calls which are chatty if not connected to an APM server.
-     - BOOL
-     - FALSE
+     - annotation=str required=False default='classic' description='Queue type for the interservice-bus.' examples=['quorum', 'classic']
    * - BUILD_VERSION
-     - Version of the pipeline.  When built this makes its way into the workflow or dag name.
-     - STR
-     - dev
+     - annotation=str required=False default='dev' description='Fallback build version for workflow tasks.'
 
 Development
 -----------

{dkist_processing_core-5.2.1 → dkist_processing_core-6.0.0}/README.rst

@@ -86,53 +86,47 @@ Environment Variables
 ---------------------
 
 .. list-table::
-   :widths: 10 70 10 10
+   :widths: 10 90
    :header-rows: 1
 
    * - Variable
-     - Description
-     - Type
-     - Default
-   * - BUILD_VERSION
-     - Build/Export pipelines only.  This is the value that will be appended to all artifacts and represents their unique version
-     - STR
-     - dev
+     - Field Info
+   * - LOGURU_LEVEL
+     - annotation=str required=False default='INFO' alias_priority=2 validation_alias='LOGURU_LEVEL' description='Log level for the application'
    * - MESH_CONFIG
-     - Provides the dkistdc cloud mesh configuration.  Specifically the location of the message broker
-     - JSON
-     - ``{}``
+     - annotation=dict[str, MeshService] required=False default_factory=dict alias_priority=2 validation_alias='MESH_CONFIG' description='Service mesh configuration' examples=[{'upstream_service_name': {'mesh_address': 'localhost', 'mesh_port': 6742}}]
+   * - RETRY_CONFIG
+     - annotation=RetryConfig required=False default_factory=RetryConfig description='Retry configuration for the service'
+   * - OTEL_SERVICE_NAME
+     - annotation=str required=False default='unknown-service-name' alias_priority=2 validation_alias='OTEL_SERVICE_NAME' description='Service name for OpenTelemetry'
+   * - DKIST_SERVICE_VERSION
+     - annotation=str required=False default='unknown-service-version' alias_priority=2 validation_alias='DKIST_SERVICE_VERSION' description='Service version for OpenTelemetry'
+   * - NOMAD_ALLOC_ID
+     - annotation=str required=False default='unknown-allocation-id' alias_priority=2 validation_alias='NOMAD_ALLOC_ID' description='Nomad allocation ID for OpenTelemetry'
+   * - OTEL_EXPORTER_OTLP_TRACES_INSECURE
+     - annotation=bool required=False default=True description='Use insecure connection for OTLP traces'
+   * - OTEL_EXPORTER_OTLP_METRICS_INSECURE
+     - annotation=bool required=False default=True description='Use insecure connection for OTLP metrics'
+   * - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='OTLP traces endpoint. Overrides mesh configuration' examples=['localhost:4317']
+   * - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='OTLP metrics endpoint. Overrides mesh configuration' examples=['localhost:4317']
+   * - OTEL_PYTHON_DISABLED_INSTRUMENTATIONS
+     - annotation=list[str] required=False default_factory=list description='List of instrumentations to disable. https://opentelemetry.io/docs/zero-code/python/configuration/' examples=[['pika', 'requests']]
+   * - OTEL_PYTHON_FASTAPI_EXCLUDED_URLS
+     - annotation=str required=False default='health' description='Comma separated list of URLs to exclude from OpenTelemetry instrumentation in FastAPI.' examples=['client/.*/info,healthcheck']
+   * - SYSTEM_METRIC_INSTRUMENTATION_CONFIG
+     - annotation=Union[dict[str, bool], NoneType] required=False default=None description='Configuration for system metric instrumentation. https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/system_metrics/system_metrics.html' examples=[{'system.memory.usage': ['used', 'free', 'cached'], 'system.cpu.time': ['idle', 'user', 'system', 'irq'], 'system.network.io': ['transmit', 'receive'], 'process.runtime.memory': ['rss', 'vms'], 'process.runtime.cpu.time': ['user', 'system'], 'process.runtime.context_switches': ['involuntary', 'voluntary']}]
    * - ISB_USERNAME
-     - Message broker user name
-     - STR
-     - guest
+     - annotation=str required=False default='guest' description='Username for the interservice-bus.'
    * - ISB_PASSWORD
-     - Message broker password
-     - STR
-     - guest
+     - annotation=str required=False default='guest' description='Password for the interservice-bus.'
    * - ISB_EXCHANGE
-     - Message Broker Exchange name for publishing messages
-     - STR
-     - master.direct.x
+     - annotation=str required=False default='master.direct.x' description='Exchange for the interservice-bus.'
    * - ISB_QUEUE_TYPE
-     - Message Broker queue type for transporting messages
-     - STR
-     - classic
-   * - ELASTIC_APM_SERVICE_NAME
-     - Service Name used by Elastic Application Performance Monitoring
-     - STR
-     -
-   * - ELASTIC_APM_OTHER_OPTIONS
-     - Dictionary of configuration for the Elastic Application Performance Monitoring client
-     - STR
-     - ``{}``
-   * - ELASTIC_APM_ENABLED
-     - Flag to disable/enable Elastic Application Performance Monitoring client calls which are chatty if not connected to an APM server.
-     - BOOL
-     - FALSE
+     - annotation=str required=False default='classic' description='Queue type for the interservice-bus.' examples=['quorum', 'classic']
    * - BUILD_VERSION
-     - Version of the pipeline.  When built this makes its way into the workflow or dag name.
-     - STR
-     - dev
+     - annotation=str required=False default='dev' description='Fallback build version for workflow tasks.'
 
 Development
 -----------

{dkist_processing_core-5.2.1 → dkist_processing_core-6.0.0}/dkist_processing_core/config.py

@@ -1,24 +1,30 @@
 """Environment controlled configurations for dkist_processing_core."""
 
-from dkist_service_configuration import MeshServiceConfigurationBase
+from dkist_service_configuration import InstrumentedMeshServiceConfigurationBase
 from dkist_service_configuration.settings import MeshService
+from opentelemetry.sdk.resources import Resource
 from pydantic import Field
 from talus import ConnectionRetryerFactory
 from talus import Exchange
 from talus.models.connection_parameters import ConnectionParameterFactory
 
 
-class DKISTProcessingCoreConfiguration(MeshServiceConfigurationBase):
+class DKISTProcessingCoreConfiguration(InstrumentedMeshServiceConfigurationBase):
     """Environment configurations for dkist_processing_core."""
 
-    isb_username: str = Field(default="guest")
-    isb_password: str = Field(default="guest")
-    isb_exchange: str = Field(default="master.direct.x")
-    isb_queue_type: str = Field(default="classic")
-    elastic_apm_service_name: str = Field(default="dkist-processing-core")
-    elastic_apm_other_options: dict = Field(default_factory=dict)
-    elastic_apm_enabled: bool = False
-    build_version: str = Field(default="dev")
+    isb_username: str = Field(default="guest", description="Username for the interservice-bus.")
+    isb_password: str = Field(default="guest", description="Password for the interservice-bus.")
+    isb_exchange: str = Field(
+        default="master.direct.x", description="Exchange for the interservice-bus."
+    )
+    isb_queue_type: str = Field(
+        default="classic",
+        description="Queue type for the interservice-bus.",
+        examples=["quorum", "classic"],
+    )
+    build_version: str = Field(
+        default="dev", description="Fallback build version for workflow tasks."
+    )
 
     @property
     def isb_mesh_service(self) -> MeshService:
@@ -36,7 +42,7 @@ class DKISTProcessingCoreConfiguration(MeshServiceConfigurationBase):
             rabbitmq_port=self.isb_mesh_service.port,
             rabbitmq_user=self.isb_username,
             rabbitmq_pass=self.isb_password,
-            connection_name="dkist-processing-core-producer",
+            connection_name=f"{self.service_name}-producer",
         )
 
     @property
@@ -64,20 +70,13 @@ class DKISTProcessingCoreConfiguration(MeshServiceConfigurationBase):
         return Exchange(name=self.isb_exchange)
 
     @property
-    def elastic_apm_server_url(self) -> str:
-        """Return the URL for the Elastic APM server."""
-        apm_server = self.service_mesh_detail(service_name="system-monitoring-log-apm")
-        return f"http://{apm_server.host}:{apm_server.port}/"
-
-    @property
-    def apm_config(self) -> dict:
-        """Return the configuration for the Elastic APM."""
-        return {
-            "SERVICE_NAME": self.elastic_apm_service_name,
-            "SERVER_URL": self.elastic_apm_server_url,
-            "ENVIRONMENT": "Workflows",
-            **self.elastic_apm_other_options,
-        }
+    def otel_resource(self) -> Resource:
+        """Open Telemetry resource attributes."""
+        old = super().otel_resource
+        updates = Resource(attributes={"service.name.alias": "dkist-processing"})
+        new = old.merge(updates)
+        return new
 
 
 core_configurations = DKISTProcessingCoreConfiguration()
+core_configurations.auto_instrument()

{dkist_processing_core-5.2.1 → dkist_processing_core-6.0.0}/dkist_processing_core/failure_callback.py

@@ -3,6 +3,7 @@
 import logging
 from contextlib import contextmanager
 from typing import Callable
+from typing import Generator
 from typing import Type
 
 from talus import Binding
@@ -35,7 +36,7 @@ class RecipeRunFailureMessage(PublishMessageBase):
 
 
 @contextmanager
-def recipe_run_failure_message_producer_factory() -> DurableProducer:
+def recipe_run_failure_message_producer_factory() -> Generator[DurableProducer, None, None]:
     """Create message producer for recipe run failure messages."""
     # Configure the queue the messages should be routed to
     recipe_run_failure_queue = Queue(

dkist_processing_core-6.0.0/dkist_processing_core/task.py

@@ -0,0 +1,258 @@
+"""
+Base class that is used to wrap the various DAG task methods.
+
+It provides support for user-defined setup and cleanup, task monitoring using OpenTelemetry,
+standardized logging, and exception handling.
+"""
+
+import logging
+from abc import ABC
+from abc import abstractmethod
+from contextlib import contextmanager
+from typing import Generator
+from typing import Sequence
+
+from opentelemetry.context.context import Context
+from opentelemetry.metrics import Counter
+from opentelemetry.metrics import Meter
+from opentelemetry.trace import Link
+from opentelemetry.trace import Span
+from opentelemetry.trace import SpanKind
+from opentelemetry.trace import StatusCode
+from opentelemetry.trace import Tracer
+from opentelemetry.util.types import Attributes
+
+from dkist_processing_core.config import core_configurations
+
+__all__ = ["TaskBase"]
+
+logger = logging.getLogger(__name__)
+
+
+class TaskBase(ABC):
+    """
+    A Task is the interface between processing code and its execution.  Processing code can follow this interface through subclassing remain agnostic to the execution environment.
+
+    Each DAG task must implement its own subclass of this abstract wrapper class.
+
+    Intended instantiation is as a context manager
+
+    >>> class RealTask(TaskBase):
+    >>>     def run(self):
+    >>>         pass
+    >>>
+    >>> with RealTask(1, "a", "b") as task:
+    >>>     task()
+
+    Task names in airflow are the same as the class name
+    Additional methods can be added but will only be called if they are referenced via run,
+    pre_run, post_run, or __exit__
+
+    overriding methods other than run, pre_run, post_run, and in special cases __exit__ is
+    discouraged as they are used internally to support the abstraction.
+    e.g. __init__ is called by the core api without user involvement so adding parameters will not
+    result in them being passed in as there is no client interface to __init__.
+
+    To use the tracing infrastructure in subclass code one would do the following:
+
+    >>> def foo(self):
+    >>>     with self.telemetry_span("do detailed work"):
+    >>>         pass # do work
+
+    Parameters
+    ----------
+    recipe_run_id : int
+        id of the recipe run used to identify the workflow run this task is part of
+    workflow_name : str
+        name of the workflow to which this instance of the task belongs
+    workflow_version : str
+        version of the workflow to which this instance of the task belongs
+
+    """
+
+    retries = 0
+    retry_delay_seconds = 60
+    tracer: Tracer = core_configurations.tracer
+    meter: Meter = core_configurations.meter
+
+    def __init__(
+        self,
+        recipe_run_id: int,
+        workflow_name: str,
+        workflow_version: str,
+    ):
+        """
+        Instantiate a Task.
+
+        The details of instantiation may vary based upon the export target but this signature is what is expected by the instantiation transformation (Node) code.
+        """
+        self.recipe_run_id = int(recipe_run_id)
+        self.workflow_name = workflow_name
+        self.workflow_version = workflow_version
+        self.task_name = self.__class__.__name__
+
+        self.base_telemetry_attributes = {
+            "recipe.run.id": self.recipe_run_id,
+            "workflow.name": self.workflow_name,
+            "workflow.version": self.workflow_version,
+            "task.name": self.task_name,
+        }
+
+        # meter instruments
+        self.task_execution_counter: Counter = self.meter.create_counter(
+            name=self.format_metric_name("tasks.executed"),
+            unit="1",
+            description="The number of tasks executed in the processing stack.",
+        )
+
+        logger.info(f"Task {self.task_name} initialized")
+
+    @contextmanager
+    def telemetry_span(
+        self,
+        name: str,
+        context: Context | None = None,
+        kind: SpanKind = SpanKind.INTERNAL,
+        attributes: Attributes = None,
+        links: Sequence[Link] | None = None,
+        start_time: int | None = None,
+        record_exception: bool = True,
+        set_status_on_exception: bool = True,
+        end_on_exit: bool = True,
+    ) -> Generator[Span, None, None]:  # noqa: D405,D407
+        """
+        Context manager for creating a new span and set it as the current span in this tracer's context.
+
+        Parameters
+        ----------
+        name
+            The name of the span to be created.
+
+        context
+            An optional Context containing the span's parent. Defaults to the global context.
+
+        kind
+            The span's kind (relationship to parent). Note that is meaningful even if there is no parent.
+
+        attributes
+            The span's attributes.
+
+        links
+            Links span to other spans
+
+        start_time
+            Sets the start time of a span
+
+        record_exception
+            Whether to record any exceptions raised within the context as error event on the span.
+
+        set_status_on_exception
+            Only relevant if the returned span is used in a with/context manager. Defines whether the span status will
+            be automatically set to ERROR when an uncaught exception is raised in the span with block. The span status
+            won't be set by this mechanism if it was previously set manually.
+
+        end_on_exit
+            Whether to end the span automatically when leaving the context manager.
+
+        Yields
+        ------
+        The newly-created span.
+        """
+        with self.tracer.start_as_current_span(
+            name=name,
+            context=context,
+            kind=kind,
+            attributes=attributes,
+            links=links,
+            start_time=start_time,
+            record_exception=record_exception,
+            set_status_on_exception=set_status_on_exception,
+            end_on_exit=end_on_exit,
+        ) as span:
+            span.set_attributes(self.base_telemetry_attributes)
+            yield span
+            span.set_status(StatusCode.OK)
+
+    def format_metric_name(self, name: str) -> str:
+        """
+        Format the metric name to include the meter name and a namespace of 'processing' for dkist-processing-* meters. Words are separated by a dot.
+
+        For example, if the meter name is "dkist.meter" and the metric name is "tasks.executed",
+        the formatted name will be "dkist.meter.processing.tasks.executed".
+        """
+        return f"{self.meter.name}.processing.{name}"
+
+    def pre_run(self) -> None:
+        """Intended to be overridden and will execute prior to run() with Open Telemetry trace span capturing."""
+
+    @abstractmethod
+    def run(self) -> None:
+        """Abstract method that must be overridden to execute the desired DAG task with Open Telemetry trace span capturing."""
+
+    def post_run(self) -> None:
+        """Intended to be overridden and will execute after run() with Open Telemetry trace span capturing."""
+
+    def rollback(self) -> None:
+        """Rollback any changes to persistent stores performed by the task."""
+
+    def __call__(self) -> None:
+        """
+        DAG task wrapper. Execution is instrumented with Open Telemetry tracing if configured.
+
+        The standard execution sequence is:
+
+        1 run
+
+        2 record provenance
+
+        Returns
+        -------
+        None
+
+        """
+        verbose_task_name = f"{self.workflow_name}.{self.task_name}"
+        logger.info(f"{verbose_task_name} started")
+
+        self.task_execution_counter.add(amount=1, attributes=self.base_telemetry_attributes)
+
+        with self.telemetry_span(f"{verbose_task_name}"):  # Root Span
+            with self.telemetry_span("Pre Run"):
+                self.pre_run()
+            with self.telemetry_span("Run"):
+                self.run()
+            with self.telemetry_span("Post Run"):
+                self.post_run()
+        logger.info(f"{verbose_task_name} complete")
+
+    def __enter__(self):
+        """
+        Override to execute setup tasks before task execution.
+
+        Only override this method with tasks that need to happen
+        regardless of tasks having an exception, ensure that no additional exception
+        will be raised, and always call super().__enter__
+        """
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """
+        Override to execute teardown tasks after task execution regardless of task execution success.
+
+        Only override this method with tasks that need to happen
+        regardless of tasks having an exception, ensure that no additional exception
+        will be raised, and always call super().__exit__
+        """
+
+    def __repr__(self):
+        """Return the representation of the task."""
+        return (
+            f"{self.__class__.__name__}("
+            f"recipe_run_id={self.recipe_run_id}, "
+            f"workflow_name={self.workflow_name}, "
+            f"workflow_version={self.workflow_version}, "
+            f")"
+        )
+
+    def __str__(self):
+        """Return a string representation of the task."""
+        return repr(self)

{dkist_processing_core-5.2.1 → dkist_processing_core-6.0.0}/dkist_processing_core/tests/conftest.py

@@ -27,7 +27,7 @@ def export_path() -> str:
 
 @pytest.fixture(scope="session")
 def task_subclass():
-    """Sub class of the abstract task base class implementing methods that are expected to be subclassed with inspect-able metadata."""
+    """Subclass of the abstract task base class implementing methods that are expected to be subclassed with inspect-able metadata."""
     return Task
 
 

{dkist_processing_core-5.2.1 → dkist_processing_core-6.0.0}/dkist_processing_core/tests/test_task.py

@@ -2,22 +2,12 @@
 
 import pytest
 
-from dkist_processing_core.config import core_configurations
 from dkist_processing_core.task import TaskBase
 
 
-@pytest.fixture(
-    params=[pytest.param(True, id="apm_enabled"), pytest.param(False, id="apm_disabled")]
-)
-def apm_enabled(request, mocker):
-    mocked_config = core_configurations.model_copy()
-    mocked_config.elastic_apm_enabled = request.param
-    mocker.patch("dkist_processing_core.task.core_configurations", mocked_config)
-
-
-def test_task_execution(task_subclass, apm_enabled):
+def test_task_execution(task_subclass):
     """
-    Given: Task subclass and parametrized APM configurations.
+    Given: Task subclass.
     When: calling the instance.
     Then: the run method is executed.
     """
@@ -30,7 +20,7 @@ def test_task_execution(task_subclass, apm_enabled):
 
 def test_task_run_failure(error_task_subclass):
     """
-    Given: Task subclass and parametrized APM configurations.
+    Given: Task subclass.
     When: calling the instance.
     Then: the run method is executed.
     """

{dkist_processing_core-5.2.1 → dkist_processing_core-6.0.0}/dkist_processing_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-core
-Version: 5.2.1
+Version: 6.0.0
 Summary: Abstraction layer used by the DKIST science data processing pipelines with Apache Airflow
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -14,12 +14,11 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.12
 Description-Content-Type: text/x-rst
 Requires-Dist: apache-airflow[celery,postgres]==2.11.0
-Requires-Dist: elastic-apm<7.0.0
 Requires-Dist: requests>=2.23
-Requires-Dist: talus<2.0,>=1.1.0
+Requires-Dist: talus<2.0,>=1.3.4
 Requires-Dist: pendulum
 Requires-Dist: nbformat>=5.9.2
-Requires-Dist: dkist-service-configuration<3.0,>=2.0.2
+Requires-Dist: dkist-service-configuration<5.0,>=4.1.7
 Requires-Dist: pydantic>2.0
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
@@ -128,53 +127,47 @@ Environment Variables
 ---------------------
 
 .. list-table::
-   :widths: 10 70 10 10
+   :widths: 10 90
    :header-rows: 1
 
    * - Variable
-     - Description
-     - Type
-     - Default
-   * - BUILD_VERSION
-     - Build/Export pipelines only.  This is the value that will be appended to all artifacts and represents their unique version
-     - STR
-     - dev
+     - Field Info
+   * - LOGURU_LEVEL
+     - annotation=str required=False default='INFO' alias_priority=2 validation_alias='LOGURU_LEVEL' description='Log level for the application'
    * - MESH_CONFIG
-     - Provides the dkistdc cloud mesh configuration.  Specifically the location of the message broker
-     - JSON
-     - ``{}``
+     - annotation=dict[str, MeshService] required=False default_factory=dict alias_priority=2 validation_alias='MESH_CONFIG' description='Service mesh configuration' examples=[{'upstream_service_name': {'mesh_address': 'localhost', 'mesh_port': 6742}}]
+   * - RETRY_CONFIG
+     - annotation=RetryConfig required=False default_factory=RetryConfig description='Retry configuration for the service'
+   * - OTEL_SERVICE_NAME
+     - annotation=str required=False default='unknown-service-name' alias_priority=2 validation_alias='OTEL_SERVICE_NAME' description='Service name for OpenTelemetry'
+   * - DKIST_SERVICE_VERSION
+     - annotation=str required=False default='unknown-service-version' alias_priority=2 validation_alias='DKIST_SERVICE_VERSION' description='Service version for OpenTelemetry'
+   * - NOMAD_ALLOC_ID
+     - annotation=str required=False default='unknown-allocation-id' alias_priority=2 validation_alias='NOMAD_ALLOC_ID' description='Nomad allocation ID for OpenTelemetry'
+   * - OTEL_EXPORTER_OTLP_TRACES_INSECURE
+     - annotation=bool required=False default=True description='Use insecure connection for OTLP traces'
+   * - OTEL_EXPORTER_OTLP_METRICS_INSECURE
+     - annotation=bool required=False default=True description='Use insecure connection for OTLP metrics'
+   * - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='OTLP traces endpoint. Overrides mesh configuration' examples=['localhost:4317']
+   * - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='OTLP metrics endpoint. Overrides mesh configuration' examples=['localhost:4317']
+   * - OTEL_PYTHON_DISABLED_INSTRUMENTATIONS
+     - annotation=list[str] required=False default_factory=list description='List of instrumentations to disable. https://opentelemetry.io/docs/zero-code/python/configuration/' examples=[['pika', 'requests']]
+   * - OTEL_PYTHON_FASTAPI_EXCLUDED_URLS
+     - annotation=str required=False default='health' description='Comma separated list of URLs to exclude from OpenTelemetry instrumentation in FastAPI.' examples=['client/.*/info,healthcheck']
+   * - SYSTEM_METRIC_INSTRUMENTATION_CONFIG
+     - annotation=Union[dict[str, bool], NoneType] required=False default=None description='Configuration for system metric instrumentation. https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/system_metrics/system_metrics.html' examples=[{'system.memory.usage': ['used', 'free', 'cached'], 'system.cpu.time': ['idle', 'user', 'system', 'irq'], 'system.network.io': ['transmit', 'receive'], 'process.runtime.memory': ['rss', 'vms'], 'process.runtime.cpu.time': ['user', 'system'], 'process.runtime.context_switches': ['involuntary', 'voluntary']}]
    * - ISB_USERNAME
-     - Message broker user name
-     - STR
-     - guest
+     - annotation=str required=False default='guest' description='Username for the interservice-bus.'
    * - ISB_PASSWORD
-     - Message broker password
-     - STR
-     - guest
+     - annotation=str required=False default='guest' description='Password for the interservice-bus.'
    * - ISB_EXCHANGE
-     - Message Broker Exchange name for publishing messages
-     - STR
-     - master.direct.x
+     - annotation=str required=False default='master.direct.x' description='Exchange for the interservice-bus.'
    * - ISB_QUEUE_TYPE
-     - Message Broker queue type for transporting messages
-     - STR
-     - classic
-   * - ELASTIC_APM_SERVICE_NAME
-     - Service Name used by Elastic Application Performance Monitoring
-     - STR
-     -
-   * - ELASTIC_APM_OTHER_OPTIONS
-     - Dictionary of configuration for the Elastic Application Performance Monitoring client
-     - STR
-     - ``{}``
-   * - ELASTIC_APM_ENABLED
-     - Flag to disable/enable Elastic Application Performance Monitoring client calls which are chatty if not connected to an APM server.
-     - BOOL
-     - FALSE
+     - annotation=str required=False default='classic' description='Queue type for the interservice-bus.' examples=['quorum', 'classic']
    * - BUILD_VERSION
-     - Version of the pipeline.  When built this makes its way into the workflow or dag name.
-     - STR
-     - dev
+     - annotation=str required=False default='dev' description='Fallback build version for workflow tasks.'
 
 Development
 -----------

{dkist_processing_core-5.2.1 → dkist_processing_core-6.0.0}/dkist_processing_core.egg-info/requires.txt

@@ -1,10 +1,9 @@
 apache-airflow[celery,postgres]==2.11.0
-elastic-apm<7.0.0
 requests>=2.23
-talus<2.0,>=1.1.0
+talus<2.0,>=1.3.4
 pendulum
 nbformat>=5.9.2
-dkist-service-configuration<3.0,>=2.0.2
+dkist-service-configuration<5.0,>=4.1.7
 pydantic>2.0
 
 [docs]

{dkist_processing_core-5.2.1 → dkist_processing_core-6.0.0}/docs/conf.py

@@ -33,9 +33,15 @@ autoapi_keep_files = True
 
 # -- Options for intersphinx extension -----------------------------------------
 intersphinx_mapping = {
+    # Official Python docs
     "python": (
         "https://docs.python.org/3/",
-        (None, "http://www.astropy.org/astropy-data/intersphinx/python3.inv"),
+        "https://docs.python.org/3/objects.inv",
+    ),
+    # OpenTelemetry (Python)
+    "opentelemetry": (
+        "https://opentelemetry-python.readthedocs.io/en/stable/",
+        "https://opentelemetry-python.readthedocs.io/en/stable/objects.inv",
     ),
 }
 # Remaining sphinx settings are in dkist-sphinx-theme conf.py

{dkist_processing_core-5.2.1 → dkist_processing_core-6.0.0}/pyproject.toml

@@ -22,12 +22,11 @@ classifiers = [
 ]
 dependencies = [
   "apache-airflow[postgres, celery] == 2.11.0",
-  "elastic-apm < 7.0.0",
   "requests >= 2.23",
-  "talus >= 1.1.0, <2.0",
+  "talus >= 1.3.4, <2.0",
   "pendulum",
   "nbformat >= 5.9.2",
-  "dkist-service-configuration >=2.0.2, <3.0",
+  "dkist-service-configuration >=4.1.7, <5.0",
   "pydantic > 2.0",
 ]
 

dkist_processing_core-5.2.1/dkist_processing_core/task.py

@@ -1,250 +0,0 @@
-"""
-Base class that is used to wrap the various DAG task methods.
-
-It provides support for user-defined setup and cleanup, task monitoring using Elastic APM,
-standardized logging and exception handling.
-"""
-
-import logging
-from abc import ABC
-from abc import abstractmethod
-from contextlib import contextmanager
-
-import elasticapm
-
-from dkist_processing_core.config import core_configurations
-
-__all__ = ["TaskBase"]
-
-logger = logging.getLogger(__name__)
-
-
-class ApmTransaction:
-    """
-    Elastic APM transaction manager for a DAG Task.
-
-    Without configuration, it disables itself.
-    """
-
-    @property
-    def apm_service_name(self) -> str:
-        """Format the service name for Elastic APM."""
-        name = f"{self._workflow_name}-{self._workflow_version}"
-        name = name.replace("_", "-")
-        name = name.replace(".", "-")
-        return name
-
-    @property
-    def apm_config(self) -> dict:
-        """Override the Elastic APM configuration with the workflow specific service name."""
-        core_config = core_configurations.apm_config
-        core_config["SERVICE_NAME"] = self.apm_service_name
-        return core_config
-
-    def __init__(self, transaction_name: str, workflow_name: str, workflow_version: str) -> None:
-        self._workflow_name = workflow_name
-        self._workflow_version = workflow_version
-        self.transaction_name = transaction_name
-
-        if core_configurations.elastic_apm_enabled:
-            self.client = elasticapm.Client(self.apm_config)
-            self.instrument()
-            self.client.begin_transaction(transaction_type="Task")
-            logger.info(f"APM Configured: {self=} {self.apm_config=}")
-        else:
-            logger.warning(f"APM Not Configured")
-
-    @contextmanager
-    def capture_span(self, name: str, *args, **kwargs):
-        if core_configurations.elastic_apm_enabled:
-            try:
-                with elasticapm.capture_span(name, *args, **kwargs):
-                    yield
-            finally:
-                pass
-        else:
-            try:
-                yield
-            finally:
-                pass
-
-    def close(self, exc_type=None):
-        if core_configurations.elastic_apm_enabled:
-            result = "Success"
-            if exc_type is not None:
-                result = "Error"  # pragma: no cover
-                self.client.capture_exception(handled=False)  # pragma: no cover
-            self.client.end_transaction(name=self.transaction_name, result=result)
-            self.client.close()
-
-    @staticmethod
-    def instrument():
-        """Vendored implementation of elasticapm.instrumentation.control.instrument changed to omit certain frameworks."""
-        omit_frameworks = {
-            "elasticapm.instrumentation.packages.redis.RedisInstrumentation",
-            "elasticapm.instrumentation.packages.redis.RedisPipelineInstrumentation",
-            "elasticapm.instrumentation.packages.redis.RedisConnectionInstrumentation",
-            "elasticapm.instrumentation.packages.asyncio.aioredis.RedisConnectionPoolInstrumentation",
-            "elasticapm.instrumentation.packages.asyncio.aioredis.RedisPipelineInstrumentation",
-            "elasticapm.instrumentation.packages.asyncio.aioredis.RedisConnectionInstrumentation",
-        }
-
-        from elasticapm.instrumentation.control import _lock
-        from elasticapm.instrumentation.register import _cls_register
-        from elasticapm.instrumentation.register import _instrumentation_singletons
-        from elasticapm.instrumentation.register import import_string
-
-        # from elasticapm.instrumentation.control.instrument
-        with _lock:
-            # update to vendored code
-            filtered_cls_register = _cls_register.difference(omit_frameworks)
-            # from elasticapm.instrumentation.register.get_instrumentation_objects
-            for cls_str in filtered_cls_register:
-                if cls_str not in _instrumentation_singletons:
-                    cls = import_string(cls_str)
-                    _instrumentation_singletons[cls_str] = cls()
-                obj = _instrumentation_singletons[cls_str]
-                # from elasticapm.instrumentation.control.instrument
-                obj.instrument()
-
-    def __repr__(self):
-        return f"{self.__class__.__name__}(transaction_name={self.transaction_name}, workflow_name={self._workflow_name}, workflow_version={self._workflow_version})"
-
-
-class TaskBase(ABC):
-    """
-    A Task is the interface between processing code and its execution.  Processing code can follow this interface through subclassing remain agnostic to the execution environment.
-
-    Each DAG task must implement its own subclass of this abstract wrapper class.
-
-    Intended instantiation is as a context manager
-
-    >>> class RealTask(TaskBase):
-    >>>     def run(self):
-    >>>         pass
-    >>>
-    >>> with RealTask(1, "a", "b") as task:
-    >>>     task()
-
-    Task names in airflow are the same as the class name
-    Additional methods can be added but will only be called if they are referenced via run,
-    pre_run, post_run, or __exit__
-
-    overriding methods other than run, pre_run, post_run, and in special cases __exit__ is
-    discouraged as they are used internally to support the abstraction.
-    e.g. __init__ is called by the core api without user involvement so adding parameters will not
-    result in them being passed in as there is no client interface to __init__.
-
-    To use the apm infrastructure in subclass code one would do the following:
-
-    >>> def foo(self):
-    >>>     with self.apm_step("do detailed work"):
-    >>>         pass # do work
-
-    Parameters
-    ----------
-    recipe_run_id : int
-        id of the recipe run used to identify the workflow run this task is part of
-    workflow_name : str
-        name of the workflow to which this instance of the task belongs
-    workflow_version : str
-        version of the workflow to which this instance of the task belongs
-
-    """
-
-    retries = 0
-    retry_delay_seconds = 60
-
-    def __init__(
-        self,
-        recipe_run_id: int,
-        workflow_name: str,
-        workflow_version: str,
-    ):
-        """
-        Instantiate a Task.
-
-        The details of instantiation may vary based upon the export target but this signature is what is expected by the intantiation transformation (Node) code.
-        """
-        self.recipe_run_id = int(recipe_run_id)
-        self.workflow_name = workflow_name
-        self.workflow_version = workflow_version
-        self.task_name = self.__class__.__name__
-        logger.info(f"Task {self.task_name} initialized")
-        self.apm = ApmTransaction(
-            transaction_name=self.task_name,
-            workflow_name=self.workflow_name,
-            workflow_version=self.workflow_version,
-        )
-        self.apm_step = self.apm.capture_span  # abbreviated syntax for capture span context mgr
-
-    def pre_run(self) -> None:
-        """Intended to be overridden and will execute prior to run() with Elastic APM span capturing."""
-
-    @abstractmethod
-    def run(self) -> None:
-        """Abstract method that must be overridden to execute the desired DAG task."""
-
-    def post_run(self) -> None:
-        """Intended to be overridden and will execute after run() with Elastic APM span capturing."""
-
-    def rollback(self) -> None:
-        """Rollback any changes to persistent stores performed by the task."""
-
-    def __call__(self) -> None:
-        """
-        DAG task wrapper. Execution is instrumented with Application Performance Monitoring if configured.
-
-        The standard execution sequence is:
-
-        1 run
-
-        2 record provenance
-
-        Returns
-        -------
-        None
-
-        """
-        logger.info(f"Task {self.task_name} started")
-        with self.apm_step("Pre Run", span_type="code.core", labels={"type": "core"}):
-            self.pre_run()
-        with self.apm_step("Run", span_type="code.core", labels={"type": "core"}):
-            self.run()
-        with self.apm_step("Post Run", span_type="code.core", labels={"type": "core"}):
-            self.post_run()
-        logger.info(f"Task {self.task_name} complete")
-
-    def __enter__(self):
-        """
-        Override to execute setup tasks before task execution.
-
-        Only override this method with tasks that need to happen
-        regardless of tasks having an exception, ensure that no additional exception
-        will be raised, and always call super().__enter__
-        """
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        """
-        Override to execute teardown tasks after task execution regardless of task execution success.
-
-        Only override this method with tasks that need to happen
-        regardless of tasks having an exception, ensure that no additional exception
-        will be raised, and always call super().__exit__
-        """
-        self.apm.close(exc_type=exc_type)
-
-    def __repr__(self):
-        """Return the representation of the task."""
-        return (
-            f"{self.__class__.__name__}("
-            f"recipe_run_id={self.recipe_run_id}, "
-            f"workflow_name={self.workflow_name}, "
-            f"workflow_version={self.workflow_version}, "
-            f")"
-        )
-
-    def __str__(self):
-        """Return a string representation of the task."""
-        return repr(self)