dkist-processing-common 11.5.0__tar.gz → 11.6.0rc1__tar.gz

{dkist_processing_common-11.5.0 → dkist_processing_common-11.6.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 11.5.0
+Version: 11.6.0rc1
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -17,9 +17,9 @@ Requires-Dist: asdf<4.0.0,>=3.5.0
 Requires-Dist: astropy>=7.0.0
 Requires-Dist: dkist-fits-specifications<5.0,>=4.0.0
 Requires-Dist: dkist-header-validator<6.0,>=5.0.0
-Requires-Dist: dkist-processing-core==5.2.1
+Requires-Dist: dkist-processing-core==6.0.0rc3
 Requires-Dist: dkist-processing-pac<4.0,>=3.1
-Requires-Dist: dkist-service-configuration<3.0,>=2.0.2
+Requires-Dist: dkist-service-configuration<5.0,>=4.1.7
 Requires-Dist: dkist-spectral-lines<4.0,>=3.0.0
 Requires-Dist: solar-wavelength-calibration<2.0,>=1.0
 Requires-Dist: globus-sdk>=3.12.0
@@ -97,6 +97,84 @@ Deployment
 
 dkist-processing-common is deployed to `PyPI <https://pypi.org/project/dkist-processing-common/>`_
 
+Environment Variables
+---------------------
+
+.. list-table::
+   :widths: 10 90
+   :header-rows: 1
+
+   * - Variable
+     - Field Info
+   * - LOGURU_LEVEL
+     - annotation=str required=False default='INFO' alias_priority=2 validation_alias='LOGURU_LEVEL' description='Log level for the application'
+   * - MESH_CONFIG
+     - annotation=dict[str, MeshService] required=False default_factory=dict alias_priority=2 validation_alias='MESH_CONFIG' description='Service mesh configuration' examples=[{'upstream_service_name': {'mesh_address': 'localhost', 'mesh_port': 6742}}]
+   * - RETRY_CONFIG
+     - annotation=RetryConfig required=False default_factory=RetryConfig description='Retry configuration for the service'
+   * - OTEL_SERVICE_NAME
+     - annotation=str required=False default='unknown-service-name' alias_priority=2 validation_alias='OTEL_SERVICE_NAME' description='Service name for OpenTelemetry'
+   * - DKIST_SERVICE_VERSION
+     - annotation=str required=False default='unknown-service-version' alias_priority=2 validation_alias='DKIST_SERVICE_VERSION' description='Service version for OpenTelemetry'
+   * - NOMAD_ALLOC_ID
+     - annotation=str required=False default='unknown-allocation-id' alias_priority=2 validation_alias='NOMAD_ALLOC_ID' description='Nomad allocation ID for OpenTelemetry'
+   * - OTEL_EXPORTER_OTLP_TRACES_INSECURE
+     - annotation=bool required=False default=True description='Use insecure connection for OTLP traces'
+   * - OTEL_EXPORTER_OTLP_METRICS_INSECURE
+     - annotation=bool required=False default=True description='Use insecure connection for OTLP metrics'
+   * - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='OTLP traces endpoint. Overrides mesh configuration' examples=['localhost:4317']
+   * - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='OTLP metrics endpoint. Overrides mesh configuration' examples=['localhost:4317']
+   * - OTEL_PYTHON_DISABLED_INSTRUMENTATIONS
+     - annotation=list[str] required=False default_factory=list description='List of instrumentations to disable. https://opentelemetry.io/docs/zero-code/python/configuration/' examples=[['pika', 'requests']]
+   * - OTEL_PYTHON_FASTAPI_EXCLUDED_URLS
+     - annotation=str required=False default='health' description='Comma separated list of URLs to exclude from OpenTelemetry instrumentation in FastAPI.' examples=['client/.*/info,healthcheck']
+   * - SYSTEM_METRIC_INSTRUMENTATION_CONFIG
+     - annotation=Union[dict[str, bool], NoneType] required=False default=None description='Configuration for system metric instrumentation. https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/system_metrics/system_metrics.html' examples=[{'system.memory.usage': ['used', 'free', 'cached'], 'system.cpu.time': ['idle', 'user', 'system', 'irq'], 'system.network.io': ['transmit', 'receive'], 'process.runtime.memory': ['rss', 'vms'], 'process.runtime.cpu.time': ['user', 'system'], 'process.runtime.context_switches': ['involuntary', 'voluntary']}]
+   * - ISB_USERNAME
+     - annotation=str required=False default='guest' description='Username for the interservice-bus.'
+   * - ISB_PASSWORD
+     - annotation=str required=False default='guest' description='Password for the interservice-bus.'
+   * - ISB_EXCHANGE
+     - annotation=str required=False default='master.direct.x' description='Exchange for the interservice-bus.'
+   * - ISB_QUEUE_TYPE
+     - annotation=str required=False default='classic' description='Queue type for the interservice-bus.' examples=['quorum', 'classic']
+   * - BUILD_VERSION
+     - annotation=str required=False default='dev' description='Fallback build version for workflow tasks.'
+   * - GQL_AUTH_TOKEN
+     - annotation=Union[str, NoneType] required=False default='dev' description='The auth token for the metadata-store-api.'
+   * - OBJECT_STORE_ACCESS_KEY
+     - annotation=Union[str, NoneType] required=False default=None description='The access key for the object store.'
+   * - OBJECT_STORE_SECRET_KEY
+     - annotation=Union[str, NoneType] required=False default=None description='The secret key for the object store.'
+   * - OBJECT_STORE_USE_SSL
+     - annotation=bool required=False default=False description='Whether to use SSL for the object store connection.'
+   * - MULTIPART_THRESHOLD
+     - annotation=Union[int, NoneType] required=False default=None description='Multipart threshold for the object store.'
+   * - S3_CLIENT_CONFIG
+     - annotation=Union[dict, NoneType] required=False default=None description='S3 client configuration for the object store.'
+   * - S3_UPLOAD_CONFIG
+     - annotation=Union[dict, NoneType] required=False default=None description='S3 upload configuration for the object store.'
+   * - S3_DOWNLOAD_CONFIG
+     - annotation=Union[dict, NoneType] required=False default=None description='S3 download configuration for the object store.'
+   * - GLOBUS_TRANSPORT_PARAMS
+     - annotation=dict required=False default_factory=dict description='Globus transfer parameters.'
+   * - GLOBUS_CLIENT_ID
+     - annotation=Union[str, NoneType] required=False default=None description='Globus client ID for inbound/outbound transfers.'
+   * - GLOBUS_CLIENT_SECRET
+     - annotation=Union[str, NoneType] required=False default=None description='Globus client secret for inbound/outbound transfers.'
+   * - OBJECT_STORE_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='Object store Globus Endpoint ID.'
+   * - SCRATCH_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='Scratch Globus Endpoint ID.'
+   * - SCRATCH_BASE_PATH
+     - annotation=str required=False default='scratch/' description='Base path for scratch storage.'
+   * - SCRATCH_INVENTORY_DB_COUNT
+     - annotation=int required=False default=16 description='Number of databases in the scratch inventory (redis).'
+   * - DOCS_BASE_URL
+     - annotation=str required=False default='my_test_url' description='Base URL for the documentation site.'
+
 Development
 -----------
 

dkist_processing_common-11.6.0rc1/README.rst

@@ -0,0 +1,194 @@
+dkist-processing-common
+=======================
+
+|codecov|
+
+This repository works in concert with `dkist-processing-core <https://pypi.org/project/dkist-processing-core/>`_ and `dkist-processing-*instrument*` to
+form the DKIST calibration processing stack.
+
+Usage
+-----
+
+The classes in this repository should be used as the base of any DKIST processing pipeline tasks. Science tasks should subclass `ScienceTaskL0ToL1Base`.
+
+Each class is built on an abstract base class with the `run` method left for a developer to fill out with the required steps that the task should take.
+This class is then used as the callable object for the workflow and scheduling engine.
+
+Example
+-------
+
+.. code-block:: python
+
+    from dkist_processing_common.tasks.base import ScienceTaskL0ToL1Base
+
+
+    class RemoveArtifacts(ScienceTaskL0ToL1Base):
+        def run(self):
+            # task code here
+            total = 2 + 5
+
+Deployment
+----------
+
+dkist-processing-common is deployed to `PyPI <https://pypi.org/project/dkist-processing-common/>`_
+
+Environment Variables
+---------------------
+
+.. list-table::
+   :widths: 10 90
+   :header-rows: 1
+
+   * - Variable
+     - Field Info
+   * - LOGURU_LEVEL
+     - annotation=str required=False default='INFO' alias_priority=2 validation_alias='LOGURU_LEVEL' description='Log level for the application'
+   * - MESH_CONFIG
+     - annotation=dict[str, MeshService] required=False default_factory=dict alias_priority=2 validation_alias='MESH_CONFIG' description='Service mesh configuration' examples=[{'upstream_service_name': {'mesh_address': 'localhost', 'mesh_port': 6742}}]
+   * - RETRY_CONFIG
+     - annotation=RetryConfig required=False default_factory=RetryConfig description='Retry configuration for the service'
+   * - OTEL_SERVICE_NAME
+     - annotation=str required=False default='unknown-service-name' alias_priority=2 validation_alias='OTEL_SERVICE_NAME' description='Service name for OpenTelemetry'
+   * - DKIST_SERVICE_VERSION
+     - annotation=str required=False default='unknown-service-version' alias_priority=2 validation_alias='DKIST_SERVICE_VERSION' description='Service version for OpenTelemetry'
+   * - NOMAD_ALLOC_ID
+     - annotation=str required=False default='unknown-allocation-id' alias_priority=2 validation_alias='NOMAD_ALLOC_ID' description='Nomad allocation ID for OpenTelemetry'
+   * - OTEL_EXPORTER_OTLP_TRACES_INSECURE
+     - annotation=bool required=False default=True description='Use insecure connection for OTLP traces'
+   * - OTEL_EXPORTER_OTLP_METRICS_INSECURE
+     - annotation=bool required=False default=True description='Use insecure connection for OTLP metrics'
+   * - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='OTLP traces endpoint. Overrides mesh configuration' examples=['localhost:4317']
+   * - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='OTLP metrics endpoint. Overrides mesh configuration' examples=['localhost:4317']
+   * - OTEL_PYTHON_DISABLED_INSTRUMENTATIONS
+     - annotation=list[str] required=False default_factory=list description='List of instrumentations to disable. https://opentelemetry.io/docs/zero-code/python/configuration/' examples=[['pika', 'requests']]
+   * - OTEL_PYTHON_FASTAPI_EXCLUDED_URLS
+     - annotation=str required=False default='health' description='Comma separated list of URLs to exclude from OpenTelemetry instrumentation in FastAPI.' examples=['client/.*/info,healthcheck']
+   * - SYSTEM_METRIC_INSTRUMENTATION_CONFIG
+     - annotation=Union[dict[str, bool], NoneType] required=False default=None description='Configuration for system metric instrumentation. https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/system_metrics/system_metrics.html' examples=[{'system.memory.usage': ['used', 'free', 'cached'], 'system.cpu.time': ['idle', 'user', 'system', 'irq'], 'system.network.io': ['transmit', 'receive'], 'process.runtime.memory': ['rss', 'vms'], 'process.runtime.cpu.time': ['user', 'system'], 'process.runtime.context_switches': ['involuntary', 'voluntary']}]
+   * - ISB_USERNAME
+     - annotation=str required=False default='guest' description='Username for the interservice-bus.'
+   * - ISB_PASSWORD
+     - annotation=str required=False default='guest' description='Password for the interservice-bus.'
+   * - ISB_EXCHANGE
+     - annotation=str required=False default='master.direct.x' description='Exchange for the interservice-bus.'
+   * - ISB_QUEUE_TYPE
+     - annotation=str required=False default='classic' description='Queue type for the interservice-bus.' examples=['quorum', 'classic']
+   * - BUILD_VERSION
+     - annotation=str required=False default='dev' description='Fallback build version for workflow tasks.'
+   * - GQL_AUTH_TOKEN
+     - annotation=Union[str, NoneType] required=False default='dev' description='The auth token for the metadata-store-api.'
+   * - OBJECT_STORE_ACCESS_KEY
+     - annotation=Union[str, NoneType] required=False default=None description='The access key for the object store.'
+   * - OBJECT_STORE_SECRET_KEY
+     - annotation=Union[str, NoneType] required=False default=None description='The secret key for the object store.'
+   * - OBJECT_STORE_USE_SSL
+     - annotation=bool required=False default=False description='Whether to use SSL for the object store connection.'
+   * - MULTIPART_THRESHOLD
+     - annotation=Union[int, NoneType] required=False default=None description='Multipart threshold for the object store.'
+   * - S3_CLIENT_CONFIG
+     - annotation=Union[dict, NoneType] required=False default=None description='S3 client configuration for the object store.'
+   * - S3_UPLOAD_CONFIG
+     - annotation=Union[dict, NoneType] required=False default=None description='S3 upload configuration for the object store.'
+   * - S3_DOWNLOAD_CONFIG
+     - annotation=Union[dict, NoneType] required=False default=None description='S3 download configuration for the object store.'
+   * - GLOBUS_TRANSPORT_PARAMS
+     - annotation=dict required=False default_factory=dict description='Globus transfer parameters.'
+   * - GLOBUS_CLIENT_ID
+     - annotation=Union[str, NoneType] required=False default=None description='Globus client ID for inbound/outbound transfers.'
+   * - GLOBUS_CLIENT_SECRET
+     - annotation=Union[str, NoneType] required=False default=None description='Globus client secret for inbound/outbound transfers.'
+   * - OBJECT_STORE_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='Object store Globus Endpoint ID.'
+   * - SCRATCH_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='Scratch Globus Endpoint ID.'
+   * - SCRATCH_BASE_PATH
+     - annotation=str required=False default='scratch/' description='Base path for scratch storage.'
+   * - SCRATCH_INVENTORY_DB_COUNT
+     - annotation=int required=False default=16 description='Number of databases in the scratch inventory (redis).'
+   * - DOCS_BASE_URL
+     - annotation=str required=False default='my_test_url' description='Base URL for the documentation site.'
+
+Development
+-----------
+
+There are two prerequisites for test execution on a local machine:
+
+
+* Redis.  A running instance of redis on the local machine is required. The tests will use the default host ip of localhost and port of 6379 to connect to the database.
+
+* RabbitMQ.  A running instance of rabbitmq on the local machine is required.  The tests will use the default host of localhost and a port of 5672 to connect to the interservice bus.
+
+
+To run the tests locally, clone the repository and install the package in editable mode with the test extras.
+
+
+.. code-block:: bash
+
+    git clone git@bitbucket.org:dkistdc/dkist-processing-common.git
+    cd dkist-processing-common
+    pre-commit install
+    pip install -e .[test]
+    # Redis must be running
+    pytest -v --cov dkist_processing_common
+
+Changelog
+#########
+
+When you make **any** change to this repository it **MUST** be accompanied by a changelog file.
+The changelog for this repository uses the `towncrier <https://github.com/twisted/towncrier>`__ package.
+Entries in the changelog for the next release are added as individual files (one per change) to the ``changelog/`` directory.
+
+Writing a Changelog Entry
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A changelog entry accompanying a change should be added to the ``changelog/`` directory.
+The name of a file in this directory follows a specific template::
+
+  <PULL REQUEST NUMBER>.<TYPE>[.<COUNTER>].rst
+
+The fields have the following meanings:
+
+* ``<PULL REQUEST NUMBER>``: This is the number of the pull request, so people can jump from the changelog entry to the diff on BitBucket.
+* ``<TYPE>``: This is the type of the change and must be one of the values described below.
+* ``<COUNTER>``: This is an optional field, if you make more than one change of the same type you can append a counter to the subsequent changes, i.e. ``100.bugfix.rst`` and ``100.bugfix.1.rst`` for two bugfix changes in the same PR.
+
+The list of possible types is defined the the towncrier section of ``pyproject.toml``, the types are:
+
+* ``feature``: This change is a new code feature.
+* ``bugfix``: This is a change which fixes a bug.
+* ``doc``: A documentation change.
+* ``removal``: A deprecation or removal of public API.
+* ``misc``: Any small change which doesn't fit anywhere else, such as a change to the package infrastructure.
+
+
+Rendering the Changelog at Release Time
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When you are about to tag a release first you must run ``towncrier`` to render the changelog.
+The steps for this are as follows:
+
+* Run `towncrier build --version vx.y.z` using the version number you want to tag.
+* Agree to have towncrier remove the fragments.
+* Add and commit your changes.
+* Tag the release.
+
+**NOTE:** If you forget to add a Changelog entry to a tagged release (either manually or automatically with ``towncrier``)
+then the Bitbucket pipeline will fail. To be able to use the same tag you must delete it locally and on the remote branch:
+
+.. code-block:: bash
+
+    # First, actually update the CHANGELOG and commit the update
+    git commit
+
+    # Delete tags
+    git tag -d vWHATEVER.THE.VERSION
+    git push --delete origin vWHATEVER.THE.VERSION
+
+    # Re-tag with the same version
+    git tag vWHATEVER.THE.VERSION
+    git push --tags origin main
+
+.. |codecov| image:: https://codecov.io/bb/dkistdc/dkist-processing-common/graph/badge.svg?token=3QSLGSEF3O
+ :target: https://codecov.io/bb/dkistdc/dkist-processing-common

dkist_processing_common-11.6.0rc1/changelog/268.misc.rst

@@ -0,0 +1 @@
+Integrate dkist-processing-core 6.0.0 which brings a swap of Elastic APM to OpenTelemetry for metrics and tracing.

{dkist_processing_common-11.5.0 → dkist_processing_common-11.6.0rc1}/dkist_processing_common/config.py

@@ -3,48 +3,65 @@
 from dkist_processing_core.config import DKISTProcessingCoreConfiguration
 from dkist_service_configuration.settings import DEFAULT_MESH_SERVICE
 from dkist_service_configuration.settings import MeshService
-from pydantic import BaseModel
 from pydantic import Field
 from talus import ConnectionRetryerFactory
 from talus import ConsumerConnectionParameterFactory
 from talus import ProducerConnectionParameterFactory
 
 
-class RetryConfig(BaseModel):
-    """Retry metadata model."""
-
-    retry_delay: int = 1
-    retry_backoff: int = 2
-    retry_jitter: tuple[int, int] = (1, 10)
-    retry_max_delay: int = 300
-    retry_tries: int = -1
-
-
 class DKISTProcessingCommonConfiguration(DKISTProcessingCoreConfiguration):
     """Common configurations."""
 
-    retry_config: RetryConfig = Field(default_factory=RetryConfig)
     # metadata-store-api
-    gql_auth_token: str | None = None
+    gql_auth_token: str | None = Field(
+        default="dev", description="The auth token for the metadata-store-api."
+    )
     # object-store-api
-    object_store_access_key: str | None = None
-    object_store_secret_key: str | None = None
-    object_store_use_ssl: bool = False
-    multipart_threshold: int | None = None
-    s3_client_config: dict | None = None
-    s3_upload_config: dict | None = None
-    s3_download_config: dict | None = None
+    object_store_access_key: str | None = Field(
+        default=None, description="The access key for the object store."
+    )
+    object_store_secret_key: str | None = Field(
+        default=None, description="The secret key for the object store."
+    )
+    object_store_use_ssl: bool = Field(
+        default=False, description="Whether to use SSL for the object store connection."
+    )
+    # start object-clerk library
+    multipart_threshold: int | None = Field(
+        default=None, description="Multipart threshold for the object store."
+    )
+    s3_client_config: dict | None = Field(
+        default=None, description="S3 client configuration for the object store."
+    )
+    s3_upload_config: dict | None = Field(
+        default=None, description="S3 upload configuration for the object store."
+    )
+    s3_download_config: dict | None = Field(
+        default=None, description="S3 download configuration for the object store."
+    )
     # globus
-    globus_transport_params: dict = Field(default_factory=dict)
-    globus_client_id: str | None = None
-    globus_client_secret: str | None = None
-    object_store_endpoint: str | None = None
-    scratch_endpoint: str | None = None
+    globus_transport_params: dict = Field(
+        default_factory=dict, description="Globus transfer parameters."
+    )
+    globus_client_id: str | None = Field(
+        default=None, description="Globus client ID for inbound/outbound transfers."
+    )
+    globus_client_secret: str | None = Field(
+        default=None, description="Globus client secret for inbound/outbound transfers."
+    )
+    object_store_endpoint: str | None = Field(
+        default=None, description="Object store Globus Endpoint ID."
+    )
+    scratch_endpoint: str | None = Field(default=None, description="Scratch Globus Endpoint ID.")
     # scratch
-    scratch_base_path: str = Field(default="scratch/")
-    scratch_inventory_db_count: int = 16
+    scratch_base_path: str = Field(default="scratch/", description="Base path for scratch storage.")
+    scratch_inventory_db_count: int = Field(
+        default=16, description="Number of databases in the scratch inventory (redis)."
+    )
     # docs
-    docs_base_url: str = Field(default="my_test_url")
+    docs_base_url: str = Field(
+        default="my_test_url", description="Base URL for the documentation site."
+    )
 
     @property
     def metadata_store_api_base(self) -> str:
@@ -106,4 +123,3 @@ class DKISTProcessingCommonConfiguration(DKISTProcessingCoreConfiguration):
 
 
 common_configurations = DKISTProcessingCommonConfiguration()
-common_configurations.log_configurations()

dkist_processing_common-11.6.0rc1/dkist_processing_common/models/telemetry.py

@@ -0,0 +1,21 @@
+"""Models to support telemetry data."""
+
+from pydantic import BaseModel
+
+
+class Progress(BaseModel, validate_assignment=True):
+    """Container for tracking progress for a metering instrument."""
+
+    current: int = 0
+    total: int = 0
+
+    def increment(self, step: int = 1) -> None:
+        """Increment the current progress by the given step."""
+        self.current += step
+
+    @property
+    def percent_complete(self) -> float:
+        """Return the percent complete as a float between 0 and 100."""
+        if self.total > 0:
+            return (self.current / self.total) * 100
+        return 0.0

{dkist_processing_common-11.5.0 → dkist_processing_common-11.6.0rc1}/dkist_processing_common/tasks/assemble_movie.py

@@ -163,7 +163,7 @@ class AssembleMovie(WorkflowTaskBase, ABC):
         relative_movie_path = f"{self.constants.dataset_id}_browse_movie.mp4"
         absolute_movie_path = str(self.scratch.absolute_path(relative_movie_path))
 
-        with self.apm_processing_step("Assembling movie frames"):
+        with self.telemetry_span("Assembling movie frames"):
             clip.write_videofile(absolute_movie_path, fps=self.FPS, codec="libx264", audio=False)
 
         self.tag(path=absolute_movie_path, tags=[Tag.movie(), Tag.output()])

{dkist_processing_common-11.5.0 → dkist_processing_common-11.6.0rc1}/dkist_processing_common/tasks/base.py

@@ -10,8 +10,14 @@ from typing import Any
 from typing import Generator
 from typing import Iterable
 from typing import Type
+from typing import TypeAlias
 
 from dkist_processing_core import TaskBase
+from opentelemetry.metrics import CallbackOptions
+from opentelemetry.metrics import Counter
+from opentelemetry.metrics import ObservableGauge
+from opentelemetry.metrics import Observation
+from pydantic import BaseModel
 
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common._util.tags import TagDB
@@ -27,7 +33,7 @@ __all__ = ["WorkflowTaskBase", "tag_type_hint"]
 
 logger = logging.getLogger(__name__)
 
-tag_type_hint = Iterable[str] | str
+tag_type_hint: TypeAlias = Iterable[str] | str
 
 
 class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
@@ -66,7 +72,6 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
             workflow_name=workflow_name,
             workflow_version=workflow_version,
         )
-        self.task_name = self.__class__.__name__
         self.scratch = WorkflowFileSystem(recipe_run_id=recipe_run_id, task_name=self.task_name)
         self.constants = self.constants_model_class(
             recipe_run_id=recipe_run_id, task_name=self.task_name
@@ -76,50 +81,16 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
             recipe_run_id=recipe_run_id, task_name=self.task_name, namespace="counter"
         )
 
-    # These apm* functions provide tagged spans for APM bliss
-    def apm_type_base(
-        self,
-        name: str,
-        *args,
-        arg_span_type: str = None,
-        arg_labels: dict[str, str] = None,
-        **kwargs,
-    ):
-        """Groom inputs to apm_step to handle various kwarg collisions."""
-        if "span_type" in kwargs:
-            raise RuntimeError(
-                f"Cannot specify 'span_type' {kwargs['span_type']} in step that forces is it to be {arg_span_type}"
-            )
-
-        if "labels" in kwargs:
-            arg_labels.update(kwargs["labels"])
-            del kwargs["labels"]
-        logger.info(
-            f"Recording APM span: {name = }, {arg_span_type = }, {arg_labels = }, "
-            f"recipe_run_id = {self.recipe_run_id}"
-        )
-        return self.apm_step(name, *args, span_type=arg_span_type, labels=arg_labels, **kwargs)
-
-    def apm_task_step(self, name: str, *args, **kwargs):
-        """Span for management/organizational/info type stuff."""
-        return self.apm_type_base(
-            name, *args, arg_span_type="code.task", arg_labels={"type": "task"}, **kwargs
-        )
-
-    def apm_processing_step(self, name: str, *args, **kwargs):
-        """Span for computations."""
-        return self.apm_type_base(
-            name,
-            *args,
-            arg_span_type="code.processing",
-            arg_labels={"type": "processing"},
-            **kwargs,
+        # meter instruments
+        self.read_counter: Counter = self.meter.create_counter(
+            name=self.format_metric_name("tasks.reads"),
+            unit="1",
+            description="The number of reads executed in the processing stack.",
         )
-
-    def apm_writing_step(self, name: str, *args, **kwargs):
-        """Span for writing to disk."""
-        return self.apm_type_base(
-            name, *args, arg_span_type="code.writing", arg_labels={"type": "writing"}, **kwargs
+        self.write_counter: Counter = self.meter.create_counter(
+            name=self.format_metric_name("tasks.writes"),
+            unit="1",
+            description="The number of writes executed in the processing stack.",
         )
 
     @property
@@ -161,7 +132,7 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
         """Execute any pre-task setup required."""
         super().pre_run()
         if self.record_provenance or self.is_task_manual:
-            with self.apm_task_step("Record Provenance"):
+            with self.telemetry_span("Record Provenance"):
                 self._record_provenance()
 
     def read(
@@ -183,7 +154,9 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
         **decoder_kwargs
             Additional arguments to pass to the `decoder` function.
         """
-        return (decoder(p, **decoder_kwargs) for p in self.scratch.find_all(tags=tags))
+        for p in self.scratch.find_all(tags=tags):
+            self.read_counter.add(amount=1, attributes=self.base_telemetry_attributes)
+            yield decoder(p, **decoder_kwargs)
 
     def write(
         self,
@@ -221,6 +194,7 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
         -------
         The path for the written file
         """
+        self.write_counter.add(amount=1, attributes=self.base_telemetry_attributes)
         file_obj = encoder(data, **encoder_kwargs)
         if isinstance(tags, str):
             tags = [tags]
@@ -372,11 +346,11 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
         Filename Counter: not rolled back but its purpose of preventing file name collisions is not impacted
         """
         super().rollback()
-        with self.apm_writing_step("Rollback Scratch"):
+        with self.telemetry_span("Rollback Scratch"):
             self.scratch.rollback()
-        with self.apm_writing_step("Rollback Constants"):
+        with self.telemetry_span("Rollback Constants"):
             self.constants._rollback()
-        with self.apm_task_step("Change Recipe Run to Inprogress"):
+        with self.telemetry_span("Change Recipe Run to Inprogress"):
             self.metadata_store_change_recipe_run_to_inprogress()
 
     def __exit__(self, exc_type, exc_val, exc_tb):

{dkist_processing_common-11.5.0 → dkist_processing_common-11.6.0rc1}/dkist_processing_common/tasks/l1_output_data.py

@@ -54,11 +54,11 @@ class TransferL1Data(TransferDataBase, GlobusMixin):
 
     def transfer_objects(self):
         """Transfer movie and L1 output frames."""
-        with self.apm_task_step("Upload movie"):
+        with self.telemetry_span("Upload movie"):
             # Movie needs to be transferred separately as the movie headers need to go with it
             self.transfer_movie()
 
-        with self.apm_task_step("Upload science frames"):
+        with self.telemetry_span("Upload science frames"):
             self.transfer_output_frames()
 
     def transfer_output_frames(self):
@@ -120,10 +120,10 @@ class AssembleQualityData(L1OutputDataBase, QualityMixin):
 
     def run(self):
         """Run method for the task."""
-        with self.apm_processing_step("Assembling quality data"):
+        with self.telemetry_span("Assembling quality data"):
             quality_data = self.quality_assemble_data(polcal_label_list=self.polcal_label_list)
 
-        with self.apm_writing_step(
+        with self.telemetry_span(
             f"Saving quality data with {len(quality_data)} metrics to the file system"
         ):
             self.write(
@@ -146,7 +146,7 @@ class SubmitDatasetMetadata(L1OutputDataBase):
 
     def run(self) -> None:
         """Run method for this task."""
-        with self.apm_writing_step(f"Storing quality data to metadata store"):
+        with self.telemetry_span(f"Storing quality data to metadata store"):
             # each quality_data file is a list - this will combine the elements of multiple lists into a single list
             quality_data = list(
                 chain.from_iterable(
@@ -156,7 +156,7 @@ class SubmitDatasetMetadata(L1OutputDataBase):
             self.metadata_store_add_quality_data(
                 dataset_id=self.constants.dataset_id, quality_data=quality_data
             )
-        with self.apm_processing_step("Count Expected Outputs"):
+        with self.telemetry_span("Count Expected Outputs"):
             dataset_id = self.constants.dataset_id
             expected_object_count = self.count(tags=Tag.output())
             if quality_data:
@@ -165,7 +165,7 @@ class SubmitDatasetMetadata(L1OutputDataBase):
             f"Adding Dataset Receipt Account: "
             f"{dataset_id=}, {expected_object_count=}, recipe_run_id={self.recipe_run_id}"
         )
-        with self.apm_task_step(
+        with self.telemetry_span(
             f"Add Dataset Receipt Account: {dataset_id = }, {expected_object_count = }"
         ):
             self.metadata_store_add_dataset_receipt_account(
@@ -245,10 +245,10 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
 
     def run(self) -> None:
         """Run method for this task."""
-        with self.apm_task_step("Gather output data"):
+        with self.telemetry_span("Gather output data"):
             frames = self.read(tags=self.output_frame_tags)
             movies = self.read(tags=[Tag.output(), Tag.movie()])
-        with self.apm_task_step("Create message objects"):
+        with self.telemetry_span("Create message objects"):
             messages = []
             messages += self.frame_messages(paths=frames)
             frame_message_count = len(messages)
@@ -257,7 +257,7 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
             dataset_has_quality_data = self.dataset_has_quality_data
             if dataset_has_quality_data:
                 messages.append(self.quality_report_message)
-        with self.apm_task_step(
+        with self.telemetry_span(
             f"Publish messages: {frame_message_count = }, {object_message_count = }, {dataset_has_quality_data = }"
         ):
             self.interservice_bus_publish(messages=messages)

{dkist_processing_common-11.5.0 → dkist_processing_common-11.6.0rc1}/dkist_processing_common/tasks/output_data_base.py

@@ -58,10 +58,10 @@ class TransferDataBase(OutputDataBase, ObjectStoreMixin, ABC):
 
     def run(self) -> None:
         """Transfer the data and cleanup any folders."""
-        with self.apm_task_step("Transfer objects"):
+        with self.telemetry_span("Transfer objects"):
             self.transfer_objects()
 
-        with self.apm_task_step("Remove folder objects"):
+        with self.telemetry_span("Remove folder objects"):
             self.remove_folder_objects()
 
     @abstractmethod