dkist-processing-common 11.5.0rc1__py3-none-any.whl → 11.6.0__py3-none-any.whl

dkist_processing_common/config.py

@@ -3,48 +3,65 @@
 from dkist_processing_core.config import DKISTProcessingCoreConfiguration
 from dkist_service_configuration.settings import DEFAULT_MESH_SERVICE
 from dkist_service_configuration.settings import MeshService
-from pydantic import BaseModel
 from pydantic import Field
 from talus import ConnectionRetryerFactory
 from talus import ConsumerConnectionParameterFactory
 from talus import ProducerConnectionParameterFactory
 
 
-class RetryConfig(BaseModel):
-    """Retry metadata model."""
-
-    retry_delay: int = 1
-    retry_backoff: int = 2
-    retry_jitter: tuple[int, int] = (1, 10)
-    retry_max_delay: int = 300
-    retry_tries: int = -1
-
-
 class DKISTProcessingCommonConfiguration(DKISTProcessingCoreConfiguration):
     """Common configurations."""
 
-    retry_config: RetryConfig = Field(default_factory=RetryConfig)
     # metadata-store-api
-    gql_auth_token: str | None = None
+    gql_auth_token: str | None = Field(
+        default="dev", description="The auth token for the metadata-store-api."
+    )
     # object-store-api
-    object_store_access_key: str | None = None
-    object_store_secret_key: str | None = None
-    object_store_use_ssl: bool = False
-    multipart_threshold: int | None = None
-    s3_client_config: dict | None = None
-    s3_upload_config: dict | None = None
-    s3_download_config: dict | None = None
+    object_store_access_key: str | None = Field(
+        default=None, description="The access key for the object store."
+    )
+    object_store_secret_key: str | None = Field(
+        default=None, description="The secret key for the object store."
+    )
+    object_store_use_ssl: bool = Field(
+        default=False, description="Whether to use SSL for the object store connection."
+    )
+    # start object-clerk library
+    multipart_threshold: int | None = Field(
+        default=None, description="Multipart threshold for the object store."
+    )
+    s3_client_config: dict | None = Field(
+        default=None, description="S3 client configuration for the object store."
+    )
+    s3_upload_config: dict | None = Field(
+        default=None, description="S3 upload configuration for the object store."
+    )
+    s3_download_config: dict | None = Field(
+        default=None, description="S3 download configuration for the object store."
+    )
     # globus
-    globus_transport_params: dict = Field(default_factory=dict)
-    globus_client_id: str | None = None
-    globus_client_secret: str | None = None
-    object_store_endpoint: str | None = None
-    scratch_endpoint: str | None = None
+    globus_transport_params: dict = Field(
+        default_factory=dict, description="Globus transfer parameters."
+    )
+    globus_client_id: str | None = Field(
+        default=None, description="Globus client ID for inbound/outbound transfers."
+    )
+    globus_client_secret: str | None = Field(
+        default=None, description="Globus client secret for inbound/outbound transfers."
+    )
+    object_store_endpoint: str | None = Field(
+        default=None, description="Object store Globus Endpoint ID."
+    )
+    scratch_endpoint: str | None = Field(default=None, description="Scratch Globus Endpoint ID.")
     # scratch
-    scratch_base_path: str = Field(default="scratch/")
-    scratch_inventory_db_count: int = 16
+    scratch_base_path: str = Field(default="scratch/", description="Base path for scratch storage.")
+    scratch_inventory_db_count: int = Field(
+        default=16, description="Number of databases in the scratch inventory (redis)."
+    )
     # docs
-    docs_base_url: str = Field(default="my_test_url")
+    docs_base_url: str = Field(
+        default="my_test_url", description="Base URL for the documentation site."
+    )
 
     @property
     def metadata_store_api_base(self) -> str:
@@ -106,4 +123,3 @@ class DKISTProcessingCommonConfiguration(DKISTProcessingCoreConfiguration):
 
 
 common_configurations = DKISTProcessingCommonConfiguration()
-common_configurations.log_configurations()

dkist_processing_common/models/telemetry.py

@@ -0,0 +1,28 @@
+"""Models to support telemetry data."""
+
+from pydantic import BaseModel
+
+
+class ObservableProgress(BaseModel, validate_assignment=True):
+    """Container for tracking progress for a metering instrument e.g. task progress."""
+
+    current: int = 0
+    total: int = 0
+
+    def increment(self, step: int = 1) -> None:
+        """Increment the current progress by the given step."""
+        self.current += step
+
+    @property
+    def percent_complete(self) -> float:
+        """Return the percent complete as a float between 0 and 100."""
+        if self.total > 0:
+            return (self.current / self.total) * 100
+        return 0.0
+
+    def set_complete(self):
+        """Set the current progress to the total."""
+        if self.total == 0:
+            self.total = self.current = 1
+        else:
+            self.current = self.total

dkist_processing_common/tasks/assemble_movie.py

@@ -163,7 +163,7 @@ class AssembleMovie(WorkflowTaskBase, ABC):
         relative_movie_path = f"{self.constants.dataset_id}_browse_movie.mp4"
         absolute_movie_path = str(self.scratch.absolute_path(relative_movie_path))
 
-        with self.apm_processing_step("Assembling movie frames"):
+        with self.telemetry_span("Assembling movie frames"):
             clip.write_videofile(absolute_movie_path, fps=self.FPS, codec="libx264", audio=False)
 
         self.tag(path=absolute_movie_path, tags=[Tag.movie(), Tag.output()])

dkist_processing_common/tasks/base.py

@@ -12,6 +12,10 @@ from typing import Iterable
 from typing import Type
 
 from dkist_processing_core import TaskBase
+from opentelemetry.metrics import CallbackOptions
+from opentelemetry.metrics import Counter
+from opentelemetry.metrics import ObservableGauge
+from opentelemetry.metrics import Observation
 
 from dkist_processing_common._util.scratch import WorkflowFileSystem
 from dkist_processing_common._util.tags import TagDB
@@ -21,6 +25,7 @@ from dkist_processing_common.config import common_configurations
 from dkist_processing_common.models.constants import ConstantsBase
 from dkist_processing_common.models.tags import StemName
 from dkist_processing_common.models.tags import Tag
+from dkist_processing_common.models.telemetry import ObservableProgress
 from dkist_processing_common.tasks.mixin.metadata_store import MetadataStoreMixin
 
 __all__ = ["WorkflowTaskBase", "tag_type_hint"]
@@ -66,7 +71,6 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
             workflow_name=workflow_name,
             workflow_version=workflow_version,
         )
-        self.task_name = self.__class__.__name__
         self.scratch = WorkflowFileSystem(recipe_run_id=recipe_run_id, task_name=self.task_name)
         self.constants = self.constants_model_class(
             recipe_run_id=recipe_run_id, task_name=self.task_name
@@ -76,50 +80,30 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
             recipe_run_id=recipe_run_id, task_name=self.task_name, namespace="counter"
         )
 
-    # These apm* functions provide tagged spans for APM bliss
-    def apm_type_base(
-        self,
-        name: str,
-        *args,
-        arg_span_type: str = None,
-        arg_labels: dict[str, str] = None,
-        **kwargs,
-    ):
-        """Groom inputs to apm_step to handle various kwarg collisions."""
-        if "span_type" in kwargs:
-            raise RuntimeError(
-                f"Cannot specify 'span_type' {kwargs['span_type']} in step that forces is it to be {arg_span_type}"
-            )
-
-        if "labels" in kwargs:
-            arg_labels.update(kwargs["labels"])
-            del kwargs["labels"]
-        logger.info(
-            f"Recording APM span: {name = }, {arg_span_type = }, {arg_labels = }, "
-            f"recipe_run_id = {self.recipe_run_id}"
+        # meter instruments
+        self.read_counter: Counter = self.meter.create_counter(
+            name=self.format_metric_name("tasks.reads"),
+            unit="1",
+            description="The number of reads executed in the processing stack.",
         )
-        return self.apm_step(name, *args, span_type=arg_span_type, labels=arg_labels, **kwargs)
-
-    def apm_task_step(self, name: str, *args, **kwargs):
-        """Span for management/organizational/info type stuff."""
-        return self.apm_type_base(
-            name, *args, arg_span_type="code.task", arg_labels={"type": "task"}, **kwargs
+        self.write_counter: Counter = self.meter.create_counter(
+            name=self.format_metric_name("tasks.writes"),
+            unit="1",
+            description="The number of writes executed in the processing stack.",
         )
-
-    def apm_processing_step(self, name: str, *args, **kwargs):
-        """Span for computations."""
-        return self.apm_type_base(
-            name,
-            *args,
-            arg_span_type="code.processing",
-            arg_labels={"type": "processing"},
-            **kwargs,
+        self.outer_loop_progress = ObservableProgress()
+        self.outer_loop_progress_gauge: ObservableGauge = self.meter.create_observable_gauge(
+            name=self.format_metric_name("tasks.outer.loop.progress"),
+            description="The progress of a task through the main processing loop.",
+            callbacks=[lambda options: self.outer_loop_run_progress(options)],
         )
 
-    def apm_writing_step(self, name: str, *args, **kwargs):
-        """Span for writing to disk."""
-        return self.apm_type_base(
-            name, *args, arg_span_type="code.writing", arg_labels={"type": "writing"}, **kwargs
+    def outer_loop_run_progress(
+        self, options: CallbackOptions
+    ) -> Generator[Observation, None, None]:
+        """Observe the progress of the current task as a percentage."""
+        yield Observation(
+            self.outer_loop_progress.percent_complete, attributes=self.base_telemetry_attributes
         )
 
     @property
@@ -161,9 +145,14 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
         """Execute any pre-task setup required."""
         super().pre_run()
         if self.record_provenance or self.is_task_manual:
-            with self.apm_task_step("Record Provenance"):
+            with self.telemetry_span("Record Provenance"):
                 self._record_provenance()
 
+    def post_run(self) -> None:
+        """Execute and post-task bookkeeping required."""
+        super().post_run()
+        self.outer_loop_progress.set_complete()
+
     def read(
         self, tags: tag_type_hint, decoder: callable = path_decoder, **decoder_kwargs
     ) -> Generator[Any, None, None]:
@@ -183,7 +172,9 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
         **decoder_kwargs
             Additional arguments to pass to the `decoder` function.
         """
-        return (decoder(p, **decoder_kwargs) for p in self.scratch.find_all(tags=tags))
+        for p in self.scratch.find_all(tags=tags):
+            self.read_counter.add(amount=1, attributes=self.base_telemetry_attributes)
+            yield decoder(p, **decoder_kwargs)
 
     def write(
         self,
@@ -221,6 +212,7 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
         -------
         The path for the written file
         """
+        self.write_counter.add(amount=1, attributes=self.base_telemetry_attributes)
         file_obj = encoder(data, **encoder_kwargs)
         if isinstance(tags, str):
             tags = [tags]
@@ -372,11 +364,11 @@ class WorkflowTaskBase(TaskBase, MetadataStoreMixin, ABC):
         Filename Counter: not rolled back but its purpose of preventing file name collisions is not impacted
         """
         super().rollback()
-        with self.apm_writing_step("Rollback Scratch"):
+        with self.telemetry_span("Rollback Scratch"):
             self.scratch.rollback()
-        with self.apm_writing_step("Rollback Constants"):
+        with self.telemetry_span("Rollback Constants"):
             self.constants._rollback()
-        with self.apm_task_step("Change Recipe Run to Inprogress"):
+        with self.telemetry_span("Change Recipe Run to Inprogress"):
             self.metadata_store_change_recipe_run_to_inprogress()
 
     def __exit__(self, exc_type, exc_val, exc_tb):

dkist_processing_common/tasks/l1_output_data.py

@@ -54,11 +54,11 @@ class TransferL1Data(TransferDataBase, GlobusMixin):
 
     def transfer_objects(self):
         """Transfer movie and L1 output frames."""
-        with self.apm_task_step("Upload movie"):
+        with self.telemetry_span("Upload movie"):
             # Movie needs to be transferred separately as the movie headers need to go with it
             self.transfer_movie()
 
-        with self.apm_task_step("Upload science frames"):
+        with self.telemetry_span("Upload science frames"):
             self.transfer_output_frames()
 
     def transfer_output_frames(self):
@@ -120,10 +120,10 @@ class AssembleQualityData(L1OutputDataBase, QualityMixin):
 
     def run(self):
         """Run method for the task."""
-        with self.apm_processing_step("Assembling quality data"):
+        with self.telemetry_span("Assembling quality data"):
             quality_data = self.quality_assemble_data(polcal_label_list=self.polcal_label_list)
 
-        with self.apm_writing_step(
+        with self.telemetry_span(
             f"Saving quality data with {len(quality_data)} metrics to the file system"
         ):
             self.write(
@@ -146,7 +146,7 @@ class SubmitDatasetMetadata(L1OutputDataBase):
 
     def run(self) -> None:
         """Run method for this task."""
-        with self.apm_writing_step(f"Storing quality data to metadata store"):
+        with self.telemetry_span(f"Storing quality data to metadata store"):
             # each quality_data file is a list - this will combine the elements of multiple lists into a single list
             quality_data = list(
                 chain.from_iterable(
@@ -156,7 +156,7 @@ class SubmitDatasetMetadata(L1OutputDataBase):
             self.metadata_store_add_quality_data(
                 dataset_id=self.constants.dataset_id, quality_data=quality_data
             )
-        with self.apm_processing_step("Count Expected Outputs"):
+        with self.telemetry_span("Count Expected Outputs"):
             dataset_id = self.constants.dataset_id
             expected_object_count = self.count(tags=Tag.output())
             if quality_data:
@@ -165,7 +165,7 @@ class SubmitDatasetMetadata(L1OutputDataBase):
             f"Adding Dataset Receipt Account: "
             f"{dataset_id=}, {expected_object_count=}, recipe_run_id={self.recipe_run_id}"
         )
-        with self.apm_task_step(
+        with self.telemetry_span(
             f"Add Dataset Receipt Account: {dataset_id = }, {expected_object_count = }"
         ):
             self.metadata_store_add_dataset_receipt_account(
@@ -245,10 +245,10 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
 
     def run(self) -> None:
         """Run method for this task."""
-        with self.apm_task_step("Gather output data"):
+        with self.telemetry_span("Gather output data"):
             frames = self.read(tags=self.output_frame_tags)
             movies = self.read(tags=[Tag.output(), Tag.movie()])
-        with self.apm_task_step("Create message objects"):
+        with self.telemetry_span("Create message objects"):
             messages = []
             messages += self.frame_messages(paths=frames)
             frame_message_count = len(messages)
@@ -257,7 +257,7 @@ class PublishCatalogAndQualityMessages(L1OutputDataBase, InterserviceBusMixin):
             dataset_has_quality_data = self.dataset_has_quality_data
             if dataset_has_quality_data:
                 messages.append(self.quality_report_message)
-        with self.apm_task_step(
+        with self.telemetry_span(
             f"Publish messages: {frame_message_count = }, {object_message_count = }, {dataset_has_quality_data = }"
         ):
             self.interservice_bus_publish(messages=messages)

dkist_processing_common/tasks/output_data_base.py

@@ -58,10 +58,10 @@ class TransferDataBase(OutputDataBase, ObjectStoreMixin, ABC):
 
     def run(self) -> None:
         """Transfer the data and cleanup any folders."""
-        with self.apm_task_step("Transfer objects"):
+        with self.telemetry_span("Transfer objects"):
             self.transfer_objects()
 
-        with self.apm_task_step("Remove folder objects"):
+        with self.telemetry_span("Remove folder objects"):
             self.remove_folder_objects()
 
     @abstractmethod

dkist_processing_common/tasks/parse_l0_input_data.py

@@ -103,21 +103,25 @@ class ParseDataBase(WorkflowTaskBase, ABC):
 
     @property
     @abstractmethod
-    def tags_for_input_frames(self) -> list[Tag]:
+    def tags_for_input_frames(self) -> list[str]:
         """Define the tags for the data that will be parsed."""
 
+    def pre_run(self) -> None:
+        """Execute pre-task setup."""
+        self.outer_loop_progress.total = self.scratch.count_all(tags=self.tags_for_input_frames)
+
     def run(self) -> None:
         """Run method for this task."""
-        with self.apm_task_step("Check that input frames exist"):
+        with self.telemetry_span("Check that input frames exist"):
             self.check_input_frames()
 
-        with self.apm_task_step("Ingest all input files"):
+        with self.telemetry_span("Ingest all input files"):
             tag_pot, constant_pot = self.make_flower_pots()
 
-        with self.apm_task_step("Update constants"):
+        with self.telemetry_span("Update constants"):
             self.update_constants(constant_pot)
 
-        with self.apm_task_step("Tag files"):
+        with self.telemetry_span("Tag files"):
             self.tag_petals(tag_pot)
 
     def make_flower_pots(self) -> tuple[FlowerPot, FlowerPot]:
@@ -128,6 +132,7 @@ class ParseDataBase(WorkflowTaskBase, ABC):
         constant_pot.stems += self.constant_buds
 
         for fits_obj in self.input_frames:
+            self.outer_loop_progress.increment()
             filepath = fits_obj.name
             tag_pot.add_dirt(filepath, fits_obj)
             constant_pot.add_dirt(filepath, fits_obj)
@@ -161,7 +166,7 @@ class ParseDataBase(WorkflowTaskBase, ABC):
         None
         """
         for stem in constant_pot:
-            with self.apm_processing_step(f"Setting value of constant {stem.stem_name}"):
+            with self.telemetry_span(f"Setting value of constant {stem.stem_name}"):
                 if len(stem.petals) == 0:
                     # There are no petals so nothing to do
                     continue
@@ -186,7 +191,7 @@ class ParseDataBase(WorkflowTaskBase, ABC):
         None
         """
         for stem in tag_pot:
-            with self.apm_processing_step(f"Applying {stem.stem_name} tag to files"):
+            with self.telemetry_span(f"Applying {stem.stem_name} tag to files"):
                 for petal in stem.petals:
                     tag = Tag.format_tag(stem.stem_name, petal.value)
                     for path in petal.keys:

dkist_processing_common/tasks/quality_metrics.py

@@ -100,10 +100,10 @@ class QualityL0Metrics(WorkflowTaskBase, QualityMixin):
         `quality_task_types` properties, respectively.
         """
         modstate_list = self.modstate_list if self.modstate_list is not None else [None]
-        with self.apm_task_step("Computing L0 Quality Metrics"):
+        with self.telemetry_span("Computing L0 Quality Metrics"):
             quality_data_list = []
             for task_type in self.quality_task_types:
-                with self.apm_processing_step(f"Working on {task_type = }"):
+                with self.telemetry_span(f"Working on {task_type = }"):
                     for modstate in modstate_list:
                         paths = self.get_paths_for_modstate_and_task(modstate, task_type)
                         quality_data = self.calculate_l0_metrics(
@@ -112,7 +112,7 @@ class QualityL0Metrics(WorkflowTaskBase, QualityMixin):
                         quality_data.modstate = modstate
                         quality_data_list.append(quality_data)
 
-        with self.apm_writing_step("Saving metrics to disk"):
+        with self.telemetry_span("Saving metrics to disk"):
             for quality_data in quality_data_list:
                 if quality_data.has_values:
                     self.save_quality_data(quality_data, modstate=quality_data.modstate)
@@ -300,17 +300,17 @@ class QualityL1Metrics(WorkflowTaskBase, QualityMixin):
             ),
         ]
 
-        with self.apm_task_step("Reading L1 frames"):
+        with self.telemetry_span("Reading L1 frames"):
             paths = list(self.read(tags=[Tag.calibrated(), Tag.frame()]))
 
-        with self.apm_task_step("Calculating L1 quality metrics"):
+        with self.telemetry_span("Calculating L1 quality metrics"):
             for metric in metrics:
-                with self.apm_processing_step(f"Calculating L1 metric {metric.value_source}"):
+                with self.telemetry_span(f"Calculating L1 metric {metric.value_source}"):
                     for path in paths:
                         frame = L1QualityFitsAccess.from_path(path)
                         metric.append_value(frame=frame)
 
-        with self.apm_task_step("Sending lists for storage"):
+        with self.telemetry_span("Sending lists for storage"):
             for metric in metrics:
                 if metric.has_values:
                     metric.store_metric()

dkist_processing_common/tasks/teardown.py

@@ -26,11 +26,11 @@ class TeardownBase(WorkflowTaskBase, ABC):
 
     def run(self) -> None:
         """Run method for Teardown class."""
-        with self.apm_task_step("Change recipe run status"):
+        with self.telemetry_span("Change recipe run status"):
             self.change_recipe_run_status_to_success()
 
         if not self.teardown_enabled:
-            with self.apm_task_step(f"Skip Teardown"):
+            with self.telemetry_span(f"Skip Teardown"):
                 return
 
         logger.info(f"Removing data and tags for recipe run {self.recipe_run_id}")
@@ -43,13 +43,13 @@ class TeardownBase(WorkflowTaskBase, ABC):
 
     def teardown(self):
         """Purge all constants and files/tags in scratch."""
-        with self.apm_task_step("Remove Data and Tags"):
+        with self.telemetry_span("Remove Data and Tags"):
             self.scratch.purge()
 
-        with self.apm_task_step("Remove File Counters"):
+        with self.telemetry_span("Remove File Counters"):
             self.filename_counter.purge()
 
-        with self.apm_task_step("Remove Constants"):
+        with self.telemetry_span("Remove Constants"):
             self.constants._purge()
 
 

dkist_processing_common/tasks/transfer_input_data.py

@@ -81,13 +81,13 @@ class TransferL0Data(WorkflowTaskBase, GlobusMixin):
 
     def run(self) -> None:
         """Execute the data transfer."""
-        with self.apm_task_step("Change Status to InProgress"):
+        with self.telemetry_span("Change Status to InProgress"):
             self.metadata_store_change_recipe_run_to_inprogress()
 
-        with self.apm_task_step("Download Input Dataset Documents"):
+        with self.telemetry_span("Download Input Dataset Documents"):
             self.download_input_dataset()
 
-        with self.apm_task_step("Build Input Dataset Transfer List"):
+        with self.telemetry_span("Build Input Dataset Transfer List"):
             observe_transfer_objects = self.build_transfer_list(
                 doc_tag=Tag.input_dataset_observe_frames()
             )
@@ -103,13 +103,13 @@ class TransferL0Data(WorkflowTaskBase, GlobusMixin):
             if len(observe_transfer_objects + calibration_transfer_objects) == 0:
                 raise ValueError("No input dataset frames found to transfer")
 
-        with self.apm_task_step("Transfer Input Frames and Parameter Files via Globus"):
+        with self.telemetry_span("Transfer Input Frames and Parameter Files via Globus"):
             self.globus_transfer_object_store_to_scratch(
                 transfer_items=self.format_transfer_items(input_dataset_objects=transfer_objects),
                 label=f"Transfer Input Objects for Recipe Run {self.recipe_run_id}",
             )
 
-        with self.apm_processing_step("Tag Input Frames and Parameter Files"):
+        with self.telemetry_span("Tag Input Frames and Parameter Files"):
             self.tag_transfer_objects(input_dataset_objects=transfer_objects)
 
     def rollback(self):

dkist_processing_common/tasks/trial_catalog.py

@@ -97,13 +97,13 @@ class CreateTrialDatasetInventory(OutputDataBase):
 
     def run(self) -> None:
         """Generate a json file simulating the dataset inventory record that would be produced when cataloging the dataset."""
-        with self.apm_task_step("Retrieve output frame headers"):
+        with self.telemetry_span("Retrieve output frame headers"):
             json_headers = list(self.frame_inventories)
-        with self.apm_processing_step("Generate dataset inventory"):
+        with self.telemetry_span("Generate dataset inventory"):
             inventory: dict = generate_inventory_from_frame_inventory(
                 bucket=self.destination_bucket, json_headers=json_headers
             )
-        with self.apm_writing_step("Save dataset inventory file"):
+        with self.telemetry_span("Save dataset inventory file"):
             self.write(
                 inventory,
                 tags=[Tag.output(), Tag.dataset_inventory()],
@@ -138,7 +138,7 @@ class CreateTrialAsdf(OutputDataBase):
 
     def run(self) -> None:
         """Generate an ASDF file simulating the ASDF file that would be produced when cataloging the dataset."""
-        with self.apm_processing_step("Generate ASDF tree"):
+        with self.telemetry_span("Generate ASDF tree"):
             tree = asdf_tree_from_filenames(
                 filenames=self.absolute_output_frame_paths,
                 hdu=1,  # compressed
@@ -156,7 +156,7 @@ class CreateTrialAsdf(OutputDataBase):
                 },
             )
         ]
-        with self.apm_writing_step("Save ASDF file"):
+        with self.telemetry_span("Save ASDF file"):
             with make_asdf_file_object(tree, extra_history=trial_history) as asdf_obj:
                 self.write(
                     asdf_obj,
@@ -192,7 +192,7 @@ class CreateTrialQualityReport(OutputDataBase):
 
     def create_trial_quality_report(self) -> None:
         """Generate a trial quality report in pdf format and save to the file system for future upload."""
-        with self.apm_processing_step(f"Building the trial quality report"):
+        with self.telemetry_span(f"Building the trial quality report"):
             # each quality_data file is a list - this will combine the elements of multiple lists into a single list
             quality_data = list(
                 chain.from_iterable(
@@ -203,7 +203,7 @@ class CreateTrialQualityReport(OutputDataBase):
                 report_data=quality_data, dataset_id=self.constants.dataset_id
             )
 
-        with self.apm_writing_step(f"Saving the trial quality report to the file system"):
+        with self.telemetry_span(f"Saving the trial quality report to the file system"):
             self.write(
                 quality_report,
                 tags=[Tag.output(), Tag.quality_report()],

dkist_processing_common/tasks/trial_output_data.py

@@ -36,10 +36,10 @@ class TransferTrialData(TransferDataBase, GlobusMixin):
 
     def transfer_objects(self) -> None:
         """Collect transfer items and send them to Globus for transfer."""
-        with self.apm_task_step("Build transfer list"):
+        with self.telemetry_span("Build transfer list"):
             transfer_manifest = self.build_transfer_list()
 
-        with self.apm_task_step("Send transfer manifest to globus"):
+        with self.telemetry_span("Send transfer manifest to globus"):
             self.transfer_all_trial_frames(transfer_manifest)
 
     @cached_property

dkist_processing_common/tasks/write_l1.py

@@ -53,7 +53,7 @@ class WriteL1Frame(WorkflowTaskBase, MetadataStoreMixin, ABC):
     def run(self) -> None:
         """Run method for this task."""
         for stokes_param in self.constants.stokes_params:
-            with self.apm_task_step(f"Get calibrated frames for stokes param {stokes_param}"):
+            with self.telemetry_span(f"Get calibrated frames for stokes param {stokes_param}"):
                 tags = [Tag.frame(), Tag.calibrated(), Tag.stokes(stokes_param)]
                 calibrated_fits_objects = self.read(
                     tags=tags,

dkist_processing_common/tests/test_base.py

@@ -66,20 +66,6 @@ def tags_and_expected_generic_name() -> (list[str], str):
     return tags, expected_base_name
 
 
-def test_apm_spans(base_task):
-    """
-    Given: A WorkflowTaskBase task
-    When: Calling the task-specific apm_steps with weird inputs
-    Then: Errors happen when they're supposed to and not when they're not supposed to
-    """
-    with pytest.raises(RuntimeError):
-        with base_task.apm_processing_step("foo", span_type="bar"):
-            pass
-
-    with base_task.apm_task_step("foo", labels={"foo": "bar"}):
-        pass
-
-
 def test_tags(base_task):
     """
     Given: A WorkflowTaskBase task

{dkist_processing_common-11.5.0rc1.dist-info → dkist_processing_common-11.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 11.5.0rc1
+Version: 11.6.0
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause
@@ -17,9 +17,9 @@ Requires-Dist: asdf<4.0.0,>=3.5.0
 Requires-Dist: astropy>=7.0.0
 Requires-Dist: dkist-fits-specifications<5.0,>=4.0.0
 Requires-Dist: dkist-header-validator<6.0,>=5.0.0
-Requires-Dist: dkist-processing-core==5.2.0rc1
+Requires-Dist: dkist-processing-core==6.0.0
 Requires-Dist: dkist-processing-pac<4.0,>=3.1
-Requires-Dist: dkist-service-configuration<3.0,>=2.0.2
+Requires-Dist: dkist-service-configuration<5.0,>=4.1.7
 Requires-Dist: dkist-spectral-lines<4.0,>=3.0.0
 Requires-Dist: solar-wavelength-calibration<2.0,>=1.0
 Requires-Dist: globus-sdk>=3.12.0
@@ -32,7 +32,7 @@ Requires-Dist: object-clerk==1.0.0
 Requires-Dist: pandas>=1.4.2
 Requires-Dist: pillow>=10.2.0
 Requires-Dist: pydantic>=2.0
-Requires-Dist: redis==4.6.0
+Requires-Dist: redis==6.4.0
 Requires-Dist: requests>=2.23
 Requires-Dist: scipy>=1.15.1
 Requires-Dist: sunpy>=3.0.0
@@ -97,6 +97,84 @@ Deployment
 
 dkist-processing-common is deployed to `PyPI <https://pypi.org/project/dkist-processing-common/>`_
 
+Environment Variables
+---------------------
+
+.. list-table::
+   :widths: 10 90
+   :header-rows: 1
+
+   * - Variable
+     - Field Info
+   * - LOGURU_LEVEL
+     - annotation=str required=False default='INFO' alias_priority=2 validation_alias='LOGURU_LEVEL' description='Log level for the application'
+   * - MESH_CONFIG
+     - annotation=dict[str, MeshService] required=False default_factory=dict alias_priority=2 validation_alias='MESH_CONFIG' description='Service mesh configuration' examples=[{'upstream_service_name': {'mesh_address': 'localhost', 'mesh_port': 6742}}]
+   * - RETRY_CONFIG
+     - annotation=RetryConfig required=False default_factory=RetryConfig description='Retry configuration for the service'
+   * - OTEL_SERVICE_NAME
+     - annotation=str required=False default='unknown-service-name' alias_priority=2 validation_alias='OTEL_SERVICE_NAME' description='Service name for OpenTelemetry'
+   * - DKIST_SERVICE_VERSION
+     - annotation=str required=False default='unknown-service-version' alias_priority=2 validation_alias='DKIST_SERVICE_VERSION' description='Service version for OpenTelemetry'
+   * - NOMAD_ALLOC_ID
+     - annotation=str required=False default='unknown-allocation-id' alias_priority=2 validation_alias='NOMAD_ALLOC_ID' description='Nomad allocation ID for OpenTelemetry'
+   * - OTEL_EXPORTER_OTLP_TRACES_INSECURE
+     - annotation=bool required=False default=True description='Use insecure connection for OTLP traces'
+   * - OTEL_EXPORTER_OTLP_METRICS_INSECURE
+     - annotation=bool required=False default=True description='Use insecure connection for OTLP metrics'
+   * - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='OTLP traces endpoint. Overrides mesh configuration' examples=['localhost:4317']
+   * - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='OTLP metrics endpoint. Overrides mesh configuration' examples=['localhost:4317']
+   * - OTEL_PYTHON_DISABLED_INSTRUMENTATIONS
+     - annotation=list[str] required=False default_factory=list description='List of instrumentations to disable. https://opentelemetry.io/docs/zero-code/python/configuration/' examples=[['pika', 'requests']]
+   * - OTEL_PYTHON_FASTAPI_EXCLUDED_URLS
+     - annotation=str required=False default='health' description='Comma separated list of URLs to exclude from OpenTelemetry instrumentation in FastAPI.' examples=['client/.*/info,healthcheck']
+   * - SYSTEM_METRIC_INSTRUMENTATION_CONFIG
+     - annotation=Union[dict[str, bool], NoneType] required=False default=None description='Configuration for system metric instrumentation. https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/system_metrics/system_metrics.html' examples=[{'system.memory.usage': ['used', 'free', 'cached'], 'system.cpu.time': ['idle', 'user', 'system', 'irq'], 'system.network.io': ['transmit', 'receive'], 'process.runtime.memory': ['rss', 'vms'], 'process.runtime.cpu.time': ['user', 'system'], 'process.runtime.context_switches': ['involuntary', 'voluntary']}]
+   * - ISB_USERNAME
+     - annotation=str required=False default='guest' description='Username for the interservice-bus.'
+   * - ISB_PASSWORD
+     - annotation=str required=False default='guest' description='Password for the interservice-bus.'
+   * - ISB_EXCHANGE
+     - annotation=str required=False default='master.direct.x' description='Exchange for the interservice-bus.'
+   * - ISB_QUEUE_TYPE
+     - annotation=str required=False default='classic' description='Queue type for the interservice-bus.' examples=['quorum', 'classic']
+   * - BUILD_VERSION
+     - annotation=str required=False default='dev' description='Fallback build version for workflow tasks.'
+   * - GQL_AUTH_TOKEN
+     - annotation=Union[str, NoneType] required=False default='dev' description='The auth token for the metadata-store-api.'
+   * - OBJECT_STORE_ACCESS_KEY
+     - annotation=Union[str, NoneType] required=False default=None description='The access key for the object store.'
+   * - OBJECT_STORE_SECRET_KEY
+     - annotation=Union[str, NoneType] required=False default=None description='The secret key for the object store.'
+   * - OBJECT_STORE_USE_SSL
+     - annotation=bool required=False default=False description='Whether to use SSL for the object store connection.'
+   * - MULTIPART_THRESHOLD
+     - annotation=Union[int, NoneType] required=False default=None description='Multipart threshold for the object store.'
+   * - S3_CLIENT_CONFIG
+     - annotation=Union[dict, NoneType] required=False default=None description='S3 client configuration for the object store.'
+   * - S3_UPLOAD_CONFIG
+     - annotation=Union[dict, NoneType] required=False default=None description='S3 upload configuration for the object store.'
+   * - S3_DOWNLOAD_CONFIG
+     - annotation=Union[dict, NoneType] required=False default=None description='S3 download configuration for the object store.'
+   * - GLOBUS_TRANSPORT_PARAMS
+     - annotation=dict required=False default_factory=dict description='Globus transfer parameters.'
+   * - GLOBUS_CLIENT_ID
+     - annotation=Union[str, NoneType] required=False default=None description='Globus client ID for inbound/outbound transfers.'
+   * - GLOBUS_CLIENT_SECRET
+     - annotation=Union[str, NoneType] required=False default=None description='Globus client secret for inbound/outbound transfers.'
+   * - OBJECT_STORE_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='Object store Globus Endpoint ID.'
+   * - SCRATCH_ENDPOINT
+     - annotation=Union[str, NoneType] required=False default=None description='Scratch Globus Endpoint ID.'
+   * - SCRATCH_BASE_PATH
+     - annotation=str required=False default='scratch/' description='Base path for scratch storage.'
+   * - SCRATCH_INVENTORY_DB_COUNT
+     - annotation=int required=False default=16 description='Number of databases in the scratch inventory (redis).'
+   * - DOCS_BASE_URL
+     - annotation=str required=False default='my_test_url' description='Base URL for the documentation site.'
+
 Development
 -----------
 

{dkist_processing_common-11.5.0rc1.dist-info → dkist_processing_common-11.6.0.dist-info}/RECORD

@@ -1,7 +1,6 @@
 changelog/.gitempty,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-changelog/266.misc.rst,sha256=5yvMMAn3qAkY4NgLSgc-7dFzJdBTfWTu1J_TwCzd_eM,107
 dkist_processing_common/__init__.py,sha256=GQ9EBnYhkOnt-qODclAoLS_g5YVhurxfg1tjVtI9rDI,320
-dkist_processing_common/config.py,sha256=OCqMjIV9mDvsWX_nvwS_v1W3chAhfA52wQVVXjCQwe8,4248
+dkist_processing_common/config.py,sha256=kkFm-Q6jGGoV3Zdf3PSfk3xNYd2vSH0J2R8dBBav3Og,5218
 dkist_processing_common/manual.py,sha256=bIVVyLsbXMh-g_2L3kGROL-1TtJe0_XviHsp7Br31x8,7023
 dkist_processing_common/_util/__init__.py,sha256=xf6JNpMKQgbhE2Jivymt-WO0WF6PpGt9rl604YpuTWk,92
 dkist_processing_common/_util/constants.py,sha256=0_bWLsvusHD8GrTx4B6V7AieKAaFbN9crcAALaS8x5Q,3245
@@ -36,6 +35,7 @@ dkist_processing_common/models/parameters.py,sha256=9An3SxUEBI-oYHjICQ_q-IIScTfp
 dkist_processing_common/models/quality.py,sha256=TmDVbvPbfl5CIIs1ioD5guLUoEOFTfiJESvDjLTLl5s,3981
 dkist_processing_common/models/tags.py,sha256=0YqiDrismOSds_3XtFBb2dfv0gjMs6CgRv2dJKsSthI,12082
 dkist_processing_common/models/task_name.py,sha256=uAl7qTK4Xx1nqPAhNAe5nAXqxwPwQzAq58YmoccX6xQ,567
+dkist_processing_common/models/telemetry.py,sha256=XVcLNgHCZsP9L7oYiklyLUoqQtWt_xjEkuf70Kbudz4,839
 dkist_processing_common/models/wavelength.py,sha256=4UhRVoNvCHZitXo5S1oRdewadbmGfmDK6wetMV06POA,967
 dkist_processing_common/parsers/__init__.py,sha256=XJQzHtPb78F6-qXXKXjyztc0x-aHVlgv1C_l4dR88tI,67
 dkist_processing_common/parsers/cs_step.py,sha256=rL2gdstKEV5aqdPDs3a5EuUaOT_6YXDJVqIPIVKSw8M,6450
@@ -54,17 +54,17 @@ dkist_processing_common/parsers/time.py,sha256=z9zHV3Fz6ebEDgiPhv6H-aAS8e-sSW3Ek
 dkist_processing_common/parsers/unique_bud.py,sha256=IkS2zZkVzn3PRsYF2ksBkUxl_HJ4TxCqBKJUs1WdL54,3310
 dkist_processing_common/parsers/wavelength.py,sha256=P5C9mG8DAKK3GB3vWNRBI5l7pAW68lJK-kw-4eqERuQ,612
 dkist_processing_common/tasks/__init__.py,sha256=l23ctjNsKJbHbbqaZBMeOPaOtw0hmITEljI_JJ-CVsU,627
-dkist_processing_common/tasks/assemble_movie.py,sha256=m8O1psUBn8bRPj0AwMd1K5wNlcAtlxBQb1PZeSKne6o,12772
-dkist_processing_common/tasks/base.py,sha256=mebbG-VZp9Iu-J7skE9S5qMCUENZy48ySJkaGh8weC0,13144
-dkist_processing_common/tasks/l1_output_data.py,sha256=jjJao1YO88sgsKfl6Lg11yurlM1bcljOZpK1-ykFWw4,10576
-dkist_processing_common/tasks/output_data_base.py,sha256=ZWdxkXKzrQdc1PN97b8cEcFBckraOJkwrBoMOPWii8o,3683
-dkist_processing_common/tasks/parse_l0_input_data.py,sha256=Apa1qG6TAVll_oCGp5aSZb_j_ywNnu1GmYNnpjvHqH4,8133
-dkist_processing_common/tasks/quality_metrics.py,sha256=5rLCkGyE6F76bJp71y2lmKbol5FJ_ynbZbkhX8IBPKA,12489
-dkist_processing_common/tasks/teardown.py,sha256=eHyOJbtu8w51wFSTVFSt8iVVOkSODqHLvrcP6W-Szcg,2355
-dkist_processing_common/tasks/transfer_input_data.py,sha256=hjJzgwCVqltF87IMa2Oi9hmKyT0mFqwvPCI8xTV2B44,5787
-dkist_processing_common/tasks/trial_catalog.py,sha256=rGIlckrWYqx7Div5dOTQr_hcKSWQfR_i5DyzoHVhE9A,8749
-dkist_processing_common/tasks/trial_output_data.py,sha256=5WwFuShXNIFO7fSh6nLAkpHS8NbZeW7AYsjRxzqRhNI,6835
-dkist_processing_common/tasks/write_l1.py,sha256=rki0og7B10LooAFHABMz64NtwCo54CSwy3qOcOsve_g,23203
+dkist_processing_common/tasks/assemble_movie.py,sha256=1ixDG-f4ODt0vywqVccG3aodLljVO5OGlvuMO9EEvcU,12767
+dkist_processing_common/tasks/base.py,sha256=itAHCvzcodo-q8_AjpWoRaM86BlcjWDpCIiUP7uwmP0,13236
+dkist_processing_common/tasks/l1_output_data.py,sha256=D4S3kH2uRKp1b8_xF2YiWO_mGj19UJS5wKQZ4OdatGs,10568
+dkist_processing_common/tasks/output_data_base.py,sha256=r1Bu3FX5zTVj66GTMWtaV_NdhxjyjSm661Bt2Mxmfi4,3685
+dkist_processing_common/tasks/parse_l0_input_data.py,sha256=Gf8tA0aKqKuhkuxJqtblLL9y_ARCKoMb1cztSt_2hSU,8343
+dkist_processing_common/tasks/quality_metrics.py,sha256=cvGF6tJ8yAvxOvkeG3tWxYwL885BrFW5X3V7_MSzL-A,12481
+dkist_processing_common/tasks/teardown.py,sha256=rwT9lWINVDF11-az_nx-Z5ykMTX_SJCchobpU6sErgk,2360
+dkist_processing_common/tasks/transfer_input_data.py,sha256=DAYfS-B1o-iBT9MXU-TiJG4Hv05Z0c_JzPrnFgvnK9g,5786
+dkist_processing_common/tasks/trial_catalog.py,sha256=iAaMT_oLnupA1O3xAtqVjsqRY5f_hyvMps-fXg6KlHU,8729
+dkist_processing_common/tasks/trial_output_data.py,sha256=CPMXXODvN5RTcu9bTF8v6AXciCl212EWP6qTiARvUNk,6837
+dkist_processing_common/tasks/write_l1.py,sha256=Xy834RTp3F95kLcW4ba5gfHMUocfZd82ZQQKnvQcP2M,23204
 dkist_processing_common/tasks/mixin/__init__.py,sha256=-g-DQbU7m1bclJYuFe3Yh757V-35GIDTbstardKQ7nU,68
 dkist_processing_common/tasks/mixin/globus.py,sha256=9ey_UCacqCfmxYZSgm6VDefdlm7dkNloC8G5DeVub8s,6592
 dkist_processing_common/tasks/mixin/interservice_bus.py,sha256=M6R922l7gJSmmU_vswUXxy-c5DWNrIRjQu9H9CSgGfU,1081
@@ -78,7 +78,7 @@ dkist_processing_common/tests/conftest.py,sha256=Tm-Yq956EAafpDtu1d7JjdVY0Unp9e4
 dkist_processing_common/tests/mock_metadata_store.py,sha256=fbCvSk1-s0ojN6l538RWodPW7dx6k4eXqipemnHKO0Y,8248
 dkist_processing_common/tests/test_assemble_movie.py,sha256=dyVhowxB-Kc6GuxlDs74UrPtK9fwdUL7y5haA3Bidz0,4065
 dkist_processing_common/tests/test_assemble_quality.py,sha256=-F22jMY6mPy65VZ1TZY2r1vsxMXOPmZHArGx70OD3BA,17832
-dkist_processing_common/tests/test_base.py,sha256=EQsIkeWoOtjk0yxr_oPkhW3Uc0p8cMsknSMwKgrJI9E,7078
+dkist_processing_common/tests/test_base.py,sha256=gsyBG2R6Ufx7CzbHeGMagUwM9yCfpN4gCSZ6-aH2q48,6643
 dkist_processing_common/tests/test_codecs.py,sha256=XuvG1sG8DECMPmxtDEi98TxlvTSAy0vrtUUFLrwnHlA,22173
 dkist_processing_common/tests/test_constants.py,sha256=I_KcJs7ScCn53GYhEO6qjWrrnfZuyC1IVYOy87Pjlg4,6565
 dkist_processing_common/tests/test_cs_step.py,sha256=RA0QD3D8eaL3YSOL_gIJ9wkngy14RQ2jbD-05KAziW4,2408
@@ -117,7 +117,7 @@ docs/landing_page.rst,sha256=aPAuXFhBx73lEZ59B6E6JXxkK0LlxzD0n-HXqHrfumQ,746
 docs/make.bat,sha256=mBAhtURwhQ7yc95pqwJzlhqBSvRknr1aqZ5s8NKvdKs,4513
 docs/requirements.txt,sha256=Kbl_X4c7RQZw035YTeNB63We6I7pvXFU4T0Uflp2yDY,29
 licenses/LICENSE.rst,sha256=piZaQplkzOMmH1NXg6QIdo9wwo9pPCoHkvm2-DmH76E,1462
-dkist_processing_common-11.5.0rc1.dist-info/METADATA,sha256=8FTiTtNr9N0SSWAkAuQTWRChbk0UUh-jW0KoO1xWGTQ,7207
-dkist_processing_common-11.5.0rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dkist_processing_common-11.5.0rc1.dist-info/top_level.txt,sha256=LJhd1W-Vn90K8HnQDIE4r52YDpUjjMWDnllAWHBByW0,48
-dkist_processing_common-11.5.0rc1.dist-info/RECORD,,
+dkist_processing_common-11.6.0.dist-info/METADATA,sha256=AQxL1zjp7254K1ZpFGOic8SHuO6bbexvQQ0KwHQGHOY,13313
+dkist_processing_common-11.6.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dkist_processing_common-11.6.0.dist-info/top_level.txt,sha256=LJhd1W-Vn90K8HnQDIE4r52YDpUjjMWDnllAWHBByW0,48
+dkist_processing_common-11.6.0.dist-info/RECORD,,

changelog/266.misc.rst

@@ -1 +0,0 @@
-Update dkist-processing-core to 5.2.0 which includes upgrades to airflow 2.11.0 and requires Python 3.12+.