dkist-processing-common 13.0.3rc2__tar.gz → 13.0.3rc3__tar.gz

{dkist_processing_common-13.0.3rc2 → dkist_processing_common-13.0.3rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 13.0.3rc2
+Version: 13.0.3rc3
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

dkist_processing_common-13.0.3rc3/changelog/322.misc.rst

@@ -0,0 +1 @@
+Make isb username and password configurable for local unit tests.

{dkist_processing_common-13.0.3rc2 → dkist_processing_common-13.0.3rc3}/dkist_processing_common/tasks/quality_metrics.py

@@ -439,36 +439,22 @@ class QualityL1Metrics(WorkflowTaskBase, QualityMixin):
             fits_access_class=L1QualityFitsAccess,
         )
 
-        # accumulate specified L1QualityFitsAccess values into lists, sorted by datetime
-        rows = sorted(
-            (
-                (
-                    # DATE-BEG
-                    datetime.fromisoformat(frame.time_obs),
-                    # LIGHTLVL
-                    frame.light_level,
-                    # DSHEALTH
-                    frame.health_status,
-                    # ATMOS_R0
-                    frame.fried_parameter,
-                    # AO_LOCK
-                    frame.ao_status,
-                    # OOBSHIFT
-                    frame.num_out_of_bounds_ao_values,
-                )
-                for frame in frames
-            ),
-            # DATE-BEG
-            key=lambda row: row[0],
-        )
+        # sort by datetime
+        sorted_frames = sorted(frames, key=lambda f: datetime.fromisoformat(f.time_obs))
 
         return L1QualityData(
-            datetimes=tuple(row[0] for row in rows),
-            light_level_values=tuple(row[1] for row in rows),
-            health_status_values=tuple(row[2] for row in rows),
-            fried_parameter_values=tuple(row[3] for row in rows),
-            ao_lock_values=tuple(row[4] for row in rows),
-            ao_oob_values=tuple(row[5] for row in rows),
+            # DATE-BEG
+            datetimes=tuple(datetime.fromisoformat(f.time_obs) for f in sorted_frames),
+            # LIGHTLVL
+            light_level_values=tuple(f.light_level for f in sorted_frames),
+            # DSHEALTH
+            health_status_values=tuple(f.health_status for f in sorted_frames),
+            # ATMOS_R0
+            fried_parameter_values=tuple(f.fried_parameter for f in sorted_frames),
+            # AO_LOCK
+            ao_lock_values=tuple(f.ao_status for f in sorted_frames),
+            # OOBSHIFT
+            ao_oob_values=tuple(f.num_out_of_bounds_ao_values for f in sorted_frames),
         )
 
     def compute_and_write_l1_quality_metrics(self, l1_quality_data: L1QualityData) -> None:
@@ -575,19 +561,21 @@ class QualityL1Metrics(WorkflowTaskBase, QualityMixin):
         rows.extend(tuple(z) for z in zip(statuses, counts))
         table_data = TableData(rows=tuple(rows))
 
+        warnings = None
+        if any(s in statuses for s in ["bad", "ill", "unknown"]):
+            warnings = [
+                "Data sourced from components with a health status of 'ill', 'bad', or 'unknown'."
+            ]
+
         metric = QualityMetric(
             name="Data Source Health",
             description="This metric contains the worst health status of the data source during "
             "data acquisition. One reading is taken per L1 frame.",
             metric_code=MetricCode.health_status,
             table_data=[table_data],
+            warnings=warnings,
         )
 
-        if any(s in statuses for s in ["bad", "ill", "unknown"]):
-            metric.warnings = [
-                "Data sourced from components with a health status of 'ill', 'bad', or 'unknown'."
-            ]
-
         self.write(metric, tags=Tag.quality("GENERIC"), encoder=basemodel_encoder)
 
     def compute_and_write_light_level(self, l1_quality_data: L1QualityData) -> None:

{dkist_processing_common-13.0.3rc2 → dkist_processing_common-13.0.3rc3}/dkist_processing_common/tests/test_interservice_bus.py

@@ -4,8 +4,9 @@ from typing import Type
 from uuid import uuid4
 
 import pytest
+from pika import ConnectionParameters
+from pika.credentials import PlainCredentials
 from pydantic import BaseModel
-from pydantic import Field
 from talus import Binding
 from talus import ConsumeMessageBase
 from talus import DurableConsumer
@@ -15,6 +16,8 @@ from talus import MessageBodyBase
 from talus import PublishMessageBase
 from talus import Queue
 
+from dkist_processing_common.config import common_configurations
+
 
 @pytest.fixture
 def sample_message_body():
@@ -59,19 +62,35 @@ def consume_message(sample_message):
 
 
 @pytest.fixture
-def producer(sample_message, publish_message):
+def connection_parameters() -> ConnectionParameters:
+    isb_username = common_configurations.isb_username
+    isb_password = common_configurations.isb_password
+    connection_parameters = ConnectionParameters(
+        credentials=PlainCredentials(isb_username, isb_password)
+    )
+    return connection_parameters
+
+
+@pytest.fixture
+def producer(sample_message, publish_message, connection_parameters):
     exchange = sample_message.exchange
     binding = Binding(queue=sample_message.queue, message=publish_message)
-    with DurableProducer(queue_bindings=binding, publish_exchange=exchange) as p:
+    with DurableProducer(
+        queue_bindings=binding,
+        publish_exchange=exchange,
+        connection_parameters=connection_parameters,
+    ) as p:
         yield p
         p.channel.queue_delete(queue=sample_message.queue.name)
         p.channel.exchange_delete(exchange=sample_message.exchange.name)
 
 
 @pytest.fixture
-def consumer(sample_message):
+def consumer(sample_message, connection_parameters):
     queue = sample_message.queue
-    with DurableConsumer(consume_queue=queue, prefetch_count=100) as c:
+    with DurableConsumer(
+        consume_queue=queue, prefetch_count=100, connection_parameters=connection_parameters
+    ) as c:
         yield c
         c.channel.queue_delete(queue=sample_message.queue.name)
         c.channel.exchange_delete(exchange=sample_message.exchange.name)

{dkist_processing_common-13.0.3rc2 → dkist_processing_common-13.0.3rc3}/dkist_processing_common/tests/test_interservice_bus_mixin.py

@@ -5,6 +5,8 @@ from typing import Dict
 from typing import Type
 
 import pytest
+from pika import ConnectionParameters
+from pika.credentials import PlainCredentials
 from pydantic import BaseModel
 from talus import ConsumeMessageBase
 from talus import DurableConsumer
@@ -77,9 +79,21 @@ def consume_message(task_message):
 
 
 @pytest.fixture
-def consumer(task_message):
+def connection_parameters() -> ConnectionParameters:
+    isb_username = common_configurations.isb_username
+    isb_password = common_configurations.isb_password
+    connection_parameters = ConnectionParameters(
+        credentials=PlainCredentials(isb_username, isb_password)
+    )
+    return connection_parameters
+
+
+@pytest.fixture
+def consumer(task_message, connection_parameters):
     queue = task_message.queue
-    with DurableConsumer(consume_queue=queue, prefetch_count=100) as c:
+    with DurableConsumer(
+        consume_queue=queue, prefetch_count=100, connection_parameters=connection_parameters
+    ) as c:
         yield c
         # Clean up all the queues, skipping queues that are not empty:
         for binding in common_message_queue_bindings:

{dkist_processing_common-13.0.3rc2 → dkist_processing_common-13.0.3rc3}/dkist_processing_common/tests/test_quality.py

@@ -737,9 +737,9 @@ def test_ao_status(quality_l1_task_with_calibrated_frames: QualityL1Metrics):
     # Then
     metric_code = "AO_STATUS"
     tags = [Tag.quality("GENERIC")]
-    metrics = [
-        metric for metric in task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
-    ]
+    metrics: list[QualityMetric] = list(
+        task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
+    )
     assert len([metric for metric in metrics if metric.metric_code in metric_code]) == 1
     for metric in metrics:
         if metric.metric_code != metric_code:
@@ -772,9 +772,9 @@ def test_fried_parameter(
     # Then
     metric_code = "FRIED_PARAMETER"
     tags = [Tag.quality("GENERIC")]
-    metrics = [
-        metric for metric in task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
-    ]
+    metrics: list[QualityMetric] = list(
+        task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
+    )
     assert len([metric for metric in metrics if metric.metric_code in metric_code]) == 1
     for metric in metrics:
         if metric.metric_code != metric_code:
@@ -823,9 +823,9 @@ def test_health_status(quality_l1_task_with_calibrated_frames: QualityL1Metrics)
     # Then
     metric_code = "HEALTH_STATUS"
     tags = [Tag.quality("GENERIC")]
-    metrics = [
-        metric for metric in task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
-    ]
+    metrics: list[QualityMetric] = list(
+        task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
+    )
     assert len([metric for metric in metrics if metric.metric_code in metric_code]) == 1
     for metric in metrics:
         if metric.metric_code != metric_code:
@@ -867,9 +867,9 @@ def test_light_level(
     # Then
     metric_code = "LIGHT_LEVEL"
     tags = [Tag.quality("GENERIC")]
-    metrics = [
-        metric for metric in task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
-    ]
+    metrics: list[QualityMetric] = list(
+        task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
+    )
     assert len([metric for metric in metrics if metric.metric_code in metric_code]) == 1
     for metric in metrics:
         if metric.metric_code != metric_code:
@@ -919,9 +919,9 @@ def test_compute_and_write_ao_status(
     task.compute_and_write_ao_status(l1_quality_data)
     # Then
     tags = [Tag.quality("GENERIC")]
-    metrics: list[QualityMetric] = [
-        metric for metric in task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
-    ]
+    metrics: list[QualityMetric] = list(
+        task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
+    )
     assert len(metrics) == 1
     metric = metrics[0]
     # basics
@@ -956,9 +956,9 @@ def test_compute_and_write_fried_parameter(
     task.compute_and_write_fried_parameter(l1_quality_data)
     # Then
     tags = [Tag.quality("GENERIC")]
-    metrics: list[QualityMetric] = [
-        metric for metric in task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
-    ]
+    metrics: list[QualityMetric] = list(
+        task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
+    )
     assert len(metrics) == 1
     metric = metrics[0]
     # basics
@@ -1013,9 +1013,9 @@ def test_compute_and_write_health_status(
     task.compute_and_write_health_status(l1_quality_data)
     # Then
     tags = [Tag.quality("GENERIC")]
-    metrics: list[QualityMetric] = [
-        metric for metric in task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
-    ]
+    metrics: list[QualityMetric] = list(
+        task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
+    )
     assert len(metrics) == 1
     metric = metrics[0]
     # basics
@@ -1069,9 +1069,9 @@ def test_compute_and_write_light_level(
     task.compute_and_write_light_level(l1_quality_data)
     # Then
     tags = [Tag.quality("GENERIC")]
-    metrics: list[QualityMetric] = [
-        metric for metric in task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
-    ]
+    metrics: list[QualityMetric] = list(
+        task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
+    )
     assert len(metrics) == 1
     metric = metrics[0]
     # basics
@@ -1122,7 +1122,7 @@ def test_no_calibration_frames_no_l1_metrics(quality_l1_task_no_scratch_data: Qu
     task()
     # Then
     tags = [Tag.quality("GENERIC")]
-    metrics = [
-        metric for metric in task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
-    ]
+    metrics: list[QualityMetric] = list(
+        task.read(tags=tags, decoder=basemodel_decoder, model=QualityMetric)
+    )
     assert len(metrics) == 0

{dkist_processing_common-13.0.3rc2 → dkist_processing_common-13.0.3rc3}/dkist_processing_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dkist-processing-common
-Version: 13.0.3rc2
+Version: 13.0.3rc3
 Summary: Common task classes used by the DKIST science data processing pipelines
 Author-email: NSO / AURA <dkistdc@nso.edu>
 License: BSD-3-Clause

{dkist_processing_common-13.0.3rc2 → dkist_processing_common-13.0.3rc3}/dkist_processing_common.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ README.rst
 bitbucket-pipelines.yml
 pyproject.toml
 changelog/.gitempty
+changelog/322.misc.rst
 changelog/326.feature.rst
 dkist_processing_common/__init__.py
 dkist_processing_common/config.py