ctao-bdms-clients 0.3.0rc1__py3-none-any.whl → 0.3.1__py3-none-any.whl

bdms/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.0rc1'
-__version_tuple__ = version_tuple = (0, 3, 0, 'rc1')
+__version__ = version = '0.3.1'
+__version_tuple__ = version_tuple = (0, 3, 1)

bdms/acada_ingest_cli.py

@@ -256,6 +256,7 @@ def setup_logging(log_level, log_file=None):
     logging.getLogger("urllib3").setLevel(logging.WARNING)
     logging.getLogger("requests").setLevel(logging.WARNING)
     logging.getLogger("watchdog").setLevel(logging.WARNING)
+    logging.getLogger("charset_normalizer").setLevel(logging.WARNING)
 
 
 def create_ingestion_client(args) -> IngestionClient:

bdms/acada_ingestion.py

@@ -13,13 +13,12 @@ import threading
 import time
 from concurrent.futures import Future, ProcessPoolExecutor
 from contextlib import ExitStack
-from enum import Enum
 from functools import partial
 from multiprocessing import cpu_count
 from pathlib import Path
 from queue import Empty, Queue
 from traceback import format_exception
-from typing import Optional, Union
+from typing import NamedTuple, Optional, Union
 
 from astropy.io import fits
 from filelock import FileLock, Timeout
@@ -42,11 +41,10 @@ from bdms.extract_fits_metadata import (
 
 LOGGER = logging.getLogger(__name__)
 
-__all__ = ["IngestionClient", "FITSVerificationError", "Ingest"]
+__all__ = ["IngestionClient", "FITSVerificationError", "Ingest", "IngestResult"]
 
 INGEST_RUNNING_MESSAGE = "Another ingestion process is already running"
 DETECTED_NEW_TRIGGER_FILE = "Detected new trigger file"
-INGEST_SUCCESS_MESSAGE = "Successfully ingested"
 TRIGGER_SUFFIX = ".trigger"
 
 # Prometheus Metrics for monitoring
@@ -58,8 +56,15 @@ N_TASKS_PROCESSED = Counter(
     "n_tasks_processed", "Total number of tasks processed by the Ingest daemon"
 )
 TASKS_IN_QUEUE = Gauge("n_tasks_queued", "Current number of queued tasks")
+BYTES_INGESTED = Counter("bytes_ingested", "Total ingested file size")
 
-IngestStatus = Enum("IngestStatus", [("SUCCESS", 0), ("FAILURE", 1), ("SKIPPED", 2)])
+
+class IngestResult(NamedTuple):
+    """Result of the ingestion of a single file."""
+
+    lfn: str
+    skipped: bool
+    file_size: int
 
 
 class IngestionClient:
@@ -257,7 +262,7 @@ class IngestionClient:
             return True
         return False
 
-    def add_onsite_replica(self, acada_path: Union[str, Path]) -> tuple[str, bool]:
+    def add_onsite_replica(self, acada_path: Union[str, Path]) -> IngestResult:
         """Register a file as a replica in Rucio on the specified RSE and retrieve its LFN.
 
         Parameters
@@ -295,7 +300,7 @@ class IngestionClient:
         # Check if the replica already exists
         if self.check_replica_exists(lfn):
             self.logger.info("Replica already exists for lfn '%s', skipping", lfn)
-            return lfn, True  # Indicate the file was skipped
+            return IngestResult(lfn=lfn, skipped=True, file_size=0)
 
         # Proceed with registering the replica if check_replica_exists returns False
         valid, metadata = verify_and_extract_metadata(acada_path)
@@ -328,7 +333,7 @@ class IngestionClient:
             self.did_client.set_metadata_bulk(scope=self.scope, name=lfn, meta=metadata)
             self.logger.info("Set metadata of %r to %r", lfn, metadata)
 
-        return lfn, False  # Indicate the file was newly ingested
+        return IngestResult(lfn=lfn, skipped=False, file_size=file_size)
 
     def add_offsite_replication_rules(
         self,
@@ -512,7 +517,7 @@ def verify_and_extract_metadata(fits_path):
 
 def process_file(
     client: IngestionClient, file_path: str, logger=None, copies: int = 2
-) -> IngestStatus:
+) -> IngestResult:
     """Process a single file with IngestionClient, clean up the trigger file, and return the ingestion status.
 
     Parameters
@@ -533,24 +538,17 @@ def process_file(
         - SKIPPED if the file was already ingested.
     """
     logger = logger or LOGGER.getChild("Ingest")
+    result = client.add_onsite_replica(file_path)
+
+    if not result.skipped:
+        client.add_offsite_replication_rules(result.lfn, copies=copies)
+
     trigger_file = Path(file_path + TRIGGER_SUFFIX)
-    try:
-        lfn, was_skipped = client.add_onsite_replica(file_path)
-        if was_skipped:
-            logger.info("Replica already exists for %s, skipping", file_path)
-            if trigger_file.exists():
-                trigger_file.unlink()
-                logger.debug("Removed trigger file %s", trigger_file)
-            return IngestStatus.SKIPPED
-        client.add_offsite_replication_rules(lfn, copies=copies)
-        logger.info("%s %s, LFN: %s", INGEST_SUCCESS_MESSAGE, file_path, lfn)
-        if trigger_file.exists():
-            trigger_file.unlink()
-            logger.debug("Removed trigger file %s", trigger_file)
-        return IngestStatus.SUCCESS
-    except Exception as e:
-        logger.exception("Exception in process_file for %s: %s", file_path, str(e))
-        return IngestStatus.FAILURE
+    if trigger_file.exists():
+        trigger_file.unlink()
+        logger.debug("Removed trigger file %s", trigger_file)
+
+    return result
 
 
 class TriggerFileHandler(FileSystemEventHandler):
@@ -756,7 +754,7 @@ class Ingest:
         rate = processed_count / elapsed_time if elapsed_time > 0 else 0
 
         # Clean up task tracking
-        self.submitted_tasks.pop(task_id, None)
+        path = self.submitted_tasks.pop(task_id, None)
         current_concurrent = len(self.submitted_tasks)
         TASKS_IN_QUEUE.dec()  # Always decrement queue counter
 
@@ -767,30 +765,33 @@ class Ingest:
         )
 
         # Process the result
+        # the order here is important, as the methods on the future object
+        # raise if the future is in the wrong state
         if future.cancelled():
             status = "cancelled"
-            N_TASKS_CANCELLED.inc()  # Increment cancellation counter
+            N_TASKS_CANCELLED.inc()
+
         elif (e := future.exception()) is not None:
             self.logger.error(
-                "Task %d failed: %s",
+                "Task %d for path %s failed: %s",
                 task_id,
+                path,
                 "".join(format_exception(type(e), e, e.__traceback__)),
             )
             status = "failed"
-            N_TASKS_FAILED.inc()  # Increment failure counter
+            N_TASKS_FAILED.inc()
+
         else:
             result = future.result()
-            if result == IngestStatus.SUCCESS:
+            if not result.skipped:
                 status = "success"
-                N_TASKS_SUCCESS.inc()  # Increment success counter
-            elif result == IngestStatus.SKIPPED:
-                status = "skipped"
-                N_TASKS_SKIPPED.inc()  # Increment skipped counter
+                N_TASKS_SUCCESS.inc()
+                BYTES_INGESTED.inc(result.file_size)
             else:
-                status = "failed"
-                N_TASKS_FAILED.inc()  # Increment failure counter
+                status = "skipped"
+                N_TASKS_SKIPPED.inc()
 
-        N_TASKS_PROCESSED.inc()  # Increment total processed counter
+        N_TASKS_PROCESSED.inc()
 
         # Summary log for all cases
         self.logger.info(
@@ -896,8 +897,8 @@ class Ingest:
             result_thread.start()
             self.logger.info("Started result processing thread")
 
-            with ProcessPoolExecutor(max_workers=self.num_workers) as executor:
-                self.executor = executor  # Store reference for submit_file method
+            self.executor = ProcessPoolExecutor(max_workers=self.num_workers)
+            with self.executor:
                 self.logger.info(
                     "Started process pool with %d workers", self.num_workers
                 )

bdms/tests/test_acada_ingestion.py

@@ -29,11 +29,9 @@ from watchdog.events import FileMovedEvent
 
 from bdms.acada_ingestion import (
     DETECTED_NEW_TRIGGER_FILE,
-    INGEST_SUCCESS_MESSAGE,
     TRIGGER_SUFFIX,
     Ingest,
     IngestionClient,
-    IngestStatus,
     TriggerFileHandler,
     process_file,
 )
@@ -238,7 +236,8 @@ def test_add_onsite_replica_with_minio_fits_file(
     reset_xrootd_permissions(storage_mount_path)
 
     # Use add_onsite_replica to register the replica
-    lfn, skipped = ingestion_client.add_onsite_replica(acada_path=acada_path)
+    lfn, skipped, size = ingestion_client.add_onsite_replica(acada_path=acada_path)
+    assert size == os.stat(acada_path).st_size
 
     # Verify the LFN matches the expected LFN
     expected_lfn = ingestion_client.acada_to_lfn(acada_path)
@@ -267,11 +266,12 @@ def test_add_onsite_replica_with_minio_fits_file(
 
     # Check for don't ingest again if its already registered
     caplog.clear()
-    lfn_check, skipped_check = ingestion_client.add_onsite_replica(
+    lfn_check, skipped_check, size = ingestion_client.add_onsite_replica(
         acada_path=acada_path
     )
     msg = f"LFN mismatch on second ingestion attempt: expected {lfn}, got {lfn_check}"
     assert lfn_check == lfn, msg
+    assert size == 0, "Expected size 0 for skipped file"
 
     msg = (
         "Expected the file to be skipped on second ingestion, but it was ingested again"
@@ -672,14 +672,17 @@ def test_process_file_success(
         scope=test_scope,
     )
 
-    acada_path, _ = onsite_test_file
+    acada_path, test_file_content = onsite_test_file
     test_file = acada_path
     trigger_file = Path(str(test_file) + TRIGGER_SUFFIX)
     trigger_file.symlink_to(test_file)
     result = process_file(ingestion_client, str(test_file))
-    assert result == IngestStatus.SUCCESS
+
+    assert result.file_size == len(test_file_content)
+    assert not result.skipped
     assert not trigger_file.exists()
-    assert INGEST_SUCCESS_MESSAGE in caplog.text
+    assert "Successfully registered the replica for lfn" in caplog.text
+    assert "Created 2 offsite replication rule(s) for LFN" in caplog.text
 
 
 @pytest.mark.usefixtures("_auth_proxy")
@@ -694,19 +697,26 @@ def test_process_file_skipped(
         scope=test_scope,
     )
 
-    acada_path, _ = onsite_test_file
+    acada_path, test_file_content = onsite_test_file
     test_file = acada_path
     trigger_file = Path(str(test_file) + TRIGGER_SUFFIX)
     trigger_file.symlink_to(test_file)
-    process_file(ingestion_client, str(test_file))
+
+    # process file for the first time
+    result = process_file(ingestion_client, str(test_file))
+    assert not result.skipped
+    assert result.file_size == len(test_file_content)
+
     caplog.clear()
+    # process file second time to verify it is skipped
     result = process_file(ingestion_client, str(test_file))
-    assert result == IngestStatus.SKIPPED
+    assert result.skipped
+    assert result.file_size == 0
     assert "Replica already exists" in caplog.text
 
 
 @pytest.mark.usefixtures("_auth_proxy")
-def test_process_file_failure(storage_mount_path, caplog, tmp_path):
+def test_process_file_failure(storage_mount_path, tmp_path):
     """Test for checking failure for invalid file paths"""
     ingestion_client = IngestionClient(
         data_path=storage_mount_path,
@@ -721,18 +731,8 @@ def test_process_file_failure(storage_mount_path, caplog, tmp_path):
     trigger_file.symlink_to(invalid_file)
 
     # The file path is outside the data_path causing a ValueError in acada_to_lfn
-    result = process_file(ingestion_client, str(invalid_file))
-
-    # Verify the function returns FAILURE status instead of raising an exception
-    assert result == IngestStatus.FAILURE
-
-    # Check for the actual error message that gets logged
-    assert "Exception in process_file" in caplog.text
-    # Verify the file path is in the error message
-    assert str(invalid_file) in caplog.text
-
-    # Verify that no success message was logged
-    assert INGEST_SUCCESS_MESSAGE not in caplog.text
+    with pytest.raises(ValueError, match="is not within data_path"):
+        process_file(ingestion_client, str(invalid_file))
 
     # Trigger file should still exist since ingestion failed
     msg = "Trigger file should not be removed when ingestion fails"

{ctao_bdms_clients-0.3.0rc1.dist-info → ctao_bdms_clients-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ctao-bdms-clients
-Version: 0.3.0rc1
+Version: 0.3.1
 Summary: Client module for the CTAO DPPS Bulk Data Management System
 Author-email: Georgios Zacharis <georgios.zacharis@inaf.it>, Stefano Gallozzi <Stefano.gallozzi@inaf.it>, Michele Mastropietro <michele.mastropietro@inaf.it>, Syed Anwar Ul Hasan <syedanwarul.hasan@cta-consortium.org>, Maximilian Linhoff <maximilian.linhoff@cta-observatory.org>, Volodymyr Savchenko <Volodymyr.Savchenko@epfl.ch>
 License-Expression: BSD-3-Clause

{ctao_bdms_clients-0.3.0rc1.dist-info → ctao_bdms_clients-0.3.1.dist-info}/RECORD

@@ -1,13 +1,13 @@
 bdms/__init__.py,sha256=7btE6tNhFqXSv2eUhZ-0m1J3nTTs4Xo6HWcQI4eh5Do,142
-bdms/_version.py,sha256=ymwdyKB404aMzKXrx7y01ltePvHS_nIJjfh_zAIIN44,521
-bdms/acada_ingest_cli.py,sha256=xkf9nT5Lk7SjcbxVeBpKJWuJ-8Luze5-MSq4yki-7_k,12866
-bdms/acada_ingestion.py,sha256=mB5ilvzJbPblFp94Jcca-IzYvrMuQlroDZxuujpFB_I,36373
+bdms/_version.py,sha256=lOWWIGJeBi0KkFopWU_n3GH71C1PsaZ-ZYDfxFkne6c,511
+bdms/acada_ingest_cli.py,sha256=_AksS4NFhG6SsHbhjFnO-pgEKsIuTDFX7m_iraK-C7A,12936
+bdms/acada_ingestion.py,sha256=Z44s9meJC2OSJ-pL0cxHiDHmwF33xYNaTbwcqKq00VE,35782
 bdms/extract_fits_metadata.py,sha256=ZGJQCFJCXkWg8N3CAb17GB-wwPj-wTvNg0JOS-MemZ0,3431
 bdms/version.py,sha256=mTfi1WzbIs991NyImM6mcMg1R39a6U1W2pKnk-Tt5Vw,765
 bdms/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bdms/tests/conftest.py,sha256=n7KN9foojCXDxFuZinI0MvhnSvLk5Mn7aFmjQKmO8eI,7364
 bdms/tests/test_acada_ingest_cli.py,sha256=SYVt1xlEDsrbPX0C5Isf0thjUcaxr7cjflyZSwpPBaw,8314
-bdms/tests/test_acada_ingestion.py,sha256=xQN07Qbx00IW_w0vCcR5r5H3qvvl_JNYmCUuWJX9xrc,59485
+bdms/tests/test_acada_ingestion.py,sha256=sJV7m_rzNTx2J7Zz5twArj7ME6Os4KiGMK7OI5Fm5Ko,59554
 bdms/tests/test_basic_rucio_functionality.py,sha256=9GIX8IO6wBJm40LKFEH2StS-fMKvC07sxFHPVR7dftU,3583
 bdms/tests/test_dpps_rel_0_0.py,sha256=2NhxpdhXQg_8lmK-tRrPQ_FcijsIEfv07x-kVlT8Zik,3138
 bdms/tests/test_extract_fits_metadata.py,sha256=A935WD2TF3lBcaeDmzGSlH2IXUF1v8qslrsW30lnEAA,3490
@@ -15,9 +15,9 @@ bdms/tests/test_file_replicas.py,sha256=NqutrSJa5ME50JpmyATNPSLqq1AOq1ruv84XSY3P
 bdms/tests/test_metadata.py,sha256=f0tSqNGlYe-ydoSDJw0k1De2kHoPl6g-GYBj_jP6kCY,3728
 bdms/tests/test_onsite_storage.py,sha256=waK7t9kBquzJbuLLYcpeNU9YuA70XTRS88RMxBWxawI,3765
 bdms/tests/utils.py,sha256=PUayWe60JGVDs5mkWmHVjFV_yqg5XUQlxoAvhT1P0OM,4101
-ctao_bdms_clients-0.3.0rc1.dist-info/licenses/LICENSE,sha256=Py9riZY_f0CmXbrZ5JreE3WgglyWkRnwUfqydvX6jxE,1556
-ctao_bdms_clients-0.3.0rc1.dist-info/METADATA,sha256=NRkliF-xYd9V8Vfkdgj2MEFQI9ee67wmUVKxhFD9tMo,2517
-ctao_bdms_clients-0.3.0rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ctao_bdms_clients-0.3.0rc1.dist-info/entry_points.txt,sha256=YZCIOePi_xXaJunA6lAQxAKh1tn3wOd4pmqymFRvah4,60
-ctao_bdms_clients-0.3.0rc1.dist-info/top_level.txt,sha256=ao0U8aA33KRHpcqmr7yrK8y2AQ6ahSu514tfaN4hDV8,5
-ctao_bdms_clients-0.3.0rc1.dist-info/RECORD,,
+ctao_bdms_clients-0.3.1.dist-info/licenses/LICENSE,sha256=Py9riZY_f0CmXbrZ5JreE3WgglyWkRnwUfqydvX6jxE,1556
+ctao_bdms_clients-0.3.1.dist-info/METADATA,sha256=U8xOFh-YkxRI9hqHftAvdDji4UbCPCPvsFwVLEUfYGo,2514
+ctao_bdms_clients-0.3.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ctao_bdms_clients-0.3.1.dist-info/entry_points.txt,sha256=YZCIOePi_xXaJunA6lAQxAKh1tn3wOd4pmqymFRvah4,60
+ctao_bdms_clients-0.3.1.dist-info/top_level.txt,sha256=ao0U8aA33KRHpcqmr7yrK8y2AQ6ahSu514tfaN4hDV8,5
+ctao_bdms_clients-0.3.1.dist-info/RECORD,,