das-setup-helper 0.1.0__py3-none-any.whl

das_setup_helper/__init__.py

@@ -0,0 +1,3 @@
+from das_setup_helper.shared.utils import set_environment_variables
+
+set_environment_variables()

das_setup_helper/data/fs/base.py

@@ -0,0 +1,72 @@
+from typing import Protocol
+
+from das_setup_helper.shared.context import Context
+
+
+class IOManager(Protocol):
+    """
+    Protocol describing minimal IO operations over a single data source.
+
+    Methods
+    -------
+    read(acquisition_id_or_s3_path)
+        Reads data from the specified acquisition ID or S3 path.
+    """
+
+    def __init__(self): ...
+
+    def read(self, acquisition_id_or_s3_path): ...
+
+
+class StorageIOManager:
+    """
+    Base class for storage-backed IOManagers (S3, local, …).
+
+    ``IOHandler`` uses ``isinstance(io, StorageIOManager)`` to dispatch
+    storage-related operations without being tied to a specific backend.
+    """
+
+    def __init__(self, context: Context):
+        self.context = context
+
+    def copy_file(self, file_path: str, new_file_path: str) -> None:
+        self.context.storage_connector.copy_file(file_path, new_file_path)
+
+    def get_crawler_configs(self) -> dict:
+        return self.context.storage_connector.get_crawler_configs()
+
+    def get_valid_wihp_config_to_create(self) -> list:
+        return self.context.storage_connector.get_wihp_sources_to_create()
+
+    def read(self, file_path: str):
+        return self.context.storage_connector.read(file_path)
+
+    def write(self, index_documents: list, file_path: str) -> bool:
+        return self.context.storage_connector.write_json(index_documents, file_path)
+
+    def save_acquisition_infos(self, acquisition_infos: dict) -> bool:
+        return self.context.storage_connector.save_acquisition_infos(acquisition_infos)
+
+    def save_crawler_info(
+        self,
+        sources_to_create,
+        duplicated_urls,
+        null_mne_names,
+        non_english_urls,
+        long_mne_names,
+        crawler_name: str,
+        crawler_id: str,
+        crawler_timestamp: str,
+        updated_sources=None,
+    ) -> bool:
+        return self.context.storage_connector.save_crawler_info(
+            sources_to_create,
+            duplicated_urls,
+            null_mne_names,
+            non_english_urls,
+            long_mne_names,
+            crawler_name,
+            crawler_id,
+            crawler_timestamp,
+            updated_sources,
+        )

das_setup_helper/data/fs/local.py

@@ -0,0 +1,16 @@
+"""
+IOManager implementation backed by the local filesystem.
+"""
+
+from das_setup_helper.data.fs.base import StorageIOManager
+
+__all__ = ["LocalIOManager"]
+
+
+class LocalIOManager(StorageIOManager):
+    """
+    IOManager implementation backed by the local filesystem.
+
+    Delegates to context.storage_connector (LocalStorageConnector).
+    Drop-in replacement for S3IOManager when STORAGE_BACKEND=local.
+    """

das_setup_helper/data/fs/s3.py

@@ -0,0 +1,12 @@
+from das_setup_helper.data.fs.base import StorageIOManager
+
+__all__ = ["S3IOManager"]
+
+
+class S3IOManager(StorageIOManager):
+    """
+    IOManager implementation backed by S3.
+
+    Provides the S3 operations needed by setup_crawler and create_acquisitions
+    pipelines: config loading, source validation, and result persistence.
+    """

das_setup_helper/data/fs/wihp_api.py

@@ -0,0 +1,46 @@
+from das_setup_helper.shared.context import Context
+
+__all__ = ["WihpAPIIOManager"]
+
+
+class WihpAPIIOManager:
+    """
+    IOManager implementation backed by the WIHP API.
+
+    Provides the crawler and acquisition lifecycle methods used by the
+    setup_crawler, create_acquisitions, and stop_acquisitions pipelines.
+    """
+
+    def __init__(self, context: Context):
+        self.context = context
+
+    def create_crawler(self, sources_to_create):
+        return self.context.wihp_api_connector.create_crawler_and_add_sources(
+            sources_to_create
+        )
+
+    def get_candidates_to_stopped(self) -> list:
+        return self.context.wihp_api_connector.get_candidates_acquisition_to_stopped()
+
+    def get_acquisition_status_counts(self, acquisition_id: str) -> list:
+        return self.context.wihp_api_connector.get_acquisition_status_counts(
+            acquisition_id
+        )
+
+    def select_acquisitions_to_stop_from_status_counts(
+        self,
+        acquisition_ids: list,
+    ) -> list:
+        return self.context.wihp_api_connector.select_acquisitions_to_stop_from_status_counts(
+            acquisition_ids,
+        )
+
+    def create_acquisitions(self, crawler_configs: dict) -> dict:
+        return self.context.wihp_api_connector.create_acquisitions(crawler_configs)
+
+    def stop_non_discovered_acquisition(
+        self, running_acquisitions_to_stop: list
+    ) -> None:
+        self.context.wihp_api_connector.stop_acquisitions(
+            running_acquisitions_to_stop,
+        )

das_setup_helper/data/io.py

@@ -0,0 +1,131 @@
+import logging
+
+from das_setup_helper.data.fs.base import IOManager, StorageIOManager
+from das_setup_helper.data.fs.wihp_api import WihpAPIIOManager
+from das_setup_helper.shared.context import Context
+from das_setup_helper.shared.logger import setup_logger
+
+logger = setup_logger("io", logging.DEBUG)
+
+
+class IOHandler:
+    """
+    Facade that routes I/O operations to the appropriate IOManager.
+
+    Only the methods required by the three public pipelines are exposed:
+
+    * setup_crawler     — create_crawler, get_validated_config,
+                          save_crawler_info
+    * create_acquisitions — get_crawler_configs, create_acquisitions,
+                            save_acquisition_infos
+    * stop_acquisitions — get_candidates_to_stopped,
+                          get_acquisition_status_counts,
+                          select_acquisitions_to_stop_from_status_counts,
+                          select_acquisitions_to_stop,
+                          stop_non_discovered_acquisition
+    """
+
+    def __init__(self, context: Context, io_manager: IOManager):
+        self.context = context
+        self.io = io_manager
+
+    # ------------------------------------------------------------------ #
+    # setup_crawler pipeline                                               #
+    # ------------------------------------------------------------------ #
+
+    def create_crawler(self, sources_to_create):
+        if not isinstance(self.io, WihpAPIIOManager):
+            msg = "Only WihpAPIIOManager supports create_crawler."
+            raise NotImplementedError(msg)
+        return self.io.create_crawler(sources_to_create)
+
+    def get_validated_config(self):
+        if not isinstance(self.io, StorageIOManager):
+            msg = "Only a StorageIOManager supports get_validated_config."
+            raise NotImplementedError(msg)
+        return self.io.get_valid_wihp_config_to_create()
+
+    def save_crawler_info(
+        self,
+        sources_to_create,
+        duplicated_urls,
+        null_mne_names,
+        non_english_urls,
+        long_mne_names,
+        crawler_name: str,
+        crawler_id: str,
+        crawler_timestamp: str,
+        updated_sources=None,
+    ):
+        if not isinstance(self.io, StorageIOManager):
+            msg = "Only a StorageIOManager supports save_crawler_info."
+            raise NotImplementedError(msg)
+        self.io.save_crawler_info(
+            sources_to_create,
+            duplicated_urls,
+            null_mne_names,
+            non_english_urls,
+            long_mne_names,
+            crawler_name,
+            crawler_id,
+            crawler_timestamp,
+            updated_sources,
+        )
+
+    # ------------------------------------------------------------------ #
+    # create_acquisitions pipeline                                         #
+    # ------------------------------------------------------------------ #
+
+    def get_crawler_configs(self) -> dict:
+        if not isinstance(self.io, StorageIOManager):
+            msg = "Only a StorageIOManager supports get_crawler_configs."
+            raise NotImplementedError(msg)
+        return self.io.get_crawler_configs()
+
+    def create_acquisitions(self, crawler_configs: dict) -> dict:
+        if not isinstance(self.io, WihpAPIIOManager):
+            msg = "Only WihpAPIIOManager supports create_acquisitions."
+            raise NotImplementedError(msg)
+        return self.io.create_acquisitions(crawler_configs)
+
+    def save_acquisition_infos(self, acquisition_infos: dict):
+        if not isinstance(self.io, StorageIOManager):
+            msg = "Only a StorageIOManager supports save_acquisition_infos."
+            raise NotImplementedError(msg)
+        self.io.save_acquisition_infos(acquisition_infos)
+
+    # ------------------------------------------------------------------ #
+    # stop_acquisitions pipeline                                         #
+    # ------------------------------------------------------------------ #
+
+    def get_candidates_to_stopped(self) -> list:
+        if not isinstance(self.io, WihpAPIIOManager):
+            msg = "Only WihpAPIIOManager supports get_candidates_to_stopped."
+            raise NotImplementedError(msg)
+        return self.io.get_candidates_to_stopped()
+
+    def get_acquisition_status_counts(self, acquisition_id: str) -> list:
+        if not isinstance(self.io, WihpAPIIOManager):
+            msg = "Only WihpAPIIOManager supports get_acquisition_status_counts."
+            raise NotImplementedError(msg)
+        return self.io.get_acquisition_status_counts(acquisition_id)
+
+    def select_acquisitions_to_stop_from_status_counts(
+        self,
+        acquisition_ids: list,
+    ) -> list:
+        if not isinstance(self.io, WihpAPIIOManager):
+            msg = (
+                "Only WihpAPIIOManager supports "
+                "select_acquisitions_to_stop_from_status_counts."
+            )
+            raise NotImplementedError(msg)
+        return self.io.select_acquisitions_to_stop_from_status_counts(
+            acquisition_ids,
+        )
+
+    def stop_non_discovered_acquisition(self, running_acquisitions_to_stop: list):
+        if not isinstance(self.io, WihpAPIIOManager):
+            msg = "Only WihpAPIIOManager supports stop_non_discovered_acquisition."
+            raise NotImplementedError(msg)
+        self.io.stop_non_discovered_acquisition(running_acquisitions_to_stop)

das_setup_helper/pipelines/create_acquisitions.py

@@ -0,0 +1,44 @@
+import asyncio
+import logging
+
+from das_setup_helper.data.fs.s3 import S3IOManager
+from das_setup_helper.data.fs.wihp_api import WihpAPIIOManager
+from das_setup_helper.data.io import IOHandler
+from das_setup_helper.shared.context import Context
+from das_setup_helper.shared.logger import setup_logger
+
+logger = setup_logger("create_acquisitions", logging.DEBUG)
+
+
+async def create_and_trigger_acquisitions():
+    """
+    Create and trigger new acquisitions in WIHP.
+
+    Retrieves crawler configurations from S3, creates new acquisitions in WIHP
+    via the API, saves the acquisition information back to S3, and returns the
+    acquisition details.
+
+    Returns:
+        dict: Acquisition information keyed by acquisition ID.
+    """
+    logger.info("Triggering new acquisitions under WIHP")
+    context = Context()
+
+    s3 = IOHandler(context, S3IOManager(context))
+    crawler_configs = s3.get_crawler_configs()
+
+    wihp_api = IOHandler(context, WihpAPIIOManager(context))
+    acquisition_infos = wihp_api.create_acquisitions(crawler_configs)
+
+    s3.save_acquisition_infos(acquisition_infos)
+    logger.info(f"New acquisitions triggered under WIHP:\n{acquisition_infos}")
+
+    return acquisition_infos
+
+
+def main():
+    asyncio.run(create_and_trigger_acquisitions())
+
+
+if __name__ == "__main__":
+    main()

das_setup_helper/pipelines/setup_crawler.py

@@ -0,0 +1,86 @@
+import asyncio
+import logging
+
+from das_setup_helper.data.fs.s3 import S3IOManager
+from das_setup_helper.data.fs.wihp_api import WihpAPIIOManager
+from das_setup_helper.data.io import IOHandler
+from das_setup_helper.shared.constant.s3 import (
+    CRAWLER_INFO_ID_KEY,
+    CRAWLER_INFO_NAME_KEY,
+    CRAWLER_INFO_TIMESTAMP_KEY,
+)
+from das_setup_helper.shared.context import Context
+from das_setup_helper.shared.logger import setup_logger
+
+__all__ = ["create_crawler"]
+
+logger = setup_logger("setup_crawler", logging.DEBUG)
+
+
+async def create_crawler():
+    """
+    Create a new crawler in WIHP.
+
+    Retrieves validated sources from S3, creates a new crawler in WIHP,
+    saves the crawler information back to S3, and returns the crawler details.
+
+    Returns:
+        dict | None: Crawler name, ID, and timestamp, or None if no
+            sources found.
+    """
+    logger.info("Creating a new crawler in WIHP")
+    context = Context()
+
+    s3 = IOHandler(context, S3IOManager(context))
+    (
+        sources_to_create,
+        duplicated_urls,
+        null_mne_names,
+        non_english_urls,
+        long_mne_names,
+    ) = s3.get_validated_config()
+
+    if sources_to_create is None:
+        logger.info("No new sources found to create. Exiting crawler creation process.")
+        return None
+
+    wihp_api = IOHandler(context, WihpAPIIOManager(context))
+    (
+        crawler_name,
+        crawler_id,
+        crawler_timestamp,
+        updated_sources,
+    ) = wihp_api.create_crawler(
+        sources_to_create,
+    )
+
+    s3.save_crawler_info(
+        sources_to_create,
+        duplicated_urls,
+        null_mne_names,
+        non_english_urls,
+        long_mne_names,
+        crawler_name,
+        crawler_id,
+        crawler_timestamp,
+        updated_sources,
+    )
+
+    crawler_infos = {
+        CRAWLER_INFO_NAME_KEY: crawler_name,
+        CRAWLER_INFO_ID_KEY: crawler_id,
+        CRAWLER_INFO_TIMESTAMP_KEY: crawler_timestamp,
+    }
+    logger.info(
+        f"New Crawler created in WIHP — name: {crawler_name}, "
+        f"id: {crawler_id}, timestamp: {crawler_timestamp}"
+    )
+    return crawler_infos
+
+
+def main():
+    asyncio.run(create_crawler())
+
+
+if __name__ == "__main__":
+    main()

das_setup_helper/pipelines/stop_acquisitions.py

@@ -0,0 +1,46 @@
+import asyncio
+import logging
+
+from das_setup_helper.data.fs.wihp_api import WihpAPIIOManager
+from das_setup_helper.data.io import IOHandler
+from das_setup_helper.shared.context import Context
+from das_setup_helper.shared.logger import setup_logger
+
+__all__ = ["stopped_acquisitions"]
+
+logger = setup_logger("stop_acquisitions", logging.DEBUG)
+
+
+async def stopped_acquisitions():
+    """
+    Stop running acquisitions that have a non-DISCOVERED index status.
+
+    Retrieves running acquisition candidates from the WIHP API, selects those
+    whose DAS stats have no discovered URLs, then calls the stop endpoint.
+    """
+    logger.info("Starting job to stop acquisitions with none 'DISCOVERED' status")
+    context = Context()
+
+    wihp_api = IOHandler(context, WihpAPIIOManager(context))
+    running_acquisitions_candidate_to_stop = wihp_api.get_candidates_to_stopped()
+
+    if not running_acquisitions_candidate_to_stop:
+        logger.info("No running acquisitions found to inspect.")
+        return
+
+    running_acquisitions_to_stop = (
+        wihp_api.select_acquisitions_to_stop_from_status_counts(
+            running_acquisitions_candidate_to_stop,
+        )
+    )
+
+    wihp_api.stop_non_discovered_acquisition(running_acquisitions_to_stop)
+    logger.info("Succeeded in stopping acquisitions with none 'DISCOVERED' status")
+
+
+def main():
+    asyncio.run(stopped_acquisitions())
+
+
+if __name__ == "__main__":
+    main()