mccode-plumber 0.6.0__py3-none-any.whl → 0.7.1__py3-none-any.whl

mccode_plumber/file_writer_control/WorkerFinder.py

@@ -0,0 +1,139 @@
+import uuid
+from datetime import datetime
+from typing import Dict, List, Optional
+
+from kafka import KafkaProducer
+from kafka.errors import NoBrokersAvailable
+from streaming_data_types.run_stop_6s4t import serialise_6s4t as serialise_stop
+
+from file_writer_control.CommandChannel import CommandChannel
+from file_writer_control.CommandHandler import CommandHandler
+from file_writer_control.CommandStatus import CommandStatus
+from file_writer_control.JobStatus import JobState, JobStatus
+from file_writer_control.KafkaTopicUrl import KafkaTopicUrl
+from file_writer_control.WorkerStatus import WorkerStatus
+from file_writer_control.WriteJob import WriteJob
+
+
+class WorkerFinderBase:
+    def __init__(
+        self,
+        command_topic: str,
+        command_channel: CommandChannel,
+        message_producer: KafkaProducer,
+    ):
+        """
+        Constructor.
+        """
+        self.command_channel = command_channel
+        self.command_topic = command_topic
+        self.message_producer = message_producer
+
+    def send_command(self, message: bytes):
+        """
+        Send a message (command) to the file-writer "command"-topic.
+        :param message: The command/message as binary data.
+        """
+        self.message_producer.send(self.command_topic, message)
+
+    def try_start_job(self, job: WriteJob) -> CommandHandler:
+        """
+        Attempts to start a file-writing job. This function is not blocking. No guarantees are given that the job will
+        be successfully started.
+        .. note:: This class must be implemented by the classes inheriting from this one.
+        :param job: The file-writing job to be started.
+        :return: A CommandHandler instance for (more) easily checking the outcome of attempting to start a write job.
+        """
+        raise NotImplementedError("Not implemented in base class.")
+
+    def try_send_stop_time(
+        self, service_id: Optional[str], job_id: str, stop_time: datetime
+    ) -> CommandHandler:
+        """
+        Sends a "set stop time" message to a file-writer running a job as identified by the parameters.
+        This function is not blocking. No guarantees are given that this command will be followed.
+        :param service_id: The (optional) service identifier of the file-writer to receive the command.
+        :param job_id: The job identifier of the currently running file-writer job.
+        :param stop_time: The new stop time.
+        :return: A CommandHandler instance for (more) easily checking the outcome of setting a new stop time.
+        """
+        command_id = str(uuid.uuid1())
+        message = serialise_stop(
+            job_id=job_id,
+            service_id=service_id,
+            command_id=command_id,
+            stop_time=stop_time,
+        )
+        self.command_channel.add_command_id(job_id=job_id, command_id=command_id)
+        self.send_command(message)
+        return CommandHandler(self.command_channel, command_id)
+
+    def try_send_stop_now(
+        self, service_id: Optional[str], job_id: str
+    ) -> CommandHandler:
+        """
+        See documentation for `try_send_abort()`.
+        """
+        return self.try_send_abort(service_id, job_id)
+
+    def try_send_abort(self, service_id: Optional[str], job_id: str) -> CommandHandler:
+        """
+        Sends a "abort" message to a file-writer running a job as identified by the parameters of this function.
+        This function is not blocking. No guarantees are given that this command will be followed.
+        :param service_id: The (optional) service identifier of the file-writer to receive the command.
+        :param job_id: The job identifier of the currently running file-writer job.
+        :return: A CommandHandler instance for (more) easily checking the outcome of the "abort" command.
+        """
+        return self.try_send_stop_time(service_id, job_id, 0)
+
+    def list_known_workers(self) -> List[WorkerStatus]:
+        """
+        :return: A list of the (known) status of the workers publishing status updates to the configured command topic.
+        """
+        return self.command_channel.list_workers()
+
+    def list_known_jobs(self) -> List[JobStatus]:
+        """
+        :return: A list of the (known) jobs and their status as published on the configured command topic.
+        """
+        return self.command_channel.list_jobs()
+
+    def list_known_commands(self) -> List[CommandStatus]:
+        """
+        :return: A list of the (known) commands and their outcomes as published on the configured command topic.
+        """
+        return self.command_channel.list_commands()
+
+    def get_job_state(self, job_id: str) -> JobState:
+        """
+        Get the state of a specific job.
+        :param job_id: The (unique) identifier of the job that we are trying to find the state of.
+        :return: The state of the job if known, JobState.UNAVAILABLE if job is not known.
+        """
+        current_job = self.command_channel.get_job(job_id)
+        if current_job is None:
+            return JobState.UNAVAILABLE
+        return current_job.state
+
+    def get_job_status(self, job_id: str) -> JobStatus:
+        """
+        Get the full (known) status of a specific job.
+        :param job_id: The (unique) identifier of the job that we are trying to find the status of.
+        :return: The status of the job if known. None if it is not.
+        """
+        return self.command_channel.get_job(job_id)
+
+
+class WorkerFinder(WorkerFinderBase):
+    def __init__(self, command_topic_url: str, kafka_config: Dict[str, str] = {}):
+        temp_cmd_ch = CommandChannel(command_topic_url, kafka_config=kafka_config)
+        command_url = KafkaTopicUrl(command_topic_url)
+        try:
+            temp_producer = KafkaProducer(
+                bootstrap_servers=[command_url.host_port], **kafka_config
+            )
+        except NoBrokersAvailable as e:
+            raise NoBrokersAvailable(
+                f'Unable to find brokers (or connect to brokers) on address: "{command_url.host_port}"'
+            ) from e
+        super().__init__(command_url.topic, temp_cmd_ch, temp_producer)

mccode_plumber/file_writer_control/WorkerJobPool.py

@@ -0,0 +1,70 @@
+from typing import Dict
+
+from kafka import KafkaProducer
+from kafka.errors import NoBrokersAvailable
+
+from file_writer_control.CommandHandler import CommandHandler
+from file_writer_control.CommandStatus import CommandState
+from file_writer_control.KafkaTopicUrl import KafkaTopicUrl
+from file_writer_control.WorkerFinder import WorkerFinder
+from file_writer_control.WriteJob import WriteJob
+
+
+class WorkerJobPool(WorkerFinder):
+    """
+    A child of WorkerFinder intended for use with "worker pool" style of starting a file-writing job.
+    """
+
+    def __init__(
+        self,
+        job_topic_url: str,
+        command_topic_url: str,
+        max_message_size: int = 1048576 * 200,
+        kafka_config: Dict[str, str] = {},
+    ):
+        """
+        :param job_topic_url: The Kafka topic that the available file-writers are listening to for write jobs.
+        :param command_topic_url: The Kafka topic that a file-writer uses to send status updates to and receive direct
+        commands from.
+        :param max_message_size: The maximum message (actually "request") size.
+        """
+        super().__init__(command_topic_url, kafka_config=kafka_config)
+        self._job_pool = KafkaTopicUrl(job_topic_url)
+        self._max_message_size = max_message_size
+        try:
+            self._pool_producer = KafkaProducer(
+                bootstrap_servers=[self._job_pool.host_port],
+                max_request_size=max_message_size,
+                buffer_memory=max_message_size,
+                **kafka_config,
+            )
+        except NoBrokersAvailable as e:
+            raise NoBrokersAvailable(
+                f'Unable to find brokers (or connect to brokers) on address: "{self._job_pool.host_port}"'
+            ) from e
+
+    def _send_pool_message(self, message: bytes):
+        """
+        Send a message to the Kafka topic that is configured as the job-pool topic.
+        .. note:: If the file-writer has been configured properly, it will only accept start-job messages to this topic.
+        :param message: The binary data of the message.
+        """
+        if len(message) >= self._max_message_size:
+            raise RuntimeError(
+                f"Unable to send Kafka message as message size is too large ({len(message)} vs"
+                f"{self._max_message_size} bytes). Increase max message size with the 'max_message_size'"
+                f"constructor argument."
+            )
+        self._pool_producer.send(self._job_pool.topic, message)
+
+    def try_start_job(self, job: WriteJob) -> CommandHandler:
+        """
+        See base class for documentation.
+        """
+        self.command_channel.add_job_id(job.job_id)
+        self.command_channel.add_command_id(job.job_id, job.job_id)
+        self.command_channel.get_command(
+            job.job_id
+        ).state = CommandState.WAITING_RESPONSE
+        self._send_pool_message(job.get_start_message())
+        return CommandHandler(self.command_channel, job.job_id)

mccode_plumber/file_writer_control/WorkerStatus.py

@@ -0,0 +1,88 @@
+from datetime import datetime, timedelta
+from enum import Enum, auto
+from typing import Optional
+
+DEFAULT_TIMEOUT = timedelta(seconds=15)
+
+
+class WorkerState(Enum):
+    """
+    The state of a worker (i.e. a file-writer instance).
+    """
+
+    IDLE = auto()
+    WRITING = auto()
+    UNKNOWN = auto()
+    UNAVAILABLE = auto()
+
+
+class WorkerStatus(object):
+    """
+    Contains general status information about a worker.
+    """
+
+    def __init__(self, service_id: str, timeout: Optional[timedelta] = DEFAULT_TIMEOUT):
+        self._last_update = datetime.now()
+        self._service_id = service_id
+        self._timeout = timeout
+        self._state = WorkerState.UNAVAILABLE
+
+    def __eq__(self, other_status: "WorkerStatus") -> bool:
+        if not isinstance(other_status, WorkerStatus):
+            raise NotImplementedError
+        return (
+            self.service_id == other_status.service_id
+            and self.state == other_status.state
+        )
+
+    def update_status(self, new_status: "WorkerStatus"):
+        """
+        Updates the status/state of this instance of the WorkerStatus class using another instance.
+        .. note:: The service identifier of both this instance and the other one must be identical.
+        :param new_status: The other instance of the WorkerStatus class.
+        """
+        if new_status.service_id != self.service_id:
+            raise RuntimeError(
+                f"Service id of status update is not correct ({self.service_id} vs {new_status.service_id})"
+            )
+        self._state = new_status.state
+        self._last_update = new_status.last_update
+
+    def check_if_outdated(self, current_time: datetime):
+        """
+        Given the current time, state and the time of the last update: Have we lost the connection?
+        :param current_time: The current time
+        """
+        if (
+            self.state != WorkerState.UNAVAILABLE
+            and current_time - self.last_update > self._timeout
+        ):
+            self._state = WorkerState.UNAVAILABLE
+            self._last_update = current_time
+
+    @property
+    def state(self) -> WorkerState:
+        """
+        The current state of the worker.
+        """
+        return self._state
+
+    @property
+    def service_id(self) -> str:
+        """
+        The service identifier of the worker that this instance of the WorkerState class represent.
+        """
+        return self._service_id
+
+    @property
+    def last_update(self) -> datetime:
+        """
+        The local time stamp of the last update of the status of the file-writer instance that this instance of the
+        WorkerStatus class represents.
+        """
+        return self._last_update
+
+    @state.setter
+    def state(self, new_state: WorkerState):
+        self._last_update = datetime.now()
+        self._state = new_state

mccode_plumber/file_writer_control/WriteJob.py

@@ -0,0 +1,83 @@
+import uuid
+from datetime import datetime, timedelta
+
+from streaming_data_types import serialise_pl72
+
+
+class WriteJob:
+    """
+    Represents a file-writer write job (before it has been started).
+    """
+
+    def __init__(
+        self,
+        nexus_structure: str,
+        file_name: str,
+        broker: str,
+        start_time: datetime,
+        stop_time: datetime = None,
+        job_id="",
+        instrument_name: str = "",
+        run_name: str = "",
+        metadata: str = "",
+        control_topic: str = "",
+    ):
+        self.structure = nexus_structure
+        self.file = file_name
+        if job_id:
+            try:
+                uuid.UUID(job_id)
+                self.job_id = job_id
+            except ValueError as e:
+                raise RuntimeError("Job ID should be a valid UUID (v1).") from e
+        else:
+            self.job_id = str(uuid.uuid1())
+        self.start = start_time
+        if stop_time is None:
+            self.stop = self.start + timedelta(days=365.25 * 10)
+        else:
+            self.stop = stop_time
+        self._service_id = ""
+        self.broker = broker
+        self.instrument_name = instrument_name
+        self.run_name = run_name
+        self.metadata = metadata
+        self.control_topic = control_topic
+
+    def generate_new_job_id(self):
+        """
+        Generate a new job id. Should be called if an attempt at starting this write job fails and another attempt is made.
+        """
+        self.job_id = str(uuid.uuid1())
+
+    @property
+    def service_id(self) -> str:
+        """
+        The service identifier that should process this job. Defaults to an empty string.
+        .. note:: Must be set if job is to be processed by a specific file-writer instance.
+        """
+        return self._service_id
+
+    @service_id.setter
+    def service_id(self, new_service_id: str):
+        self._service_id = new_service_id
+
+    def get_start_message(self) -> bytes:
+        """
+        Generate the (flatbuffer) start message that will start this job. If you are sending the message to a specific
+        file-writer instance, you have to set the service_id property first or the message will fail to start the job.
+        :return: A flatbuffer message that holds the necessary information for starting a write job.
+        """
+        return serialise_pl72(
+            self.job_id,
+            self.file,
+            self.start,
+            self.stop,
+            nexus_structure=self.structure,
+            service_id=self.service_id,
+            broker=self.broker,
+            instrument_name=self.instrument_name,
+            run_name=self.run_name,
+            metadata=self.metadata,
+            control_topic=self.control_topic,
+        )

mccode_plumber/file_writer_control/__init__.py

@@ -0,0 +1,13 @@
+from .CommandStatus import CommandState
+from .JobHandler import JobHandler
+from .JobStatus import JobState
+from .WorkerJobPool import WorkerJobPool
+from .WriteJob import WriteJob
+
+__all__ = [
+    "JobHandler",
+    "WorkerJobPool",
+    "WriteJob",
+    "CommandState",
+    "JobState",
+]

mccode_plumber/writer.py

@@ -138,7 +138,7 @@ def insert_events_in_nexus_structure(ns: dict, config: dict):
 
 
 def get_writer_pool(broker: str = None, job: str = None, command: str = None):
-    from file_writer_control import WorkerJobPool
+    from .file_writer_control import WorkerJobPool
     pool = WorkerJobPool(f"{broker}/{job}", f"{broker}/{command}")
     return pool
 
@@ -175,7 +175,7 @@ def start_pool_writer(start_time_string, structure, filename=None, stop_time_str
     from time import sleep
     from json import dumps
     from datetime import datetime, timedelta
-    from file_writer_control import JobHandler, WriteJob, CommandState
+    from .file_writer_control import JobHandler, WriteJob, CommandState
 
     start_time = datetime.fromisoformat(start_time_string)
     if filename is None:
@@ -301,7 +301,7 @@ def wait_on_writer():
     from os import EX_OK, EX_UNAVAILABLE
     from time import sleep
     from datetime import datetime, timedelta
-    from file_writer_control import JobHandler, CommandState
+    from .file_writer_control import JobHandler, CommandState
 
     from argparse import ArgumentParser
     parser = ArgumentParser()

{mccode_plumber-0.6.0.dist-info → mccode_plumber-0.7.1.dist-info}/METADATA

@@ -1,11 +1,12 @@
 Metadata-Version: 2.4
 Name: mccode-plumber
-Version: 0.6.0
+Version: 0.7.1
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 Classifier: License :: OSI Approved :: BSD License
 Description-Content-Type: text/markdown
 Requires-Dist: p4p
-Requires-Dist: file-writer-control>=1.3.0
+Requires-Dist: kafka-python>=2.0
+Requires-Dist: ess-streaming-data-types>=0.14.0
 Requires-Dist: restage>=0.4.0
 Requires-Dist: mccode-to-kafka>=0.2.1
 Requires-Dist: moreniius>=0.2.3

mccode_plumber-0.7.1.dist-info/RECORD

@@ -0,0 +1,27 @@
+mccode_plumber/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mccode_plumber/conductor.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mccode_plumber/epics.py,sha256=5j_ictmDloxUGbGNe2ESlkSDQsPknzgPAbUsoZEUWO0,4441
+mccode_plumber/forwarder.py,sha256=1PdW91qLksDnA41-gOFMtgSae4wDdSLVvgAlrF23Uks,3764
+mccode_plumber/kafka.py,sha256=8fHTy2Zzk638XXj29pfrk-7fUxafuse9OYWmaRgvV7M,1195
+mccode_plumber/mccode.py,sha256=WXlu89Cs6a3s0aKtfRGuzqcOWoFZtdZWibLgBQnaXCI,2210
+mccode_plumber/splitrun.py,sha256=MsR1cUvoayQGNCJNl5fIPG92Wfcy9V_8QNO_ITQTvtA,1295
+mccode_plumber/utils.py,sha256=E8NoGo_3Z-pPEpzicVGSWfQOX8p3eR-GxElT33-kX5U,2167
+mccode_plumber/writer.py,sha256=TSWw7Q1mlXcwOx4OuvNGN-yyB_X10Vmt1KywA79RmBs,16359
+mccode_plumber/file_writer_control/CommandChannel.py,sha256=rrQxmQ6VJsl9OC5n45XkL3g7MXtICT_8tsv_xuGdIKI,9119
+mccode_plumber/file_writer_control/CommandHandler.py,sha256=ETPGNiD3umt2XazCWztoaO6qgtilBzrB1N1EH5EAKUw,2245
+mccode_plumber/file_writer_control/CommandStatus.py,sha256=biNK58vY1H-USYAl9uB_JNQyMohBj1cwJwmXktndd4E,4517
+mccode_plumber/file_writer_control/InThreadStatusTracker.py,sha256=sLb8v1JBs3QSWXu-_kIznku3h_ZqVNwx0o9-RoxaKu4,10196
+mccode_plumber/file_writer_control/JobHandler.py,sha256=H6fr0O6lcGw4EKRUuR9Ak4gk5Rknozru0PGaNxNITeg,4296
+mccode_plumber/file_writer_control/JobStatus.py,sha256=ncLYEcTAOdrIASQvY0jJAX3fC4qF9tyzLavm5YUOhlQ,4416
+mccode_plumber/file_writer_control/KafkaTopicUrl.py,sha256=dG6Aj2tYo1UGZferxyvURUAr77z9dhBN_SRzizzt1Vo,736
+mccode_plumber/file_writer_control/StateExtractor.py,sha256=FaroeEuZCGTIi7kEIxeS7QHjnD7CxU1ONvKRQqRCmrc,2068
+mccode_plumber/file_writer_control/WorkerFinder.py,sha256=vMv-_YQSdchGDo5TAHQfJODWsatjCyCbLiCwtJJGZi4,6093
+mccode_plumber/file_writer_control/WorkerJobPool.py,sha256=2P6qYNgRtcolSO910a2BejoZyJf4H2ZiSfsVgYzwq8g,2966
+mccode_plumber/file_writer_control/WorkerStatus.py,sha256=uw1q-Pvf1o2hxpMIPVwdtnAXLcO7VeEpcZ3wKjxxdsk,2826
+mccode_plumber/file_writer_control/WriteJob.py,sha256=-2tQvfajctf6Bn19c9hT9N1lOP0uTo7SZRMnXV1A-aA,2721
+mccode_plumber/file_writer_control/__init__.py,sha256=Wp8A7JOB0vgpwAbvI9ik5cBlZa6lY-cq8dxYGRTRs0M,285
+mccode_plumber-0.7.1.dist-info/METADATA,sha256=MJPyAqAG4OUUFWy5hs-s9fZfzo9z7qQny3cUsNcUMss,552
+mccode_plumber-0.7.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+mccode_plumber-0.7.1.dist-info/entry_points.txt,sha256=V-ULMAQo7W1O3CLd39iSJcRWNjx9p4OuzEC_8y3T-Zs,520
+mccode_plumber-0.7.1.dist-info/top_level.txt,sha256=kCCIpYtKHCKWxiPEqX9J1UaGEm-ze0Qb-cemBCEPhDA,15
+mccode_plumber-0.7.1.dist-info/RECORD,,

mccode_plumber-0.6.0.dist-info/RECORD

@@ -1,14 +0,0 @@
-mccode_plumber/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-mccode_plumber/conductor.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-mccode_plumber/epics.py,sha256=THzech5V1UCLIN9u1r3GiEKxUVCDkYVXhfOQV71aQn4,3989
-mccode_plumber/forwarder.py,sha256=1PdW91qLksDnA41-gOFMtgSae4wDdSLVvgAlrF23Uks,3764
-mccode_plumber/kafka.py,sha256=8fHTy2Zzk638XXj29pfrk-7fUxafuse9OYWmaRgvV7M,1195
-mccode_plumber/mccode.py,sha256=WXlu89Cs6a3s0aKtfRGuzqcOWoFZtdZWibLgBQnaXCI,2210
-mccode_plumber/splitrun.py,sha256=MsR1cUvoayQGNCJNl5fIPG92Wfcy9V_8QNO_ITQTvtA,1295
-mccode_plumber/utils.py,sha256=E8NoGo_3Z-pPEpzicVGSWfQOX8p3eR-GxElT33-kX5U,2167
-mccode_plumber/writer.py,sha256=-gdh6fyOtr6mBhm5AZOs4x2D-bo4_zRSgGkNk-X0ez8,16356
-mccode_plumber-0.6.0.dist-info/METADATA,sha256=7NnOPT9uLE_ogTmdR3pSx0-L3T1oUNj9j4cep7pT4LU,513
-mccode_plumber-0.6.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-mccode_plumber-0.6.0.dist-info/entry_points.txt,sha256=V-ULMAQo7W1O3CLd39iSJcRWNjx9p4OuzEC_8y3T-Zs,520
-mccode_plumber-0.6.0.dist-info/top_level.txt,sha256=kCCIpYtKHCKWxiPEqX9J1UaGEm-ze0Qb-cemBCEPhDA,15
-mccode_plumber-0.6.0.dist-info/RECORD,,