mccode-plumber 0.11.0__tar.gz → 0.12.0__tar.gz

{mccode_plumber-0.11.0/src/mccode_plumber.egg-info → mccode_plumber-0.12.0}/PKG-INFO

@@ -1,16 +1,17 @@
 Metadata-Version: 2.4
 Name: mccode-plumber
-Version: 0.11.0
+Version: 0.12.0
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 Classifier: License :: OSI Approved :: BSD License
 Description-Content-Type: text/markdown
 Requires-Dist: p4p
 Requires-Dist: kafka-python>=2.2.11
 Requires-Dist: ess-streaming-data-types>=0.14.0
-Requires-Dist: restage>=0.7.0
+Requires-Dist: restage>=0.7.2
 Requires-Dist: mccode-to-kafka>=0.2.2
 Requires-Dist: moreniius>=0.4.0
 Requires-Dist: icecream
+Requires-Dist: ephemeral-port-reserve
 
 # McCode Plumber
 Setup, run, and teardown the infrastructure for splitrun McCode scans sending data through Kafka into NeXus

{mccode_plumber-0.11.0 → mccode_plumber-0.12.0}/pyproject.toml

@@ -8,10 +8,11 @@ dependencies = [
     'p4p',
     'kafka-python>=2.2.11',
     'ess-streaming-data-types>=0.14.0',
-    'restage>=0.7.0',
+    'restage>=0.7.2',
     'mccode-to-kafka>=0.2.2',
     'moreniius>=0.4.0',
     'icecream',
+    'ephemeral-port-reserve',
 ]
 readme = "README.md"
 authors = [
@@ -25,6 +26,7 @@ dynamic = ['version']
 [project.scripts]
 mp-splitrun = 'mccode_plumber.splitrun:main'
 mp-epics = 'mccode_plumber.epics:run'
+mp-epics-strings = 'mccode_plumber.epics:run_strings'
 mp-epics-update = 'mccode_plumber.epics:update'
 mp-forwarder-setup = 'mccode_plumber.forwarder:setup'
 mp-forwarder-teardown = 'mccode_plumber.forwarder:teardown'
@@ -36,6 +38,8 @@ mp-writer-kill = 'mccode_plumber.writer:kill_job'
 mp-writer-killall = 'mccode_plumber.writer:kill_all'
 mp-register-topics = 'mccode_plumber.kafka:register_topics'
 mp-insert-hdf5-instr = 'mccode_plumber.mccode:insert'
+mp-nexus-splitrun = 'mccode_plumber.manage.orchestrate:main'
+mp-nexus-services = 'mccode_plumber.manage.orchestrate:services'
 
 [tool.setuptools_scm]
 

{mccode_plumber-0.11.0 → mccode_plumber-0.12.0}/src/mccode_plumber/epics.py

@@ -5,10 +5,9 @@ from p4p.server.thread import SharedPV
 from pathlib import Path
 from typing import Union
 
-
-def convert_instr_parameters_to_nt(parameters):
+def instr_par_to_nt_primitive(parameters):
     from mccode_antlr.common.expression import DataType, ShapeType
-    out = {}
+    out = []
     for p in parameters:
         expr = p.value
         if expr.is_str:
@@ -21,7 +20,37 @@ def convert_instr_parameters_to_nt(parameters):
             raise ValueError(f"Unknown parameter type {expr.data_type}")
         if expr.shape_type == ShapeType.vector:
             t, d = 'a' + t, [d]
-        out[p.name] = NTScalar(t).wrap(expr.value if expr.has_value else d)
+        out.append((p.name, t, d))
+    return out
+
+def instr_par_nt_to_strings(parameters):
+    return [f'{n}:{t}:{d}'.replace(' ','') for n, t, d in instr_par_to_nt_primitive(parameters)]
+
+def strings_to_instr_par_nt(strings):
+    out = []
+    for string in strings:
+        name, t, dstr = string.split(':')
+        trans = None
+        if 'i' in t:
+            trans = int
+        elif 'd' in t:
+            trans = float
+        elif 's' in t:
+            trans = str
+        else:
+            ValueError(f"Unknown type in {string}")
+        if t.startswith('a'):
+            d = [trans(x) for x in dstr.translate(str.maketrans(',',' ','[]')).split()]
+        else:
+            d = trans(dstr)
+        out.append((name, t, d))
+    return out
+
+def convert_strings_to_nt(strings):
+    return {n: NTScalar(t).wrap(d) for n, t, d in strings_to_instr_par_nt(strings)}
+
+def convert_instr_parameters_to_nt(parameters):
+    out = {n: NTScalar(t).wrap(d) for n, t, d in instr_par_to_nt_primitive(parameters)}
     return out
 
 
@@ -67,14 +96,16 @@ def parse_args():
     return parameters, args
 
 
-def main(names: dict[str, NTScalar], prefix: str = None):
+def main(names: dict[str, NTScalar], prefix: str = None, filename_required: bool = True):
     provider = StaticProvider('mailbox')  # 'mailbox' is an arbitrary name
 
+    if filename_required and 'mcpl_filename' not in names:
+        names['mcpl_filename'] = NTScalar('s').wrap('')
+
     pvs = []  # we must keep a reference in order to keep the Handler from being collected
     for name, value in names.items():
         pv = SharedPV(initial=value, handler=MailboxHandler())
         provider.add(f'{prefix}{name}' if prefix else name, pv)
-        print(f'Add mailbox for {prefix}{name}')
         pvs.append(pv)
 
     print(f'Start mailbox server for {len(pvs)} PVs with prefix {prefix}')
@@ -135,5 +166,21 @@ def update():
     ctx.disconnect()
 
 
+def get_strings_parser():
+    from argparse import ArgumentParser
+    from mccode_plumber import __version__
+    p = ArgumentParser()
+    p.add_argument('strings', type=str, nargs='+', help='The string encoded NTScalars to read, each name:type-char:default')
+    p.add_argument('-p', '--prefix', type=str, help='The EPICS PV prefix to use', default='mcstas:')
+    p.add_argument('-v', '--version', action='version', version=__version__)
+    return p
+
+
+def run_strings():
+    args = get_strings_parser().parse_args()
+    main(convert_strings_to_nt(args.strings), prefix=args.prefix)
+
+
+
 if __name__ == '__main__':
     run()

{mccode_plumber-0.11.0 → mccode_plumber-0.12.0}/src/mccode_plumber/file_writer_control/WorkerJobPool.py

@@ -19,7 +19,7 @@ class WorkerJobPool(WorkerFinder):
         self,
         job_topic_url: str,
         command_topic_url: str,
-        max_message_size: int = 1048576 * 200,
+        max_message_size: int = 104857600, # matching the default for Kafka -- previously was 2x larger
         kafka_config: Dict[str, str] = {},
     ):
         """

{mccode_plumber-0.11.0 → mccode_plumber-0.12.0}/src/mccode_plumber/forwarder.py

@@ -59,6 +59,15 @@ def reset_forwarder(pvs: list[dict], config=None, prefix=None, topic=None):
     return pvs
 
 
+def forwarder_partial_streams(prefix, topic, parameters):
+    names = [p.name for p in parameters]
+    if 'mcpl_filename' not in names:
+        names.append("mcpl_filename")
+    # Minimal information used by the forwarder for stream setup:
+    partial = [dict(source=f'{prefix}{n}', module='f144', topic=topic) for n in names]
+    return partial
+
+
 def parse_registrar_args():
     from argparse import ArgumentParser
     from .mccode import get_mccode_instr_parameters
@@ -72,11 +81,7 @@ def parse_registrar_args():
     parser.add_argument('-v', '--version', action='version', version=__version__)
 
     args = parser.parse_args()
-    parameter_names = [p.name for p in get_mccode_instr_parameters(args.instrument)]
-    if 'mcpl_filename' not in parameter_names:
-        parameter_names.append('mcpl_filename')
-    # the forwarder only cares about: "source", "module", "topic"
-    params = [{'source': f'{args.prefix}{name}', 'module': 'f144', 'topic': args.topic} for name in parameter_names]
+    params = forwarder_partial_streams(args.prefix, args.topic, get_mccode_instr_parameters(args.instrument))
     return params, args
 
 

mccode_plumber-0.12.0/src/mccode_plumber/kafka.py

@@ -0,0 +1,68 @@
+from enum import Enum
+
+
+class KafkaTopic(Enum):
+    CREATED = 1
+    EXISTS = 2
+    ERROR = 3
+    UNKNOWN = 4
+
+
+def all_exist(topic_enums):
+    if any(not isinstance(v, KafkaTopic) for v in topic_enums):
+        raise ValueError('Only KafkaTopic enumerated values supported')
+    return all(v == KafkaTopic.EXISTS or v == KafkaTopic.CREATED for v in topic_enums)
+
+
+def parse_kafka_topic_args():
+    from argparse import ArgumentParser
+    from mccode_plumber import __version__
+    parser = ArgumentParser(description="Prepare the named Kafka broker to host one or more topics")
+    parser.add_argument('-b', '--broker', type=str, help='The Kafka broker server to interact with')
+    parser.add_argument('topic', nargs="+", type=str, help='The Kafka topic(s) to register')
+    parser.add_argument('-q', '--quiet', action='store_true', help='Quiet (positive) failure')
+    parser.add_argument('-v', '--version', action='version', version=__version__)
+
+    args = parser.parse_args()
+    return args
+
+
+def register_kafka_topics(broker: str, topics: list[str]):
+    from confluent_kafka.admin import AdminClient, NewTopic
+    client = AdminClient({"bootstrap.servers": broker})
+    config = {
+        # 'cleanup.policy': 'delete',
+        # 'delete.retention.ms': 60000,
+        'max.message.bytes': 104857600,
+        # 'retention.bytes': 10737418240,
+        # 'retention.ms': 30000,
+        # 'segment.bytes': 104857600,
+        # 'segment.ms': 60000
+    }
+    new_ts = [NewTopic(t, num_partitions=1, replication_factor=1, config=config) for t in topics]
+    futures = client.create_topics(new_ts)
+    results = {}
+    for topic, future in futures.items():
+        try:
+            future.result()
+            results[topic] = KafkaTopic.CREATED
+        except Exception as e:
+            from confluent_kafka.error import KafkaError
+            if e.args[0] == KafkaError.TOPIC_ALREADY_EXISTS:
+                results[topic] = KafkaTopic.EXISTS
+            else:
+                results[topic] = e.args[0]
+    return results
+
+
+def register_topics():
+    args = parse_kafka_topic_args()
+    results = register_kafka_topics(args.broker, args.topic)
+    if not args.quiet:
+        for topic, result in results.items():
+            if result == KafkaTopic.CREATED:
+                print(f'Created topic {topic}')
+            elif result == KafkaTopic.EXISTS:
+                print(f'Topic {topic} already exists')
+            else:
+                print(f'Failed to register topic "{topic}"? {result}')

mccode_plumber-0.12.0/src/mccode_plumber/manage/__init__.py

@@ -0,0 +1,26 @@
+from .manager import Manager
+from .efu import EventFormationUnit
+from .epics import EPICSMailbox
+from .forwarder import Forwarder
+from .writer import KafkaToNexus
+from .ensure import (
+    ensure_accessible_directory, ensure_accessible_file, ensure_executable,
+    ensure_readable_directory, ensure_readable_file,
+    ensure_writable_directory, ensure_writable_file
+)
+
+
+__all__ = (
+    "Manager",
+    "EventFormationUnit",
+    "EPICSMailbox",
+    "Forwarder",
+    "KafkaToNexus",
+    "ensure_accessible_directory",
+    "ensure_accessible_file",
+    "ensure_executable",
+    "ensure_readable_directory",
+    "ensure_readable_file",
+    "ensure_writable_directory",
+    "ensure_writable_file",
+)

mccode_plumber-0.12.0/src/mccode_plumber/manage/efu.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+from pathlib import Path
+from ephemeral_port_reserve import reserve
+from .manager import Manager
+from .ensure import ensure_readable_file, ensure_executable
+
+@dataclass
+class EventFormationUnitConfig:
+    name: str
+    binary: Path
+    config: Path
+    calibration: Path
+    topic: str
+    samples_topic: str
+    port: int
+    monitor_every: int
+    monitor_consecutive: int
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        required = ('binary', 'config', 'calibration', 'topic', 'port')
+        if any(req not in data for req in required):
+            msg = [req for req in required if req not in data]
+            msg = ', '.join(msg)
+            raise ValueError(f"Missing required value{'' if len(msg)==1 else 's'}: {msg}")
+        binary = ensure_readable_file(data['binary'])
+        config = ensure_readable_file(data['config'])
+        calibration = ensure_readable_file(data['calibration'])
+        topic = data['topic']
+        port = int(data['port'])
+        monitor_every = int(data.get('monitor_every', 1000))
+        monitor_consecutive = int(data.get('monitor_consecutive', 2))
+        name = data.get('name', binary.stem)
+        samples_topic = data.get('samples_topic', f'{topic}_samples')
+        return cls(name, binary, config, calibration, topic, samples_topic, port, monitor_every, monitor_consecutive)
+
+    def to_dict(self):
+        d = {
+            'name': self.name,
+            'binary': self.binary.as_posix(),
+            'config': self.config.as_posix(),
+            'calibration': self.calibration.as_posix(),
+            'topic': self.topic,
+            'samples_topic': self.samples_topic,
+            'port': self.port,
+            'monitor_every': self.monitor_every,
+            'monitor_consecutive': self.monitor_consecutive,
+        }
+        return d
+
+    def to_cli_str(self):
+        from json import dumps
+        return dumps(self.to_dict()).translate(str.maketrans(',',';',' {}"'))
+
+    @classmethod
+    def from_cli_str(cls, cli_str: str):
+        data ={k: v for k, v in [z.split(':') for z in [x for x in cli_str.split(';')]]}
+        return cls.from_dict(data)
+
+
+
+@dataclass
+class EventFormationUnit(Manager):
+    """
+    Command and control of an Event Formation Unit
+
+    Properties
+    ----------
+    binary: the full path to a binary file which is the EFU
+    config: the full path to its configuration JSON file
+    calibration: the full path to its calibration JSON file
+    broker: the domain name or IP and port of the Kafka broker
+    topic: the EV44 detector data Kafka stream topic
+    samples_topic: the raw AR51 detector data Kafka stream topic
+    port: the UDP port at which the EFU will listen for Readout messages
+    command: the TCP port the EFU will use to listen for command messages, e.g. EXIT
+    monitor_every: For every `monitor_every`th Readout packet
+    monitor_consecutive: Send `monitor_consecutive` raw packets to `samples_topic`
+    """
+    binary: Path
+    config: Path
+    calibration: Path
+    broker: str | None = None
+    topic: str | None = None
+    samples_topic: str | None = None
+    port: int = 9000
+    command: int = field(default_factory=reserve)
+    monitor_every: int = 1000
+    monitor_consecutive: int = 2
+
+    def __post_init__(self):
+        self.binary = ensure_executable(self.binary)
+        self.config = ensure_readable_file(self.config)
+        self.calibration = ensure_readable_file(self.calibration)
+        if self.broker is None:
+            self.broker = 'localhost:9092'
+        if self.topic is None:
+            self.topic = self.binary.stem
+        if self.samples_topic is None:
+            self.samples_topic = f'{self.topic}_samples'
+
+    def __run_command__(self):
+        argv = [self.binary.as_posix(),
+                '-b', self.broker,
+                '-t', self.topic,
+                '--ar51_topic', self.samples_topic,
+                '--file', self.config.as_posix(),
+                '--calibration', self.calibration.as_posix(),
+                '--port', str(self.port),
+                '--cmdport', str(self.command),
+                '--monitor_every', str(self.monitor_every),
+                '--monitor_consecutive', str(self.monitor_consecutive),
+                '--nohwcheck']
+        return argv
+
+    def finalize(self):
+        import socket
+        message = f"Check your system status manager whether {self.binary} is active."
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+            try:
+                sock.settimeout(1.0)
+                sock.connect(('localhost', self.command))
+                sock.sendall(bytes("EXIT\n", "utf-8"))
+                received = str(sock.recv(1024), "utf-8")
+            except TimeoutError:
+                print(f"Communication timed out, is the EFU running? {message}")
+                return
+            except ConnectionRefusedError:
+                # the server is already dead or was not started?
+                received = '<OK>'
+        if received.strip() != "<OK>":
+            print(f"EFU responded '{received.strip()}' when asked to exit. {message}")

mccode_plumber-0.12.0/src/mccode_plumber/manage/ensure.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+from pathlib import Path
+from os import access, R_OK, W_OK, X_OK
+
+def message(mode) -> str:
+    return {R_OK: 'readable', W_OK: 'writable', X_OK: 'executable'}.get(mode, 'unknown')
+
+def ensure_executable(path: str| Path) -> Path:
+    from shutil import which
+    found = which(path)
+    if found is None:
+        raise FileNotFoundError(path)
+    return Path(found)
+
+def ensure_accessible_file(path: str| Path, mode, must_exist=True) -> Path:
+    if isinstance(path, str):
+        path = Path(path)
+    if not isinstance(path, Path):
+        raise ValueError(f'{path} is not a Path object')
+    if must_exist:
+        if not path.exists():
+            raise ValueError(f'{path} does not exist')
+        if not path.is_file():
+            raise ValueError(f'{path} is not a file')
+        if not access(path, mode):
+            raise ValueError(f'{path} is not {message(mode)}')
+    return path
+
+def ensure_accessible_directory(path: str| Path, mode) -> Path:
+    if isinstance(path, str):
+        path = Path(path)
+    if not isinstance(path, Path):
+        raise ValueError(f'{path} is not a Path object')
+    if not path.exists():
+        raise ValueError(f'{path} does not exist')
+    if not path.is_dir():
+        raise ValueError(f'{path} is not a directory')
+    if not access(path, mode):
+        raise ValueError(f'{path} is not a {message(mode)} directory')
+    return path
+
+def ensure_readable_file(path: str| Path) -> Path:
+    return ensure_accessible_file(path, R_OK)
+
+def ensure_writable_file(path: str| Path) -> Path:
+    if not isinstance(path, Path):
+        path = Path(path)
+    return (
+            ensure_accessible_directory(path.parent, W_OK)
+            and ensure_accessible_file(path, W_OK, must_exist=False)
+    )
+
+def ensure_readable_directory(path: str| Path) -> Path:
+    return ensure_accessible_directory(path, R_OK)
+
+def ensure_writable_directory(path: str| Path) -> Path:
+    return ensure_accessible_directory(path, W_OK)
+
+
+def ensure_path(path: str| Path, access_type, is_dir: bool = False) -> Path:
+    if isinstance(path, str):
+        path = Path(path)
+    if not isinstance(path, Path):
+        raise ValueError(f'{path} is not a Path object')
+    if not path.exists():
+        raise ValueError(f'{path} does not exist')
+    if is_dir and not path.is_dir():
+        raise ValueError(f'{path} is not a directory')
+    if not is_dir and not path.is_file():
+        raise ValueError(f'{path} is not a file')
+    if not access(path, access_type):
+        raise ValueError(f'{path} does not support {access_type}')
+    return path

mccode_plumber-0.12.0/src/mccode_plumber/manage/epics.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+from pathlib import Path
+from mccode_antlr.common import InstrumentParameter
+from .manager import Manager
+from .ensure import ensure_executable
+
+@dataclass
+class EPICSMailbox(Manager):
+    """
+    Command and control of an EPICS Mailbox server for an instrument
+
+    Parameters
+    ----------
+    parameters: the instrument parameters which define the PV values
+    prefix:     a PV value prefix to use with all instrument-defined parameters
+    strings:    optional list of NT parameter information to configure the
+                mailbox when the instrument parameters are not available for
+                use in determining the same information.
+    """
+    parameters: tuple[InstrumentParameter, ...]
+    prefix: str
+    strings: list[str] = field(default_factory=list)
+    _command: Path = field(default_factory=lambda: Path('mp-epics-strings'))
+
+    def __post_init__(self):
+        from mccode_plumber.epics import instr_par_nt_to_strings
+        self._command = ensure_executable(self._command)
+        if not len(self.strings):
+            self.strings = instr_par_nt_to_strings(self.parameters)
+
+    def __run_command__(self) -> list[str]:
+        return [self._command.as_posix(), '--prefix', self.prefix] + self.strings

mccode_plumber-0.12.0/src/mccode_plumber/manage/forwarder.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+from pathlib import Path
+from .manager import Manager
+from .ensure import ensure_executable
+
+
+@dataclass
+class Forwarder(Manager):
+    """
+    Manage the execution of a Forwarder to send EPICS PV updates to Kafka
+
+    Parameters
+    ----------
+    broker:     the name or address and port of the broker to which updated
+                EPICS values will be sent, once configured. (localhost:9092)
+    config:     the broker and topic used for configuring the forwarder
+                (localhost:9092/ForwardConfig)
+    status:     the broker and topic used for forwarder status messages
+                (localhost:9092/ForwardStatus)
+    retrieve:   Retrieve values from Kafka at configuration (False == don't)
+    verbosity:  Control if (Trace, Debug, Warning, Error, or Critical) messages
+                should be printed to STDOUT
+
+    Note
+    ----
+    `config` and `status` can be provided as _only_ their topic if they use the same
+    broker as PV updates. In such a case, there will be no '/' character in their input
+    value and `lambda value = f'{broker}/{value}'` will replace them.
+
+    """
+    broker: str | None = None
+    config: str | None = None
+    status: str | None = None
+    retrieve: bool = False
+    verbosity: str | None = None
+    _command: Path = field(default_factory=lambda: Path('forwarder-launch'))
+
+    def __post_init__(self):
+        from mccode_plumber.kafka import register_kafka_topics, all_exist
+        self._command =ensure_executable(self._command)
+        if self.broker is None:
+            self.broker = 'localhost:9092'
+        if self.config is None:
+            self.config = 'ForwardConfig'
+        if self.status is None:
+            self.status = 'ForwardStatus'
+        if '/' not in self.config:
+            self.config = f'{self.broker}/{self.config}'
+        if '/' not in self.status:
+            self.status = f'{self.broker}/{self.status}'
+
+        for broker_topic in (self.config, self.status):
+            b, t = broker_topic.split('/')
+            res = register_kafka_topics(b, [t])
+            if not all_exist(res.values()):
+                raise RuntimeError(f'Missing Kafka topics? {res}')
+
+
+    def __run_command__(self) -> list[str]:
+        args = [
+            self._command.as_posix(),
+            '--config-topic', self.config,
+            '--status-topic', self.status,
+            '--output-broker', self.broker,
+        ]
+        if not self.retrieve:
+            args.append('--skip-retrieval')
+        if (v:=forwarder_verbosity(self.verbosity)) is not None:
+            args.extend(['-v', v])
+        return args
+
+
+def forwarder_verbosity(v):
+    if isinstance(v, str):
+        for k in ('Trace', 'Debug', 'Warning', 'Error', 'Critical'):
+            if k.lower() == v.lower():
+                return k
+    return None