mccode-plumber 0.11.1__py3-none-any.whl → 0.13.0__py3-none-any.whl

mccode_plumber/epics.py

@@ -5,10 +5,9 @@ from p4p.server.thread import SharedPV
 from pathlib import Path
 from typing import Union
 
-
-def convert_instr_parameters_to_nt(parameters):
+def instr_par_to_nt_primitive(parameters):
     from mccode_antlr.common.expression import DataType, ShapeType
-    out = {}
+    out = []
     for p in parameters:
         expr = p.value
         if expr.is_str:
@@ -21,7 +20,37 @@ def convert_instr_parameters_to_nt(parameters):
             raise ValueError(f"Unknown parameter type {expr.data_type}")
         if expr.shape_type == ShapeType.vector:
             t, d = 'a' + t, [d]
-        out[p.name] = NTScalar(t).wrap(expr.value if expr.has_value else d)
+        out.append((p.name, t, d))
+    return out
+
+def instr_par_nt_to_strings(parameters):
+    return [f'{n}:{t}:{d}'.replace(' ','') for n, t, d in instr_par_to_nt_primitive(parameters)]
+
+def strings_to_instr_par_nt(strings):
+    out = []
+    for string in strings:
+        name, t, dstr = string.split(':')
+        trans = None
+        if 'i' in t:
+            trans = int
+        elif 'd' in t:
+            trans = float
+        elif 's' in t:
+            trans = str
+        else:
+            ValueError(f"Unknown type in {string}")
+        if t.startswith('a'):
+            d = [trans(x) for x in dstr.translate(str.maketrans(',',' ','[]')).split()]
+        else:
+            d = trans(dstr)
+        out.append((name, t, d))
+    return out
+
+def convert_strings_to_nt(strings):
+    return {n: NTScalar(t).wrap(d) for n, t, d in strings_to_instr_par_nt(strings)}
+
+def convert_instr_parameters_to_nt(parameters):
+    out = {n: NTScalar(t).wrap(d) for n, t, d in instr_par_to_nt_primitive(parameters)}
     return out
 
 
@@ -67,14 +96,16 @@ def parse_args():
     return parameters, args
 
 
-def main(names: dict[str, NTScalar], prefix: str = None):
+def main(names: dict[str, NTScalar], prefix: str = None, filename_required: bool = True):
     provider = StaticProvider('mailbox')  # 'mailbox' is an arbitrary name
 
+    if filename_required and 'mcpl_filename' not in names:
+        names['mcpl_filename'] = NTScalar('s').wrap('')
+
     pvs = []  # we must keep a reference in order to keep the Handler from being collected
     for name, value in names.items():
         pv = SharedPV(initial=value, handler=MailboxHandler())
         provider.add(f'{prefix}{name}' if prefix else name, pv)
-        print(f'Add mailbox for {prefix}{name}')
         pvs.append(pv)
 
     print(f'Start mailbox server for {len(pvs)} PVs with prefix {prefix}')
@@ -135,5 +166,21 @@ def update():
     ctx.disconnect()
 
 
+def get_strings_parser():
+    from argparse import ArgumentParser
+    from mccode_plumber import __version__
+    p = ArgumentParser()
+    p.add_argument('strings', type=str, nargs='+', help='The string encoded NTScalars to read, each name:type-char:default')
+    p.add_argument('-p', '--prefix', type=str, help='The EPICS PV prefix to use', default='mcstas:')
+    p.add_argument('-v', '--version', action='version', version=__version__)
+    return p
+
+
+def run_strings():
+    args = get_strings_parser().parse_args()
+    main(convert_strings_to_nt(args.strings), prefix=args.prefix)
+
+
+
 if __name__ == '__main__':
     run()

mccode_plumber/file_writer_control/WorkerJobPool.py

@@ -19,7 +19,7 @@ class WorkerJobPool(WorkerFinder):
         self,
         job_topic_url: str,
         command_topic_url: str,
-        max_message_size: int = 1048576 * 200,
+        max_message_size: int = 104857600, # matching the default for Kafka -- previously was 2x larger
         kafka_config: Dict[str, str] = {},
     ):
         """

mccode_plumber/forwarder.py

@@ -59,6 +59,15 @@ def reset_forwarder(pvs: list[dict], config=None, prefix=None, topic=None):
     return pvs
 
 
+def forwarder_partial_streams(prefix, topic, parameters):
+    names = [p.name for p in parameters]
+    if 'mcpl_filename' not in names:
+        names.append("mcpl_filename")
+    # Minimal information used by the forwarder for stream setup:
+    partial = [dict(source=f'{prefix}{n}', module='f144', topic=topic) for n in names]
+    return partial
+
+
 def parse_registrar_args():
     from argparse import ArgumentParser
     from .mccode import get_mccode_instr_parameters
@@ -72,11 +81,7 @@ def parse_registrar_args():
     parser.add_argument('-v', '--version', action='version', version=__version__)
 
     args = parser.parse_args()
-    parameter_names = [p.name for p in get_mccode_instr_parameters(args.instrument)]
-    if 'mcpl_filename' not in parameter_names:
-        parameter_names.append('mcpl_filename')
-    # the forwarder only cares about: "source", "module", "topic"
-    params = [{'source': f'{args.prefix}{name}', 'module': 'f144', 'topic': args.topic} for name in parameter_names]
+    params = forwarder_partial_streams(args.prefix, args.topic, get_mccode_instr_parameters(args.instrument))
     return params, args
 
 

mccode_plumber/kafka.py

@@ -1,3 +1,19 @@
+from enum import Enum
+
+
+class KafkaTopic(Enum):
+    CREATED = 1
+    EXISTS = 2
+    ERROR = 3
+    UNKNOWN = 4
+
+
+def all_exist(topic_enums):
+    if any(not isinstance(v, KafkaTopic) for v in topic_enums):
+        raise ValueError('Only KafkaTopic enumerated values supported')
+    return all(v == KafkaTopic.EXISTS or v == KafkaTopic.CREATED for v in topic_enums)
+
+
 def parse_kafka_topic_args():
     from argparse import ArgumentParser
     from mccode_plumber import __version__
@@ -11,19 +27,42 @@ def parse_kafka_topic_args():
     return args
 
 
-def register_topics():
+def register_kafka_topics(broker: str, topics: list[str]):
     from confluent_kafka.admin import AdminClient, NewTopic
-    args = parse_kafka_topic_args()
-
-    client = AdminClient({"bootstrap.servers": args.broker})
-    topics = [NewTopic(t, num_partitions=1, replication_factor=1) for t in args.topic]
-    futures = client.create_topics(topics)
-
+    client = AdminClient({"bootstrap.servers": broker})
+    config = {
+        # 'cleanup.policy': 'delete',
+        # 'delete.retention.ms': 60000,
+        'max.message.bytes': 104857600,
+        # 'retention.bytes': 10737418240,
+        # 'retention.ms': 30000,
+        # 'segment.bytes': 104857600,
+        # 'segment.ms': 60000
+    }
+    new_ts = [NewTopic(t, num_partitions=1, replication_factor=1, config=config) for t in topics]
+    futures = client.create_topics(new_ts)
+    results = {}
     for topic, future in futures.items():
         try:
             future.result()
-            print(f"Topic {topic} created")
+            results[topic] = KafkaTopic.CREATED
         except Exception as e:
             from confluent_kafka.error import KafkaError
-            if not (args.quiet and e.args[0] == KafkaError.TOPIC_ALREADY_EXISTS):
-                print(f"Failed to create topic {topic}: {e.args[0].str()}")
+            if e.args[0] == KafkaError.TOPIC_ALREADY_EXISTS:
+                results[topic] = KafkaTopic.EXISTS
+            else:
+                results[topic] = e.args[0]
+    return results
+
+
+def register_topics():
+    args = parse_kafka_topic_args()
+    results = register_kafka_topics(args.broker, args.topic)
+    if not args.quiet:
+        for topic, result in results.items():
+            if result == KafkaTopic.CREATED:
+                print(f'Created topic {topic}')
+            elif result == KafkaTopic.EXISTS:
+                print(f'Topic {topic} already exists')
+            else:
+                print(f'Failed to register topic "{topic}"? {result}')

mccode_plumber/manage/__init__.py

@@ -0,0 +1,26 @@
+from .manager import Manager
+from .efu import EventFormationUnit
+from .epics import EPICSMailbox
+from .forwarder import Forwarder
+from .writer import KafkaToNexus
+from .ensure import (
+    ensure_accessible_directory, ensure_accessible_file, ensure_executable,
+    ensure_readable_directory, ensure_readable_file,
+    ensure_writable_directory, ensure_writable_file
+)
+
+
+__all__ = (
+    "Manager",
+    "EventFormationUnit",
+    "EPICSMailbox",
+    "Forwarder",
+    "KafkaToNexus",
+    "ensure_accessible_directory",
+    "ensure_accessible_file",
+    "ensure_executable",
+    "ensure_readable_directory",
+    "ensure_readable_file",
+    "ensure_writable_directory",
+    "ensure_writable_file",
+)

mccode_plumber/manage/efu.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+from pathlib import Path
+from ephemeral_port_reserve import reserve
+from .manager import Manager
+from .ensure import ensure_readable_file, ensure_executable
+
+@dataclass
+class EventFormationUnitConfig:
+    name: str
+    binary: Path
+    config: Path
+    calibration: Path
+    topic: str
+    samples_topic: str
+    port: int
+    monitor_every: int
+    monitor_consecutive: int
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        required = ('binary', 'config', 'calibration', 'topic', 'port')
+        if any(req not in data for req in required):
+            msg = [req for req in required if req not in data]
+            msg = ', '.join(msg)
+            raise ValueError(f"Missing required value{'' if len(msg)==1 else 's'}: {msg}")
+        binary = ensure_readable_file(data['binary'])
+        config = ensure_readable_file(data['config'])
+        calibration = ensure_readable_file(data['calibration'])
+        topic = data['topic']
+        port = int(data['port'])
+        monitor_every = int(data.get('monitor_every', 1000))
+        monitor_consecutive = int(data.get('monitor_consecutive', 2))
+        name = data.get('name', binary.stem)
+        samples_topic = data.get('samples_topic', f'{topic}_samples')
+        return cls(name, binary, config, calibration, topic, samples_topic, port, monitor_every, monitor_consecutive)
+
+    def to_dict(self):
+        d = {
+            'name': self.name,
+            'binary': self.binary.as_posix(),
+            'config': self.config.as_posix(),
+            'calibration': self.calibration.as_posix(),
+            'topic': self.topic,
+            'samples_topic': self.samples_topic,
+            'port': self.port,
+            'monitor_every': self.monitor_every,
+            'monitor_consecutive': self.monitor_consecutive,
+        }
+        return d
+
+    def to_cli_str(self):
+        from json import dumps
+        return dumps(self.to_dict()).translate(str.maketrans('','',' {}"'))
+
+    @classmethod
+    def from_cli_str(cls, cli_str: str):
+        """Split a command-line argument string into a EventFormationUnitConfig
+
+        Note
+        ----
+        A command-line argument string is expected to have the following format:
+            "arg1:value1,arg2:value2,...,argN:valueN"
+        That is, key-value pairs separated from each other by commas and internally
+        by a colon. This allows for _values_ which contain colons, but not keys.
+        """
+        data =dict(z.split(':', maxsplit=1) for z in [x for x in cli_str.split(',')])
+        return cls.from_dict(data)
+
+
+
+@dataclass
+class EventFormationUnit(Manager):
+    """
+    Command and control of an Event Formation Unit
+
+    Properties
+    ----------
+    binary: the full path to a binary file which is the EFU
+    config: the full path to its configuration JSON file
+    calibration: the full path to its calibration JSON file
+    broker: the domain name or IP and port of the Kafka broker
+    topic: the EV44 detector data Kafka stream topic
+    samples_topic: the raw AR51 detector data Kafka stream topic
+    port: the UDP port at which the EFU will listen for Readout messages
+    command: the TCP port the EFU will use to listen for command messages, e.g. EXIT
+    monitor_every: For every `monitor_every`th Readout packet
+    monitor_consecutive: Send `monitor_consecutive` raw packets to `samples_topic`
+    """
+    binary: Path
+    config: Path
+    calibration: Path
+    broker: str | None = None
+    topic: str | None = None
+    samples_topic: str | None = None
+    port: int = 9000
+    command: int = field(default_factory=reserve)
+    monitor_every: int = 1000
+    monitor_consecutive: int = 2
+
+    def __post_init__(self):
+        self.binary = ensure_executable(self.binary)
+        self.config = ensure_readable_file(self.config)
+        self.calibration = ensure_readable_file(self.calibration)
+        if self.broker is None:
+            self.broker = 'localhost:9092'
+        if self.topic is None:
+            self.topic = self.binary.stem
+        if self.samples_topic is None:
+            self.samples_topic = f'{self.topic}_samples'
+
+    def __run_command__(self):
+        argv = [self.binary.as_posix(),
+                '-b', self.broker,
+                '-t', self.topic,
+                '--ar51_topic', self.samples_topic,
+                '--file', self.config.as_posix(),
+                '--calibration', self.calibration.as_posix(),
+                '--port', str(self.port),
+                '--cmdport', str(self.command),
+                '--monitor_every', str(self.monitor_every),
+                '--monitor_consecutive', str(self.monitor_consecutive),
+                '--nohwcheck']
+        return argv
+
+    def finalize(self):
+        import socket
+        message = f"Check your system status manager whether {self.binary} is active."
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+            try:
+                sock.settimeout(1.0)
+                sock.connect(('localhost', self.command))
+                sock.sendall(bytes("EXIT\n", "utf-8"))
+                received = str(sock.recv(1024), "utf-8")
+            except TimeoutError:
+                print(f"Communication timed out, is the EFU running? {message}")
+                return
+            except ConnectionRefusedError:
+                # the server is already dead or was not started?
+                received = '<OK>'
+        if received.strip() != "<OK>":
+            print(f"EFU responded '{received.strip()}' when asked to exit. {message}")

mccode_plumber/manage/ensure.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+from pathlib import Path
+from os import access, R_OK, W_OK, X_OK
+
+def message(mode) -> str:
+    return {R_OK: 'readable', W_OK: 'writable', X_OK: 'executable'}.get(mode, 'unknown')
+
+def ensure_executable(path: str| Path) -> Path:
+    from shutil import which
+    found = which(path)
+    if found is None:
+        raise FileNotFoundError(path)
+    return Path(found)
+
+def ensure_accessible_file(path: str| Path, mode, must_exist=True) -> Path:
+    if isinstance(path, str):
+        path = Path(path)
+    if not isinstance(path, Path):
+        raise ValueError(f'{path} is not a Path object')
+    if must_exist:
+        if not path.exists():
+            raise ValueError(f'{path} does not exist')
+        if not path.is_file():
+            raise ValueError(f'{path} is not a file')
+        if not access(path, mode):
+            raise ValueError(f'{path} is not {message(mode)}')
+    return path
+
+def ensure_accessible_directory(path: str| Path, mode) -> Path:
+    if isinstance(path, str):
+        path = Path(path)
+    if not isinstance(path, Path):
+        raise ValueError(f'{path} is not a Path object')
+    if not path.exists():
+        raise ValueError(f'{path} does not exist')
+    if not path.is_dir():
+        raise ValueError(f'{path} is not a directory')
+    if not access(path, mode):
+        raise ValueError(f'{path} is not a {message(mode)} directory')
+    return path
+
+def ensure_readable_file(path: str| Path) -> Path:
+    return ensure_accessible_file(path, R_OK)
+
+def ensure_writable_file(path: str| Path) -> Path:
+    if not isinstance(path, Path):
+        path = Path(path)
+    return (
+            ensure_accessible_directory(path.parent, W_OK)
+            and ensure_accessible_file(path, W_OK, must_exist=False)
+    )
+
+def ensure_readable_directory(path: str| Path) -> Path:
+    return ensure_accessible_directory(path, R_OK)
+
+def ensure_writable_directory(path: str| Path) -> Path:
+    return ensure_accessible_directory(path, W_OK)
+
+
+def ensure_path(path: str| Path, access_type, is_dir: bool = False) -> Path:
+    if isinstance(path, str):
+        path = Path(path)
+    if not isinstance(path, Path):
+        raise ValueError(f'{path} is not a Path object')
+    if not path.exists():
+        raise ValueError(f'{path} does not exist')
+    if is_dir and not path.is_dir():
+        raise ValueError(f'{path} is not a directory')
+    if not is_dir and not path.is_file():
+        raise ValueError(f'{path} is not a file')
+    if not access(path, access_type):
+        raise ValueError(f'{path} does not support {access_type}')
+    return path

mccode_plumber/manage/epics.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+from pathlib import Path
+from mccode_antlr.common import InstrumentParameter
+from .manager import Manager
+from .ensure import ensure_executable
+
+@dataclass
+class EPICSMailbox(Manager):
+    """
+    Command and control of an EPICS Mailbox server for an instrument
+
+    Parameters
+    ----------
+    parameters: the instrument parameters which define the PV values
+    prefix:     a PV value prefix to use with all instrument-defined parameters
+    strings:    optional list of NT parameter information to configure the
+                mailbox when the instrument parameters are not available for
+                use in determining the same information.
+    """
+    parameters: tuple[InstrumentParameter, ...]
+    prefix: str
+    strings: list[str] = field(default_factory=list)
+    _command: Path = field(default_factory=lambda: Path('mp-epics-strings'))
+
+    def __post_init__(self):
+        from mccode_plumber.epics import instr_par_nt_to_strings
+        self._command = ensure_executable(self._command)
+        if not len(self.strings):
+            self.strings = instr_par_nt_to_strings(self.parameters)
+
+    def __run_command__(self) -> list[str]:
+        return [self._command.as_posix(), '--prefix', self.prefix] + self.strings

mccode_plumber/manage/forwarder.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+from pathlib import Path
+from .manager import Manager
+from .ensure import ensure_executable
+
+
+@dataclass
+class Forwarder(Manager):
+    """
+    Manage the execution of a Forwarder to send EPICS PV updates to Kafka
+
+    Parameters
+    ----------
+    broker:     the name or address and port of the broker to which updated
+                EPICS values will be sent, once configured. (localhost:9092)
+    config:     the broker and topic used for configuring the forwarder
+                (localhost:9092/ForwardConfig)
+    status:     the broker and topic used for forwarder status messages
+                (localhost:9092/ForwardStatus)
+    retrieve:   Retrieve values from Kafka at configuration (False == don't)
+    verbosity:  Control if (Trace, Debug, Warning, Error, or Critical) messages
+                should be printed to STDOUT
+
+    Note
+    ----
+    `config` and `status` can be provided as _only_ their topic if they use the same
+    broker as PV updates. In such a case, there will be no '/' character in their input
+    value and `lambda value = f'{broker}/{value}'` will replace them.
+
+    """
+    broker: str | None = None
+    config: str | None = None
+    status: str | None = None
+    retrieve: bool = False
+    verbosity: str | None = None
+    _command: Path = field(default_factory=lambda: Path('forwarder-launch'))
+
+    def __post_init__(self):
+        from mccode_plumber.kafka import register_kafka_topics, all_exist
+        self._command =ensure_executable(self._command)
+        if self.broker is None:
+            self.broker = 'localhost:9092'
+        if self.config is None:
+            self.config = 'ForwardConfig'
+        if self.status is None:
+            self.status = 'ForwardStatus'
+        if '/' not in self.config:
+            self.config = f'{self.broker}/{self.config}'
+        if '/' not in self.status:
+            self.status = f'{self.broker}/{self.status}'
+
+        for broker_topic in (self.config, self.status):
+            b, t = broker_topic.split('/')
+            res = register_kafka_topics(b, [t])
+            if not all_exist(res.values()):
+                raise RuntimeError(f'Missing Kafka topics? {res}')
+
+
+    def __run_command__(self) -> list[str]:
+        args = [
+            self._command.as_posix(),
+            '--config-topic', self.config,
+            '--status-topic', self.status,
+            '--output-broker', self.broker,
+        ]
+        if not self.retrieve:
+            args.append('--skip-retrieval')
+        if (v:=forwarder_verbosity(self.verbosity)) is not None:
+            args.extend(['-v', v])
+        return args
+
+
+def forwarder_verbosity(v):
+    if isinstance(v, str):
+        for k in ('Trace', 'Debug', 'Warning', 'Error', 'Critical'):
+            if k.lower() == v.lower():
+                return k
+    return None

mccode_plumber/manage/manager.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from pathlib import Path
+from multiprocessing import Process, Pipe
+from multiprocessing.connection import Connection
+from enum import Enum
+from colorama import Fore, Back, Style
+
+class IOType(Enum):
+    stdout = 1
+    stderr = 2
+
+
+@dataclass
+class Manager:
+    """
+    Command and control of a process
+
+    Properties
+    ----------
+    _process:   a multiprocessing.Process instance, which is undefined for a short
+                period during instance creation inside the `start` class method
+    """
+    name: str
+    style: Style
+    _process: Process | None
+    _connection: Connection | None
+
+    def __run_command__(self) -> list[str]:
+        pass
+
+    def finalize(self):
+        pass
+
+    @classmethod
+    def fieldnames(cls) -> list[str]:
+        from dataclasses import fields
+        return [field.name for field in fields(cls)]
+
+    @classmethod
+    def start(cls, **config):
+        names = cls.fieldnames()
+        kwargs = {k: config[k] for k in names if k in config}
+        if any(k not in names for k in config):
+            raise ValueError(f'{config} expected to contain only {names}')
+        if '_process' not in kwargs:
+            kwargs['_process'] = None
+        if '_connection' not in kwargs:
+            kwargs['_connection'] = None
+        if 'name' not in kwargs:
+            kwargs['name'] = 'Managed process'
+        if 'style' not in kwargs:
+            kwargs['style'] = Fore.WHITE + Back.BLACK
+        manager = cls(**kwargs)
+        manager._connection, child_conn = Pipe()
+        manager._process = Process(target=manager.run, args=(child_conn,))
+        manager._process.start()
+        return manager
+
+    def stop(self):
+        self.finalize()
+        self._process.terminate()
+
+    def poll(self):
+        from sys import stderr
+        attn = Fore.BLACK + Back.RED + Style.BRIGHT
+        # check for anything received on our end of the connection
+        while self._connection.poll():
+            # examine what was returned:
+            try:
+                ret = self._connection.recv()
+            except EOFError:
+                print(f'{attn}{self.name}: [unexpected halt]{Style.RESET_ALL}')
+                return False
+            if len(ret) == 2:
+                t, line = ret
+                line = f'{self.style}{self.name}:{Style.RESET_ALL} {line}'
+                if t == IOType.stdout:
+                    print(line, end='')
+                else:
+                    print(line, file=stderr, end='')
+            else:
+                print(f'{attn}{self.name}: [unknown received data on connection]{Style.RESET_ALL}')
+        return self._process.is_alive()
+
+    def run(self, conn):
+        from subprocess import Popen, PIPE
+        from select import select
+        argv = self.__run_command__()
+
+        shell = isinstance(argv, str)
+        conn.send((IOType.stdout, f'Starting {argv if shell else " ".join(argv)}\n'))
+        process = Popen(argv, shell=shell, stdout=PIPE, stderr=PIPE, bufsize=1, universal_newlines=True, )
+        out, err = process.stdout.fileno(), process.stderr.fileno()
+        check = [process.stdout, process.stderr]
+        while process.poll() is None:
+            r, w, x = select(check, [], check, 0.5,)
+            for stream in r:
+                if stream.fileno() == out:
+                    conn.send((IOType.stdout, process.stdout.readline()))
+                elif stream.fileno() == err:
+                    conn.send((IOType.stderr, process.stderr.readline()))
+            for stream in x:
+                if stream.fileno() == out:
+                    conn.send((IOType.stdout, "EXCEPTION ON STDOUT"))
+                elif stream.fileno() == err:
+                    conn.send((IOType.stderr, "EXCEPTION ON STDERR"))
+        # Process finished, but the buffers may still contain data:
+        for stream in check:
+            if stream.fileno() == out:
+                map(lambda line: conn.send(IOType.stdout, line), stream.readlines())
+            elif stream.fileno() == err:
+                map(lambda line: conn.send(IOType.stderr, line), stream.readlines())