mccode-plumber 0.14.4__tar.gz → 0.15.0__tar.gz

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/.github/workflows/pip.yml

@@ -2,10 +2,10 @@ name: Pip
 
 on:
   workflow_dispatch:
-    #  pull_request:
-    #  push:
-    #    branches:
-    #      - main
+  pull_request:
+  push:
+    branches:
+      - main
 
 jobs:
   build:
@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         platform: [windows-latest, macos-latest, ubuntu-latest]
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12", "3.13"]
 
     steps:
     - uses: actions/checkout@v6
@@ -24,15 +24,10 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Set min macOS version
-      if: runner.os == 'macOS'
-      run: |
-        echo "MACOS_DEPLOYMENT_TARGET=10.14" >> $GITHUB_ENV
-
     - name: Build and install
       run: pip install --verbose .
 
     - name: Test
       run: |
-        python -m pip install pytest
+        python -m pip install pytest "niess>=0.1.4"
         python -m pytest

{mccode_plumber-0.14.4/src/mccode_plumber.egg-info → mccode_plumber-0.15.0}/PKG-INFO

@@ -1,17 +1,25 @@
 Metadata-Version: 2.4
 Name: mccode-plumber
-Version: 0.14.4
+Version: 0.15.0
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Description-Content-Type: text/markdown
 Requires-Dist: p4p
 Requires-Dist: kafka-python>=2.2.11
 Requires-Dist: ess-streaming-data-types>=0.14.0
-Requires-Dist: restage>=0.9.0
+Requires-Dist: restage>=0.10.0
 Requires-Dist: mccode-to-kafka>=0.3.1
-Requires-Dist: moreniius>=0.6.1
+Requires-Dist: moreniius>=0.6.2
 Requires-Dist: icecream
 Requires-Dist: ephemeral-port-reserve
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: niess>=0.1.4; extra == "test"
 
 # McCode Plumber
 Setup, run, and teardown the infrastructure for splitrun McCode scans sending data through Kafka into NeXus

mccode_plumber-0.15.0/mypy.ini

@@ -0,0 +1,2 @@
+[mypy]
+disable_error_code = import-untyped

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/pyproject.toml

@@ -8,9 +8,9 @@ dependencies = [
     'p4p',
     'kafka-python>=2.2.11',
     'ess-streaming-data-types>=0.14.0',
-    'restage>=0.9.0',
+    'restage>=0.10.0',
     'mccode-to-kafka>=0.3.1',
-    'moreniius>=0.6.1',
+    'moreniius>=0.6.2',
     'icecream',
     'ephemeral-port-reserve',
 ]
@@ -20,9 +20,17 @@ authors = [
 ]
 classifiers = [
     "License :: OSI Approved :: BSD License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
 ]
 dynamic = ['version']
 
+[project.optional-dependencies]
+test = ["pytest", "niess>=0.1.4"]
+
 [project.scripts]
 mp-splitrun = 'mccode_plumber.splitrun:main'
 mp-epics = 'mccode_plumber.epics:run'
@@ -44,3 +52,25 @@ mp-nexus-services = 'mccode_plumber.manage.orchestrate:services'
 
 [tool.setuptools_scm]
 
+[tool.tox]
+legacy_tox_ini = """
+    [tox]
+    min_version = 4.0
+    env_list =
+        py313
+        py312
+        py311
+        type
+
+    [testenv]
+    deps =
+        pytest
+        niess>=0.1.4
+    commands = pytest tests
+
+    [testenv:type]
+    deps =
+        mypy
+        types-PyYAML
+    commands = mypy src
+"""

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber/epics.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+from __future__ import annotations
+
 from p4p.nt import NTScalar
 from p4p.server import Server, StaticProvider
 from p4p.server.thread import SharedPV
@@ -96,7 +98,7 @@ def parse_args():
     return parameters, args
 
 
-def main(names: dict[str, NTScalar], prefix: str = None, filename_required: bool = True):
+def main(names: dict[str, NTScalar], prefix: str | None = None, filename_required: bool = True):
     provider = StaticProvider('mailbox')  # 'mailbox' is an arbitrary name
 
     if filename_required and 'mcpl_filename' not in names:
@@ -118,7 +120,7 @@ def run():
     main(parameters, prefix=args.prefix)
 
 
-def start(parameters, prefix: str = None):
+def start(parameters, prefix: str | None = None):
     from multiprocessing import Process
     proc = Process(target=main, args=(parameters, prefix))
     proc.start()

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber/file_writer_control/CommandChannel.py

@@ -74,8 +74,8 @@ class CommandChannel(object):
         :param command_topic_url: The url of the Kafka topic to where the file-writer status/command messages are published.
         """
         kafka_address = KafkaTopicUrl(command_topic_url)
-        self.status_queue = Queue()
-        self.to_thread_queue = Queue()
+        self.status_queue: Queue = Queue()
+        self.to_thread_queue: Queue = Queue()
         thread_kwargs = {
             "host_port": kafka_address.host_port,
             "topic": kafka_address.topic,

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber/file_writer_control/CommandHandler.py

@@ -32,7 +32,10 @@ class CommandHandler:
         """
         :return: True if the command completed successfully. False otherwise.
         """
-        current_state = self.command_channel.get_command(self.command_id).state
+        command = self.command_channel.get_command(self.command_id)
+        if command is None:
+            return False
+        current_state = command.state
         if current_state == CommandState.ERROR:
             raise RuntimeError(
                 f'Command failed with error message "{self.get_message()}".'
@@ -52,7 +55,10 @@ class CommandHandler:
         return command.message
 
     def set_timeout(self, new_timeout: timedelta):
-        self.command_channel.get_command(self.command_id).timeout = new_timeout
+        if command := self.command_channel.get_command(self.command_id):
+            command.timeout = new_timeout
 
     def get_timeout(self):
-        return self.command_channel.get_command(self.command_id).timeout
+        if command := self.command_channel.get_command(self.command_id):
+            return command.timeout
+        return None

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber/file_writer_control/CommandStatus.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from datetime import datetime, timedelta
 from enum import Enum, auto
 from typing import Optional
@@ -18,7 +20,7 @@ class CommandState(Enum):
     SUCCESS = auto()
 
 
-class CommandStatus(object):
+class CommandStatus:
     """
     The status of a command.
     """
@@ -35,11 +37,11 @@ class CommandStatus(object):
         self._last_update = datetime.now()
         self._state = CommandState.NO_COMMAND
         self._message = ""
-        self._response_code = None
+        self._response_code: int | None = None
 
-    def __eq__(self, other_status: "CommandStatus"):
+    def __eq__(self, other_status):
         if not isinstance(other_status, CommandStatus):
-            raise NotImplementedError
+            return NotImplemented
         return (
             other_status.command_id == self.command_id
             and other_status.job_id == self.job_id

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber/file_writer_control/JobStatus.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from datetime import datetime, timedelta
 from enum import Enum, auto
 from typing import Dict, Optional
@@ -48,7 +50,7 @@ class JobStatus:
         if new_status.message:
             self._message = new_status.message
         self._service_id = new_status.service_id
-        self._file_name = new_status.file_name
+        self._file_name = new_status.file_name or ""
         self._last_update = new_status.last_update
         self._metadata = new_status.metadata
 
@@ -61,7 +63,7 @@ class JobStatus:
             self.state != JobState.DONE
             and self.state != JobState.ERROR
             and self.state != JobState.TIMEOUT
-            and current_time - self.last_update > self._timeout
+            and self._timeout and current_time - self.last_update > self._timeout
         ):
             self._state = JobState.TIMEOUT
             self._last_update = current_time
@@ -106,8 +108,13 @@ class JobStatus:
         """
         return self._state
 
+    @state.setter
+    def state(self, new_state: JobState) -> None:
+        self._state = new_state
+        self._last_update = datetime.now()
+
     @property
-    def file_name(self) -> str:
+    def file_name(self) -> str | None:
         """
         The file name of the job. None if the file name is not known.
         """
@@ -120,11 +127,6 @@ class JobStatus:
         self._file_name = new_file_name
         self._last_update = datetime.now()
 
-    @state.setter
-    def state(self, new_state: JobState) -> None:
-        self._state = new_state
-        self._last_update = datetime.now()
-
     @property
     def message(self) -> str:
         """
@@ -132,6 +134,12 @@ class JobStatus:
         """
         return self._message
 
+    @message.setter
+    def message(self, new_message: str) -> None:
+        if new_message:
+            self._message = new_message
+            self._last_update = datetime.now()
+
     @property
     def metadata(self) -> Optional[Dict]:
         return self._metadata
@@ -140,8 +148,3 @@ class JobStatus:
     def metadata(self, metadata: Dict) -> None:
         self._metadata = metadata
 
-    @message.setter
-    def message(self, new_message: str) -> None:
-        if new_message:
-            self._message = new_message
-            self._last_update = datetime.now()

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber/file_writer_control/WorkerFinder.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import uuid
 from datetime import datetime
 from typing import Dict, List, Optional
@@ -84,7 +86,8 @@ class WorkerFinderBase:
         :param job_id: The job identifier of the currently running file-writer job.
         :return: A CommandHandler instance for (more) easily checking the outcome of the "abort" command.
         """
-        return self.try_send_stop_time(service_id, job_id, 0)
+        from datetime import datetime, UTC
+        return self.try_send_stop_time(service_id, job_id, datetime.fromtimestamp(0, UTC))
 
     def list_known_workers(self) -> List[WorkerStatus]:
         """
@@ -115,7 +118,7 @@ class WorkerFinderBase:
             return JobState.UNAVAILABLE
         return current_job.state
 
-    def get_job_status(self, job_id: str) -> JobStatus:
+    def get_job_status(self, job_id: str) -> JobStatus | None:
         """
         Get the full (known) status of a specific job.
         :param job_id: The (unique) identifier of the job that we are trying to find the status of.

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber/file_writer_control/WorkerJobPool.py

@@ -63,8 +63,7 @@ class WorkerJobPool(WorkerFinder):
         """
         self.command_channel.add_job_id(job.job_id)
         self.command_channel.add_command_id(job.job_id, job.job_id)
-        self.command_channel.get_command(
-            job.job_id
-        ).state = CommandState.WAITING_RESPONSE
+        if command := self.command_channel.get_command(job.job_id):
+            command.state = CommandState.WAITING_RESPONSE
         self._send_pool_message(job.get_start_message())
         return CommandHandler(self.command_channel, job.job_id)

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber/file_writer_control/WorkerStatus.py

@@ -16,7 +16,7 @@ class WorkerState(Enum):
     UNAVAILABLE = auto()
 
 
-class WorkerStatus(object):
+class WorkerStatus:
     """
     Contains general status information about a worker.
     """
@@ -27,9 +27,9 @@ class WorkerStatus(object):
         self._timeout = timeout
         self._state = WorkerState.UNAVAILABLE
 
-    def __eq__(self, other_status: "WorkerStatus") -> bool:
+    def __eq__(self, other_status) -> bool:
         if not isinstance(other_status, WorkerStatus):
-            raise NotImplementedError
+            return NotImplemented
         return (
             self.service_id == other_status.service_id
             and self.state == other_status.state
@@ -55,18 +55,11 @@ class WorkerStatus(object):
         """
         if (
             self.state != WorkerState.UNAVAILABLE
-            and current_time - self.last_update > self._timeout
+            and self._timeout and current_time - self.last_update > self._timeout
         ):
             self._state = WorkerState.UNAVAILABLE
             self._last_update = current_time
 
-    @property
-    def state(self) -> WorkerState:
-        """
-        The current state of the worker.
-        """
-        return self._state
-
     @property
     def service_id(self) -> str:
         """
@@ -82,7 +75,13 @@ class WorkerStatus(object):
         """
         return self._last_update
 
+    @property
+    def state(self) -> WorkerState:
+        return self._state
+
     @state.setter
     def state(self, new_state: WorkerState):
         self._last_update = datetime.now()
         self._state = new_state
+
+

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber/file_writer_control/WriteJob.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import uuid
 from datetime import datetime, timedelta
 
@@ -15,7 +17,7 @@ class WriteJob:
         file_name: str,
         broker: str,
         start_time: datetime,
-        stop_time: datetime = None,
+        stop_time: datetime | None = None,
         job_id="",
         instrument_name: str = "",
         run_name: str = "",
@@ -33,10 +35,7 @@ class WriteJob:
         else:
             self.job_id = str(uuid.uuid1())
         self.start = start_time
-        if stop_time is None:
-            self.stop = self.start + timedelta(days=365.25 * 10)
-        else:
-            self.stop = stop_time
+        self.stop = stop_time or self.start + timedelta(days=365.25 * 10)
         self._service_id = ""
         self.broker = broker
         self.instrument_name = instrument_name

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber/kafka.py

@@ -33,7 +33,7 @@ def register_kafka_topics(broker: str, topics: list[str]):
     config = {
         # 'cleanup.policy': 'delete',
         # 'delete.retention.ms': 60000,
-        'max.message.bytes': 104857600,
+        'max.message.bytes': '104857600',
         # 'retention.bytes': 10737418240,
         # 'retention.ms': 30000,
         # 'segment.bytes': 104857600,

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber/manage/efu.py

@@ -21,9 +21,10 @@ class EventFormationUnitConfig:
     def from_dict(cls, data: dict):
         required = ('binary', 'config', 'calibration', 'topic', 'port')
         if any(req not in data for req in required):
-            msg = [req for req in required if req not in data]
-            msg = ', '.join(msg)
-            raise ValueError(f"Missing required value{'' if len(msg)==1 else 's'}: {msg}")
+            missing = [req for req in required if req not in data]
+            msg = ', '.join(missing)
+            val = f"value{'' if len(missing) == 1 else 's'}"
+            raise ValueError(f"Missing required {val}: {msg}")
         binary = ensure_readable_file(data['binary'])
         config = ensure_readable_file(data['config'])
         calibration = ensure_readable_file(data['calibration'])

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber/manage/ensure.py

@@ -7,7 +7,23 @@ def message(mode) -> str:
 
 def ensure_executable(path: str| Path) -> Path:
     from shutil import which
-    found = which(path)
+    import os
+    p = Path(path)
+    # If the path exists as given, accept it (handles absolute and relative files)
+    if p.exists():
+        return p
+
+    # On Windows try PATHEXT extensions for provided path (handles .py etc.)
+    if os.name == "nt":
+        pathext = os.environ.get("PATHEXT", ".COM;.EXE;.BAT;.CMD;.PY;.PYW").split(
+            os.pathsep)
+        for ext in pathext:
+            candidate = Path(str(p) + ext)
+            if candidate.exists():
+                return candidate
+
+    # Fallback to shutil.which (searches PATH and PATHEXT)
+    found = which(str(path))
     if found is None:
         raise FileNotFoundError(path)
     return Path(found)

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber/manage/forwarder.py

@@ -35,20 +35,26 @@ class Forwarder(Manager):
     retrieve: bool = False
     verbosity: str | None = None
     _command: Path = field(default_factory=lambda: Path('forwarder-launch'))
+    _broker: str = field(default='localhost:9092')
+    _config: str = field(default='ForwardConfig')
+    _status: str = field(default='ForwardStatus')
 
     def __post_init__(self):
         from mccode_plumber.kafka import register_kafka_topics, all_exist
         self._command =ensure_executable(self._command)
         if self.broker is None:
-            self.broker = 'localhost:9092'
+            self.broker = self._broker
         if self.config is None:
-            self.config = 'ForwardConfig'
+            self.config = self._config
         if self.status is None:
-            self.status = 'ForwardStatus'
+            self.status = self._status
         if '/' not in self.config:
             self.config = f'{self.broker}/{self.config}'
         if '/' not in self.status:
             self.status = f'{self.broker}/{self.status}'
+        self._broker = self.broker
+        self._config = self.config
+        self._status = self.status
 
         for broker_topic in (self.config, self.status):
             b, t = broker_topic.split('/')
@@ -58,11 +64,11 @@ class Forwarder(Manager):
 
 
     def __run_command__(self) -> list[str]:
-        args = [
+        args: list[str] = [
             self._command.as_posix(),
-            '--config-topic', self.config,
-            '--status-topic', self.status,
-            '--output-broker', self.broker,
+            '--config-topic', self._config,
+            '--status-topic', self._status,
+            '--output-broker', self._broker,
         ]
         if not self.retrieve:
             args.append('--skip-retrieval')

mccode_plumber-0.15.0/src/mccode_plumber/manage/manager.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from subprocess import Popen, PIPE
+from threading import Thread
+from enum import Enum
+from colorama import Fore, Back, Style
+from colorama.ansi import AnsiStyle
+
+
+class IOType(Enum):
+    stdout = 1
+    stderr = 2
+
+
+@dataclass
+class Manager:
+    """
+    Command and control of a process
+
+    Properties
+    ----------
+    _process:   a subprocess.Popen instance
+    """
+    name: str
+    style: AnsiStyle
+    _process: Popen | None
+    _stdout_thread: Thread | None
+    _stderr_thread: Thread | None
+
+    def __run_command__(self) -> list[str]:
+        return []
+
+    def finalize(self):
+        pass
+
+    @classmethod
+    def fieldnames(cls) -> list[str]:
+        from dataclasses import fields
+        return [field.name for field in fields(cls)]
+
+    def _read_stream(self, stream, io_type: IOType):
+        """Read lines from stream and print them until EOF.
+
+        This replaces the previous behaviour of sending lines over a
+        multiprocessing Connection. Printing directly from the reader
+        threads is sufficient because the manager previously only used
+        the connection to relay subprocess stdout/stderr back to the
+        parent process for display.
+        """
+        try:
+            for line in iter(stream.readline, ''):
+                if not line:
+                    break
+                # format and print the line, preserving original behaviour
+                formatted = f'{self.style}{self.name}:{Style.RESET_ALL} {line}'
+                if io_type == IOType.stdout:
+                    print(formatted, end='')
+                else:
+                    from sys import stderr
+                    print(formatted, file=stderr, end='')
+        except ValueError:
+            pass  # stream closed
+        finally:
+            try:
+                stream.close()
+            except Exception:
+                pass
+
+    @classmethod
+    def start(cls, **config):
+        names = cls.fieldnames()
+        kwargs = {k: config[k] for k in names if k in config}
+        if any(k not in names for k in config):
+            raise ValueError(f'{config} expected to contain only {names}')
+        for p in ('_process', '_stdout_thread', '_stderr_thread'):
+            if p not in kwargs:
+                kwargs[p] = None
+        if 'name' not in kwargs:
+            kwargs['name'] = 'Managed process'
+        if 'style' not in kwargs:
+            kwargs['style'] = Fore.WHITE + Back.BLACK
+
+        manager = cls(**kwargs)
+
+        argv = manager.__run_command__()
+        shell = isinstance(argv, str)
+        # announce start directly instead of sending via a Connection
+        print(f'Starting {argv if shell else " ".join(argv)}')
+
+        manager._process = Popen(
+            argv, shell=shell, stdout=PIPE, stderr=PIPE, bufsize=1,
+            universal_newlines=True,
+        )
+        manager._stdout_thread = Thread(
+            target=manager._read_stream,
+            args=(manager._process.stdout, IOType.stdout),
+            daemon=True,
+        )
+        manager._stderr_thread = Thread(
+            target=manager._read_stream,
+            args=(manager._process.stderr, IOType.stderr),
+            daemon=True,
+        )
+        manager._stdout_thread.start()
+        manager._stderr_thread.start()
+        return manager
+
+    def stop(self):
+        self.finalize()
+        if self._process:
+            self._process.terminate()
+            self._process.wait()
+
+    def poll(self):
+        """Check whether the managed process is still running.
+
+        Previously this drained and printed any messages received over a
+        multiprocessing Connection. Reader threads now handle printing,
+        so poll only needs to report process liveness.
+        """
+        if not self._process:
+            return False
+        return self._process.poll() is None

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber/manage/orchestrate.py

@@ -17,17 +17,17 @@ TOPICS = {
 }
 PREFIX = 'mcstas:'
 
-def guess_instr_config(name: str):
+def guess_instr_config(name: str) -> Path:
     guess = f'/event-formation-unit/configs/{name}/configs/{name}.json'
     return ensure_readable_file(Path(guess))
 
 
-def guess_instr_calibration(name: str):
+def guess_instr_calibration(name: str) -> Path:
     guess = f'/event-formation-unit/configs/{name}/configs/{name}nullcalib.json'
     return ensure_readable_file(Path(guess))
 
 
-def guess_instr_efu(name: str):
+def guess_instr_efu(name: str) -> Path:
     guess = name.split('_')[0].split('.')[0].split('-')[0].lower()
     return ensure_executable(Path(guess))
 
@@ -193,8 +193,10 @@ def efu_parameter(s: str):
     # likely to be needed. Finally, the config file can also be supplied to change, e.g.,
     # number of pixels or rings, etc.
     parts = s.split(',')
-    data = {'topic': TOPICS['event'], 'port': 9000, 'binary': ensure_executable(parts[0]),}
-    data['name'] = data['binary'].stem
+    binary: Path = ensure_executable(parts[0])
+    data : dict[str, int | str | Path] = {
+        'topic': TOPICS['event'], 'port': 9000, 'binary': binary, 'name': binary.stem
+    }
 
     if len(parts) > 1 and (len(parts) > 2 or not parts[1].isnumeric()):
         data['calibration'] = parts[1]

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber/splitrun.py

@@ -1,3 +1,8 @@
+from __future__ import annotations
+
+from typing import Union
+
+
 def make_parser():
     from mccode_plumber import __version__
     from restage.splitrun import make_splitrun_parser
@@ -16,7 +21,7 @@ def monitors_to_kafka_callback_with_arguments(
 ):
     from mccode_to_kafka.sender import send_histograms
 
-    partial_kwargs = {'broker': broker}
+    partial_kwargs: dict[str, Union[str,list[str]]] = {'broker': broker}
     if topic is not None and source is not None and names is not None and len(names) > 1:
         raise ValueError("Cannot specify both topic/source and multiple names simultaneously.")
 

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber/writer.py

@@ -137,7 +137,7 @@ def insert_events_in_nexus_structure(ns: dict, config: dict):
     return ns
 
 
-def get_writer_pool(broker: str = None, job: str = None, command: str = None):
+def get_writer_pool(broker: str | None = None, job: str | None = None, command: str | None = None):
     from .file_writer_control import WorkerJobPool
     print(f'Create a Writer pool for {broker=} {job=} {command=}')
     pool = WorkerJobPool(f"{broker}/{job}", f"{broker}/{command}")
@@ -151,17 +151,17 @@ def make_define_nexus_structure():
     def define_nexus_structure(
             instr: Path | str,
             pvs: list[dict],
-            title: str = None,
-            event_stream: dict[str, str] = None,
+            title: str | None = None,
+            event_stream: dict[str, str] | None = None,
             file: Path | None = None,
             func: Callable[[Instr], dict] | None = None,
             binary: Path | None = None,
-            origin: str = None):
+            origin: str | None = None):
         import json
         from .mccode import get_mcstas_instr
         if file is not None and file.exists():
-            with open(file, 'r') as file:
-                nexus_structure = json.load(file)
+            with open(file, 'r') as f:
+                nexus_structure = json.load(f)
         elif func is not None:
             nexus_structure = func(get_mcstas_instr(instr))
         elif binary is not None and binary.exists():
@@ -173,7 +173,7 @@ def make_define_nexus_structure():
         else:
             nexus_structure = default_nexus_structure(get_mcstas_instr(instr), origin=origin)
         nexus_structure = add_pvs_to_nexus_structure(nexus_structure, pvs)
-        nexus_structure = add_title_to_nexus_structure(nexus_structure, title)
+        nexus_structure = add_title_to_nexus_structure(nexus_structure, title or 'Unknown title')
         # nexus_structure = insert_events_in_nexus_structure(nexus_structure, event_stream)
         return nexus_structure
     return define_nexus_structure

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0/src/mccode_plumber.egg-info}/PKG-INFO

@@ -1,17 +1,25 @@
 Metadata-Version: 2.4
 Name: mccode-plumber
-Version: 0.14.4
+Version: 0.15.0
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Description-Content-Type: text/markdown
 Requires-Dist: p4p
 Requires-Dist: kafka-python>=2.2.11
 Requires-Dist: ess-streaming-data-types>=0.14.0
-Requires-Dist: restage>=0.9.0
+Requires-Dist: restage>=0.10.0
 Requires-Dist: mccode-to-kafka>=0.3.1
-Requires-Dist: moreniius>=0.6.1
+Requires-Dist: moreniius>=0.6.2
 Requires-Dist: icecream
 Requires-Dist: ephemeral-port-reserve
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: niess>=0.1.4; extra == "test"
 
 # McCode Plumber
 Setup, run, and teardown the infrastructure for splitrun McCode scans sending data through Kafka into NeXus

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber.egg-info/SOURCES.txt

@@ -1,5 +1,6 @@
 .gitignore
 README.md
+mypy.ini
 pyproject.toml
 .github/dependabot.yml
 .github/workflows/pip.yml
@@ -41,7 +42,10 @@ src/mccode_plumber/manage/forwarder.py
 src/mccode_plumber/manage/manager.py
 src/mccode_plumber/manage/orchestrate.py
 src/mccode_plumber/manage/writer.py
+tests/fake_efu.py
+tests/fake_manager.py
 tests/test_epics.py
+tests/test_management.py
 tests/test_orchestration_utils.py
 tests/test_splitrun.py
 tests/test_writer.py

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/src/mccode_plumber.egg-info/requires.txt

@@ -1,8 +1,12 @@
 p4p
 kafka-python>=2.2.11
 ess-streaming-data-types>=0.14.0
-restage>=0.9.0
+restage>=0.10.0
 mccode-to-kafka>=0.3.1
-moreniius>=0.6.1
+moreniius>=0.6.2
 icecream
 ephemeral-port-reserve
+
+[test]
+pytest
+niess>=0.1.4

mccode_plumber-0.15.0/tests/fake_efu.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+
+"""
+A fake EFU server for testing purposes. Arguments needed for a real EFU are accepted,
+but only the command port is utilized.
+
+Usage:
+------
+Extra carriage returns inserted to keep client and server timelines in sync vertically
+
+(client)                               |  (server)
+$                                      |  $ python fake_efu.py --cmdport 10123
+$ echo hello | netcat localhost 10123  |  1. connection from ('127.0.0.1', xxxxx)
+hello                                  |  ('127.0.0.1', xxxxx) says: hello
+$                                      |
+$ echo EXIT | netcat localhost 10123   |  1. connection from ('127.0.0.1', YYYYY)
+<OK>$                                  |  ('127.0.0.1', YYYYY) says: EXIT
+$                                      |
+$                                      |  $
+$                                      |  $ python fake_efu.py --cmdport 10456
+$ netcat localhost 10456               |  1. connection from ('127.0.0.1', ZZZZZ)
+Hello?                                 |  ('127.0.0.1', ZZZZZ) says: Hello
+Hello?                                 |
+Goodbye                                |  ('127.0.0.1', ZZZZZ) says: Goodbye
+Goodbye                                |
+EXIT                                   |
+$ echo EXIT | netcat localhost 10123   |  ('127.0.0.1', ZZZZZ) says: EXIT
+<OK>$                                  |
+$                                      |  $
+
+(Note that 'EXIT' within interactive netcat is a command and is not sent.)
+"""
+
+def main():
+    import sys
+    from argparse import ArgumentParser
+    parser = ArgumentParser(prog='fake_efu', description='Fake Efu commands')
+    parser.add_argument('--cmdport', type=int)
+    args, unknown = parser.parse_known_args()
+    if unknown:
+        print(f"Ignoring unknown args: {unknown}", file=sys.stderr)
+    serve(args.cmdport)
+
+
+def serve(p):
+    import socket
+    l = []
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    # Since we're undoubtedly calling this after an ephemeral_port_reserve.reserve() call
+    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    s.bind(('', p))
+    s.listen(1)
+    while 1:
+        (c, a) = s.accept()
+        l.append(c)
+        print(f'{len(l)}: connection from {a}')
+        received = str(c.recv(1024), 'utf-8')
+        print(f'{a} says: {received}')
+        if received == 'EXIT\n':
+            c.sendall(bytes('<OK>', 'utf-8'))
+            exit(0)
+        else:
+            c.sendall(bytes(received, 'utf-8'))
+        c.close()
+
+
+if __name__ == '__main__':
+    main()
+else:
+    print(f'Imported? {__name__}')

mccode_plumber-0.15.0/tests/fake_manager.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+
+import socket
+def sender(port, message):
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        try:
+            sock.settimeout(1.0)
+            sock.connect(('localhost', port))
+            sock.sendall(bytes(message, 'utf-8'))
+            recv = str(sock.recv(1024), "utf-8")
+        except TimeoutError:
+            print("Timeout!")
+            exit(1)
+        except ConnectionRefusedError:
+            print('Connection Refused')
+            exit(1)
+    print(recv)
+
+if __name__ == '__main__':
+    from argparse import ArgumentParser
+    parser = ArgumentParser()
+    parser.add_argument('p', type=int)
+    parser.add_argument('message', type=str, default='message')
+    args = parser.parse_args()
+    sender(args.p, args.message)
+
+
+

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/tests/test_epics.py

@@ -1,17 +1,28 @@
+import multiprocessing
 import unittest
 
 
+def main_for_tests(instr: str, prefix: str):
+    """Avoid needing to pickle p4p.nt.scalar.NTScalar values"""
+    from mccode_plumber.epics import main, convert_instr_parameters_to_nt
+    from mccode_antlr.loader.loader import parse_mcstas_instr
+    pvs = convert_instr_parameters_to_nt(parse_mcstas_instr(instr).parameters)
+    return main(pvs, prefix)
+
+
 class EPICSTestCase(unittest.TestCase):
     def setUp(self):
         from uuid import uuid4
-        from multiprocessing import Process
-        from mccode_plumber.epics import main, convert_instr_parameters_to_nt
-        from mccode_antlr.loader.loader import parse_mccode_instr_parameters, parse_mcstas_instr
+        from mccode_antlr.loader.loader import parse_mccode_instr_parameters
+
+        # Ensure we use the same multiprocessing method on all systems, Python versions
+        # Thus ensuring only pickle-able types are provided to multiprocessing.Process
+        ctx = multiprocessing.get_context('spawn')
+
         instr = 'define instrument blah(par1, double par2, int par3=1, string par4="string", double par5=5.5) trace end'
         self.pars = parse_mccode_instr_parameters(instr)
-        self.pvs = convert_instr_parameters_to_nt(parse_mcstas_instr(instr).parameters)
         self.prefix = f"test{str(uuid4()).replace('-', '')}:"
-        self.proc = Process(target=main, args=(self.pvs, self.prefix))
+        self.proc = ctx.Process(target=main_for_tests, args=(instr, self.prefix))
         self.proc.start()
 
     def tearDown(self):

mccode_plumber-0.15.0/tests/test_management.py

@@ -0,0 +1,69 @@
+import os
+import time
+import pytest
+from pathlib import Path
+from ephemeral_port_reserve import reserve
+from mccode_plumber.manage import EventFormationUnit
+
+
+def command_args(*args):
+    largs = [a if isinstance(a, str) else str(a) for a in args]
+    if os.name == 'nt':
+        return ["python"] + largs
+    return largs
+
+
+def assert_faker_working(port, message, retries=5, timeout=0.1):
+    from subprocess import run
+    binary = Path(__file__).resolve().parent / 'fake_manager.py'
+    res = 'Not-run yet'
+    for _ in range(retries):
+        res = run(command_args(binary, port, message), capture_output=True, text=True)
+        if res.returncode == 0:
+            return res.stdout.strip() == message
+        time.sleep(timeout)
+    print(f'Failed after {retries} attempts: {res}')
+    return False
+
+
+def test_all_fake():
+    from subprocess import Popen
+    port = reserve()
+    binary = Path(__file__).resolve().parent / 'fake_efu.py'
+    proc = Popen(command_args(binary, '--cmdport', port))
+    time.sleep(0.1) # startup
+    assert proc.poll() is None # ensure it's still running
+    assert assert_faker_working(port, "fake it till you make it")
+    proc.terminate()
+    proc.wait()
+    assert proc.poll() is not None
+
+
+@pytest.mark.skipif(os.name == 'nt', reason="fake_efu.py needs the python")
+def test_efu_management():
+    service = EventFormationUnit.start(
+        binary=Path(__file__).parent / "fake_efu.py",
+        config=Path(__file__),
+        calibration=Path(__file__),
+        broker='localhost:9092',
+        topic='TestEvents',
+        samples_topic='TestEvents_samples',
+        port=-1,
+        command=reserve(),
+        name='efu'
+    )
+    assert service.poll()
+    assert assert_faker_working(service.command, "test_management")
+    service.stop()
+
+
+@pytest.mark.skipif(os.name == 'nt', reason="fake_efu.py needs the python")
+def test_minimal_efu_management():
+    service = EventFormationUnit.start(
+        binary=Path(__file__).parent / "fake_efu.py",
+        config=Path(__file__),
+        calibration=Path(__file__),
+    )
+    assert service.poll()
+    assert assert_faker_working(service.command, "test_management")
+    service.stop()

{mccode_plumber-0.14.4 → mccode_plumber-0.15.0}/tests/test_splitrun.py

@@ -16,8 +16,9 @@ class SplitrunTestCase(unittest.TestCase):
 
     def test_mixed_order_throws(self):
         parser = make_parser()
-        parser.prog = "{{This failed before but works now? Why did it stop throwing?}}"
-        pa = parser.parse_args
+        parser.prog = "{{This failed before Python 3.12}}"
+        # Pre Python 3.12 was more-strict about argument position mixing
+        pa = getattr(parser, "parse_intermixed_args", parser.parse_args)
         # These also output usage information to stdout -- don't be surprised by the 'extra' test output.
         pa(['inst.h5', '--broker', 'l:9092', '--source', 'm', '-n', '10000',
             'a=1:4', 'b=2:5'
@@ -64,6 +65,11 @@ class SplitrunTestCase(unittest.TestCase):
         self.assertEqual(args.nmin, 1)
         self.assertEqual(args.nmax, 2**20)
 
+    def test_parsing_with_explicit_list(self):
+        parser = make_parser()
+        args = args_fixup(parser.parse_args(['--broker', 'l:9092', '--source', 'm', '-n', '10000', 'inst.h5', '--', 'a=1:4', 'b=2:5', 'c=1,2,3,4,5']))
+        self.assertEqual(args.parameters, ['a=1:4', 'b=2:5', 'c=1,2,3,4,5'])
+
 
 if __name__ == '__main__':
     unittest.main()

mccode_plumber-0.14.4/src/mccode_plumber/manage/manager.py

@@ -1,113 +0,0 @@
-from __future__ import annotations
-from dataclasses import dataclass
-from pathlib import Path
-from multiprocessing import Process, Pipe
-from multiprocessing.connection import Connection
-from enum import Enum
-from colorama import Fore, Back, Style
-
-class IOType(Enum):
-    stdout = 1
-    stderr = 2
-
-
-@dataclass
-class Manager:
-    """
-    Command and control of a process
-
-    Properties
-    ----------
-    _process:   a multiprocessing.Process instance, which is undefined for a short
-                period during instance creation inside the `start` class method
-    """
-    name: str
-    style: Style
-    _process: Process | None
-    _connection: Connection | None
-
-    def __run_command__(self) -> list[str]:
-        pass
-
-    def finalize(self):
-        pass
-
-    @classmethod
-    def fieldnames(cls) -> list[str]:
-        from dataclasses import fields
-        return [field.name for field in fields(cls)]
-
-    @classmethod
-    def start(cls, **config):
-        names = cls.fieldnames()
-        kwargs = {k: config[k] for k in names if k in config}
-        if any(k not in names for k in config):
-            raise ValueError(f'{config} expected to contain only {names}')
-        if '_process' not in kwargs:
-            kwargs['_process'] = None
-        if '_connection' not in kwargs:
-            kwargs['_connection'] = None
-        if 'name' not in kwargs:
-            kwargs['name'] = 'Managed process'
-        if 'style' not in kwargs:
-            kwargs['style'] = Fore.WHITE + Back.BLACK
-        manager = cls(**kwargs)
-        manager._connection, child_conn = Pipe()
-        manager._process = Process(target=manager.run, args=(child_conn,))
-        manager._process.start()
-        return manager
-
-    def stop(self):
-        self.finalize()
-        self._process.terminate()
-
-    def poll(self):
-        from sys import stderr
-        attn = Fore.BLACK + Back.RED + Style.BRIGHT
-        # check for anything received on our end of the connection
-        while self._connection.poll():
-            # examine what was returned:
-            try:
-                ret = self._connection.recv()
-            except EOFError:
-                print(f'{attn}{self.name}: [unexpected halt]{Style.RESET_ALL}')
-                return False
-            if len(ret) == 2:
-                t, line = ret
-                line = f'{self.style}{self.name}:{Style.RESET_ALL} {line}'
-                if t == IOType.stdout:
-                    print(line, end='')
-                else:
-                    print(line, file=stderr, end='')
-            else:
-                print(f'{attn}{self.name}: [unknown received data on connection]{Style.RESET_ALL}')
-        return self._process.is_alive()
-
-    def run(self, conn):
-        from subprocess import Popen, PIPE
-        from select import select
-        argv = self.__run_command__()
-
-        shell = isinstance(argv, str)
-        conn.send((IOType.stdout, f'Starting {argv if shell else " ".join(argv)}\n'))
-        process = Popen(argv, shell=shell, stdout=PIPE, stderr=PIPE, bufsize=1, universal_newlines=True, )
-        out, err = process.stdout.fileno(), process.stderr.fileno()
-        check = [process.stdout, process.stderr]
-        while process.poll() is None:
-            r, w, x = select(check, [], check, 0.5,)
-            for stream in r:
-                if stream.fileno() == out:
-                    conn.send((IOType.stdout, process.stdout.readline()))
-                elif stream.fileno() == err:
-                    conn.send((IOType.stderr, process.stderr.readline()))
-            for stream in x:
-                if stream.fileno() == out:
-                    conn.send((IOType.stdout, "EXCEPTION ON STDOUT"))
-                elif stream.fileno() == err:
-                    conn.send((IOType.stderr, "EXCEPTION ON STDERR"))
-        # Process finished, but the buffers may still contain data:
-        for stream in check:
-            if stream.fileno() == out:
-                map(lambda line: conn.send(IOType.stdout, line), stream.readlines())
-            elif stream.fileno() == err:
-                map(lambda line: conn.send(IOType.stderr, line), stream.readlines())