mccode-plumber 0.14.0__tar.gz

mccode_plumber-0.14.0/.github/workflows/pip.yml

@@ -0,0 +1,38 @@
+name: Pip
+
+on:
+  workflow_dispatch:
+    #  pull_request:
+    #  push:
+    #    branches:
+    #      - main
+
+jobs:
+  build:
+    name: Build with Pip
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [windows-latest, macos-latest, ubuntu-latest]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Set min macOS version
+      if: runner.os == 'macOS'
+      run: |
+        echo "MACOS_DEPLOYMENT_TARGET=10.14" >> $GITHUB_ENV
+
+    - name: Build and install
+      run: pip install --verbose .
+
+    - name: Test
+      run: |
+        python -m pip install pytest
+        python -m pytest

mccode_plumber-0.14.0/.github/workflows/wheels.yml

@@ -0,0 +1,52 @@
+name: Wheels
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+     - main
+  release:
+    types:
+      - published
+
+jobs:
+  build_sdist_and_wheel:
+    name: Build SDist and Wheel
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        submodules: true
+
+    - name: Build SDist
+      run: pipx run build
+
+    - name: Check metadata
+      run: pipx run twine check dist/*
+
+    - uses: actions/upload-artifact@v4
+      with:
+        path: dist/*
+
+  upload_all:
+    name: Upload if release
+    needs: [build_sdist_and_wheel]
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/mccode-plumber
+    permissions:
+      id-token: write
+    if: github.event_name == 'release' && github.event.action == 'published'
+
+    steps:
+    - uses: actions/setup-python@v5
+
+    - uses: actions/download-artifact@v4
+      with:
+        name: artifact
+        path: dist
+
+    - uses: pypa/gh-action-pypi-publish@release/v1

mccode_plumber-0.14.0/.gitignore

@@ -0,0 +1,5 @@
+build/
+venv/
+__pycache__/
+.idea/
+*.egg-info

mccode_plumber-0.14.0/PKG-INFO

@@ -0,0 +1,17 @@
+Metadata-Version: 2.4
+Name: mccode-plumber
+Version: 0.14.0
+Author-email: Gregory Tucker <gregory.tucker@ess.eu>
+Classifier: License :: OSI Approved :: BSD License
+Description-Content-Type: text/markdown
+Requires-Dist: p4p
+Requires-Dist: kafka-python>=2.2.11
+Requires-Dist: ess-streaming-data-types>=0.14.0
+Requires-Dist: restage>=0.9.0
+Requires-Dist: mccode-to-kafka>=0.2.2
+Requires-Dist: moreniius>=0.6.0
+Requires-Dist: icecream
+Requires-Dist: ephemeral-port-reserve
+
+# McCode Plumber
+Setup, run, and teardown the infrastructure for splitrun McCode scans sending data through Kafka into NeXus

mccode_plumber-0.14.0/README.md

@@ -0,0 +1,2 @@
+# McCode Plumber
+Setup, run, and teardown the infrastructure for splitrun McCode scans sending data through Kafka into NeXus

mccode_plumber-0.14.0/pyproject.toml

@@ -0,0 +1,46 @@
+[build-system]
+requires = ['setuptools>=60', 'setuptools_scm>=8.0']
+build-backend = 'setuptools.build_meta'
+
+[project]
+name = 'mccode-plumber'
+dependencies = [
+    'p4p',
+    'kafka-python>=2.2.11',
+    'ess-streaming-data-types>=0.14.0',
+    'restage>=0.9.0',
+    'mccode-to-kafka>=0.2.2',
+    'moreniius>=0.6.0',
+    'icecream',
+    'ephemeral-port-reserve',
+]
+readme = "README.md"
+authors = [
+    { name = "Gregory Tucker", email = "gregory.tucker@ess.eu" },
+]
+classifiers = [
+    "License :: OSI Approved :: BSD License",
+]
+dynamic = ['version']
+
+[project.scripts]
+mp-splitrun = 'mccode_plumber.splitrun:main'
+mp-epics = 'mccode_plumber.epics:run'
+mp-epics-strings = 'mccode_plumber.epics:run_strings'
+mp-epics-update = 'mccode_plumber.epics:update'
+mp-epics-watch = 'mccode_plumber.epics_watcher:run_instr'
+mp-forwarder-setup = 'mccode_plumber.forwarder:setup'
+mp-forwarder-teardown = 'mccode_plumber.forwarder:teardown'
+mp-writer-from = 'mccode_plumber.writer:print_time'
+mp-writer-write = 'mccode_plumber.writer:start_writer'
+mp-writer-wait = 'mccode_plumber.writer:wait_on_writer'
+mp-writer-list = 'mccode_plumber.writer:list_status'
+mp-writer-kill = 'mccode_plumber.writer:kill_job'
+mp-writer-killall = 'mccode_plumber.writer:kill_all'
+mp-register-topics = 'mccode_plumber.kafka:register_topics'
+mp-insert-hdf5-instr = 'mccode_plumber.mccode:insert'
+mp-nexus-splitrun = 'mccode_plumber.manage.orchestrate:main'
+mp-nexus-services = 'mccode_plumber.manage.orchestrate:services'
+
+[tool.setuptools_scm]
+

mccode_plumber-0.14.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

mccode_plumber-0.14.0/src/mccode_plumber/__init__.py

@@ -0,0 +1,6 @@
+from importlib.metadata import version, PackageNotFoundError
+
+try:
+    __version__ = version('mccode-plumber')
+except PackageNotFoundError:
+    pass

mccode_plumber-0.14.0/src/mccode_plumber/epics.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+from p4p.nt import NTScalar
+from p4p.server import Server, StaticProvider
+from p4p.server.thread import SharedPV
+from pathlib import Path
+from typing import Union
+
+def instr_par_to_nt_primitive(parameters):
+    from mccode_antlr.common.expression import DataType, ShapeType
+    out = []
+    for p in parameters:
+        expr = p.value
+        if expr.is_str:
+            t, d = 's', ''
+        elif expr.data_type == DataType.int:
+            t, d = 'i', 0
+        elif expr.data_type == DataType.float:
+            t, d = 'd', 0.0
+        else:
+            raise ValueError(f"Unknown parameter type {expr.data_type}")
+        if expr.shape_type == ShapeType.vector:
+            t, d = 'a' + t, [d]
+        out.append((p.name, t, d))
+    return out
+
+def instr_par_nt_to_strings(parameters):
+    return [f'{n}:{t}:{d}'.replace(' ','') for n, t, d in instr_par_to_nt_primitive(parameters)]
+
+def strings_to_instr_par_nt(strings):
+    out = []
+    for string in strings:
+        name, t, dstr = string.split(':')
+        trans = None
+        if 'i' in t:
+            trans = int
+        elif 'd' in t:
+            trans = float
+        elif 's' in t:
+            trans = str
+        else:
+            ValueError(f"Unknown type in {string}")
+        if t.startswith('a'):
+            d = [trans(x) for x in dstr.translate(str.maketrans(',',' ','[]')).split()]
+        else:
+            d = trans(dstr)
+        out.append((name, t, d))
+    return out
+
+def convert_strings_to_nt(strings):
+    return {n: NTScalar(t).wrap(d) for n, t, d in strings_to_instr_par_nt(strings)}
+
+def convert_instr_parameters_to_nt(parameters):
+    out = {n: NTScalar(t).wrap(d) for n, t, d in instr_par_to_nt_primitive(parameters)}
+    return out
+
+
+def parse_instr_nt_values(instr: Union[Path, str]):
+    """Get the instrument parameters from an Instr a or a parseable Instr file and convert to NTScalar values"""
+    from .mccode import get_mccode_instr_parameters
+    nts = convert_instr_parameters_to_nt(get_mccode_instr_parameters(instr))
+    if 'mcpl_filename' not in nts:
+        nts['mcpl_filename'] = NTScalar('s').wrap('')
+    return nts
+
+
+class MailboxHandler:
+    @staticmethod
+    def put(pv, op):
+        from datetime import datetime, timezone
+        val = op.value()
+
+        if pv.nt is None:
+            # Assume that this means wrap wasn't provided ...
+            pv.nt = NTScalar(val.type()['value'])
+            pv._wrap = pv.nt.wrap
+
+        # Notify any subscribers of the new value, adding the timestamp, so they know when it was set.
+        pv.post(val, timestamp=datetime.now(timezone.utc).timestamp())
+        # Notify the client making this PUT operation that it has now completed
+        op.done()
+
+
+def get_parser():
+    from argparse import ArgumentParser
+    from mccode_plumber import __version__
+    p = ArgumentParser()
+    p.add_argument('instr', type=str, help='The instrument file to read')
+    p.add_argument('-p', '--prefix', type=str, help='The EPICS PV prefix to use', default='mcstas:')
+    p.add_argument('-v', '--version', action='version', version=__version__)
+    return p
+
+
+def parse_args():
+    args = get_parser().parse_args()
+    parameters = parse_instr_nt_values(args.instr)
+    return parameters, args
+
+
+def main(names: dict[str, NTScalar], prefix: str = None, filename_required: bool = True):
+    provider = StaticProvider('mailbox')  # 'mailbox' is an arbitrary name
+
+    if filename_required and 'mcpl_filename' not in names:
+        names['mcpl_filename'] = NTScalar('s').wrap('')
+
+    pvs = []  # we must keep a reference in order to keep the Handler from being collected
+    for name, value in names.items():
+        pv = SharedPV(initial=value, handler=MailboxHandler())
+        provider.add(f'{prefix}{name}' if prefix else name, pv)
+        pvs.append(pv)
+
+    print(f'Start mailbox server for {len(pvs)} PVs with prefix {prefix}')
+    Server.forever(providers=[provider])
+    print('Done')
+
+
+def run():
+    parameters, args = parse_args()
+    main(parameters, prefix=args.prefix)
+
+
+def start(parameters, prefix: str = None):
+    from multiprocessing import Process
+    proc = Process(target=main, args=(parameters, prefix))
+    proc.start()
+    return proc
+
+
+def stop(proc):
+    proc.terminate()
+    proc.join(1)
+    proc.close()
+
+
+def update():
+    from argparse import ArgumentParser
+    from p4p.client.thread import Context
+    parser = ArgumentParser(description="Update the mailbox server with new values")
+    parser.add_argument('address value', type=str, nargs='+', help='The mailbox address and value to be updated')
+    args = parser.parse_args()
+    addresses_values = getattr(args, 'address value')
+
+    if len(addresses_values) == 0:
+        parser.print_help()
+        return
+
+    addresses = addresses_values[::2]
+    values = addresses_values[1::2]
+
+    if len(addresses_values) % 2:
+        print(f'Please provide address-value pairs. Provided {addresses=} {values=}')
+
+    ctx = Context('pva')
+    for address, value in zip(addresses, values):
+        pv = ctx.get(address, throw=False)
+        if isinstance(pv, float):
+            ctx.put(address, float(value))
+        elif isinstance(pv, int):
+            ctx.put(address, int(value))
+        elif isinstance(pv, str):
+            ctx.put(address, str(value))
+        elif isinstance(pv, TimeoutError):
+            print(f'[Timeout] Failed to update {address} with {value} (Unknown to EPICS?)')
+        else:
+            raise ValueError(f'Address {address} has unknown type {type(pv)}')
+
+    ctx.disconnect()
+
+
+def get_strings_parser():
+    from argparse import ArgumentParser
+    from mccode_plumber import __version__
+    p = ArgumentParser()
+    p.add_argument('strings', type=str, nargs='+', help='The string encoded NTScalars to read, each name:type-char:default')
+    p.add_argument('-p', '--prefix', type=str, help='The EPICS PV prefix to use', default='mcstas:')
+    p.add_argument('-v', '--version', action='version', version=__version__)
+    return p
+
+
+def run_strings():
+    args = get_strings_parser().parse_args()
+    main(convert_strings_to_nt(args.strings), prefix=args.prefix)
+
+
+
+if __name__ == '__main__':
+    run()

mccode_plumber-0.14.0/src/mccode_plumber/epics_watcher.py

@@ -0,0 +1,125 @@
+from textual.app import App, ComposeResult
+from textual.containers import VerticalScroll
+from textual.widgets import Static
+from textual.reactive import reactive
+from textual import events
+
+from p4p.client.thread import Context
+from p4p.client.thread import Disconnected
+
+import asyncio
+import threading
+
+
+class PVWidget(Static):
+    value: reactive[str] = reactive("Connecting...")
+
+    def __init__(self, pvname: str):
+        super().__init__()
+        self.pvname = pvname
+        self.sid = pvname.replace(':', '')
+        self.set_class(True, "pv-widget")
+        self.value_widget = None  # Store reference directly
+
+    def compose(self) -> ComposeResult:
+        yield Static(f"[b]{self.pvname}[/b]", id=f"label-{self.sid}")
+        self.value_widget = Static(self.value, id=f"value-{self.sid}")
+        yield self.value_widget
+
+    def watch_value(self, value: str):
+        if self.value_widget:
+            self.value_widget.update(value)
+
+    def update_value(self, new_value: str):
+        self.value = new_value
+
+class PVMonitorApp(App):
+    CSS = """
+    .pv-widget {
+        padding: 1 1;
+        border: round $primary;
+        margin: 1;
+    }
+    """
+
+    def __init__(self, prefix: str, names: list[str]):
+        super().__init__()
+        self.pv_widgets = {}
+        self.ctx = Context("pva")  # Or 'ca' if using Channel Access
+        self.prefix = prefix
+        self.names = names
+        self.grid = None
+
+    def compose(self) -> ComposeResult:
+        from textual.containers import Grid
+        self.grid = Grid(id="pv-grid")
+        yield self.grid
+
+    def on_mount(self) -> None:
+        self.grid.styles.grid_columns = ["1fr", "1fr", "1fr"]  # 3 columns
+        self.grid.styles.grid_gap = (1, 1)
+        for name in self.names:
+            pv = f'{self.prefix}{name}'
+            widget = PVWidget(pv)
+            self.pv_widgets[pv] = widget
+            self.grid.mount(widget)
+            threading.Thread(target=self.monitor_pv, args=(pv,), daemon=True).start()
+
+    def monitor_pv(self, pvname: str):
+        def callback(value):
+            if isinstance(value, Disconnected):
+                new_value = "Disconnected"
+            else:
+                new_value = str(value)
+            # Schedule update on the main thread
+            asyncio.run_coroutine_threadsafe(
+                self.update_widget_value(pvname, new_value),
+                self._loop,
+            )
+
+        self.ctx.monitor(pvname, callback)
+
+    async def update_widget_value(self, pvname: str, new_value: str):
+        widget = self.pv_widgets.get(pvname)
+        if widget:
+            widget.update_value(new_value)
+
+    async def on_shutdown(self) -> None:
+        self.ctx.close()
+
+
+
+def get_names_parser():
+    from argparse import ArgumentParser
+    from mccode_plumber import __version__
+    p = ArgumentParser()
+    p.add_argument('name', type=str, nargs='+', help='The NTScalar names to watch')
+    p.add_argument('-p', '--prefix', type=str, help='The EPICS PV prefix to use', default='mcstas:')
+    p.add_argument('-v', '--version', action='version', version=__version__)
+    return p
+
+
+def run_strings():
+    args = get_names_parser().parse_args()
+    PVMonitorApp(args.prefix, args.name).run()
+
+
+def get_instr_parser():
+    from argparse import ArgumentParser
+    from mccode_plumber import __version__
+    p = ArgumentParser()
+    p.add_argument('instr', type=str, help='The instrument which defines names to watch')
+    p.add_argument('-p', '--prefix', type=str, help='The EPICS PV prefix to use', default='mcstas:')
+    p.add_argument('-v', '--version', action='version', version=__version__)
+    return p
+
+def run_instr():
+    from mccode_plumber.manage.orchestrate import get_instr_name_and_parameters
+    args = get_instr_parser().parse_args()
+    _, parameters = get_instr_name_and_parameters(args.instr)
+    names = [p.name for p in parameters]
+    PVMonitorApp(args.prefix, names).run()
+
+
+if __name__ == '__main__':
+    run_instr()