mccode-plumber 0.14.0__tar.gz → 0.14.1__tar.gz

mccode_plumber-0.14.1/.github/dependabot.yml

@@ -0,0 +1,17 @@
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    groups:
+      python-packages:
+        patterns:
+          - "*"

{mccode_plumber-0.14.0 → mccode_plumber-0.14.1}/.github/workflows/pip.yml

@@ -18,9 +18,9 @@ jobs:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
 

{mccode_plumber-0.14.0 → mccode_plumber-0.14.1}/.github/workflows/wheels.yml

@@ -15,7 +15,7 @@ jobs:
     name: Build SDist and Wheel
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 0
         submodules: true
@@ -26,7 +26,7 @@ jobs:
     - name: Check metadata
       run: pipx run twine check dist/*
 
-    - uses: actions/upload-artifact@v4
+    - uses: actions/upload-artifact@v6
       with:
         path: dist/*
 
@@ -42,9 +42,9 @@ jobs:
     if: github.event_name == 'release' && github.event.action == 'published'
 
     steps:
-    - uses: actions/setup-python@v5
+    - uses: actions/setup-python@v6
 
-    - uses: actions/download-artifact@v4
+    - uses: actions/download-artifact@v7
       with:
         name: artifact
         path: dist

{mccode_plumber-0.14.0/src/mccode_plumber.egg-info → mccode_plumber-0.14.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mccode-plumber
-Version: 0.14.0
+Version: 0.14.1
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 Classifier: License :: OSI Approved :: BSD License
 Description-Content-Type: text/markdown
@@ -8,8 +8,8 @@ Requires-Dist: p4p
 Requires-Dist: kafka-python>=2.2.11
 Requires-Dist: ess-streaming-data-types>=0.14.0
 Requires-Dist: restage>=0.9.0
-Requires-Dist: mccode-to-kafka>=0.2.2
-Requires-Dist: moreniius>=0.6.0
+Requires-Dist: mccode-to-kafka>=0.3.0
+Requires-Dist: moreniius>=0.6.1
 Requires-Dist: icecream
 Requires-Dist: ephemeral-port-reserve
 

{mccode_plumber-0.14.0 → mccode_plumber-0.14.1}/pyproject.toml

@@ -9,8 +9,8 @@ dependencies = [
     'kafka-python>=2.2.11',
     'ess-streaming-data-types>=0.14.0',
     'restage>=0.9.0',
-    'mccode-to-kafka>=0.2.2',
-    'moreniius>=0.6.0',
+    'mccode-to-kafka>=0.3.0',
+    'moreniius>=0.6.1',
     'icecream',
     'ephemeral-port-reserve',
 ]

{mccode_plumber-0.14.0 → mccode_plumber-0.14.1}/src/mccode_plumber/manage/orchestrate.py

@@ -121,31 +121,31 @@ def start_writer(start_time: datetime,
     return job_id, success
 
 
-def get_topics_iter(data: list | tuple):
+def get_stream_pairs_list(data: list | tuple):
     topics = set()
     for entry in data:
         if isinstance(entry, dict):
-            topics.update(get_topics_dict(entry))
+            topics.update(get_stream_pairs_dict(entry))
         elif isinstance(entry, (list, tuple)):
-            topics.update(get_topics_iter(entry))
+            topics.update(get_stream_pairs_list(entry))
     return topics
 
 
-def get_topics_dict(data: dict):
+def get_stream_pairs_dict(data: dict):
     topics = set()
+    if all(k in data for k in ('topic', 'source')):
+        topics.add((data['topic'], data['source']))
     for k, v in data.items():
         if isinstance(v, dict):
-            topics.update(get_topics_dict(v))
+            topics.update(get_stream_pairs_dict(v))
         elif isinstance(v, (list, tuple)):
-            topics.update(get_topics_iter(list(v)))
-        elif k == 'topic':
-            topics.add(v)
+            topics.update(get_stream_pairs_list(list(v)))
     return topics
 
 
-def get_topics_json(data: dict) -> list[str]:
-    """Traverse a loaded JSON object and return the found list of topic names"""
-    return list(get_topics_dict(data))
+def get_stream_pairs(data: dict) -> list[tuple[str, str]]:
+    """Traverse a loaded JSON object and return the found list of (topic, source) pairs."""
+    return list(get_stream_pairs_dict(data))
 
 
 def load_file_json(file: str | Path):
@@ -360,16 +360,22 @@ def main():
     instr = get_mcstas_instr(args.instrument)
 
     structure = load_file_json(args.structure if args.structure else Path(args.instrument).with_suffix('.json'))
+
+    streams = get_stream_pairs(structure)
+    # All monitors should use a single topic:
+    monitor_topic = f'{instr.name}_beam_monitor'
+    if {monitor_topic} != {s[0] for s in streams}:
+        raise ValueError(f'All monitor streams must use the same topic {monitor_topic}, found {streams}')
+    monitor_names = [s[1] for s in streams]
+
     broker = 'localhost:9092'
-    monitor_source = 'mccode-to-kafka'
-    callback_topics = get_topics_json(structure)  # all structure-topics might be monitor topics?
-    if len(callback_topics):
-        print(f'register {callback_topics}')
-        register_topics(broker, callback_topics) # ensure the topics are known to Kafka
-    else:
-        print('no callback topics registered')
+    # monitor_source = 'mccode-to-kafka'  # old-style single source multi-topic
+    register_topics(broker, [monitor_topic]) # ensure the topics are known to Kafka
 
-    callback, callback_args = monitors_to_kafka_callback_with_arguments(broker, monitor_source, callback_topics)
+    # Configure the callback to send monitor data to Kafka, using the common topic with source names as monitor names
+    callback, callback_args = monitors_to_kafka_callback_with_arguments(
+        broker=broker, topic=monitor_topic, source=None, names=monitor_names
+    )
     splitrun_kwargs = {
         'args': args, 'parameters': parameters, 'precision': precision,
         'callback': callback, 'callback_arguments': callback_args,

{mccode_plumber-0.14.0 → mccode_plumber-0.14.1}/src/mccode_plumber/splitrun.py

@@ -5,17 +5,27 @@ def make_parser():
     parser.prog = 'mp-splitrun'
     parser.add_argument('--broker', type=str, help='The Kafka broker to send monitors to', default=None)
     parser.add_argument('--source', type=str, help='The Kafka source name to use for monitors', default=None)
-    parser.add_argument('--topic', type=str, help='The Kafka topic name(s) to use for monitors', default=None, action='append')
+    parser.add_argument('--topic', type=str, help='The Kafka topic name to use for monitors', default=None)
+    parser.add_argument('--names', type=str, help='The monitor name(s) to send to Kafka', default=None, action='append')
     parser.add_argument('-v', '--version', action='version', version=__version__)
     return parser
 
 
-def monitors_to_kafka_callback_with_arguments(broker: str, source: str, topics: list[str]):
+def monitors_to_kafka_callback_with_arguments(
+        broker: str, topic: str | None, source: str | None, names: list[str] | None
+):
     from mccode_to_kafka.sender import send_histograms
 
-    partial_kwargs = {'broker': broker, 'source': source}
-    if topics is not None and len(topics) > 0:
-        partial_kwargs['names'] = topics
+    partial_kwargs = {'broker': broker}
+    if topic is not None and source is not None and names is not None and len(names) > 1:
+        raise ValueError("Cannot specify both topic/source and multiple names simultaneously.")
+
+    if topic is not None:
+        partial_kwargs['topic'] = topic
+    if source is not None:
+        partial_kwargs['source'] = source
+    if names is not None and len(names) > 0:
+        partial_kwargs['names'] = names
 
     def callback(*args, **kwargs):
         return send_histograms(*args, **partial_kwargs, **kwargs)
@@ -28,5 +38,5 @@ def main():
     from restage.splitrun import splitrun_args, parse_splitrun
     args, parameters, precision = parse_splitrun(make_parser())
     instr = get_mcstas_instr(args.instrument)
-    callback, callback_args = monitors_to_kafka_callback_with_arguments(args.broker, args.source, args.topic)
+    callback, callback_args = monitors_to_kafka_callback_with_arguments(args.broker, args.topic, args.source, args.names)
     return splitrun_args(instr, parameters, precision, args, callback=callback, callback_arguments=callback_args)

{mccode_plumber-0.14.0 → mccode_plumber-0.14.1/src/mccode_plumber.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mccode-plumber
-Version: 0.14.0
+Version: 0.14.1
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 Classifier: License :: OSI Approved :: BSD License
 Description-Content-Type: text/markdown
@@ -8,8 +8,8 @@ Requires-Dist: p4p
 Requires-Dist: kafka-python>=2.2.11
 Requires-Dist: ess-streaming-data-types>=0.14.0
 Requires-Dist: restage>=0.9.0
-Requires-Dist: mccode-to-kafka>=0.2.2
-Requires-Dist: moreniius>=0.6.0
+Requires-Dist: mccode-to-kafka>=0.3.0
+Requires-Dist: moreniius>=0.6.1
 Requires-Dist: icecream
 Requires-Dist: ephemeral-port-reserve
 

{mccode_plumber-0.14.0 → mccode_plumber-0.14.1}/src/mccode_plumber.egg-info/SOURCES.txt

@@ -1,6 +1,7 @@
 .gitignore
 README.md
 pyproject.toml
+.github/dependabot.yml
 .github/workflows/pip.yml
 .github/workflows/wheels.yml
 src/mccode_plumber/__init__.py
@@ -41,5 +42,6 @@ src/mccode_plumber/manage/manager.py
 src/mccode_plumber/manage/orchestrate.py
 src/mccode_plumber/manage/writer.py
 tests/test_epics.py
+tests/test_orchestration_utils.py
 tests/test_splitrun.py
 tests/test_writer.py

{mccode_plumber-0.14.0 → mccode_plumber-0.14.1}/src/mccode_plumber.egg-info/requires.txt

@@ -2,7 +2,7 @@ p4p
 kafka-python>=2.2.11
 ess-streaming-data-types>=0.14.0
 restage>=0.9.0
-mccode-to-kafka>=0.2.2
-moreniius>=0.6.0
+mccode-to-kafka>=0.3.0
+moreniius>=0.6.1
 icecream
 ephemeral-port-reserve

mccode_plumber-0.14.1/tests/test_orchestration_utils.py

@@ -0,0 +1,66 @@
+from mccode_antlr.instr import Instr
+
+
+def get_registries():
+    from mccode_antlr.reader import GitHubRegistry
+
+    registries = ['mcstas-chopper-lib', 'mcstas-detector-tubes', 'mcstas-frame-tof-monitor', 'mccode-mcpl-filter',]
+    registries = [GitHubRegistry(
+        name,
+        url=f'https://github.com/mcdotstar/{name}',
+        filename='pooch-registry.txt',
+        version='main'
+    ) for name in registries]
+
+    return registries
+
+
+def instr_to_nexus_structure_json(instrument: Instr):
+    from tempfile import TemporaryDirectory
+    from pathlib import Path
+    from json import load
+    import moreniius
+    import moreniius.additions
+
+    moreniius.additions.BIFROST_DETECTOR_TOPIC='SimulatedEvents'
+
+    nx = moreniius.MorEniius.from_mccode(
+        instrument,
+        origin='sample_origin',
+        only_nx=False,
+        absolute_depends_on=False,
+    )
+
+    with TemporaryDirectory() as tmpdir:
+        json_file = Path(tmpdir)/f'{instrument.name}.json'
+        nx.to_json(json_file.as_posix())
+        with open(json_file, 'r') as f:
+            return load(f)
+
+
+def test_monitor_streams():
+    from mccode_antlr import Flavor
+    from mccode_plumber.manage.orchestrate import get_stream_pairs
+    from mccode_antlr.assembler import Assembler
+    from niess.bifrost import Primary
+    from scipp import scalar
+
+    assembler = Assembler("bifrost", flavor=Flavor.MCSTAS, registries=get_registries())
+
+    primary = Primary.from_calibration()
+    primary.source.n_pulses = 1
+    primary.source.accelerator_power = scalar(2.0, unit='MW')
+
+    primary.to_mccode(assembler)
+
+    instr = instr_to_nexus_structure_json(assembler.instrument)
+    streams = get_stream_pairs(instr)
+    topics = set(t for t, _ in streams)
+    sources = set(n for _, n in streams)
+
+    assert topics == {'bifrost_beam_monitor'}
+    assert sources == {f'{x}_monitor' for x in ['psc', 'overlap', 'bandwidth', 'normalization']}
+
+
+if __name__ == '__main__':
+    test_monitor_streams()

{mccode_plumber-0.14.0 → mccode_plumber-0.14.1}/tests/test_writer.py

@@ -29,9 +29,9 @@ class WriterTestCase(unittest.TestCase):
         self.assertEqual(len(struct['children']), 1)
         self.assertEqual(struct['children'][0]['name'], 'entry')
         self.assertEqual(struct['children'][0]['children'][0]['name'], 'instrument')
-        self.assertEqual(struct['children'][0]['children'][0]['children'][1]['name'], '0_origin')
-        self.assertEqual(struct['children'][0]['children'][0]['children'][2]['name'], '1_source')
-        self.assertEqual(struct['children'][0]['children'][0]['children'][3]['name'], '2_monitor')
+        self.assertEqual(struct['children'][0]['children'][0]['children'][1]['name'], 'origin')
+        self.assertEqual(struct['children'][0]['children'][0]['children'][2]['name'], 'source')
+        self.assertEqual(struct['children'][0]['children'][0]['children'][3]['name'], 'monitor')
         mon = struct['children'][0]['children'][0]['children'][3]
         self.assertEqual(len(mon['children']), 4)  # removed 'mccode' property 5->4
         idx = [i for i, ch in enumerate(mon['children']) if 'name' in ch and 'data' == ch['name']]