mccode-plumber 0.14.0__py3-none-any.whl → 0.14.1__py3-none-any.whl

mccode_plumber/manage/orchestrate.py

@@ -121,31 +121,31 @@ def start_writer(start_time: datetime,
     return job_id, success
 
 
-def get_topics_iter(data: list | tuple):
+def get_stream_pairs_list(data: list | tuple):
     topics = set()
     for entry in data:
         if isinstance(entry, dict):
-            topics.update(get_topics_dict(entry))
+            topics.update(get_stream_pairs_dict(entry))
         elif isinstance(entry, (list, tuple)):
-            topics.update(get_topics_iter(entry))
+            topics.update(get_stream_pairs_list(entry))
     return topics
 
 
-def get_topics_dict(data: dict):
+def get_stream_pairs_dict(data: dict):
     topics = set()
+    if all(k in data for k in ('topic', 'source')):
+        topics.add((data['topic'], data['source']))
     for k, v in data.items():
         if isinstance(v, dict):
-            topics.update(get_topics_dict(v))
+            topics.update(get_stream_pairs_dict(v))
         elif isinstance(v, (list, tuple)):
-            topics.update(get_topics_iter(list(v)))
-        elif k == 'topic':
-            topics.add(v)
+            topics.update(get_stream_pairs_list(list(v)))
     return topics
 
 
-def get_topics_json(data: dict) -> list[str]:
-    """Traverse a loaded JSON object and return the found list of topic names"""
-    return list(get_topics_dict(data))
+def get_stream_pairs(data: dict) -> list[tuple[str, str]]:
+    """Traverse a loaded JSON object and return the found list of (topic, source) pairs."""
+    return list(get_stream_pairs_dict(data))
 
 
 def load_file_json(file: str | Path):
@@ -360,16 +360,22 @@ def main():
     instr = get_mcstas_instr(args.instrument)
 
     structure = load_file_json(args.structure if args.structure else Path(args.instrument).with_suffix('.json'))
+
+    streams = get_stream_pairs(structure)
+    # All monitors should use a single topic:
+    monitor_topic = f'{instr.name}_beam_monitor'
+    if {monitor_topic} != {s[0] for s in streams}:
+        raise ValueError(f'All monitor streams must use the same topic {monitor_topic}, found {streams}')
+    monitor_names = [s[1] for s in streams]
+
     broker = 'localhost:9092'
-    monitor_source = 'mccode-to-kafka'
-    callback_topics = get_topics_json(structure)  # all structure-topics might be monitor topics?
-    if len(callback_topics):
-        print(f'register {callback_topics}')
-        register_topics(broker, callback_topics) # ensure the topics are known to Kafka
-    else:
-        print('no callback topics registered')
+    # monitor_source = 'mccode-to-kafka'  # old-style single source multi-topic
+    register_topics(broker, [monitor_topic]) # ensure the topics are known to Kafka
 
-    callback, callback_args = monitors_to_kafka_callback_with_arguments(broker, monitor_source, callback_topics)
+    # Configure the callback to send monitor data to Kafka, using the common topic with source names as monitor names
+    callback, callback_args = monitors_to_kafka_callback_with_arguments(
+        broker=broker, topic=monitor_topic, source=None, names=monitor_names
+    )
     splitrun_kwargs = {
         'args': args, 'parameters': parameters, 'precision': precision,
         'callback': callback, 'callback_arguments': callback_args,

mccode_plumber/splitrun.py

@@ -5,17 +5,27 @@ def make_parser():
     parser.prog = 'mp-splitrun'
     parser.add_argument('--broker', type=str, help='The Kafka broker to send monitors to', default=None)
     parser.add_argument('--source', type=str, help='The Kafka source name to use for monitors', default=None)
-    parser.add_argument('--topic', type=str, help='The Kafka topic name(s) to use for monitors', default=None, action='append')
+    parser.add_argument('--topic', type=str, help='The Kafka topic name to use for monitors', default=None)
+    parser.add_argument('--names', type=str, help='The monitor name(s) to send to Kafka', default=None, action='append')
     parser.add_argument('-v', '--version', action='version', version=__version__)
     return parser
 
 
-def monitors_to_kafka_callback_with_arguments(broker: str, source: str, topics: list[str]):
+def monitors_to_kafka_callback_with_arguments(
+        broker: str, topic: str | None, source: str | None, names: list[str] | None
+):
     from mccode_to_kafka.sender import send_histograms
 
-    partial_kwargs = {'broker': broker, 'source': source}
-    if topics is not None and len(topics) > 0:
-        partial_kwargs['names'] = topics
+    partial_kwargs = {'broker': broker}
+    if topic is not None and source is not None and names is not None and len(names) > 1:
+        raise ValueError("Cannot specify both topic/source and multiple names simultaneously.")
+
+    if topic is not None:
+        partial_kwargs['topic'] = topic
+    if source is not None:
+        partial_kwargs['source'] = source
+    if names is not None and len(names) > 0:
+        partial_kwargs['names'] = names
 
     def callback(*args, **kwargs):
         return send_histograms(*args, **partial_kwargs, **kwargs)
@@ -28,5 +38,5 @@ def main():
     from restage.splitrun import splitrun_args, parse_splitrun
     args, parameters, precision = parse_splitrun(make_parser())
     instr = get_mcstas_instr(args.instrument)
-    callback, callback_args = monitors_to_kafka_callback_with_arguments(args.broker, args.source, args.topic)
+    callback, callback_args = monitors_to_kafka_callback_with_arguments(args.broker, args.topic, args.source, args.names)
     return splitrun_args(instr, parameters, precision, args, callback=callback, callback_arguments=callback_args)

{mccode_plumber-0.14.0.dist-info → mccode_plumber-0.14.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mccode-plumber
-Version: 0.14.0
+Version: 0.14.1
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 Classifier: License :: OSI Approved :: BSD License
 Description-Content-Type: text/markdown
@@ -8,8 +8,8 @@ Requires-Dist: p4p
 Requires-Dist: kafka-python>=2.2.11
 Requires-Dist: ess-streaming-data-types>=0.14.0
 Requires-Dist: restage>=0.9.0
-Requires-Dist: mccode-to-kafka>=0.2.2
-Requires-Dist: moreniius>=0.6.0
+Requires-Dist: mccode-to-kafka>=0.3.0
+Requires-Dist: moreniius>=0.6.1
 Requires-Dist: icecream
 Requires-Dist: ephemeral-port-reserve
 

{mccode_plumber-0.14.0.dist-info → mccode_plumber-0.14.1.dist-info}/RECORD

@@ -5,7 +5,7 @@ mccode_plumber/epics_watcher.py,sha256=Jiz761A5NfoUzJ6ZzBGg8_BewFTmHDo5qYgh_DZHx
 mccode_plumber/forwarder.py,sha256=yzjb--r8M6vNSsWvvOi5IjXqpRn8MvT13cv89ezNEeU,3994
 mccode_plumber/kafka.py,sha256=Q3oAuk7c-PS7b7zWSDhOR3dOO8R6Q8Y3HPI7K-Kt-RU,2499
 mccode_plumber/mccode.py,sha256=vc4Gb5WhdOfuDRjEejU3Flps0C2A60sfFbMrxmKLhn0,2189
-mccode_plumber/splitrun.py,sha256=xL8k99NGRcU7tMCfdJCuj2a0-MvOZ7cPa6J11ACqpOA,1515
+mccode_plumber/splitrun.py,sha256=K5VetcWpw8Aa-4d2BQLyshkwXkiVdzzJRB1mr2eNyP0,1971
 mccode_plumber/utils.py,sha256=E8NoGo_3Z-pPEpzicVGSWfQOX8p3eR-GxElT33-kX5U,2167
 mccode_plumber/writer.py,sha256=YN2I_gvoXn8XkHX_BVbdvCGhJC1aWbwU9VtiYFrcGS0,21185
 mccode_plumber/file_writer_control/CommandChannel.py,sha256=U32mvk8Ctlw8vg38_ULP__-UAG2funWIkhEFo2xahiY,9024
@@ -27,10 +27,10 @@ mccode_plumber/manage/ensure.py,sha256=0HaxcHYzvcDs6hBvBI39EZsIDt_rVA9CjHm5_yvOZ
 mccode_plumber/manage/epics.py,sha256=oQt_hL-7KgtI9Kmwyw8L7a7RxQe7YJWRbmHnKIqMLpQ,1310
 mccode_plumber/manage/forwarder.py,sha256=YYvHaOJ4djBXM7PFF2NBbnNDO6nnrwNOfSHaVOh16Ro,2876
 mccode_plumber/manage/manager.py,sha256=zzrduroUL-jwQ9BrPTdAm1IW4dGv5bi8-ieIQ6qiQ6M,4141
-mccode_plumber/manage/orchestrate.py,sha256=bumvHN6jPEOqd8V96f7gEmc5IpWOZ6mSo7_bosi3jY8,17662
+mccode_plumber/manage/orchestrate.py,sha256=rd6tSAyqT6B0gqj4uXtk0o6plaoVfisCy1xrmlymlXk,18088
 mccode_plumber/manage/writer.py,sha256=SEv1U14L01Y9-BcaJKPei4Ah2LFfwexDy9FTjpvtSEs,2245
-mccode_plumber-0.14.0.dist-info/METADATA,sha256=tceaL7T1Li53p4HVOLaUOpKgvrEx12o8Npoqui_LgDY,594
-mccode_plumber-0.14.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mccode_plumber-0.14.0.dist-info/entry_points.txt,sha256=k3LDo_9HG2v4-IgMYlNukphdMmaAT6zkJZYaB1zJh3c,900
-mccode_plumber-0.14.0.dist-info/top_level.txt,sha256=kCCIpYtKHCKWxiPEqX9J1UaGEm-ze0Qb-cemBCEPhDA,15
-mccode_plumber-0.14.0.dist-info/RECORD,,
+mccode_plumber-0.14.1.dist-info/METADATA,sha256=BbSQyG3UF1fPgzVR6XbfwB9hqicfaqYtSw8aoxC0Bic,594
+mccode_plumber-0.14.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mccode_plumber-0.14.1.dist-info/entry_points.txt,sha256=k3LDo_9HG2v4-IgMYlNukphdMmaAT6zkJZYaB1zJh3c,900
+mccode_plumber-0.14.1.dist-info/top_level.txt,sha256=kCCIpYtKHCKWxiPEqX9J1UaGEm-ze0Qb-cemBCEPhDA,15
+mccode_plumber-0.14.1.dist-info/RECORD,,