mccode-plumber 0.7.0__tar.gz → 0.8.0__tar.gz

{mccode_plumber-0.7.0/src/mccode_plumber.egg-info → mccode_plumber-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mccode-plumber
-Version: 0.7.0
+Version: 0.8.0
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 Classifier: License :: OSI Approved :: BSD License
 Description-Content-Type: text/markdown
@@ -8,7 +8,7 @@ Requires-Dist: p4p
 Requires-Dist: kafka-python>=2.0
 Requires-Dist: ess-streaming-data-types>=0.14.0
 Requires-Dist: restage>=0.4.0
-Requires-Dist: mccode-to-kafka>=0.2.1
+Requires-Dist: mccode-to-kafka>=0.2.2
 Requires-Dist: moreniius>=0.2.3
 Requires-Dist: icecream
 

{mccode_plumber-0.7.0 → mccode_plumber-0.8.0}/pyproject.toml

@@ -9,7 +9,7 @@ dependencies = [
     'kafka-python>=2.0',
     'ess-streaming-data-types>=0.14.0',
     'restage>=0.4.0',
-    'mccode-to-kafka>=0.2.1',
+    'mccode-to-kafka>=0.2.2',
     'moreniius>=0.2.3',
     'icecream',
 ]
@@ -31,6 +31,9 @@ mp-forwarder-teardown = 'mccode_plumber.forwarder:teardown'
 mp-writer-from = 'mccode_plumber.writer:print_time'
 mp-writer-write = 'mccode_plumber.writer:start_writer'
 mp-writer-wait = 'mccode_plumber.writer:wait_on_writer'
+mp-writer-list = 'mccode_plumber.writer:list_status'
+mp-writer-kill = 'mccode_plumber.writer:kill_job'
+mp-writer-killall = 'mccode_plumber.writer:kill_all'
 mp-register-topics = 'mccode_plumber.kafka:register_topics'
 mp-insert-hdf5-instr = 'mccode_plumber.mccode:insert'
 

{mccode_plumber-0.7.0 → mccode_plumber-0.8.0}/src/mccode_plumber/epics.py

@@ -72,6 +72,7 @@ def main(names: dict[str, NTScalar], prefix: str = None):
     for name, value in names.items():
         pv = SharedPV(initial=value, handler=MailboxHandler())
         provider.add(f'{prefix}{name}' if prefix else name, pv)
+        print(f'Add mailbox for {prefix}{name}')
         pvs.append(pv)
 
     print(f'Start mailbox server for {len(pvs)} PVs with prefix {prefix}')
@@ -101,20 +102,33 @@ def update():
     from argparse import ArgumentParser
     from p4p.client.thread import Context
     parser = ArgumentParser(description="Update the mailbox server with new values")
-    parser.add_argument('address', type=str, help='The mailbox address of the value to be updated')
-    parser.add_argument('value', type=str, help='The new value to be assigned to the mailbox')
+    parser.add_argument('address value', type=str, nargs='+', help='The mailbox address and value to be updated')
     args = parser.parse_args()
+    addresses_values = getattr(args, 'address value')
+
+    if len(addresses_values) == 0:
+        parser.print_help()
+        return
+
+    addresses = addresses_values[::2]
+    values = addresses_values[1::2]
+
+    if len(addresses_values) % 2:
+        print(f'Please provide address-value pairs. Provided {addresses=} {values=}')
 
     ctx = Context('pva')
-    pv = ctx.get(args.address, throw=False)
-    if isinstance(pv, float):
-        ctx.put(args.address, float(args.value))
-    elif isinstance(pv, int):
-        ctx.put(args.address, int(args.value))
-    elif isinstance(pv, str):
-        ctx.put(args.address, str(args.value))
-    else:
-        raise ValueError(f'Unknown type {type(pv)} (this is likely a vector that I can not handle yet?)')
+    for address, value in zip(addresses, values):
+        pv = ctx.get(address, throw=False)
+        if isinstance(pv, float):
+            ctx.put(address, float(value))
+        elif isinstance(pv, int):
+            ctx.put(address, int(value))
+        elif isinstance(pv, str):
+            ctx.put(address, str(value))
+        elif isinstance(pv, TimeoutError):
+            print(f'[Timeout] Failed to update {address} with {value} (Unknown to EPICS?)')
+        else:
+            raise ValueError(f'Address {address} has unknown type {type(pv)}')
 
     ctx.disconnect()
 

{mccode_plumber-0.7.0 → mccode_plumber-0.8.0}/src/mccode_plumber/file_writer_control/CommandChannel.py

@@ -7,14 +7,14 @@ from typing import Dict, List, Optional, Union
 from kafka import KafkaConsumer
 from kafka.errors import NoBrokersAvailable
 
-from file_writer_control.CommandStatus import CommandState, CommandStatus
-from file_writer_control.InThreadStatusTracker import (
+from .CommandStatus import CommandState, CommandStatus
+from .InThreadStatusTracker import (
     DEAD_ENTITY_TIME_LIMIT,
     InThreadStatusTracker,
 )
-from file_writer_control.JobStatus import JobStatus
-from file_writer_control.KafkaTopicUrl import KafkaTopicUrl
-from file_writer_control.WorkerStatus import WorkerStatus
+from .JobStatus import JobStatus
+from .KafkaTopicUrl import KafkaTopicUrl
+from .WorkerStatus import WorkerStatus
 
 
 def thread_function(

{mccode_plumber-0.7.0 → mccode_plumber-0.8.0}/src/mccode_plumber/file_writer_control/CommandHandler.py

@@ -1,7 +1,7 @@
 from datetime import timedelta
 
-from file_writer_control.CommandChannel import CommandChannel
-from file_writer_control.CommandStatus import CommandState
+from .CommandChannel import CommandChannel
+from .CommandStatus import CommandState
 
 
 class CommandHandler:

{mccode_plumber-0.7.0 → mccode_plumber-0.8.0}/src/mccode_plumber/file_writer_control/InThreadStatusTracker.py

@@ -22,14 +22,14 @@ from streaming_data_types.status_x5f2 import FILE_IDENTIFIER as STAT_IDENTIFIER
 from streaming_data_types.status_x5f2 import StatusMessage
 from streaming_data_types.utils import get_schema
 
-from file_writer_control.CommandStatus import CommandState, CommandStatus
-from file_writer_control.JobStatus import JobState, JobStatus
-from file_writer_control.StateExtractor import (
+from .CommandStatus import CommandState, CommandStatus
+from .JobStatus import JobState, JobStatus
+from .StateExtractor import (
     extract_job_state_from_answer,
     extract_state_from_command_answer,
     extract_worker_state_from_status,
 )
-from file_writer_control.WorkerStatus import WorkerState, WorkerStatus
+from .WorkerStatus import WorkerState, WorkerStatus
 
 DEAD_ENTITY_TIME_LIMIT = timedelta(hours=1)
 
@@ -223,6 +223,6 @@ class InThreadStatusTracker:
             current_job.state = JobState.ERROR
         else:
             current_job.state = JobState.DONE
-            current_job.metadata = json.loads(stopped.metadata)
+            current_job.metadata = json.loads(stopped.metadata) if stopped.metadata is not None else None
         current_job.message = stopped.message
         self.known_workers[stopped.service_id].state = WorkerState.IDLE

{mccode_plumber-0.7.0 → mccode_plumber-0.8.0}/src/mccode_plumber/file_writer_control/JobHandler.py

@@ -1,9 +1,9 @@
 from datetime import datetime
 
-from file_writer_control.CommandHandler import CommandHandler
-from file_writer_control.JobStatus import JobState
-from file_writer_control.WorkerFinder import WorkerFinder
-from file_writer_control.WriteJob import WriteJob
+from .CommandHandler import CommandHandler
+from .JobStatus import JobState
+from .WorkerFinder import WorkerFinder
+from .WriteJob import WriteJob
 
 
 class JobHandler:

{mccode_plumber-0.7.0 → mccode_plumber-0.8.0}/src/mccode_plumber/file_writer_control/StateExtractor.py

@@ -8,9 +8,9 @@ from streaming_data_types.action_response_answ import (
 )
 from streaming_data_types.status_x5f2 import StatusMessage
 
-from file_writer_control.CommandStatus import CommandState
-from file_writer_control.JobStatus import JobState
-from file_writer_control.WorkerStatus import WorkerState
+from .CommandStatus import CommandState
+from .JobStatus import JobState
+from .WorkerStatus import WorkerState
 
 
 def extract_worker_state_from_status(status: StatusMessage) -> WorkerState:

{mccode_plumber-0.7.0 → mccode_plumber-0.8.0}/src/mccode_plumber/file_writer_control/WorkerFinder.py

@@ -6,13 +6,13 @@ from kafka import KafkaProducer
 from kafka.errors import NoBrokersAvailable
 from streaming_data_types.run_stop_6s4t import serialise_6s4t as serialise_stop
 
-from file_writer_control.CommandChannel import CommandChannel
-from file_writer_control.CommandHandler import CommandHandler
-from file_writer_control.CommandStatus import CommandStatus
-from file_writer_control.JobStatus import JobState, JobStatus
-from file_writer_control.KafkaTopicUrl import KafkaTopicUrl
-from file_writer_control.WorkerStatus import WorkerStatus
-from file_writer_control.WriteJob import WriteJob
+from .CommandChannel import CommandChannel
+from .CommandHandler import CommandHandler
+from .CommandStatus import CommandStatus
+from .JobStatus import JobState, JobStatus
+from .KafkaTopicUrl import KafkaTopicUrl
+from .WorkerStatus import WorkerStatus
+from .WriteJob import WriteJob
 
 
 class WorkerFinderBase:

{mccode_plumber-0.7.0 → mccode_plumber-0.8.0}/src/mccode_plumber/file_writer_control/WorkerJobPool.py

@@ -3,11 +3,11 @@ from typing import Dict
 from kafka import KafkaProducer
 from kafka.errors import NoBrokersAvailable
 
-from file_writer_control.CommandHandler import CommandHandler
-from file_writer_control.CommandStatus import CommandState
-from file_writer_control.KafkaTopicUrl import KafkaTopicUrl
-from file_writer_control.WorkerFinder import WorkerFinder
-from file_writer_control.WriteJob import WriteJob
+from .CommandHandler import CommandHandler
+from .CommandStatus import CommandState
+from .KafkaTopicUrl import KafkaTopicUrl
+from .WorkerFinder import WorkerFinder
+from .WriteJob import WriteJob
 
 
 class WorkerJobPool(WorkerFinder):

{mccode_plumber-0.7.0 → mccode_plumber-0.8.0}/src/mccode_plumber/splitrun.py

@@ -4,18 +4,20 @@ def make_parser():
     parser.prog = 'mp-splitrun'
     parser.add_argument('--broker', type=str, help='The Kafka broker to send monitors to', default=None)
     parser.add_argument('--source', type=str, help='The Kafka source name to use for monitors', default=None)
+    parser.add_argument('--topic', type=str, help='The Kafka topic name(s) to use for monitors', default=None, action='append')
     return parser
 
 
-def monitors_to_kafka_callback_with_arguments(broker: str, source: str):
-    from functools import partial
+def monitors_to_kafka_callback_with_arguments(broker: str, source: str, topics: list[str]):
     from mccode_to_kafka.sender import send_histograms
 
+    partial_kwargs = {'broker': broker, 'source': source}
+    if topics is not None and len(topics) > 0:
+        partial_kwargs['names'] = topics
+
     def callback(*args, **kwargs):
-        print(f'monitors to kafka callback called with {args} and {kwargs}')
-        return send_histograms(*args, broker=broker, source=source, **kwargs)
+        return send_histograms(*args, **partial_kwargs, **kwargs)
 
-    # return partial(send_histograms, broker=broker, source=source), {'dir': 'root'}
     return callback, {'dir': 'root'}
 
 
@@ -24,5 +26,5 @@ def main():
     from restage.splitrun import splitrun_args, parse_splitrun
     args, parameters, precision = parse_splitrun(make_parser())
     instr = get_mcstas_instr(args.instrument[0])
-    callback, callback_args = monitors_to_kafka_callback_with_arguments(args.broker, args.source)
+    callback, callback_args = monitors_to_kafka_callback_with_arguments(args.broker, args.source, args.topic)
     return splitrun_args(instr, parameters, precision, args, callback=callback, callback_arguments=callback_args)

{mccode_plumber-0.7.0 → mccode_plumber-0.8.0}/src/mccode_plumber/writer.py

@@ -4,6 +4,8 @@ from pathlib import Path
 from typing import Union, Callable
 from mccode_antlr.instr import Instr
 
+from .file_writer_control import WorkerJobPool
+
 
 def _is_group(x, group):
     """Is a (dict) object a (NeXus) group with the specified name?"""
@@ -334,3 +336,116 @@ def wait_on_writer():
         # raise RuntimeError(e.__str__() + f" The message was: {stop.get_message()}")
         exit(EX_UNAVAILABLE)
     exit(EX_OK)
+
+
+def kill_job():
+    import time
+    from argparse import ArgumentParser
+    from .file_writer_control import WorkerJobPool
+    parser = ArgumentParser()
+    parser.add_argument('-b', '--broker', help="Kafka broker", default='localhost:9092', type=str)
+    parser.add_argument('-c', '--command', help="Writer command topic", default="WriterCommand", type=str)
+    parser.add_argument('-t', '--topic', help='Writer job topic', default='WriterJobs', type=str)
+    parser.add_argument('-s', '--sleep', help='Post pool creation sleep time (s)', default=1, type=int)
+    parser.add_argument('service_id', type=str, help='Writer service id to stop')
+    parser.add_argument('job_id', type=str, help='Writer job id to stop')
+
+    args = parser.parse_args()
+    pool = WorkerJobPool(f'{args.broker}/{args.topic}', f'{args.broker}/{args.command}')
+    time.sleep(args.sleep)
+    pool.try_send_stop_now(args.service_id, args.job_id)
+
+    
+def print_columns(titles: list | tuple, values: list[list | tuple] | tuple[list | tuple, ...]):
+    if not len(values) or not len(titles):
+        return
+    widths = [len(str(x)) for x in titles]
+    for row in values:
+        for i, v in enumerate(row):
+            n = len(str(v))
+            if n > widths[i]:
+                widths[i] = n
+    w_format = ''.join([f'{{:{n + 1:d}s}}' for n in widths])
+    print(w_format.format(*[str(x) for x in titles]))
+    print(w_format.format(*['-' * n for n in widths]))
+    for row in values:
+        print(w_format.format(*[str(x) for x in row]))
+    print()
+
+
+def print_workers(workers):
+    if len(workers):
+        print("Known workers")
+        print_columns(("Service id", "Current state"),
+                      [(w.service_id, w.state) for w in workers])
+    else:
+        print("No workers")
+
+
+def print_jobs(jobs):
+    if len(jobs):
+        print("Known jobs")
+        job_info = [(j.service_id, j.job_id, j.state,
+                     j.file_name if j.file_name else j.message) for j in jobs]
+        print_columns(("Service id", "Job id", "Current state", "File name or message"),
+                      job_info)
+    else:
+        print("No jobs")
+
+
+def print_commands(commands):
+    if len(commands):
+        print("Known commands")
+        print_columns(("Job id", "Command id", "Current state", "Message"),
+                      [(c.job_id, c.command_id, c.state, c.message) for c in
+                       commands])
+    else:
+        print("No commands")
+
+
+def print_current_state(channel: WorkerJobPool):
+    print_workers(channel.list_known_workers())
+    print_jobs(channel.list_known_jobs())
+    print_commands(channel.list_known_commands())
+
+
+def kill_all():
+    import time
+    from argparse import ArgumentParser
+    from .file_writer_control import WorkerJobPool
+    parser = ArgumentParser()
+    parser.add_argument('-b', '--broker', help="Kafka broker", default='localhost:9092', type=str)
+    parser.add_argument('-c', '--command', help="Writer command topic", default="WriterCommand", type=str)
+    parser.add_argument('-t', '--topic', help='Writer job topic', default='WriterJobs', type=str)
+    parser.add_argument('-s', '--sleep', help='Post pool creation sleep time (s)', default=1, type=int)
+    parser.add_argument('-v', '--verbose', help='Verbose output', action='store_true')
+
+    args = parser.parse_args()
+    pool = WorkerJobPool(f'{args.broker}/{args.topic}', f'{args.broker}/{args.command}')
+    time.sleep(args.sleep)
+    if args.verbose:
+        print_current_state(pool)
+    jobs = pool.list_known_jobs()
+    for job in jobs:
+        print(f'Kill {job.service_id} {job.job_id}')
+        pool.try_send_stop_now(job.service_id, job.job_id)
+        time.sleep(args.sleep)
+    if len(jobs) == 0:
+        print("No jobs")
+
+    if args.verbose:
+        print_current_state(pool)
+
+
+def list_status():
+    import time
+    from argparse import ArgumentParser
+    parser = ArgumentParser()
+    parser.add_argument('-b', '--broker', help="Kafka broker", default='localhost:9092', type=str)
+    parser.add_argument('-c', '--command', help="Writer command topic", default="WriterCommand", type=str)
+    parser.add_argument('-t', '--topic', help='Writer job topic', default='WriterJobs', type=str)
+    parser.add_argument('-s', '--sleep', type=int, help='Post pool creation sleep time', default=1)
+    args = parser.parse_args()
+    pool = WorkerJobPool(f'{args.broker}/{args.topic}', f'{args.broker}/{args.command}')
+    time.sleep(args.sleep)
+    print_current_state(pool)

{mccode_plumber-0.7.0 → mccode_plumber-0.8.0/src/mccode_plumber.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mccode-plumber
-Version: 0.7.0
+Version: 0.8.0
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 Classifier: License :: OSI Approved :: BSD License
 Description-Content-Type: text/markdown
@@ -8,7 +8,7 @@ Requires-Dist: p4p
 Requires-Dist: kafka-python>=2.0
 Requires-Dist: ess-streaming-data-types>=0.14.0
 Requires-Dist: restage>=0.4.0
-Requires-Dist: mccode-to-kafka>=0.2.1
+Requires-Dist: mccode-to-kafka>=0.2.2
 Requires-Dist: moreniius>=0.2.3
 Requires-Dist: icecream
 

{mccode_plumber-0.7.0 → mccode_plumber-0.8.0}/src/mccode_plumber.egg-info/entry_points.txt

@@ -7,5 +7,8 @@ mp-insert-hdf5-instr = mccode_plumber.mccode:insert
 mp-register-topics = mccode_plumber.kafka:register_topics
 mp-splitrun = mccode_plumber.splitrun:main
 mp-writer-from = mccode_plumber.writer:print_time
+mp-writer-kill = mccode_plumber.writer:kill_job
+mp-writer-killall = mccode_plumber.writer:kill_all
+mp-writer-list = mccode_plumber.writer:list_status
 mp-writer-wait = mccode_plumber.writer:wait_on_writer
 mp-writer-write = mccode_plumber.writer:start_writer

{mccode_plumber-0.7.0 → mccode_plumber-0.8.0}/src/mccode_plumber.egg-info/requires.txt

@@ -2,6 +2,6 @@ p4p
 kafka-python>=2.0
 ess-streaming-data-types>=0.14.0
 restage>=0.4.0
-mccode-to-kafka>=0.2.1
+mccode-to-kafka>=0.2.2
 moreniius>=0.2.3
 icecream