mccode-plumber 0.7.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

mccode_plumber/file_writer_control/CommandChannel.py

@@ -7,14 +7,14 @@ from typing import Dict, List, Optional, Union
 from kafka import KafkaConsumer
 from kafka.errors import NoBrokersAvailable
 
-from file_writer_control.CommandStatus import CommandState, CommandStatus
-from file_writer_control.InThreadStatusTracker import (
+from .CommandStatus import CommandState, CommandStatus
+from .InThreadStatusTracker import (
     DEAD_ENTITY_TIME_LIMIT,
     InThreadStatusTracker,
 )
-from file_writer_control.JobStatus import JobStatus
-from file_writer_control.KafkaTopicUrl import KafkaTopicUrl
-from file_writer_control.WorkerStatus import WorkerStatus
+from .JobStatus import JobStatus
+from .KafkaTopicUrl import KafkaTopicUrl
+from .WorkerStatus import WorkerStatus
 
 
 def thread_function(

mccode_plumber/file_writer_control/CommandHandler.py

@@ -1,7 +1,7 @@
 from datetime import timedelta
 
-from file_writer_control.CommandChannel import CommandChannel
-from file_writer_control.CommandStatus import CommandState
+from .CommandChannel import CommandChannel
+from .CommandStatus import CommandState
 
 
 class CommandHandler:

mccode_plumber/file_writer_control/InThreadStatusTracker.py

@@ -22,14 +22,14 @@ from streaming_data_types.status_x5f2 import FILE_IDENTIFIER as STAT_IDENTIFIER
 from streaming_data_types.status_x5f2 import StatusMessage
 from streaming_data_types.utils import get_schema
 
-from file_writer_control.CommandStatus import CommandState, CommandStatus
-from file_writer_control.JobStatus import JobState, JobStatus
-from file_writer_control.StateExtractor import (
+from .CommandStatus import CommandState, CommandStatus
+from .JobStatus import JobState, JobStatus
+from .StateExtractor import (
     extract_job_state_from_answer,
     extract_state_from_command_answer,
     extract_worker_state_from_status,
 )
-from file_writer_control.WorkerStatus import WorkerState, WorkerStatus
+from .WorkerStatus import WorkerState, WorkerStatus
 
 DEAD_ENTITY_TIME_LIMIT = timedelta(hours=1)
 
@@ -223,6 +223,6 @@ class InThreadStatusTracker:
             current_job.state = JobState.ERROR
         else:
             current_job.state = JobState.DONE
-            current_job.metadata = json.loads(stopped.metadata)
+            current_job.metadata = json.loads(stopped.metadata) if stopped.metadata is not None else None
         current_job.message = stopped.message
         self.known_workers[stopped.service_id].state = WorkerState.IDLE

mccode_plumber/file_writer_control/JobHandler.py

@@ -1,9 +1,9 @@
 from datetime import datetime
 
-from file_writer_control.CommandHandler import CommandHandler
-from file_writer_control.JobStatus import JobState
-from file_writer_control.WorkerFinder import WorkerFinder
-from file_writer_control.WriteJob import WriteJob
+from .CommandHandler import CommandHandler
+from .JobStatus import JobState
+from .WorkerFinder import WorkerFinder
+from .WriteJob import WriteJob
 
 
 class JobHandler:

mccode_plumber/file_writer_control/StateExtractor.py

@@ -8,9 +8,9 @@ from streaming_data_types.action_response_answ import (
 )
 from streaming_data_types.status_x5f2 import StatusMessage
 
-from file_writer_control.CommandStatus import CommandState
-from file_writer_control.JobStatus import JobState
-from file_writer_control.WorkerStatus import WorkerState
+from .CommandStatus import CommandState
+from .JobStatus import JobState
+from .WorkerStatus import WorkerState
 
 
 def extract_worker_state_from_status(status: StatusMessage) -> WorkerState:

mccode_plumber/file_writer_control/WorkerFinder.py

@@ -6,13 +6,13 @@ from kafka import KafkaProducer
 from kafka.errors import NoBrokersAvailable
 from streaming_data_types.run_stop_6s4t import serialise_6s4t as serialise_stop
 
-from file_writer_control.CommandChannel import CommandChannel
-from file_writer_control.CommandHandler import CommandHandler
-from file_writer_control.CommandStatus import CommandStatus
-from file_writer_control.JobStatus import JobState, JobStatus
-from file_writer_control.KafkaTopicUrl import KafkaTopicUrl
-from file_writer_control.WorkerStatus import WorkerStatus
-from file_writer_control.WriteJob import WriteJob
+from .CommandChannel import CommandChannel
+from .CommandHandler import CommandHandler
+from .CommandStatus import CommandStatus
+from .JobStatus import JobState, JobStatus
+from .KafkaTopicUrl import KafkaTopicUrl
+from .WorkerStatus import WorkerStatus
+from .WriteJob import WriteJob
 
 
 class WorkerFinderBase:

mccode_plumber/file_writer_control/WorkerJobPool.py

@@ -3,11 +3,11 @@ from typing import Dict
 from kafka import KafkaProducer
 from kafka.errors import NoBrokersAvailable
 
-from file_writer_control.CommandHandler import CommandHandler
-from file_writer_control.CommandStatus import CommandState
-from file_writer_control.KafkaTopicUrl import KafkaTopicUrl
-from file_writer_control.WorkerFinder import WorkerFinder
-from file_writer_control.WriteJob import WriteJob
+from .CommandHandler import CommandHandler
+from .CommandStatus import CommandState
+from .KafkaTopicUrl import KafkaTopicUrl
+from .WorkerFinder import WorkerFinder
+from .WriteJob import WriteJob
 
 
 class WorkerJobPool(WorkerFinder):

mccode_plumber/splitrun.py

@@ -4,18 +4,20 @@ def make_parser():
     parser.prog = 'mp-splitrun'
     parser.add_argument('--broker', type=str, help='The Kafka broker to send monitors to', default=None)
     parser.add_argument('--source', type=str, help='The Kafka source name to use for monitors', default=None)
+    parser.add_argument('--topic', type=str, help='The Kafka topic name(s) to use for monitors', default=None, action='append')
     return parser
 
 
-def monitors_to_kafka_callback_with_arguments(broker: str, source: str):
-    from functools import partial
+def monitors_to_kafka_callback_with_arguments(broker: str, source: str, topics: list[str]):
     from mccode_to_kafka.sender import send_histograms
 
+    partial_kwargs = {'broker': broker, 'source': source}
+    if topics is not None and len(topics) > 0:
+        partial_kwargs['names'] = topics
+
     def callback(*args, **kwargs):
-        print(f'monitors to kafka callback called with {args} and {kwargs}')
-        return send_histograms(*args, broker=broker, source=source, **kwargs)
+        return send_histograms(*args, **partial_kwargs, **kwargs)
 
-    # return partial(send_histograms, broker=broker, source=source), {'dir': 'root'}
     return callback, {'dir': 'root'}
 
 
@@ -24,5 +26,5 @@ def main():
     from restage.splitrun import splitrun_args, parse_splitrun
     args, parameters, precision = parse_splitrun(make_parser())
     instr = get_mcstas_instr(args.instrument[0])
-    callback, callback_args = monitors_to_kafka_callback_with_arguments(args.broker, args.source)
+    callback, callback_args = monitors_to_kafka_callback_with_arguments(args.broker, args.source, args.topic)
     return splitrun_args(instr, parameters, precision, args, callback=callback, callback_arguments=callback_args)

mccode_plumber/writer.py

@@ -4,6 +4,8 @@ from pathlib import Path
 from typing import Union, Callable
 from mccode_antlr.instr import Instr
 
+from .file_writer_control import WorkerJobPool
+
 
 def _is_group(x, group):
     """Is a (dict) object a (NeXus) group with the specified name?"""
@@ -334,3 +336,116 @@ def wait_on_writer():
         # raise RuntimeError(e.__str__() + f" The message was: {stop.get_message()}")
         exit(EX_UNAVAILABLE)
     exit(EX_OK)
+
+
+def kill_job():
+    import time
+    from argparse import ArgumentParser
+    from .file_writer_control import WorkerJobPool
+    parser = ArgumentParser()
+    parser.add_argument('-b', '--broker', help="Kafka broker", default='localhost:9092', type=str)
+    parser.add_argument('-c', '--command', help="Writer command topic", default="WriterCommand", type=str)
+    parser.add_argument('-t', '--topic', help='Writer job topic', default='WriterJobs', type=str)
+    parser.add_argument('-s', '--sleep', help='Post pool creation sleep time (s)', default=1, type=int)
+    parser.add_argument('service_id', type=str, help='Writer service id to stop')
+    parser.add_argument('job_id', type=str, help='Writer job id to stop')
+
+    args = parser.parse_args()
+    pool = WorkerJobPool(f'{args.broker}/{args.topic}', f'{args.broker}/{args.command}')
+    time.sleep(args.sleep)
+    pool.try_send_stop_now(args.service_id, args.job_id)
+
+    
+def print_columns(titles: list | tuple, values: list[list | tuple] | tuple[list | tuple, ...]):
+    if not len(values) or not len(titles):
+        return
+    widths = [len(str(x)) for x in titles]
+    for row in values:
+        for i, v in enumerate(row):
+            n = len(str(v))
+            if n > widths[i]:
+                widths[i] = n
+    w_format = ''.join([f'{{:{n + 1:d}s}}' for n in widths])
+    print(w_format.format(*[str(x) for x in titles]))
+    print(w_format.format(*['-' * n for n in widths]))
+    for row in values:
+        print(w_format.format(*[str(x) for x in row]))
+    print()
+
+
+def print_workers(workers):
+    if len(workers):
+        print("Known workers")
+        print_columns(("Service id", "Current state"),
+                      [(w.service_id, w.state) for w in workers])
+    else:
+        print("No workers")
+
+
+def print_jobs(jobs):
+    if len(jobs):
+        print("Known jobs")
+        job_info = [(j.service_id, j.job_id, j.state,
+                     j.file_name if j.file_name else j.message) for j in jobs]
+        print_columns(("Service id", "Job id", "Current state", "File name or message"),
+                      job_info)
+    else:
+        print("No jobs")
+
+
+def print_commands(commands):
+    if len(commands):
+        print("Known commands")
+        print_columns(("Job id", "Command id", "Current state", "Message"),
+                      [(c.job_id, c.command_id, c.state, c.message) for c in
+                       commands])
+    else:
+        print("No commands")
+
+
+def print_current_state(channel: WorkerJobPool):
+    print_workers(channel.list_known_workers())
+    print_jobs(channel.list_known_jobs())
+    print_commands(channel.list_known_commands())
+
+
+def kill_all():
+    import time
+    from argparse import ArgumentParser
+    from .file_writer_control import WorkerJobPool
+    parser = ArgumentParser()
+    parser.add_argument('-b', '--broker', help="Kafka broker", default='localhost:9092', type=str)
+    parser.add_argument('-c', '--command', help="Writer command topic", default="WriterCommand", type=str)
+    parser.add_argument('-t', '--topic', help='Writer job topic', default='WriterJobs', type=str)
+    parser.add_argument('-s', '--sleep', help='Post pool creation sleep time (s)', default=1, type=int)
+    parser.add_argument('-v', '--verbose', help='Verbose output', action='store_true')
+
+    args = parser.parse_args()
+    pool = WorkerJobPool(f'{args.broker}/{args.topic}', f'{args.broker}/{args.command}')
+    time.sleep(args.sleep)
+    if args.verbose:
+        print_current_state(pool)
+    jobs = pool.list_known_jobs()
+    for job in jobs:
+        print(f'Kill {job.service_id} {job.job_id}')
+        pool.try_send_stop_now(job.service_id, job.job_id)
+        time.sleep(args.sleep)
+    if len(jobs) == 0:
+        print("No jobs")
+
+    if args.verbose:
+        print_current_state(pool)
+
+
+def list_status():
+    import time
+    from argparse import ArgumentParser
+    parser = ArgumentParser()
+    parser.add_argument('-b', '--broker', help="Kafka broker", default='localhost:9092', type=str)
+    parser.add_argument('-c', '--command', help="Writer command topic", default="WriterCommand", type=str)
+    parser.add_argument('-t', '--topic', help='Writer job topic', default='WriterJobs', type=str)
+    parser.add_argument('-s', '--sleep', type=int, help='Post pool creation sleep time', default=1)
+    args = parser.parse_args()
+    pool = WorkerJobPool(f'{args.broker}/{args.topic}', f'{args.broker}/{args.command}')
+    time.sleep(args.sleep)
+    print_current_state(pool)

{mccode_plumber-0.7.1.dist-info → mccode_plumber-0.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mccode-plumber
-Version: 0.7.1
+Version: 0.8.0
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 Classifier: License :: OSI Approved :: BSD License
 Description-Content-Type: text/markdown
@@ -8,7 +8,7 @@ Requires-Dist: p4p
 Requires-Dist: kafka-python>=2.0
 Requires-Dist: ess-streaming-data-types>=0.14.0
 Requires-Dist: restage>=0.4.0
-Requires-Dist: mccode-to-kafka>=0.2.1
+Requires-Dist: mccode-to-kafka>=0.2.2
 Requires-Dist: moreniius>=0.2.3
 Requires-Dist: icecream
 

{mccode_plumber-0.7.1.dist-info → mccode_plumber-0.8.0.dist-info}/RECORD

@@ -4,24 +4,24 @@ mccode_plumber/epics.py,sha256=5j_ictmDloxUGbGNe2ESlkSDQsPknzgPAbUsoZEUWO0,4441
 mccode_plumber/forwarder.py,sha256=1PdW91qLksDnA41-gOFMtgSae4wDdSLVvgAlrF23Uks,3764
 mccode_plumber/kafka.py,sha256=8fHTy2Zzk638XXj29pfrk-7fUxafuse9OYWmaRgvV7M,1195
 mccode_plumber/mccode.py,sha256=WXlu89Cs6a3s0aKtfRGuzqcOWoFZtdZWibLgBQnaXCI,2210
-mccode_plumber/splitrun.py,sha256=MsR1cUvoayQGNCJNl5fIPG92Wfcy9V_8QNO_ITQTvtA,1295
+mccode_plumber/splitrun.py,sha256=s1ShM-Y5d9DzndVIYuWXqPkYPmxzG9Cjjcxo7Pss6Q0,1393
 mccode_plumber/utils.py,sha256=E8NoGo_3Z-pPEpzicVGSWfQOX8p3eR-GxElT33-kX5U,2167
-mccode_plumber/writer.py,sha256=TSWw7Q1mlXcwOx4OuvNGN-yyB_X10Vmt1KywA79RmBs,16359
-mccode_plumber/file_writer_control/CommandChannel.py,sha256=rrQxmQ6VJsl9OC5n45XkL3g7MXtICT_8tsv_xuGdIKI,9119
-mccode_plumber/file_writer_control/CommandHandler.py,sha256=ETPGNiD3umt2XazCWztoaO6qgtilBzrB1N1EH5EAKUw,2245
+mccode_plumber/writer.py,sha256=XUTrq1u1aQYX6jTDY5Rie3dQAuN8iE4nxxdbFmLRhQk,20886
+mccode_plumber/file_writer_control/CommandChannel.py,sha256=U32mvk8Ctlw8vg38_ULP__-UAG2funWIkhEFo2xahiY,9024
+mccode_plumber/file_writer_control/CommandHandler.py,sha256=4dPsMyBYx5hKuiVw5HoPMOFJ0QFxmrI3XK3TA_a8ZGE,2207
 mccode_plumber/file_writer_control/CommandStatus.py,sha256=biNK58vY1H-USYAl9uB_JNQyMohBj1cwJwmXktndd4E,4517
-mccode_plumber/file_writer_control/InThreadStatusTracker.py,sha256=sLb8v1JBs3QSWXu-_kIznku3h_ZqVNwx0o9-RoxaKu4,10196
-mccode_plumber/file_writer_control/JobHandler.py,sha256=H6fr0O6lcGw4EKRUuR9Ak4gk5Rknozru0PGaNxNITeg,4296
+mccode_plumber/file_writer_control/InThreadStatusTracker.py,sha256=CT20ch2Kv3VPvrM6reNMKfdH3Lbi4y6J_JXV10RALXM,10162
+mccode_plumber/file_writer_control/JobHandler.py,sha256=KKBTaXWSzZpfuj2ak1kboRkXKmuSuOLmT3KHwUqNHLc,4220
 mccode_plumber/file_writer_control/JobStatus.py,sha256=ncLYEcTAOdrIASQvY0jJAX3fC4qF9tyzLavm5YUOhlQ,4416
 mccode_plumber/file_writer_control/KafkaTopicUrl.py,sha256=dG6Aj2tYo1UGZferxyvURUAr77z9dhBN_SRzizzt1Vo,736
-mccode_plumber/file_writer_control/StateExtractor.py,sha256=FaroeEuZCGTIi7kEIxeS7QHjnD7CxU1ONvKRQqRCmrc,2068
-mccode_plumber/file_writer_control/WorkerFinder.py,sha256=vMv-_YQSdchGDo5TAHQfJODWsatjCyCbLiCwtJJGZi4,6093
-mccode_plumber/file_writer_control/WorkerJobPool.py,sha256=2P6qYNgRtcolSO910a2BejoZyJf4H2ZiSfsVgYzwq8g,2966
+mccode_plumber/file_writer_control/StateExtractor.py,sha256=DmHYtfzUuEX4kkqOKSw2Q2CWagvS2XJX5R3I6fBBglU,2011
+mccode_plumber/file_writer_control/WorkerFinder.py,sha256=nhItfO9_o8UAlvNyCSV6OvbcftHRKGpe48U2PasWsQ0,5960
+mccode_plumber/file_writer_control/WorkerJobPool.py,sha256=4VKzAhreGzunYT52te5kG1Hiql-xgVwLvm2hFO4U2FI,2871
 mccode_plumber/file_writer_control/WorkerStatus.py,sha256=uw1q-Pvf1o2hxpMIPVwdtnAXLcO7VeEpcZ3wKjxxdsk,2826
 mccode_plumber/file_writer_control/WriteJob.py,sha256=-2tQvfajctf6Bn19c9hT9N1lOP0uTo7SZRMnXV1A-aA,2721
 mccode_plumber/file_writer_control/__init__.py,sha256=Wp8A7JOB0vgpwAbvI9ik5cBlZa6lY-cq8dxYGRTRs0M,285
-mccode_plumber-0.7.1.dist-info/METADATA,sha256=MJPyAqAG4OUUFWy5hs-s9fZfzo9z7qQny3cUsNcUMss,552
-mccode_plumber-0.7.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-mccode_plumber-0.7.1.dist-info/entry_points.txt,sha256=V-ULMAQo7W1O3CLd39iSJcRWNjx9p4OuzEC_8y3T-Zs,520
-mccode_plumber-0.7.1.dist-info/top_level.txt,sha256=kCCIpYtKHCKWxiPEqX9J1UaGEm-ze0Qb-cemBCEPhDA,15
-mccode_plumber-0.7.1.dist-info/RECORD,,
+mccode_plumber-0.8.0.dist-info/METADATA,sha256=STOHlVURtT4_6ixD8jijei4Me30rIB3v3YsvuJCxQSg,552
+mccode_plumber-0.8.0.dist-info/WHEEL,sha256=wXxTzcEDnjrTwFYjLPcsW_7_XihufBwmpiBeiXNBGEA,91
+mccode_plumber-0.8.0.dist-info/entry_points.txt,sha256=VwqJ6B-ppH6Fezql9ZDKNcjw6QG2aQ8qvDxdqVGM-jk,670
+mccode_plumber-0.8.0.dist-info/top_level.txt,sha256=kCCIpYtKHCKWxiPEqX9J1UaGEm-ze0Qb-cemBCEPhDA,15
+mccode_plumber-0.8.0.dist-info/RECORD,,

{mccode_plumber-0.7.1.dist-info → mccode_plumber-0.8.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (80.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{mccode_plumber-0.7.1.dist-info → mccode_plumber-0.8.0.dist-info}/entry_points.txt

@@ -7,5 +7,8 @@ mp-insert-hdf5-instr = mccode_plumber.mccode:insert
 mp-register-topics = mccode_plumber.kafka:register_topics
 mp-splitrun = mccode_plumber.splitrun:main
 mp-writer-from = mccode_plumber.writer:print_time
+mp-writer-kill = mccode_plumber.writer:kill_job
+mp-writer-killall = mccode_plumber.writer:kill_all
+mp-writer-list = mccode_plumber.writer:list_status
 mp-writer-wait = mccode_plumber.writer:wait_on_writer
 mp-writer-write = mccode_plumber.writer:start_writer