mccode-plumber 0.16.0__tar.gz → 0.17.1__tar.gz

{mccode_plumber-0.16.0/src/mccode_plumber.egg-info → mccode_plumber-0.17.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mccode-plumber
-Version: 0.16.0
+Version: 0.17.1
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Programming Language :: Python :: 3
@@ -13,8 +13,8 @@ Requires-Dist: p4p
 Requires-Dist: kafka-python>=2.2.11
 Requires-Dist: ess-streaming-data-types>=0.14.0
 Requires-Dist: restage>=0.10.1
-Requires-Dist: mccode-to-kafka>=0.3.1
-Requires-Dist: moreniius>=0.6.3
+Requires-Dist: mccode-to-kafka>=0.5.0
+Requires-Dist: moreniius>=0.7.0
 Requires-Dist: icecream
 Requires-Dist: ephemeral-port-reserve
 Provides-Extra: test

{mccode_plumber-0.16.0 → mccode_plumber-0.17.1}/pyproject.toml

@@ -9,8 +9,8 @@ dependencies = [
     'kafka-python>=2.2.11',
     'ess-streaming-data-types>=0.14.0',
     'restage>=0.10.1',
-    'mccode-to-kafka>=0.3.1',
-    'moreniius>=0.6.3',
+    'mccode-to-kafka>=0.5.0',
+    'moreniius>=0.7.0',
     'icecream',
     'ephemeral-port-reserve',
 ]

{mccode_plumber-0.16.0 → mccode_plumber-0.17.1}/src/mccode_plumber/manage/manager.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from subprocess import Popen, PIPE
 from threading import Thread
 from enum import Enum
@@ -12,6 +12,54 @@ class IOType(Enum):
     stderr = 2
 
 
+@dataclass
+class Triage:
+    level: str = field(default=lambda: 'info')
+    ignore: list[str] = field(default_factory=list)
+    patterns: dict[str, list[str]] = field(default_factory=lambda: {
+        'critical': [r'\bcritical\b', r'^cri'],
+        'error': [r'\berror\b', r'exception', r'traceback', r'^err' ],
+        'warning': [r'\bwarn(ing)?\b', r'deprecated', r'^war'],
+        'notice': [r'\bnotice\b', r'^not'],
+        'info': [r'\binfo\b', r'starting', r'done'],
+        'hint': [r'\bhint\b'],
+        'debug': [r'\bdebug\b', r'^deb', r'^dbg'],
+    })
+    styles: dict[str, str] = field(default_factory=lambda: {
+        'critical': Fore.MAGENTA + Style.BRIGHT,
+        'error': Fore.RED + Style.BRIGHT,
+        'warning': Fore.YELLOW + Style.BRIGHT,
+        'notice': Fore.CYAN + Style.BRIGHT,
+        'info': Fore.GREEN,
+        'hint': Fore.BLUE,
+        'debug': Fore.WHITE + Style.BRIGHT,
+        'default': Fore.RESET,
+    })
+
+    def _filtered_level(self, level: str) -> bool:
+        def _level_value(v: str):
+            for i, lvl in enumerate(self.patterns.keys()):
+                if v == lvl:
+                    return i
+            return -1
+        return _level_value(level) > _level_value(self.level)
+
+    def _style_line(self, level: str, line: str):
+        return self.styles.get(level, '') + line + Style.RESET_ALL
+
+    def __call__(self, line: str) -> tuple[bool, str | None]:
+        import re
+        # If the line contains an ignored keyword, ignore it.
+        if any(kw in line for kw in self.ignore):
+            return True, None
+        # Check if we can identify the status level of this message
+        for level, patterns in self.patterns.items():
+            for pattern in patterns:
+                if re.search(pattern, line, re.IGNORECASE):
+                    return self._filtered_level(level), self._style_line(level, line)
+        return self._filtered_level('default'), self._style_line('default', line)
+
+
 @dataclass
 class Manager:
     """
@@ -19,13 +67,24 @@ class Manager:
 
     Properties
     ----------
+    name: str
+        The name of the process, used as a prefix for all printed status messages
+    style: AnsiStyle
+        Format string to style the printed process name
+    _triage: Triage
+        An object to filter status messages and identify severity levels
+        applying its own message styling based on the identified level
     _process:   a subprocess.Popen instance
+    _stdout_thread: Thread
+    _stderr_thread: Thread
     """
     name: str
     style: AnsiStyle
+    triage: Triage
     _process: Popen | None
     _stdout_thread: Thread | None
     _stderr_thread: Thread | None
+    _name_padding: int
 
     def __run_command__(self) -> list[str]:
         return []
@@ -38,6 +97,18 @@ class Manager:
         from dataclasses import fields
         return [field.name for field in fields(cls)]
 
+    @property
+    def name_padding(self):
+        return self._name_padding
+
+    @name_padding.setter
+    def name_padding(self, value: int):
+        self._name_padding = value
+
+    def _pretty_name(self):
+        padding = ' ' * self.name_padding
+        return f'{self.style}{self.name}:{Style.RESET_ALL}{padding}'
+
     def _read_stream(self, stream, io_type: IOType):
         """Read lines from stream and print them until EOF.
 
@@ -51,8 +122,10 @@ class Manager:
             for line in iter(stream.readline, ''):
                 if not line:
                     break
-                # format and print the line, preserving original behaviour
-                formatted = f'{self.style}{self.name}:{Style.RESET_ALL} {line}'
+                ignored, line = self.triage(line)
+                if ignored:
+                    continue
+                formatted = f'{self._pretty_name()} {line}'
                 if io_type == IOType.stdout:
                     print(formatted, end='')
                 else:
@@ -79,6 +152,10 @@ class Manager:
             kwargs['name'] = 'Managed process'
         if 'style' not in kwargs:
             kwargs['style'] = Fore.WHITE + Back.BLACK
+        if 'triage' not in kwargs:
+            kwargs['triage'] = Triage()
+        if '_name_padding' not in kwargs:
+            kwargs['_name_padding'] = 0
 
         manager = cls(**kwargs)
 

{mccode_plumber-0.16.0 → mccode_plumber-0.17.1}/src/mccode_plumber/manage/orchestrate.py

@@ -261,6 +261,7 @@ def load_in_wait_load_out(
         )
         from mccode_plumber.manage.forwarder import forwarder_verbosity
         from mccode_plumber.manage.writer import writer_verbosity
+        from mccode_plumber.manage.manager import Triage
 
         # Start up services if they should be managed locally
         if manage:
@@ -283,7 +284,10 @@ def load_in_wait_load_out(
                 efu = [EventFormationUnitConfig.from_dict(data)]
             things = tuple(
                 EventFormationUnit.start(
-                    style=Fore.BLUE, broker=broker, **x.to_dict()
+                    style=Fore.BLUE,
+                    broker=broker,
+                    triage=Triage(ignore=["graphite", ":2003 failed"]),
+                    **x.to_dict()
                 ) for x in efu) + (
                 Forwarder.start(
                     name='FWD',
@@ -302,6 +306,7 @@ def load_in_wait_load_out(
                 KafkaToNexus.start(
                     name='K2N',
                     style=Fore.RED + Style.DIM,
+                    triage=Triage(ignore=["ignored by this consumer instance"]),
                     broker=broker,
                     work=work,
                     command=TOPICS['command'],
@@ -309,6 +314,9 @@ def load_in_wait_load_out(
                     verbosity=writer_verbosity(verbosity_writer),
                 ),
             )
+            longest_name = max(len(thing.name) for thing in things)
+            for thing in things:
+                thing.name_padding = longest_name - len(thing.name)
         else:
             things = ()
 

mccode_plumber-0.17.1/src/mccode_plumber/manage/triage_demo.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+"""Small demo utility to print all styles known to the Triage class.
+
+Run as module:
+    python -m mccode_plumber.manage.triage_demo
+
+This prints a sample line for each style key defined in Triage.styles.
+"""
+from __future__ import annotations
+
+from colorama import init
+
+from mccode_plumber.manage.manager import Triage
+
+
+def main() -> None:
+    init(autoreset=True)
+    triage = Triage()
+
+    print("Triage styles (sample lines):")
+    # Print each known style with a short sample message
+    for level in sorted(triage.styles.keys()):
+        sample = f"[{level.upper()}] This is a sample {level} message."
+        # Use internal helper to apply style consistently with Manager output
+        styled = triage._style_line(level, sample)
+        print(styled)
+
+
+if __name__ == "__main__":
+    main()

{mccode_plumber-0.16.0 → mccode_plumber-0.17.1}/src/mccode_plumber/splitrun.py

@@ -17,11 +17,15 @@ def make_parser():
 
 
 def monitors_to_kafka_callback_with_arguments(
-        broker: str, topic: str | None, source: str | None, names: list[str] | None
+        broker: str, topic: str | None, source: str | None, names: list[str] | None,
+        delete_after_sending: bool = True,
 ):
     from mccode_to_kafka.sender import send_histograms
 
-    partial_kwargs: dict[str, Union[str,list[str]]] = {'broker': broker}
+    partial_kwargs: dict[str, Union[str,list[str]]] = {
+        'broker': broker,
+        'remove': delete_after_sending,
+    }
     if topic is not None and source is not None and names is not None and len(names) > 1:
         raise ValueError("Cannot specify both topic/source and multiple names simultaneously.")
 
@@ -41,7 +45,15 @@ def monitors_to_kafka_callback_with_arguments(
 def main():
     from .mccode import get_mcstas_instr
     from restage.splitrun import splitrun_args, parse_splitrun
-    args, parameters, precision = parse_splitrun(make_parser())
+    parser = make_parser()
+    parser.add_argument('--keep-after-send', action='store_true', help='Keep after sending histograms', default=False)
+    args, parameters, precision = parse_splitrun(parser)
     instr = get_mcstas_instr(args.instrument)
-    callback, callback_args = monitors_to_kafka_callback_with_arguments(args.broker, args.topic, args.source, args.names)
+    callback, callback_args = monitors_to_kafka_callback_with_arguments(
+        broker=args.broker,
+        topic=args.topic,
+        source=args.source,
+        names=args.names,
+        delete_after_sending=not args.keep_after_send
+    )
     return splitrun_args(instr, parameters, precision, args, callback=callback, callback_arguments=callback_args)

{mccode_plumber-0.16.0 → mccode_plumber-0.17.1/src/mccode_plumber.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mccode-plumber
-Version: 0.16.0
+Version: 0.17.1
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Programming Language :: Python :: 3
@@ -13,8 +13,8 @@ Requires-Dist: p4p
 Requires-Dist: kafka-python>=2.2.11
 Requires-Dist: ess-streaming-data-types>=0.14.0
 Requires-Dist: restage>=0.10.1
-Requires-Dist: mccode-to-kafka>=0.3.1
-Requires-Dist: moreniius>=0.6.3
+Requires-Dist: mccode-to-kafka>=0.5.0
+Requires-Dist: moreniius>=0.7.0
 Requires-Dist: icecream
 Requires-Dist: ephemeral-port-reserve
 Provides-Extra: test

{mccode_plumber-0.16.0 → mccode_plumber-0.17.1}/src/mccode_plumber.egg-info/SOURCES.txt

@@ -41,6 +41,7 @@ src/mccode_plumber/manage/epics.py
 src/mccode_plumber/manage/forwarder.py
 src/mccode_plumber/manage/manager.py
 src/mccode_plumber/manage/orchestrate.py
+src/mccode_plumber/manage/triage_demo.py
 src/mccode_plumber/manage/writer.py
 tests/fake_efu.py
 tests/fake_manager.py

{mccode_plumber-0.16.0 → mccode_plumber-0.17.1}/src/mccode_plumber.egg-info/requires.txt

@@ -2,8 +2,8 @@ p4p
 kafka-python>=2.2.11
 ess-streaming-data-types>=0.14.0
 restage>=0.10.1
-mccode-to-kafka>=0.3.1
-moreniius>=0.6.3
+mccode-to-kafka>=0.5.0
+moreniius>=0.7.0
 icecream
 ephemeral-port-reserve
 

{mccode_plumber-0.16.0 → mccode_plumber-0.17.1}/tests/test_splitrun.py

@@ -70,6 +70,61 @@ class SplitrunTestCase(unittest.TestCase):
         args = args_fixup(parser.parse_args(['--broker', 'l:9092', '--source', 'm', '-n', '10000', 'inst.h5', '--', 'a=1:4', 'b=2:5', 'c=1,2,3,4,5']))
         self.assertEqual(args.parameters, ['a=1:4', 'b=2:5', 'c=1,2,3,4,5'])
 
+    # New tests for the --keep-after-send flag and how it controls the 'delete' kwarg
+    def test_keep_after_send_defaults_to_false_and_delete_true(self):
+        # make parser match main() which adds this argument
+        parser = make_parser()
+        parser.add_argument('--keep-after-send', action='store_true', help='Keep after sending histograms', default=False)
+        args = args_fixup(parser.parse_args(['--broker', 'l:9092', '--source', 'm', '-n', '10', 'inst.h5', '--', 'a=1:4']))
+        # flag not passed, should be False
+        self.assertFalse(args.keep_after_send)
+
+        from mccode_plumber.splitrun import monitors_to_kafka_callback_with_arguments
+        callback, callback_args = monitors_to_kafka_callback_with_arguments(
+            broker=args.broker, topic=args.topic, source=args.source, names=args.names,
+            delete_after_sending=not args.keep_after_send
+        )
+
+        # inspect closure to find the dict with 'delete'
+        delete_value = None
+        for cell in (callback.__closure__ or ()):  # pragma: no branch - defensive
+            try:
+                val = cell.cell_contents
+            except ValueError:
+                continue
+            if isinstance(val, dict) and 'remove' in val:
+                delete_value = val['remove']
+                break
+
+        self.assertIsNotNone(delete_value)
+        self.assertTrue(delete_value)
+
+    def test_keep_after_send_passed_sets_delete_false(self):
+        parser = make_parser()
+        parser.add_argument('--keep-after-send', action='store_true', help='Keep after sending histograms', default=False)
+        args = args_fixup(parser.parse_args(['--keep-after-send', '--broker', 'l:9092', '--source', 'm', '-n', '10', 'inst.h5', '--', 'a=1:4']))
+        # flag passed, should be True
+        self.assertTrue(args.keep_after_send)
+
+        from mccode_plumber.splitrun import monitors_to_kafka_callback_with_arguments
+        callback, callback_args = monitors_to_kafka_callback_with_arguments(
+            broker=args.broker, topic=args.topic, source=args.source, names=args.names,
+            delete_after_sending=not args.keep_after_send
+        )
+
+        delete_value = None
+        for cell in (callback.__closure__ or ()):  # pragma: no branch - defensive
+            try:
+                val = cell.cell_contents
+            except ValueError:
+                continue
+            if isinstance(val, dict) and 'remove' in val:
+                delete_value = val['remove']
+                break
+
+        self.assertIsNotNone(delete_value)
+        self.assertFalse(delete_value)
+
 
 if __name__ == '__main__':
     unittest.main()