mccode-plumber 0.15.1__tar.gz → 0.17.0__tar.gz

{mccode_plumber-0.15.1/src/mccode_plumber.egg-info → mccode_plumber-0.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mccode-plumber
-Version: 0.15.1
+Version: 0.17.0
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Programming Language :: Python :: 3
@@ -13,8 +13,8 @@ Requires-Dist: p4p
 Requires-Dist: kafka-python>=2.2.11
 Requires-Dist: ess-streaming-data-types>=0.14.0
 Requires-Dist: restage>=0.10.1
-Requires-Dist: mccode-to-kafka>=0.3.1
-Requires-Dist: moreniius>=0.6.3
+Requires-Dist: mccode-to-kafka>=0.5.0
+Requires-Dist: moreniius>=0.7.0
 Requires-Dist: icecream
 Requires-Dist: ephemeral-port-reserve
 Provides-Extra: test

{mccode_plumber-0.15.1 → mccode_plumber-0.17.0}/pyproject.toml

@@ -9,8 +9,8 @@ dependencies = [
     'kafka-python>=2.2.11',
     'ess-streaming-data-types>=0.14.0',
     'restage>=0.10.1',
-    'mccode-to-kafka>=0.3.1',
-    'moreniius>=0.6.3',
+    'mccode-to-kafka>=0.5.0',
+    'moreniius>=0.7.0',
     'icecream',
     'ephemeral-port-reserve',
 ]

{mccode_plumber-0.15.1 → mccode_plumber-0.17.0}/src/mccode_plumber/forwarder.py

@@ -7,6 +7,7 @@ list of EPICS PVs to monitor.
 Alternatively, the same functionality can be accessed from Python using the configure_forwarder and reset_forwarder
 functions. Which take PV information and Forwarder/Kafka configuration as arguments.
 """
+from mccode_antlr.common import InstrumentParameter
 
 
 def normalise_pvs(pvs: list[dict], config=None, prefix=None, topic=None):
@@ -59,10 +60,19 @@ def reset_forwarder(pvs: list[dict], config=None, prefix=None, topic=None):
     return pvs
 
 
-def forwarder_partial_streams(prefix, topic, parameters):
-    names = [p.name for p in parameters]
-    if 'mcpl_filename' not in names:
-        names.append("mcpl_filename")
+def forwarder_partial_streams(prefix: str, topic: str, parameters: list[InstrumentParameter]):
+    from mccode_antlr.common import DataType
+    # The streaming-data-type f144 only supports numeric data, so we need to
+    # filter out string-valued data types to avoid annoying error messages in the
+    # forwarder's log output.
+    names = [p.name for p in parameters if p.value.data_type is not DataType.str]
+
+    # splitrun adds an instrument parameter named 'mcpl_filename', but we also
+    # can not forward this since it is a string-valued parameter.
+    #
+    #if 'mcpl_filename' not in names:
+    #    names.append("mcpl_filename")
+
     # Minimal information used by the forwarder for stream setup:
     partial = [dict(source=f'{prefix}{n}', module='f144', topic=topic) for n in names]
     return partial

{mccode_plumber-0.15.1 → mccode_plumber-0.17.0}/src/mccode_plumber/manage/manager.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from subprocess import Popen, PIPE
 from threading import Thread
 from enum import Enum
@@ -12,6 +12,54 @@ class IOType(Enum):
     stderr = 2
 
 
+@dataclass
+class Triage:
+    level: str = field(default=lambda: 'info')
+    ignore: list[str] = field(default_factory=list)
+    patterns: dict[str, list[str]] = field(default_factory=lambda: {
+        'critical': [r'\bcritical\b', r'^cri'],
+        'error': [r'\berror\b', r'exception', r'traceback', r'^err' ],
+        'warning': [r'\bwarn(ing)?\b', r'deprecated', r'^war'],
+        'notice': [r'\bnotice\b', r'^not'],
+        'info': [r'\binfo\b', r'starting', r'done'],
+        'hint': [r'\bhint\b'],
+        'debug': [r'\bdebug\b', r'^deb', r'^dbg'],
+    })
+    styles: dict[str, str] = field(default_factory=lambda: {
+        'critical': Fore.MAGENTA + Style.BRIGHT,
+        'error': Fore.RED + Style.BRIGHT,
+        'warning': Fore.YELLOW + Style.BRIGHT,
+        'notice': Fore.CYAN + Style.BRIGHT,
+        'info': Fore.GREEN,
+        'hint': Fore.BLUE,
+        'debug': Fore.WHITE + Style.BRIGHT,
+        'default': Fore.WHITE,
+    })
+
+    def _filtered_level(self, level: str) -> bool:
+        def _level_value(v: str):
+            for i, lvl in enumerate(self.patterns.keys()):
+                if v == lvl:
+                    return i
+            return -1
+        return _level_value(level) > _level_value(self.level)
+
+    def _style_line(self, level: str, line: str):
+        return self.styles.get(level, '') + line + Style.RESET_ALL
+
+    def __call__(self, line: str) -> tuple[bool, str | None]:
+        import re
+        # If the line contains an ignored keyword, ignore it.
+        if any(kw in line for kw in self.ignore):
+            return True, None
+        # Check if we can identify the status level of this message
+        for level, patterns in self.patterns.items():
+            for pattern in patterns:
+                if re.search(pattern, line, re.IGNORECASE):
+                    return self._filtered_level(level), self._style_line(level, line)
+        return self._filtered_level('default'), self._style_line('default', line)
+
+
 @dataclass
 class Manager:
     """
@@ -19,13 +67,24 @@ class Manager:
 
     Properties
     ----------
+    name: str
+        The name of the process, used as a prefix for all printed status messages
+    style: AnsiStyle
+        Format string to style the printed process name
+    _triage: Triage
+        An object to filter status messages and identify severity levels
+        applying its own message styling based on the identified level
     _process:   a subprocess.Popen instance
+    _stdout_thread: Thread
+    _stderr_thread: Thread
     """
     name: str
     style: AnsiStyle
+    triage: Triage
     _process: Popen | None
     _stdout_thread: Thread | None
     _stderr_thread: Thread | None
+    _name_padding: int
 
     def __run_command__(self) -> list[str]:
         return []
@@ -38,6 +97,18 @@ class Manager:
         from dataclasses import fields
         return [field.name for field in fields(cls)]
 
+    @property
+    def name_padding(self):
+        return self._name_padding
+
+    @name_padding.setter
+    def name_padding(self, value: int):
+        self._name_padding = value
+
+    def _pretty_name(self):
+        padding = ' ' * self.name_padding
+        return f'{self.style}{self.name}:{Style.RESET_ALL}{padding}'
+
     def _read_stream(self, stream, io_type: IOType):
         """Read lines from stream and print them until EOF.
 
@@ -51,8 +122,10 @@ class Manager:
             for line in iter(stream.readline, ''):
                 if not line:
                     break
-                # format and print the line, preserving original behaviour
-                formatted = f'{self.style}{self.name}:{Style.RESET_ALL} {line}'
+                ignored, line = self.triage(line)
+                if ignored:
+                    continue
+                formatted = f'{self._pretty_name()} {line}'
                 if io_type == IOType.stdout:
                     print(formatted, end='')
                 else:
@@ -79,6 +152,10 @@ class Manager:
             kwargs['name'] = 'Managed process'
         if 'style' not in kwargs:
             kwargs['style'] = Fore.WHITE + Back.BLACK
+        if 'triage' not in kwargs:
+            kwargs['triage'] = Triage()
+        if '_name_padding' not in kwargs:
+            kwargs['_name_padding'] = 0
 
         manager = cls(**kwargs)
 

{mccode_plumber-0.15.1 → mccode_plumber-0.17.0}/src/mccode_plumber/manage/orchestrate.py

@@ -261,6 +261,7 @@ def load_in_wait_load_out(
         )
         from mccode_plumber.manage.forwarder import forwarder_verbosity
         from mccode_plumber.manage.writer import writer_verbosity
+        from mccode_plumber.manage.manager import Triage
 
         # Start up services if they should be managed locally
         if manage:
@@ -283,7 +284,10 @@ def load_in_wait_load_out(
                 efu = [EventFormationUnitConfig.from_dict(data)]
             things = tuple(
                 EventFormationUnit.start(
-                    style=Fore.BLUE, broker=broker, **x.to_dict()
+                    style=Fore.BLUE,
+                    broker=broker,
+                    triage=Triage(ignore=["graphite", ":2003 failed"]),
+                    **x.to_dict()
                 ) for x in efu) + (
                 Forwarder.start(
                     name='FWD',
@@ -309,6 +313,9 @@ def load_in_wait_load_out(
                     verbosity=writer_verbosity(verbosity_writer),
                 ),
             )
+            longest_name = max(len(thing.name) for thing in things)
+            for thing in things:
+                thing.name_padding = longest_name - len(thing.name)
         else:
             things = ()
 

{mccode_plumber-0.15.1 → mccode_plumber-0.17.0}/src/mccode_plumber/splitrun.py

@@ -17,11 +17,15 @@ def make_parser():
 
 
 def monitors_to_kafka_callback_with_arguments(
-        broker: str, topic: str | None, source: str | None, names: list[str] | None
+        broker: str, topic: str | None, source: str | None, names: list[str] | None,
+        delete_after_sending: bool = True,
 ):
     from mccode_to_kafka.sender import send_histograms
 
-    partial_kwargs: dict[str, Union[str,list[str]]] = {'broker': broker}
+    partial_kwargs: dict[str, Union[str,list[str]]] = {
+        'broker': broker,
+        'delete': delete_after_sending,
+    }
     if topic is not None and source is not None and names is not None and len(names) > 1:
         raise ValueError("Cannot specify both topic/source and multiple names simultaneously.")
 
@@ -41,7 +45,15 @@ def monitors_to_kafka_callback_with_arguments(
 def main():
     from .mccode import get_mcstas_instr
     from restage.splitrun import splitrun_args, parse_splitrun
-    args, parameters, precision = parse_splitrun(make_parser())
+    parser = make_parser()
+    parser.add_argument('--keep-after-send', action='store_true', help='Keep after sending histograms', default=False)
+    args, parameters, precision = parse_splitrun(parser)
     instr = get_mcstas_instr(args.instrument)
-    callback, callback_args = monitors_to_kafka_callback_with_arguments(args.broker, args.topic, args.source, args.names)
+    callback, callback_args = monitors_to_kafka_callback_with_arguments(
+        broker=args.broker,
+        topic=args.topic,
+        source=args.source,
+        names=args.names,
+        delete_after_sending=not args.keep_after_send
+    )
     return splitrun_args(instr, parameters, precision, args, callback=callback, callback_arguments=callback_args)

{mccode_plumber-0.15.1 → mccode_plumber-0.17.0}/src/mccode_plumber/writer.py

@@ -311,8 +311,16 @@ def construct_writer_pv_dicts(instr: Path | str, prefix: str, topic: str):
 
 
 def construct_writer_pv_dicts_from_parameters(parameters, prefix: str, topic: str):
+    from mccode_antlr.common import DataType
+
     def strip_quotes(s):
         return s[1:-1] if s is not None and len(s) > 2 and (s[0] == s[-1] == '"' or s[0] == s[-1] == "'") else s
+
+    # Remove string-valued parameters from the provided list since they are not
+    # supported by streaming-data-type f144. In the future we could specify a different
+    # module for them instead.
+    parameters = [p for p in parameters if p.value.data_type is not DataType.str]
+
     return [dict(name=p.name, dtype=p.value.data_type.name, source=f'{prefix}{p.name}', topic=topic,
                  description=parameter_description(p), module='f144', unit=strip_quotes(p.unit)) for p in parameters]
 

{mccode_plumber-0.15.1 → mccode_plumber-0.17.0/src/mccode_plumber.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mccode-plumber
-Version: 0.15.1
+Version: 0.17.0
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Programming Language :: Python :: 3
@@ -13,8 +13,8 @@ Requires-Dist: p4p
 Requires-Dist: kafka-python>=2.2.11
 Requires-Dist: ess-streaming-data-types>=0.14.0
 Requires-Dist: restage>=0.10.1
-Requires-Dist: mccode-to-kafka>=0.3.1
-Requires-Dist: moreniius>=0.6.3
+Requires-Dist: mccode-to-kafka>=0.5.0
+Requires-Dist: moreniius>=0.7.0
 Requires-Dist: icecream
 Requires-Dist: ephemeral-port-reserve
 Provides-Extra: test

{mccode_plumber-0.15.1 → mccode_plumber-0.17.0}/src/mccode_plumber.egg-info/requires.txt

@@ -2,8 +2,8 @@ p4p
 kafka-python>=2.2.11
 ess-streaming-data-types>=0.14.0
 restage>=0.10.1
-mccode-to-kafka>=0.3.1
-moreniius>=0.6.3
+mccode-to-kafka>=0.5.0
+moreniius>=0.7.0
 icecream
 ephemeral-port-reserve
 

{mccode_plumber-0.15.1 → mccode_plumber-0.17.0}/tests/test_splitrun.py

@@ -70,6 +70,61 @@ class SplitrunTestCase(unittest.TestCase):
         args = args_fixup(parser.parse_args(['--broker', 'l:9092', '--source', 'm', '-n', '10000', 'inst.h5', '--', 'a=1:4', 'b=2:5', 'c=1,2,3,4,5']))
         self.assertEqual(args.parameters, ['a=1:4', 'b=2:5', 'c=1,2,3,4,5'])
 
+    # New tests for the --keep-after-send flag and how it controls the 'delete' kwarg
+    def test_keep_after_send_defaults_to_false_and_delete_true(self):
+        # make parser match main() which adds this argument
+        parser = make_parser()
+        parser.add_argument('--keep-after-send', action='store_true', help='Keep after sending histograms', default=False)
+        args = args_fixup(parser.parse_args(['--broker', 'l:9092', '--source', 'm', '-n', '10', 'inst.h5', '--', 'a=1:4']))
+        # flag not passed, should be False
+        self.assertFalse(args.keep_after_send)
+
+        from mccode_plumber.splitrun import monitors_to_kafka_callback_with_arguments
+        callback, callback_args = monitors_to_kafka_callback_with_arguments(
+            broker=args.broker, topic=args.topic, source=args.source, names=args.names,
+            delete_after_sending=not args.keep_after_send
+        )
+
+        # inspect closure to find the dict with 'delete'
+        delete_value = None
+        for cell in (callback.__closure__ or ()):  # pragma: no branch - defensive
+            try:
+                val = cell.cell_contents
+            except ValueError:
+                continue
+            if isinstance(val, dict) and 'delete' in val:
+                delete_value = val['delete']
+                break
+
+        self.assertIsNotNone(delete_value)
+        self.assertTrue(delete_value)
+
+    def test_keep_after_send_passed_sets_delete_false(self):
+        parser = make_parser()
+        parser.add_argument('--keep-after-send', action='store_true', help='Keep after sending histograms', default=False)
+        args = args_fixup(parser.parse_args(['--keep-after-send', '--broker', 'l:9092', '--source', 'm', '-n', '10', 'inst.h5', '--', 'a=1:4']))
+        # flag passed, should be True
+        self.assertTrue(args.keep_after_send)
+
+        from mccode_plumber.splitrun import monitors_to_kafka_callback_with_arguments
+        callback, callback_args = monitors_to_kafka_callback_with_arguments(
+            broker=args.broker, topic=args.topic, source=args.source, names=args.names,
+            delete_after_sending=not args.keep_after_send
+        )
+
+        delete_value = None
+        for cell in (callback.__closure__ or ()):  # pragma: no branch - defensive
+            try:
+                val = cell.cell_contents
+            except ValueError:
+                continue
+            if isinstance(val, dict) and 'delete' in val:
+                delete_value = val['delete']
+                break
+
+        self.assertIsNotNone(delete_value)
+        self.assertFalse(delete_value)
+
 
 if __name__ == '__main__':
     unittest.main()

{mccode_plumber-0.15.1 → mccode_plumber-0.17.0}/tests/test_writer.py

@@ -61,10 +61,13 @@ class WriterUnitsTestCase(unittest.TestCase):
     def test_parse(self):
         from mccode_plumber.writer import construct_writer_pv_dicts_from_parameters
         params = construct_writer_pv_dicts_from_parameters(self.instr.parameters, 'mcstas:', 'topic')
-        self.assertEqual(len(params), 4)
-        for p, x in zip(params, [('a', 'Hz'), ('b', 'm'), ('c', None), ('d', None)]):
+        # Only non-string valued parameters should be extracted since f144 only
+        # supports numeric-valued data
+        self.assertEqual(len(params), 3)
+        for p, x in zip(params, [('a', 'Hz'), ('b', 'm'), ('c', None)]):
             self.assertEqual(p['name'], x[0])
             self.assertEqual(p['unit'], x[1])
+            self.assertEqual(p['module'], 'f144')
 
 
 if __name__ == '__main__':