mccode-plumber 0.15.1__py3-none-any.whl → 0.17.0__py3-none-any.whl

mccode_plumber/forwarder.py

@@ -7,6 +7,7 @@ list of EPICS PVs to monitor.
 Alternatively, the same functionality can be accessed from Python using the configure_forwarder and reset_forwarder
 functions. Which take PV information and Forwarder/Kafka configuration as arguments.
 """
+from mccode_antlr.common import InstrumentParameter
 
 
 def normalise_pvs(pvs: list[dict], config=None, prefix=None, topic=None):
@@ -59,10 +60,19 @@ def reset_forwarder(pvs: list[dict], config=None, prefix=None, topic=None):
     return pvs
 
 
-def forwarder_partial_streams(prefix, topic, parameters):
-    names = [p.name for p in parameters]
-    if 'mcpl_filename' not in names:
-        names.append("mcpl_filename")
+def forwarder_partial_streams(prefix: str, topic: str, parameters: list[InstrumentParameter]):
+    from mccode_antlr.common import DataType
+    # The streaming-data-type f144 only supports numeric data, so we need to
+    # filter out string-valued data types to avoid annoying error messages in the
+    # forwarder's log output.
+    names = [p.name for p in parameters if p.value.data_type is not DataType.str]
+
+    # splitrun adds an instrument parameter named 'mcpl_filename', but we also
+    # can not forward this since it is a string-valued parameter.
+    #
+    #if 'mcpl_filename' not in names:
+    #    names.append("mcpl_filename")
+
     # Minimal information used by the forwarder for stream setup:
     partial = [dict(source=f'{prefix}{n}', module='f144', topic=topic) for n in names]
     return partial

mccode_plumber/manage/manager.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from subprocess import Popen, PIPE
 from threading import Thread
 from enum import Enum
@@ -12,6 +12,54 @@ class IOType(Enum):
     stderr = 2
 
 
+@dataclass
+class Triage:
+    level: str = field(default=lambda: 'info')
+    ignore: list[str] = field(default_factory=list)
+    patterns: dict[str, list[str]] = field(default_factory=lambda: {
+        'critical': [r'\bcritical\b', r'^cri'],
+        'error': [r'\berror\b', r'exception', r'traceback', r'^err' ],
+        'warning': [r'\bwarn(ing)?\b', r'deprecated', r'^war'],
+        'notice': [r'\bnotice\b', r'^not'],
+        'info': [r'\binfo\b', r'starting', r'done'],
+        'hint': [r'\bhint\b'],
+        'debug': [r'\bdebug\b', r'^deb', r'^dbg'],
+    })
+    styles: dict[str, str] = field(default_factory=lambda: {
+        'critical': Fore.MAGENTA + Style.BRIGHT,
+        'error': Fore.RED + Style.BRIGHT,
+        'warning': Fore.YELLOW + Style.BRIGHT,
+        'notice': Fore.CYAN + Style.BRIGHT,
+        'info': Fore.GREEN,
+        'hint': Fore.BLUE,
+        'debug': Fore.WHITE + Style.BRIGHT,
+        'default': Fore.WHITE,
+    })
+
+    def _filtered_level(self, level: str) -> bool:
+        def _level_value(v: str):
+            for i, lvl in enumerate(self.patterns.keys()):
+                if v == lvl:
+                    return i
+            return -1
+        return _level_value(level) > _level_value(self.level)
+
+    def _style_line(self, level: str, line: str):
+        return self.styles.get(level, '') + line + Style.RESET_ALL
+
+    def __call__(self, line: str) -> tuple[bool, str | None]:
+        import re
+        # If the line contains an ignored keyword, ignore it.
+        if any(kw in line for kw in self.ignore):
+            return True, None
+        # Check if we can identify the status level of this message
+        for level, patterns in self.patterns.items():
+            for pattern in patterns:
+                if re.search(pattern, line, re.IGNORECASE):
+                    return self._filtered_level(level), self._style_line(level, line)
+        return self._filtered_level('default'), self._style_line('default', line)
+
+
 @dataclass
 class Manager:
     """
@@ -19,13 +67,24 @@ class Manager:
 
     Properties
     ----------
+    name: str
+        The name of the process, used as a prefix for all printed status messages
+    style: AnsiStyle
+        Format string to style the printed process name
+    _triage: Triage
+        An object to filter status messages and identify severity levels
+        applying its own message styling based on the identified level
     _process:   a subprocess.Popen instance
+    _stdout_thread: Thread
+    _stderr_thread: Thread
     """
     name: str
     style: AnsiStyle
+    triage: Triage
     _process: Popen | None
     _stdout_thread: Thread | None
     _stderr_thread: Thread | None
+    _name_padding: int
 
     def __run_command__(self) -> list[str]:
         return []
@@ -38,6 +97,18 @@ class Manager:
         from dataclasses import fields
         return [field.name for field in fields(cls)]
 
+    @property
+    def name_padding(self):
+        return self._name_padding
+
+    @name_padding.setter
+    def name_padding(self, value: int):
+        self._name_padding = value
+
+    def _pretty_name(self):
+        padding = ' ' * self.name_padding
+        return f'{self.style}{self.name}:{Style.RESET_ALL}{padding}'
+
     def _read_stream(self, stream, io_type: IOType):
         """Read lines from stream and print them until EOF.
 
@@ -51,8 +122,10 @@ class Manager:
             for line in iter(stream.readline, ''):
                 if not line:
                     break
-                # format and print the line, preserving original behaviour
-                formatted = f'{self.style}{self.name}:{Style.RESET_ALL} {line}'
+                ignored, line = self.triage(line)
+                if ignored:
+                    continue
+                formatted = f'{self._pretty_name()} {line}'
                 if io_type == IOType.stdout:
                     print(formatted, end='')
                 else:
@@ -79,6 +152,10 @@ class Manager:
             kwargs['name'] = 'Managed process'
         if 'style' not in kwargs:
             kwargs['style'] = Fore.WHITE + Back.BLACK
+        if 'triage' not in kwargs:
+            kwargs['triage'] = Triage()
+        if '_name_padding' not in kwargs:
+            kwargs['_name_padding'] = 0
 
         manager = cls(**kwargs)
 

mccode_plumber/manage/orchestrate.py

@@ -261,6 +261,7 @@ def load_in_wait_load_out(
         )
         from mccode_plumber.manage.forwarder import forwarder_verbosity
         from mccode_plumber.manage.writer import writer_verbosity
+        from mccode_plumber.manage.manager import Triage
 
         # Start up services if they should be managed locally
         if manage:
@@ -283,7 +284,10 @@ def load_in_wait_load_out(
                 efu = [EventFormationUnitConfig.from_dict(data)]
             things = tuple(
                 EventFormationUnit.start(
-                    style=Fore.BLUE, broker=broker, **x.to_dict()
+                    style=Fore.BLUE,
+                    broker=broker,
+                    triage=Triage(ignore=["graphite", ":2003 failed"]),
+                    **x.to_dict()
                 ) for x in efu) + (
                 Forwarder.start(
                     name='FWD',
@@ -309,6 +313,9 @@ def load_in_wait_load_out(
                     verbosity=writer_verbosity(verbosity_writer),
                 ),
             )
+            longest_name = max(len(thing.name) for thing in things)
+            for thing in things:
+                thing.name_padding = longest_name - len(thing.name)
         else:
             things = ()
 

mccode_plumber/splitrun.py

@@ -17,11 +17,15 @@ def make_parser():
 
 
 def monitors_to_kafka_callback_with_arguments(
-        broker: str, topic: str | None, source: str | None, names: list[str] | None
+        broker: str, topic: str | None, source: str | None, names: list[str] | None,
+        delete_after_sending: bool = True,
 ):
     from mccode_to_kafka.sender import send_histograms
 
-    partial_kwargs: dict[str, Union[str,list[str]]] = {'broker': broker}
+    partial_kwargs: dict[str, Union[str,list[str]]] = {
+        'broker': broker,
+        'delete': delete_after_sending,
+    }
     if topic is not None and source is not None and names is not None and len(names) > 1:
         raise ValueError("Cannot specify both topic/source and multiple names simultaneously.")
 
@@ -41,7 +45,15 @@ def monitors_to_kafka_callback_with_arguments(
 def main():
     from .mccode import get_mcstas_instr
     from restage.splitrun import splitrun_args, parse_splitrun
-    args, parameters, precision = parse_splitrun(make_parser())
+    parser = make_parser()
+    parser.add_argument('--keep-after-send', action='store_true', help='Keep after sending histograms', default=False)
+    args, parameters, precision = parse_splitrun(parser)
     instr = get_mcstas_instr(args.instrument)
-    callback, callback_args = monitors_to_kafka_callback_with_arguments(args.broker, args.topic, args.source, args.names)
+    callback, callback_args = monitors_to_kafka_callback_with_arguments(
+        broker=args.broker,
+        topic=args.topic,
+        source=args.source,
+        names=args.names,
+        delete_after_sending=not args.keep_after_send
+    )
     return splitrun_args(instr, parameters, precision, args, callback=callback, callback_arguments=callback_args)

mccode_plumber/writer.py

@@ -311,8 +311,16 @@ def construct_writer_pv_dicts(instr: Path | str, prefix: str, topic: str):
 
 
 def construct_writer_pv_dicts_from_parameters(parameters, prefix: str, topic: str):
+    from mccode_antlr.common import DataType
+
     def strip_quotes(s):
         return s[1:-1] if s is not None and len(s) > 2 and (s[0] == s[-1] == '"' or s[0] == s[-1] == "'") else s
+
+    # Remove string-valued parameters from the provided list since they are not
+    # supported by streaming-data-type f144. In the future we could specify a different
+    # module for them instead.
+    parameters = [p for p in parameters if p.value.data_type is not DataType.str]
+
     return [dict(name=p.name, dtype=p.value.data_type.name, source=f'{prefix}{p.name}', topic=topic,
                  description=parameter_description(p), module='f144', unit=strip_quotes(p.unit)) for p in parameters]
 

{mccode_plumber-0.15.1.dist-info → mccode_plumber-0.17.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mccode-plumber
-Version: 0.15.1
+Version: 0.17.0
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Programming Language :: Python :: 3
@@ -13,8 +13,8 @@ Requires-Dist: p4p
 Requires-Dist: kafka-python>=2.2.11
 Requires-Dist: ess-streaming-data-types>=0.14.0
 Requires-Dist: restage>=0.10.1
-Requires-Dist: mccode-to-kafka>=0.3.1
-Requires-Dist: moreniius>=0.6.3
+Requires-Dist: mccode-to-kafka>=0.5.0
+Requires-Dist: moreniius>=0.7.0
 Requires-Dist: icecream
 Requires-Dist: ephemeral-port-reserve
 Provides-Extra: test

{mccode_plumber-0.15.1.dist-info → mccode_plumber-0.17.0.dist-info}/RECORD

@@ -2,12 +2,12 @@ mccode_plumber/__init__.py,sha256=ZQMwyPQ6GyoEkKXj7aWETPqhXfwUbDOD5EpvlXj1c18,14
 mccode_plumber/conductor.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 mccode_plumber/epics.py,sha256=BqkT4A0i1VRMshZU-L0NlJy3BoVDcpHkoKm6gnVB4C8,6217
 mccode_plumber/epics_watcher.py,sha256=Jiz761A5NfoUzJ6ZzBGg8_BewFTmHDo5qYgh_DZHx_4,3973
-mccode_plumber/forwarder.py,sha256=yzjb--r8M6vNSsWvvOi5IjXqpRn8MvT13cv89ezNEeU,3994
+mccode_plumber/forwarder.py,sha256=Nccjof4GydaZLFyk_Ij9xBXy8QUpGigOV6lOPGw0PAQ,4513
 mccode_plumber/kafka.py,sha256=1BVTsGmiFJPkl-5RVJbkLm68AKRCp2NyngPjMcSutBs,2501
 mccode_plumber/mccode.py,sha256=vc4Gb5WhdOfuDRjEejU3Flps0C2A60sfFbMrxmKLhn0,2189
-mccode_plumber/splitrun.py,sha256=E9O88_GnoZ6vdmtgqW2kcHhg0ytJ_FH-oQDtMbSzbAI,2067
+mccode_plumber/splitrun.py,sha256=kJShCw1Cd_QawAeHooxa79XimtslLeWxeuJ3W4clsKk,2424
 mccode_plumber/utils.py,sha256=E8NoGo_3Z-pPEpzicVGSWfQOX8p3eR-GxElT33-kX5U,2167
-mccode_plumber/writer.py,sha256=10Y6r6s9oXWM76HYJcBSW_a0em-l_E9WJ8y22vn_-JA,21240
+mccode_plumber/writer.py,sha256=uNU1nICANNbB-gTa2WKptfls6tzmoHkUIdzmIeaFWXE,21569
 mccode_plumber/file_writer_control/CommandChannel.py,sha256=Ge5O71tZqpPmgcyYVcQe6XnxkE9r71Tf3he1ndvOFlQ,9038
 mccode_plumber/file_writer_control/CommandHandler.py,sha256=LAmr6SpriPfVJd6C38XZ2TsSoO2Dh0iSIzyo8Q3gfrc,2376
 mccode_plumber/file_writer_control/CommandStatus.py,sha256=acvS-KTH4hYD4xNw3i3FX0d4IukheWvKBmZ--rh1aj0,4536
@@ -26,11 +26,11 @@ mccode_plumber/manage/efu.py,sha256=y4V-LBQ9Xt5YATEfW0XkZ9KcPGo230-5BIemthSaMD8,
 mccode_plumber/manage/ensure.py,sha256=rS1kjUGhbks42_jsAZ2GjnhzJmpYqn59h6y3HJgshLs,3166
 mccode_plumber/manage/epics.py,sha256=oQt_hL-7KgtI9Kmwyw8L7a7RxQe7YJWRbmHnKIqMLpQ,1310
 mccode_plumber/manage/forwarder.py,sha256=stDKEVVfepCOqs5vBCVepaO3d63KJT-6okxooIl9dGs,3136
-mccode_plumber/manage/manager.py,sha256=hkvtH-brtn1IXQB6vN9VB1UF95nSrs48uiIebS48wJI,3876
-mccode_plumber/manage/orchestrate.py,sha256=Xqu0mm1eBzwf_pRYJzomk0YtnRtlBeQE5dvw78IuqiE,17985
+mccode_plumber/manage/manager.py,sha256=oQal8gmVqWy2Qd9BjTRZEliRkRzWjZbG6wtPD3__EX0,6762
+mccode_plumber/manage/orchestrate.py,sha256=knODbPBgSZNSCCpBjqc6yiSivWQGQMMLHidShVu-aCc,18323
 mccode_plumber/manage/writer.py,sha256=SEv1U14L01Y9-BcaJKPei4Ah2LFfwexDy9FTjpvtSEs,2245
-mccode_plumber-0.15.1.dist-info/METADATA,sha256=S6rNqxvR7-kvA6JZ9vCbC1CsntBynSgRciR0kJENu0g,957
-mccode_plumber-0.15.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mccode_plumber-0.15.1.dist-info/entry_points.txt,sha256=k3LDo_9HG2v4-IgMYlNukphdMmaAT6zkJZYaB1zJh3c,900
-mccode_plumber-0.15.1.dist-info/top_level.txt,sha256=kCCIpYtKHCKWxiPEqX9J1UaGEm-ze0Qb-cemBCEPhDA,15
-mccode_plumber-0.15.1.dist-info/RECORD,,
+mccode_plumber-0.17.0.dist-info/METADATA,sha256=0erkj44mHc3c9oyNUArbp16YA2BOaWfLMAcn6Pyr8Qk,957
+mccode_plumber-0.17.0.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
+mccode_plumber-0.17.0.dist-info/entry_points.txt,sha256=k3LDo_9HG2v4-IgMYlNukphdMmaAT6zkJZYaB1zJh3c,900
+mccode_plumber-0.17.0.dist-info/top_level.txt,sha256=kCCIpYtKHCKWxiPEqX9J1UaGEm-ze0Qb-cemBCEPhDA,15
+mccode_plumber-0.17.0.dist-info/RECORD,,

{mccode_plumber-0.15.1.dist-info → mccode_plumber-0.17.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.1)
 Root-Is-Purelib: true
 Tag: py3-none-any