pytrms 0.9.0__py3-none-any.whl → 0.9.2__py3-none-any.whl

pytrms/__init__.py

@@ -1,4 +1,4 @@
-_version = '0.9.0'
+_version = '0.9.2'
 
 __all__ = ['load', 'connect']
 
@@ -6,14 +6,16 @@ __all__ = ['load', 'connect']
 def load(path):
     '''Open a datafile for post-analysis or batch processing.
 
-    returns a `Measurement`.
+    `path` may be a glob-expression to collect a whole batch.
+
+    returns a `Measurement` instance.
     '''
-    from .measurement import OfflineMeasurement
-    from .readers import IoniTOFReader
+    import glob
+    from .measurement import FinishedMeasurement
 
-    reader = IoniTOFReader(path)
+    files = glob.glob(path)
 
-    return OfflineMeasurement(reader)
+    return FinishedMeasurement(*files)
 
 def connect(host=None, method='webapi'):
     '''Connect a client to a running measurement server.
@@ -23,7 +25,6 @@ def connect(host=None, method='webapi'):
     returns an `Instrument` if connected successfully.
     '''
     from .instrument import Instrument
-    from .helpers import PTRConnectionError
 
     if method.lower() == 'webapi':
         from .clients.ioniclient import IoniClient

pytrms/_base/mqttclient.py

@@ -6,9 +6,8 @@ from collections import deque
 from itertools import cycle
 from threading import Condition, RLock
 from datetime import datetime as dt
-from abc import ABC, abstractmethod
 
-import paho.mqtt.client as mqtt
+import paho.mqtt.client
 
 from .ioniclient import IoniClientBase
 
@@ -39,7 +38,6 @@ def _on_publish(client, self, mid):
 class MqttClientBase(IoniClientBase):
 
     @property
-    @abstractmethod
     def is_connected(self):
         '''Returns `True` if connected to the server.
 
@@ -49,12 +47,27 @@ class MqttClientBase(IoniClientBase):
         return (True
             and self.client.is_connected())
 
-    def __init__(self, host, subscriber_functions,
+    def __init__(self, host, port, subscriber_functions,
             on_connect, on_subscribe, on_publish, 
             connect_timeout_s=10):
-        super().__init__(host, port=1883)
-        # configure connection...
-        self.client = mqtt.Client(clean_session=True)
+        assert len(subscriber_functions) > 0, "no subscribers: for some unknown reason this causes disconnects"
+        super().__init__(host, port)
+
+        # Note: Version 2.0 of paho-mqtt introduced versioning of the user-callback to fix
+        #  some inconsistency in callback arguments and to provide better support for MQTTv5.
+        #  VERSION1 of the callback is deprecated, but is still supported in version 2.x.
+        #  If you want to upgrade to the newer version of the API callback, you will need
+        #  to update your callbacks:
+        paho_version = int(paho.mqtt.__version__.split('.')[0])
+        if paho_version == 1:
+            self.client = paho.mqtt.client.Client(clean_session=True)
+        elif paho_version == 2:
+            self.client = paho.mqtt.client.Client(paho.mqtt.client.CallbackAPIVersion.VERSION1,
+                    clean_session=True)
+        else:
+            # see https://eclipse.dev/paho/files/paho.mqtt.python/html/migrations.html
+            raise NotImplementedError("API VERSION2 for MQTTv5 (use paho-mqtt 2.x or implement user callbacks)")
+
         # clean_session is a boolean that determines the client type. If True,
         # the broker will remove all information about this client when it
         # disconnects. If False, the client is a persistent client and

pytrms/clients/__init__.py

@@ -7,7 +7,7 @@ assert os.path.exists(_par_id_file), "par-id file not found: please re-install P
 
 import logging as _logging
 
-_logging.TRACE = 0  # overwrites logging.NOTSET
+_logging.TRACE = 5  # even more verbose than logging.DEBUG
 
 def enable_extended_logging(log_level=_logging.DEBUG):
     '''make output of http-requests more talkative.
@@ -22,7 +22,7 @@ def enable_extended_logging(log_level=_logging.DEBUG):
         requests_log.setLevel(log_level)
         requests_log.propagate = True
     
-    if log_level == _logging.TRACE:
+    if log_level <= _logging.TRACE:
         # Enabling debugging at http.client level (requests->urllib3->http.client)
         # you will see the REQUEST, including HEADERS and DATA, and RESPONSE with
         # HEADERS but without DATA. the only thing missing will be the response.body,

pytrms/clients/db_api.py

@@ -4,6 +4,7 @@ import json
 import requests
 
 from . import _logging
+from .ssevent import SSEventListener
 from .._base import IoniClientBase
 
 log = _logging.getLogger(__name__)
@@ -82,7 +83,7 @@ class IoniConnect(IoniClientBase):
         if not endpoint.startswith('/'):
             endpoint = '/' + endpoint
         if not isinstance(data, str):
-            data = json.dumps(data)
+            data = json.dumps(data, ensure_ascii=False)  # default is `True`, escapes Umlaute!
         if 'headers' not in kwargs:
             kwargs['headers'] = {'content-type': 'application/hal+json'}
         elif 'content-type' not in (k.lower() for k in kwargs['headers']):
@@ -94,93 +95,89 @@ class IoniConnect(IoniClientBase):
 
         return r
 
-    def iter_events(self):
-        """Follow the server-sent-events (SSE) on the DB-API."""
-        r = self.session.request('GET', self.url + "/api/events",
-                headers={'accept': 'text/event-stream'}, stream=True)
-        r.raise_for_status()
-        kv_pair = dict()
-        for line in r.iter_lines():
-            # empty newlines serve as keep-alive and end-of-entry:
-            if not line:
-                if kv_pair:
-                    yield kv_pair
-                    kv_pair = dict()
-                else:
-                    log.debug("sse: still kept alive...")
+    def sync(self, peaktable):
+        """Compare and upload any differences in `peaktable` to the database."""
+        from pytrms.peaktable import Peak, PeakTable
+        from operator import attrgetter
+
+        # Note: a `Peak` is a hashable object that serves as a key that
+        #  distinguishes between peaks as defined by PyTRMS:
+        make_key = lambda peak: Peak(center=peak['center'], label=peak['name'], shift=peak['shift'])
+
+        if isinstance(peaktable, str):
+            log.info(f"loading peaktable '{peaktable}'...")
+            peaktable = PeakTable.from_file(peaktable)
+
+        # get the PyTRMS- and IoniConnect-peaks on the same page:
+        conv = {
+            'name':   attrgetter('label'),
+            'center': attrgetter('center'),
+            'kRate':  attrgetter('k_rate'),
+            'low':    lambda p: p.borders[0],
+            'high':   lambda p: p.borders[1],
+            'shift':  attrgetter('shift'),
+            'multiplier': attrgetter('multiplier'),
+        }
+        # normalize the input argument and create a hashable set:
+        updates = dict()
+        for peak in peaktable:
+            payload = {k: conv[k](peak) for k in conv}
+            updates[make_key(payload)] = {'payload': payload}
+
+        log.info(f"fetching current peaktable from the server...")
+        # create a comparable collection of peaks already on the database by
+        # reducing the keys in the response to what we actually want to update:
+        db_peaks = {make_key(p): {
+                    'payload': {k: p[k] for k in conv.keys()},
+                    'self':   p['_links']['self'],
+                    'parent': p['_links'].get('parent'),
+                    } for p in self.get('/api/peaks')['_embedded']['peaks']}
+
+        to_update = updates.keys() & db_peaks.keys()
+        to_upload = updates.keys() - db_peaks.keys()
+        updated = 0
+        for key in sorted(to_update):
+            # check if an existing peak needs an update
+            if db_peaks[key]['payload'] == updates[key]['payload']:
+                # nothing to do..
+                log.debug(f"up-to-date: {key}")
                 continue
 
-            key, val = line.decode().split(':')
-            kv_pair[key] = val.strip()
-
-    def refresh_comp_dict(self):
-        j = self.get('/api/components')
-        self.comp_dict = {component["shortName"]: component
-            for component in j["_embedded"]["components"]}
-    
-    def get_component(self, short_name):
-        if not len(self.comp_dict):
-            self.refresh_comp_dict()
-    
-        return self.comp_dict[short_name]
-
-    def create_component(self, short_name):
-        payload = {
-            "shortName": short_name
+            self.put(db_peaks[key]['self']['href'], updates[key]['payload'])
+            log.info(f"updated:    {key}")
+            updated += 1
+
+        if len(to_upload):
+            # Note: POSTing the embedded-collection is *miles faster*
+            #  than doing separate requests for each peak!
+            payload = {'_embedded': {'peaks': [updates[key]['payload'] for key in sorted(to_upload)]}}
+            self.post('/api/peaks', payload)
+            for key in sorted(to_upload): log.info(f"added new:  {key}")
+
+        # Note: this disregards the peak-parent-relationship, but in
+        #  order to implement this correctly, one would need to check
+        #  if the parent-peak with a specific 'parentID' is already
+        #  uploaded and search it.. there's an endpoint
+        #   'LINK /api/peaks/{parentID} Location: /api/peaks/{childID}'
+        #  to link a child to its parent, but it remains complicated.
+        # TODO :: maybe later implement parent-peaks!?
+
+        return {
+                'added': len(to_upload),
+                'updated': updated,
+                'up-to-date': len(to_update) - updated,
         }
-        self.post('/api/components', payload)
-        self.refresh_comp_dict()
-
-    def create_average(self, endpoint, run, step, action=0, use_mean=True):
 
-        params = {'run': int(run), 'step': int(step), 'usemean': bool(use_mean)}
-        if (action != 0):
-            params['action'] = int(action)
+    def iter_events(self, event_re=r".*"):
+        """Follow the server-sent-events (SSE) on the DB-API.
 
-        timecycles = self.get(endpoint, params)
-        self.current_avg_endpoint = self.post('/api/averages', timecycles)
+        `event_re`  a regular expression to filter events (default: matches everything)
 
-    def save_component_values(self, new_values):
+        Note: This will block until a matching event is received.
+         Especially, it cannot be cancelled by KeyboardInterrupt (due to the `requests`
+         stream-implementation), unless the server sends a keep-alive at regular
+         intervals (as every well-behaved server should be doing)!
         """
-        Post Components to the database.
-
-        `new_values`    dictionary {name~>value}
-        """
-        if self.current_avg_endpoint is None:
-            raise Exception("create average first")
-    
-        payload = {
-            "quantities": [
-                {
-                    "componentID": self.get_component(name)["componentID"],
-                    "value": value
-                } for name, value in new_values.items()
-            ]
-        }
-        endpoint = self.current_avg_endpoint + '/component_traces'
-        self.put(endpoint, payload)
-
-    def save_instrument_values(self, new_values):
-        """
-        Post Parameters to the database.
-
-        `new_values`    dictionary {name~>value}
-        """
-        # 13.07.: SCHNELL, SCHNELL (es ist 17 Uhr 57 und ich will die Modbus-instrument
-        #  daten noch hochladen):
-        #  this expects a namedtuple as defined in Modbus client: .set, .act, .par_id
-        if self.current_avg_endpoint is None:
-            raise Exception("create average first")
-    
-        payload = {
-            "quantities": [
-                {
-                    "parameterID": item.par_id,
-                    "setValue": item.set,
-                    "actMean": item.act
-                } for name, item in new_values.items()
-            ]
-        }
-        endpoint = self.current_avg_endpoint + '/parameter_traces'  # on the DB it's called parameter... :\
-        self.put(endpoint, payload)
+        yield from SSEventListener(event_re, host_url=self.url, endpoint="/api/events",
+                session=self.session)
 

pytrms/clients/ioniclient.py

@@ -18,17 +18,17 @@ class IoniClient:
     Access the Ionicon WebAPI.
 
     Usage:
-    >>> client = IoniClient()
-    >>> client.get('TPS_Pull_H')
+    > client = IoniClient()
+    > client.get('TPS_Pull_H')
     {'TPS_Pull_H': 123.45, ... }
 
-    >>> client.set('TPS_Pull_H', 42)
+    > client.set('TPS_Pull_H', 42)
     {'TPS_Pull_H': 42.0, ... }
 
-    >>> client.start_measurement()
+    > client.start_measurement()
     ACK
 
-    >>> client.host, client.port
+    > client.host, client.port
     ('localhost', 8002)
     
     '''
@@ -85,19 +85,3 @@ class IoniClient:
 
     def stop_measurement(self):
         return self.set('ACQ_SRV_Stop_Meas', 1)
-
-
-if __name__ == '__main__':
-    import sys
-    client = IoniClient()
-
-    if len(sys.argv) == 2:
-        print(client.get(sys.argv[1]))
-    elif len(sys.argv) == 3:
-        print(client.set(sys.argv[1], sys.argv[2]))
-    else:
-        print(f"""\
-                usage:
-                  python {sys.argv[0]} <varname> [<value>]
-              """)
-

pytrms/clients/modbus.py

@@ -9,7 +9,7 @@ from collections import namedtuple
 from functools import lru_cache
 from itertools import tee
 
-from pyModbusTCP import client
+import pyModbusTCP.client
 
 from . import _par_id_file
 from .._base.ioniclient import IoniClientBase
@@ -19,9 +19,28 @@ log = logging.getLogger(__name__)
 __all__ = ['IoniconModbus']
 
 
+def _patch_is_open():
+    # Note: the .is_open and .timeout attributes were changed
+    #  from a function to a property!
+    # 
+    # 0.2.0 2022-06-05
+    # 
+    #  - ModbusClient: parameters are now properties instead of methods (more intuitive).
+    # 
+    # from the [changelog](https://github.com/sourceperl/pyModbusTCP/blob/master/CHANGES):
+    major, minor, patch = pyModbusTCP.__version__.split('.')
+    if int(minor) < 2:
+        return lambda mc: mc.is_open()
+    else:
+        return lambda mc: mc.is_open
+
+_is_open = _patch_is_open()
+
 with open(_par_id_file) as f:
     it = iter(f)
-    assert next(it).startswith('ID\tName'), "Modbus parameter file is corrupt: " + f.name
+    assert next(it).startswith('ID\tName'), ("Modbus parameter file is corrupt: "
+            + f.name
+            + "\n\ntry re-installing the PyTRMS python package to fix it!")
     _id_to_descr = {int(id_): name for id_, name, *_ in (line.strip().split('\t') for line in it)}
 
 # look-up-table for c_structs (see docstring of struct-module for more info).
@@ -59,10 +78,10 @@ def _unpack(registers, format='>f'):
     representation, respectively.
 
     >>> _unpack([17448, 0], 'float')
-    672.
+    672.0
 
     >>> _unpack([17446, 32768], 'float')
-    666.
+    666.0
     
     >>> _unpack([16875, 61191, 54426, 37896], 'double')
     3749199524.83057
@@ -132,7 +151,7 @@ class IoniconModbus(IoniClientBase):
 
     @property
     def is_connected(self):
-        if not self.mc.is_open:
+        if not _is_open(self.mc):
             return False
 
         # wait for the IoniTOF alive-counter to change (1 second max)...
@@ -175,7 +194,18 @@ class IoniconModbus(IoniClientBase):
 
     def __init__(self, host='localhost', port=502):
         super().__init__(host, port)
-        self.mc = client.ModbusClient(host=self.host, port=self.port)
+        # Note: we patch the behaviour such, that it behaves like pre-0.2
+        #  (from the time of development of this module), BUT we skip the
+        #  auto_close-feature for the sake of speed:
+        # 
+        # 0.2.0 2022-06-05
+        # 
+        #  - ModbusClient: now TCP auto open mode is active by default (auto_open=True, auto_close=False).
+        #
+        # from the [changelog](https://github.com/sourceperl/pyModbusTCP/blob/master/CHANGES)
+        self.mc = pyModbusTCP.client.ModbusClient(host=self.host, port=self.port,
+                auto_open = False, auto_close = False
+        )
         # try connect immediately:
         try:
             self.connect()
@@ -185,8 +215,11 @@ class IoniconModbus(IoniClientBase):
 
     def connect(self, timeout_s=10):
         log.info(f"[{self}] connecting to Modbus server...")
-        self.mc.timeout = timeout_s
-        self.mc.auto_open = True
+        # Note: .timeout-attribute changed to a property with 0.2.0 (see comments above)
+        if callable(self.mc.timeout):
+            self.mc.timeout(timeout_s)
+        else:
+            self.mc.timeout = timeout_s
         if not self.mc.open():
             raise TimeoutError(f"[{self}] no connection to modbus socket")
 
@@ -201,7 +234,7 @@ class IoniconModbus(IoniClientBase):
             raise TimeoutError(f"[{self}] no connection to IoniTOF");
 
     def disconnect(self):
-        if self.mc.is_open:
+        if _is_open(self.mc):
             self.mc.close()
 
     @property
@@ -452,9 +485,9 @@ class IoniconModbus(IoniClientBase):
         _read = self.mc.read_holding_registers if is_holding_register else self.mc.read_input_registers
         
         register = _read(addr, n_bytes)
-        if register is None and self.mc.is_open:
+        if register is None and _is_open(self.mc):
             raise IOError(f"unable to read ({n_bytes}) registers at [{addr}] from connection")
-        elif register is None and not self.mc.is_open:
+        elif register is None and not _is_open(self.mc):
             raise IOError("trying to read from closed Modbus-connection")
 
         return _unpack(register, c_format)

pytrms/clients/mqtt.py

@@ -325,9 +325,14 @@ def follow_state(client, self, msg):
     log.debug(f"[{self}] new server-state: " + str(state))
     # replace the current state with the new element:
     self._server_state.append(state)
-    if state == "ACQ_Aquire":  # yes, there's a typo, plz keep it :)
-        self._calcconzinfo.append(_NOT_INIT)  # invalidate
-        # Note: this signals to the relevant thread that we need an update
+    meas_running = (state == "ACQ_Aquire")  # yes, there's a typo, plz keep it :)
+    just_started = (meas_running and not msg.retain)
+    if meas_running:
+        # signal the relevant thread(s) that we need an update:
+        self._calcconzinfo.append(_NOT_INIT)
+    if just_started:
+        # invalidate the source-file until we get a new one:
+        self._sf_filename.append(_NOT_INIT)
 
 follow_state.topics = ["DataCollection/Act/ACQ_SRV_CurrentState"]
 
@@ -356,6 +361,10 @@ def follow_act_set_values(client, self, msg):
             # Note: this topic doesn't strictly follow the convention and is handled separately
             return
 
+        if server == "Sequencer":
+            # Note: this is a separate program and will be ignored (has its own AUTO_-numbers et.c.)
+            return
+
         if parID == "PTR_CalcConzInfo":
             # another "special" topic handled in 'follow_calc_conz_info' ...
             return
@@ -404,8 +413,8 @@ _NOT_INIT = object()
 class MqttClient(MqttClientBase):
     """a simplified client for the Ionicon MQTT API.
 
-    >>> mq = MqttClient()
-    >>> mq.write('TCP_MCP_B', 3400)
+    > mq = MqttClient()
+    > mq.write('TCP_MCP_B', 3400)
     ValueError()
 
     """
@@ -463,11 +472,27 @@ class MqttClient(MqttClientBase):
 
     @property
     def current_sourcefile(self):
-        '''Returns the path to the hdf5-file that is currently (or soon to be) written.
+        '''Returns the path to the hdf5-file that is currently being written.
         
-        May be an empty string if no sourcefile has yet been set.
+        Returns an empty string if no measurement is running.
         '''
-        return self._sf_filename[0]
+        if not self.is_running:
+            return ""
+
+        if self._sf_filename[0] is not _NOT_INIT:
+            return self._sf_filename[0]
+
+        # Note: '_NOT_INIT' is set by us on start of acquisition, so we'd expect
+        #  to receive the source-file-topic after a (generous) timeout:
+        timeout_s = 15
+        started_at = time.monotonic()
+        while time.monotonic() < started_at + timeout_s:
+            if self._sf_filename[0] is not _NOT_INIT:
+                return self._sf_filename[0]
+    
+            time.sleep(10e-3)
+        else:
+            raise TimeoutError(f"[{self}] unable to retrieve source-file after ({timeout_s = })");
 
     @property
     def current_cycle(self):
@@ -476,9 +501,9 @@ class MqttClient(MqttClientBase):
             return self._overallcycle[0]
         return 0
 
-    def __init__(self, host='127.0.0.1'):
+    def __init__(self, host='127.0.0.1', port=1883):
         # this sets up the mqtt connection with default callbacks:
-        super().__init__(host, _subscriber_functions, None, None, None)
+        super().__init__(host, port, _subscriber_functions, None, None, None)
         log.debug(f"connection check ({self.is_connected}) :: {self._server_state = } / {self._sched_cmds = }");
 
     def disconnect(self):
@@ -508,12 +533,10 @@ class MqttClient(MqttClientBase):
         if _lut is self.set_values and is_read_only:
             raise ValueError(f"'{parID}' is read-only, did you mean `kind='act'`?")
 
-        # Note: The values should need NO! time to be populated from the MQTT topics,
-        #  because all topics are published as *retained* by the PTR-server.
-        #  However, a short timeout is respected before raising a `KeyError`:
-        try:
-            return _lut[parID]
-        except KeyError as exc:
+        if not parID in _lut:
+            # Note: The values should need NO! time to be populated from the MQTT topics,
+            #  because all topics are published as *retained* by the PTR-server.
+            #  However, a short timeout is respected before raising a `KeyError`:
             time.sleep(200e-3)
             rv = _lut.get(parID)
             if rv is not None:
@@ -525,6 +548,7 @@ class MqttClient(MqttClientBase):
                 "set" if parID in self.set_values else
                 "")
             raise KeyError(str(parID) + (' (did you mean `kind="%s"`?)' % error_hint) if error_hint else "")
+        return _lut[parID]
 
     def get_table(self, table_name):
         timeout_s = 10