naeural-client 2.0.0__py3-none-any.whl

naeural_client/default/session/mqtt_session.py

@@ -0,0 +1,72 @@
+import json
+
+from ...base import GenericSession
+from ...comm import MQTTWrapper
+from ...const import comms as comm_ct
+
+
+class MqttSession(GenericSession):
+  def startup(self):
+    self._default_communicator = MQTTWrapper(
+        log=self.log,
+        config=self._config,
+        send_channel_name=comm_ct.COMMUNICATION_CONFIG_CHANNEL,
+        recv_channel_name=comm_ct.COMMUNICATION_PAYLOADS_CHANNEL,
+        comm_type=comm_ct.COMMUNICATION_DEFAULT,
+        recv_buff=self._payload_messages,
+        connection_name=self.name,
+        verbosity=self._verbosity,
+    )
+
+    self._heartbeats_communicator = MQTTWrapper(
+        log=self.log,
+        config=self._config,
+        recv_channel_name=comm_ct.COMMUNICATION_CTRL_CHANNEL,
+        comm_type=comm_ct.COMMUNICATION_HEARTBEATS,
+        recv_buff=self._hb_messages,
+        connection_name=self.name,
+        verbosity=self._verbosity,
+    )
+
+    self._notifications_communicator = MQTTWrapper(
+        log=self.log,
+        config=self._config,
+        recv_channel_name=comm_ct.COMMUNICATION_NOTIF_CHANNEL,
+        comm_type=comm_ct.COMMUNICATION_NOTIFICATIONS,
+        recv_buff=self._notif_messages,
+        connection_name=self.name,
+        verbosity=self._verbosity,
+    )
+    return super(MqttSession, self).startup()
+
+  @property
+  def _connected(self):
+    """
+    Check if the session is connected to the communication server.
+    """
+    return self._default_communicator.connected and self._heartbeats_communicator.connected and self._notifications_communicator.connected
+
+  def _connect(self) -> None:
+    if self._default_communicator.connection is None:
+      self._default_communicator.server_connect()
+      self._default_communicator.subscribe()
+    if self._heartbeats_communicator.connection is None:
+      self._heartbeats_communicator.server_connect()
+      self._heartbeats_communicator.subscribe()
+    if self._notifications_communicator.connection is None:
+      self._notifications_communicator.server_connect()
+      self._notifications_communicator.subscribe()
+    return
+
+  def _communication_close(self, **kwargs):
+    self._default_communicator.release()
+    self._heartbeats_communicator.release()
+    self._notifications_communicator.release()
+    return
+
+  def _send_payload(self, to, msg):
+    payload = json.dumps(msg)
+
+    self._default_communicator._send_to = to
+    self._default_communicator.send(payload)
+    return

naeural_client/io_formatter/__init__.py

@@ -0,0 +1,2 @@
+from .io_formatter_manager import IOFormatterWrapper
+from .base import BaseFormatter

naeural_client/io_formatter/base/__init__.py

@@ -0,0 +1 @@
+from .base_formatter import BaseFormatter

naeural_client/io_formatter/base/base_formatter.py

@@ -0,0 +1,80 @@
+import traceback
+from time import time
+
+from ...const import PAYLOAD_DATA
+
+
+class BaseFormatter(object):
+
+  def __init__(self, log, signature, **kwargs):
+    self.signature = signature
+    self.log = log
+    super(BaseFormatter, self).__init__()
+    return
+
+  def P(self, *args, **kwargs):
+    return self.log.P(*args, **kwargs)
+
+  def _decode_streams(self, dct_config_streams):
+    """
+    Maybe implement
+    """
+    return dct_config_streams
+
+  def _encode_output(self, output):
+    """
+    Maybe implement
+    """
+    return output
+
+  def _decode_output(self, encoded_output):
+    """
+    Maybe implement
+    """
+    return encoded_output
+
+  def decode_streams(self, dct_config_streams):
+    try:
+      dct_config_streams = self._decode_streams(dct_config_streams)
+    except Exception as e:
+      dct_config_streams = {}
+      msg = "ERROR! Could not decode streams!\n{}".format(e)
+      self.P(msg)
+      self.P(traceback.format_exc(), color='r')
+
+    return dct_config_streams
+
+  def encode_output(self, output):
+    tm = time()
+    self.log.start_timer('encode', section='Formatter_' + str(self.signature))
+    try:
+      encoded_output = self._encode_output(output)
+    except Exception as e:
+      encoded_output = {}
+      msg = "ERROR! Could not encode output {}\n{}".format(output, e)
+      self.P(msg)
+      self.P(traceback.format_exc(), color='r')
+    # end try-except
+
+    elapsed = time() - tm
+    self.log.stop_timer('encode', section='Formatter_' + str(self.signature))
+    return encoded_output, elapsed
+
+  def decode_output(self, encoded_output):
+    ee_impl = encoded_output.get(PAYLOAD_DATA.EE_FORMATTER, encoded_output.get(PAYLOAD_DATA.SB_IMPLEMENTATION, None))
+    if ee_impl is None or (isinstance(ee_impl, str) and ee_impl.lower() != self.signature.lower()):
+      return encoded_output
+
+    self.log.start_timer('decode', section='Formatter_' + str(self.signature))
+    try:
+      output = self._decode_output(encoded_output)
+    except Exception as e:
+      output = {}
+      msg = "ERROR! Could not decode {}\n{}".format(encoded_output, e)
+      self.P(msg)
+      self.P(traceback.format_exc(), color='r')
+    # end try-except
+    self.log.stop_timer('decode', section='Formatter_' + str(self.signature))
+    if isinstance(output, dict):
+      output[PAYLOAD_DATA.EE_FORMATTER] = ee_impl
+    return output

naeural_client/io_formatter/default/__init__.py

@@ -0,0 +1,3 @@
+from .a_dummy import ADummyFormatter
+from .default import DefaultFormatter
+from .aixp1 import Aixp1Formatter

naeural_client/io_formatter/default/a_dummy.py

@@ -0,0 +1,51 @@
+
+from ...io_formatter.base import BaseFormatter
+import re
+
+
+def camel_to_upper_snake(name):
+  name = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
+  return re.sub('([a-z0-9])([A-Z])', r'\1_\2', name).upper()
+
+
+def snake_to_camel(name):
+  name = name.lower()
+
+  words = []
+  for i, x in enumerate(name.split('_')):
+    if i > 0:
+      words.append(x.title())
+    else:
+      words.append(x)
+
+  return ''.join(words)
+
+
+class ADummyFormatter(BaseFormatter):
+
+  def __init__(self, **kwargs):
+    super(ADummyFormatter, self).__init__(**kwargs)
+    return
+
+  def _decode_streams(self, dct_config_streams):
+    return dct_config_streams
+
+  def _encode_output(self, output):
+    # transforms the first level keys to camelCase
+    new_output = {}
+
+    for k, v in output.items():
+      k_camel_case = snake_to_camel(k)
+      new_output[k_camel_case] = v
+
+    return new_output
+
+  def _decode_output(self, encoded_output):
+    # it is used mainly for map-reduce jobs in reduce phase!
+    decoded_output = {}
+
+    for k, v in encoded_output.items():
+      k_snake_case = camel_to_upper_snake(k)
+      decoded_output[k_snake_case] = v
+
+    return decoded_output

naeural_client/io_formatter/default/aixp1.py

@@ -0,0 +1,113 @@
+# local dependencies
+from ...io_formatter.base import BaseFormatter
+
+
+class Aixp1Formatter(BaseFormatter):
+
+  def __init__(self, log, **kwargs):
+    super(Aixp1Formatter, self).__init__(
+        log=log, prefix_log='[INV-FORM]', **kwargs)
+    return
+
+  def startup(self):
+    pass
+
+  def _encode_output(self, output):
+    event_type = output.pop('EE_EVENT_TYPE', None)
+
+    # below fields are not required as they will be decorated post-formatting anyway
+    output.pop('EE_MESSAGE_ID', None)
+    output.pop('EE_MESSAGE_SEQ', None)
+    output.pop('EE_TOTAL_MESSAGES', None)
+
+    output.pop('EE_TIMESTAMP', None)
+    output.pop('EE_ID', None)
+    output.pop('STREAM_NAME', None)
+    output.pop('SIGNATURE', None)
+    output.pop('INSTANCE_ID', None)
+
+    output.pop('EE_TIMEZONE', None)
+    output.pop('EE_VERSION', None)
+    output.pop('EE_TZ', None)
+
+    output.pop('INITIATOR_ID', None)
+    output.pop('SESSION_ID', None)
+    # end non-managed fields
+
+    lvl_0_dct = {
+      "DATA": {
+      },
+    }
+
+    lvl_1_dct = lvl_0_dct['DATA']
+
+    if event_type == 'PAYLOAD':
+      # add payload context
+      output.pop('STREAM')
+      output.pop('PIPELINE')
+
+      # Plugin meta
+      if True:
+        lvl_1_dct['PLUGIN_META'] = {}
+        plugin_meta_keys = [k for k in output.keys() if k.startswith('_P_')]
+        for k in plugin_meta_keys:
+          lvl_1_dct['PLUGIN_META'][k] = output.pop(k, None)
+        # end for
+
+      # Pipeline meta
+      if True:
+        lvl_1_dct['PIPELINE_META'] = {}
+        pipeline_meta_keys = [k for k in output.keys() if k.startswith('_C_')]
+        for k in pipeline_meta_keys:
+          lvl_1_dct['PIPELINE_META'][k] = output.pop(k, None)
+        # end for
+
+    # endif payload
+
+    for k, v in output.items():
+      lvl_1_dct[k] = v
+
+    return lvl_0_dct
+
+  def _decode_output(self, encoded_output):
+    # Pop the unimportant stuff
+    encoded_output.get('EE_FORMATTER', None)
+
+    node_id, pipeline, signature, instance_id = encoded_output.get('EE_PAYLOAD_PATH', [None, None, None, None])
+
+    encoded_output['EE_ID'] = node_id
+
+    if encoded_output['EE_EVENT_TYPE'] != 'HEARTBEAT':
+      encoded_output['STREAM_NAME'] = pipeline
+
+    if pipeline is not None:
+      encoded_output['SIGNATURE'] = signature
+
+    if instance_id is not None:
+      encoded_output['INSTANCE_ID'] = instance_id
+
+    lvl_1_dct = encoded_output.pop('DATA')
+    if encoded_output['EE_EVENT_TYPE'] == 'PAYLOAD':
+      encoded_output['STREAM'] = pipeline
+      encoded_output['PIPELINE'] = pipeline
+
+      # Plugin meta
+      plugin_meta = lvl_1_dct.pop('PLUGIN_META', {}) or {}
+      for k, v in plugin_meta.items():
+        encoded_output[k] = v
+
+      # Pipeline meta
+      pipeline_meta = lvl_1_dct.pop('PIPELINE_META', {}) or {}
+      for k, v in pipeline_meta.items():
+        encoded_output[k] = v
+
+      encoded_output['PLUGIN_CATEGORY'] = 'general'
+      a = 1
+
+    for k, v in lvl_1_dct.items():
+      encoded_output[k] = v
+
+    return encoded_output
+
+  def _decode_streams(self, dct_config_streams):
+    return dct_config_streams

naeural_client/io_formatter/default/default.py

@@ -0,0 +1,22 @@
+# local dependencies
+from ...io_formatter.base import BaseFormatter
+
+
+class DefaultFormatter(BaseFormatter):
+
+  def __init__(self, log, **kwargs):
+    super(DefaultFormatter, self).__init__(
+        log=log, prefix_log='[DEFAULT-FMT]', **kwargs)
+    return
+
+  def startup(self):
+    pass
+
+  def _encode_output(self, output):
+    return output
+
+  def _decode_output(self, encoded_output):
+    return encoded_output
+
+  def _decode_streams(self, dct_config_streams):
+    return dct_config_streams

naeural_client/io_formatter/io_formatter_manager.py

@@ -0,0 +1,96 @@
+from time import time
+
+from ..plugins_manager_mixin import _PluginsManagerMixin
+from ..const import PAYLOAD_DATA
+from ..io_formatter.default import Aixp1Formatter, DefaultFormatter
+
+
+class IOFormatterWrapper(_PluginsManagerMixin):
+  FORMATTER_CLASSES = [DefaultFormatter, Aixp1Formatter]
+
+  def __init__(self, log, plugin_search_locations=['plugins.io_formatters'], plugin_search_suffix='Formatter', **kwargs):
+    super(IOFormatterWrapper, self).__init__()
+    self._dct_formatters = {}
+    self.log = log
+    self.plugin_search_locations = plugin_search_locations
+    self.plugin_search_suffix = plugin_search_suffix
+
+    self._last_search_invalid_formatter = {}
+
+    self.__init_formatters()
+    return
+
+  def __init_formatters(self):
+    formatter_names_and_classes = [(cls.__name__.lower().split('formatter')[0], cls) for cls in self.FORMATTER_CLASSES]
+    for formatter_name, formatter_class in formatter_names_and_classes:
+      try:
+        self._dct_formatters[formatter_name] = formatter_class(log=self.log, signature=formatter_name)
+        self.D("Successfully created IO formatter {}.".format(formatter_name))
+      except Exception as exc:
+        msg = "Exception '{}' when initializing io_formatter plugin {}".format(exc, formatter_name)
+        self.P(msg, color='r')
+
+    return
+
+  def _get_formatter_name_from_payload(self, msg):
+    return msg.get(PAYLOAD_DATA.EE_FORMATTER, msg.get(PAYLOAD_DATA.SB_IMPLEMENTATION, ''))
+
+  def _get_plugin_class(self, name):
+    _module_name, _class_name, _class_def, _class_config = self._get_module_name_and_class(
+        locations=self.plugin_search_locations,
+        name=name,
+        suffix=self.plugin_search_suffix,
+        safe_locations=[],
+    )
+
+    if _class_def is None:
+      msg = "Error loading io_formatter plugin '{}'".format(name)
+      self.D(msg, color='r')
+    return _class_def
+
+  def formatter_ready(self, name):
+    return name in self._dct_formatters or name is None or name == ''
+
+  def get_formatter_by_name(self, name):
+    return self._create_formatter(name)
+
+  def _create_formatter(self, name):
+    # TODO: change name to maybe_create_formatter
+    if name is None or name == '':
+      # check if we want to create a default formatter
+      return self._dct_formatters['default']
+
+    if name in self._dct_formatters:
+      # formatter already created
+      if self._dct_formatters[name] is None:
+        # formatter is not available
+        if name not in self._last_search_invalid_formatter:
+          self._last_search_invalid_formatter[name] = 0
+        if time() - self._last_search_invalid_formatter[name] < 10 * 60:
+          return self._dct_formatters[name]
+      else:
+        return self._dct_formatters[name]
+    # end if name in self._dct_formatters
+
+    self.D("Creating formatter '{}'".format(name))
+    _cls = self._get_plugin_class(name)
+
+    formatter = None
+
+    if _cls is not None:
+      formatter = _cls(log=self.log, signature=name.lower())
+      self.D("Successfully created IO formatter {}.".format(name))
+    else:
+      self._last_search_invalid_formatter[name] = time()
+    self._dct_formatters[name] = formatter
+    return formatter
+
+  def get_required_formatter_from_payload(self, payload):
+    name = self._get_formatter_name_from_payload(payload)
+    return self._create_formatter(name)
+
+  def D(self, *args, **kwargs):
+    return self.log.D(*args, **kwargs)
+
+  def P(self, *args, **kwargs):
+    return self.log.P(*args, **kwargs)

naeural_client/logging/__init__.py

@@ -0,0 +1 @@
+from .small_logger import Logger