naeural-client 2.0.0__py3-none-any.whl

naeural_client/logging/logger_mixins/pickle_serialization_mixin.py

@@ -0,0 +1,301 @@
+import os
+import bz2
+import pickle
+
+class _PickleSerializationMixin(object):
+  """
+  Mixin for pickle serialization functionalities that are attached to `pye2.Logger`.
+
+  This mixin cannot be instantiated because it is built just to provide some additional
+  functionalities for `pye2.Logger`
+
+  In this mixin we can use any attribute/method of the Logger.
+  """
+
+  def __init__(self):
+    super(_PickleSerializationMixin, self).__init__()
+    return
+
+  def _save_compressed_pickle(self, full_filename, myobj, locking=False):
+    """
+    save object to file using pickle
+
+    @param full_filename: name of destination file
+    @param myobj: object to save (has to be pickleable)
+    """
+    with self.managed_lock_resource(full_filename, condition=locking):
+      try:
+        fhandle = bz2.BZ2File(full_filename, 'wb')
+        pickle.dump(myobj, fhandle, protocol=pickle.HIGHEST_PROTOCOL)
+        fhandle.close()
+      except:
+        self.P('ERROR: File ' + full_filename + ' cannot be written!')
+        return False
+    # endwith conditional lock
+    return True
+
+
+  def _load_compressed_pickle(self, full_filename, locking=False):
+    """
+    Load from filename using pickle
+
+    @param full_filename: name of file to load from
+    """
+    with self.managed_lock_resource(full_filename, condition=locking):
+      try:
+        fhandle = bz2.BZ2File(full_filename, 'rb')
+        myobj = pickle.load(fhandle)
+        fhandle.close()
+      except:
+        self.P('ERROR: File ' + full_filename + ' cannot be read!')
+        return None
+    # endwith conditional lock
+
+    return myobj
+
+
+  def save_pickle(self, data, fn, folder=None,
+                  use_prefix=False, verbose=True,
+                  compressed=False,
+                  subfolder_path=None,
+                  locking=True,
+                  ):
+    """
+    compressed: True if compression is required OR you can just add '.pklz' to `fn`
+    """
+
+    def P(s):
+      if verbose:
+        self.P(s)
+      return
+
+    # enddef
+
+    lfld = self.get_target_folder(target=folder)
+
+    if lfld is None:
+      P("Assuming `fn` param ({}) is a full path".format(fn))
+      datafile = fn
+    else:
+      if use_prefix:
+        fn = self.file_prefix + '_' + fn
+      datafolder = lfld
+      if subfolder_path is not None:
+        datafolder = os.path.join(datafolder, subfolder_path.lstrip('/'))
+        os.makedirs(datafolder, exist_ok=True)
+      datafile = os.path.join(datafolder, fn)
+
+    os.makedirs(os.path.split(datafile)[0], exist_ok=True)
+
+    if compressed or '.pklz' in fn:
+      if not compressed:
+        P("Saving pickle with compression=True forced due to extension")
+      else:
+        P("Saving pickle with compression...")
+      if self._save_compressed_pickle(datafile, myobj=data, locking=locking):
+        P("  Compressed pickle {} saved in {}".format(fn, folder))
+      else:
+        P("  FAILED compressed pickle save!")
+    else:
+      P("Saving uncompressed pikle (lock:{}) : {} ".format(locking, datafile))
+      with self.managed_lock_resource(datafile, condition=locking):
+        try:
+          with open(datafile, 'wb') as fhandle:
+            pickle.dump(data, fhandle, protocol=pickle.HIGHEST_PROTOCOL)
+        except:
+          pass
+    # endwith conditional lock
+      if verbose:
+        P("  Saved pickle '{}' in '{}' folder".format(fn, folder))
+    return datafile
+
+
+  def save_pickle_to_data(self, data, fn, compressed=False, verbose=True, 
+                          subfolder_path=None, locking=True):
+    """
+    compressed: True if compression is required OR you can just add '.pklz' to `fn`
+    """
+    return self.save_pickle(
+      data, fn,
+      folder='data',
+      compressed=compressed,
+      subfolder_path=subfolder_path,
+      verbose=verbose,
+      locking=locking,
+    )
+
+
+  def save_pickle_to_models(self, data, fn, compressed=False, verbose=True, 
+                            subfolder_path=None, locking=True):
+    """
+    compressed: True if compression is required OR you can just add '.pklz' to `fn`
+    """
+    return self.save_pickle(
+      data, fn,
+      folder='models',
+      compressed=compressed,
+      subfolder_path=subfolder_path,
+      verbose=verbose,
+      locking=locking,
+    )
+
+
+  def save_pickle_to_output(self, data, fn, compressed=False, verbose=True, 
+                            subfolder_path=None, locking=True):
+    """
+    compressed: True if compression is required OR you can just add '.pklz' to `fn`
+    """
+    return self.save_pickle(
+      data, fn,
+      folder='output',
+      compressed=compressed,
+      subfolder_path=subfolder_path,
+      verbose=verbose,
+      locking=locking,
+    )
+
+
+  def load_pickle_from_models(self, fn, decompress=False, verbose=True, 
+                              subfolder_path=None, locking=True):
+    """
+     decompressed : True if the file was saved with `compressed=True` or you can just use '.pklz'
+    """
+    return self.load_pickle(
+      fn,
+      folder='models',
+      decompress=decompress,
+      verbose=verbose,
+      subfolder_path=subfolder_path,
+      locking=locking,
+    )
+
+
+  def load_pickle_from_data(self, fn, decompress=False, verbose=True, 
+                            subfolder_path=None, locking=True):
+    """
+     decompressed : True if the file was saved with `compressed=True` or you can just use '.pklz'
+    """
+    return self.load_pickle(
+      fn,
+      folder='data',
+      decompress=decompress,
+      verbose=verbose,
+      subfolder_path=subfolder_path,
+      locking=locking,
+    )
+
+
+  def load_pickle_from_output(self, fn, decompress=False, verbose=True, 
+                              subfolder_path=None, locking=True):
+    """
+     decompressed : True if the file was saved with `compressed=True` or you can just use '.pklz'
+    """
+    return self.load_pickle(
+      fn,
+      folder='output',
+      decompress=decompress,
+      verbose=verbose,
+      subfolder_path=subfolder_path,
+      locking=locking,
+    )
+
+
+  def load_pickle(self, fn, folder=None, decompress=False, verbose=True,
+                  subfolder_path=None, locking=True):
+    """
+     load_from: 'data', 'output', 'models'
+     decompressed : True if the file was saved with `compressed=True` or you can just use '.pklz'
+    """
+    if verbose:
+      P = self.P
+    else:
+      P = lambda x, color=None: x
+
+    lfld = self.get_target_folder(target=folder)
+
+    if lfld is None:
+      P("Loading pickle ... Assuming `fn` param ({}) is a full path".format(fn))
+      datafile = fn
+    else:
+      datafolder = lfld
+      if subfolder_path is not None:
+        datafolder = os.path.join(datafolder, subfolder_path.lstrip('/'))
+      datafile = os.path.join(datafolder, fn)
+      P("Loading pickle (locked:{}) from {}".format(locking, datafile))
+    #endif full path or not
+    data = None
+    exc = None
+    if os.path.isfile(datafile):
+      if decompress or '.pklz' in datafile:
+        if not decompress:
+          P("Loading pickle with decompress=True forced due to extension")
+        else:
+          P("Loading pickle with decompression...")
+        data = self._load_compressed_pickle(datafile)
+      else:
+        with self.managed_lock_resource(datafile, condition=locking):
+          try:
+            with open(datafile, "rb") as f:
+              data = pickle.load(f)
+          except Exception as _exc:
+            data = None
+            exc = _exc
+        # endwith conditional lock
+      #endif decompress or not
+      if data is None:
+        P("  {} load failed with error {}".format(datafile, exc), color='r')
+      else:
+        P("  Loaded: {}".format(datafile))
+      #endif data is None
+    else:
+      P("  File not found! Pickle load failed.", color='r')
+    return data
+  
+  
+  def update_pickle_from_data(self, 
+                              fn, 
+                              update_callback, 
+                              decompress=False, 
+                              verbose=False, 
+                              subfolder_path=None,
+                              force_update=False):
+    assert update_callback is not None, "update_callback must be defined!"
+    datafile = self.get_file_path(
+      fn=fn,
+      folder='data',
+      subfolder_path=subfolder_path,
+      force=True
+      )
+    if datafile is None:
+      self.P("update_pickle_from_data failed due to missing {}".format(datafile), color='error')
+      return False
+
+    with self.managed_lock_resource(datafile):
+      result = None
+      try:
+        data = self.load_pickle_from_data(
+          fn=fn,
+          decompress=decompress,
+          verbose=verbose,
+          subfolder_path=subfolder_path,
+          locking=False,
+          )
+        
+        if data is not None or force_update:
+          data = update_callback(data)
+          
+          self.save_pickle_to_data(
+            data=data, 
+            fn=fn,
+            compressed=decompress,
+            verbose=verbose,
+            subfolder_path=subfolder_path,
+            locking=False,
+            )
+          result = True
+      except Exception as e:
+        self.P("update_pickle_from_data failed: {}".format(e), color='error')
+        result = False
+    # endwith lock
+    return result
+      

naeural_client/logging/logger_mixins/process_mixin.py

@@ -0,0 +1,63 @@
+import os
+import sys
+
+class _ProcessMixin(object):
+  """
+  Mixin for process functionalities that are attached to `pye2.Logger`.
+
+  This mixin cannot be instantiated because it is built just to provide some additional
+  functionalities for `pye2.Logger`
+
+  In this mixin we can use any attribute/method of the Logger.
+  """
+
+  def __init__(self):
+    super(_ProcessMixin, self).__init__()
+    self._close_callback = None
+    return
+
+  @staticmethod
+  def runs_from_ipython():
+    try:
+      __IPYTHON__
+      return True
+    except NameError:
+      return False
+
+  @staticmethod
+  def runs_with_debugger():
+    gettrace = getattr(sys, 'gettrace', None)
+    if gettrace is None:
+      return False
+    else:
+      return not gettrace() is None
+
+  @staticmethod
+  def get_current_process_memory(mb=True):
+    import psutil
+    process = psutil.Process(os.getpid())
+    mem = process.memory_info().rss / pow(1024, 2 if mb else 3)
+    return mem
+
+  def _default_close_callback(self, sig, frame):
+    self.P("SIGINT/Ctrl-C received. Script closing")
+    if self._close_callback is None:
+      self.P(
+        "WARNING: `register_close_callback` received and will force close. Please provide a callback where you can stop the script loop and deallocate nicely.")
+      sys.exit(0)
+    else:
+      self._close_callback()
+    return
+
+  def register_close_callback(self, func=None):
+    """
+    will register a SIGINT/Ctrl-C callback or will default to the one in Logger
+    """
+    import signal
+    if func is None:
+      self.P(
+        "WARNING: register_close_callback received NO callback. The script will not behave nice. Please provide a callback where you can stop the script nicely. ")
+    self._close_callback = func
+    signal.signal(signal.SIGINT, self._default_close_callback)
+    self.P("Registered {} SIGINT/Ctrl-C callback".format('custom' if func else 'default'))
+    return

naeural_client/logging/logger_mixins/resource_size_mixin.py

@@ -0,0 +1,81 @@
+import os
+
+class _ResourceSizeMixin(object):
+  """
+  Mixin for resource size functionalities that are attached to `pye2.Logger`.
+
+  This mixin cannot be instantiated because it is built just to provide some additional
+  functionalities for `pye2.Logger`
+
+  In this mixin we can use any attribute/method of the Logger.
+  """
+
+  def __init__(self):
+    super(_ResourceSizeMixin, self).__init__()
+
+  @staticmethod
+  def compute_size_units(size):
+    units = ['B', 'KB', 'MB', 'GB', 'TB']
+    i = 0
+    unit = units[i]
+    while size >= 1024 and i < len(units) - 1:
+      i += 1
+      size /= 1024
+      unit = units[i]
+
+    size = round(size, 3)
+    return size, unit
+
+  @staticmethod
+  def cast_size_units(size,
+                      initial_unit,
+                      target_unit):
+    assert initial_unit in ['B', 'KB', 'MB', 'GB', 'TB']
+    assert target_unit in ['B', 'KB', 'MB', 'GB', 'TB']
+
+    if initial_unit == 'TB':
+      size_bytes = size * (2 ** 40)
+    elif initial_unit == 'GB':
+      size_bytes = size * (2 ** 30)
+    elif initial_unit == 'MB':
+      size_bytes = size * (2 ** 20)
+    elif initial_unit == 'KB':
+      size_bytes = size * (2 ** 10)
+    elif initial_unit == 'B':
+      size_bytes = size
+
+    if target_unit == 'TB':
+      target_size = size_bytes / (2 ** 40)
+    elif target_unit == 'GB':
+      target_size = size_bytes / (2 ** 30)
+    elif target_unit == 'MB':
+      target_size = size_bytes / (2 ** 20)
+    elif target_unit == 'KB':
+      target_size = size_bytes / (2 ** 10)
+    elif target_unit == 'B':
+      target_size = size_bytes
+
+    return target_size
+
+  def get_file_size(self, fn=None, target='data'):
+    file_path = os.path.join(self.get_target_folder(target=target), fn)
+    size = os.path.getsize(file_path)
+    return self.compute_size_units(size)
+
+  @staticmethod
+  def get_folder_size(start_path=None):
+    if start_path is None:
+      start_path = '.'
+
+    total_size = 0
+    for dirpath, dirnames, filenames in os.walk(start_path):
+      for f in filenames:
+        fp = os.path.join(dirpath, f)
+        # skip if it is symbolic link
+        if not os.path.islink(fp):
+          try:
+            total_size += os.path.getsize(fp)
+          except:
+            pass
+
+    return _ResourceSizeMixin.compute_size_units(total_size)