python-misc-utils 0.2__py3-none-any.whl

py_misc_utils/iter_file.py

@@ -0,0 +1,117 @@
+import bisect
+import os
+
+from . import alog
+from . import assert_checks as tas
+
+
+class IterFile:
+
+  def __init__(self, data_gen):
+    offset, blocks, offsets = 0, [], []
+    for block in data_gen:
+      blocks.append(block)
+      offsets.append(offset)
+      offset += len(block)
+
+    offsets.append(offset)
+
+    self._blocks = tuple(blocks)
+    self._offsets = tuple(offsets)
+    self._size = offset
+    self._offset = 0
+    self._block = None
+    self._block_offset = 0
+
+  def close(self):
+    self._blocks = None
+
+  @property
+  def closed(self):
+    return self._blocks is None
+
+  def seek(self, pos, whence=os.SEEK_SET):
+    if whence == os.SEEK_SET:
+      offset = pos
+    elif whence == os.SEEK_CUR:
+      offset = self._offset + pos
+    elif whence == os.SEEK_END:
+      offset = self._size + pos
+    else:
+      alog.xraise(ValueError, f'Invalid seek mode: {whence}')
+
+    tas.check_le(offset, self._size, msg=f'Offset out of range')
+    tas.check_ge(offset, 0, msg=f'Offset out of range')
+
+    self._offset = offset
+
+    return offset
+
+  def tell(self):
+    return self._offset
+
+  def _ensure_buffer(self, offset):
+    boffset = offset - self._block_offset
+    if self._block is None or boffset < 0 or boffset >= len(self._block):
+      pos = bisect.bisect(self._offsets, offset) - 1
+      if pos >= len(self._blocks):
+        self._block_offset = self._size
+        self._block = b''
+      else:
+        self._block_offset = self._offsets[pos]
+        self._block = memoryview(self._blocks[pos])
+
+      boffset = offset - self._block_offset
+
+    return boffset
+
+  def read(self, size=-1):
+    if size < 0:
+      rsize = self._size - self._offset
+    else:
+      rsize = min(size, self._size - self._offset)
+
+    parts = []
+    while rsize > 0:
+      boffset = self._ensure_buffer(self._offset)
+
+      csize = min(rsize, len(self._block) - boffset)
+      parts.append(self._block[boffset: boffset + csize])
+      self._offset += csize
+      rsize -= csize
+
+    return b''.join(parts)
+
+  def read1(self, size=-1):
+    return self.read(size=size)
+
+  def peek(self, size=0):
+    if size > 0:
+      boffset = self._ensure_buffer(self._offset)
+      csize = min(size, len(self._block) - boffset)
+
+      return self._block[boffset: boffset + csize].tobytes()
+
+    return b''
+
+  def flush(self):
+    pass
+
+  def readable(self):
+    return not self.closed
+
+  def seekable(self):
+    return not self.closed
+
+  def writable(self):
+    return False
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, *exc):
+    self.close()
+
+    return False
+
+

py_misc_utils/key_wrap.py

@@ -0,0 +1,46 @@
+from . import utils as ut
+
+_KW_TEMPLATE = '''
+class $CLASS:
+
+  def __init__(self, $KEY, **kwargs):
+    self.$KEY = $KEY
+    for k, v in kwargs.items():
+      setattr(self, k, v)
+
+  def __lt__(self, other):
+    return self.$KEY < other.$KEY
+
+  def __le__(self, other):
+    return self.$KEY <= other.$KEY
+
+  def __gt__(self, other):
+    return self.$KEY > other.$KEY
+
+  def __ge__(self, other):
+    return self.$KEY >= other.$KEY
+
+  def __eq__(self, other):
+    return self.$KEY == other.$KEY
+
+  def __ne__(self, other):
+    return self.$KEY != other.$KEY
+
+  def __hash__(self):
+    return hash(self.$KEY)
+
+  def __repr__(self):
+    args = self.__dict__.copy()
+    args.pop('$KEY', None)
+
+    return f'$KEY=[{self.$KEY}] : {args}'
+'''
+
+
+def key_wrap(cname, key_name):
+  replaces = dict(CLASS=cname, KEY=key_name)
+
+  results, = ut.compile(_KW_TEMPLATE, cname, vals=replaces)
+
+  return results
+

py_misc_utils/lazy_import.py

@@ -0,0 +1,25 @@
+import importlib.util
+import inspect
+import re
+import sys
+
+from . import traceback as tb
+
+
+def lazy_import(modname, package=None):
+  if package is not None and re.match(r'\.+$', package):
+    parent_frame = tb.get_frame(1)
+    parent_packages = inspect.getmodule(parent_frame).__package__.split('.')
+    package_path = parent_packages[: len(parent_packages) - len(package) + 1]
+    modname = '.'.join(package_path + [modname])
+    package = None
+
+  spec = importlib.util.find_spec(modname, package=package)
+  loader = importlib.util.LazyLoader(spec.loader)
+  spec.loader = loader
+  module = importlib.util.module_from_spec(spec)
+  sys.modules[modname] = module
+  loader.exec_module(module)
+
+  return module
+

py_misc_utils/lockfile.py

@@ -0,0 +1,164 @@
+import hashlib
+import os
+import psutil
+import time
+import yaml
+
+from . import alog
+from . import assert_checks as tas
+from . import fs_utils as fsu
+from . import obj
+from . import osfd
+from . import rnd_utils as rngu
+from . import tempdir as tmpd
+
+
+_LOCKDIR = tmpd.fastfs_dir(name='.locks')
+
+def _lockfile(name):
+  lhash = hashlib.sha1(name.encode()).hexdigest()
+
+  return os.path.join(_LOCKDIR, lhash)
+
+
+class Meta(obj.Obj):
+  pass
+
+
+_CMDLINE = list(psutil.Process().cmdline())
+_ACQUIRE_TIMEOUT = float(os.getenv('LOCKF_AQTIMEO', 0.5))
+_CHECK_TIMEOUT = float(os.getenv('LOCKF_CKTIMEO', 5.0))
+
+class LockFile:
+
+  def __init__(self, name, acquire_timeout=None, check_timeout=None):
+    acquire_timeout = _ACQUIRE_TIMEOUT if acquire_timeout is None else acquire_timeout
+    check_timeout = _CHECK_TIMEOUT if check_timeout is None else check_timeout
+
+    self._name = name
+    self._lockfile = _lockfile(name)
+    self._acquire_timeout = rngu.uniform(acquire_timeout, pct=0.2)
+    self._check_timeout = rngu.uniform(check_timeout, pct=0.2)
+
+  def _mkmeta(self):
+    tag = dict(pid=os.getpid(), cmdline=_CMDLINE, time=time.time())
+    stag = yaml.dump(tag, default_flow_style=False)
+
+    return stag.encode()
+
+  def _parse_meta(self, data):
+    if data:
+      try:
+        tag = yaml.safe_load(data.decode())
+
+        return Meta(**tag)
+      except:
+        pass
+
+  def _alive_pid(self, meta):
+    is_alive = False
+    if psutil.pid_exists(meta.pid):
+      try:
+        proc = psutil.Process(meta.pid)
+        cmdline = list(proc.cmdline())
+
+        is_alive = cmdline == meta.cmdline and proc.create_time() <= meta.time
+      except psutil.NoSuchProcess:
+        pass
+
+    if not is_alive:
+      alog.warning(f'Process {meta.pid} ({meta.cmdline}) holding lock on ' \
+                   f'{self._name} seems vanished')
+
+    return is_alive
+
+  def acquire(self, timeout=None):
+    quit_time = timeout + time.time() if timeout is not None else None
+    check_time = time.time() + self._check_timeout
+    while True:
+      try:
+        with osfd.OsFd(self._lockfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL) as fd:
+          os.write(fd, self._mkmeta())
+
+        return True
+      except OSError:
+        time.sleep(self._acquire_timeout)
+
+        now = time.time()
+        if now >= check_time:
+          check_time = now + self._check_timeout
+          meta = self._locking_meta()
+          if meta is None or not self._alive_pid(meta):
+            if self._try_force_lock(meta):
+              return True
+
+        if quit_time is not None and now >= quit_time:
+          alog.debug(f'Giving up on lock {self._name} after {timeout} seconds')
+          return False
+
+  def release(self):
+    meta = self._locking_meta()
+    if meta is not None:
+      if meta.pid == os.getpid() and meta.cmdline == _CMDLINE:
+        try:
+          os.remove(self._lockfile)
+          return True
+        except OSError:
+          pass
+      else:
+        alog.warning(f'Trying to release lock on {self._name} by pid {os.getpid()} but ' \
+                     f'it was held by {meta.pid} ({meta.cmdline})')
+    else:
+      alog.warning(f'Trying to release lock on {self._name} from pid {os.getpid()} but ' \
+                   f'no lock metadata was found')
+
+    return False
+
+  def _try_force_lock(self, meta):
+    if meta is not None:
+      alog.warning(f'Trying to override gone process {meta.pid} on {self._name}')
+
+    upath = f'{self._lockfile}.ENFORCER'
+    created = result = False
+    try:
+      with osfd.OsFd(upath, os.O_WRONLY | os.O_CREAT | os.O_EXCL) as fd:
+        created = True
+
+      with osfd.OsFd(self._lockfile, os.O_RDWR) as fd:
+        data = fsu.readall(fd)
+
+        lmeta = self._parse_meta(data)
+        if lmeta is None or (meta is not None and lmeta == meta):
+          os.lseek(fd, 0, os.SEEK_SET)
+          os.truncate(fd, 0)
+          os.write(fd, self._mkmeta())
+          result = True
+
+          alog.info(f'Successfull override on {self._name}')
+    except OSError:
+      pass
+    finally:
+      if created:
+        os.remove(upath)
+
+    return result
+
+  def _locking_meta(self):
+    try:
+      with osfd.OsFd(self._lockfile, os.O_RDONLY) as fd:
+        data = fsu.readall(fd)
+
+      return self._parse_meta(data)
+    except OSError:
+      pass
+
+  def __enter__(self):
+    self.acquire()
+
+    return self
+
+  def __exit__(self, *exc):
+    self.release()
+
+    return False
+

py_misc_utils/mem_size.py

@@ -0,0 +1,64 @@
+import array
+import sys
+import threading
+
+import numpy as np
+
+from . import core_utils as cu
+
+
+_LOCK = threading.Lock()
+_TYPES_SIZE = dict()
+
+def register(otype, sizefn):
+  with _LOCK:
+    _TYPES_SIZE[otype] = sizefn
+
+
+def _get_sizefn(otype):
+  with _LOCK:
+    return _TYPES_SIZE.get(otype)
+
+
+def std_sizefn(obj):
+  return sys.getsizeof(obj)
+
+
+_SIZE_AWARE = {
+  float,
+  int,
+  str,
+  bytes,
+  bytearray,
+  array.array,
+  np.ndarray,
+}
+
+def _get_size(obj, seen):
+  oid = id(obj)
+  if oid in seen:
+    return 0
+  seen.add(oid)
+
+  otype = type(obj)
+  if sizefn := _get_sizefn(otype):
+    size = sizefn(obj)
+  else:
+    size = sys.getsizeof(obj)
+    if otype not in _SIZE_AWARE:
+      if cu.isdict(obj):
+        size += sum(_get_size(v, seen) + _get_size(k, seen) for k, v in obj.items())
+      elif ustg := getattr(obj, 'untyped_storage', None):
+        # Handle PyTorch tensors.
+        size += sys.getsizeof(ustg())
+      elif hasattr(obj, '__dict__'):
+        size += _get_size(obj.__dict__, seen)
+      elif hasattr(obj, '__iter__'):
+        size += sum(_get_size(x, seen) for x in obj)
+
+  return size
+
+
+def get_size(obj):
+  return _get_size(obj, set())
+

py_misc_utils/mirror_from.py

@@ -0,0 +1,72 @@
+import functools
+
+
+# Copied from fsstat.
+# Used to forward calls from  a class, to a memeber of the class itself.
+# Example:
+#   @mirror_from(
+#     'stream',
+#     [
+#        'read',
+#        'seek',
+#        'write',
+#     ],
+#   )
+#   class MyStream:
+#     def __init__(self, stream):
+#       self.stream = stream
+#
+def mirror_from(origin_name, methods):
+
+  def origin_getter(method, obj):
+    origin = getattr(obj, origin_name)
+
+    return getattr(origin, method)
+
+  def wrapper(cls):
+    for method in methods:
+      wrapped_method = functools.partial(origin_getter, method)
+      setattr(cls, method, property(wrapped_method))
+
+    return cls
+
+  return wrapper
+
+
+def mirror_attributes(src, dest, attributes):
+  for attr in attributes:
+    setattr(dest, attr, getattr(src, attr))
+
+
+def _mirrored_field(name):
+  return f'_mirrored_{name}'
+
+
+def mirror_all(src, dest, excludes=(), includes=(), name=None):
+  excludes = set(excludes)
+  for attr in dir(dest):
+    if not attr.startswith('_'):
+      excludes.add(attr)
+
+  includes = set(includes)
+  mirrored = []
+  for attr in dir(src):
+    if (not attr.startswith('_') or attr in includes) and attr not in excludes:
+      setattr(dest, attr, getattr(src, attr))
+      mirrored.append(attr)
+
+  if name is not None:
+    setattr(dest, _mirrored_field(name), tuple(mirrored))
+
+  return tuple(mirrored)
+
+
+def unmirror(dest, attributes=None, name=None):
+  if name is not None:
+    mfield = _mirrored_field(name)
+    attributes = getattr(dest, mfield, None)
+    delattr(dest, mfield)
+
+  for attr in attributes or ():
+    delattr(dest, attr)
+

py_misc_utils/mmap.py

@@ -0,0 +1,16 @@
+import mmap
+import os
+
+from . import alog
+from . import osfd
+
+
+def file_view(path):
+  with osfd.OsFd(path, os.O_RDONLY) as fd:
+    mm = mmap.mmap(fd, 0, access=mmap.ACCESS_READ)
+
+  # We can close the fd, but we cannot close the mmap. When the memoryview
+  # will be garbage collected, the buffer protocol used by memoryview will
+  # decref the mmap object which will be automatically released.
+  return memoryview(mm)
+

py_misc_utils/module_utils.py

@@ -0,0 +1,196 @@
+import functools
+import importlib
+import importlib.util
+import inspect
+import os
+import sys
+
+from . import alog
+from . import assert_checks as tas
+from . import core_utils as cu
+from . import fs_utils as fsu
+from . import utils as ut
+
+
+_PYINIT = '__init__.py'
+
+
+def split_module_name(name):
+  pos = name.rfind('.')
+  if pos < 0:
+    return '', name
+
+  return name[: pos], name[pos + 1:]
+
+
+def add_sys_path(path):
+  cu.append_if_missing(sys.path, path)
+
+
+def find_module_parent(path):
+  parts = fsu.path_split(path)
+  modname = fsu.drop_ext(parts.pop(), '.py')
+
+  for i in range(len(parts)):
+    ipath = os.path.join(*parts[: i + 1], _PYINIT)
+    if os.path.isfile(ipath):
+      return ipath, parts[i + 1:] + [modname]
+
+
+def find_module(path):
+  apath = os.path.abspath(path)
+
+  for modname, module in sys.modules.items():
+    mpath = module_file(module)
+    if mpath is not None and mpath == apath:
+      return modname, module
+
+
+def install_module(modname, module):
+  xmodule = sys.modules.get(modname)
+  if xmodule is not None:
+    modfile, xmodfile = module_file(module), module_file(xmodule)
+    tas.check_eq(modfile, xmodfile,
+                 msg=f'Module "{modname}" already defined at "{xmodfile}"')
+  else:
+    alog.debug(f'Installing module at "{module_file(module)}" with "{modname}" name')
+    sys.modules[modname] = module
+
+
+def load_module(path, modname=None, install=None, add_syspath=None):
+  install = install or True
+  add_syspath = add_syspath or False
+
+  name_and_module = find_module(path)
+  if name_and_module is not None:
+    fmodname, module = name_and_module
+    alog.debug(f'Found existing module "{fmodname}" for "{path}"')
+    if install and modname is not None and modname != fmodname:
+      install_module(modname, module)
+
+    return module
+
+  apath = os.path.abspath(path)
+
+  parent = find_module_parent(apath) if os.path.basename(apath) != _PYINIT else None
+  if parent is not None:
+    init_path, mod_path = parent
+
+    parent_module = load_module(init_path,
+                                install=True,
+                                add_syspath=add_syspath)
+
+    for i in range(len(mod_path) - 1):
+      partial = mod_path[: i + 1]
+      ipath = os.path.join(os.path.dirname(parent_module.__file__), *partial, _PYINIT)
+      if os.path.isfile(ipath):
+        imodname = parent_module.__name__ + '.' + '.'.join(partial)
+        load_module(ipath, modname=imodname, install=True)
+
+    imodname = parent_module.__name__ + '.' + '.'.join(mod_path)
+    alog.debug(f'Importing sub-module "{imodname}"')
+    module = importlib.import_module(imodname)
+  else:
+    pathdir = syspath = os.path.dirname(apath)
+
+    if modname is None:
+      modname = fsu.drop_ext(os.path.basename(apath), '.py')
+      if modname == '__init__':
+        syspath, modname = os.path.split(pathdir)
+
+    alog.debug(f'Installing module "{apath}" with name "{modname}" (syspath is "{syspath}")')
+
+    if add_syspath:
+      add_sys_path(syspath)
+
+    modspec = importlib.util.spec_from_file_location(
+      modname, apath,
+      submodule_search_locations=[pathdir])
+    module = importlib.util.module_from_spec(modspec)
+
+    if install:
+      install_module(modname, module)
+
+    modspec.loader.exec_module(module)
+
+  return module
+
+
+def import_module(name_or_path,
+                  modname=None,
+                  install=None,
+                  add_syspath=None,
+                  package=None):
+  if os.path.isfile(name_or_path):
+    module = load_module(name_or_path,
+                         modname=modname,
+                         install=install,
+                         add_syspath=add_syspath)
+  else:
+    alog.debug(f'Loading module "{name_or_path}')
+    module = importlib.import_module(name_or_path, package=package)
+
+    if modname is not None and install in (True, None):
+      install_module(modname, module)
+
+  return module
+
+
+def _module_getter(dot_path, obj):
+  for name in dot_path.split('.'):
+    try:
+      obj = getattr(obj, name)
+    except AttributeError:
+      if inspect.ismodule(obj):
+        obj = importlib.import_module(obj.__name__ + '.' + name)
+      else:
+        raise
+
+  return obj
+
+
+def module_getter(dot_path):
+  return functools.partial(_module_getter, dot_path)
+
+
+def import_module_names(modname, names=None):
+  if names is None:
+    npos = modname.find('.')
+    tas.check_gt(npos, 0)
+    names = [modname[npos + 1:]]
+    modname = modname[: npos]
+  else:
+    names = ut.expand_strings(names)
+
+  module = importlib.import_module(modname)
+
+  return tuple(module_getter(name)(module) for name in names)
+
+
+def rel_import_module(path, ppath,
+                      modname=None,
+                      install=None,
+                      add_syspath=None):
+  ipath = os.path.abspath(os.path.join(os.path.dirname(ppath), f'{path}.py'))
+
+  return import_module(ipath, modname=modname, install=install, add_syspath=add_syspath)
+
+
+def maybe_import(modname, package=None):
+  try:
+    return importlib.import_module(modname, package=package)
+  except:
+    alog.spam(f'Unable to load module "{modname}" in package "{package}"')
+
+
+def module_file(module):
+  return getattr(module, '__file__', None)
+
+
+def clone_module(modname):
+  specs = importlib.util.find_spec(modname)
+  module = importlib.util.module_from_spec(specs)
+  specs.loader.exec_module(module)
+
+  return module
+