python-misc-utils 0.2__py3-none-any.whl

py_misc_utils/executor.py

@@ -0,0 +1,239 @@
+import collections
+import heapq
+import os
+import threading
+import time
+import weakref
+
+from . import alog
+from . import cond_waiter as cwait
+from . import utils as ut
+
+
+_ExceptionWrapper = collections.namedtuple('ExceptionWrapper', 'exception')
+
+
+class Task:
+
+  def __init__(self, fn, args=None, kwargs=None, aresult=None):
+    self._fn = fn
+    self._args = args or ()
+    self._kwargs = kwargs or dict()
+    self._aresult = aresult
+
+  def __call__(self):
+    try:
+      fnres = self._fn(*self._args, **self._kwargs)
+    except Exception as ex:
+      alog.exception(ex, exmsg=f'Exception while running task')
+      fnres = _ExceptionWrapper(exception=ex)
+
+    if self._aresult is not None:
+      self._aresult.set(fnres)
+
+
+class _Void:
+  pass
+
+VOID = _Void()
+
+class AsyncResult:
+
+  def __init__(self):
+    self._cond = threading.Condition(lock=threading.Lock())
+    self._result = VOID
+
+  def set(self, result):
+    with self._cond:
+      self._result = result
+      self._cond.notify_all()
+
+  def wait(self, timeout=None):
+    with self._cond:
+      # No need for a loop here, as the condition is signaled only when result
+      # is set by the producer.
+      if self._result is VOID:
+        self._cond.wait(timeout=timeout)
+
+      if isinstance(self._result, _ExceptionWrapper):
+        raise self._result.exception
+
+      return self._result
+
+
+class Queue:
+
+  def __init__(self):
+    self._lock = threading.Lock()
+    self._cond = threading.Condition(lock=self._lock)
+    self._queue = collections.deque()
+    self._stopped = 0
+
+  def put(self, task):
+    with self._lock:
+      self._queue.append(task)
+      self._cond.notify()
+
+      return len(self._queue)
+
+  def get(self, timeout=None):
+    with self._lock:
+      while True:
+        # Even in case of stopped queue, always return pending items if available.
+        if self._queue:
+          return self._queue.popleft()
+        if self._stopped > 0 or not self._cond.wait(timeout=timeout):
+          break
+
+  def start(self):
+    with self._lock:
+      self._stopped -= 1
+
+  def stop(self):
+    with self._lock:
+      self._stopped += 1
+      self._cond.notify_all()
+
+  def __len__(self):
+    with self._lock:
+      return len(self._queue)
+
+
+class _Worker:
+
+  def __init__(self, executor, queue, name, idle_timeout=None):
+    self.executor = executor
+    self.queue = queue
+    self.idle_timeout = idle_timeout
+    self.thread = threading.Thread(target=self._run, name=name, daemon=True)
+    self.thread.start()
+
+  def _unregister(self):
+    executor = self.executor()
+    if executor is not None:
+      executor._unregister_worker(self)
+
+  def _run(self):
+    while True:
+      task = self.queue.get(timeout=self.idle_timeout)
+
+      if task is None:
+        break
+
+      task()
+      del task
+
+    self._unregister()
+
+  @property
+  def ident(self):
+    return self.thread.ident
+
+  def join(self):
+    self.thread.join()
+
+
+def _compute_num_threads(min_threads, max_threads):
+  if max_threads is None:
+    max_threads = max(8, int(os.cpu_count() * 1.5))
+  if min_threads is None:
+    min_threads = max(2, max_threads // 4)
+
+  return min_threads, max_threads
+
+
+class Executor:
+
+  def __init__(self, max_threads=None, min_threads=None, name_prefix=None,
+               idle_timeout=None):
+    self._min_threads, self._max_threads = _compute_num_threads(min_threads, max_threads)
+    self._name_prefix = name_prefix or 'Executor'
+    self._idle_timeout = idle_timeout or ut.getenv('EXECUTOR_IDLE_TIMEOUT', dtype=int, defval=5)
+    self._lock = threading.Lock()
+    self._queue = Queue()
+    self._workers = dict()
+    self._thread_counter = 0
+    self._idle_cond = threading.Condition(lock=self._lock)
+
+  def _unregister_worker(self, worker):
+    alog.spam(f'Unregistering worker thread {worker.ident}')
+    with self._lock:
+      self._workers.pop(worker.ident, None)
+      if not self._workers:
+        self._idle_cond.notify_all()
+
+  def _new_name(self):
+    self._thread_counter += 1
+
+    return f'{self._name_prefix}-{self._thread_counter}'
+
+  def _maybe_add_worker(self, queued):
+    num_threads = len(self._workers)
+    if ((queued > 1 and num_threads < self._max_threads) or num_threads < self._min_threads):
+      # Up to min_threads the workers should never quit, so they get None as
+      # timeout, while the one after that will get _idle_timeout which will
+      # make them quit if no task is fetched within such timeout.
+      idle_timeout = self._idle_timeout if num_threads > self._min_threads else None
+
+      worker = _Worker(weakref.ref(self), self._queue, self._new_name(),
+                       idle_timeout=idle_timeout)
+
+      self._workers[worker.ident] = worker
+
+      alog.spam(f'New thread #{num_threads} with ID {worker.ident}')
+
+  def _submit_task(self, task):
+    with self._lock:
+      queued = self._queue.put(task)
+      self._maybe_add_worker(queued)
+
+  def submit(self, fn, *args, **kwargs):
+    self._submit_task(Task(fn, args=args, kwargs=kwargs))
+
+  def submit_result(self, fn, *args, **kwargs):
+    aresult = AsyncResult()
+
+    self._submit_task(Task(fn, args=args, kwargs=kwargs, aresult=aresult))
+
+    return aresult
+
+  def shutdown(self):
+    alog.debug0(f'Stopping executor')
+    self._queue.stop()
+    with self._lock:
+      alog.debug0(f'Waiting executor workers exit')
+      while self._workers:
+        self._idle_cond.wait()
+
+  def wait_for_idle(self, timeout=None, timegen=None, waiter=None):
+    alog.debug0(f'Waiting for idle ...')
+
+    waiter = waiter or cwait.CondWaiter(timeout=timeout, timegen=timegen)
+    self._queue.stop()
+    try:
+      with self._lock:
+        while self._workers:
+          if not waiter.wait(self._idle_cond):
+            return False
+    finally:
+      self._queue.start()
+      alog.debug0(f'Waiting for idle ... done')
+
+    return True
+
+
+_LOCK = threading.Lock()
+_EXECUTOR = None
+
+def common_executor():
+  global _EXECUTOR
+
+  with _LOCK:
+    if _EXECUTOR is None:
+      _EXECUTOR = Executor(
+        max_threads=ut.getenv('EXECUTOR_WORKERS', dtype=int),
+        name_prefix=os.getenv('EXECUTOR_NAME', 'CommonExecutor'),
+      )
+
+    return _EXECUTOR
+

py_misc_utils/file_overwrite.py

@@ -0,0 +1,29 @@
+from . import gfs
+
+
+class FileOverwrite:
+
+  def __init__(self, dest, mode='w', **kwargs):
+    self._dest = dest
+    self._path = gfs.path_of(dest)
+    self._mode = mode
+    self._kwargs = kwargs
+    self._tmpfile = None
+
+  def __enter__(self):
+    if self._path is not None:
+      self._tmpfile = gfs.TempFile(nspath=self._path, mode=self._mode, **self._kwargs)
+
+      return self._tmpfile.open()
+    else:
+      return self._dest
+
+  def __exit__(self, *exc):
+    if self._tmpfile is not None:
+      try:
+        self._tmpfile.replace(self._path)
+      finally:
+        self._tmpfile.close()
+
+    return False
+

py_misc_utils/fin_wrap.py

@@ -0,0 +1,77 @@
+import weakref
+
+from . import assert_checks as tas
+
+
+def _finalizer_name(name):
+  return f'__{name}_finalizer'
+
+
+class fin_wrap:
+
+  def __init__(self, parent, name, obj, finfn=None, cleanup=False):
+    setattr(parent, name, obj)
+    fname = _finalizer_name(name)
+    if obj is not None:
+      tas.check_is_not_none(finfn, msg=f'Missing finalize function argument')
+
+      setattr(parent, fname, self)
+      self._finalizer = weakref.finalize(self, finfn)
+    else:
+      fwrap = getattr(parent, fname, None)
+      if fwrap is not None:
+        delattr(parent, fname)
+        if cleanup:
+          fwrap._finalizer()
+        else:
+          fwrap._finalizer.detach()
+
+
+class _Wrapper:
+  pass
+
+def fin_wrap_np(obj, finfn, name='v'):
+  wrapper = _Wrapper()
+  fin_wrap(wrapper, name, obj, finfn=finfn)
+
+  return wrapper
+
+
+_OBJ_NAME = 'wrapped_obj'
+_RESERVED_NAMES = {_OBJ_NAME, _finalizer_name(_OBJ_NAME)}
+
+class FinWrapper:
+
+  def __init__(self, obj, finfn):
+    fin_wrap(self, _OBJ_NAME, obj, finfn=finfn)
+
+  def __getattribute__(self, name):
+    pd = super().__getattribute__('__dict__')
+    obj = pd[_OBJ_NAME]
+
+    return getattr(obj, name)
+
+  def __getattr__(self, name):
+    pd = super().__getattribute__('__dict__')
+    obj = pd[_OBJ_NAME]
+
+    return getattr(obj, name)
+
+  def __setattr__(self, name, value):
+    pd = super().__getattribute__('__dict__')
+    if name in _RESERVED_NAMES:
+      pd[name] = value
+    else:
+      obj = pd[_OBJ_NAME]
+
+      setattr(obj, name, value)
+
+  def __delattr__(self, name):
+    pd = super().__getattribute__('__dict__')
+    if name in _RESERVED_NAMES:
+      pd.pop(name)
+    else:
+      obj = pd[_OBJ_NAME]
+
+      delattr(obj, name)
+

py_misc_utils/fp_utils.py

@@ -0,0 +1,47 @@
+import math
+
+
+def exp_bias(nx):
+  return (1 << (nx - 1)) - 1
+
+
+def real_to_bits(v, nx, nm):
+  xm, xe = math.frexp(math.fabs(v))
+
+  e = (xe + exp_bias(nx) - 1) if xm != 0 else 0
+  m = int(xm * (1 << (nm + 1)))
+
+  return 1 if v < 0 else 0, e, m
+
+
+def pack_bits(s, e, m, nx, nm):
+  return (s << (nx + nm)) | (e << nm) | m
+
+
+def real_to_packedbits(v, nx, nm):
+  s, e, m = real_to_bits(v, nx, nm)
+
+  return pack_bits(s, e, m, nx, nm)
+
+
+def _bits(v, pos, n):
+  return (v >> pos) & ((1 << n) - 1)
+
+
+def packedbits_to_real(v, nx, nm):
+  s, e, m = _bits(v, nx + nm, 1), _bits(v, nm, nx), _bits(v, 0, nm)
+
+  if e == 0 and m == 0:
+    return 0.0
+  if e == ((1 << nx) - 1):
+    return math.inf if m == 0 else math.nan
+
+  xm = float(m | (1 << nm))
+  re = e - exp_bias(nx) - nm - 1
+  if re >= 0:
+    rm = xm * (1 << re)
+  else:
+    rm = xm / (1 << (-re))
+
+  return rm if s == 0 else -rm
+

py_misc_utils/fs/file_fs.py

@@ -0,0 +1,127 @@
+import hashlib
+import os
+import shutil
+
+from .. import alog
+from .. import assert_checks as tas
+from .. import cached_file as chf
+from .. import fs_base as fsb
+from .. import fs_utils as fsu
+from .. import osfd
+
+
+class FileReader:
+
+  def __init__(self, path):
+    self._path = path
+
+  @classmethod
+  def tag(cls, sres):
+    return chf.make_tag(size=sres.st_size, mtime=sres.st_mtime)
+
+  def support_blocks(self):
+    return True
+
+  def read_block(self, bpath, offset, size):
+    if offset != chf.CachedBlockFile.WHOLE_OFFSET:
+      with (osfd.OsFd(self._path, os.O_RDONLY) as rfd,
+            osfd.OsFd(bpath, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, mode=0o440) as wfd):
+        if os.lseek(rfd, offset, os.SEEK_SET) != offset:
+          alog.xraise(RuntimeError, f'Unable to seek {self._path} at offset {offset}')
+        data = os.read(rfd, size)
+        os.write(wfd, data)
+
+        return len(data)
+    else:
+      with open(self._path, mode='rb') as rfd:
+        bfd = os.open(bpath, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, mode=0o440)
+        with open(bfd, mode='wb') as wfd:
+          shutil.copyfileobj(rfd, wfd)
+
+      return os.path.getsize(bpath)
+
+
+class FileFs(fsb.FsBase):
+
+  ID = 'file'
+  IDS = (ID,)
+
+  def __init__(self, cache_iface=None, **kwargs):
+    super().__init__(cache_iface=cache_iface, **kwargs)
+
+  def norm_url(self, url):
+    return fsu.normpath(url)
+
+  def _create_tag(self, sres):
+    return FileReader.tag(sres)
+
+  def stat(self, url):
+    sres = os.stat(url)
+
+    return fsb.DirEntry(name=os.path.basename(url),
+                        path=url,
+                        etag=self._create_tag(sres),
+                        st_mode=sres.st_mode,
+                        st_size=sres.st_size,
+                        st_ctime=sres.st_ctime,
+                        st_mtime=sres.st_mtime)
+
+  def open(self, url, mode, **kwargs):
+    return open(url, mode=mode)
+
+  def remove(self, url):
+    os.remove(url)
+
+  def rename(self, src_url, dest_url):
+    os.rename(src_url, dest_url)
+
+  def replace(self, src_url, dest_url):
+    os.replace(src_url, dest_url)
+
+  def mkdir(self, url, mode=None):
+    os.mkdir(url, mode=mode or 0o777)
+
+  def makedirs(self, url, mode=None, exist_ok=None):
+    os.makedirs(url, mode=mode or 0o777, exist_ok=exist_ok or False)
+
+  def rmdir(self, url):
+    os.rmdir(url)
+
+  def rmtree(self, url, ignore_errors=None):
+    fsu.safe_rmtree(url, ignore_errors=ignore_errors or False)
+
+  def list(self, url):
+    with os.scandir(url) as sdit:
+      for de in sdit:
+        sres = de.stat()
+
+        yield fsb.DirEntry(name=de.name,
+                           path=os.path.join(url, de.name),
+                           etag=self._create_tag(sres),
+                           st_mode=sres.st_mode,
+                           st_size=sres.st_size,
+                           st_ctime=sres.st_ctime,
+                           st_mtime=sres.st_mtime)
+
+  def put_file(self, url, data_gen):
+    with open(url, mode='wb') as fd:
+      for data in data_gen:
+        fd.write(data)
+
+  def get_file(self, url):
+    with open(url, mode='rb') as fd:
+      for data in fsu.enum_chunks(fd):
+        yield data
+
+  def as_local(self, url, **kwargs):
+    return url
+
+  def link(self, src_url, dest_url):
+    os.link(src_url, dest_url)
+
+  def symlink(self, src_url, dest_url):
+    os.symlink(src_url, dest_url)
+
+
+FILE_SYSTEMS = (FileFs,)
+