python-misc-utils 0.2__py3-none-any.whl

py_misc_utils/cmd.py

@@ -0,0 +1,97 @@
+import functools
+import os
+import shlex
+import signal
+import string
+import subprocess
+import sys
+
+from . import alog
+from . import fs_utils as fsu
+from . import signal as sgn
+from . import template_replace as tr
+
+
+class _SigHandler:
+
+  def __init__(self, proc, logfd=None):
+    self._proc = proc
+    self._logfd = logfd
+    self._sent = set()
+
+  def __call__(self, sig, frame):
+    if sig not in self._sent:
+      self._sent.add(sig)
+      alog.async_log(alog.WARNING,
+                     f'{signal.strsignal(sig)} received. Forwarding it to running ' \
+                     f'child {self._proc.pid} ...',
+                     file=self._logfd)
+
+      self._proc.send_signal(sig)
+
+    return sgn.HANDLED
+
+
+class _Writer:
+
+  def __init__(self, fd):
+    self.fd = fd
+    self._is_binary = fsu.is_binary(fd)
+
+  def write(self, data):
+    if self._is_binary:
+      if isinstance(data, str):
+        data = data.encode()
+    elif not isinstance(data, str):
+      data = data.decode()
+
+    self.fd.write(data)
+    self.fd.flush()
+
+
+class _Reader:
+
+  def __init__(self, fd):
+    self._read = getattr(fd, 'read1', None)
+    if self._read is None:
+      self._read = getattr(fd, 'readline', None)
+
+  def read(self):
+    return self._read()
+
+
+def _lookup(tmpl_envs, key, defval=None):
+  for env in tmpl_envs:
+    value = env.get(key)
+    if value is not None:
+      return value
+
+  alog.xraise(KeyError, f'Unable to lookup "{key}" while substituting command ' \
+              f'line arguments')
+
+
+def run(cmd, outfd=None, tmpl_envs=None, **kwargs):
+  if isinstance(cmd, str):
+    tmpl_envs = tmpl_envs or (os.environ,)
+    cmd = tr.template_replace(cmd, lookup_fn=functools.partial(_lookup, tmpl_envs))
+    cmd = shlex.split(cmd)
+
+  alog.debug(f'Running: {cmd}')
+
+  proc = subprocess.Popen(cmd,
+                          stdout=subprocess.PIPE,
+                          stderr=subprocess.STDOUT,
+                          **kwargs)
+
+  reader = _Reader(proc.stdout)
+  writer = _Writer(outfd or sys.stdout)
+  with sgn.Signals('INT, TERM', _SigHandler(proc, logfd=writer.fd)):
+    while True:
+      data = reader.read()
+      if data:
+        writer.write(data)
+      elif proc.poll() is not None:
+        break
+
+  return proc.returncode
+

py_misc_utils/compression.py

@@ -0,0 +1,116 @@
+import bz2
+import collections
+import gzip
+import contextlib
+import lzma
+import os
+import shutil
+
+from . import gfs
+
+
+def fgzip(src, dest):
+  with gfs.open(src, mode='rb') as infd, gfs.open(dest, mode='wb') as outfd:
+    with gzip.open(outfd, mode='wb') as zfd:
+      shutil.copyfileobj(infd, zfd)
+
+
+def fgunzip(src, dest):
+  with gfs.open(src, mode='rb') as infd, gfs.open(dest, mode='wb') as outfd:
+    with gzip.open(infd, mode='rb') as zfd:
+      shutil.copyfileobj(zfd, outfd)
+
+
+def fbzip2(src, dest):
+  with gfs.open(src, mode='rb') as infd, gfs.open(dest, mode='wb') as outfd:
+    with bz2.open(outfd, mode='wb') as zfd:
+      shutil.copyfileobj(infd, zfd)
+
+
+def fbunzip2(src, dest):
+  with gfs.open(src, mode='rb') as infd, gfs.open(dest, mode='wb') as outfd:
+    with bz2.open(infd, mode='rb') as zfd:
+      shutil.copyfileobj(zfd, outfd)
+
+
+def fxzip(src, dest):
+  with gfs.open(src, mode='rb') as infd, gfs.open(dest, mode='wb') as outfd:
+    with lzma.open(outfd, mode='wb') as zfd:
+      shutil.copyfileobj(infd, zfd)
+
+
+def fxunzip(src, dest):
+  with gfs.open(src, mode='rb') as infd, gfs.open(dest, mode='wb') as outfd:
+    with lzma.open(infd, mode='rb') as zfd:
+      shutil.copyfileobj(zfd, outfd)
+
+
+_Processor = collections.namedtuple('Processor', 'processor, module')
+
+_COMPRESSORS = {
+  '.bz2': _Processor(processor=fbzip2, module=bz2),
+  '.bzip': _Processor(processor=fbzip2, module=bz2),
+  '.gz': _Processor(processor=fgzip, module=gzip),
+  '.xz': _Processor(processor=fxzip, module=lzma),
+}
+
+def compressor(ext):
+  return _COMPRESSORS.get(ext)
+
+
+_DECOMPRESSORS = {
+  '.bz2': _Processor(processor=fbunzip2, module=bz2),
+  '.bzip': _Processor(processor=fbunzip2, module=bz2),
+  '.gz': _Processor(processor=fgunzip, module=gzip),
+  '.xz': _Processor(processor=fxunzip, module=lzma),
+}
+
+def decompressor(ext):
+  return _DECOMPRESSORS.get(ext)
+
+
+def compress(src, dest):
+  _, ext = os.path.splitext(dest)
+  comp = compressor(ext)
+  if comp is not None:
+    comp.processor(src, dest)
+  else:
+    shutil.copyfile(src, dest)
+
+
+def decompress(src, dest):
+  _, ext = os.path.splitext(src)
+  decomp = decompressor(ext)
+  if decomp is not None:
+    decomp.processor(src, dest)
+  else:
+    shutil.copyfile(src, dest)
+
+
+@contextlib.contextmanager
+def dopen(path, mode='r', **kwargs):
+  _, ext = os.path.splitext(path)
+
+  decomp = decompressor(ext)
+  if decomp is None:
+    with gfs.open(path, mode=mode, **kwargs) as fd:
+      yield fd
+  else:
+    with gfs.open(path, mode='rb', **kwargs) as fd:
+      with decomp.module.open(fd, mode=mode) as zfd:
+        yield zfd
+
+
+@contextlib.contextmanager
+def copen(path, mode='w', **kwargs):
+  _, ext = os.path.splitext(path)
+
+  comp = compressor(ext)
+  if comp is None:
+    with gfs.open(path, mode=mode, **kwargs) as fd:
+      yield fd
+  else:
+    with gfs.open(path, mode='wb', **kwargs) as fd:
+      with comp.module.open(fd, mode=mode) as zfd:
+        yield zfd
+

py_misc_utils/cond_waiter.py

@@ -0,0 +1,13 @@
+from . import abs_timeout as abst
+from . import timegen as tg
+
+
+class CondWaiter:
+
+  def __init__(self, timeout=None, timegen=None):
+    self._timegen = tg.TimeGen() if timegen is None else timegen
+    self._timeo = abst.AbsTimeout(timeout, timefn=self._timegen.now)
+
+  def wait(self, cond):
+    return self._timegen.wait(cond, timeout=self._timeo.get())
+

py_misc_utils/context_base.py

@@ -0,0 +1,18 @@
+from . import utils as ut
+
+
+class ContextBase:
+
+  def __init__(self, default_ctx):
+    self._ctx = ut.make_object(**default_ctx)
+
+  def _new_ctx(self, **kwargs):
+    ctx = self._ctx
+    args = ctx.__dict__.copy()
+    args.update(**kwargs)
+
+    nctx = ut.make_object(**args)
+    self._ctx = nctx
+
+    return nctx
+

py_misc_utils/context_managers.py

@@ -0,0 +1,67 @@
+import contextlib
+
+
+class CtxManager:
+
+  def __init__(self, infn, outfn):
+    self._infn = infn
+    self._outfn = outfn
+
+  def __enter__(self):
+    return self._infn()
+
+  def __exit__(self, *exc):
+    return self._outfn(*exc)
+
+
+class Wrapper:
+
+  def __init__(self, v, attr=None, finfn=None):
+    self.v = v
+    self._finfn = finfn or getattr(v, attr or 'close')
+
+  def __enter__(self):
+    return self
+
+  def detach(self):
+    self._finfn = None
+
+    return self.v
+
+  def __exit__(self, *exc):
+    if self._finfn is not None:
+      self._finfn()
+
+    return False
+
+
+class Pack(contextlib.ExitStack):
+
+  def __init__(self, *wrap_ctxs, wrap_obj=None, wrap_idx=None):
+    super().__init__()
+    self._wrap_ctxs = wrap_ctxs
+    self._wrap_obj = wrap_obj
+    self._wrap_idx = wrap_idx
+
+  def __enter__(self):
+    super().__enter__()
+
+    try:
+      wres = [self.enter_context(ctx) for ctx in self._wrap_ctxs]
+    except:
+      self.close()
+      raise
+
+    if self._wrap_obj is not None:
+      return self._wrap_obj
+
+    return wres[-1] if self._wrap_idx is None else wres[self._wrap_idx]
+
+
+def detach(obj):
+  return obj.detach() if isinstance(obj, CtxManagerProxy) else obj
+
+
+def cond(value, ctxfn, *args, **kwargs):
+  return ctxfn(*args, **kwargs) if value else contextlib.nullcontext()
+