python-misc-utils 0.2__py3-none-any.whl

py_misc_utils/http_server.py

@@ -0,0 +1,196 @@
+import argparse
+import base64
+import copy
+import http.server
+import os
+
+
+def _sanitize_path(path):
+  if '..' not in path and not path.endswith('/'):
+    return path
+
+
+def _read_stream(headers, stream, chunked_headers, chunk_size=None):
+  length = headers['Content-Length']
+  encoding = headers['Transfer-Encoding']
+  if length is not None:
+    length = int(length)
+    chunk_size = chunk_size or 16 * 1024**2
+    while length > 0:
+      rsize = min(length, chunk_size)
+
+      yield stream.read(rsize)
+
+      length -= rsize
+
+  elif encoding == 'chunked':
+    while True:
+      size = int(stream.readline().strip(), 16)
+      if size == 0:
+        break
+
+      yield stream.read(size)
+
+      while True:
+        ln = stream.readline().strip()
+        if not ln:
+          break
+        parts = ln.split(':', maxsplit=1)
+        if len(parts) == 2:
+          chunked_headers[part[0].strip()] = part[1].strip()
+
+  else:
+    raise RuntimeError(f'Unable to read data: {headers}')
+
+
+class HandlerException(Exception):
+
+  def __init__(self, code, message, explain):
+    super().__init__()
+    self.code = code
+    self.message = message
+    self.explain = explain
+
+
+class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
+
+  def _check_authorization(self, op, path):
+    hauth = self.headers['Authorization']
+    if hauth is None:
+      raise HandlerException(401, 'Unauthorized', f'Action not allowed on "{self.path}"\n')
+    if self._args.auth is None:
+      self.log_message(f'Client request needs authorization but server not configured ' \
+                       f'for it: {op} {self.path}')
+      raise HandlerException(403, 'Forbidden', f'Action not allowed on "{self.path}"\n')
+
+    try:
+      auth_type, auth_config = [x.strip() for x in hauth.split(' ', maxsplit=1)]
+
+      auth_type = auth_type.lower()
+      if auth_type == 'bearer':
+        token = base64.b64decode(auth_config).decode()
+        if token != self._args.auth:
+          self.log_message(f'Client authorization invalid: {op} {self.path} {token}')
+          raise HandlerException(403, 'Forbidden', f'Action not allowed on "{self.path}"\n')
+      elif auth_type == 'basic':
+        creds = base64.b64decode(auth_config).decode()
+        user, passwd = creds.split(':', maxsplit=1)
+
+        # Very crude auth!
+        if passwd != self._args.auth:
+          self.log_message(f'Client authorization invalid: {op} {self.path} {user}:{passwd}')
+          raise HandlerException(403, 'Forbidden', f'Action not allowed on "{self.path}"\n')
+      else:
+        raise HandlerException(403, 'Forbidden', f'Authorization not supported: {hauth}\n')
+    except HandlerException:
+      raise
+    except Exception as ex:
+      raise HandlerException(500, 'Internal Server Error', f'Internal error: {ex}\n')
+
+  def do_OPTIONS(self):
+    self.send_response(200, 'OK')
+    self.send_header('Allow', 'GET,POST,PUT,OPTIONS,HEAD,DELETE')
+    self.send_header('Content-Length', '0')
+    self.end_headers()
+
+  def do_DELETE(self):
+    path = _sanitize_path(self.translate_path(self.path))
+    if path is None:
+      self.send_error(403,
+                      message='Forbidden',
+                      explain=f'Action not allowed on "{self.path}"\n')
+    else:
+      try:
+        self._check_authorization('DELETE', path)
+
+        os.remove(path)
+
+        self.send_response(200, 'OK')
+        self.send_header('Content-Length', '0')
+        self.end_headers()
+      except HandlerException as ex:
+        self.send_error(ex.code,
+                        message=ex.message,
+                        explain=ex.explain)
+      except OSError as ex:
+        self.send_error(500,
+                        message='Internal Server Error',
+                        explain=f'Internal error: {ex}\n')
+
+  def do_PUT(self):
+    path = _sanitize_path(self.translate_path(self.path))
+    if path is None:
+      self.send_error(403,
+                      message='Forbidden',
+                      explain=f'Action not allowed on "{self.path}"\n')
+    else:
+      try:
+        self._check_authorization('PUT', path)
+
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+
+        chunked_headers = dict()
+        with open(path, 'wb') as f:
+          for data in _read_stream(self.headers, self.rfile, chunked_headers):
+            f.write(data)
+
+        self.send_response(201, 'Created')
+        self.end_headers()
+      except HandlerException as ex:
+        self.send_error(ex.code,
+                        message=ex.message,
+                        explain=ex.explain)
+      except Exception as ex:
+        self.send_error(500,
+                        message='Internal Server Error',
+                        explain=f'Internal error: {ex}\n')
+
+
+class ClassWrapper:
+
+  PRIVATE_ATTRIBUTES = {'_cls', '_kwargs'}
+
+  def __init__(self, cls, **kwargs):
+    self._cls = cls
+    self._kwargs = kwargs
+
+  def __setattr__(self, name, value):
+    sdict = super().__getattribute__('__dict__')
+    sdict[name] = value
+    if name not in ClassWrapper.PRIVATE_ATTRIBUTES:
+      sdict['_kwargs'][name] = value
+
+  def __call__(self, *args, **kwargs):
+    obj = self._cls.__new__(self._cls, *args, **kwargs)
+    # Add ctor supplied data and data planted by the http.server.test() API
+    # (talking about bad API design).
+    obj.__dict__.update(self._kwargs)
+    obj.__init__(*args, **kwargs)
+
+    return obj
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser(description='Simple HTTP Server For Testing')
+  parser.add_argument('--bind', default='0.0.0.0',
+                      help='Specify alternate bind address')
+  parser.add_argument('--port', type=int, default=8000,
+                      help='Specify alternate port')
+  parser.add_argument('--protocol', default='HTTP/1.1',
+                      help='Conform to this HTTP version')
+  parser.add_argument('--auth',
+                      help='The authentication to be used when performing HTTP write operations')
+
+  args = parser.parse_args()
+
+  # Wrap the handler and server classes, to allow adding the args (and also because
+  # the http.server.test() API plants data inside the class global namespace!
+  req_handler = ClassWrapper(HTTPRequestHandler, _args=args)
+  server_class = ClassWrapper(http.server.ThreadingHTTPServer)
+
+  http.server.test(HandlerClass=req_handler,
+                   ServerClass=server_class,
+                   port=args.port,
+                   bind=args.bind,
+                   protocol=args.protocol)
+

py_misc_utils/http_utils.py

@@ -0,0 +1,143 @@
+import collections
+import os
+import re
+import requests
+import time
+
+
+ACCEPT_RANGES = 'Accept-Ranges'
+AUTHORIZATION = 'Authorization'
+CONTENT_LENGTH = 'Content-Length'
+CONTENT_TYPE = 'Content-Type'
+CONTENT_ENCODING = 'Content-Encoding'
+LAST_MODIFIED = 'Last-Modified'
+XLINKED_SIZE = 'X-Linked-Size'
+XLINKED_ETAG = 'X-Linked-ETag'
+ETAG = 'ETag'
+RANGE = 'Range'
+CONTENT_RANGE = 'Content-Range'
+
+
+def support_ranges(headers):
+  return 'bytes' in headers.get(ACCEPT_RANGES, '')
+
+
+def content_length(headers, defval=None):
+  if (length := headers.get(XLINKED_SIZE)) is not None:
+    return int(length)
+  if (length := headers.get(CONTENT_LENGTH)) is not None:
+    return int(length)
+
+  return defval
+
+
+def etag(headers, defval=None):
+  if (etag_value := headers.get(XLINKED_ETAG)) is not None:
+    return etag_value.strip('"\'')
+  if (etag_value := headers.get(ETAG)) is not None:
+    return etag_value.strip('"\'')
+
+  return defval
+
+
+def last_modified(headers, defval=None):
+  if (mtime := headers.get(LAST_MODIFIED)) is not None:
+    return date_to_epoch(mtime)
+
+  return defval
+
+
+def add_range(headers, start, end):
+  headers[RANGE] = f'bytes={start}-{end - 1}'
+
+  return headers
+
+
+Range = collections.namedtuple('Range', 'start, stop, length')
+
+def range(headers):
+  hrange = headers.get(CONTENT_RANGE)
+  if hrange is None:
+    if (length := content_length(headers)) is not None:
+      return Range(start=0, stop=length - 1, length=length)
+  else:
+    m = re.match(r'bytes\s+(\d+)\-(\d+)(/(\d+))?', hrange)
+    if m:
+      hlength = m.group(4)
+      length = int(hlength) if hlength else None
+
+      return Range(start=int(m.group(1)), stop=int(m.group(2)), length=length)
+
+
+def range_data(start, stop, headers, data):
+  hrange = range(headers)
+  if hrange is not None:
+    dstart = start - hrange.start
+    size = min(stop - start, hrange.stop - hrange.start) + 1
+    dstop = dstart + size - 1
+
+    if dstart != start or dstop != stop:
+      return memoryview(data)[dstart: dstop + 1]
+
+  return data
+
+
+_HTTP_DATE_FMT ='%a, %d %b %Y %H:%M:%S %Z'
+
+def date_to_epoch(http_date):
+  htime = time.strptime(http_date, _HTTP_DATE_FMT)
+
+  return time.mktime(htime)
+
+
+def epoch_to_date(epoch_time=None):
+  return time.strftime(_HTTP_DATE_FMT, time.gmtime(epoch_time or time.time()))
+
+
+def filter_request_args(kwargs, key=None):
+  req_kwargs = kwargs.get(key or 'requests')
+  if req_kwargs is None:
+    req_kwargs = dict()
+    for arg in ('headers', 'timeout', 'auth', 'cookies', 'allow_redirects',
+                'proxies', 'verify', 'cert'):
+      argv = kwargs.get(arg)
+      if argv is not None:
+        req_kwargs[arg] = argv
+
+  return req_kwargs
+
+
+def info(url, mod=None, headers=None, **kwargs):
+  mod = mod or requests
+  req_headers = headers.copy() if headers else dict()
+
+  add_range(req_headers, 0, 1024)
+
+  try:
+    resp = mod.get(url, headers=req_headers, **kwargs)
+    resp.raise_for_status()
+
+    hrange = range(resp.headers)
+    if hrange is not None and hrange.length is not None:
+      resp.headers[CONTENT_LENGTH] = hrange.length
+      resp.headers[ACCEPT_RANGES] = 'bytes'
+    else:
+      resp = None
+  except requests.exceptions.HTTPError:
+    resp = None
+
+  if resp is None:
+    resp = mod.head(url, headers=headers, **kwargs)
+    resp.raise_for_status()
+
+  return resp
+
+
+def get(url, mod=None, **kwargs):
+  mod = mod or requests
+
+  resp = mod.get(url, **kwargs)
+  resp.raise_for_status()
+
+  return resp.content
+

py_misc_utils/img_utils.py

@@ -0,0 +1,20 @@
+import io
+
+import PIL.Image as Image
+
+from . import http_utils as hu
+
+
+def from_bytes(data, convert=None):
+  try:
+    img = Image.open(io.BytesIO(data))
+  except Exception as ex:
+    ex.add_note(f'Unable to load image: data={data[: 16]}...')
+    raise
+
+  return img if convert is None else img.convert(convert)
+
+
+def from_url(url, convert=None, **kwargs):
+  return from_bytes(hu.get(url, **kwargs), convert=convert)
+

py_misc_utils/infix_op.py

@@ -0,0 +1,20 @@
+
+class InfixOp:
+  def __init__(self, opfn):
+    self._opfn = opfn
+
+  def __ror__(self, lhs):
+    return InfixOp(lambda x, self=self, lhs=lhs: self._opfn(lhs, x))
+
+  def __or__(self, rhs):
+    return self._opfn(rhs)
+
+  def __rlshift__(self, lhs):
+    return InfixOp(lambda x, self=self, lhs=lhs: self._opfn(lhs, x))
+
+  def __rshift__(self, rhs):
+    return self._opfn(rhs)
+
+  def __call__(self, value1, value2):
+    return self._opfn(value1, value2)
+

py_misc_utils/inspect_utils.py

@@ -0,0 +1,205 @@
+import inspect
+import itertools
+import sys
+import types
+
+from . import assert_checks as tas
+from . import traceback as tb
+
+
+_NONE = object()
+
+
+def classof(obj):
+  return obj if inspect.isclass(obj) else getattr(obj, '__class__', None)
+
+
+def moduleof(obj):
+  cls = classof(obj)
+
+  return getattr(cls, '__module__', None) if cls is not None else None
+
+
+def cname(obj):
+  cls = classof(obj)
+
+  return cls.__name__ if cls is not None else None
+
+
+def func_name(func):
+  fname = getattr(func, '__name__', None)
+
+  return fname if fname is not None else cname(func)
+
+
+_BUILTIN_NAMES = {'__builtin__', 'builtins'}
+
+def _qual_name(obj, builtin_strip=False):
+  module = getattr(obj, '__module__', None)
+  name = getattr(obj, '__qualname__', None)
+  if name is None:
+    name = getattr(obj, '__name__', None)
+    tas.check_is_not_none(name, msg=f'Unable to reference name: {obj}')
+  if module is not None and not (builtin_strip and module in _BUILTIN_NAMES):
+    name = module + '.' + name
+
+  return name
+
+
+def qual_name(obj, builtin_strip=False):
+  if (inspect.isclass(obj) or inspect.isfunction(obj) or
+      inspect.ismethod(obj) or inspect.ismodule(obj)):
+    ref = obj
+  else:
+    ref = obj.__class__
+
+  return _qual_name(ref, builtin_strip=builtin_strip)
+
+
+def qual_mro(obj, builtin_strip=False):
+  cls = obj if inspect.isclass(obj) else obj.__class__
+
+  for scls in cls.__mro__:
+    yield _qual_name(scls, builtin_strip=builtin_strip)
+
+
+def is_subclass(cls, cls_group):
+  return inspect.isclass(cls) and issubclass(cls, cls_group)
+
+
+def _fn_lookup(frame, name):
+  xns, xname = None, name
+  while True:
+    dpos = xname.find('.')
+    if dpos > 0:
+      cname, fname = xname[: dpos], xname[dpos + 1:]
+      if cname == '<locals>':
+        code = getattr(xns, '__code__', None)
+        if code is not None:
+          for cv in code.co_consts:
+            if inspect.iscode(cv) and cv.co_name == fname:
+              return types.FunctionType(cv, frame.f_globals, fname)
+
+        return None
+      else:
+        xns = frame.f_globals[cname] if xns is None else getattr(xns, cname)
+        xname = fname
+    else:
+      xns = frame.f_globals[xname] if xns is None else getattr(xns, xname)
+      break
+
+  return xns
+
+
+def get_caller_function(back=0, frame=None):
+  if frame is None:
+    frame = tb.get_frame(back + 1)
+
+  return _fn_lookup(frame, frame.f_code.co_qualname)
+
+
+def current_module():
+  return inspect.getmodule(tb.get_frame(1))
+
+
+def fetch_args(func, locs, input_args=()):
+  sig = inspect.signature(func)
+
+  def args_append(args, n):
+    if len(args) < len(input_args):
+      args.append(input_args[len(args)])
+    else:
+      pv = locs.get(n, _NONE)
+      if pv is not _NONE:
+        args.append(pv)
+      elif args:
+        alog.xraise(RuntimeError, f'Missing argument: {n}')
+
+  def kwargs_assign(kwargs, n, p):
+    pv = locs.get(n, _NONE)
+    if pv is _NONE or (pv is None and p.default is not inspect.Signature.empty):
+      pv = p.default
+    if pv is not inspect.Signature.empty:
+      kwargs[n] = pv
+
+  args, kwargs = [], dict()
+  for n, p in sig.parameters.items():
+    if p.kind == p.POSITIONAL_ONLY:
+      args_append(args, n)
+    elif p.kind == p.POSITIONAL_OR_KEYWORD:
+      if p.default is inspect.Signature.empty:
+        args_append(args, n)
+      else:
+        kwargs_assign(kwargs, n, p)
+    else:
+      kwargs_assign(kwargs, n, p)
+
+  return args, kwargs
+
+
+def fetch_call(func, locs, input_args=()):
+  args, kwargs = fetch_args(func, locs, input_args=input_args)
+
+  return func(*args, **kwargs)
+
+
+def get_fn_kwargs(args, func, prefix=None, roffset=None):
+  aspec = inspect.getfullargspec(func)
+
+  sdefaults = aspec.defaults or ()
+  sargs = aspec.args or ()
+  ndelta = len(sargs) - len(sdefaults)
+
+  fnargs = dict()
+  for i, an in enumerate(sargs):
+    if i != 0 or an != 'self':
+      nn = f'{prefix}.{an}' if prefix else an
+      di = i - ndelta
+      if di >= 0:
+        fnargs[an] = args.get(nn, sdefaults[di])
+      elif roffset is not None and i >= roffset:
+        aval = args.get(nn, _NONE)
+        tas.check(aval is not _NONE,
+                  msg=f'The "{an}" argument must be present as "{nn}": {args}')
+        fnargs[an] = aval
+
+  if aspec.kwonlyargs:
+    for an in aspec.kwonlyargs:
+      nn = f'{prefix}.{an}' if prefix else an
+      aval = args.get(nn, aspec.kwonlydefaults.get(an, inspect.Signature.empty))
+      if aval is not inspect.Signature.empty:
+        fnargs[an] = aval
+
+  return fnargs
+
+
+def get_defaulted_params(func):
+  sig = inspect.signature(func)
+
+  return tuple(p for p in sig.parameters.values()
+               if p.default is not inspect.Signature.empty)
+
+
+def parent_locals(level=0):
+  frame = tb.get_frame(level + 2)
+
+  return frame.f_locals
+
+
+def parent_globals(level=0):
+  frame = tb.get_frame(level + 2)
+
+  return frame.f_globals
+
+
+def parent_coords(level=0):
+  frame = tb.get_frame(level + 2)
+
+  return frame.f_code.co_filename, frame.f_lineno
+
+
+def class_slots(cls):
+  slots = itertools.chain.from_iterable(getattr(mcls, '__slots__', []) for mcls in cls.__mro__)
+
+  return tuple(slots)
+

py_misc_utils/iostream.py

@@ -0,0 +1,21 @@
+import functools
+import os
+
+
+class IOStream:
+
+  def __init__(self, fd):
+    if (wfn := getattr(fd, 'write', None)) is not None:
+      self.write = wfn
+    elif (wfn := getattr(fd, 'send', None)) is not None:
+      self.write = wfn
+    else:
+      self.write = functools.partial(os.write, fd)
+
+    if (rfn := getattr(fd, 'read', None)) is not None:
+      self.read = rfn
+    elif (rfn := getattr(fd, 'recv', None)) is not None:
+      self.read = rfn
+    else:
+      self.read = functools.partial(os.read, fd)
+