iripau 0.1.0__py3-none-any.whl

iripau/executable.py

@@ -0,0 +1,108 @@
+"""
+Execute commands as Python functions
+"""
+
+from shlex import split
+from random import choice
+from itertools import chain
+
+from iripau.command import host_run
+
+
+class Command:
+    """ Run an executable command as a Python callable """
+
+    def __init__(self, parent, command):
+        self._parent = parent
+        self._command = parent._mk_command(command)
+        self._mk_command = parent._mk_command
+
+    def __getattr__(self, command):
+        child = Command(self, command)
+        setattr(self, command, child)
+        return child
+
+    def __call__(self, *args, **kwargs):
+        return self._parent(self._command, *args, **kwargs)
+
+
+def make_command(command):
+    return command.replace("_", "-")
+
+
+def make_option(option):
+    return "--" + option.replace("_", "-"),
+
+
+class Executable:
+    """ Run an executable as a Python callable """
+
+    def __init__(
+        self, executable, make_command=make_command, make_option=make_option,
+        alias=None, run_args_prefix="_", run_function=None, **kwargs
+    ):
+        self._run = run_function or host_run
+        self._exe = split(executable) if isinstance(executable, str) else executable
+        self._alias = split(alias) if isinstance(alias, str) else alias
+        self._kwargs = kwargs
+
+        self._prefix = run_args_prefix
+        self._mk_option = make_option
+        self._mk_command = make_command
+
+    def __getattr__(self, command):
+        child = Command(self, command)
+        setattr(self, command, child)
+        return child
+
+    def __call__(self, *args, **kwargs):
+        optionals = chain.from_iterable(
+            self._make_arg(self._mk_option(key), value)
+            for key, value in kwargs.items()
+            if not key.startswith(self._prefix)
+        )
+
+        positionals = list(map(str, args))
+        optionals = list(optionals)
+
+        kwargs = {
+            key[len(self._prefix):]: value
+            for key, value in kwargs.items()
+            if key.startswith(self._prefix)
+        }
+
+        if self._alias:
+            kwargs.setdefault("alias", self._alias + positionals + optionals)
+
+        cmd = self._exe + positionals + optionals
+        return self._run(cmd, **self._kwargs, **kwargs)
+
+    @staticmethod
+    def _is_iterable(value):
+        if isinstance(value, (str, bytes)):
+            return False
+        return hasattr(value, "__iter__")
+
+    @classmethod
+    def _make_arg(cls, options, value=None):
+        """ Return a list of tokens. Randomly choose a short or long option.
+            If a 'value' is given it's appended appropriately.
+            If 'value' is iterable, the option will be repeated for each item.
+        """
+        if cls._is_iterable(value):
+            return chain.from_iterable(
+                cls._make_arg(options, item)
+                for item in value
+            )
+
+        if value in {None, False}:
+            return []
+
+        option = choice(options)
+        if value is True:
+            return [option]
+
+        value = str(value)
+        if option.startswith("--"):
+            return [option + "=" + value]
+        return [option, value]

iripau/functools.py

@@ -0,0 +1,89 @@
+"""
+Function utilities
+"""
+
+import sys
+import uuid
+import operator
+
+from functools import wraps
+from inspect import getsource, isgeneratorfunction
+from time import time, sleep
+
+
+def wait_for(
+    condition, *args,
+    _timeout=None, _outcome=True, _poll_time=10, _stop_condition=None, **kwargs
+):
+    last = time()
+    end = _timeout and last + _timeout
+    operation = operator.not_ if _outcome else operator.truth
+    while operation(condition(*args, **kwargs)):
+        if _stop_condition and _stop_condition():
+            message = "No reason to keep waiting since the following condition was met:\n{0}"
+            raise InterruptedError(message.format(getsource(_stop_condition)))
+        now = time()
+        if end and now > end:
+            message = "The following condition was not {0} after {1} seconds: \n{2}"
+            raise TimeoutError(
+                message.format(_outcome, _timeout, getsource(condition))
+            )
+        sleep_time = max(0, _poll_time - (now - last))
+        sleep(sleep_time)
+        last = now + sleep_time
+
+
+def retry(tries, exceptions=Exception, retry_condition=None, backoff_time=0):
+    """
+        Call the decorated function until it succeeds.
+
+        It will only retry if the expected exceptions are risen and the condition
+        is fulfilled. To verify the condition is fulfilled, it will be called
+        with the caught exception.
+
+        The decorated function can have two parts divided by 'yield', in that
+        case the expected exceptions will only be caught in the second part.
+    """
+    def decorator(function):
+
+        @wraps(function)
+        def helper(*args, **kwargs):
+            yield
+            return function(*args, **kwargs)
+
+        f = function if isgeneratorfunction(function) else helper
+
+        @wraps(function)
+        def wrapper(*args, **kwargs):
+            t = tries
+            while t > 0:
+                gen = f(*args, **kwargs)
+                next(gen)
+                try:
+                    next(gen)
+                except StopIteration as e:
+                    return e.value
+                except exceptions as e:
+                    if retry_condition is None or retry_condition(e):
+                        if t == 1:
+                            raise
+                        t = t - 1
+                        sleep(backoff_time)
+                    else:
+                        raise
+        return wrapper
+    return decorator
+
+
+def globalize(function):
+    """
+        Make function globally available in the module it was defined so it can
+        be serialized. Useful when calling local functions with multiprocessing.
+    """
+    @wraps(function)
+    def wrapper(*args, **kwargs):
+        return function(*args, **kwargs)
+
+    wrapper.__name__ = wrapper.__qualname__ = uuid.uuid4().hex
+    setattr(sys.modules[function.__module__], wrapper.__name__, wrapper)
+    return wrapper

iripau/logging.py

@@ -0,0 +1,86 @@
+"""
+Logging utilities
+"""
+
+import os
+import re
+import logging
+import threading
+
+from collections import OrderedDict
+
+
+class LoggerFile:
+    """ File that logs every line written to it """
+
+    def __new__(cls, logger, level):
+        r, w = os.pipe()
+        read_file = os.fdopen(r, "r")
+        write_file = os.fdopen(w, "w")
+
+        thread = threading.Thread(
+            target=cls.log,
+            args=(read_file, logger, level),
+            name=threading.current_thread().name
+        )
+
+        # Close write_file when the thread joins
+        cls.patch(thread, "join", write_file.close)
+
+        # Join the thread when the file is closed
+        write_file.close = thread.join
+
+        thread.start()
+        return write_file
+
+    @staticmethod
+    def log(read_file, logger, level):
+        with read_file:
+            for line in read_file:
+                logger.log(level, line.splitlines()[0])
+
+    @staticmethod
+    def patch(instance, function_name, callback):
+        original_function = getattr(instance, function_name)
+
+        def new_function():
+            callback()
+            original_function()
+        setattr(instance, function_name, new_function)
+
+
+class SimpleThreadNameFormatter(logging.Formatter):
+    """ The same logging.Formatter but threadName is just the first token after
+        splitting it: record.threadName.split(maxsplit=1)[0]
+    """
+
+    def format(self, record):
+        if record.threadName:
+            record.threadName = record.threadName.split(maxsplit=1)[0]
+        return super().format(record)
+
+
+def group_log_lines(lines, thread_id_regex, main_thread_id="MainThread"):
+    """ For a log file containing entries from several threads, group the lines
+        so that the lines coming from the same thread are contiguous, preserving
+        order within the group. Logs coming from MainThread will not be grouped.
+    """
+    lines_map = OrderedDict()
+    for i, line in enumerate(lines):
+        match = re.match(thread_id_regex, line)
+        if not match:
+            raise ValueError(f"Invalid log line {i}: '{line}'")
+
+        thread = match.groups()[0]
+        if main_thread_id == thread:
+            for thread_lines in lines_map.values():
+                for thread_line in thread_lines:
+                    yield thread_line
+            yield line
+            lines_map.clear()
+        else:
+            lines_map[thread] = lines_map.get(thread, []) + [line]
+
+    for thread_lines in lines_map.values():
+        for thread_line in thread_lines:
+            yield thread_line

iripau/random.py

@@ -0,0 +1,38 @@
+"""
+Handle random data
+"""
+
+import math
+import random
+import string
+
+
+def one(items):
+    """ Return a new list containing only one element from items """
+    return [random.choice(items)]
+
+
+def some(items, percentage=50, at_least=2, at_most=None):
+    """ Return a new list containing some elements from items.
+        Ordinality is preserved as much as possible.
+    """
+    count = math.ceil(percentage * len(items) / 100)
+    if at_least and at_least > count:
+        count = at_least
+    if at_most and at_most < count:
+        count = at_most
+    return sorted(random.sample(items, count), key=items.index)
+
+
+def shuffled(items):
+    """ Return a new list containing all of the elements from items but in a
+        different order.
+    """
+    return random.sample(items, len(items))
+
+
+def random_string(length, chars=string.ascii_letters):
+    """ Return a string of the desired length containing randomly chosen
+        characters from chars.
+    """
+    return "".join(random.choice(chars) for _ in range(length))

iripau/shutil.py

@@ -0,0 +1,111 @@
+"""
+Shell utilities
+"""
+
+import os
+import errno
+import select
+import shutil
+import contextlib
+
+from iripau.functools import wait_for
+
+
+class FileLock:
+    poll_time = 0.05
+
+    def __init__(self, file_name, timeout=None):
+        self.file_name = file_name
+        self.timeout = timeout
+        self.acquired = False
+
+    def __enter__(self):
+        self.acquire(self.timeout)
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.release()
+
+    def __del__(self):
+        self.release()
+
+    def _lock_file_created(self):
+        """ Lock file could be created by us """
+        try:
+            self.fd = os.open(
+                self.file_name,
+                os.O_CREAT | os.O_EXCL | os.O_RDWR
+            )
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+        else:
+            return True
+        return False
+
+    def acquire(self, timeout=None):
+        if not self.acquired:
+            wait_for(
+                self._lock_file_created,
+                _timeout=timeout or self.timeout,
+                _poll_time=self.poll_time
+            )
+            self.acquired = True
+
+    def release(self):
+        if self.acquired:
+            os.close(self.fd)
+            os.unlink(self.file_name)
+            self.acquired = False
+
+
+def create_file(file_name, content=""):
+    mode = "wb" if bytes == type(content) else "w"
+    with open(file_name, mode) as f:
+        f.write(content)
+
+
+def read_file(file_name, binary=False):
+    mode = "rb" if binary else "r"
+    with open(file_name, mode) as f:
+        return f.read()
+
+
+def remove_file(file_name):
+    with contextlib.suppress(FileNotFoundError):
+        os.remove(file_name)
+
+
+def remove_tree(root):
+    with contextlib.suppress(FileNotFoundError):
+        if os.path.isdir(root) and not os.path.islink(root):
+            shutil.rmtree(root)
+        else:
+            os.remove(root)
+
+
+@contextlib.contextmanager
+def file_created(file_name, content=""):
+    create_file(file_name, content)
+    try:
+        yield file_name
+    finally:
+        os.remove(file_name)
+
+
+def wait_for_file(file_obj, *args, **kwargs):
+    poll_obj = select.poll()
+    poll_obj.register(file_obj, select.POLLIN)
+    wait_for(lambda: poll_obj.poll(0), *args, **kwargs)
+
+
+def _rotate(root, ext, seq=0):
+    old = seq and f"{root}.{seq}{ext}" or root + ext
+    if os.path.exists(old):
+        seq += 1
+        _rotate(root, ext, seq)
+        os.rename(old, f"{root}.{seq}{ext}")
+
+
+def rotate(path, seq=0):
+    _rotate(*os.path.splitext(path))