danielutils 1.1.28__py3-none-any.whl

danielutils/async_/async_layered_command.py

@@ -0,0 +1,125 @@
+import sys
+import logging
+from typing import Optional, Tuple, List
+
+from .async_cmd import async_cmd
+from ..logging_.utils import get_logger
+logger = get_logger(__name__)
+
+
+class AsyncLayeredCommand:
+    class_capture_stdout: bool = True
+    class_capture_stderr: bool = True
+    class_raise_on_fail: bool = False
+    class_verbose: bool = False
+    _class_prev_instance: Optional['AsyncLayeredCommand'] = None
+    _id: int = 0
+
+    @property
+    def prev(self):
+        return self._prev_instance
+
+    @prev.setter
+    def prev(self, value: Optional['AsyncLayeredCommand'] = None):
+        self._prev_instance = value
+
+    def __init__(
+            self,
+            command: Optional[str] = None,
+            *,
+            prev_instance: Optional['AsyncLayeredCommand'] = None,
+            instance_capture_stdout: Optional[bool] = None,
+            instance_capture_stderr: Optional[bool] = None,
+            instance_raise_on_fail: Optional[bool] = None,
+            instance_verbose: Optional[bool] = None
+    ):
+        logger.debug("Initializing AsyncLayeredCommand with command='%s', prev_instance=%s", command, prev_instance is not None)
+        self._command = command if command is not None else ""
+        self._instance_capture_stdout = instance_capture_stdout
+        self._instance_capture_stderr = instance_capture_stderr
+        self._instance_raise_on_fail = instance_raise_on_fail
+        self._instance_verbose = instance_verbose
+        self._prev_instance = prev_instance if prev_instance is not None else AsyncLayeredCommand._class_prev_instance
+        self._cur_class_prev_instance = AsyncLayeredCommand._class_prev_instance
+        AsyncLayeredCommand._class_prev_instance = self
+        self._has_entered: bool = False
+        logger.debug("AsyncLayeredCommand initialized successfully")
+
+    def __enter__(self):
+        logger.debug("Entering AsyncLayeredCommand context")
+        self._open()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        logger.debug("Exiting AsyncLayeredCommand context, exc_type=%s", exc_type)
+        if self.prev is self._cur_class_prev_instance:
+            AsyncLayeredCommand._class_prev_instance = self.prev
+
+    def _open(self) -> None:
+        logger.debug("Opening AsyncLayeredCommand")
+        self._has_entered = True
+
+    def _build_command(self, *commands: str) -> str:
+        logger.debug("Building command with %d additional commands", len(commands))
+        res = ""
+        if self.prev is not None:
+            prev = self.prev._build_command()
+            res += f"{prev} & " if prev != "" else ""
+        if self._command != "":
+            result = res + " & ".join([self._command, *commands])
+        else:
+            result = res + " & ".join(commands)
+        logger.debug("Built command: %s", result)
+        return result
+
+    def _error(self, predicate: bool, command: str, code: int, command_verbose: Optional[bool]) -> None:
+        if predicate:
+            logger.error("Command '%s' failed with exit code %d", command, code)
+            verbose = self._merge_values(command_verbose, self._instance_verbose, AsyncLayeredCommand.class_verbose)
+            if verbose:
+                raise RuntimeError(f"command '{command}' failed with exit code {code}")
+            sys.exit(1)
+
+    @staticmethod
+    def _merge_values(command: Optional[bool], instance: Optional[bool], class_: bool) -> bool:
+        return command if command is not None else (instance if instance is not None else class_)
+
+    async def execute(
+            self,
+            *commands: str,
+            command_capture_stdout: Optional[bool] = None,
+            command_capture_stderr: Optional[bool] = None,
+            command_raise_on_fail: Optional[bool] = None,
+            command_verbose: Optional[bool] = None
+    ) -> Tuple[int, List[str], List[str]]:
+        logger.info("Executing AsyncLayeredCommand with %d commands", len(commands))
+        if not self._has_entered:
+            logger.error("LayeredCommand must be used with a context manager")
+            raise RuntimeError(
+                "LayeredCommand must be used with a context manager. Use as: `with LayeredCommand(...) as l1:`")
+        capture_stdout = self._merge_values(command_capture_stdout, self._instance_capture_stdout,
+                                            self.class_capture_stdout)
+        capture_stderr = self._merge_values(command_capture_stderr, self._instance_capture_stderr,
+                                            self.class_capture_stderr)
+        raise_on_fail = self._merge_values(command_raise_on_fail, self._instance_raise_on_fail,
+                                           self.class_raise_on_fail)
+
+        command = self._build_command(*commands)
+        logger.debug("Executing command with capture_stdout=%s, capture_stderr=%s", capture_stdout, capture_stderr)
+        if not capture_stdout and not capture_stderr:
+            code = (await async_cmd(command))[0]
+            logger.debug("Command executed with return code %d", code)
+            self._error(raise_on_fail and code != 0, command, code, command_verbose)
+            return code, [], []
+
+        code, stdout, stderr = await async_cmd(command, capture_stdout=True, capture_stderr=True)
+        logger.info("Command execution completed with return code %d", code)
+        return code, stdout.decode().splitlines() if stdout else [], stderr.decode().splitlines() if stderr else []
+
+    async def __call__(self, *args, **kwargs) -> Tuple[int, List[str], List[str]]:
+        return await self.execute(*args, **kwargs)
+
+
+__all__ = [
+    'AsyncLayeredCommand'
+]

danielutils/async_/async_retry_executor.py

@@ -0,0 +1,86 @@
+import asyncio
+import functools
+import logging
+from datetime import datetime
+from typing import Literal, Optional, Any, Mapping, Iterable, Callable, Coroutine
+
+from ..custom_types import Supplier
+from ..decorators import normalize_decorator
+from .time_strategy import LinearTimeStrategy, ConstantTimeStrategy
+from ..versioned_imports import ParamSpec
+from ..logging_.utils import get_logger
+
+P = ParamSpec("P")
+logger = get_logger(__name__)
+
+
+class AsyncRetryExecutor:
+    def __init__(
+            self,
+            timeout_strategy: Supplier[float] = LinearTimeStrategy(30, 5),
+            delay_strategy: Supplier[float] = ConstantTimeStrategy(0)
+    ) -> None:
+        logger.info("Initializing AsyncRetryExecutor with timeout_strategy=%s, delay_strategy=%s", type(timeout_strategy).__name__, type(delay_strategy).__name__)
+        self.timeout_strategy = timeout_strategy
+        self.delay_strategy = delay_strategy
+        logger.debug("AsyncRetryExecutor initialized successfully")
+
+    def is_transient(self, e: Exception) -> bool:
+        """
+        This function will return true if the exception that was raised at a specific attempt should be ignored and we should try again with respet to the amount of retries left
+        Args:
+            e: exception caught
+
+        Returns:
+            boolean
+        """
+        return False
+
+    async def execute(
+            self,
+            func: Callable[P, Coroutine],
+            *,
+            args: Optional[Iterable] = None,
+            kwargs: Optional[Mapping] = None,
+            max_tries: int = 5
+    ) -> Optional[Any]:
+        args = list(args) if args else []
+        kwargs = dict(kwargs) if kwargs else {}
+        logger.info("Starting async retry execution for %s with max_tries=%d", func.__name__, max_tries)
+        
+        for i in range(1, max_tries + 1):
+            timeout = self.timeout_strategy()
+            delay = self.delay_strategy()
+            logger.debug("Attempt %d/%d with timeout=%ss, delay=%ss", i, max_tries, timeout, delay)
+            try:
+                result = await asyncio.wait_for(func(*args, **kwargs), timeout=timeout)
+                logger.info("Async retry execution succeeded on attempt %d", i)
+                return result
+            except Exception as e:
+                if self.is_transient(e):
+                    logger.warning("Failed attempt %d/%d with transient exception: %s: %s", i, max_tries, type(e).__name__, e)
+                    if i < max_tries - 1 and delay > 0:
+                        logger.debug("Waiting %ss before next attempt", delay)
+                        await asyncio.sleep(delay)
+                else:
+                    logger.error("Non-transient exception on attempt %d: %s: %s", i, type(e).__name__, e)
+                    raise e
+        logger.error("Failed all %d attempts for %s", max_tries, func.__name__)
+        raise RuntimeError(f"Failed all attempts")
+
+
+@normalize_decorator
+def with_async_retry(func, *retry_executor_args, max_tries: int = 5, **retry_executor_kwargs):
+    retry_executor = AsyncRetryExecutor(*retry_executor_args, **retry_executor_kwargs)
+
+    @functools.wraps(func)
+    async def wrapper(*args, **kwargs):
+        return await retry_executor.execute(func, args=args, kwargs=kwargs, max_tries=max_tries)
+
+    return wrapper
+
+
+__all__ = [
+    "AsyncRetryExecutor",
+    "with_async_retry"
+]

danielutils/async_/async_worker_pool.py

@@ -0,0 +1,110 @@
+import asyncio
+import json
+import logging
+from collections import defaultdict
+from typing import Callable, Optional, Coroutine, List, Iterable, Any, Mapping, Tuple
+
+try:
+    from tqdm import tqdm
+except ImportError:
+    from ..mock_ import MockImportObject
+
+    tqdm = MockImportObject("'tqdm' is not installed. Please install 'tqdm' to use AsyncWorkerPool feature.")  # type: ignore
+
+from ..logging_.utils import get_logger
+
+class AsyncWorkerPool:
+    DEFAULT_ORDER_IF_KEY_EXISTS = (
+        "pool", "timestamp", "worker_id", "task_id", "task_name", "num_tasks", "tasks", "level", "message", "exception"
+    )
+    
+    # Default logger instance
+    _logger = get_logger(__name__)
+    
+    @staticmethod
+    def log(level: int, message: str, *args, **kwargs) -> None:
+        """
+        Static logging method that delegates to the default logger.
+        End users can override this method to customize logging behavior.
+        
+        Args:
+            level: Log level (DEBUG=10, INFO=20, WARNING=30, ERROR=40, CRITICAL=50)
+            message: Log message
+            *args: Additional arguments for message formatting
+            **kwargs: Additional keyword arguments for logging
+        """
+        AsyncWorkerPool._logger.log(level, message, *args, **kwargs)
+
+    def __init__(self, pool_name: str, num_workers: int = 5, show_pbar: bool = False) -> None:
+        self.log(logging.INFO, "Initializing AsyncWorkerPool '%s' with %d workers, show_pbar=%s", pool_name, num_workers, show_pbar)
+        self._num_workers: int = num_workers
+        self._pool_name: str = pool_name
+        self._show_pbar: bool = show_pbar
+        self._pbar: Optional[tqdm] = None
+        self._queue: asyncio.Queue[
+            Optional[Tuple[Callable, Iterable[Any], Mapping[Any, Any], Optional[str]]]] = asyncio.Queue()
+        self._workers: List = []
+        self.log(logging.DEBUG, "AsyncWorkerPool '%s' initialized successfully", pool_name)
+
+    async def worker(self, worker_id) -> None:
+        """Worker coroutine that continuously fetches and executes tasks from the queue."""
+        self.log(logging.DEBUG, "Worker %d starting", worker_id)
+        task_index = 0
+        tasks = defaultdict(list)
+        while True:
+            task = await self._queue.get()
+            if task is None:  # Sentinel value to shut down the worker
+                self.log(logging.DEBUG, "Worker %d received shutdown signal", worker_id)
+                break
+            func, args, kwargs, name = task
+            task_index += 1
+            self.log(logging.INFO, "Task %d '%s' started on worker %d", task_index, name, worker_id)
+            try:
+                await func(*args, **kwargs)
+                tasks["success"].append(name)
+                self.log(logging.INFO, "Task %d '%s' finished on worker %d", task_index, name, worker_id)
+            except Exception as e:
+                self.log(logging.ERROR, "Task %d '%s' failed on worker %d: %s: %s", task_index, name, worker_id, type(e).__name__, e)
+                tasks["failure"].append(name)
+
+            if self._pbar:
+                self._pbar.update(1)
+            self._queue.task_done()
+        self.log(logging.INFO, "Worker %d completed %d tasks (success: %d, failure: %d)", worker_id, task_index, len(tasks['success']), len(tasks['failure']))
+        if tasks['success']:
+            self.log(logging.INFO, "Worker %d successful tasks: [%s]", worker_id, ', '.join(map(str, tasks['success'])))
+        if tasks['failure']:
+            self.log(logging.WARNING, "Worker %d failed tasks: [%s]", worker_id, ', '.join(map(str, tasks['failure'])))
+
+    async def start(self) -> None:
+        """Starts the worker pool."""
+        self.log(logging.INFO, "Starting worker pool '%s' with %d workers", self._pool_name, self._num_workers)
+        if self._show_pbar:
+            self._pbar = tqdm(total=self._queue.qsize(), desc="#Tasks")
+        self._workers = [asyncio.create_task(self.worker(i + 1)) for i in range(self._num_workers)]
+        self.log(logging.INFO, "Worker pool '%s' started successfully", self._pool_name)
+
+    async def submit(
+            self,
+            func: Callable[..., Coroutine[None, None, None]],
+            args: Optional[Iterable[Any]] = None,
+            kwargs: Optional[Mapping[Any, Any]] = None,
+            name: Optional[str] = None
+    ) -> None:
+        """Submit a new task to the queue."""
+        self.log(logging.DEBUG, "Adding new job '%s' to queue", name)
+        await self._queue.put((func, args or (), kwargs or {}, name))
+
+    async def join(self) -> None:
+        """Stops the worker pool by waiting for all tasks to complete and shutting down workers."""
+        self.log(logging.INFO, "Starting join process for worker pool '%s'", self._pool_name)
+        await self._queue.join()  # Wait until all tasks are processed
+        for _ in range(self._num_workers):
+            await self._queue.put(None)  # Send sentinel values to stop workers
+        await asyncio.gather(*self._workers)  # Wait for workers to finish
+        self.log(logging.INFO, "Join process completed for worker pool '%s'", self._pool_name)
+
+
+__all__ = [
+    "AsyncWorkerPool",
+]

danielutils/async_/time_strategy.py

@@ -0,0 +1,73 @@
+from abc import ABC, abstractmethod
+import logging
+from ..logging_.utils import get_logger
+logger = get_logger(__name__)
+
+
+class TimeStrategy(ABC):
+    @abstractmethod
+    def next(self): ...
+
+    def __call__(self, *args, **kwargs):
+        return self.next()
+
+    @abstractmethod
+    def reset(self): ...
+
+
+class ConstantTimeStrategy(TimeStrategy):
+    def __init__(self, timeout: float):
+        logger.debug("Initializing ConstantTimeStrategy with timeout=%s", timeout)
+        self.timeout = timeout
+
+    def next(self) -> float:
+        logger.debug("ConstantTimeStrategy returning timeout=%s", self.timeout)
+        return self.timeout
+
+    def reset(self) -> None:
+        logger.debug("ConstantTimeStrategy reset called (no-op)")
+        pass  # No state to reset
+
+
+class LinearTimeStrategy(TimeStrategy):
+    def __init__(self, base_timeout: float, step: float):
+        logger.debug("Initializing LinearTimeStrategy with base_timeout=%s, step=%s", base_timeout, step)
+        self.base_timeout = base_timeout
+        self.step = step
+        self.current_timeout = base_timeout
+
+    def next(self) -> float:
+        timeout = self.current_timeout
+        self.current_timeout += self.step
+        logger.debug("LinearTimeStrategy returning timeout=%s, next will be %s", timeout, self.current_timeout)
+        return timeout
+
+    def reset(self) -> None:
+        logger.debug("LinearTimeStrategy resetting to base_timeout=%s", self.base_timeout)
+        self.current_timeout = self.base_timeout
+
+
+class MultiplicativeTimeStrategy(TimeStrategy):
+    def __init__(self, base_timeout: float, factor: float):
+        logger.debug("Initializing MultiplicativeTimeStrategy with base_timeout=%s, factor=%s", base_timeout, factor)
+        self.base_timeout = base_timeout
+        self.factor = factor
+        self.current_timeout = base_timeout
+
+    def next(self) -> float:
+        timeout = self.current_timeout
+        self.current_timeout *= self.factor
+        logger.debug("MultiplicativeTimeStrategy returning timeout=%s, next will be %s", timeout, self.current_timeout)
+        return timeout
+
+    def reset(self) -> None:
+        logger.debug("MultiplicativeTimeStrategy resetting to base_timeout=%s", self.base_timeout)
+        self.current_timeout = self.base_timeout
+
+
+__all__ = [
+    "TimeStrategy",
+    "ConstantTimeStrategy",
+    "LinearTimeStrategy",
+    "MultiplicativeTimeStrategy"
+]

danielutils/async_/utils.py

@@ -0,0 +1,57 @@
+import asyncio
+import logging
+from asyncio import Task
+from typing import List, Coroutine, Any, Tuple, Optional, Set, AsyncIterator, Iterator, TypeVar
+from ..logging_.utils import get_logger
+logger = get_logger(__name__)
+
+
+async def return_first(coros: List[Coroutine], timeout: Optional[int] = None) -> List[Tuple[int, Any]]:
+    logger.debug("return_first called with %d coroutines, timeout=%s", len(coros), timeout)
+    tasks: List[Task] = [asyncio.create_task(coro) for coro in coros]
+    result: Tuple[Set[Task], Set[Task]] = await asyncio.wait(tasks, timeout=timeout,
+                                                             return_when=asyncio.FIRST_COMPLETED)
+    done: Set[Task] = result[0]
+    # pending: Set[Task] = result[1]
+
+    res = []
+    for task in done:
+        res.append((tasks.index(task), task.result()))
+
+    logger.debug("return_first completed with %d results", len(res))
+    return res
+
+
+async def return_all(coros: List[Coroutine], timeout: Optional[int] = None) -> List[Any]:
+    logger.debug("return_all called with %d coroutines, timeout=%s", len(coros), timeout)
+    tasks: List[Task] = [asyncio.create_task(coro) for coro in coros]
+    result: Tuple[Set[Task], Set[Task]] = await asyncio.wait(tasks, timeout=timeout,
+                                                             return_when=asyncio.ALL_COMPLETED)
+    done: Set[Task] = result[0]
+
+    res = [task.result() for task in done]
+    logger.debug("return_all completed with %d results", len(res))
+    return res
+
+
+async def cast_aiter(itr: Iterator) -> AsyncIterator:
+    logger.debug("cast_aiter called")
+    for x in itr:
+        yield x
+
+
+T = TypeVar("T")
+
+
+async def async_enumerate(iterable: AsyncIterator[T], start: int = 0) -> AsyncIterator[Tuple[int, T]]:
+    index = start
+    async for item in iterable:
+        yield index, item
+        index += 1
+
+
+__all__ = [
+    "return_first",
+    "return_all",
+    'cast_aiter'
+]

danielutils/better_builtins/__init__.py

@@ -0,0 +1,3 @@
+from .typed_builtins import *
+from .frange import *
+from .counter import *

danielutils/better_builtins/counter.py

@@ -0,0 +1,48 @@
+from typing import Union
+from ..metaclasses import AtomicClassMeta
+
+
+class Counter:
+    """A simple counter class
+    """
+
+    def __init__(self, initial_value: Union[int, float] = 0, increment_amount: Union[int, float] = 1) -> None:
+        self.value = initial_value
+        self.increment_value = increment_amount
+
+    def increment(self) -> None:
+        """increments the stored value by the increment amount
+        """
+        self.value += self.increment_value
+
+    def decrement(self) -> None:
+        """decrements the stored value by the increment amount
+        """
+        self.value -= self.increment_value
+
+    def get(self) -> Union[int, float]:
+        """returns the current value of the counter
+
+        Returns:
+            Union[int, float]: value
+        """
+        return self.value
+
+    def set(self, value: Union[int, float]):
+        """sets the values of the counter
+
+        Args:
+            value (Union[int, float]): value to set
+        """
+        self.value = value
+
+
+class AtomicCounter(Counter, metaclass=AtomicClassMeta):  # type:ignore
+    """A Counter Class which is Atomic
+    """
+
+
+__all__ = [
+    "Counter",
+    "AtomicCounter"
+]