danielutils 1.1.28__py3-none-any.whl

danielutils/abstractions/multiprogramming/worker_pool.py

@@ -0,0 +1,98 @@
+from queue import Queue
+import logging
+from typing import Optional, Any, Type as t_type, Tuple as Tuple, List as List
+from threading import Semaphore
+from .worker import Worker
+from ...reflection import get_python_version
+from ...logging_.utils import get_logger
+
+if get_python_version() >= (3, 9):
+    from builtins import type as t_type, tuple as Tuple, list as List  # type:ignore
+logger = get_logger(__name__)
+
+
+class WorkerPool:
+    """A worker pool class
+    """
+
+    def __init__(self, num_workers: int, worker_class: t_type[Worker], w_kwargs: dict, global_variables: dict) -> None:
+        logger.info("Initializing WorkerPool with %s workers, worker_class=%s", num_workers, worker_class.__name__)
+        self.num_workers = num_workers
+        self.global_variables: dict = global_variables
+        self.q: Queue[Tuple[Any]] = Queue()
+        self.worker_class = worker_class
+        self.workers: List[Worker] = []
+        self.sem = Semaphore(0)
+        self.w_kwargs = w_kwargs
+        logger.debug("WorkerPool initialized: num_workers=%s, global_vars_count=%s, worker_kwargs_count=%s", num_workers, len(global_variables), len(w_kwargs))
+
+    def submit(self, job: Any) -> None:
+        """submit a job to the pool
+        the object can be anything you want as long as you use it 
+        correctly in your implemented worker class
+
+        Args:
+            job (Any): job object
+        """
+        logger.debug("Submitting job to pool: job_type=%s", type(job).__name__)
+        # we create a tuple to signal that it is indeed a job object and we haven't just got None
+        # see Worker._loop
+        self.q.put((job,))
+        self.sem.release()
+        logger.debug("Job submitted successfully, queue_size=%s, semaphore_value=%s", self.q.qsize(), self.sem._value)
+
+    def _acquire(self) -> Optional[Tuple[Any]]:
+        """acquire a new job from the pool
+
+        Returns:
+            Optional[tuple[Any]]: optional tuple of job object
+        """
+        logger.debug("Acquiring job from pool")
+        self.sem.acquire()
+        if self.q.unfinished_tasks > 0:
+            job = self.q.get()
+            logger.debug("Job acquired successfully, remaining_tasks=%s", self.q.unfinished_tasks)
+            return job
+        logger.debug("No jobs available in pool")
+        return None
+
+    def start(self) -> None:
+        """starts running the pool of workers
+        """
+        logger.info("Starting worker pool with %s workers", self.num_workers)
+        for i in range(self.num_workers):
+            logger.debug("Creating worker %s/%s", i+1, self.num_workers)
+            w = self.worker_class(i, self, **self.w_kwargs)
+            w.run()
+            self.workers.append(w)
+            logger.debug("Worker %s started successfully", i+1)
+        logger.info("Worker pool started with %s workers", len(self.workers))
+
+    def _notify(self) -> None:
+        """a function that the worker calls after finishing processing a job object (Any)
+        this function is called automatically from Worker.work()
+        """
+        logger.debug("Worker notification received, unfinished_tasks=%s", self.q.unfinished_tasks)
+        self.q.task_done()
+        if self.q.unfinished_tasks <= 0:
+            logger.debug("All tasks completed, releasing all workers")
+            self.sem.release(self.num_workers)
+        logger.debug("Notification processed, remaining_tasks=%s", self.q.unfinished_tasks)
+
+    def join(self) -> None:
+        """
+        waits for all the workers to finish and will return afterwards
+        Returns:
+            None
+        """
+        logger.info("Joining worker pool with %s workers", len(self.workers))
+        for i, w in enumerate(self.workers):
+            logger.debug("Joining worker %s/%s", i+1, len(self.workers))
+            w.thread.join()
+            logger.debug("Worker %s joined successfully", i+1)
+        logger.info("All workers joined successfully")
+
+
+__all__ = [
+    "WorkerPool"
+]

danielutils/abstractions/repl.py

@@ -0,0 +1,125 @@
+import copy, re
+import logging
+from typing import Any, Callable, Union, Tuple as Tuple, List as List, Dict as Dict
+from ..reflection import get_python_version  # pylint :disable=relative-beyond-top-level
+from ..logging_.utils import get_logger
+
+if get_python_version() >= (3, 9):
+    from builtins import tuple as Tuple, list as List, dict as Dict  # type:ignore
+logger = get_logger(__name__)
+
+
+class Argument:
+    """a class to wrap an argument"""
+
+    def __init__(self, name: str, optional: bool = False, flag: bool = False) -> None:
+        logger.debug("Creating Argument: name=%s, optional=%s, flag=%s", name, optional, flag)
+        self.name = name
+        self.optional = optional
+        self.flag = flag
+        logger.debug("Argument created successfully: %s", name)
+
+
+class Command:
+    """a class to wrap a command
+    """
+
+    def __init__(self, command: Union[Argument, str], callback: Callable,
+                 explanation: str = "", *, options: Tuple[Argument, ...] = tuple()) -> None:
+        logger.debug("Creating Command: command=%s, explanation_length=%d, options_count=%d", command if isinstance(command, str) else command.name, len(explanation), len(options))
+        self.command = command if isinstance(
+            command, Argument) else Argument(command)
+        self.callback = callback
+        self.explanation = explanation
+        self.options = options
+        logger.debug("Command created successfully: %s", self.command.name)
+
+    def __call__(self, *args: Any, **kwargs: Any) -> Any:
+        logger.debug("Command '%s' called with args_count=%d, kwargs_count=%d", self.command.name, len(args), len(kwargs))
+        if len(args) > 0:
+            if args[0] == "help":
+                logger.info("Help requested for command: %s", self.command.name)
+                if self.explanation != "":
+                    print(self.explanation)
+                    logger.debug("Help explanation displayed for command: %s", self.command.name)
+                    return None
+        try:
+            result = self.callback(*args, **kwargs)
+            logger.debug("Command '%s' executed successfully", self.command.name)
+            return result
+        except Exception as e:
+            logger.error("Command '%s' failed with %s: %s", self.command.name, type(e).__name__, e)
+            raise
+
+
+class REPL:
+    """a class to easily create a shell application and get functionality for free
+    """
+
+    # pylint: disable=dangerous-default-value
+    def __init__(self, routes: List[Command], *, prompt_symbol: str = ">>> ", exit_keywords: set = {"exit", "quit"}):
+        logger.info("Initializing REPL with %d commands, prompt='%s', exit_keywords=%s", len(routes), prompt_symbol, exit_keywords)
+        self.prompt_symbol = prompt_symbol
+        self.exit_keywords = copy.copy(exit_keywords)
+        self.routes: Dict[str, Command] = {
+            com.command.name: com for com in routes}
+        logger.debug("REPL initialized successfully with commands: %s", list(self.routes.keys()))
+
+    def run(self) -> None:
+        """runs the main loop for the shell
+
+        Raises:
+            e: any error if there is any
+        """
+        logger.info("Starting REPL main loop")
+        while True:
+            prompt = input(self.prompt_symbol)
+            logger.debug("User input received: '%s'", prompt)
+            
+            if prompt in self.exit_keywords:
+                logger.info("Exit keyword '%s' received, stopping REPL", prompt)
+                break
+
+            if prompt == "help":
+                logger.info("Help command requested")
+                print("Available commands:")
+                for com in list(self.routes.keys()) + list(self.exit_keywords):
+                    print(f"\t{com}")
+                logger.debug("Help displayed with %d commands", len(self.routes) + len(self.exit_keywords))
+                continue
+
+            prompt_parts = prompt.split()
+            command = prompt_parts[0]
+            logger.debug("Processing command: '%s' with %d arguments", command, len(prompt_parts)-1)
+            
+            if command in self.routes:
+                try:
+                    logger.debug("Executing command: %s", command)
+                    self.routes[command](*prompt_parts[1:])
+                    logger.debug("Command '%s' completed successfully", command)
+                except TypeError as e:
+                    msg = str(e)
+                    logger.warning("TypeError in command '%s': %s", command, msg)
+                    if re.match(r".*missing.*required.*argument.*", msg):
+                        print(f"'{command}' " + msg[msg.find("missing"):])
+                    elif re.match(r".*takes.*arguments but.*given", msg):
+                        print(f"'{command}' " + msg[msg.find("takes"):])
+                    else:
+                        logger.error("Unhandled TypeError in command '%s': %s", command, e)
+                        raise e
+                except Exception as e:
+                    logger.error("Unexpected error in command '%s': %s: %s", command, type(e).__name__, e)
+                    raise e
+            else:
+                logger.warning("Invalid command '%s' received", command)
+                print(
+                    "Invalid command. for help type 'help'.\nOr additionally you may try a command and then 'help'")
+        
+        logger.info("REPL main loop ended")
+
+
+__all__ = [
+    "REPL",
+    "Command",
+    "Argument"
+]

danielutils/aliases.py

@@ -0,0 +1,174 @@
+# import inspect
+# from abc import ABC, abstractmethod
+# from typing import cast, Generic, ParamSpec, TypeVar, Callable
+# from .functions import isoftype
+# from .reflection.Function import get_caller_name, get_function_return_type
+# P = ParamSpec("P")
+# A = TypeVar("A")
+# B = TypeVar("B")
+
+
+# class SubscribableBase(ABC):
+#     """An abstract class that need __instanceCheck__ to be implemented
+#         will create default functions such that isinstance will work with the new type
+#         while it is used as all of the python's generic types
+#     """
+#     __slots__ = ("params",)
+
+#     def __init__(self, *params):
+#         if get_caller_name() != "__class_getitem__":
+#             raise ValueError(
+#                 f"{self.__class__.__name__} is a type-hint class, Cant instantiate.")
+#         self.params = params[0] if type(params) in {list, tuple} else params
+
+#     @classmethod
+#     def __class_getitem__(cls, *params) -> type:
+#         res: type = cast(type, cls(*params))
+#         return res
+
+#     @abstractmethod
+#     def __instancecheck__(self, value) -> bool: ...
+
+
+# class Supplier(SubscribableBase):
+#     """create a type for functions to be a supplier function
+#     """
+
+#     def __instancecheck__(self, func) -> bool:
+#         if not callable(func):
+#             return False
+#         signature = inspect.signature(func)
+#         if len(signature.parameters) != 0:
+#             return False
+
+#         return_type = get_function_return_type(func)
+#         if return_type is not None:
+#             try:
+#                 return isoftype(return_type(), self.params)
+#             except:
+#                 return False
+#         return self.params is None
+
+
+# class Consumer(SubscribableBase):
+#     """create a type for functions to be a consumer function
+#     """
+
+#     def __instancecheck__(self, func) -> bool:
+#         if not callable(func):
+#             return False
+#         if get_function_return_type(func) is not type(None):
+#             return False
+#         signature = inspect.signature(func)
+#         if len(signature.parameters) != 1:
+#             return False
+#         param_type = next(iter(signature.parameters.values())).annotation
+#         try:
+#             return isoftype(param_type(), self.params)
+#         except:
+#             return False
+
+
+# class BinaryConsumer(SubscribableBase, Generic[A, B]):
+#     """create a type for functions to be a binary consumer function
+#     """
+
+#     def __instancecheck__(self, func) -> bool:
+#         if self.params[-1] is not None:
+#             return False
+#         self.params = self.params[0]
+#         if not callable(func):
+#             return False
+#         if get_function_return_type(func) is not type(None):
+#             return False
+#         signature = inspect.signature(func)
+#         if len(signature.parameters) != 2:
+#             return False
+#         param_iter = iter(signature.parameters.values())
+#         param_types = (next(param_iter).annotation(),
+#                        next(param_iter).annotation())
+#         try:
+#             for value, value_type in zip(param_types, self.params):
+#                 if not isoftype(value, value_type):
+#                     return False
+#             return True
+#         except:
+#             return False
+
+# # class Comparer(SubscribableBase):
+# #     def __instancecheck__(self, func) -> bool:
+# #         if not callable(func):
+# #             return False
+# #         if get_function_return_type(func) not in {int, float, Union[int, float]}:
+# #             return False
+# #         signature = inspect.signature(func)
+# #         if len(signature.parameters) != 2:
+# #             return False
+# #         param_iter = iter(signature.parameters.values())
+# #         param1_type = next(param_iter).annotation
+# #         param2_type = next(param_iter).annotation
+# #         try:
+# #             return param1_type == param2_type == self.params[0] == self.params[1]
+# #         except:
+# #             return False
+# # class Predicate(SubscribableBase):
+# #     """create a type for functions to be a consumer function
+# #     """
+
+# #     def __instancecheck__(self, func) -> bool:
+# #         if not callable(func):
+# #             return False
+# #         signature = inspect.signature(func)
+# #         if len(signature.parameters) != 0:
+# #             return False
+
+# #         return_type = get_function_return_type(func)
+# #         try:
+# #             return isoftype(return_type(), self.params)
+# #         except:
+# #             return False
+
+
+# # ListTupleType: TypeAlias = Union[list, tuple]
+# __all__ = [
+#     "SubscribableBase",
+#     "Supplier",
+#     "Consumer",
+#     "BinaryConsumer",
+#     # "Predicate"
+#     # "ListTupleType"
+# ]
+from typing import Callable, TypeVar, Union
+
+Number = Union[int, float, complex]
+
+A = TypeVar("A")
+B = TypeVar("B")
+U = TypeVar("U")
+T = TypeVar("T")
+# P = ParamSpec("P")
+
+Supplier = Callable[[], T]
+Runnable = Supplier[None]
+Consumer = Callable[[T], None]
+UnaryFunction = Callable[[U], T]
+UnaryOperator = Callable[[T], T]
+Predicate = Callable[[T], bool]
+
+BinaryFunction = Callable[[A, B], T]
+BinaryOperator = Callable[[T, T], T]
+Comparator = Callable[[A, B], int]
+BinaryConsumer = Callable[[T, T], None]
+
+__all__ = [
+    "UnaryFunction",
+    "BinaryFunction",
+    "UnaryOperator",
+    "BinaryOperator",
+    "Supplier",
+    "Predicate",
+    "Comparator",
+    "Consumer",
+    "BinaryConsumer",
+    "Runnable",
+]

danielutils/async_/__init__.py

@@ -0,0 +1,7 @@
+from .async_cmd import *
+from .async_command import *
+from .async_layered_command import *
+from .async_retry_executor import *
+from .async_worker_pool import *
+from .time_strategy import *
+from .utils import *

danielutils/async_/async_cmd.py

@@ -0,0 +1,28 @@
+import asyncio
+import logging
+from typing import Tuple, Optional
+from ..logging_.utils import get_logger
+logger = get_logger(__name__)
+
+
+async def async_cmd(
+        cmd: str,
+        *,
+        capture_stdout: bool = False,
+        capture_stderr: bool = False
+) -> Tuple[int, Optional[bytes], Optional[bytes]]:
+    logger.debug("Executing async command: %s, capture_stdout=%s, capture_stderr=%s", cmd, capture_stdout, capture_stderr)
+    kwargs = {}
+    if capture_stdout:
+        kwargs['stdout'] = asyncio.subprocess.PIPE
+    if capture_stderr:
+        kwargs['stderr'] = asyncio.subprocess.PIPE
+    process = await asyncio.create_subprocess_shell(cmd, **kwargs)  # type:ignore
+    stdout, stderr = await process.communicate()
+    logger.debug("Command completed with returncode=%s", process.returncode)
+    return process.returncode or 0, stdout, stderr
+
+
+__all__ = [
+    'async_cmd',
+]