danielutils 1.0.35__tar.gz → 1.0.37__tar.gz

{danielutils-1.0.35/danielutils.egg-info → danielutils-1.0.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: danielutils
-Version: 1.0.35
+Version: 1.0.37
 Summary: A python utils library for things I find useful
 Author-email: danielnachumdev <danielnachumdev@gmail.com>
 License: MIT License

danielutils-1.0.37/danielutils/async_/__init__.py

@@ -0,0 +1,5 @@
+from .async_worker_pool import *
+from .async_worker_pool import *
+from .time_strategy import *
+from .utils import *
+from .async_cmd import *

danielutils-1.0.37/danielutils/async_/async_cmd.py

@@ -0,0 +1,23 @@
+import asyncio
+from typing import Tuple, Optional
+
+
+async def async_cmd(
+        cmd: str,
+        *,
+        capture_stdout: bool = False,
+        capture_stderr: bool = False
+) -> Tuple[int, Optional[bytes], Optional[bytes]]:
+    kwargs = {}
+    if capture_stdout:
+        kwargs['stdout'] = asyncio.subprocess.PIPE
+    if capture_stderr:
+        kwargs['stderr'] = asyncio.subprocess.PIPE
+    process = await asyncio.create_subprocess_shell(cmd, **kwargs)  # type:ignore
+    stdout, stderr = await process.communicate()
+    return process.returncode, stdout, stderr
+
+
+__all__ = [
+    'async_cmd',
+]

danielutils-1.0.37/danielutils/async_/async_layered_command.py

@@ -0,0 +1,109 @@
+import sys, os
+from typing import Optional, Tuple, List
+
+from .async_cmd import async_cmd
+from danielutils.context_managers import TemporaryFile
+import random
+
+
+class AsyncLayeredCommand:
+    class_capture_stdout: bool = True
+    class_capture_stderr: bool = True
+    class_raise_on_fail: bool = False
+    class_verbose: bool = False
+    _class_prev_instance: Optional['AsyncLayeredCommand'] = None
+    _id: int = 0
+
+    @property
+    def prev(self):
+        return self._prev_instance
+
+    @prev.setter
+    def prev(self, value: Optional['AsyncLayeredCommand'] = None):
+        self._prev_instance = value
+
+    def __init__(
+            self,
+            command: Optional[str] = None,
+            *,
+            prev_instance: Optional['AsyncLayeredCommand'] = None,
+            instance_capture_stdout: Optional[bool] = None,
+            instance_capture_stderr: Optional[bool] = None,
+            instance_raise_on_fail: Optional[bool] = None,
+            instance_verbose: Optional[bool] = None
+    ):
+        self._command = command if command is not None else ""
+        self._instance_capture_stdout = instance_capture_stdout
+        self._instance_capture_stderr = instance_capture_stderr
+        self._instance_raise_on_fail = instance_raise_on_fail
+        self._instance_verbose = instance_verbose
+        self._prev_instance = prev_instance if prev_instance is not None else AsyncLayeredCommand._class_prev_instance
+        self._cur_class_prev_instance = AsyncLayeredCommand._class_prev_instance
+        AsyncLayeredCommand._class_prev_instance = self
+        self._has_entered: bool = False
+
+    def __enter__(self):
+        self._open()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.prev is self._cur_class_prev_instance:
+            AsyncLayeredCommand._class_prev_instance = self.prev
+
+    def _open(self) -> None:
+        self._has_entered = True
+
+    def _build_command(self, *commands: str) -> str:
+        res = ""
+        if self.prev is not None:
+            prev = self.prev._build_command()
+            res += f"{prev} & " if prev != "" else ""
+        if self._command != "":
+            return res + " & ".join([self._command, *commands])
+        return res + " & ".join(commands)
+
+    def _error(self, predicate: bool, command: str, code: int, command_verbose: Optional[bool]) -> None:
+        if predicate:
+            verbose = self._merge_values(command_verbose, self._instance_verbose, AsyncLayeredCommand.class_verbose)
+            if verbose:
+                raise RuntimeError(f"command '{command}' failed with exit code {code}")
+            sys.exit(1)
+
+    @staticmethod
+    def _merge_values(command: Optional[bool], instance: Optional[bool], class_: bool) -> bool:
+        return command if command is not None else (instance if instance is not None else class_)
+
+    async def execute(
+            self,
+            *commands: str,
+            command_capture_stdout: Optional[bool] = None,
+            command_capture_stderr: Optional[bool] = None,
+            command_raise_on_fail: Optional[bool] = None,
+            command_verbose: Optional[bool] = None
+    ) -> Tuple[int, List[str], List[str]]:
+        if not self._has_entered:
+            raise RuntimeError(
+                "LayeredCommand must be used with a context manager. Use as: `with LayeredCommand(...) as l1:`")
+        capture_stdout = self._merge_values(command_capture_stdout, self._instance_capture_stdout,
+                                            self.class_capture_stdout)
+        capture_stderr = self._merge_values(command_capture_stderr, self._instance_capture_stderr,
+                                            self.class_capture_stderr)
+        raise_on_fail = self._merge_values(command_raise_on_fail, self._instance_raise_on_fail,
+                                           self.class_raise_on_fail)
+
+        command = self._build_command(*commands)
+        if not capture_stdout and not capture_stderr:
+            code = (await async_cmd(command))[0]
+            self._error(raise_on_fail and code != 0, command, code, command_verbose)
+            return code, [], []
+
+        code, stdout, stderr = await async_cmd(command, capture_stdout=True, capture_stderr=True)
+        return code, stdout.decode().splitlines(), stderr.decode().splitlines()
+
+    async def __call__(self, *args, **kwargs) -> Tuple[int, List[str], List[str]]:
+        return await self.execute(*args, **kwargs)
+
+
+__all__ = [
+    'AsyncLayeredCommand'
+]

danielutils-1.0.35/danielutils/async_/async_retry_executor_base.py → danielutils-1.0.37/danielutils/async_/async_retry_executor.py

@@ -12,7 +12,7 @@ from ..versioned_imports import ParamSpec
 P = ParamSpec("P")
 
 
-class AsyncRetryExecutorBase:
+class AsyncRetryExecutor:
     def __init__(
             self,
             timeout_strategy: Supplier[float] = LinearTimeStrategy(30, 5),
@@ -75,7 +75,7 @@ class AsyncRetryExecutorBase:
 
 @normalize_decorator
 def with_async_retry(func, *retry_executor_args, max_tries: int = 5, **retry_executor_kwargs):
-    retry_executor = AsyncRetryExecutorBase(*retry_executor_args, **retry_executor_kwargs)
+    retry_executor = AsyncRetryExecutor(*retry_executor_args, **retry_executor_kwargs)
 
     @functools.wraps(func)
     async def wrapper(*args, **kwargs):
@@ -85,6 +85,6 @@ def with_async_retry(func, *retry_executor_args, max_tries: int = 5, **retry_exe
 
 
 __all__ = [
-    "AsyncRetryExecutorBase",
+    "AsyncRetryExecutor",
     "with_async_retry"
 ]

danielutils-1.0.35/danielutils/async_/async_worker_pool_base.py → danielutils-1.0.37/danielutils/async_/async_worker_pool.py

@@ -2,33 +2,33 @@ import asyncio
 import json
 from typing import Callable, Literal, Optional, Coroutine
 
-from .async_retry_executor_base import AsyncRetryExecutorBase
 
-
-class AsyncWorkerPoolBase:
-    def __init__(self, num_workers: int, retry_executor: Optional[AsyncRetryExecutorBase] = None) -> None:
+class AsyncWorkerPool:
+    def __init__(self, pool_name: str, num_workers: int = 5) -> None:
         self.num_workers = num_workers
+        self.pool_name = pool_name
         self.queue = asyncio.Queue()
         self.workers = []
-        self.retry_executor = retry_executor
 
     async def worker(self, worker_id) -> None:
         """Worker coroutine that continuously fetches and executes tasks from the queue."""
-        count = 0
+        task_count = 0
         while True:
             task = await self.queue.get()
             if task is None:  # Sentinel value to shut down the worker
                 break
-            count += 1
+            task_count += 1
             func, args, kwargs = task
-            self.info(f"Worker {worker_id} started with task {count}")
-            if self.retry_executor:
-                await self.retry_executor.execute(func, args=args, kwargs=kwargs)
-            else:
+            self.info(f"Worker {worker_id} started with task {task_count}")
+
+            try:
                 await func(*args, **kwargs)
-            self.info(f"Worker {worker_id} finished with task {count}")
+            except Exception as e:
+                self.error(f"Worker {worker_id} failed task {task_count}", exception=e)
+
+            self.info(f"Worker {worker_id} finished with task {task_count}")
             self.queue.task_done()
-        self.info(f"Worker {worker_id} done.")
+        self.info(f"Worker {worker_id} done executed {task_count} tasks.")
 
     async def start(self) -> None:
         """Starts the worker pool."""
@@ -48,6 +48,7 @@ class AsyncWorkerPoolBase:
     def log(self, level: Literal["INFO", "WARNING", "ERROR"], message: str, **kwargs) -> None:
         kwargs["level"] = level
         kwargs["message"] = message
+        kwargs["pool"] = self.pool_name
         print(json.dumps(kwargs, default=str))
 
     def info(self, message: str, **kwargs) -> None:
@@ -61,5 +62,5 @@ class AsyncWorkerPoolBase:
 
 
 __all__ = [
-    "AsyncWorkerPoolBase",
+    "AsyncWorkerPool",
 ]

{danielutils-1.0.35 → danielutils-1.0.37}/danielutils/reflection/class_/class_reflection.py

@@ -115,7 +115,7 @@ class FunctionDeclaration:
             generics=self.generics,
         )
         dct.update(override_kwargs)
-        return FunctionDeclaration(**dct)
+        return FunctionDeclaration(**dct)  # type:ignore
 
     @property
     def has_generics(self) -> bool:
@@ -141,13 +141,13 @@ class FunctionDeclaration:
                         parameters.append(arg)
         res = []
         for code, name, args, ret in func_pattern.findall(src):
-            name: str = name.strip()
-            args: Optional[str] = remove_whitespace(args) if args is not None else None
+            name: str = name.strip()  # type:ignore
+            args: Optional[str] = remove_whitespace(args) if args is not None else None  # type:ignore
             arguments = []
             if args is not None:
                 for arg in split_args(args):
                     arguments.append(Argument(*arg_pattern.match(remove_whitespace(arg)).groups()))
-            ret: Optional[str] = ret.strip() if ret is not None and len(ret) != 0 else None
+            ret: Optional[str] = ret.strip() if ret is not None and len(ret) != 0 else None # type:ignore
             res.append(FunctionDeclaration(name, tuple(arguments), ret, decorators=None, generics=tuple(parameters)))
 
         return res

{danielutils-1.0.35 → danielutils-1.0.37/danielutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: danielutils
-Version: 1.0.35
+Version: 1.0.37
 Summary: A python utils library for things I find useful
 Author-email: danielnachumdev <danielnachumdev@gmail.com>
 License: MIT License

{danielutils-1.0.35 → danielutils-1.0.37}/danielutils.egg-info/SOURCES.txt

@@ -35,8 +35,10 @@ danielutils/abstractions/multiprogramming/multi_id.py
 danielutils/abstractions/multiprogramming/worker.py
 danielutils/abstractions/multiprogramming/worker_pool.py
 danielutils/async_/__init__.py
-danielutils/async_/async_retry_executor_base.py
-danielutils/async_/async_worker_pool_base.py
+danielutils/async_/async_cmd.py
+danielutils/async_/async_layered_command.py
+danielutils/async_/async_retry_executor.py
+danielutils/async_/async_worker_pool.py
 danielutils/async_/time_strategy.py
 danielutils/async_/utils.py
 danielutils/better_builtins/__init__.py

{danielutils-1.0.35 → danielutils-1.0.37}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "danielutils"
-version = "1.0.35"
+version = "1.0.37"
 authors = [
     { name = "danielnachumdev", email = "danielnachumdev@gmail.com" },
 ]

danielutils-1.0.35/danielutils/async_/__init__.py

@@ -1,4 +0,0 @@
-from .async_worker_pool_base import *
-from .async_worker_pool_base import *
-from .time_strategy import *
-from .utils import *