mm-std 0.3.12__py3-none-any.whl → 0.3.14__py3-none-any.whl

mm_std/__init__.py

@@ -1,12 +1,13 @@
-from .async_concurrency import AsyncScheduler as AsyncScheduler
-from .async_concurrency import async_synchronized as async_synchronized
 from .command import CommandResult as CommandResult
 from .command import run_command as run_command
 from .command import run_ssh_command as run_ssh_command
-from .concurrency import ConcurrentTasks as ConcurrentTasks
-from .concurrency import Scheduler as Scheduler
-from .concurrency import synchronized as synchronized
-from .concurrency import synchronized_parameter as synchronized_parameter
+from .concurrency.async_decorators import async_synchronized as async_synchronized
+from .concurrency.async_scheduler import AsyncScheduler as AsyncScheduler
+from .concurrency.async_task_runner import AsyncTaskRunner as AsyncTaskRunner
+from .concurrency.sync_decorators import synchronized as synchronized
+from .concurrency.sync_decorators import synchronized_parameter as synchronized_parameter
+from .concurrency.sync_scheduler import Scheduler as Scheduler
+from .concurrency.sync_task_runner import ConcurrentTasks as ConcurrentTasks
 from .config import BaseConfig as BaseConfig
 from .crypto import fernet_decrypt as fernet_decrypt
 from .crypto import fernet_encrypt as fernet_encrypt

mm_std/concurrency/async_decorators.py

@@ -0,0 +1,19 @@
+import functools
+from collections.abc import Awaitable, Callable
+from typing import ParamSpec, TypeVar
+
+import anyio
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+
+def async_synchronized(func: Callable[P, Awaitable[R]]) -> Callable[P, Awaitable[R]]:
+    lock = anyio.Lock()
+
+    @functools.wraps(func)
+    async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+        async with lock:
+            return await func(*args, **kwargs)
+
+    return wrapper

mm_std/{async_concurrency.py → concurrency/async_scheduler.py}

@@ -1,17 +1,12 @@
-import functools
 import threading
-from collections.abc import Awaitable, Callable
 from dataclasses import dataclass, field
-from datetime import UTC, datetime
+from datetime import datetime
 from logging import Logger
-from typing import ParamSpec, TypeVar
 
 import anyio
 
-# Type aliases for clarity
-AsyncFunc = Callable[..., Awaitable[object]]
-Args = tuple[object, ...]
-Kwargs = dict[str, object]
+from mm_std.date import utc_now
+from mm_std.types_ import Args, AsyncFunc, Kwargs
 
 
 class AsyncScheduler:
@@ -23,6 +18,7 @@ class AsyncScheduler:
         args: Args = ()
         kwargs: Kwargs = field(default_factory=dict)
         run_count: int = 0
+        error_count: int = 0
         last_run: datetime | None = None
         running: bool = False
 
@@ -45,15 +41,16 @@ class AsyncScheduler:
         """Internal loop for running a single task repeatedly."""
         task = self.tasks[task_id]
         while self._running:
-            task.last_run = datetime.now(UTC)
+            task.last_run = utc_now()
             task.run_count += 1
             try:
                 await task.func(*task.args, **task.kwargs)
             except Exception:
+                task.error_count += 1
                 self._logger.exception("AsyncScheduler exception")
 
             # Calculate elapsed time and sleep if needed so that tasks never overlap.
-            elapsed = (datetime.now(UTC) - task.last_run).total_seconds()
+            elapsed = (utc_now() - task.last_run).total_seconds()
             sleep_time = task.interval - elapsed
             if sleep_time > 0:
                 try:
@@ -107,18 +104,3 @@ class AsyncScheduler:
             self._thread.join(timeout=5)
             self._thread = None
         self._logger.debug("AsyncScheduler stopped")
-
-
-P = ParamSpec("P")
-R = TypeVar("R")
-
-
-def async_synchronized(func: Callable[P, Awaitable[R]]) -> Callable[P, Awaitable[R]]:
-    lock = anyio.Lock()
-
-    @functools.wraps(func)
-    async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
-        async with lock:
-            return await func(*args, **kwargs)
-
-    return wrapper

mm_std/concurrency/async_task_runner.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from typing import Any
+
+import anyio
+
+
+class AsyncTaskRunner:
+    """
+    AsyncTaskRunner executes a batch of asynchronous tasks with controlled concurrency.
+    Note: This runner is designed for one-time use. Create a new instance for each batch of tasks.
+    """
+
+    @dataclass
+    class Result:
+        results: dict[str, Any]  # Maps task_id to result
+        exceptions: dict[str, Any]  # Maps task_id to exception (if any)
+        is_ok: bool  # True if no exception and no timeout occurred
+        is_timeout: bool  # True if at least one task was cancelled due to timeout
+
+    @dataclass
+    class Task:
+        """Individual task representation"""
+
+        task_id: str
+        async_func: Callable[..., Awaitable[Any]]
+        args: tuple[Any, ...]
+        kwargs: dict[str, Any]
+
+    def __init__(self, max_concurrent_tasks: int, timeout: float | None = None) -> None:
+        """
+        :param max_concurrent_tasks: Maximum number of tasks that can run concurrently.
+        :param timeout: Optional overall timeout in seconds for running all tasks.
+        """
+        if timeout is not None and timeout <= 0:
+            raise ValueError("Timeout must be positive if specified.")
+        self.max_concurrent_tasks: int = max_concurrent_tasks
+        self.timeout: float | None = timeout
+        self.limiter: anyio.CapacityLimiter = anyio.CapacityLimiter(max_concurrent_tasks)
+        self._tasks: list[AsyncTaskRunner.Task] = []
+        self._was_run: bool = False
+        self._task_ids: set[str] = set()
+
+    def add_task(
+        self,
+        task_id: str,
+        async_func: Callable[..., Awaitable[Any]],
+        *args: object,
+        **kwargs: object,
+    ) -> None:
+        """
+        Adds a task to the runner that will be executed when run() is called.
+
+        :param task_id: Unique identifier for the task.
+        :param async_func: The asynchronous function to execute.
+        :param args: Positional arguments for async_func.
+        :param kwargs: Keyword arguments for async_func.
+        :raises RuntimeError: If the runner has already been used.
+        :raises ValueError: If task_id is empty or already exists.
+        """
+        if self._was_run:
+            raise RuntimeError("This AsyncTaskRunner has already been used. Create a new instance for new tasks.")
+
+        if not task_id:
+            raise ValueError("Task ID cannot be empty")
+
+        if task_id in self._task_ids:
+            raise ValueError(f"Task ID '{task_id}' already exists. All task IDs must be unique.")
+
+        self._task_ids.add(task_id)
+        self._tasks.append(AsyncTaskRunner.Task(task_id, async_func, args, kwargs))
+
+    async def run(self) -> AsyncTaskRunner.Result:
+        """
+        Executes all added tasks with concurrency limited by the capacity limiter.
+        If a timeout is specified, non-finished tasks are cancelled.
+
+        :return: AsyncTaskRunner.Result containing task results, exceptions, and flags indicating overall status.
+        :raises RuntimeError: If the runner has already been used.
+        """
+        if self._was_run:
+            raise RuntimeError("This AsyncTaskRunner instance can only be run once. Create a new instance for new tasks.")
+
+        self._was_run = True
+        results: dict[str, Any] = {}
+        exceptions: dict[str, Any] = {}
+        is_timeout: bool = False
+
+        async def run_task(task: AsyncTaskRunner.Task) -> None:
+            async with self.limiter:
+                try:
+                    res: Any = await task.async_func(*task.args, **task.kwargs)
+                    results[task.task_id] = res
+                except Exception as e:
+                    exceptions[task.task_id] = e
+
+        try:
+            if self.timeout is not None:
+                with anyio.fail_after(self.timeout):
+                    async with anyio.create_task_group() as tg:
+                        for task in self._tasks:
+                            tg.start_soon(run_task, task)
+            else:
+                async with anyio.create_task_group() as tg:
+                    for task in self._tasks:
+                        tg.start_soon(run_task, task)
+        except TimeoutError:
+            is_timeout = True
+
+        is_ok: bool = (not exceptions) and (not is_timeout)
+        return AsyncTaskRunner.Result(results=results, exceptions=exceptions, is_ok=is_ok, is_timeout=is_timeout)

mm_std/concurrency/sync_decorators.py

@@ -0,0 +1,35 @@
+import functools
+from collections import defaultdict
+from collections.abc import Callable
+from threading import Lock
+
+
+def synchronized_parameter[T, **P](arg_index: int = 0, skip_if_locked: bool = False) -> Callable[..., Callable[P, T | None]]:
+    locks: dict[object, Lock] = defaultdict(Lock)
+
+    def outer(func: Callable[P, T]) -> Callable[P, T | None]:
+        @functools.wraps(func)
+        def wrapper(*args: P.args, **kwargs: P.kwargs) -> T | None:
+            if skip_if_locked and locks[args[arg_index]].locked():
+                return None
+            try:
+                with locks[args[arg_index]]:
+                    return func(*args, **kwargs)
+            finally:
+                locks.pop(args[arg_index], None)
+
+        wrapper.locks = locks  # type: ignore[attr-defined]
+        return wrapper
+
+    return outer
+
+
+def synchronized[T, **P](fn: Callable[P, T]) -> Callable[P, T]:
+    lock = Lock()
+
+    @functools.wraps(fn)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+        with lock:
+            return fn(*args, **kwargs)
+
+    return wrapper

mm_std/concurrency/sync_scheduler.py

@@ -0,0 +1,73 @@
+import time
+from dataclasses import dataclass, field
+from datetime import datetime
+from logging import Logger
+from threading import Thread
+
+from mm_std.date import is_too_old, utc_now
+from mm_std.types_ import Func
+
+
+class Scheduler:
+    def __init__(self, log: Logger, loop_delay: float = 0.5, debug: bool = False) -> None:
+        self.log = log
+        self.debug = debug
+        self.loop_delay = loop_delay
+        self.stopped = False
+        self.jobs: list[Scheduler.Job] = []
+        self.run_immediately_jobs: list[Scheduler.Job] = []
+        self._debug("init")
+
+    @dataclass
+    class Job:
+        func: Func
+        args: tuple[object, ...]
+        interval: int
+        is_running: bool = False
+        last_at: datetime = field(default_factory=utc_now)
+
+        def __str__(self) -> str:
+            return str(self.func)
+
+    def add_job(self, func: Func, interval: int, args: tuple[object, ...] = (), run_immediately: bool = False) -> None:
+        job = Scheduler.Job(func, args, interval)
+        self.jobs.append(job)
+        if run_immediately:
+            self.run_immediately_jobs.append(job)
+
+    def _run_job(self, job: Job) -> None:
+        self._debug(f"_run_job: {job}")
+        if self.stopped:
+            return
+        try:
+            job.func(*job.args)
+            self._debug(f"_run_job: {job} done")
+        except Exception:
+            self.log.exception("scheduler error")
+            self._debug(f"_run_job: {job} error")
+        finally:
+            job.is_running = False
+
+    def _start(self) -> None:
+        self._debug(f"_start: jobs={len(self.jobs)}, run_immediately_jobs={len(self.run_immediately_jobs)}")
+        for j in self.run_immediately_jobs:
+            j.is_running = True
+            j.last_at = utc_now()
+            Thread(target=self._run_job, args=(j,)).start()
+        while not self.stopped:
+            for j in self.jobs:
+                if not j.is_running and is_too_old(j.last_at, j.interval):
+                    j.is_running = True
+                    j.last_at = utc_now()
+                    Thread(target=self._run_job, args=(j,)).start()
+            time.sleep(self.loop_delay)
+
+    def _debug(self, message: str) -> None:
+        if self.debug:
+            self.log.debug("Scheduler: %s", message)
+
+    def start(self) -> None:
+        Thread(target=self._start).start()
+
+    def stop(self) -> None:
+        self.stopped = True

mm_std/concurrency/sync_task_runner.py

@@ -0,0 +1,45 @@
+import concurrent
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+
+from mm_std.types_ import Args, Func, Kwargs
+
+
+class ConcurrentTasks:
+    def __init__(self, max_workers: int = 5, timeout: int | None = None, thread_name_prefix: str = "concurrent_tasks") -> None:
+        self.max_workers = max_workers
+        self.timeout = timeout
+        self.thread_name_prefix = thread_name_prefix
+        self.tasks: list[ConcurrentTasks.Task] = []
+        self.exceptions: dict[str, Exception] = {}
+        self.error = False
+        self.timeout_error = False
+        self.result: dict[str, object] = {}
+
+    @dataclass
+    class Task:
+        key: str
+        func: Func
+        args: Args
+        kwargs: Kwargs
+
+    def add_task(self, key: str, func: Func, args: Args = (), kwargs: Kwargs | None = None) -> None:
+        if kwargs is None:
+            kwargs = {}
+        self.tasks.append(ConcurrentTasks.Task(key, func, args, kwargs))
+
+    def execute(self) -> None:
+        with ThreadPoolExecutor(self.max_workers, thread_name_prefix=self.thread_name_prefix) as executor:
+            future_to_key = {executor.submit(task.func, *task.args, **task.kwargs): task.key for task in self.tasks}
+            try:
+                result_map = concurrent.futures.as_completed(future_to_key, timeout=self.timeout)
+                for future in result_map:
+                    key = future_to_key[future]
+                    try:
+                        self.result[key] = future.result()
+                    except Exception as err:
+                        self.error = True
+                        self.exceptions[key] = err
+            except concurrent.futures.TimeoutError:
+                self.error = True
+                self.timeout_error = True

mm_std/types_.py

@@ -1,4 +1,8 @@
-from collections.abc import Callable
+from collections.abc import Awaitable, Callable
 from typing import Any
 
 type CallableAny = Callable[..., Any]
+type Func = Callable[..., object]
+type AsyncFunc = Callable[..., Awaitable[object]]
+type Args = tuple[object, ...]
+type Kwargs = dict[str, object]

{mm_std-0.3.12.dist-info → mm_std-0.3.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mm-std
-Version: 0.3.12
+Version: 0.3.14
 Requires-Python: >=3.12
 Requires-Dist: anyio>=4.9.0
 Requires-Dist: cryptography~=44.0.2

{mm_std-0.3.12.dist-info → mm_std-0.3.14.dist-info}/RECORD

@@ -1,7 +1,5 @@
-mm_std/__init__.py,sha256=BJzVoqux6-YWxGMyXJS34v6CgKh3V7SrpV7eXIUVfbw,2718
-mm_std/async_concurrency.py,sha256=IfnOx1Cwkc134P9F9_giNTyEYIXPj-LdzafPZAwA-5Y,4365
+mm_std/__init__.py,sha256=UkRzp_lm7yddXImD8nRlJtBnx9YlopWJKqukcbqjRyo,2881
 mm_std/command.py,sha256=ze286wjUjg0QSTgIu-2WZks53_Vclg69UaYYgPpQvCU,1283
-mm_std/concurrency.py,sha256=4kKLhde6YQYsjJJjH6K5eMQj6FtegEz55Mo5TmhQMM0,5242
 mm_std/config.py,sha256=4ox4D2CgGR76bvZ2n2vGQOYUDagFnlKEDb87to5zpxE,1871
 mm_std/crypto.py,sha256=jdk0_TCmeU0pPXMyz9xH6kQHSjjZ9GcGClBwQps5vBo,340
 mm_std/date.py,sha256=976eEkSONuNqHQBgSRu8hrtH23tJqztbmHFHLdbP2TY,1879
@@ -18,8 +16,15 @@ mm_std/random_.py,sha256=OuUX4VJeSd13NZBya4qrGpR2TfN7_87tfebOY6DBUnI,1113
 mm_std/result.py,sha256=KLnPWjICYFkP6CAhq7Ifs22XSD-PQ9RkG6n1-cZcXkM,7625
 mm_std/str.py,sha256=BEjJ1p5O4-uSYK0h-enasSSDdwzkBbiwdQ4_dsrlEE8,3257
 mm_std/toml.py,sha256=CNznWKR0bpOxS6e3VB5LGS-Oa9lW-wterkcPUFtPcls,610
-mm_std/types_.py,sha256=hvZlnvBWyB8CL_MeEWWD0Y0nN677plibYn3yD-5g7xs,99
+mm_std/types_.py,sha256=9FGd2q47a8M9QQgsWJR1Kq34jLxBAkYSoJuwih4PPqg,257
 mm_std/zip.py,sha256=axzF1BwcIygtfNNTefZH7hXKaQqwe-ZH3ChuRWr9dnk,396
-mm_std-0.3.12.dist-info/METADATA,sha256=G3VmqlSd0N1ctXXLUyQWU8pJYHBLW98sYKoPsz97clE,410
-mm_std-0.3.12.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-mm_std-0.3.12.dist-info/RECORD,,
+mm_std/concurrency/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mm_std/concurrency/async_decorators.py,sha256=tJ5u-_G9mAF3zrFr1aBUhYh03rB-K_6oiDKGdmGdlGk,449
+mm_std/concurrency/async_scheduler.py,sha256=NsR9nrxAEdiYjGL-lTdDlQAXJHJZBUYnm26h-0zaxCg,3932
+mm_std/concurrency/async_task_runner.py,sha256=D4fD_hwSVy55sM_-U1_sdVYlkahnQeigXdMfbIi8Rpc,4492
+mm_std/concurrency/sync_decorators.py,sha256=syCQBOmN7qPO55yzgJB2rbkh10CVww376hmyvs6e5tA,1080
+mm_std/concurrency/sync_scheduler.py,sha256=j4tBL_cBI1spr0cZplTA7N2CoYsznuORMeRN8rpR6gY,2407
+mm_std/concurrency/sync_task_runner.py,sha256=s5JPlLYLGQGHIxy4oDS-PN7O9gcy-yPZFoNm8RQwzcw,1780
+mm_std-0.3.14.dist-info/METADATA,sha256=sH9DFjRqCXpYR3okXUVWFsTNlbyEk98QxYIfYGbFiXg,410
+mm_std-0.3.14.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+mm_std-0.3.14.dist-info/RECORD,,

mm_std/concurrency.py

@@ -1,152 +0,0 @@
-import concurrent.futures
-import functools
-import time
-from collections import defaultdict
-from collections.abc import Callable
-from concurrent.futures.thread import ThreadPoolExecutor
-from dataclasses import dataclass, field
-from datetime import datetime
-from logging import Logger
-from threading import Lock, Thread
-
-from .date import is_too_old, utc_now
-
-type Func = Callable[..., object]
-type Args = tuple[object, ...]
-type Kwargs = dict[str, object]
-
-
-class ConcurrentTasks:
-    def __init__(self, max_workers: int = 5, timeout: int | None = None, thread_name_prefix: str = "concurrent_tasks") -> None:
-        self.max_workers = max_workers
-        self.timeout = timeout
-        self.thread_name_prefix = thread_name_prefix
-        self.tasks: list[ConcurrentTasks.Task] = []
-        self.exceptions: dict[str, Exception] = {}
-        self.error = False
-        self.timeout_error = False
-        self.result: dict[str, object] = {}
-
-    @dataclass
-    class Task:
-        key: str
-        func: Func
-        args: Args
-        kwargs: Kwargs
-
-    def add_task(self, key: str, func: Func, args: Args = (), kwargs: Kwargs | None = None) -> None:
-        if kwargs is None:
-            kwargs = {}
-        self.tasks.append(ConcurrentTasks.Task(key, func, args, kwargs))
-
-    def execute(self) -> None:
-        with ThreadPoolExecutor(self.max_workers, thread_name_prefix=self.thread_name_prefix) as executor:
-            future_to_key = {executor.submit(task.func, *task.args, **task.kwargs): task.key for task in self.tasks}
-            try:
-                result_map = concurrent.futures.as_completed(future_to_key, timeout=self.timeout)
-                for future in result_map:
-                    key = future_to_key[future]
-                    try:
-                        self.result[key] = future.result()
-                    except Exception as err:
-                        self.error = True
-                        self.exceptions[key] = err
-            except concurrent.futures.TimeoutError:
-                self.error = True
-                self.timeout_error = True
-
-
-def synchronized_parameter[T, **P](arg_index: int = 0, skip_if_locked: bool = False) -> Callable[..., Callable[P, T | None]]:
-    locks: dict[object, Lock] = defaultdict(Lock)
-
-    def outer(func: Callable[P, T]) -> Callable[P, T | None]:
-        @functools.wraps(func)
-        def wrapper(*args: P.args, **kwargs: P.kwargs) -> T | None:
-            if skip_if_locked and locks[args[arg_index]].locked():
-                return None
-            try:
-                with locks[args[arg_index]]:
-                    return func(*args, **kwargs)
-            finally:
-                locks.pop(args[arg_index], None)
-
-        wrapper.locks = locks  # type: ignore[attr-defined]
-        return wrapper
-
-    return outer
-
-
-def synchronized[T, **P](fn: Callable[P, T]) -> Callable[P, T]:
-    lock = Lock()
-
-    @functools.wraps(fn)
-    def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
-        with lock:
-            return fn(*args, **kwargs)
-
-    return wrapper
-
-
-class Scheduler:
-    def __init__(self, log: Logger, loop_delay: float = 0.5, debug: bool = False) -> None:
-        self.log = log
-        self.debug = debug
-        self.loop_delay = loop_delay
-        self.stopped = False
-        self.jobs: list[Scheduler.Job] = []
-        self.run_immediately_jobs: list[Scheduler.Job] = []
-        self._debug("init")
-
-    @dataclass
-    class Job:
-        func: Func
-        args: tuple[object, ...]
-        interval: int
-        is_running: bool = False
-        last_at: datetime = field(default_factory=utc_now)
-
-        def __str__(self) -> str:
-            return str(self.func)
-
-    def add_job(self, func: Func, interval: int, args: tuple[object, ...] = (), run_immediately: bool = False) -> None:
-        job = Scheduler.Job(func, args, interval)
-        self.jobs.append(job)
-        if run_immediately:
-            self.run_immediately_jobs.append(job)
-
-    def _run_job(self, job: Job) -> None:
-        self._debug(f"_run_job: {job}")
-        if self.stopped:
-            return
-        try:
-            job.func(*job.args)
-            self._debug(f"_run_job: {job} done")
-        except Exception:
-            self.log.exception("scheduler error")
-            self._debug(f"_run_job: {job} error")
-        finally:
-            job.is_running = False
-
-    def _start(self) -> None:
-        self._debug(f"_start: jobs={len(self.jobs)}, run_immediately_jobs={len(self.run_immediately_jobs)}")
-        for j in self.run_immediately_jobs:
-            j.is_running = True
-            j.last_at = utc_now()
-            Thread(target=self._run_job, args=(j,)).start()
-        while not self.stopped:
-            for j in self.jobs:
-                if not j.is_running and is_too_old(j.last_at, j.interval):
-                    j.is_running = True
-                    j.last_at = utc_now()
-                    Thread(target=self._run_job, args=(j,)).start()
-            time.sleep(self.loop_delay)
-
-    def _debug(self, message: str) -> None:
-        if self.debug:
-            self.log.debug("Scheduler: %s", message)
-
-    def start(self) -> None:
-        Thread(target=self._start).start()
-
-    def stop(self) -> None:
-        self.stopped = True