mpengine 0.4.1__tar.gz → 0.5.0__tar.gz

{mpengine-0.4.1/src/mpengine.egg-info → mpengine-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mpengine
-Version: 0.4.1
+Version: 0.5.0
 Summary: A small, general-purpose multiprocessing engine: dispatch any callable across processes with run manifests, per-worker logs, on-disk outputs and failure isolation
 Author: Amandeep Singh
 License-Expression: MIT
@@ -46,7 +46,7 @@ Every run produces three things:
 
 | what | where |
 |---|---|
-| a manifest of exactly what was launched, and how it ended | `<base>/manifests/<run_id>.txt` |
+| human-readable and versioned JSON manifests | `<base>/manifests/<run_id>.txt` and `.json` |
 | each job's result, saved to disk | `<base>/outputs/<run_id>/<label>` |
 | one log file per **worker process** | `<base>/logs/<run_id>/worker_<pid>.log` |
 | a log of the run as a whole | `<base>/logs/<run_id>/run.log` |
@@ -133,6 +133,10 @@ run with a custom `save_fn`, pass the matching reader as
 | `n_workers` | worker *process* count (not threads — see below); defaults to `os.cpu_count()`, clamped down to the number of jobs if there are fewer jobs than that |
 | `debug` | run sequentially in-process — real tracebacks you can attach a debugger to, no pool |
 | `show_progress` | live terminal display: one overall bar for the whole run, plus a live rate number per worker process. Ignored when `debug=True` |
+| `retries`, `retry_delay` | retry transient callback failures, optionally pausing between attempts |
+| `failure_policy` | `continue` (default), `raise` after all jobs, or `fail_fast` |
+| `timeout` | wall-clock limit for the complete parallel dispatch |
+| `cancel_token` | a thread-safe `CancellationToken` another thread can use to abort the run |
 
 Develop with `debug=True`, then flip it off. Chasing a bug through a process
 pool means reading a traceback re-raised from a worker that has already exited,
@@ -152,7 +156,7 @@ rather than quoting a remembered number.
 | `broadcast` | a `dict` of values shipped once per *worker* instead of once per job, delivered to your function as keyword arguments | 200 jobs + 80 MB panel **7.97s → 0.55s** |
 | `reuse_pool` | keep the pool (and its broadcast payload) alive between calls instead of rebuilding it. Off by default | 10 dispatches **9.86s → 0.02s** |
 | `initializer`, `initargs` | run once per worker before its first job — and unlike the stdlib's, may be a closure | — |
-| `max_tasks_per_child` | recycle a worker every N jobs, so a slow leak can't end the run (Python 3.11+) | — |
+| `max_tasks_per_child` | recycle a worker every N user jobs, so a slow leak can't end the run; disables batching to keep the count exact (Python 3.11+) | — |
 
 **BLAS threads** is the one that catches people out. numpy hands matrix work to
 a native library that is itself multi-threaded, and each worker process loads
@@ -176,6 +180,45 @@ it is a different order of magnitude — 0.55s on the 80 MB case, level with
 joblib's memmapped 0.58s. Call `shutdown_pools()` when you're done with a
 reused pool, or let `atexit` do it.
 
+For a large NumPy array, wrap it in `Shared` to copy it once into OS shared
+memory and let every worker read the same physical bytes:
+
+```python
+from mpengine import Shared
+
+summary = run(score, param_sets, base_dir="runs",
+              broadcast={"panel": Shared(panel)})
+```
+
+Shared arrays are read-only in workers and must be C-contiguous with a
+non-object dtype. Keep the original array alive while a reused pool needs it.
+
+### Reliability controls
+
+```python
+from mpengine import CancellationToken, RunFailedError, run
+
+token = CancellationToken()
+try:
+    summary = run(
+        score,
+        param_sets,
+        base_dir="runs",
+        retries=2,
+        retry_delay=0.25,
+        failure_policy="raise",
+        timeout=300,
+        cancel_token=token,
+    )
+except RunFailedError as exc:
+    summary = exc.summary  # outputs and both manifests are already durable
+```
+
+`failure_policy="continue"` preserves the original behavior. `"raise"` runs
+everything and then raises if any job failed. `"fail_fast"` terminates the
+pool after the first failed result. Timeout and cancellation also terminate
+the pool and raise exceptions carrying a partial `summary`.
+
 ### Logging
 
 The library logs its whole lifecycle through the standard `logging` module and
@@ -227,10 +270,13 @@ That's a cost trade-off to be aware of, not a limitation on what's allowed.
 you want to drive the pool yourself:
 
 ```python
-from mpengine import expand_call, process_jobs, process_jobs_
+from mpengine import expand_call, iter_jobs, process_jobs, process_jobs_
 
 jobs = [{"func": my_task, "x": 1, "y": 2}, {"func": other_task, "n": 5}]
 results = process_jobs(jobs)     # n_workers defaults to os.cpu_count(); or process_jobs_ to stay sequential
+
+for result in iter_jobs(jobs):   # consume each result as soon as it completes
+    store(result)
 ```
 
 A job is just a dict carrying its own callback plus that callback's kwargs, so a

{mpengine-0.4.1 → mpengine-0.5.0}/README.md

@@ -12,7 +12,7 @@ Every run produces three things:
 
 | what | where |
 |---|---|
-| a manifest of exactly what was launched, and how it ended | `<base>/manifests/<run_id>.txt` |
+| human-readable and versioned JSON manifests | `<base>/manifests/<run_id>.txt` and `.json` |
 | each job's result, saved to disk | `<base>/outputs/<run_id>/<label>` |
 | one log file per **worker process** | `<base>/logs/<run_id>/worker_<pid>.log` |
 | a log of the run as a whole | `<base>/logs/<run_id>/run.log` |
@@ -99,6 +99,10 @@ run with a custom `save_fn`, pass the matching reader as
 | `n_workers` | worker *process* count (not threads — see below); defaults to `os.cpu_count()`, clamped down to the number of jobs if there are fewer jobs than that |
 | `debug` | run sequentially in-process — real tracebacks you can attach a debugger to, no pool |
 | `show_progress` | live terminal display: one overall bar for the whole run, plus a live rate number per worker process. Ignored when `debug=True` |
+| `retries`, `retry_delay` | retry transient callback failures, optionally pausing between attempts |
+| `failure_policy` | `continue` (default), `raise` after all jobs, or `fail_fast` |
+| `timeout` | wall-clock limit for the complete parallel dispatch |
+| `cancel_token` | a thread-safe `CancellationToken` another thread can use to abort the run |
 
 Develop with `debug=True`, then flip it off. Chasing a bug through a process
 pool means reading a traceback re-raised from a worker that has already exited,
@@ -118,7 +122,7 @@ rather than quoting a remembered number.
 | `broadcast` | a `dict` of values shipped once per *worker* instead of once per job, delivered to your function as keyword arguments | 200 jobs + 80 MB panel **7.97s → 0.55s** |
 | `reuse_pool` | keep the pool (and its broadcast payload) alive between calls instead of rebuilding it. Off by default | 10 dispatches **9.86s → 0.02s** |
 | `initializer`, `initargs` | run once per worker before its first job — and unlike the stdlib's, may be a closure | — |
-| `max_tasks_per_child` | recycle a worker every N jobs, so a slow leak can't end the run (Python 3.11+) | — |
+| `max_tasks_per_child` | recycle a worker every N user jobs, so a slow leak can't end the run; disables batching to keep the count exact (Python 3.11+) | — |
 
 **BLAS threads** is the one that catches people out. numpy hands matrix work to
 a native library that is itself multi-threaded, and each worker process loads
@@ -142,6 +146,45 @@ it is a different order of magnitude — 0.55s on the 80 MB case, level with
 joblib's memmapped 0.58s. Call `shutdown_pools()` when you're done with a
 reused pool, or let `atexit` do it.
 
+For a large NumPy array, wrap it in `Shared` to copy it once into OS shared
+memory and let every worker read the same physical bytes:
+
+```python
+from mpengine import Shared
+
+summary = run(score, param_sets, base_dir="runs",
+              broadcast={"panel": Shared(panel)})
+```
+
+Shared arrays are read-only in workers and must be C-contiguous with a
+non-object dtype. Keep the original array alive while a reused pool needs it.
+
+### Reliability controls
+
+```python
+from mpengine import CancellationToken, RunFailedError, run
+
+token = CancellationToken()
+try:
+    summary = run(
+        score,
+        param_sets,
+        base_dir="runs",
+        retries=2,
+        retry_delay=0.25,
+        failure_policy="raise",
+        timeout=300,
+        cancel_token=token,
+    )
+except RunFailedError as exc:
+    summary = exc.summary  # outputs and both manifests are already durable
+```
+
+`failure_policy="continue"` preserves the original behavior. `"raise"` runs
+everything and then raises if any job failed. `"fail_fast"` terminates the
+pool after the first failed result. Timeout and cancellation also terminate
+the pool and raise exceptions carrying a partial `summary`.
+
 ### Logging
 
 The library logs its whole lifecycle through the standard `logging` module and
@@ -193,10 +236,13 @@ That's a cost trade-off to be aware of, not a limitation on what's allowed.
 you want to drive the pool yourself:
 
 ```python
-from mpengine import expand_call, process_jobs, process_jobs_
+from mpengine import expand_call, iter_jobs, process_jobs, process_jobs_
 
 jobs = [{"func": my_task, "x": 1, "y": 2}, {"func": other_task, "n": 5}]
 results = process_jobs(jobs)     # n_workers defaults to os.cpu_count(); or process_jobs_ to stay sequential
+
+for result in iter_jobs(jobs):   # consume each result as soon as it completes
+    store(result)
 ```
 
 A job is just a dict carrying its own callback plus that callback's kwargs, so a

{mpengine-0.4.1 → mpengine-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "mpengine"
-version = "0.4.1"
+version = "0.5.0"
 description = "A small, general-purpose multiprocessing engine: dispatch any callable across processes with run manifests, per-worker logs, on-disk outputs and failure isolation"
 readme = "README.md"
 license = "MIT"
@@ -55,6 +55,10 @@ testpaths = ["tests"]
 markers = ["slow: spawns real process pools; deselect with -m 'not slow'"]
 filterwarnings = ["error::DeprecationWarning:mpengine.*"]
 
+[[tool.mypy.overrides]]
+module = ["cloudpickle", "threadpoolctl", "tqdm", "tqdm.contrib.logging"]
+ignore_missing_imports = true
+
 [project.urls]
 Homepage = "https://github.com/singhamandeep-kgp/multiprocessor"
 Repository = "https://github.com/singhamandeep-kgp/multiprocessor"

mpengine-0.5.0/src/mpengine/__init__.py

@@ -0,0 +1,69 @@
+"""A multiprocessing engine with durable run orchestration."""
+
+import logging as _logging
+
+# A library must never configure logging for its host application - it only
+# names its loggers and attaches a no-op handler so that a caller who has set
+# nothing up sees no "No handlers could be found" noise. To actually see any
+# of it: logging.basicConfig(level=logging.INFO). Loggers are
+# "mpengine.engine" and "mpengine.orchestrator", so either half can be tuned
+# or silenced on its own.
+_logging.getLogger("mpengine").addHandler(_logging.NullHandler())
+
+from mpengine.engine import (
+    BroadcastRef,
+    CancellationToken,
+    Shared,
+    expand_call,
+    iter_jobs,
+    process_jobs,
+    process_jobs_,
+    report_progress,
+    shutdown_pools,
+)
+from mpengine.orchestrator import (
+    JobResult,
+    RunCancelledError,
+    RunFailedError,
+    RunSummary,
+    RunTimeoutError,
+    WorkerStats,
+    load_pickle,
+    load_run_outputs,
+    run,
+    save_pickle,
+)
+from mpengine.partition import equal_chunks, lin_parts, nested_parts, parts_to_molecules
+
+__all__ = [  # noqa: RUF022 - grouped by public API role, not alphabetically
+    # the usual entry point
+    "run",
+    "RunSummary",
+    "RunFailedError",
+    "RunCancelledError",
+    "RunTimeoutError",
+    "JobResult",
+    "WorkerStats",
+    "save_pickle",
+    # reading a finished run's outputs back
+    "load_run_outputs",
+    "load_pickle",
+    # lower-level dispatch, if you want to drive the pool yourself
+    "expand_call",
+    "iter_jobs",
+    "process_jobs",
+    "process_jobs_",
+    "report_progress",
+    # sending one large shared payload per worker instead of per job
+    "BroadcastRef",
+    "CancellationToken",
+    "Shared",
+    "shutdown_pools",
+    # atom -> molecule partitioning
+    "equal_chunks",
+    "lin_parts",
+    "nested_parts",
+    "parts_to_molecules",
+]
+
+__version__ = "0.5.0"

mpengine-0.5.0/src/mpengine/banner.py

@@ -0,0 +1,22 @@
+"""Terminal banner helpers."""
+
+from __future__ import annotations
+
+import sys
+from importlib import resources
+
+
+def _load_spider() -> str:
+    """Read the packaged banner art."""
+    return (
+        resources.files("mpengine").joinpath("spider.txt").read_text(encoding="utf-8")
+    )
+
+
+def print_banner(task: str, n_workers: int, debug: bool, file=sys.stdout) -> None:
+    """Print the banner and launch summary."""
+    print(_load_spider(), file=file)
+    if debug:
+        print(f"  mpengine - '{task}' sequentially (debug mode)\n", file=file)
+    else:
+        print(f"  mpengine - '{task}' on {n_workers} workers\n", file=file)