mpengine 0.1.2__tar.gz → 0.2.0__tar.gz

{mpengine-0.1.2/src/mpengine.egg-info → mpengine-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mpengine
-Version: 0.1.2
+Version: 0.2.0
 Summary: A small, general-purpose multiprocessing engine: dispatch any callable across processes with run manifests, per-worker logs, on-disk outputs and failure isolation
 Author: Amandeep Singh
 License-Expression: MIT
@@ -122,7 +122,7 @@ run with a custom `save_fn`, pass the matching reader as
 | `save_fn` | how each result is written. Default `save_pickle`; supply your own (e.g. parquet) |
 | `labels` | names for each job; defaults to `job_0000`, `job_0001`, … |
 | `task` | names the run; defaults to `func.__name__` |
-| `n_threads` | worker count; defaults to `min(cpu_count, 8)` |
+| `n_workers` | worker *process* count (not threads — see below); defaults to `os.cpu_count()`, clamped down to the number of jobs if there are fewer jobs than that |
 | `debug` | run sequentially in-process — real tracebacks you can attach a debugger to, no pool |
 | `show_progress` | live terminal display: one overall bar for the whole run, plus a live rate number per worker process. Ignored when `debug=True` |
 
@@ -150,7 +150,7 @@ you want to drive the pool yourself:
 from mpengine import expand_call, process_jobs, process_jobs_
 
 jobs = [{"func": my_task, "x": 1, "y": 2}, {"func": other_task, "n": 5}]
-results = process_jobs(jobs, n_threads=8)     # or process_jobs_ to stay sequential
+results = process_jobs(jobs)     # n_workers defaults to os.cpu_count(); or process_jobs_ to stay sequential
 ```
 
 A job is just a dict carrying its own callback plus that callback's kwargs, so a

{mpengine-0.1.2 → mpengine-0.2.0}/README.md

@@ -93,7 +93,7 @@ run with a custom `save_fn`, pass the matching reader as
 | `save_fn` | how each result is written. Default `save_pickle`; supply your own (e.g. parquet) |
 | `labels` | names for each job; defaults to `job_0000`, `job_0001`, … |
 | `task` | names the run; defaults to `func.__name__` |
-| `n_threads` | worker count; defaults to `min(cpu_count, 8)` |
+| `n_workers` | worker *process* count (not threads — see below); defaults to `os.cpu_count()`, clamped down to the number of jobs if there are fewer jobs than that |
 | `debug` | run sequentially in-process — real tracebacks you can attach a debugger to, no pool |
 | `show_progress` | live terminal display: one overall bar for the whole run, plus a live rate number per worker process. Ignored when `debug=True` |
 
@@ -121,7 +121,7 @@ you want to drive the pool yourself:
 from mpengine import expand_call, process_jobs, process_jobs_
 
 jobs = [{"func": my_task, "x": 1, "y": 2}, {"func": other_task, "n": 5}]
-results = process_jobs(jobs, n_threads=8)     # or process_jobs_ to stay sequential
+results = process_jobs(jobs)     # n_workers defaults to os.cpu_count(); or process_jobs_ to stay sequential
 ```
 
 A job is just a dict carrying its own callback plus that callback's kwargs, so a

{mpengine-0.1.2 → mpengine-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "mpengine"
-version = "0.1.2"
+version = "0.2.0"
 description = "A small, general-purpose multiprocessing engine: dispatch any callable across processes with run manifests, per-worker logs, on-disk outputs and failure isolation"
 readme = "README.md"
 license = "MIT"
@@ -49,11 +49,11 @@ Issues = "https://github.com/singhamandeep-kgp/multiprocessor/issues"
 where = ["src"]
 include = ["mpengine*"]
 
-# worker_names.json is data, not code - setuptools' package discovery above
-# only picks up .py files by default, so it needs to be listed explicitly to
-# actually ship inside a built wheel.
+# worker_names.json and spider.txt are data, not code - setuptools' package
+# discovery above only picks up .py files by default, so they need listing
+# explicitly to actually ship inside a built wheel.
 [tool.setuptools.package-data]
-mpengine = ["worker_names.json"]
+mpengine = ["worker_names.json", "spider.txt"]
 
 # `statarb` (needed only by src/learning) is installed separately as an editable
 # path install (pip install -e ../Equity_StatArb) rather than declared here,

{mpengine-0.1.2 → mpengine-0.2.0}/src/mpengine/__init__.py

@@ -73,4 +73,4 @@ __all__ = [
     "parts_to_molecules",
 ]
 
-__version__ = "0.1.2"
+__version__ = "0.2.0"

mpengine-0.2.0/src/mpengine/banner.py

@@ -0,0 +1,41 @@
+"""The run banner - purely cosmetic, deliberately kept out of everything else.
+
+`orchestrator.py` imports exactly one name from here and calls it once. All the
+ASCII art, formatting and printing lives in this module, so the engine and the
+orchestration layer stay free of decoration.
+
+The spider is the mascot: a multiprocessing engine, and a creature famous for
+having eight legs. The art itself lives in `spider.txt` rather than as a string
+literal here - it is data, not code, the same call made for `worker_names.json`.
+"""
+
+from __future__ import annotations
+
+import sys
+from importlib import resources
+
+
+def _load_spider() -> str:
+    """Read the art from packaged data.
+
+    `read_text` applies universal-newline translation, so the file's stored
+    CRLF endings come back as `\\n` and render correctly on any OS without the
+    stored bytes ever being modified - which matters, because the art is kept
+    byte-for-byte as supplied.
+    """
+    return resources.files("mpengine").joinpath("spider.txt").read_text(encoding="utf-8")
+
+
+def print_banner(task: str, n_workers: int, debug: bool, file=sys.stdout) -> None:
+    """Print the spider, then one line saying what is being launched.
+
+    Goes to stdout by default, matching the other run-level reporting `run()`
+    does (the stored-here paths, the worker ranking). The stderr stream stays
+    reserved for the live-redraw progress display, which is a different kind of
+    output and would interleave badly with anything else written there.
+    """
+    print(_load_spider(), file=file)
+    if debug:
+        print(f"  mpengine - '{task}' sequentially (debug mode)\n", file=file)
+    else:
+        print(f"  mpengine - '{task}' on {n_workers} workers\n", file=file)

{mpengine-0.1.2 → mpengine-0.2.0}/src/mpengine/engine.py

@@ -87,11 +87,17 @@ def _run_from_blob(blob: bytes) -> tuple[int, float, Any]:
 def process_jobs(
     jobs: list[dict[str, Any]],
     task: str | None = None,
-    n_threads: int = 24,
+    n_workers: int = os.cpu_count() or 4,
     on_progress: Callable[[int, float, Any], None] | None = None,
 ) -> list[Any]:
     """Ch.20 Snippet 20.9 (`processJobs`) - real `mp.Pool` + `imap_unordered`.
 
+    Named `n_workers`, not `n_threads`: this spawns OS *processes* via
+    `mp.Pool`, each with its own interpreter and memory space, not threads
+    sharing one. That distinction matters here specifically - ex02 measured
+    that CPython's GIL makes threads add nothing for CPU-bound work, which is
+    exactly what this engine dispatches.
+
     Results arrive in *completion* order, not submission order, which is what
     lets `report_progress` report honestly as each job lands - with uneven job
     costs (see ex03), an early-submitted heavy job can finish long after
@@ -124,7 +130,13 @@ def process_jobs(
     if task is None:
         task = jobs[0]["func"].__name__
     blobs = [cloudpickle.dumps(job) for job in jobs]
-    pool = mp.Pool(processes=n_threads)
+    # `Pool(processes=n)` spawns exactly n OS processes immediately, whether
+    # or not there's n jobs' worth of work - never spin up more workers than
+    # there are jobs to hand them. This only ever clamps downward: when jobs
+    # outnumber n_workers, no clamp is needed at all - the same fixed pool
+    # keeps pulling jobs off the queue, one per worker at a time, until the
+    # whole list is done. Worker count is sized to hardware, not job count.
+    pool = mp.Pool(processes=min(n_workers, len(jobs)))
     out: list[Any] = []
     time0 = time.time()
     try:

{mpengine-0.1.2 → mpengine-0.2.0}/src/mpengine/orchestrator.py

@@ -37,9 +37,10 @@ from typing import Any, Callable, Iterator
 
 from tqdm import tqdm
 
+from mpengine.banner import print_banner
 from mpengine.engine import expand_call, process_jobs, process_jobs_
 
-N_WORKERS = min(os.cpu_count() or 4, 8)
+N_WORKERS = os.cpu_count() or 4
 
 # one FileHandler-backed logger per worker *process*, lazily created on that
 # process's first job and reused for every job after - keyed by pid, which is
@@ -302,7 +303,7 @@ def run(
     save_fn: Callable[[Any, Path], None] = save_pickle,
     labels: list[str] | None = None,
     task: str | None = None,
-    n_threads: int = N_WORKERS,
+    n_workers: int = N_WORKERS,
     debug: bool = False,
     show_progress: bool = False,
 ) -> RunSummary:
@@ -385,7 +386,7 @@ def run(
         f.write(f"launched: {dt.datetime.now().isoformat(sep=' ', timespec='seconds')}\n")
         f.write(f"func: {func.__name__}\n")
         f.write(f"task: {task}\n")
-        f.write(f"n_threads: {n_threads}\n")
+        f.write(f"n_workers: {n_workers}\n")
         f.write(f"debug: {debug}\n")
         f.write(f"n_jobs: {len(jobs)}\n")
         f.write(f"output_dir: {output_path}\n")
@@ -397,14 +398,16 @@ def run(
 
     worker_stats: dict[int, WorkerStats] = {}
 
+    print_banner(task, n_workers, debug)
+
     t0 = time.perf_counter()
     if debug:
         raw_results = process_jobs_(jobs)
     elif show_progress:
         with _progress_renderer(len(jobs), task, worker_stats) as on_progress:
-            raw_results = process_jobs(jobs, task=task, n_threads=n_threads, on_progress=on_progress)
+            raw_results = process_jobs(jobs, task=task, n_workers=n_workers, on_progress=on_progress)
     else:
-        raw_results = process_jobs(jobs, task=task, n_threads=n_threads)
+        raw_results = process_jobs(jobs, task=task, n_workers=n_workers)
     elapsed_s = time.perf_counter() - t0
 
     if show_progress and not debug:

mpengine-0.2.0/src/mpengine/spider.txt

@@ -0,0 +1,31 @@
+           ;               ,           
+         ,;                 '.         
+        ;:                   :;        
+       ::                     ::       
+       ::                     ::       
+       ':                     :        
+        :.                    :        
+     ;' ::                   ::  '     
+    .'  ';                   ;'  '.    
+   ::    :;                 ;:    ::   
+   ;      :;.             ,;:     ::   
+   :;      :;:           ,;"      ::   
+   ::.      ':;  ..,.;  ;:'     ,.;:   
+    "'"...   '::,::::: ;:   .;.;""'    
+        '"""....;:::::;,;.;"""         
+    .:::.....'"':::::::'",...;::::;.   
+   ;:' '""'"";.,;:::::;.'""""""  ':;   
+  ::'         ;::;:::;::..         :;  
+ ::         ,;:::::::::::;:..       :: 
+ ;'     ,;;:;::::::::::::::;";..    ':.
+::     ;:"  ::::::"""'::::::  ":     ::
+ :.    ::   ::::::;  :::::::   :     ; 
+  ;    ::   :::::::  :::::::   :    ;  
+   '   ::   ::::::....:::::'  ,:   '   
+    '  ::    :::::::::::::"   ::       
+       ::     ':::::::::"'    ::       
+       ':       """""""'      ::       
+        ::                   ;:        
+        ':;                 ;:"        
+          ';              ,;'          
+            "'           '"            

{mpengine-0.1.2 → mpengine-0.2.0/src/mpengine.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mpengine
-Version: 0.1.2
+Version: 0.2.0
 Summary: A small, general-purpose multiprocessing engine: dispatch any callable across processes with run manifests, per-worker logs, on-disk outputs and failure isolation
 Author: Amandeep Singh
 License-Expression: MIT
@@ -122,7 +122,7 @@ run with a custom `save_fn`, pass the matching reader as
 | `save_fn` | how each result is written. Default `save_pickle`; supply your own (e.g. parquet) |
 | `labels` | names for each job; defaults to `job_0000`, `job_0001`, … |
 | `task` | names the run; defaults to `func.__name__` |
-| `n_threads` | worker count; defaults to `min(cpu_count, 8)` |
+| `n_workers` | worker *process* count (not threads — see below); defaults to `os.cpu_count()`, clamped down to the number of jobs if there are fewer jobs than that |
 | `debug` | run sequentially in-process — real tracebacks you can attach a debugger to, no pool |
 | `show_progress` | live terminal display: one overall bar for the whole run, plus a live rate number per worker process. Ignored when `debug=True` |
 
@@ -150,7 +150,7 @@ you want to drive the pool yourself:
 from mpengine import expand_call, process_jobs, process_jobs_
 
 jobs = [{"func": my_task, "x": 1, "y": 2}, {"func": other_task, "n": 5}]
-results = process_jobs(jobs, n_threads=8)     # or process_jobs_ to stay sequential
+results = process_jobs(jobs)     # n_workers defaults to os.cpu_count(); or process_jobs_ to stay sequential
 ```
 
 A job is just a dict carrying its own callback plus that callback's kwargs, so a

{mpengine-0.1.2 → mpengine-0.2.0}/src/mpengine.egg-info/SOURCES.txt

@@ -2,9 +2,11 @@ LICENSE
 README.md
 pyproject.toml
 src/mpengine/__init__.py
+src/mpengine/banner.py
 src/mpengine/engine.py
 src/mpengine/orchestrator.py
 src/mpengine/partition.py
+src/mpengine/spider.txt
 src/mpengine/worker_names.json
 src/mpengine.egg-info/PKG-INFO
 src/mpengine.egg-info/SOURCES.txt