mpengine 0.1.0__py3-none-any.whl

mpengine/__init__.py

@@ -0,0 +1,76 @@
+"""mpengine - a small, general-purpose multiprocessing engine.
+
+Two layers, which you can use at whichever level suits you:
+
+- ``engine``        the primitives. A job is just a dict carrying its own
+                    callback plus that callback's kwargs; ``expand_call`` turns
+                    such a dict into a call, and ``process_jobs`` /
+                    ``process_jobs_`` dispatch them in parallel or sequentially.
+- ``orchestrator``  the production layer. ``run()`` adds a manifest of exactly
+                    what was launched, one log file per worker process, results
+                    saved to disk, and failure isolation so one bad job cannot
+                    lose the rest of the run.
+
+Most callers only ever need ``run``::
+
+    from mpengine import run
+
+    def my_task(x, y):
+        return x * y
+
+    summary = run(
+        my_task,
+        [{"x": 2, "y": 3}, {"x": 4, "y": 5}],
+        base_dir="runs",
+    )
+    print(summary.n_ok, summary.n_failed)
+
+That writes ``runs/manifests/<run_id>.txt``, ``runs/outputs/<run_id>/`` and
+``runs/logs/<run_id>/``.
+
+Worker targets - ``func`` itself, and any custom ``save_fn`` - can be closures
+or lambdas, not just module-level functions. Jobs are serialized with
+``cloudpickle`` before crossing the process boundary, which can pickle a
+function by value (not just by reference). The one caveat: whatever a closure
+captures travels with every job that uses it.
+
+The dispatch core follows Lopez de Prado, *Advances in Financial Machine
+Learning*, Ch.20 - the docstrings name the specific snippets - but nothing here
+is finance-specific; it parallelizes any callable.
+"""
+
+from mpengine.engine import expand_call, process_jobs, process_jobs_, report_progress
+from mpengine.orchestrator import (
+    JobResult,
+    RunSummary,
+    WorkerStats,
+    load_pickle,
+    load_run_outputs,
+    run,
+    save_pickle,
+)
+from mpengine.partition import equal_chunks, lin_parts, nested_parts, parts_to_molecules
+
+__all__ = [
+    # the usual entry point
+    "run",
+    "RunSummary",
+    "JobResult",
+    "WorkerStats",
+    "save_pickle",
+    # reading a finished run's outputs back
+    "load_run_outputs",
+    "load_pickle",
+    # lower-level dispatch, if you want to drive the pool yourself
+    "expand_call",
+    "process_jobs",
+    "process_jobs_",
+    "report_progress",
+    # atom -> molecule partitioning
+    "equal_chunks",
+    "lin_parts",
+    "nested_parts",
+    "parts_to_molecules",
+]
+
+__version__ = "0.1.0"

mpengine/engine.py

@@ -0,0 +1,140 @@
+"""Ch.20 SS20.5 - the multiprocessing engine (exercises 6+7).
+
+expand_call / process_jobs_ / process_jobs / report_progress - a generic
+job-dict-to-function-call pipeline, reusable by any exercise rather than
+tied to one atom/molecule shape. This is the book's own point: stop writing
+a bespoke parallelization wrapper per function, and build one library that
+can parallelize unknown functions regardless of their arguments or output.
+
+A "job" here is just a dict: one 'func' entry (the callback) plus whatever
+kwargs that callback needs. `expand_call` turns such a dict into a call,
+which is the hinge the whole engine turns on.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import multiprocessing as mp
+import os
+import sys
+import time
+from typing import Any, Callable
+
+import cloudpickle
+
+
+def expand_call(kargs: dict[str, Any]) -> Any:
+    """Ch.20 Snippet 20.10 (`expandCall`) - turn a job dict into a call.
+
+    Deviation from the book: `kargs` is copied before popping. The book's
+    literal `kargs.pop('func')` mutates the *caller's* dict in place, so the
+    same job list can only ever be run once - a second pass raises
+    KeyError('func'). ex04 deliberately runs one job list through both
+    `process_jobs_` and `process_jobs` to compare them, so the copy matters.
+    """
+    kargs = dict(kargs)
+    func = kargs.pop("func")
+    return func(**kargs)
+
+
+def process_jobs_(jobs: list[dict[str, Any]]) -> list[Any]:
+    """Ch.20 Snippet 20.8 (`processJobs_`) - sequential fallback, for debugging.
+
+    Runs every job in-process, one at a time, in submission order. No pool,
+    no pickling, no spawn - so a failing job raises immediately, at its exact
+    position in the list, with an ordinary live traceback you can attach a
+    debugger to. That clarity is the entire reason this mode exists.
+    """
+    return [expand_call(job) for job in jobs]
+
+
+def report_progress(job_num: int, num_jobs: int, time0: float, task: str) -> None:
+    """Ch.20 Snippet 20.9's `reportProgress`, modernized.
+
+    Overwrites its own line via '\\r' until the final job, then emits '\\n'.
+    Units stay the book's minutes even though a fast demo run always shows
+    "0.00 minutes" - the snippet is sized for real multi-minute workloads.
+    """
+    frac = job_num / num_jobs
+    minutes_elapsed = (time.time() - time0) / 60.0
+    minutes_remaining = minutes_elapsed * (1 / frac - 1) if frac > 0 else 0.0
+    timestamp = dt.datetime.now().isoformat(sep=" ", timespec="seconds")
+    msg = (
+        f"{timestamp} {frac * 100:5.1f}% {task} done after {minutes_elapsed:.2f} "
+        f"minutes. Remaining {minutes_remaining:.2f} minutes."
+    )
+    print(msg, end="\n" if job_num >= num_jobs else "\r", file=sys.stderr, flush=True)
+
+
+def _run_from_blob(blob: bytes) -> tuple[int, float, Any]:
+    """Pool target: undo the cloudpickle wrapping `process_jobs` applies before
+    submission, then dispatch exactly as `expand_call` always has.
+
+    Returns `(pid, atom_seconds, result)`. The pid attributes the completion to
+    a specific worker process; `atom_seconds` is measured around the call
+    itself, *inside* the worker, so it is pure compute time - it cannot include
+    queueing, spawn cost, or the idle gap between this atom finishing and the
+    rest of the run finishing. That distinction is the whole point: timing the
+    same thing from the parent would measure when the result *arrived*, not how
+    long the work actually took.
+    """
+    job = cloudpickle.loads(blob)
+    t0 = time.perf_counter()
+    result = expand_call(job)
+    return os.getpid(), time.perf_counter() - t0, result
+
+
+def process_jobs(
+    jobs: list[dict[str, Any]],
+    task: str | None = None,
+    n_threads: int = 24,
+    on_progress: Callable[[int, float, Any], None] | None = None,
+) -> list[Any]:
+    """Ch.20 Snippet 20.9 (`processJobs`) - real `mp.Pool` + `imap_unordered`.
+
+    Results arrive in *completion* order, not submission order, which is what
+    lets `report_progress` report honestly as each job lands - with uneven job
+    costs (see ex03), an early-submitted heavy job can finish long after
+    several later-submitted light ones.
+
+    Each job is serialized with `cloudpickle` before submission (rather than
+    relying on `Pool`'s own stdlib-`pickle` handling of `jobs`), so a job's
+    'func' - or a nested callable, like `orchestrator.run`'s `save_fn` - can be
+    a closure or a lambda, not just a module-level function. `_run_from_blob`
+    is itself a plain module-level function, so it still pickles by reference
+    with no dependence on `__main__` spawn fixup.
+
+    `on_progress`, if given, is called as `on_progress(pid, atom_seconds,
+    result)` for every completed job - `pid` identifies which worker process
+    produced it and `atom_seconds` is that job's pure compute time as measured
+    inside the worker, which is what lets a caller (see `orchestrator.run`'s
+    `show_progress`) drive a per-worker live display with real timings. The
+    return value here is unaffected either way - still the plain `list[Any]`
+    of results, never the `(pid, atom_seconds, result)` triples.
+    Supplying `on_progress` means the caller is rendering its own progress
+    display, so the book's own text-based `report_progress` is skipped for
+    that call - the two would otherwise both write to stderr and garble each
+    other's redraws.
+
+    Uses explicit close()+join() rather than `with mp.Pool(...) as pool:`,
+    whose __exit__ calls terminate() - an abrupt kill, not the book's graceful
+    shutdown. The try/finally still lets a job's exception propagate while
+    guaranteeing the workers are torn down.
+    """
+    if task is None:
+        task = jobs[0]["func"].__name__
+    blobs = [cloudpickle.dumps(job) for job in jobs]
+    pool = mp.Pool(processes=n_threads)
+    out: list[Any] = []
+    time0 = time.time()
+    try:
+        for i, (pid, atom_s, out_) in enumerate(pool.imap_unordered(_run_from_blob, blobs), 1):
+            out.append(out_)
+            if on_progress is not None:
+                on_progress(pid, atom_s, out_)
+            else:
+                report_progress(i, len(jobs), time0, task)
+    finally:
+        pool.close()
+        pool.join()
+    return out

mpengine/orchestrator.py

@@ -0,0 +1,442 @@
+"""A production orchestration layer on top of `mpengine.engine`.
+
+`engine.py` stays the pure, book-faithful engine (`expand_call`/
+`process_jobs_`/`process_jobs`) and is unchanged by this module. `run()` is
+the one entry point on top of it: give it a function and a list of parameter
+sets, and it handles everything a real multiprocessing job needs beyond the
+book's scope -
+
+  1. a manifest .txt recording exactly what was launched (and, once done,
+     what happened)
+  2. every job's result saved to disk, in whatever format the caller chooses
+  3. one log file per *worker process* (not per job), so a failure is
+     attributable to a specific process, not just a stack trace
+
+Unlike `engine.py`'s `process_jobs`, a single job failing here does not abort
+the run - it's recorded as a failed `JobResult` and the rest continue. That is
+the actual point of "know which one broke and which didn't": you want the N-1
+good results even when job N blew up.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import json
+import logging
+import os
+import pickle
+import random
+import sys
+import time
+import traceback
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from importlib import resources
+from pathlib import Path
+from typing import Any, Callable, Iterator
+
+from tqdm import tqdm
+
+from mpengine.engine import expand_call, process_jobs, process_jobs_
+
+N_WORKERS = min(os.cpu_count() or 4, 8)
+
+# one FileHandler-backed logger per worker *process*, lazily created on that
+# process's first job and reused for every job after - keyed by pid, which is
+# always fresh in a newly spawned worker, so no cross-run coordination needed
+_worker_logger: logging.Logger | None = None
+_worker_log_dir: Path | None = None
+
+
+@dataclass
+class JobResult:
+    label: str
+    status: str  # "ok" | "error"
+    output_path: str | None = None
+    error: str | None = None
+
+
+@dataclass
+class RunSummary:
+    run_id: str
+    manifest_path: str
+    output_dir: str
+    log_dir: str
+    n_jobs: int
+    n_ok: int
+    n_failed: int
+    elapsed_s: float
+    results: list[JobResult] = field(default_factory=list)
+    # keyed by worker pid; only populated when show_progress=True, since that
+    # is what installs the per-atom timing hook
+    worker_stats: dict[int, "WorkerStats"] = field(default_factory=dict)
+
+
+def save_pickle(obj: Any, path: Path) -> None:
+    """Default save_fn. A caller-supplied save_fn can be a lambda or a closure
+    too - like any other job field, it crosses the process boundary via
+    `cloudpickle`, not stdlib `pickle`."""
+    with open(path, "wb") as f:
+        pickle.dump(obj, f)
+
+
+def load_pickle(path: Path) -> Any:
+    """Counterpart to `save_pickle`, and the default `load_fn` for
+    `load_run_outputs`. Runs in the calling process, not a worker, so the
+    ordinary `pickle` caveat applies: whatever types were saved must be
+    importable here to be reconstructed."""
+    with open(path, "rb") as f:
+        return pickle.load(f)
+
+
+def load_run_outputs(
+    run_dir: str | Path,
+    load_fn: Callable[[Path], Any] = load_pickle,
+) -> dict[str, Any]:
+    """Read a finished run's outputs back as `{label: result}`.
+
+    `run_dir` is that run's output directory - pass `RunSummary.output_dir`,
+    or the path `run()` printed as "Output stored here". Keys are the labels
+    `run()` assigned (`job_0000`, ...), so a result can be matched straight
+    back to the `JobResult` that produced it.
+
+    Only successful jobs appear. A failed job never wrote a file, so its label
+    is simply missing here - check `RunSummary.results` to see which failed and
+    why, rather than inferring it from a gap in these keys.
+
+    `load_fn` must match whatever `save_fn` wrote the run; `save_pickle` and
+    `load_pickle` are the matched default pair.
+    """
+    path = Path(run_dir)
+    if not path.is_dir():
+        raise NotADirectoryError(f"not a run output directory: {path}")
+    return {p.name: load_fn(p) for p in sorted(path.iterdir()) if p.is_file()}
+
+
+def _get_worker_logger(log_dir: Path) -> logging.Logger:
+    global _worker_logger, _worker_log_dir
+    if _worker_logger is not None and _worker_log_dir == log_dir:
+        return _worker_logger
+
+    pid = os.getpid()
+    logger = logging.getLogger(f"mpengine.orchestrator.worker.{pid}")
+    logger.setLevel(logging.INFO)
+    logger.propagate = False
+    if not logger.handlers:
+        handler = logging.FileHandler(log_dir / f"worker_{pid}.log")
+        handler.setFormatter(logging.Formatter("%(asctime)s %(message)s"))
+        logger.addHandler(handler)
+
+    _worker_logger, _worker_log_dir = logger, log_dir
+    return logger
+
+
+def _run_and_save_job(job: dict) -> JobResult:
+    """The actual Pool/sequential target. Unwraps its own bookkeeping fields,
+    calls the caller's function via `expand_call` (reusing engine.py rather
+    than reimplementing the dict-to-call trick), saves the result, and always
+    returns a JobResult - exceptions are caught here, never re-raised, so one
+    bad job cannot abort the run.
+    """
+    label = job["label"]
+    output_dir = job["output_dir"]
+    log_dir = job["log_dir"]
+    save_fn = job["save_fn"]
+    inner_job = job["inner_job"]
+
+    logger = _get_worker_logger(log_dir)
+    logger.info("starting %s", label)
+    try:
+        result = expand_call(inner_job)
+    except Exception as exc:
+        logger.error("failed %s: %s\n%s", label, exc, traceback.format_exc())
+        return JobResult(label=label, status="error", error=str(exc))
+
+    output_path = output_dir / label
+    try:
+        save_fn(result, output_path)
+    except Exception as exc:
+        logger.error("save failed %s: %s\n%s", label, exc, traceback.format_exc())
+        return JobResult(label=label, status="error", error=f"save failed: {exc}")
+
+    logger.info("completed %s -> %s", label, output_path)
+    return JobResult(label=label, status="ok", output_path=str(output_path))
+
+
+def _load_worker_names() -> list[str]:
+    """Cool, memorable codenames for the progress display - purely cosmetic,
+    so a worker reads as "Worker Jarvis (PID 12345)" rather than a bare pid.
+    Lives in `worker_names.json` (not inline here) so the name pool can be
+    edited without touching this module. Sized generously past any realistic
+    core count; `_next_worker_name` falls back to a numbered name if a run
+    somehow has more workers than names.
+    """
+    data = resources.files("mpengine").joinpath("worker_names.json").read_text(encoding="utf-8")
+    return json.loads(data)
+
+
+_WORKER_NAMES = _load_worker_names()
+
+
+@dataclass
+class WorkerStats:
+    """Per-worker-process tallies, accumulated live as atoms complete."""
+
+    name: str
+    pid: int
+    n_atoms: int = 0
+    busy_s: float = 0.0
+    last_atom_s: float = 0.0
+
+    @property
+    def avg_atom_s(self) -> float:
+        return self.busy_s / self.n_atoms if self.n_atoms else 0.0
+
+    @property
+    def atoms_per_s(self) -> float:
+        return self.n_atoms / self.busy_s if self.busy_s else 0.0
+
+    def line(self) -> str:
+        plural = "atom " if self.n_atoms == 1 else "atoms"
+        return (
+            f"Worker {self.name} (PID {self.pid}): {self.n_atoms} {plural} | "
+            f"avg {self.avg_atom_s:.2f}s/atom | last {self.last_atom_s:.2f}s"
+        )
+
+
+@contextmanager
+def _progress_renderer(
+    n_jobs: int, task: str, stats_out: dict[int, WorkerStats]
+) -> Iterator[Callable[[int, float, Any], None]]:
+    """tqdm-based live display for `run(show_progress=True)`: one overall bar
+    (total=n_jobs), plus a per-worker line created lazily the first time a pid
+    is seen. Workers get no fixed total - `Pool` assigns jobs to them
+    dynamically as they free up, so there's no way to know one in advance.
+
+    The per-worker numbers are computed here from `atom_seconds` measured
+    inside the worker, *not* from tqdm's own rate. tqdm's rate would divide by
+    the bar's lifetime - creation to close - and since every bar is closed
+    together at the end of the run, that span is mostly idle waiting, which
+    makes an early-finishing worker look slow and the last-finishing worker
+    look fast. Dividing real atom counts by real compute time avoids that
+    inversion entirely.
+
+    Every line is recomputed and redrawn the instant its worker finishes an
+    atom (`mininterval=0` so tqdm cannot throttle the redraw). A worker's line
+    holds steady while it is mid-atom - there is no partial-progress signal
+    from inside a running atom, only completions.
+
+    Each worker gets a random, unique codename for the run (shuffled once per
+    call, so it varies run to run); the pid is shown alongside it, since the
+    name itself is only decorative. `stats_out` is populated in place so the
+    caller keeps the tallies after the display is torn down.
+    """
+    overall = tqdm(total=n_jobs, desc=task, position=0, file=sys.stderr, mininterval=0)
+    worker_bars: dict[int, tqdm] = {}
+    available_names = _WORKER_NAMES.copy()
+    random.shuffle(available_names)
+
+    def _next_worker_name() -> str:
+        if available_names:
+            return available_names.pop()
+        return f"Worker-{len(worker_bars) + 1}"
+
+    def on_progress(pid: int, atom_s: float, _result: Any) -> None:
+        if pid not in worker_bars:
+            stats_out[pid] = WorkerStats(name=_next_worker_name(), pid=pid)
+            worker_bars[pid] = tqdm(
+                total=None,
+                position=len(worker_bars) + 1,
+                bar_format="{desc}",
+                file=sys.stderr,
+                mininterval=0,
+            )
+
+        stats = stats_out[pid]
+        stats.n_atoms += 1
+        stats.busy_s += atom_s
+        stats.last_atom_s = atom_s
+
+        bar = worker_bars[pid]
+        bar.set_description_str(stats.line(), refresh=True)
+        overall.update(1)
+
+    try:
+        yield on_progress
+    finally:
+        overall.close()
+        for bar in worker_bars.values():
+            bar.close()
+
+
+def _print_worker_ranking(stats: dict[int, WorkerStats]) -> None:
+    """Rank workers fastest-to-slowest by seconds per atom.
+
+    Note the caveat printed alongside: with uneven atom sizes, a low
+    s/atom can mean small atoms rather than a genuinely faster worker.
+    """
+    if not stats:
+        return
+
+    ranked = sorted(stats.values(), key=lambda s: s.avg_atom_s)
+    print("\nworkers, fastest to slowest (by avg seconds per atom):")
+    for i, s in enumerate(ranked):
+        tag = ""
+        if len(ranked) > 1:
+            tag = "  <- fastest" if i == 0 else ("  <- slowest" if i == len(ranked) - 1 else "")
+        print(
+            f"  {s.name:<12} (PID {s.pid:>6})  {s.n_atoms:>3} atoms  "
+            f"avg {s.avg_atom_s:6.2f}s/atom  {s.atoms_per_s:6.2f} atoms/s{tag}"
+        )
+    print("  (uneven atom sizes: a low s/atom can mean small atoms, not a faster worker)")
+
+
+def run(
+    func: Callable[..., Any],
+    param_sets: list[dict[str, Any]],
+    *,
+    base_dir: str | Path | None = None,
+    output_dir: str | Path | None = None,
+    log_dir: str | Path | None = None,
+    manifest_dir: str | Path | None = None,
+    save_fn: Callable[[Any, Path], None] = save_pickle,
+    labels: list[str] | None = None,
+    task: str | None = None,
+    n_threads: int = N_WORKERS,
+    debug: bool = False,
+    show_progress: bool = False,
+) -> RunSummary:
+    """Run `func(**params)` for every dict in `param_sets`, organized.
+
+    Destinations: pass `base_dir` and the three output locations are derived
+    as `<base_dir>/outputs`, `<base_dir>/logs` and `<base_dir>/manifests`; or
+    pass `output_dir`/`log_dir`/`manifest_dir` explicitly to place them
+    independently. An explicit path always wins over the derived one, so you
+    can give a `base_dir` and still redirect just the logs elsewhere.
+
+    `func` can be a closure or a lambda, not just a module-level function -
+    `engine.process_jobs` serializes jobs with `cloudpickle`, which can pickle
+    a function by value. The only cost: whatever a closure captures travels
+    with every job that uses it.
+
+    Every destination directory gets a `<run_id>` subfolder (`task` + a
+    timestamp), so successive runs never collide and a worker's log file
+    (named by its pid) can never be confused with a previous run's.
+
+    `show_progress=True` (only meaningful when `debug=False` - ignored
+    otherwise, since debug mode is one sequential in-process run with no
+    "workers" to distinguish) renders a live terminal display: one overall bar
+    for the whole run, plus a per-worker line showing atoms done, average
+    seconds per atom and the last atom's time, each recomputed and redrawn the
+    moment that worker finishes an atom. It also prints a fastest-to-slowest
+    ranking at the end, and fills `RunSummary.worker_stats` (keyed by pid) so
+    the same numbers are available programmatically.
+    """
+    if base_dir is not None:
+        base = Path(base_dir)
+        output_dir = output_dir if output_dir is not None else base / "outputs"
+        log_dir = log_dir if log_dir is not None else base / "logs"
+        manifest_dir = manifest_dir if manifest_dir is not None else base / "manifests"
+
+    missing = [
+        name
+        for name, value in (
+            ("output_dir", output_dir),
+            ("log_dir", log_dir),
+            ("manifest_dir", manifest_dir),
+        )
+        if value is None
+    ]
+    if missing:
+        raise ValueError(
+            f"pass base_dir, or all of output_dir/log_dir/manifest_dir (missing: {', '.join(missing)})"
+        )
+
+    task = task or func.__name__
+    run_id = f"{task}_{dt.datetime.now():%Y%m%d_%H%M%S}"
+
+    output_path = Path(output_dir) / run_id
+    log_path = Path(log_dir) / run_id
+    output_path.mkdir(parents=True, exist_ok=True)
+    log_path.mkdir(parents=True, exist_ok=True)
+    Path(manifest_dir).mkdir(parents=True, exist_ok=True)
+    manifest_path = Path(manifest_dir) / f"{run_id}.txt"
+
+    labels = labels or [f"job_{i:04d}" for i in range(len(param_sets))]
+    if len(labels) != len(param_sets):
+        raise ValueError(f"got {len(labels)} labels for {len(param_sets)} param sets")
+
+    jobs = [
+        {
+            "func": _run_and_save_job,
+            "job": {
+                "label": label,
+                "output_dir": output_path,
+                "log_dir": log_path,
+                "save_fn": save_fn,
+                "inner_job": {"func": func, **params},
+            },
+        }
+        for label, params in zip(labels, param_sets)
+    ]
+
+    with open(manifest_path, "w") as f:
+        f.write(f"run_id: {run_id}\n")
+        f.write(f"launched: {dt.datetime.now().isoformat(sep=' ', timespec='seconds')}\n")
+        f.write(f"func: {func.__name__}\n")
+        f.write(f"task: {task}\n")
+        f.write(f"n_threads: {n_threads}\n")
+        f.write(f"debug: {debug}\n")
+        f.write(f"n_jobs: {len(jobs)}\n")
+        f.write(f"output_dir: {output_path}\n")
+        f.write(f"log_dir: {log_path}\n")
+        f.write("\nparameters:\n")
+        for label, params in zip(labels, param_sets):
+            f.write(f"  {label}: {params}\n")
+        f.write("\n")
+
+    worker_stats: dict[int, WorkerStats] = {}
+
+    t0 = time.perf_counter()
+    if debug:
+        raw_results = process_jobs_(jobs)
+    elif show_progress:
+        with _progress_renderer(len(jobs), task, worker_stats) as on_progress:
+            raw_results = process_jobs(jobs, task=task, n_threads=n_threads, on_progress=on_progress)
+    else:
+        raw_results = process_jobs(jobs, task=task, n_threads=n_threads)
+    elapsed_s = time.perf_counter() - t0
+
+    if show_progress and not debug:
+        _print_worker_ranking(worker_stats)
+
+    n_ok = sum(1 for r in raw_results if r.status == "ok")
+    n_failed = len(raw_results) - n_ok
+
+    with open(manifest_path, "a") as f:
+        f.write("results:\n")
+        for r in raw_results:
+            if r.status == "ok":
+                f.write(f"  {r.label}: ok -> {r.output_path}\n")
+            else:
+                f.write(f"  {r.label}: ERROR - {r.error}\n")
+        f.write(f"\nn_ok: {n_ok}\n")
+        f.write(f"n_failed: {n_failed}\n")
+        f.write(f"elapsed_s: {elapsed_s:.3f}\n")
+
+    print(f"Logs stored here     - {log_path}")
+    print(f"Output stored here   - {output_path}")
+    print(f"Manifest stored here - {manifest_path}")
+
+    return RunSummary(
+        run_id=run_id,
+        manifest_path=str(manifest_path),
+        output_dir=str(output_path),
+        log_dir=str(log_path),
+        n_jobs=len(jobs),
+        n_ok=n_ok,
+        n_failed=n_failed,
+        elapsed_s=elapsed_s,
+        results=raw_results,
+        worker_stats=worker_stats,
+    )

mpengine/partition.py

@@ -0,0 +1,73 @@
+"""Atom -> molecule partitioning helpers, used by every exercise."""
+
+from __future__ import annotations
+
+from typing import Sequence, TypeVar
+
+import numpy as np
+
+T = TypeVar("T")
+
+
+def equal_chunks(seq: Sequence[T], n: int) -> list[list[T]]:
+    """Split `seq` into up to `n` contiguous, roughly equal-size chunks.
+
+    Appropriate when atoms have uniform cost (linParts territory) - not the
+    book's actual triangular-cost `nestedParts`, which gets its own exercise.
+    """
+    n = max(1, min(n, len(seq)))
+    q, r = divmod(len(seq), n)
+    chunks = []
+    start = 0
+    for i in range(n):
+        size = q + (1 if i < r else 0)
+        if size == 0:
+            break
+        chunks.append(list(seq[start:start + size]))
+        start += size
+    return chunks
+
+
+def lin_parts(num_atoms: int, num_threads: int) -> np.ndarray:
+    """Ch.20 Snippet 20.5 (`linParts`) - boundary indices for up to
+    `num_threads` equal-*count* partitions of `num_atoms` atoms of uniform
+    cost. Molecule m (1-indexed) is `atoms[parts[m-1]:parts[m]]` - see
+    `parts_to_molecules`.
+    """
+    num_threads_ = min(num_threads, num_atoms)
+    parts = np.linspace(0, num_atoms, num_threads_ + 1)
+    return np.ceil(parts).astype(int)
+
+
+def nested_parts(num_atoms: int, num_threads: int, upper_triang: bool = False) -> np.ndarray:
+    """Ch.20 Snippet 20.6 (`nestedParts`) - boundary indices for up to
+    `num_threads` equal-*work* partitions of a triangular-cost workload where
+    atom i costs O(i) (e.g. an expanding-window computation over i atoms).
+
+    Each boundary r_m is the positive root of
+        (1/2)(r_m + r_{m-1} + 1)(r_m - r_{m-1}) = N(N+1) / (2M)
+    solved iteratively from r_0 = 0, keeping intermediate boundaries as
+    floats and rounding once at the end (rounding progressively would
+    compound error).
+
+    `upper_triang=True` reverses which end carries the heavy rows - use it
+    when atom cost *decreases* with atom index instead of increasing.
+    """
+    num_threads_ = min(num_threads, num_atoms)
+    parts = [0.0]
+    for _ in range(num_threads_):
+        prev = parts[-1]
+        part = 1 + 4 * (prev**2 + prev + num_atoms * (num_atoms + 1) / num_threads_)
+        part = (-1 + part**0.5) / 2
+        parts.append(part)
+    parts_arr = np.round(parts).astype(int)
+    if upper_triang:
+        parts_arr = np.concatenate(([0], np.cumsum(np.diff(parts_arr)[::-1])))
+    return parts_arr
+
+
+def parts_to_molecules(atoms: Sequence[T], parts: np.ndarray) -> list[list[T]]:
+    """Slice a flat atom list into molecules given boundary indices `parts`
+    (as returned by `lin_parts`/`nested_parts`): molecule m = atoms[parts[m-1]:parts[m]].
+    """
+    return [list(atoms[parts[i - 1]:parts[i]]) for i in range(1, len(parts))]

mpengine/worker_names.json

@@ -0,0 +1,10 @@
+[
+  "Jarvis", "Enigma", "Falcon", "Nimbus", "Phoenix", "Vortex", "Titan", "Orion",
+  "Nova", "Cipher", "Raven", "Atlas", "Zephyr", "Comet", "Blaze", "Echo",
+  "Griffin", "Hydra", "Kraken", "Mirage", "Nebula", "Onyx", "Pulsar", "Quasar",
+  "Rogue", "Specter", "Talon", "Umbra", "Viper", "Wraith", "Zenith", "Aurora",
+  "Basilisk", "Cyclone", "Draco", "Ember", "Fenrir", "Gargoyle", "Helix", "Icarus",
+  "Juggernaut", "Karma", "Labyrinth", "Maverick", "Nexus", "Oracle", "Paradox", "Quantum",
+  "Ronin", "Sentinel", "Talisman", "Ultra", "Valkyrie", "Warden", "Xenon", "Yeti",
+  "Zodiac", "Anomaly", "Banshee", "Catalyst", "Dynamo", "Eclipse", "Frost", "Glitch"
+]

mpengine-0.1.0.dist-info/METADATA

@@ -0,0 +1,170 @@
+Metadata-Version: 2.4
+Name: mpengine
+Version: 0.1.0
+Summary: A small, general-purpose multiprocessing engine: dispatch any callable across processes with run manifests, per-worker logs, on-disk outputs and failure isolation
+Author: Amandeep Singh
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/singhamandeep-kgp/multiprocessor
+Project-URL: Repository, https://github.com/singhamandeep-kgp/multiprocessor
+Project-URL: Issues, https://github.com/singhamandeep-kgp/multiprocessor/issues
+Keywords: multiprocessing,parallel,concurrency,process-pool,vectorization
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26
+Requires-Dist: cloudpickle>=3.0
+Requires-Dist: tqdm>=4.60
+Dynamic: license-file
+
+# mpengine
+
+*(This repo is named `multiprocessor`; the package it ships is `mpengine` — see below.)*
+
+A small, general-purpose multiprocessing engine. Give it any callable and a list
+of parameter sets; it runs them across processes and leaves behind a record of
+what happened.
+
+Every run produces three things:
+
+| what | where |
+|---|---|
+| a manifest of exactly what was launched, and how it ended | `<base>/manifests/<run_id>.txt` |
+| each job's result, saved to disk | `<base>/outputs/<run_id>/<label>` |
+| one log file per **worker process** | `<base>/logs/<run_id>/worker_<pid>.log` |
+
+A single job failing does **not** abort the run — it is recorded as a failed
+job and the rest continue.
+
+## Install
+
+```bash
+pip install -e path/to/mpengine        # editable, for local development
+pip install path/to/mpengine           # or a plain install
+```
+
+`numpy`, `cloudpickle`, and `tqdm` are required.
+
+## Use
+
+```python
+from mpengine import run
+
+def my_task(x, y):
+    return x * y
+
+if __name__ == "__main__":          # required - see below
+    summary = run(
+        my_task,
+        [{"x": 2, "y": 3}, {"x": 4, "y": 5}],
+        base_dir="runs",
+    )
+
+    print(summary.n_ok, summary.n_failed)
+    for r in summary.results:
+        print(r.label, r.status, r.output_path or r.error)
+```
+
+### The `if __name__ == "__main__":` guard
+
+On Windows and macOS, Python starts worker processes with `spawn`, which
+re-imports your module in every worker. Without the guard, that re-import runs
+your `run(...)` call again in each worker, which spawns more workers, and so on
+until the process dies. Put anything that *calls* `run()` inside the guard;
+your task functions themselves stay at module level, as normal.
+
+### Placing the outputs
+
+`base_dir` derives all three locations. To place them independently, pass any of
+`output_dir`, `log_dir`, `manifest_dir` — an explicit path always wins, so you
+can give a `base_dir` and still redirect just the logs.
+
+### Reading the results back
+
+Results are written to disk, one file per job (pickled by default). To load a
+finished run back into memory as `{label: result}`:
+
+```python
+from mpengine import load_run_outputs
+
+outputs = load_run_outputs(summary.output_dir)   # or the path run() printed
+print(outputs["job_0000"])
+```
+
+Only successful jobs appear — a failed job never wrote a file, so its label is
+absent; check `summary.results` to see which failed and why. If you wrote the
+run with a custom `save_fn`, pass the matching reader as
+`load_run_outputs(..., load_fn=your_loader)`.
+
+### Other options
+
+| argument | meaning |
+|---|---|
+| `save_fn` | how each result is written. Default `save_pickle`; supply your own (e.g. parquet) |
+| `labels` | names for each job; defaults to `job_0000`, `job_0001`, … |
+| `task` | names the run; defaults to `func.__name__` |
+| `n_threads` | worker count; defaults to `min(cpu_count, 8)` |
+| `debug` | run sequentially in-process — real tracebacks you can attach a debugger to, no pool |
+| `show_progress` | live terminal display: one overall bar for the whole run, plus a live rate number per worker process. Ignored when `debug=True` |
+
+Develop with `debug=True`, then flip it off. Chasing a bug through a process
+pool means reading a `RemoteTraceback` from a worker that has already exited,
+and it never tells you which job dict was at fault.
+
+### Closures and lambdas
+
+`func`, and any custom `save_fn`, can be a closure or a lambda — not just a
+module-level function. Jobs are serialized with `cloudpickle` before crossing
+the process boundary, which (unlike stdlib `pickle`) can serialize a function
+by *value* (its bytecode plus whatever it captured), not just by reference.
+
+The one caveat: whatever a closure captures travels with **every job** that
+uses it — a closure capturing a large array re-serializes that array per job.
+That's a cost trade-off to be aware of, not a limitation on what's allowed.
+
+## Going lower-level
+
+`run()` is the convenient layer. The primitives underneath are exported too, if
+you want to drive the pool yourself:
+
+```python
+from mpengine import expand_call, process_jobs, process_jobs_
+
+jobs = [{"func": my_task, "x": 1, "y": 2}, {"func": other_task, "n": 5}]
+results = process_jobs(jobs, n_threads=8)     # or process_jobs_ to stay sequential
+```
+
+A job is just a dict carrying its own callback plus that callback's kwargs, so a
+single call can dispatch entirely different functions with different signatures
+and return types.
+
+Partitioning helpers are available for splitting work into chunks:
+
+```python
+from mpengine import lin_parts, nested_parts, parts_to_molecules
+```
+
+`lin_parts` gives equal-count chunks. `nested_parts` gives equal-*work* chunks
+for triangular workloads — where item `i` costs `O(i)`, such as an
+expanding-window computation — which keeps workers from idling while one
+overloaded worker finishes.
+
+## Provenance
+
+The dispatch core follows López de Prado, *Advances in Financial Machine
+Learning*, Chapter 20; docstrings name the specific snippets. Nothing in
+`mpengine` is finance-specific.
+
+The `src/learning/` package in this repo holds the exercises the engine grew out
+of. It depends on a separate private project and is deliberately **not**
+packaged — installing `mpengine` elsewhere never pulls it in.

mpengine-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+mpengine/__init__.py,sha256=mtX0qBUp5rdYiNr0a664WOnOj4sKU1o01xOrtPz70YY,2473
+mpengine/engine.py,sha256=yoeK2RL9XSPtVYhiEAXjpcz4qzNQy7GXHcsOuYbeTHU,6214
+mpengine/orchestrator.py,sha256=z0DzIHZtHd2cNftGGepzS-A_yBlRNoJJJGm7mV7NaqM,16591
+mpengine/partition.py,sha256=kBpFK2RPuu0kHBlzn01dJfjHfl6ionIq7CNKlLnhjSc,2742
+mpengine/worker_names.json,sha256=XVbkv6Hl5Ue-Aq2srlLK1rh17S-VRwwuRYf-Aiq2eNM,661
+mpengine-0.1.0.dist-info/licenses/LICENSE,sha256=XNuwVT7LsfBWrgCPvzXjnGlYmvy9B6gKqCuioy_rN20,1071
+mpengine-0.1.0.dist-info/METADATA,sha256=YzDYSyYKX4Xb0NkaPTHrXLtg3_kj2mGvG6ny5GaSQwA,6788
+mpengine-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+mpengine-0.1.0.dist-info/top_level.txt,sha256=m7XGsQ_E8SrYdLMFnsiS_xOEjOljYLtFUi9HRkb0x0w,9
+mpengine-0.1.0.dist-info/RECORD,,

mpengine-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

mpengine-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Amandeep Singh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

mpengine-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+mpengine