openprocess 0.13.0__tar.gz → 0.13.1__tar.gz

{openprocess-0.13.0/openprocess.egg-info → openprocess-0.13.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openprocess
-Version: 0.13.0
+Version: 0.13.1
 Summary: Process mining, Petri nets and workflows in one open app: every algorithm visible, every result reproducible, every step teachable
 Author: Sam Struthers
 License: MIT
@@ -938,6 +938,17 @@ def last_two(log: EventLog, representation: str = "multiset") -> TransitionSyste
     ...
 ```
 
+Work over many independent items uses every core with one call, and gives
+exactly the result of a loop, notes on the *How* tab included:
+
+```python
+rows = flow.parallel_map(fit_of, variants, shared=model)   # fit_of(model, variant), at the top level
+```
+
+See [Parallel work](docs/workflows.md#parallel-work) for the rules (each item
+from its own inputs, `flow.item_seed` for randomness). OpenProcess's own
+alignments and token replay run this way on large logs.
+
 `pip install -e ".[science]"` adds the pandas, NumPy, SciPy and matplotlib
 boxes (describe, bootstrap intervals, tests, plots).
 
@@ -1025,7 +1036,9 @@ the clone is older than the latest release. See
 
 OpenProcess comes with an **MCP server**, so Claude (Desktop or Code), Cursor
 or any other assistant that speaks the Model Context Protocol can use it the
-way you do:
+way you do. In the app: **Help ▸ Connect an AI Assistant…**, then **Connect**
+next to your assistant, and restart it. No terminal needed: the connector is
+inside the app. From a pip install:
 
 ```bash
 pip install "openprocess[app,mcp]"

{openprocess-0.13.0 → openprocess-0.13.1}/README.md

@@ -896,6 +896,17 @@ def last_two(log: EventLog, representation: str = "multiset") -> TransitionSyste
     ...
 ```
 
+Work over many independent items uses every core with one call, and gives
+exactly the result of a loop, notes on the *How* tab included:
+
+```python
+rows = flow.parallel_map(fit_of, variants, shared=model)   # fit_of(model, variant), at the top level
+```
+
+See [Parallel work](docs/workflows.md#parallel-work) for the rules (each item
+from its own inputs, `flow.item_seed` for randomness). OpenProcess's own
+alignments and token replay run this way on large logs.
+
 `pip install -e ".[science]"` adds the pandas, NumPy, SciPy and matplotlib
 boxes (describe, bootstrap intervals, tests, plots).
 
@@ -983,7 +994,9 @@ the clone is older than the latest release. See
 
 OpenProcess comes with an **MCP server**, so Claude (Desktop or Code), Cursor
 or any other assistant that speaks the Model Context Protocol can use it the
-way you do:
+way you do. In the app: **Help ▸ Connect an AI Assistant…**, then **Connect**
+next to your assistant, and restart it. No terminal needed: the connector is
+inside the app. From a pip install:
 
 ```bash
 pip install "openprocess[app,mcp]"

{openprocess-0.13.0 → openprocess-0.13.1}/openprocess/__init__.py

@@ -48,7 +48,7 @@ from .sim.simulator import Simulator
 #: The one place the version number is written (pyproject.toml, the update
 #: check and packaging/build.py read it from here).  A release is tagged
 #: "v" + this, e.g. v0.2.0.
-__version__ = "0.13.0"
+__version__ = "0.13.1"
 
 __all__ = [
     "Arc", "BindingElement", "CPNet", "Marking", "Multiset", "Page", "Place",

{openprocess-0.13.0 → openprocess-0.13.1}/openprocess/flow/__init__.py

@@ -26,12 +26,16 @@ Python::
     print(run.value(fit).metrics)
     save(wf, "demo.cpnflow", run)      # with the record that makes it reproducible
 
+Work over many independent items (variants, cases, resamples) runs on all
+cores with :func:`parallel_map`, with exactly the result of a loop.
+
 See ``docs/workflows.md``.
 """
 
 from .box import Box, BoxError, BoxSpec, Port, Setting, box
 from .explain import Explanation, current, note, show, steps
 from .library import Library, library_for, standard_library
+from .parallel import ParallelError, item_seed, parallel_map
 from .record import check, differences, load, save
 from .runner import Cache, Result, Run, Runner
 from .sweep import Sweep
@@ -47,5 +51,6 @@ __all__ = [
     "Port", "Predictions", "Predictor", "ProcessTree", "Regions", "ReplayResult", "Result", "Run",
     "Runner", "Scores", "Setting", "SimpleLog", "Sweep", "Table", "Text", "TransitionSystem", "Workflow",
     "WorkflowError", "box", "check", "current", "differences", "library_for", "load", "note",
+    "ParallelError", "item_seed", "parallel_map",
     "register_type", "save", "show", "standard_library", "steps", "to_python", "workflow",
 ]

{openprocess-0.13.0 → openprocess-0.13.1}/openprocess/flow/boxes/input.py

@@ -32,15 +32,11 @@ def open_log(file: Path) -> EventLog:
     file: the log file. Choose… picks it; a name alone means a file in the workflow's folder
     """
     path = _resolve(file)
-    lower = path.name.lower()
-    if lower.endswith(".csv"):
-        from ...mining.csv_import import read_csv
-        log = read_csv(path)
-    elif lower.endswith(".txt"):
-        log = EventLog.from_simple_log(parse_simple_log(path.read_text(encoding="utf-8")), path.stem)
-    else:
-        from ...mining.xes import read_xes
-        log = read_xes(path)
+    # Read once for everyone: the app's log page and other workflows reading
+    # the same file share it, and in the app a large log is read in a process
+    # of its own (openprocess.flow.loading).
+    from ..loading import read_log
+    log = read_log(path)
     flow.note(f"{path.name}: {len(log)} cases, {log.event_count} events, "
               f"classifier {log.default_classifier().name}")
     return log

openprocess-0.13.1/openprocess/flow/loading.py

@@ -0,0 +1,239 @@
+"""Reading event logs once, and large ones in a process of their own.
+
+Why
+---
+Reading a large XES file is pure Python.  In a thread it holds Python's
+interpreter lock most of the time, and the window, which needs the lock for
+every widget it builds and every repaint, waits its turn thousands of times:
+the app seems frozen although the read is "in the background".  The BPI
+Challenge 2019 log (728 MB) froze OpenProcess for over half a minute at
+launch, twice over, because the log page and a workflow reading the same
+file each parsed it.
+
+What this module does
+---------------------
+* :func:`read_log` reads a file **once** per content: a second request for
+  the same file (same path, size and modification time) gets the same
+  :class:`~openprocess.mining.log.EventLog` while anyone still holds it, and
+  a request made while it is being read waits for that read instead of
+  starting another (the log page and the workflow's *Open log* box share one).
+* With :data:`use_processes` set (the app sets it), a log of
+  :data:`PROCESS_FROM` bytes or more is read by a **worker process**, which
+  has its own interpreter lock, so the window stays responsive.  The worker
+  also hashes the file and fingerprints the log, the two things the
+  reproducibility badge needs, so the app does not redo them on the window's
+  thread.  The log comes back pickled: a columnar log is a few arrays and
+  string tables, which unpickle in a fraction of a second.
+* Scripts, notebooks and ``openprocess run`` read in-process as before
+  (``use_processes`` is False): no process to start, the same result.
+
+Logs are read-only by convention (a box never changes its inputs, §9 of
+``docs/framework.md``; filters make new logs), which is what makes sharing
+one object safe.
+"""
+
+from __future__ import annotations
+
+import os
+import pickle
+import subprocess
+import sys
+import tempfile
+import threading
+import weakref
+from concurrent.futures import Future
+from pathlib import Path
+from typing import Callable
+
+from ..mining.log import EventLog
+
+#: Read large logs in a worker process (the app turns this on; scripts do not need it).
+use_processes = False
+#: From this size a log is read by a worker process (when :data:`use_processes`).
+PROCESS_FROM = 20 * 1024 * 1024
+#: First argument that makes a standalone app run as a log-reading worker.
+WORKER_FLAG = "--openprocess-log-worker"
+
+_lock = threading.Lock()
+_cache: "weakref.WeakValueDictionary[tuple, EventLog]" = weakref.WeakValueDictionary()
+_reading: dict[tuple, Future] = {}
+
+
+class Cancelled(Exception):
+    """The read was stopped (the loading page's *Cancel*)."""
+
+
+def file_key(path: str | Path) -> tuple:
+    """What identifies a file's content here: its resolved path, size and
+    modification time."""
+    resolved = Path(path).resolve()
+    stat = resolved.stat()
+    return (str(resolved), stat.st_size, stat.st_mtime_ns)
+
+
+def cached(path: str | Path) -> EventLog | None:
+    """The log already read from this file (unchanged since), or None."""
+    try:
+        return _cache.get(file_key(path))
+    except OSError:
+        return None
+
+
+def read_log(path: str | Path, progress: Callable | None = None) -> EventLog:
+    """The event log in ``path`` (``.xes``, ``.xes.gz``, ``.csv``, ``.txt`` in course
+    notation), read once for everyone who asks while it is in use.
+
+    ``progress(cases, share)`` is called while reading, as by
+    :func:`openprocess.mining.xes.read_xes`; it may raise to stop the read
+    (then :class:`Cancelled` or that exception comes out, for this caller;
+    another caller waiting on the same read gets it too)."""
+    key = file_key(path)
+    with _lock:
+        log = _cache.get(key)
+        if log is not None:
+            return log
+        pending = _reading.get(key)
+        if pending is None:
+            pending = _reading[key] = Future()
+            mine = True
+        else:
+            mine = False
+    if not mine:
+        return pending.result()                       # waits without holding the interpreter lock
+    try:
+        log = _read(Path(path), key, progress)
+    except BaseException as error:
+        with _lock:
+            _reading.pop(key, None)
+        pending.set_exception(error)
+        raise
+    with _lock:
+        _reading.pop(key, None)
+        _cache[key] = log
+    pending.set_result(log)
+    return log
+
+
+def _read(path: Path, key: tuple, progress) -> EventLog:
+    lower = path.name.lower()
+    if lower.endswith(".csv"):
+        from ..mining.csv_import import read_csv
+        return read_csv(path)
+    if lower.endswith(".txt"):
+        from ..mining.log import parse_simple_log
+        return EventLog.from_simple_log(parse_simple_log(path.read_text(encoding="utf-8")), path.stem)
+    if use_processes and key[1] >= PROCESS_FROM:
+        return _read_in_process(path, progress)
+    from ..mining.xes import read_xes
+    return read_xes(path, progress=progress)
+
+
+# ---------------------------------------------------------------------------
+# The worker process
+# ---------------------------------------------------------------------------
+def worker_command(*arguments: str) -> list[str]:
+    """How to start a worker: this module run by the same Python, or the
+    standalone app run again with :data:`WORKER_FLAG`."""
+    if getattr(sys, "frozen", False):
+        return [sys.executable, WORKER_FLAG, *arguments]
+    return [sys.executable, "-m", "openprocess.flow.loading", *arguments]
+
+
+def _read_in_process(path: Path, progress) -> EventLog:
+    from . import record
+    with tempfile.TemporaryDirectory(prefix="openprocess-log-", ignore_cleanup_errors=True) as folder:
+        output = Path(folder) / "log.pickle"
+        environment = dict(os.environ, PYTHONIOENCODING="utf-8")
+        process = subprocess.Popen(worker_command(str(path), str(output)), stdin=subprocess.PIPE,
+                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
+                                   encoding="utf-8", env=environment,
+                                   creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
+        error_lines: list[str] = []
+
+        def drain_errors() -> None:
+            for line in process.stderr:
+                error_lines.append(line)
+        threading.Thread(target=drain_errors, daemon=True).start()
+        outcome = None
+        try:
+            for line in process.stdout:
+                words = line.split()
+                if not words:
+                    continue
+                if words[0] == "progress" and progress is not None and len(words) >= 3:
+                    share = None if words[2] == "-" else float(words[2])
+                    progress(int(words[1]), share)                  # may raise: Cancel
+                elif words[0] in ("done", "error"):
+                    outcome = line.strip()
+                    break
+        except BaseException:
+            process.kill()
+            process.wait()
+            raise
+        process.wait()
+        if outcome is None or outcome.startswith("error"):
+            detail = outcome[6:] if outcome else ("".join(error_lines[-8:]).strip() or
+                                                  f"the reader stopped (exit {process.returncode})")
+            raise ValueError(detail)
+        with open(output, "rb") as handle:
+            log, file_hash, digest, summary = pickle.load(handle)
+    # What the worker worked out, so the badge does not redo it on the window's thread.
+    stat = path.resolve().stat()
+    record._file_hashes[(str(path.resolve()), stat.st_size, stat.st_mtime_ns)] = file_hash
+    record._remember(log, digest)
+    log.source_path = str(path)
+    # The overview's summary under the log's default classifier, worked out by the
+    # same summarise() in the worker: the log page shows it without computing it.
+    log.precomputed_summaries = {summary_key(log.default_classifier()): summary}
+    return log
+
+
+def summary_key(classifier) -> tuple:
+    """How a precomputed summary is found again: the classifier's identity."""
+    return (classifier.name, tuple(getattr(classifier, "keys", ()) or ()),
+            tuple(sorted(getattr(classifier, "lifecycles", None) or ())))
+
+
+def _child(path: str, output: str) -> int:
+    """The worker: read the log, hash the file, fingerprint the log and
+    summarise it (the overview), pickle all four to ``output``; ``progress <cases> <share>`` lines while reading,
+    then ``done`` or ``error <message>``.  Stops when the parent goes away."""
+    for stream in (sys.stdout, sys.stdin):
+        if hasattr(stream, "reconfigure"):
+            stream.reconfigure(encoding="utf-8")
+
+    def watch_parent() -> None:
+        for _line in sys.stdin:
+            pass
+        os._exit(1)                                   # stdin closed: the parent is gone or cancelled
+    threading.Thread(target=watch_parent, daemon=True).start()
+
+    def report(cases: int, share) -> None:
+        print(f"progress {cases} {'-' if share is None else f'{share:.4f}'}", flush=True)
+    try:
+        from ..mining.xes import read_xes
+        from .record import fingerprint, sha256_file
+        log = read_xes(path, progress=report)
+        print(f"progress {len(log)} 1.0", flush=True)
+        from ..mining.stats import summarise
+        payload = (log, sha256_file(path), fingerprint(log), summarise(log, log.default_classifier()))
+        with open(output, "wb") as handle:
+            pickle.dump(payload, handle, protocol=pickle.HIGHEST_PROTOCOL)
+    except Exception as error:  # noqa: BLE001 - say what went wrong, in one line
+        print("error " + f"{type(error).__name__}: {error}".replace("\n", " "), flush=True)
+        return 1
+    print("done", flush=True)
+    return 0
+
+
+def run_worker(arguments: list[str]) -> int:
+    """The standalone app's entry for :data:`WORKER_FLAG` (``[PATH, OUT]``)."""
+    return _child(arguments[0], arguments[1])
+
+
+if __name__ == "__main__":  # pragma: no cover - the worker process
+    raise SystemExit(_child(sys.argv[1], sys.argv[2]))
+
+
+__all__ = ["Cancelled", "PROCESS_FROM", "WORKER_FLAG", "cached", "file_key", "read_log", "run_worker",
+           "use_processes", "worker_command"]

openprocess-0.13.1/openprocess/flow/parallel.py

@@ -0,0 +1,117 @@
+"""Parallel work in a box: ``flow.parallel_map``.
+
+A box whose work is the same computation over many independent items (the
+variants of a log, the cases, the settings of a sweep inside one box, the
+resamples of a bootstrap) can spread them over the machine's cores::
+
+    from openprocess import flow
+    from openprocess.flow import box, EventLog, PetriNet, Table
+
+
+    def score(model: PetriNet, variant) -> float:          # top level: workers find it by name
+        trace, count = variant
+        ...
+        flow.note(f"{trace}: ...")                          # still reaches the How tab
+        return value
+
+
+    @box(group="Check")
+    def my_check(model: PetriNet, log: EventLog) -> Table:
+        variants = sorted(log.simple_log().items())
+        values = flow.parallel_map(score, variants, shared=model)
+        return Table("Scores", ["variant", "score"], [[v, s] for (v, _n), s in zip(variants, values)])
+
+What it promises: the result is ``[score(model, v) for v in variants]``, the
+same values in the same order, whatever the number of workers -- so the
+workflow's record and ``openprocess run --check`` see the same numbers -- and
+the notes, steps and values the item function reports appear on the box's
+*How* tab in item order, as if the loop had run here.  What it asks of the
+function (``docs/workflows.md``, *Parallel work*): define it at the top level
+of a module (a folder's ``boxes/`` file is fine); compute each item from the
+item and ``shared`` alone; for randomness, seed per item with
+:func:`item_seed`.
+
+The machinery is :mod:`openprocess.parallel` (worker processes, batches,
+``OPENPROCESS_WORKERS``); this module adds the *How* tab.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Iterable
+
+from .. import parallel as _core
+from ..parallel import ParallelError, item_seed
+from .explain import Explanation, current, explaining
+
+
+def parallel_map(function: Callable, items: Iterable, *, shared: Any = None, setup: Callable | None = None,
+                 workers: int | None = None, progress: Callable[[int, int], None] | None = None,
+                 cost: Callable[[Any], float] | None = None) -> list:
+    """``[function(item) for item in items]`` on the machine's cores, in the
+    items' order; with ``shared`` (or ``setup``), ``function(context, item)``.
+
+    ``shared``: data every item needs (a model, a log), sent to each worker
+    once.  ``setup(shared)``: run once per worker, its result is the
+    ``context`` (a compiled model, a cache).  ``workers``: how many processes
+    (default: the cores less one, at most 8; ``1`` runs the loop here).
+    ``progress(done, total)``: called as items finish.  ``cost(item)``: an
+    estimate of an item's cost, so the expensive ones start first.
+
+    Inside a running box, what ``function`` reports with :func:`flow.note`,
+    :func:`flow.steps` and :func:`flow.show` is added to the box's *How* tab
+    in item order.  An item that fails raises :class:`ParallelError` with the
+    worker's traceback."""
+    items = list(items)
+    explanation = current()
+    if explanation is None or _core.reference(function) is None:
+        # Outside a box there is nothing to collect; a function the workers
+        # cannot find by name runs here, where its notes reach the tab directly.
+        return _core.map_items(function, items, shared=shared, setup=setup, workers=workers, progress=progress,
+                               cost=cost)
+    setup_ref = _core.reference(setup) if setup is not None else None
+    if setup is not None and setup_ref is None:
+        return _core.map_items(function, items, shared=shared, setup=setup, workers=1, progress=progress)
+    package = (_core.reference(function), setup_ref, shared, shared is not None or setup is not None)
+    outcomes = _core.map_items(_explained, items, shared=package, setup=_explained_setup, workers=workers,
+                               progress=progress, cost=cost)
+    values = []
+    for value, entries in outcomes:
+        _replay(explanation, entries)
+        values.append(value)
+    return values
+
+
+def _explained_setup(package):
+    """In a worker: the user's function and its context."""
+    function_ref, setup_ref, shared, with_context = package
+    function = _core.resolve(function_ref)
+    context = _core.resolve(setup_ref)(shared) if setup_ref is not None else shared
+    return function, context, with_context
+
+
+def _explained(prepared, item):
+    """In a worker: one item, with what it reported."""
+    function, context, with_context = prepared
+    with explaining() as explanation:
+        value = function(context, item) if with_context else function(item)
+    return value, explanation.entries
+
+
+def _replay(explanation: Explanation, entries: list) -> None:
+    """Add a worker's report to the box's, as if it had been made here."""
+    for entry in entries:
+        explanation.entries.append(entry)
+        if entry[0] == "note":
+            explanation.notes.append(entry[1])
+        elif entry[0] == "steps":
+            explanation.steps.extend(entry[1])
+        elif entry[0] == "show":
+            explanation.shows.append((entry[1], entry[2]))
+
+
+def workers() -> int:
+    """How many worker processes :func:`parallel_map` uses by default."""
+    return _core.workers()
+
+
+__all__ = ["ParallelError", "item_seed", "parallel_map", "workers"]

{openprocess-0.13.0 → openprocess-0.13.1}/openprocess/gui/flow/page.py

@@ -161,7 +161,20 @@ class WorkflowPage(QWidget):
         root.setSpacing(0)
         self.header = PageHeader(self.workflow.name, self._subtitle())
         #: Reproducibility, as last computed (a run finishing, a save): see refresh_title.
-        self._repro = reproducibility.status(self.workflow, getattr(document, "record", None), None, self.folder)
+        #: The first check hashes the inputs on disk (a 700 MB log takes a second or two),
+        #: so it runs in the background; hashing does not hold Python's lock.
+        record = getattr(document, "record", None)
+        self._repro = reproducibility.Status("unchecked", "Checking the inputs…") if record is not None \
+            and record.results else reproducibility.status(self.workflow, record, None, self.folder)
+        if record is not None and record.results:
+            workflow, folder = self.workflow, self.folder
+
+            def first_check(status) -> None:
+                if self.run is None:                     # a run finished meanwhile has the newer word
+                    self._repro = status
+                    self.refresh_title()
+            run_in_background(lambda: reproducibility.status(workflow, record, None, folder), first_check,
+                              lambda _message: None)
         # Canvas: the boxes and wires.  Summary: every result's figure, and the process map.
         self.summary_switch = SegmentedControl(["Canvas", "Summary", "Runs"], compact=True)
         self.summary_switch.setToolTip("Canvas: the boxes and wires.\nSummary: every result's key "
@@ -579,8 +592,19 @@ class WorkflowPage(QWidget):
         def work():
             if blocked:
                 # Custom boxes wait for the user's OK: run with them marked unavailable.
-                return self._run_with_blocked(workflow, changed, previous, stop, relay, blocked)
-            return self.runner.run(workflow, changed, previous, stop, on_status=relay.status.emit)
+                run = self._run_with_blocked(workflow, changed, previous, stop, relay, blocked)
+            else:
+                run = self.runner.run(workflow, changed, previous, stop, on_status=relay.status.emit)
+            # The reproducibility badge, the snapshot and a save fingerprint every result;
+            # do it here, off the window's thread, and they find the codes remembered.
+            from ...flow.record import fingerprint
+            for result in list(run.results.values()):
+                if result.status == DONE:
+                    try:
+                        fingerprint(result.value, remember=True)
+                    except Exception:  # noqa: BLE001 - a value that cannot be fingerprinted is reported later
+                        pass
+            return run
 
         run_in_background(work, relay.finished.emit, lambda message: relay.finished.emit(message))