openprocess 0.7.0__py3-none-any.whl

openprocess/flow/datasets.py

@@ -0,0 +1,198 @@
+"""Public event logs by name, with their fingerprints.
+
+"BPI Challenge 2012" in two papers should mean the same bytes.  This module
+knows the standard logs by name, with where they are published (a DOI on
+4TU.ResearchData), their file name and, once fetched, their SHA-256.  The
+*Open dataset* box fetches a log once into a shared cache, checks the hash,
+and records the dataset's name and hash in the workflow file.
+
+Downloading
+-----------
+4TU serves the files behind a DOI landing page, so this module does not
+guess download addresses: :func:`fetch` looks in the cache first, then at
+``url`` when the entry (or the user's own ``datasets.json``) has one, and
+otherwise says where to download the file by hand and where to put it.  The
+hash is recorded on first use and checked on every later one, so a file
+swapped afterwards is noticed.
+
+Add your own entries in ``~/.openprocess/datasets/datasets.json``::
+
+    {"my log": {"file": "my_log.xes.gz", "url": "https://…", "sha256": null,
+                "description": "...", "citation": "..."}}
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shutil
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable
+
+from ..mining.log import EventLog
+
+
+@dataclass
+class DatasetInfo:
+    name: str
+    file: str
+    doi: str = ""
+    url: str = ""
+    sha256: str | None = None
+    description: str = ""
+    citation: str = ""
+    size: str = ""
+
+    @property
+    def page(self) -> str:
+        return f"https://doi.org/{self.doi}" if self.doi else self.url
+
+
+#: The standard logs, by the name people use for them.
+REGISTRY: dict[str, DatasetInfo] = {}
+
+
+def _add(name, file, doi, description, citation="", size=""):
+    REGISTRY[name] = DatasetInfo(name, file, doi, "", None, description, citation, size)
+
+
+_add("BPI Challenge 2012", "BPI_Challenge_2012.xes.gz", "10.4121/uuid:3926db30-f712-4394-aebc-75976070e91f",
+     "Loan applications at a Dutch financial institute: 13,087 cases, 262,200 events.",
+     "van Dongen, B.F. (2012). BPI Challenge 2012. 4TU.ResearchData.", "29 MB")
+_add("BPI Challenge 2013 incidents", "BPI_Challenge_2013_incidents.xes.gz",
+     "10.4121/uuid:500573e6-accc-4b0c-9576-aa5468b10cee",
+     "Volvo IT incident management: 7,554 cases, 65,533 events.",
+     "Steeman, W. (2013). BPI Challenge 2013, incidents. 4TU.ResearchData.", "3 MB")
+_add("BPI Challenge 2017", "BPI Challenge 2017.xes.gz", "10.4121/uuid:5f3067df-f10b-45da-b98b-86ae4c7a310b",
+     "Loan applications, the 2012 process five years on: 31,509 cases, 1,202,267 events.",
+     "van Dongen, B.F. (2017). BPI Challenge 2017. 4TU.ResearchData.", "70 MB")
+_add("BPI Challenge 2019", "BPI_Challenge_2019.xes", "10.4121/uuid:d06aff4b-79f0-45e6-8ec8-e19730c248f1",
+     "Purchase order handling at a paints and coatings company: 251,734 cases, 1,595,923 events.",
+     "van Dongen, B.F. (2019). BPI Challenge 2019. 4TU.ResearchData.", "1.5 GB")
+_add("BPI Challenge 2020 domestic declarations", "DomesticDeclarations.xes.gz",
+     "10.4121/uuid:3f422315-ed9d-4882-891f-e180a5b4d6c9",
+     "Travel expense declarations at TU/e: 10,500 cases, 56,437 events.",
+     "van Dongen, B.F. (2020). BPI Challenge 2020: Domestic Declarations. 4TU.ResearchData.", "2 MB")
+_add("Sepsis cases", "Sepsis Cases - Event Log.xes.gz", "10.4121/uuid:915d2bfb-7e84-49ad-a286-dc35f063a460",
+     "Sepsis patients in a Dutch hospital: 1,050 cases, 15,214 events.",
+     "Mannhardt, F. (2016). Sepsis Cases - Event Log. 4TU.ResearchData.", "1 MB")
+_add("Road traffic fine management", "Road_Traffic_Fine_Management_Process.xes.gz",
+     "10.4121/uuid:270fd440-1057-4fb9-89a9-b699b47990f5",
+     "Fines by an Italian police force: 150,370 cases, 561,470 events.",
+     "de Leoni, M., Mannhardt, F. (2015). Road Traffic Fine Management Process. 4TU.ResearchData.", "8 MB")
+_add("Hospital billing", "Hospital Billing - Event Log.xes.gz", "10.4121/uuid:76c46b83-c930-4798-a1c9-4be94dfeb741",
+     "Billing of medical services in a hospital: 100,000 cases, 451,359 events.",
+     "Mannhardt, F. (2017). Hospital Billing - Event Log. 4TU.ResearchData.", "17 MB")
+
+
+def cache_dir() -> Path:
+    folder = os.environ.get("OPENPROCESS_DATASETS") or os.environ.get("CPNPY_DATASETS")
+    if not folder:
+        folder = Path.home() / ".openprocess" / "datasets"
+        old = Path.home() / ".cpnpy" / "datasets"           # CPNpy, before 0.7
+        if old.is_dir() and not folder.exists():
+            folder = old
+    folder = Path(folder)
+    folder.mkdir(parents=True, exist_ok=True)
+    return folder
+
+
+def _user_entries() -> dict[str, DatasetInfo]:
+    path = cache_dir() / "datasets.json"
+    if not path.is_file():
+        return {}
+    data = json.loads(path.read_text(encoding="utf-8"))
+    return {name: DatasetInfo(name, item["file"], item.get("doi", ""), item.get("url", ""),
+                              item.get("sha256"), item.get("description", ""), item.get("citation", ""),
+                              item.get("size", "")) for name, item in data.items()}
+
+
+def _hashes_path() -> Path:
+    return cache_dir() / "hashes.json"
+
+
+def recorded_hashes() -> dict[str, str]:
+    path = _hashes_path()
+    return json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {}
+
+
+def names() -> list[str]:
+    return sorted(set(REGISTRY) | set(_user_entries()))
+
+
+def info(name: str) -> DatasetInfo:
+    entries = {**REGISTRY, **_user_entries()}
+    key = next((k for k in entries if k.lower() == name.lower()), None)
+    if key is None:
+        raise KeyError(f"No dataset called {name!r}. Known: " + ", ".join(names()))
+    entry = entries[key]
+    if entry.sha256 is None:
+        entry.sha256 = recorded_hashes().get(entry.name)
+    return entry
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for chunk in iter(lambda: handle.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+class DatasetMissing(FileNotFoundError):
+    """The file is not in the cache and cannot be downloaded by itself."""
+
+
+def fetch(name: str, downloader: Callable[[str, Path], None] | None = None) -> Path:
+    """The dataset's file in the cache, downloading it if a URL is known.
+
+    Raises :class:`DatasetMissing` with instructions when it is not there
+    and there is no URL, and ``ValueError`` when the file's hash differs
+    from the recorded one.
+    """
+    entry = info(name)
+    path = cache_dir() / entry.file
+    if not path.is_file():
+        if not entry.url:
+            raise DatasetMissing(
+                f"{entry.name} is not in the cache. Download “{entry.file}” from {entry.page} "
+                f"and put it in {cache_dir()}, or add a direct url for it in "
+                f"{cache_dir() / 'datasets.json'}.")
+        downloader = downloader or _download
+        temporary = path.with_name(path.name + ".part")
+        downloader(entry.url, temporary)
+        temporary.replace(path)
+    digest = sha256_file(path)
+    if entry.sha256 and digest != entry.sha256:
+        raise ValueError(f"{entry.file} does not match the recorded fingerprint "
+                         f"({entry.sha256[:12]}… expected, {digest[:12]}… found). "
+                         f"Delete it from {cache_dir()} to fetch it again.")
+    if not entry.sha256:
+        hashes = recorded_hashes()
+        hashes[entry.name] = digest
+        _hashes_path().write_text(json.dumps(hashes, indent=2), encoding="utf-8")
+    return path
+
+
+def _download(url: str, target: Path) -> None:
+    with urllib.request.urlopen(url) as response, open(target, "wb") as handle:    # noqa: S310
+        shutil.copyfileobj(response, handle)
+
+
+def open_dataset(name: str) -> EventLog:
+    """Fetch (if needed) and read the dataset as an :class:`EventLog`."""
+    from ..mining.xes import read_xes
+    entry = info(name)
+    path = fetch(name)
+    log = read_xes(path)
+    log.attributes.setdefault("concept:name", entry.name)
+    log.attributes["openprocess:dataset"] = entry.name
+    log.attributes["openprocess:sha256"] = entry.sha256 or recorded_hashes().get(entry.name, "")
+    return log
+
+
+__all__ = ["REGISTRY", "DatasetInfo", "DatasetMissing", "cache_dir", "fetch", "info", "names",
+           "open_dataset", "recorded_hashes"]

openprocess/flow/explain.py

@@ -0,0 +1,115 @@
+"""How a box got to its result: notes, intermediate values and steps.
+
+A box shows what it did, not just what it produced.  Two mechanisms, both
+optional:
+
+* **A result with steps.**  If a box's result has a ``steps()`` method that
+  returns ``(title, content)`` rows, they are shown; ``AlphaResult`` already
+  has one (the eight steps).  A ``warnings`` attribute becomes notes.
+* **A trace of calls.**  While it runs, a box may call :func:`note`,
+  :func:`show` and :func:`steps`::
+
+      flow.note("Split on XOR: {a, e} | {b, c}")
+      flow.show(dfg, "Filtered DFG")
+
+  Everything goes into the :class:`Explanation` of the box that is running,
+  in order.  Outside a running box the calls do nothing, so a box works as a
+  plain function in a script.
+"""
+
+from __future__ import annotations
+
+import contextvars
+from dataclasses import dataclass, field
+from typing import Any
+
+_current: contextvars.ContextVar["Explanation | None"] = contextvars.ContextVar(
+    "openprocess.flow.explanation", default=None)
+
+
+@dataclass
+class Explanation:
+    """What a box reported while it ran, in order."""
+
+    #: ``(title, content)`` rows, as the α-algorithm's eight steps.
+    steps: list[tuple[str, str]] = field(default_factory=list)
+    notes: list[str] = field(default_factory=list)
+    #: Intermediate values with a label, each shown in its own viewer.
+    shows: list[tuple[str, Any]] = field(default_factory=list)
+    #: Every entry in the order it was made: ("note", text), ("show", label, value), ("steps", rows).
+    entries: list[tuple] = field(default_factory=list)
+
+    @property
+    def empty(self) -> bool:
+        return not (self.steps or self.notes or self.shows)
+
+
+def current() -> Explanation | None:
+    """The explanation of the box running now, or None outside a box."""
+    return _current.get()
+
+
+def note(text: str) -> None:
+    """Record one line of what is happening."""
+    explanation = _current.get()
+    if explanation is not None:
+        explanation.notes.append(str(text))
+        explanation.entries.append(("note", str(text)))
+
+
+def show(value: Any, label: str = "") -> Any:
+    """Record an intermediate value (a DFG, a tree, a table) to be shown
+    under *How*; returns the value, so it can wrap an expression."""
+    explanation = _current.get()
+    if explanation is not None:
+        explanation.shows.append((label, value))
+        explanation.entries.append(("show", label, value))
+    return value
+
+
+def steps(rows) -> None:
+    """Record ``(title, content)`` rows: the derivation as the book writes it."""
+    explanation = _current.get()
+    if explanation is not None:
+        rows = [(str(title), str(content)) for title, content in rows]
+        explanation.steps.extend(rows)
+        explanation.entries.append(("steps", rows))
+
+
+class explaining:
+    """``with explaining() as explanation:`` runs a box and collects what it
+    reports."""
+
+    def __init__(self, explanation: Explanation | None = None) -> None:
+        self.explanation = explanation or Explanation()
+        self._token = None
+
+    def __enter__(self) -> Explanation:
+        self._token = _current.set(self.explanation)
+        return self.explanation
+
+    def __exit__(self, *_exc) -> None:
+        _current.reset(self._token)
+
+
+def collect_result(value: Any, explanation: Explanation) -> None:
+    """Add what a result object can say about itself: its ``steps()`` and
+    its ``warnings``.  Only when the box did not record steps itself."""
+    if not explanation.steps:
+        method = getattr(value, "steps", None)
+        if callable(method):
+            try:
+                rows = method()
+                if isinstance(rows, list):
+                    steps_rows = [(str(t), str(c)) for t, c in rows]
+                    explanation.steps.extend(steps_rows)
+                    explanation.entries.append(("steps", steps_rows))
+            except Exception:       # noqa: BLE001 - a result's own method; never fatal
+                pass
+    warnings = getattr(value, "warnings", None)
+    if isinstance(warnings, list):
+        for warning in warnings:
+            text = f"Warning: {warning}"
+            if text not in explanation.notes:
+                explanation.notes.append(text)
+                explanation.entries.append(("note", text))

openprocess/flow/library.py

@@ -0,0 +1,222 @@
+"""Where boxes come from: OpenProcess's own, a folder's ``boxes/``, and packages.
+
+A :class:`Library` holds :class:`~.box.BoxSpec` objects by id.  The
+standard library is OpenProcess's own boxes (``openprocess.flow.boxes``).  On top of
+it, :meth:`Library.load_folder` reads every ``.py`` file in a folder (the
+open folder's ``boxes/`` subfolder in the app) and
+:meth:`Library.load_entry_points` every installed package that declares a
+``openprocess.boxes`` entry point, so a research group can publish a box pack on
+PyPI.
+
+A file with a mistake does not stop the others: it is kept as a
+:class:`BrokenBox` with the reason in plain words, and the app lists it
+greyed out.  A box whose library is missing (``needs="pandas"``) is listed
+too, greyed out, with "needs pandas".
+"""
+
+from __future__ import annotations
+
+import hashlib
+import importlib
+import importlib.metadata
+import importlib.util
+import sys
+import traceback
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable
+
+from .box import Box, BoxError, BoxSpec
+
+#: The modules of the standard library, in the order the box list shows their groups.
+STANDARD_MODULES = ("input", "filter", "discover", "check", "compare", "output",
+                    "science", "predict", "cpn", "sweeps")
+GROUP_ORDER = ("Input", "Filter", "Discover", "Check", "Compare", "Output", "Science",
+               "Predict", "Coloured nets", "Sweep", "Yours")
+
+
+@dataclass
+class BrokenBox:
+    """A box file that could not be loaded, and why."""
+
+    file: str
+    reason: str
+    name: str = ""
+
+    @property
+    def id(self) -> str:
+        return f"broken:{self.file}"
+
+
+class Library:
+    def __init__(self) -> None:
+        self.specs: dict[str, BoxSpec] = {}
+        self.broken: list[BrokenBox] = []
+        #: Where each custom box file came from, with its content hash (for the record).
+        self.files: dict[str, str] = {}
+
+    # -- lookup -------------------------------------------------------------------
+    def get(self, box_id: str) -> BoxSpec:
+        try:
+            return self.specs[box_id]
+        except KeyError:
+            raise KeyError(f"No box {box_id!r}") from None
+
+    def resolve(self, box) -> BoxSpec:
+        """A :class:`Box`, a :class:`BoxSpec`, an id or a bare function name."""
+        if isinstance(box, Box):
+            if box.spec.id not in self.specs:
+                self.register(box)
+            return box.spec
+        if isinstance(box, BoxSpec):
+            if box.id not in self.specs:
+                self.specs[box.id] = box
+            return box
+        if box in self.specs:
+            return self.specs[box]
+        matches = [s for s in self.specs.values() if s.function.__name__ == box or s.name == box]
+        if len(matches) == 1:
+            return matches[0]
+        if not matches:
+            raise KeyError(f"No box {box!r}")
+        raise KeyError(f"Several boxes are called {box!r}: " + ", ".join(m.id for m in matches))
+
+    def __contains__(self, box_id: str) -> bool:
+        return box_id in self.specs
+
+    def __iter__(self):
+        return iter(self.specs.values())
+
+    def __len__(self) -> int:
+        return len(self.specs)
+
+    def by_group(self) -> dict[str, list[BoxSpec]]:
+        groups: dict[str, list[BoxSpec]] = {}
+        for spec in self.specs.values():
+            groups.setdefault(spec.group, []).append(spec)
+        ordered = {g: groups[g] for g in GROUP_ORDER if g in groups}
+        ordered.update({g: specs for g, specs in groups.items() if g not in ordered})
+        return ordered
+
+    # -- registering -----------------------------------------------------------------
+    def register(self, *boxes: Box | BoxSpec, custom: bool | None = None) -> None:
+        for item in boxes:
+            spec = item.spec if isinstance(item, Box) else item
+            if custom is not None:
+                spec.custom = custom
+            self.specs[spec.id] = spec
+
+    def register_module(self, module, custom: bool = False) -> list[BoxSpec]:
+        """Register the boxes *defined* in ``module`` (a box it merely imports,
+        such as a standard box used by a saved group, stays as it is)."""
+        from .workflow import Recorded
+        own = module.__name__
+
+        def defined_here(value) -> bool:
+            fn = value.fn if isinstance(value, Recorded) else value.spec.function
+            return getattr(fn, "__module__", own) == own
+
+        found = [value for value in vars(module).values() if isinstance(value, Box) and defined_here(value)]
+        for value in vars(module).values():
+            if isinstance(value, Recorded) and defined_here(value):   # a @workflow function with typed parameters
+                box = value.as_box()
+                if box is not None:
+                    box.spec.id = f"{module.__name__}.{value.__name__}"
+                    found.append(box)
+        found.sort(key=lambda b: b.spec.line)
+        self.register(*found, custom=custom)
+        return [b.spec for b in found]
+
+    def load_standard(self) -> "Library":
+        for name in STANDARD_MODULES:
+            module = importlib.import_module(f"openprocess.flow.boxes.{name}")
+            self.register_module(module)
+        return self
+
+    def load_file(self, path: str | Path) -> list[BoxSpec]:
+        """Load one box file.  Raises nothing: a mistake becomes a BrokenBox."""
+        path = Path(path)
+        text = path.read_bytes()
+        self.files[str(path)] = hashlib.sha256(text).hexdigest()
+        module_name = "openprocess_boxes_" + hashlib.sha256(str(path.resolve()).encode()).hexdigest()[:12]
+        self.broken = [b for b in self.broken if b.file != str(path)]
+        for spec_id in [s for s, spec in self.specs.items() if spec.file == str(path)]:
+            del self.specs[spec_id]
+        try:
+            spec = importlib.util.spec_from_file_location(module_name, path)
+            module = importlib.util.module_from_spec(spec)
+            sys.modules[module_name] = module
+            spec.loader.exec_module(module)
+        except BoxError as error:
+            self.broken.append(BrokenBox(str(path), str(error), path.stem))
+            return []
+        except SyntaxError as error:
+            self.broken.append(BrokenBox(str(path), f"line {error.lineno}: {error.msg}", path.stem))
+            return []
+        except Exception as error:   # noqa: BLE001 - whatever the file did
+            detail = "".join(traceback.format_exception_only(type(error), error)).strip()
+            self.broken.append(BrokenBox(str(path), detail, path.stem))
+            return []
+        specs = self.register_module(module, custom=True)
+        for s in specs:
+            s.file = str(path)
+            s.group = s.group or "Yours"
+        if not specs:
+            self.broken.append(BrokenBox(str(path), "the file has no @box function", path.stem))
+        return specs
+
+    def load_folder(self, folder: str | Path) -> list[BoxSpec]:
+        """Every ``.py`` file directly in ``folder`` (``boxes/`` in the app)."""
+        folder = Path(folder)
+        if not folder.is_dir():
+            return []
+        loaded = []
+        for path in sorted(folder.glob("*.py")):
+            if path.name.startswith("_"):
+                continue
+            loaded.extend(self.load_file(path))
+        return loaded
+
+    def load_entry_points(self, group: str = "openprocess.boxes") -> list[BoxSpec]:
+        """Boxes from installed packages that declare ``[project.entry-points."openprocess.boxes"]``."""
+        loaded = []
+        try:
+            points = importlib.metadata.entry_points(group=group)
+        except TypeError:                       # Python 3.9 style
+            points = importlib.metadata.entry_points().get(group, [])
+        for point in points:
+            try:
+                module = point.load()
+            except Exception as error:   # noqa: BLE001
+                self.broken.append(BrokenBox(point.value, f"package {point.name}: {error}", point.name))
+                continue
+            specs = self.register_module(module, custom=True)
+            for s in specs:
+                s.group = s.group or point.name
+            loaded.extend(specs)
+        return loaded
+
+
+_standard: Library | None = None
+
+
+def standard_library() -> Library:
+    """OpenProcess's own boxes (loaded once)."""
+    global _standard
+    if _standard is None:
+        _standard = Library().load_standard()
+    return _standard
+
+
+def library_for(folder: str | Path | None = None, packages: bool = True) -> Library:
+    """The standard boxes plus a folder's ``boxes/`` and installed packages."""
+    library = Library().load_standard()
+    if packages:
+        library.load_entry_points()
+    if folder is not None:
+        boxes = Path(folder) / "boxes" if (Path(folder) / "boxes").is_dir() else Path(folder)
+        library.load_folder(boxes)
+    return library
+
+
+__all__ = ["BrokenBox", "GROUP_ORDER", "Library", "library_for", "standard_library"]