openprocess 0.7.0__py3-none-any.whl

openprocess/flow/boxes/predict.py

@@ -0,0 +1,186 @@
+"""Predict boxes: the prediction pipeline of predictive process monitoring.
+
+*Prefixes* turns a log into a dataset of prefix encodings with a label
+(the next activity, the remaining time, or the outcome); *Split by time*
+makes a train and a test set by case start, never at random; *Evaluate
+predictions* scores a model's predictions per prefix length.  A model of
+your own goes in between as a box that takes a :class:`~..types.Dataset`
+and gives a :class:`~..types.Predictor` (a scikit-learn estimator fits the
+shape; see ``examples/boxes/next_activity_sklearn.py``).  *Frequency
+model* is a baseline that needs no library.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Literal
+
+from ... import flow
+from ...mining.log import EventLog
+from ..box import box
+from ..types import Dataset, Predictions, Predictor, Table
+
+END = "⟂"       # the label of "the case ends here"
+
+
+@box(name="Prefixes", group="Predict")
+def prefixes(log: EventLog, label: Literal["next activity", "remaining time", "outcome"] = "next activity",
+             encoding: Literal["index", "one-hot", "frequency"] = "frequency",
+             min_prefix: int = 1, max_prefix: int = 20) -> Dataset:
+    """One row per prefix of every case, encoded as features, with the
+    label to predict: the next activity (⟂ when the case ends), the
+    remaining time in hours, or the outcome (the last activity).
+
+    encoding: frequency = how often each activity occurred so far, plus the last activity; index = the last k activities as indices; one-hot = the last k activities as 0/1 columns
+    min_prefix: the shortest prefix to include
+    max_prefix: the longest prefix to include (longer cases give more rows)
+    """
+    activities = sorted({a for seq in log.sequences() for a in seq})
+    index = {a: i for i, a in enumerate(activities)}
+    k = max_prefix if encoding != "frequency" else 1
+    if encoding == "frequency":
+        names = [f"count {a}" for a in activities] + ["prefix length"] + [f"last is {a}" for a in activities]
+    elif encoding == "index":
+        names = [f"activity −{i}" for i in range(k, 0, -1)]
+    else:
+        names = [f"−{i} is {a}" for i in range(k, 0, -1) for a in activities]
+    X, y, case_ids, lengths, starts = [], [], [], [], []
+    classifier = log.default_classifier()
+    for trace in log:
+        events = [e for e in trace if classifier.accepts(e)]
+        seq = [classifier.label(e) for e in events]
+        times = [e.timestamp for e in events]
+        start = times[0] if times and times[0] else None
+        end_time = times[-1] if times and times[-1] else None
+        for n in range(min_prefix, min(len(seq), max_prefix) + 1):
+            prefix = seq[:n]
+            if encoding == "frequency":
+                counts = Counter(prefix)
+                row = [counts[a] for a in activities] + [n] + [1 if prefix[-1] == a else 0 for a in activities]
+            elif encoding == "index":
+                window = ([-1] * k + [index[a] for a in prefix])[-k:]
+                row = window
+            else:
+                window = ([None] * k + prefix)[-k:]
+                row = [1 if w == a else 0 for w in window for a in activities]
+            if label == "next activity":
+                target = seq[n] if n < len(seq) else END
+            elif label == "outcome":
+                target = seq[-1]
+            else:
+                if end_time is None or times[n - 1] is None:
+                    continue
+                target = (end_time - times[n - 1]).total_seconds() / 3600
+            X.append(row)
+            y.append(target)
+            case_ids.append(trace.case_id)
+            lengths.append(n)
+            starts.append(start)
+    kind = "regression" if label == "remaining time" else "classification"
+    classes = sorted(set(y)) if kind == "classification" else None
+    flow.note(f"{len(X)} prefixes from {len(log)} cases, {len(names)} features, label: {label}")
+    return Dataset(X, y, names, label, kind, case_ids, lengths, starts, activities, classes,
+                   name=f"Prefixes of {log.name}")
+
+
+@dataclass
+class Split:
+    train: Dataset
+    test: Dataset
+
+
+@box(name="Split by time", group="Predict")
+def split_by_time(dataset: Dataset, train_fraction: float = 0.7) -> Split:
+    """Train and test sets by case start: the earliest cases train, the
+    latest test, so the model never sees the future. Without timestamps the
+    cases are split in log order."""
+    cases = list(dict.fromkeys(dataset.case_ids))
+    starts = {c: s for c, s in zip(dataset.case_ids, dataset.case_starts)}
+    if any(isinstance(starts.get(c), datetime) for c in cases):
+        cases.sort(key=lambda c: (starts.get(c) is None, starts.get(c) or datetime.min))
+    cut = max(1, int(round(len(cases) * train_fraction)))
+    train_cases = set(cases[:cut])
+    train = [i for i, c in enumerate(dataset.case_ids) if c in train_cases]
+    test = [i for i, c in enumerate(dataset.case_ids) if c not in train_cases]
+    flow.note(f"{len(train_cases)} cases ({len(train)} prefixes) train, "
+              f"{len(cases) - len(train_cases)} cases ({len(test)} prefixes) test")
+    return Split(dataset.subset(train, "train"), dataset.subset(test, "test"))
+
+
+class FrequencyModel(Predictor):
+    """Predicts, for each last activity, what most often came next (or the
+    mean remaining time): the baseline every paper has to beat."""
+
+    name = "Frequency model"
+
+    def __init__(self) -> None:
+        self.table: dict = {}
+        self.default = None
+        self.kind = "classification"
+
+    def _key(self, row, dataset: Dataset):
+        if dataset.feature_names and dataset.feature_names[-1].startswith("last is "):
+            n = len(dataset.activities)
+            last = row[-n:]
+            return next((a for a, flag in zip(dataset.activities, last) if flag), None)
+        return tuple(row[-1:])
+
+    def fit(self, dataset: Dataset) -> "FrequencyModel":
+        self.kind = dataset.kind
+        seen: dict = {}
+        for row, label in zip(dataset.X, dataset.y):
+            seen.setdefault(self._key(row, dataset), []).append(label)
+        if dataset.kind == "classification":
+            self.table = {key: Counter(labels).most_common(1)[0][0] for key, labels in seen.items()}
+            self.default = Counter(dataset.y).most_common(1)[0][0] if dataset.y else None
+        else:
+            self.table = {key: sum(labels) / len(labels) for key, labels in seen.items()}
+            self.default = sum(dataset.y) / len(dataset.y) if dataset.y else 0.0
+        return self
+
+    def predict(self, dataset: Dataset) -> list:
+        return [self.table.get(self._key(row, dataset), self.default) for row in dataset.X]
+
+
+@box(name="Frequency model", group="Predict")
+def frequency_model(train: Dataset) -> Predictor:
+    """The baseline: for each last activity, the most frequent next activity
+    (or the mean remaining time) in the training set."""
+    model = FrequencyModel().fit(train)
+    flow.note(f"{len(model.table)} last activities seen in {len(train)} prefixes")
+    return model
+
+
+@box(name="Predict", group="Predict")
+def predict(model: Predictor, dataset: Dataset) -> Predictions:
+    """Applies a fitted model to a dataset (the test set)."""
+    values = list(model.predict(dataset))
+    flow.note(f"{len(values)} predictions by {getattr(model, 'name', type(model).__name__)}")
+    return Predictions(values, dataset, getattr(model, "name", type(model).__name__))
+
+
+@box(name="Evaluate predictions", group="Predict")
+def evaluate_predictions(predictions: Predictions) -> Table:
+    """Accuracy (classification) or mean absolute error in hours
+    (regression), overall and per prefix length."""
+    dataset = predictions.dataset
+    if len(predictions.values) != len(dataset):
+        raise ValueError("The predictions and the dataset have different lengths")
+    by_length: dict[int, list] = {}
+    for value, truth, length in zip(predictions.values, dataset.y, dataset.prefix_lengths or [0] * len(dataset)):
+        by_length.setdefault(length, []).append((value, truth))
+    classification = dataset.kind == "classification"
+
+    def score(pairs) -> float:
+        if classification:
+            return sum(1 for v, t in pairs if v == t) / len(pairs)
+        return sum(abs(float(v) - float(t)) for v, t in pairs) / len(pairs)
+    overall = score(list(zip(predictions.values, dataset.y)))
+    metric = "accuracy" if classification else "MAE (hours)"
+    rows = [["all", len(dataset), round(overall, 4)]]
+    rows += [[str(length), len(pairs), round(score(pairs), 4)] for length, pairs in sorted(by_length.items())]
+    flow.note(f"{metric} over {len(dataset)} prefixes: {overall:.3f}")
+    return Table(f"{predictions.name}: {metric}", ["prefix length", "prefixes", metric], rows,
+                 note=f"{metric} of {predictions.name} on {dataset.name}: {overall:.3f}.")

openprocess/flow/boxes/science.py

@@ -0,0 +1,159 @@
+"""Science boxes: how any Python library plugs in.
+
+Each box here is a few lines around one library call, so it reads as a
+recipe: copy it, change three lines, save it in your folder's ``boxes/``.
+They live behind ``pip install openprocess[science]``; without a library the box
+is listed greyed out with "needs pandas" (the ``needs=`` of ``@box``).
+"""
+
+from __future__ import annotations
+
+import random
+from typing import Literal
+
+from ... import flow
+from ...mining.conformance.token_replay import token_replay
+from ...mining.log import EventLog
+from ...mining.petrinet import PetriNet
+from ..box import box
+from ..convert import convert
+from ..types import Figure, Scores, Table
+
+
+@box(name="Describe log", group="Science", needs="pandas")
+def describe_log(log: EventLog) -> Table:
+    """pandas' ``describe()`` of the events per case and, when the log has
+    timestamps, of the case durations in hours. The log goes in as a
+    DataFrame through the converter; the table comes back as a Table."""
+    import pandas as pd
+    frame = convert(log, "pandas.DataFrame")
+    per_case = frame.groupby("case:concept:name", sort=False).size().rename("events per case")
+    columns = {"events per case": per_case.describe()}
+    if "time:timestamp" in frame.columns and frame["time:timestamp"].notna().any():
+        span = frame.groupby("case:concept:name")["time:timestamp"].agg(["min", "max"])
+        hours = ((span["max"] - span["min"]).dt.total_seconds() / 3600).rename("case duration (h)")
+        columns["case duration (h)"] = hours.describe()
+    described = pd.DataFrame(columns)
+    flow.note(f"df: {len(frame)} rows × {len(frame.columns)} columns; per_case = df.groupby(case).size()")
+    rows = [[stat, *[round(float(described.loc[stat, c]), 3) for c in described.columns]] for stat in described.index]
+    return Table("Describe", ["statistic", *described.columns], rows)
+
+
+@box(name="Bootstrap fitness", group="Science", needs="numpy")
+def bootstrap_fitness(model: PetriNet, log: EventLog, samples: int = 200, seed: int = 0) -> Table:
+    """Fitness with a 95 % bootstrap interval: resample the cases with
+    replacement *samples* times and replay each sample, so a number comes
+    with its uncertainty.
+
+    samples: how many resamples (more: a steadier interval)
+    seed: the random seed, recorded in the workflow file
+    """
+    import numpy as np
+    sequences = log.sequences()
+    rng = np.random.default_rng(seed)
+    values = []
+    for _ in range(samples):
+        picked = rng.integers(0, len(sequences), len(sequences))
+        from collections import Counter
+        sample = Counter(sequences[i] for i in picked)
+        values.append(token_replay(model, sample).fitness)
+    values = np.array(values)
+    low, high = np.percentile(values, [2.5, 97.5])
+    flow.note(f"{samples} samples of {len(sequences)} cases, seed {seed}; "
+              f"min {values.min():.3f}, max {values.max():.3f}")
+    table = Table(f"Bootstrap fitness of {model.name}", ["statistic", "fitness"],
+                  [["mean", round(float(values.mean()), 4)], ["2.5 %", round(float(low), 4)],
+                   ["97.5 %", round(float(high), 4)], ["samples", samples], ["seed", seed]],
+                  samples=[float(v) for v in values],
+                  note=f"Fitness of {model.name}: {values.mean():.3f} (95 % interval {low:.3f} to {high:.3f}).")
+    return table
+
+
+@box(name="Compare samples", group="Science", needs="scipy")
+def compare_samples(a: Table, b: Table,
+                    test: Literal["Mann–Whitney U", "t-test"] = "Mann–Whitney U") -> Table:
+    """Are two samples (two bootstrap tables) from the same distribution?
+    A Mann–Whitney U test, or Welch's t-test, from SciPy."""
+    from scipy import stats
+    if not a.samples or not b.samples:
+        raise ValueError("Both tables need samples (connect two Bootstrap fitness boxes)")
+    x, y = a.samples, b.samples
+    if test == "t-test":
+        result = stats.ttest_ind(x, y, equal_var=False)
+        statistic_name = "t"
+    else:
+        result = stats.mannwhitneyu(x, y, alternative="two-sided")
+        statistic_name = "U"
+    p = float(result.pvalue)
+    mean_a, mean_b = sum(x) / len(x), sum(y) / len(y)
+    verdict = (f"The samples differ (p {'< 0.001' if p < 0.001 else f'= {p:.3f}'}): "
+               f"{a.name if mean_a > mean_b else b.name} is higher, and the bootstrap says it is not luck."
+               if p < 0.05 else f"No difference the test can see (p = {p:.3f}).")
+    flow.note(f"A = {a.name} ({len(x)} samples, mean {mean_a:.3f}); B = {b.name} ({len(y)} samples, mean {mean_b:.3f})")
+    return Table(test, ["", "value"], [[statistic_name, round(float(result.statistic), 3)],
+                                      ["p-value", "< 0.001" if p < 0.001 else round(p, 4)],
+                                      ["mean A", round(mean_a, 4)], ["mean B", round(mean_b, 4)]], note=verdict)
+
+
+@box(name="Correlate", group="Science", needs="pandas")
+def correlate(table: Table) -> Table:
+    """Correlations between the numeric columns of a table (pandas), e.g.
+    a swept noise level against fitness."""
+    frame = convert(table, "pandas.DataFrame").select_dtypes("number")
+    if frame.shape[1] < 2:
+        raise ValueError("The table needs at least two numeric columns")
+    matrix = frame.corr()
+    rows = [[row, *[round(float(v), 3) for v in matrix.loc[row]]] for row in matrix.index]
+    return Table("Correlations", ["", *matrix.columns], rows)
+
+
+@box(name="Plot", group="Science", needs="matplotlib")
+def plot(scores: list[Scores] = (), tables: list[Table] = (),
+         x: str = "", y: str = "") -> Figure:
+    """Bars per model for scores, histograms for bootstrap tables, or a line
+    of column *y* against column *x* of a table (a sweep). The figure is a
+    matplotlib figure, converted to SVG; its code is this box's code.
+
+    x: a column of the table to put on the x axis (lines only)
+    y: the column to plot against it
+    """
+    import matplotlib
+    matplotlib.use("Agg")
+    import matplotlib.pyplot as plt
+    fig, ax = plt.subplots(figsize=(5.5, 3.2))
+    scores, tables = list(scores), list(tables)
+    if scores:
+        metrics = [m for m in dict.fromkeys(k for s in scores for k in s.metrics)
+                   if all(isinstance(s.metrics.get(m, 0.0), (int, float)) for s in scores)]
+        width = 0.8 / max(1, len(metrics))
+        for j, metric in enumerate(metrics):
+            ax.bar([i + j * width for i in range(len(scores))], [float(s.metrics.get(metric, 0)) for s in scores],
+                   width=width, label=metric)
+        ax.set_xticks([i + 0.4 - width / 2 for i in range(len(scores))])
+        ax.set_xticklabels([s.model for s in scores], rotation=15, ha="right")
+        ax.set_ylim(0, 1.05)
+        ax.legend(fontsize=8)
+        caption = f"{len(scores)} models, {len(metrics)} scores each."
+    elif tables and x and y:
+        for table in tables:
+            ax.plot(table.column(x), table.column(y), marker="o", label=table.name)
+        ax.set_xlabel(x)
+        ax.set_ylabel(y)
+        if len(tables) > 1:
+            ax.legend(fontsize=8)
+        caption = f"{y} against {x}."
+    elif tables:
+        for table in tables:
+            if table.samples:
+                ax.hist(table.samples, bins=12, alpha=0.6, label=table.name)
+        ax.set_xlabel("fitness over bootstrap samples")
+        ax.legend(fontsize=8)
+        caption = f"{len(tables)} bootstrap distribution{'s' if len(tables) != 1 else ''}."
+    else:
+        raise ValueError("Connect scores, or tables with samples or an x and y column")
+    fig.tight_layout()
+    figure = convert(fig, Figure)
+    plt.close(fig)
+    figure.caption = caption
+    flow.note(f"matplotlib {matplotlib.__version__}: {caption}")
+    return figure

openprocess/flow/boxes/sweeps.py

@@ -0,0 +1,18 @@
+"""Sweep table: the scores of every swept value, stacked."""
+
+from __future__ import annotations
+
+from ... import flow
+from ..box import box
+from ..types import Scores, Table
+
+
+@box(name="Sweep table", group="Sweep")
+def sweep_table(scores: list[Scores]) -> Table:
+    """Collects the scores of every value of a sweep into one table: the
+    swept settings as columns (from the scores' context), then the scores.
+    Feed it to Plot (with x and y) or Correlate."""
+    table = Table.from_scores(scores, "Sweep")
+    swept = [c for c in table.columns if c != "model" and any(c in s.context for s in scores)]
+    flow.note(f"{len(table)} rows; swept: {', '.join(swept) or 'nothing'}")
+    return table

openprocess/flow/convert.py

@@ -0,0 +1,187 @@
+"""Converters between OpenProcess's types and other libraries' objects.
+
+A box that wraps a pandas, PM4Py or matplotlib function asks for the
+conversion in one line::
+
+    df = convert(log, "pandas.DataFrame")       # one row per event
+    return convert(fig, Figure)                  # a matplotlib figure as SVG
+
+Foreign types are named by their dotted path, so this module imports none
+of them until a conversion is asked for.  Register your own with
+:func:`register`: a function from one type to another, keyed by the source
+class and the target (a class, or a dotted name).
+"""
+
+from __future__ import annotations
+
+import importlib
+from typing import Any, Callable
+
+from ..mining.log import KEY_NAME, KEY_RESOURCE, KEY_TIME, EventLog, SimpleLog
+from ..mining.petrinet import PetriNet
+from .types import Figure, Table
+
+_converters: dict[tuple[str, str], Callable] = {}
+
+
+def _name(target) -> str:
+    if isinstance(target, str):
+        return target
+    return f"{target.__module__}.{target.__qualname__}"
+
+
+def register(source, target, function: Callable) -> None:
+    """``function(value) -> converted`` for a value of class ``source``."""
+    _converters[(_name(source), _name(target))] = function
+
+
+def _class_names(value) -> list[str]:
+    return [f"{cls.__module__}.{cls.__qualname__}" for cls in type(value).__mro__]
+
+
+def can_convert(value: Any, target) -> bool:
+    return any((name, _name(target)) in _converters for name in _class_names(value))
+
+
+def convert(value: Any, target):
+    """Convert ``value`` to ``target`` (a class or a dotted name)."""
+    wanted = _name(target)
+    for name in _class_names(value):
+        if name == wanted:
+            return value
+        function = _converters.get((name, wanted))
+        if function is not None:
+            return function(value)
+    raise TypeError(f"No converter from {type(value).__name__} to {wanted}")
+
+
+def converters() -> list[tuple[str, str]]:
+    return sorted(_converters)
+
+
+# ---------------------------------------------------------------------------
+# Built-in converters
+# ---------------------------------------------------------------------------
+def _simple_to_log(simple: SimpleLog) -> EventLog:
+    return EventLog.from_simple_log(simple)
+
+
+def _log_to_simple(log: EventLog) -> SimpleLog:
+    return log.simple_log()
+
+
+def _log_to_dataframe(log: EventLog):
+    pd = importlib.import_module("pandas")
+    rows = []
+    for trace in log:
+        case = trace.case_id
+        for event in trace:
+            row = {"case:concept:name": case}
+            row.update(event.attributes)
+            rows.append(row)
+    frame = pd.DataFrame(rows)
+    if KEY_TIME in frame.columns:
+        frame[KEY_TIME] = pd.to_datetime(frame[KEY_TIME], utc=True, errors="coerce")
+    return frame
+
+
+def _dataframe_to_log(frame) -> EventLog:
+    from ..mining.log import Event, Trace
+    case_column = "case:concept:name" if "case:concept:name" in frame.columns else frame.columns[0]
+    log = EventLog(attributes={KEY_NAME: "DataFrame"})
+    for case, group in frame.groupby(case_column, sort=False):
+        trace = Trace({KEY_NAME: str(case)})
+        for record in group.drop(columns=[case_column]).to_dict("records"):
+            attributes = {k: (v.to_pydatetime() if hasattr(v, "to_pydatetime") else v)
+                          for k, v in record.items() if v == v}       # drop NaN
+            trace.events.append(Event(attributes))
+        log.traces.append(trace)
+    return log
+
+
+def _table_to_dataframe(table: Table):
+    pd = importlib.import_module("pandas")
+    return pd.DataFrame(table.rows, columns=table.columns)
+
+
+def _dataframe_to_table(frame) -> Table:
+    columns = [str(c) for c in frame.columns]
+    rows = [[_plain(v) for v in row] for row in frame.itertuples(index=False, name=None)]
+    return Table("DataFrame", columns, rows)
+
+
+def _series_to_table(series) -> Table:
+    name = str(series.name or "value")
+    return Table(name, ["index", name], [[_plain(k), _plain(v)] for k, v in series.items()])
+
+
+def _plain(value):
+    item = getattr(value, "item", None)          # NumPy scalars -> Python
+    if callable(item):
+        try:
+            return item()
+        except (ValueError, TypeError):
+            return value
+    return value
+
+
+def _ndarray_to_table(array) -> Table:
+    rows = array.tolist()
+    if rows and not isinstance(rows[0], list):
+        rows = [[r] for r in rows]
+    width = len(rows[0]) if rows else 0
+    return Table("array", [f"c{i}" for i in range(width)], rows)
+
+
+def _mpl_figure_to_figure(fig) -> Figure:
+    import io
+    buffer = io.StringIO()
+    fig.savefig(buffer, format="svg", bbox_inches="tight")
+    svg = buffer.getvalue()
+    png = io.BytesIO()
+    try:
+        fig.savefig(png, format="png", dpi=160, bbox_inches="tight")
+        png_bytes = png.getvalue()
+    except Exception:    # noqa: BLE001 - no PNG backend
+        png_bytes = None
+    title = ""
+    try:
+        title = fig.axes[0].get_title() if fig.axes else ""
+    except Exception:    # noqa: BLE001
+        pass
+    return Figure(title or "Figure", svg=svg, png=png_bytes)
+
+
+def _petri_to_pm4py(net: PetriNet):
+    import os
+    import tempfile
+    pm4py = importlib.import_module("pm4py")
+    from ..mining.pnml import write_pnml
+    handle, path = tempfile.mkstemp(suffix=".pnml")
+    os.close(handle)
+    try:
+        write_pnml(net, path)
+        return pm4py.read_pnml(path)
+    finally:
+        os.unlink(path)
+
+
+def _pm4py_to_petri(triple) -> PetriNet:
+    from ..mining.pm4py_bridge import _to_native
+    net, im, fm = triple
+    return _to_native(net, im, fm, getattr(net, "name", None) or "PM4Py net")
+
+
+register(SimpleLog, EventLog, _simple_to_log)
+register(EventLog, SimpleLog, _log_to_simple)
+register(EventLog, "pandas.core.frame.DataFrame", _log_to_dataframe)
+register("pandas.core.frame.DataFrame", EventLog, _dataframe_to_log)
+register(Table, "pandas.core.frame.DataFrame", _table_to_dataframe)
+register("pandas.core.frame.DataFrame", Table, _dataframe_to_table)
+register("pandas.core.series.Series", Table, _series_to_table)
+register("numpy.ndarray", Table, _ndarray_to_table)
+register("matplotlib.figure.Figure", Figure, _mpl_figure_to_figure)
+register(PetriNet, "pm4py", _petri_to_pm4py)
+register(tuple, "openprocess.mining.petrinet.PetriNet", _pm4py_to_petri)
+
+__all__ = ["can_convert", "convert", "converters", "register"]