openprocess 0.7.0__py3-none-any.whl

openprocess/mining/conformance/alignments.py

@@ -0,0 +1,263 @@
+"""Alignments (Carmona, van Dongen, Solti & Weidlich, ch. 7; course reading 5).
+
+An **alignment** pairs up a trace with a complete run of the model, move by
+move.  Each move is one of:
+
+=============  =================  =============================================
+move           written            meaning
+=============  =================  =============================================
+synchronous    ``(a, a)``         the log did a, the model did a -- agreement
+log move       ``(a, ≫)``         the log did a, the model could not
+model move     ``(≫, a)``         the model had to do a, the log did not
+silent move    ``(≫, τ)``         the model fired a τ transition (free)
+=============  =================  =============================================
+
+With the **standard cost function** (log and model moves cost 1, synchronous
+and silent moves cost 0), an *optimal* alignment is one with minimal total
+cost: the fewest deviations needed to explain the trace with the model.
+
+How it is computed
+------------------
+Search the **synchronous product**: states are ``(marking, i)`` where ``i``
+is how many events have been explained so far.  From each state:
+
+* a synchronous move fires a transition labelled ``trace[i]`` and moves to
+  ``i + 1``;
+* a log move stays in the marking and moves to ``i + 1``;
+* a model move fires any enabled transition and stays at ``i``.
+
+The goal is ``(final marking, len(trace))``.  Costs are non-negative, so
+Dijkstra/A* returns an optimal path.  As in PM4Py, costs are scaled: a real
+deviation costs 10000 and a τ move costs 1, so that among equally good
+alignments the one with fewest τ moves wins; the reported cost is the
+integer division by 10000.
+
+Heuristic: every remaining event whose label does not occur in the model
+*must* become a log move, so their count is a lower bound on the remaining
+cost -- admissible, so A* stays optimal.
+
+Fitness
+-------
+``fitness(σ) = 1 − cost(σ) / (|σ| + cost(⟨⟩))`` where ``cost(⟨⟩)`` is the
+cost of aligning the empty trace -- the length of the shortest visible run of
+the model.  The denominator is the worst case: every event a log move plus a
+shortest run of model moves.
+"""
+
+from __future__ import annotations
+
+import heapq
+import itertools
+from dataclasses import dataclass, field
+
+from ..log import SimpleLog
+from ..petrinet import Marking, PetriNet
+
+SKIP = "≫"
+DEVIATION = 10_000
+TAU_COST = 1
+
+
+@dataclass
+class Move:
+    log: str               # activity, or SKIP
+    model: str             # label, "τ", or SKIP
+    transition: str | None  # transition id for model/sync moves
+
+    @property
+    def kind(self) -> str:
+        if self.log == SKIP:
+            return "silent" if self.model == "τ" else "model"
+        if self.model == SKIP:
+            return "log"
+        return "sync"
+
+
+@dataclass
+class Alignment:
+    trace: tuple[str, ...]
+    count: int
+    moves: list[Move]
+    cost: int                  # number of deviations (standard cost function)
+    fitness: float
+    states_visited: int = 0
+    #: False if the search gave up (state limit); then moves are empty
+    complete: bool = True
+
+    @property
+    def log_moves(self) -> int:
+        return sum(1 for m in self.moves if m.kind == "log")
+
+    @property
+    def model_moves(self) -> int:
+        return sum(1 for m in self.moves if m.kind == "model")
+
+
+@dataclass
+class AlignmentResult:
+    net: PetriNet
+    alignments: list[Alignment] = field(default_factory=list)   # per variant
+    empty_trace_cost: int = 0
+
+    @property
+    def trace_count(self) -> int:
+        return sum(a.count for a in self.alignments)
+
+    @property
+    def average_fitness(self) -> float:
+        """Mean of the per-trace fitness values (weighted by frequency)."""
+        total = self.trace_count
+        return sum(a.fitness * a.count for a in self.alignments) / total if total else 1.0
+
+    @property
+    def log_fitness(self) -> float:
+        """1 − total cost / total worst-case cost."""
+        cost = sum(a.cost * a.count for a in self.alignments)
+        worst = sum((len(a.trace) + self.empty_trace_cost) * a.count for a in self.alignments)
+        return 1 - cost / worst if worst else 1.0
+
+    @property
+    def fitting_traces(self) -> int:
+        return sum(a.count for a in self.alignments if a.cost == 0 and a.complete)
+
+    def move_statistics(self) -> tuple[dict[str, int], dict[str, int], dict[str, int]]:
+        """(sync, log-move, model-move) counts per activity, frequency-weighted."""
+        sync: dict[str, int] = {}
+        log_moves: dict[str, int] = {}
+        model_moves: dict[str, int] = {}
+        for alignment in self.alignments:
+            for move in alignment.moves:
+                if move.kind == "sync":
+                    sync[move.log] = sync.get(move.log, 0) + alignment.count
+                elif move.kind == "log":
+                    log_moves[move.log] = log_moves.get(move.log, 0) + alignment.count
+                elif move.kind == "model":
+                    model_moves[move.model] = model_moves.get(move.model, 0) + alignment.count
+        return sync, log_moves, model_moves
+
+
+class _CompiledNet:
+    """The net flattened into integer arrays for a fast inner loop.
+
+    A marking becomes a tuple of token counts indexed by place number, and
+    each transition becomes its input requirements plus its effect vector.
+    Tuples hash and compare much faster than :class:`Marking` objects, which
+    matters because an alignment search can visit hundreds of thousands of
+    states.
+    """
+
+    def __init__(self, net: PetriNet) -> None:
+        self.places = list(net.places)
+        index = {p: i for i, p in enumerate(self.places)}
+        self.transitions = list(net.transitions.values())
+        self.pre = [tuple((index[p], n) for p, n in net.pre(t.id).items())
+                    for t in self.transitions]
+        self.delta = []
+        for t in self.transitions:
+            change = [0] * len(self.places)
+            for p, n in net.pre(t.id).items():
+                change[index[p]] -= n
+            for p, n in net.post(t.id).items():
+                change[index[p]] += n
+            self.delta.append(tuple((i, d) for i, d in enumerate(change) if d))
+        self.initial = self.encode(net.initial_marking, index)
+        self.final = self.encode(net.final_marking, index)
+
+    def encode(self, marking: Marking, index: dict[str, int]) -> tuple[int, ...]:
+        vector = [0] * len(self.places)
+        for p, n in marking.items():
+            vector[index[p]] = n
+        return tuple(vector)
+
+    def enabled(self, marking: tuple[int, ...]) -> list[int]:
+        return [k for k, pre in enumerate(self.pre)
+                if all(marking[i] >= n for i, n in pre)]
+
+    def fire(self, marking: tuple[int, ...], k: int) -> tuple[int, ...]:
+        vector = list(marking)
+        for i, d in self.delta[k]:
+            vector[i] += d
+        return tuple(vector)
+
+
+def align_trace(net: PetriNet, trace: tuple[str, ...], count: int = 1,
+                max_states: int = 500_000, compiled: "_CompiledNet | None" = None) -> Alignment:
+    """An optimal alignment of one trace (A* on the synchronous product)."""
+    c = compiled or _CompiledNet(net)
+    labels = net.labels()
+    # Suffix heuristic: events not in the model are unavoidable log moves.
+    unavoidable = [0] * (len(trace) + 1)
+    for i in range(len(trace) - 1, -1, -1):
+        unavoidable[i] = unavoidable[i + 1] + (DEVIATION if trace[i] not in labels else 0)
+
+    start = (c.initial, 0)
+    tie = itertools.count()
+    frontier: list = [(unavoidable[0], 0, next(tie), start)]
+    best_cost: dict = {start: 0}
+    parent: dict = {start: None}
+    closed: set = set()
+    goal = (c.final, len(trace))
+
+    while frontier:
+        _, cost, _, state = heapq.heappop(frontier)
+        if state in closed:
+            continue
+        closed.add(state)
+        if state == goal:
+            moves = []
+            node = state
+            while parent[node] is not None:
+                previous, move = parent[node]
+                moves.append(move)
+                node = previous
+            return Alignment(trace, count, moves[::-1], cost // DEVIATION, 0.0, len(closed))
+        if len(closed) > max_states:
+            break
+
+        marking, position = state
+        enabled = c.enabled(marking)
+        successors = []
+        if position < len(trace):
+            activity = trace[position]
+            successors.append(((marking, position + 1), DEVIATION, Move(activity, SKIP, None)))
+            for k in enabled:
+                t = c.transitions[k]
+                if t.label == activity:
+                    successors.append(((c.fire(marking, k), position + 1), 0,
+                                       Move(activity, activity, t.id)))
+        for k in enabled:
+            t = c.transitions[k]
+            step_cost = TAU_COST if t.label is None else DEVIATION
+            successors.append(((c.fire(marking, k), position), step_cost,
+                               Move(SKIP, "τ" if t.label is None else t.label, t.id)))
+
+        for successor, step_cost, move in successors:
+            if successor in closed:
+                continue
+            new_cost = cost + step_cost
+            if new_cost < best_cost.get(successor, 1 << 62):
+                best_cost[successor] = new_cost
+                parent[successor] = (state, move)
+                heapq.heappush(frontier, (new_cost + unavoidable[successor[1]], new_cost,
+                                          next(tie), successor))
+
+    return Alignment(trace, count, [], len(trace), 0.0, len(closed), complete=False)
+
+
+def align_log(net: PetriNet, log: SimpleLog, max_states: int = 500_000,
+              progress=None) -> AlignmentResult:
+    """Align every variant; fitness per trace uses the empty-trace cost."""
+    compiled = _CompiledNet(net)
+    empty = align_trace(net, (), 1, max_states, compiled)
+    result = AlignmentResult(net, empty_trace_cost=empty.cost)
+    variants = sorted(log.items(), key=lambda item: (-item[1], item[0]))
+    for number, (trace, count) in enumerate(variants, 1):
+        alignment = align_trace(net, trace, count, max_states, compiled)
+        worst = len(trace) + empty.cost
+        alignment.fitness = 1 - alignment.cost / worst if worst else 1.0
+        if not alignment.complete:
+            alignment.fitness = 0.0
+        result.alignments.append(alignment)
+        if progress is not None:
+            progress(number, len(variants))
+    return result

openprocess/mining/conformance/quality.py

@@ -0,0 +1,145 @@
+"""Precision, generalisation and simplicity -- the other three quality dimensions.
+
+Fitness (token replay, alignments) asks "can the model replay the log?".
+Three more dimensions complete the picture (course: the four quality
+criteria):
+
+precision
+    Does the model allow *only* what the log shows?  A flower model fits
+    everything but has terrible precision.
+generalisation
+    Does the model generalise beyond the examples, rather than overfitting
+    exactly the traces seen?
+simplicity
+    Is the model as simple as possible (Occam's razor)?
+
+Precision: escaping edges (ETConformance, Muñoz-Gama & Carmona)
+---------------------------------------------------------------
+Walk through every *prefix* of every trace.  After prefix ``h``, the log
+continued with some set of activities ``log_next(h)``; the model, in the
+marking reached by replaying ``h``, would *allow* a set ``model_next(h)``.
+Anything the model allows but the log never did is an **escaping edge**::
+
+                     Σ_h  freq(h) · |model_next(h) \\ log_next(h)|
+    precision = 1 −  ─────────────────────────────────────────────
+                     Σ_h  freq(h) · |model_next(h)|
+
+Prefixes that do not fit the model are skipped (there is no meaningful
+marking to compare against).  ``model_next`` looks through τ transitions.
+
+Generalisation (token-based, as in PM4Py)
+-----------------------------------------
+A transition that fired only a few times during replay gives little evidence
+that the model is right about it.  With ``n_t`` the number of firings of
+transition t::
+
+    generalisation = 1 − mean_t ( 1 / sqrt(n_t) )
+
+Simplicity (arc degree, as in PM4Py)
+------------------------------------
+The mean number of arcs per node; a pure sequence has degree 2.  Anything
+above that makes the model harder to read::
+
+    simplicity = 1 / (1 + max(mean degree − 2, 0))
+
+These last two are rough proxies -- which is also what the literature says
+about them -- but they make the trade-off between the four dimensions
+visible when comparing models discovered from the same log.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import Counter, defaultdict, deque
+
+from ..log import SimpleLog
+from ..petrinet import Marking, PetriNet
+from .token_replay import ReplayResult, silent_path
+
+
+def _visible_enabled(net: PetriNet, marking: Marking, max_states: int = 2000) -> set[str]:
+    """Labels that can fire next, possibly after τ moves."""
+    result: set[str] = set()
+    seen = {marking}
+    queue = deque([marking])
+    while queue and len(seen) < max_states:
+        current = queue.popleft()
+        for transition in net.enabled(current):
+            label = net.transitions[transition].label
+            if label is not None:
+                result.add(label)
+            else:
+                successor = net.fire(current, transition)
+                if successor not in seen:
+                    seen.add(successor)
+                    queue.append(successor)
+    return result
+
+
+def _replay_prefix(net: PetriNet, prefix: tuple[str, ...],
+                   cache: dict[tuple[str, ...], Marking | None]) -> Marking | None:
+    """Marking after replaying ``prefix`` without missing tokens, or None."""
+    if prefix in cache:
+        return cache[prefix]
+    if not prefix:
+        cache[prefix] = net.initial_marking
+        return net.initial_marking
+    before = _replay_prefix(net, prefix[:-1], cache)
+    result: Marking | None = None
+    if before is not None:
+        activity = prefix[-1]
+        candidates = net.transitions_with_label(activity)
+        for transition in candidates:
+            if net.is_enabled(before, transition):
+                result = net.fire(before, transition)
+                break
+        else:
+            for transition in candidates:
+                path = silent_path(net, before, lambda m, t=transition: net.is_enabled(m, t))
+                if path is not None:
+                    marking = before
+                    for step in path:
+                        marking = net.fire(marking, step)
+                    result = net.fire(marking, transition)
+                    break
+    cache[prefix] = result
+    return result
+
+
+def precision(net: PetriNet, log: SimpleLog) -> float:
+    """ETConformance precision with escaping edges."""
+    log_next: dict[tuple[str, ...], set[str]] = defaultdict(set)
+    frequency: Counter = Counter()
+    for trace, count in log.items():
+        for i, activity in enumerate(trace):
+            prefix = trace[:i]
+            log_next[prefix].add(activity)
+            frequency[prefix] += count
+
+    cache: dict[tuple[str, ...], Marking | None] = {}
+    escaping = allowed = 0
+    for prefix, observed in log_next.items():
+        marking = _replay_prefix(net, prefix, cache)
+        if marking is None:
+            continue
+        enabled = _visible_enabled(net, marking)
+        allowed += frequency[prefix] * len(enabled)
+        escaping += frequency[prefix] * len(enabled - observed)
+    return 1.0 if allowed == 0 else 1 - escaping / allowed
+
+
+def generalisation(replay: ReplayResult) -> float:
+    fired, _ = replay.transition_totals()
+    net = replay.net
+    if not net.transitions:
+        return 1.0
+    terms = [1 / math.sqrt(fired[t]) if fired[t] else 1.0 for t in net.transitions]
+    return 1 - sum(terms) / len(terms)
+
+
+def simplicity(net: PetriNet) -> float:
+    nodes = len(net.places) + len(net.transitions)
+    if nodes == 0:
+        return 1.0
+    mean_degree = 2 * len(net.arcs) / nodes
+    return 1 / (1 + max(mean_degree - 2, 0))

openprocess/mining/conformance/token_replay.py

@@ -0,0 +1,252 @@
+"""Token-based replay (van der Aalst, *Process Mining*, §8.2).
+
+Replay each trace on the net and *count tokens*:
+
+``p`` produced
+    tokens put into places (the environment first produces the initial
+    marking, then every fired transition produces its output tokens);
+``c`` consumed
+    tokens taken out (every fired transition consumes its inputs, and at the
+    end the environment consumes the final marking);
+``m`` missing
+    tokens that had to be *created artificially* because the transition for
+    the next event was not enabled -- evidence the log did something the
+    model did not allow;
+``r`` remaining
+    tokens left behind after the final marking was consumed -- evidence the
+    model expected something the log did not do.
+
+Fitness of a trace σ on net N::
+
+    fitness(σ, N) = ½ (1 − m / c) + ½ (1 − r / p)
+
+It is 1 exactly when nothing was missing or remaining.  For a whole log the
+four counts are summed over all traces (weighted by frequency) before the
+formula is applied.
+
+Silent transitions and duplicates
+---------------------------------
+Events never refer to τ transitions, so the replayer may fire them on its
+own: when the transition for the next event is not enabled, it first looks
+for a (shortest) sequence of silent transitions that enables it.  With
+duplicate labels, it prefers a transition that is enabled, then one reachable
+through silent moves, then the one missing the fewest tokens.  These are
+heuristics -- token replay is fast but not always optimal, which is why
+alignments exist (see :mod:`alignments`).
+
+Diagnostics
+-----------
+Per place we keep the total missing and remaining tokens, and per transition
+how often it fired with missing tokens.  The app paints these on the model
+(ProM's "replay" view): a red ``−n`` badge means the log wanted a token here
+that was not there; a ``+n`` badge means a token was left behind.
+"""
+
+from __future__ import annotations
+
+from collections import Counter, deque
+from dataclasses import dataclass, field
+
+from ..log import SimpleLog
+from ..petrinet import Marking, PetriNet
+
+
+@dataclass
+class TraceReplay:
+    trace: tuple[str, ...]
+    count: int
+    produced: int
+    consumed: int
+    missing: int
+    remaining: int
+    #: activities that do not occur in the model at all
+    unknown_activities: list[str] = field(default_factory=list)
+    #: indices of events whose transition was not enabled
+    problem_events: list[int] = field(default_factory=list)
+    missing_by_place: Counter = field(default_factory=Counter)
+    remaining_by_place: Counter = field(default_factory=Counter)
+    fired: Counter = field(default_factory=Counter)             # transition id -> times
+    forced: Counter = field(default_factory=Counter)            # fired with missing tokens
+
+    @property
+    def fitness(self) -> float:
+        return _fitness(self.produced, self.consumed, self.missing, self.remaining)
+
+    @property
+    def fits(self) -> bool:
+        return self.missing == 0 and self.remaining == 0 and not self.unknown_activities
+
+
+def _fitness(p: int, c: int, m: int, r: int) -> float:
+    part_missing = 1 - m / c if c else 1.0
+    part_remaining = 1 - r / p if p else 1.0
+    return 0.5 * part_missing + 0.5 * part_remaining
+
+
+@dataclass
+class ReplayResult:
+    net: PetriNet
+    traces: list[TraceReplay]           # one per variant, most frequent first
+
+    def _total(self, attribute: str) -> int:
+        return sum(getattr(t, attribute) * t.count for t in self.traces)
+
+    @property
+    def produced(self) -> int:
+        return self._total("produced")
+
+    @property
+    def consumed(self) -> int:
+        return self._total("consumed")
+
+    @property
+    def missing(self) -> int:
+        return self._total("missing")
+
+    @property
+    def remaining(self) -> int:
+        return self._total("remaining")
+
+    @property
+    def fitness(self) -> float:
+        """Log-level fitness from the summed counts."""
+        return _fitness(self.produced, self.consumed, self.missing, self.remaining)
+
+    @property
+    def fitting_traces(self) -> int:
+        return sum(t.count for t in self.traces if t.fits)
+
+    @property
+    def trace_count(self) -> int:
+        return sum(t.count for t in self.traces)
+
+    def place_totals(self) -> tuple[Counter, Counter]:
+        """(missing, remaining) per place, frequency-weighted."""
+        missing, remaining = Counter(), Counter()
+        for t in self.traces:
+            for place, n in t.missing_by_place.items():
+                missing[place] += n * t.count
+            for place, n in t.remaining_by_place.items():
+                remaining[place] += n * t.count
+        return missing, remaining
+
+    def transition_totals(self) -> tuple[Counter, Counter]:
+        """(fired, forced) per transition, frequency-weighted."""
+        fired, forced = Counter(), Counter()
+        for t in self.traces:
+            for transition, n in t.fired.items():
+                fired[transition] += n * t.count
+            for transition, n in t.forced.items():
+                forced[transition] += n * t.count
+        return fired, forced
+
+
+# ---------------------------------------------------------------------------
+# Silent moves
+# ---------------------------------------------------------------------------
+def silent_path(net: PetriNet, marking: Marking, goal, max_states: int = 2000) -> list[str] | None:
+    """Shortest sequence of τ transitions from ``marking`` to a marking where
+    ``goal(marking)`` holds.  ``[]`` if it already holds, ``None`` if none found."""
+    if goal(marking):
+        return []
+    silent = [t for t, tr in net.transitions.items() if tr.silent]
+    if not silent:
+        return None
+    parent: dict[Marking, tuple[Marking, str] | None] = {marking: None}
+    queue = deque([marking])
+    while queue and len(parent) < max_states:
+        current = queue.popleft()
+        for transition in silent:
+            if not net.is_enabled(current, transition):
+                continue
+            successor = net.fire(current, transition)
+            if successor in parent:
+                continue
+            parent[successor] = (current, transition)
+            if goal(successor):
+                path = []
+                node = successor
+                while parent[node] is not None:
+                    previous, step = parent[node]  # type: ignore[misc]
+                    path.append(step)
+                    node = previous
+                return path[::-1]
+            queue.append(successor)
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Replay
+# ---------------------------------------------------------------------------
+def replay_trace(net: PetriNet, trace: tuple[str, ...], count: int = 1) -> TraceReplay:
+    marking = net.initial_marking
+    result = TraceReplay(trace, count, produced=marking.total, consumed=0, missing=0,
+                         remaining=0)
+    by_label: dict[str, list[str]] = {}
+    for transition in net.transitions.values():
+        if transition.label is not None:
+            by_label.setdefault(transition.label, []).append(transition.id)
+
+    def fire(transition: str, current: Marking) -> Marking:
+        pre, post = net.pre(transition), net.post(transition)
+        short = Marking({p: n - current[p] for p, n in pre.items() if current[p] < n})
+        if short:
+            result.missing += short.total
+            result.missing_by_place.update(dict(short.items()))
+            result.forced[transition] += 1
+            current = current + short
+        result.consumed += pre.total
+        result.produced += post.total
+        result.fired[transition] += 1
+        return current - pre + post
+
+    for index, activity in enumerate(trace):
+        candidates = by_label.get(activity)
+        if not candidates:
+            result.unknown_activities.append(activity)
+            result.problem_events.append(index)
+            continue
+        chosen: str | None = next((t for t in candidates if net.is_enabled(marking, t)), None)
+        if chosen is None:
+            # Try enabling one of the candidates with silent moves.
+            best_path = None
+            for transition in candidates:
+                path = silent_path(net, marking, lambda m, t=transition: net.is_enabled(m, t))
+                if path is not None and (best_path is None or len(path) < len(best_path[1])):
+                    best_path = (transition, path)
+            if best_path is not None:
+                chosen, path = best_path
+                for step in path:
+                    marking = fire(step, marking)
+        if chosen is None:
+            # Force the transition missing the fewest tokens.
+            chosen = min(candidates, key=lambda t: sum(
+                max(0, n - marking[p]) for p, n in net.pre(t).items()))
+            result.problem_events.append(index)
+        marking = fire(chosen, marking)
+
+    # Finish: move silently to the final marking if possible, then let the
+    # environment consume it.
+    final = net.final_marking
+    path = silent_path(net, marking, lambda m: m == final)
+    if path is None:
+        path = silent_path(net, marking, lambda m: m >= final) or []
+    for step in path:
+        marking = fire(step, marking)
+    short = Marking({p: n - marking[p] for p, n in final.items() if marking[p] < n})
+    if short:
+        result.missing += short.total
+        result.missing_by_place.update(dict(short.items()))
+        marking = marking + short
+    result.consumed += final.total
+    leftover = marking - final
+    result.remaining = leftover.total
+    result.remaining_by_place.update(dict(leftover.items()))
+    return result
+
+
+def token_replay(net: PetriNet, log: SimpleLog) -> ReplayResult:
+    """Replay every variant once and weight the results by frequency."""
+    traces = [replay_trace(net, trace, count)
+              for trace, count in sorted(log.items(), key=lambda item: (-item[1], item[0]))]
+    return ReplayResult(net, traces)