openprocess 0.7.0__py3-none-any.whl

openprocess/mining/columns.py

@@ -0,0 +1,282 @@
+"""A columnar store for large logs: one array per attribute, no objects.
+
+A million-event log read as :class:`~.log.Trace` and :class:`~.log.Event`
+objects costs a million dicts.  The :class:`ColumnStore` keeps one array
+per attribute instead -- activities, resources and lifecycles as codes into
+a table of strings, timestamps as seconds since the epoch -- and the
+events of a case are a slice ``offsets[i]:offsets[i+1]``.
+
+:class:`LazyTraces` makes a store look like the list of traces every view
+in the app reads: ``log.traces[i]`` builds the ``Trace`` with its events on
+first access and keeps it; ``len``, iteration and slicing work; and the
+first *edit* (append, insert, delete) materialises everything, so the
+editor keeps working.  The things discovery needs most -- the activity
+sequences, the directly-follows counts, the event count -- are read from
+the arrays without building any object (:meth:`LazyTraces.sequences`).
+"""
+
+from __future__ import annotations
+
+from array import array
+from collections.abc import MutableSequence
+from datetime import datetime, timedelta, timezone
+from typing import Any, Iterator
+
+#: Standard keys kept in typed arrays; everything else goes to ``extra``.
+KEY_NAME = "concept:name"
+KEY_TIME = "time:timestamp"
+KEY_LIFECYCLE = "lifecycle:transition"
+KEY_RESOURCE = "org:resource"
+
+
+class Codes:
+    """Strings as small integers, with the table to read them back."""
+
+    __slots__ = ("values", "_index", "codes")
+
+    def __init__(self) -> None:
+        self.values: list[str] = []
+        self._index: dict[str, int] = {}
+        self.codes = array("i")
+
+    def add(self, value: str | None) -> None:
+        if value is None:
+            self.codes.append(-1)
+            return
+        code = self._index.get(value)
+        if code is None:
+            code = self._index[value] = len(self.values)
+            self.values.append(value)
+        self.codes.append(code)
+
+    def get(self, index: int) -> str | None:
+        code = self.codes[index]
+        return None if code < 0 else self.values[code]
+
+    def __len__(self) -> int:
+        return len(self.codes)
+
+
+class ColumnStore:
+    """The columns of a log.  Fill it event by event, case by case."""
+
+    def __init__(self) -> None:
+        self.case_ids: list[str] = []
+        self.case_attributes: list[dict[str, Any]] = []
+        self.offsets = array("q", [0])
+        self.activity = Codes()
+        self.lifecycle = Codes()
+        self.resource = Codes()
+        self.micros = array("q")          # microseconds since the epoch; NO_TIME when absent
+        self.offset_minutes = array("i")  # the timestamp's UTC offset
+        #: Other event attributes: key -> list of values (None where absent).
+        self.extra: dict[str, list[Any]] = {}
+        #: Event attribute keys in the order they first appeared, so a trace
+        #: built from the columns writes its attributes in the file's order.
+        self.key_order: list[str] = []
+        self._known: set[str] = set()
+
+    # -- filling -------------------------------------------------------------------
+    def add_event(self, attributes: dict[str, Any]) -> None:
+        """One event from its attribute dict (the reader has a faster path)."""
+        self.add(attributes.get(KEY_NAME), attributes.get(KEY_TIME), attributes.get(KEY_LIFECYCLE),
+                 attributes.get(KEY_RESOURCE),
+                 [(k, v) for k, v in attributes.items() if k not in _STANDARD])
+
+    def add(self, activity, stamp, lifecycle, resource, extras: list[tuple[str, Any]],
+            order: list[str] | None = None) -> None:
+        if order is not None:
+            for key in order:
+                if key not in self._known:
+                    self._known.add(key)
+                    self.key_order.append(key)
+        self.activity.add(_text(activity))
+        self.lifecycle.add(_text(lifecycle))
+        self.resource.add(_text(resource))
+        if isinstance(stamp, datetime):
+            self.micros.append(_micros(stamp))
+            offset = stamp.utcoffset()
+            self.offset_minutes.append(int(offset.total_seconds() // 60) if offset is not None else 0)
+        else:
+            self.micros.append(NO_TIME)
+            self.offset_minutes.append(0)
+        position = len(self.micros) - 1
+        if extras:
+            for key, value in extras:
+                column = self.extra.get(key)
+                if column is None:
+                    column = self.extra[key] = [None] * position
+                column.append(value)
+        if len(self.extra) != len(extras):
+            for column in self.extra.values():
+                if len(column) <= position:
+                    column.append(None)
+
+    def end_case(self, attributes: dict[str, Any]) -> None:
+        self.case_attributes.append(attributes)
+        self.case_ids.append(_text(attributes.get(KEY_NAME)) or "")
+        self.offsets.append(len(self.micros))
+
+    # -- reading -------------------------------------------------------------------
+    @property
+    def case_count(self) -> int:
+        return len(self.case_ids)
+
+    @property
+    def event_count(self) -> int:
+        return len(self.micros)
+
+    def timestamp(self, index: int) -> datetime | None:
+        micros = self.micros[index]
+        if micros == NO_TIME:
+            return None
+        zone = timezone(timedelta(minutes=self.offset_minutes[index]))
+        return _EPOCH.astimezone(zone) + timedelta(microseconds=micros)
+
+    def event_attributes(self, index: int) -> dict[str, Any]:
+        """The attributes of one event, as the object reader would give them."""
+        attributes: dict[str, Any] = {}
+        activity = self.activity.get(index)
+        if activity is not None:
+            attributes[KEY_NAME] = activity
+        stamp = self.timestamp(index)
+        if stamp is not None:
+            attributes[KEY_TIME] = stamp
+        lifecycle = self.lifecycle.get(index)
+        if lifecycle is not None:
+            attributes[KEY_LIFECYCLE] = lifecycle
+        resource = self.resource.get(index)
+        if resource is not None:
+            attributes[KEY_RESOURCE] = resource
+        for key, column in self.extra.items():
+            value = column[index]
+            if value is not None:
+                attributes[key] = value
+        if self.key_order:
+            ordered = {key: attributes[key] for key in self.key_order if key in attributes}
+            ordered.update({k: v for k, v in attributes.items() if k not in ordered})
+            return ordered
+        return attributes
+
+    def trace(self, case: int):
+        from .log import Event, Trace
+        start, end = self.offsets[case], self.offsets[case + 1]
+        return Trace(dict(self.case_attributes[case]),
+                     [Event(self.event_attributes(i)) for i in range(start, end)])
+
+    def has_lifecycle_pairs(self) -> bool:
+        seen = {v.lower() for v in self.lifecycle.values}
+        return {"start", "complete"} <= seen
+
+    def sequences(self, keys: tuple[str, ...], lifecycles: frozenset[str] | None) -> list[tuple[str, ...]]:
+        """Activity sequences under a classifier, straight from the arrays."""
+        activity, lifecycle, resource = self.activity, self.lifecycle, self.resource
+        columns = []
+        for key in keys:
+            if key == KEY_NAME:
+                columns.append((activity.codes, activity.values))
+            elif key == KEY_LIFECYCLE:
+                columns.append((lifecycle.codes, lifecycle.values))
+            elif key == KEY_RESOURCE:
+                columns.append((resource.codes, resource.values))
+            else:
+                extra = self.extra.get(key)
+                columns.append((None, extra))
+        wanted = None if lifecycles is None else {v.lower() for v in lifecycles}
+        out = []
+        offsets = self.offsets
+        for case in range(len(self.case_ids)):
+            labels = []
+            for i in range(offsets[case], offsets[case + 1]):
+                if wanted is not None:
+                    code = lifecycle.codes[i]
+                    if code >= 0 and lifecycle.values[code].lower() not in wanted:
+                        continue
+                parts = []
+                for codes, values in columns:
+                    if codes is None:
+                        value = values[i] if values is not None else None
+                        parts.append("" if value is None else str(value))
+                    else:
+                        code = codes[i]
+                        parts.append("" if code < 0 else values[code])
+                labels.append("+".join(parts))
+            out.append(tuple(labels))
+        return out
+
+
+_STANDARD = frozenset((KEY_NAME, KEY_TIME, KEY_LIFECYCLE, KEY_RESOURCE))
+NO_TIME = -(1 << 62)
+_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
+
+
+def _micros(stamp: datetime) -> int:
+    delta = stamp - _EPOCH
+    return (delta.days * 86400 + delta.seconds) * 1_000_000 + delta.microseconds
+
+
+def _text(value) -> str | None:
+    return None if value is None else str(value)
+
+
+class LazyTraces(MutableSequence):
+    """The traces of a log, built from a :class:`ColumnStore` on demand."""
+
+    def __init__(self, store: ColumnStore) -> None:
+        self.store = store
+        self._built: list = [None] * store.case_count
+        self._list: list | None = None          # set once the log was edited
+
+    # -- the fast paths -------------------------------------------------------------
+    @property
+    def modified(self) -> bool:
+        return self._list is not None
+
+    @property
+    def event_count(self) -> int:
+        if self._list is not None:
+            return sum(len(t) for t in self._list)
+        return self.store.event_count
+
+    def sequences(self, keys: tuple[str, ...], lifecycles: frozenset[str] | None):
+        return self.store.sequences(keys, lifecycles) if self._list is None else None
+
+    def materialise(self) -> list:
+        """Every trace as an object; from then on this is a plain list."""
+        if self._list is None:
+            self._list = [self[i] for i in range(len(self))]
+            self._built = []
+        return self._list
+
+    # -- the sequence protocol ------------------------------------------------------
+    def __len__(self) -> int:
+        return len(self._list) if self._list is not None else self.store.case_count
+
+    def __getitem__(self, index):
+        if self._list is not None:
+            return self._list[index]
+        if isinstance(index, slice):
+            return [self[i] for i in range(*index.indices(len(self)))]
+        if index < 0:
+            index += len(self)
+        trace = self._built[index]
+        if trace is None:
+            trace = self._built[index] = self.store.trace(index)
+        return trace
+
+    def __iter__(self) -> Iterator:
+        if self._list is not None:
+            return iter(self._list)
+        return (self[i] for i in range(len(self)))
+
+    def __setitem__(self, index, value) -> None:
+        self.materialise()[index] = value
+
+    def __delitem__(self, index) -> None:
+        del self.materialise()[index]
+
+    def insert(self, index: int, value) -> None:
+        self.materialise().insert(index, value)
+
+    def __repr__(self) -> str:
+        return f"<LazyTraces {len(self)} cases, {self.event_count} events>"

openprocess/mining/compare_nets.py

@@ -0,0 +1,246 @@
+"""Compare two Petri nets on behaviour, not on how they are drawn.
+
+Two nets *behave the same* when they have the same **complete traces**:
+the sequences of visible labels of the firing sequences that lead from the
+initial marking to the final marking.  Silent (τ) transitions leave no
+trace, and transitions are matched by label, so a net laid out differently
+from another -- or with other place names, or an extra τ -- still matches.
+
+How
+---
+Each net is turned into an automaton over labels: its states are markings,
+τ-steps are free moves, and a state accepts when it is the final marking.
+Removing the τ-steps and making the automaton deterministic (the subset
+construction) gives, for each sequence of labels, the set of markings it can
+lead to.  Both automata are explored together, breadth-first, so the first
+sequence found that one accepts and the other does not is a **shortest
+differing trace**.
+
+* For **bounded** nets both reachability graphs are finite, so this is
+  exact: when nothing differs, the languages are equal.
+* For **unbounded** nets (or ones too large to explore) only traces up to a
+  length limit are compared, and the result says so.
+
+When does a net *end*?  At its final marking if it has one; for a WF-net
+without one, at one token in the sink; otherwise in any marking where
+nothing is enabled.  A net with no tokens that is a WF-net starts from one
+token in its source.
+
+Labels are compared after trimming spaces and ignoring case; a mapping can
+rename one net's labels to the other's (``register`` → ``Register request``).
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass, field
+
+from .analysis import check_workflow_net, reachability_graph
+from .petrinet import Marking, PetriNet
+
+#: How many differing traces to collect each way.
+EXAMPLES = 3
+
+
+def normalise(label: str) -> str:
+    return " ".join(label.split()).casefold()
+
+
+@dataclass
+class NetComparison:
+    #: True when the complete traces are the same (up to the length limit
+    #: when :attr:`exact` is False).
+    equivalent: bool
+    #: True when the comparison covers every trace (both nets bounded).
+    exact: bool
+    #: Shortest traces the first net (yours) allows and the second does not.
+    only_first: list[tuple[str, ...]] = field(default_factory=list)
+    #: Shortest traces the second net (the answer) allows and the first does not.
+    only_second: list[tuple[str, ...]] = field(default_factory=list)
+    #: Traces compared up to this many labels (None when exact).
+    max_length: int | None = None
+    #: Visible labels that occur in only one of the nets (after normalising
+    #: and the mapping).
+    labels_only_first: list[str] = field(default_factory=list)
+    labels_only_second: list[str] = field(default_factory=list)
+    notes: list[str] = field(default_factory=list)
+
+    def summary(self, first: str = "yours", second: str = "the answer") -> str:
+        if self.equivalent:
+            scope = "" if self.exact else f" (traces up to {self.max_length} steps compared)"
+            return f"Same behaviour{scope}: every complete trace of one is a trace of the other."
+        return f"Differs: {first} and {second} do not allow the same complete traces."
+
+
+def _start_and_end(net: PetriNet) -> tuple[Marking, Marking | None, list[str]]:
+    """The initial marking, the final marking (None: any dead marking), and notes."""
+    notes = []
+    initial, final = net.initial_marking, net.final_marking or None
+    workflow = check_workflow_net(net)
+    if not initial and workflow.is_workflow_net:
+        initial = Marking({workflow.source: 1})
+        notes.append(f"{net.name} has no tokens: it starts from one token in its source place.")
+    if final is None and workflow.is_workflow_net:
+        final = Marking({workflow.sink: 1})
+    if final is None:
+        notes.append(f"{net.name} has no final marking: its traces end where nothing is enabled.")
+    return initial, final, notes
+
+
+class _Automaton:
+    """The net as a (lazily explored) automaton over normalised labels."""
+
+    def __init__(self, net: PetriNet, mapping: dict[str, str] | None = None,
+                 closure_limit: int = 5_000) -> None:
+        self.net = net
+        self.initial, self.final, self.notes = _start_and_end(net)
+        mapping = {normalise(k): normalise(v) for k, v in (mapping or {}).items()}
+        self.label: dict[str, str | None] = {}
+        self.spelling: dict[str, str] = {}
+        for transition in net.transitions.values():
+            if transition.label is None or not transition.label.strip():
+                self.label[transition.id] = None
+                continue
+            key = normalise(transition.label)
+            key = mapping.get(key, key)
+            self.label[transition.id] = key
+            self.spelling.setdefault(key, transition.label.strip())
+        self.closure_limit = closure_limit
+        self.overflow = False
+        self._closures: dict[Marking, frozenset[Marking]] = {}
+
+    def labels(self) -> set[str]:
+        return {label for label in self.label.values() if label is not None}
+
+    def closure(self, markings) -> frozenset[Marking]:
+        """Every marking reachable by τ-steps alone."""
+        seen = set(markings)
+        queue = deque(markings)
+        while queue:
+            marking = queue.popleft()
+            for transition in self.net.enabled(marking):
+                if self.label[transition] is None:
+                    following = self.net.fire(marking, transition)
+                    if following not in seen:
+                        if len(seen) >= self.closure_limit:
+                            self.overflow = True
+                            return frozenset(seen)
+                        seen.add(following)
+                        queue.append(following)
+        return frozenset(seen)
+
+    def start(self) -> frozenset[Marking]:
+        return self.closure([self.initial])
+
+    def step(self, state: frozenset[Marking], label: str) -> frozenset[Marking]:
+        following = []
+        for marking in state:
+            for transition in self.net.enabled(marking):
+                if self.label[transition] == label:
+                    following.append(self.net.fire(marking, transition))
+        return self.closure(following) if following else frozenset()
+
+    def accepts(self, state: frozenset[Marking]) -> bool:
+        if self.final is not None:
+            return self.final in state
+        return any(not self.net.enabled(m) for m in state)
+
+
+def _bounded(net: PetriNet, initial: Marking, max_states: int) -> bool:
+    graph = reachability_graph(net, initial, max_states=max_states)
+    return not graph.truncated and not graph.has_omega
+
+
+def compare_nets(first: PetriNet, second: PetriNet, mapping: dict[str, str] | None = None,
+                 max_states: int = 20_000, max_length: int = 12,
+                 max_pairs: int = 200_000) -> NetComparison:
+    """Compare the complete traces of ``first`` (yours) and ``second`` (the answer).
+
+    ``mapping`` renames labels of ``first`` to labels of ``second``.  Exact
+    when both nets are bounded with at most ``max_states`` reachable
+    markings; otherwise traces of up to ``max_length`` labels are compared.
+    """
+    one, two = _Automaton(first, mapping), _Automaton(second)
+    exact = _bounded(first, one.initial, max_states) and _bounded(second, two.initial,
+                                                                   max_states)
+    labels = sorted(one.labels() | two.labels())
+    result = NetComparison(True, exact, max_length=None if exact else max_length)
+    result.notes = one.notes + two.notes
+    result.labels_only_first = sorted(one.spelling[x] for x in one.labels() - two.labels())
+    result.labels_only_second = sorted(two.spelling[x] for x in two.labels() - one.labels())
+
+    def spell(trace: tuple[str, ...], mine: bool) -> tuple[str, ...]:
+        """Each net's traces in its own spelling."""
+        first, second = (one, two) if mine else (two, one)
+        return tuple(first.spelling.get(x) or second.spelling.get(x, x) for x in trace)
+
+    start = (one.start(), two.start())
+    seen = {start}
+    queue: deque[tuple[tuple[frozenset, frozenset], tuple[str, ...]]] = deque([(start, ())])
+    while queue:
+        (a, b), trace = queue.popleft()
+        accept_a, accept_b = one.accepts(a), two.accepts(b)
+        if accept_a and not accept_b and len(result.only_first) < EXAMPLES:
+            result.only_first.append(spell(trace, True))
+        if accept_b and not accept_a and len(result.only_second) < EXAMPLES:
+            result.only_second.append(spell(trace, False))
+        if len(result.only_first) >= EXAMPLES and len(result.only_second) >= EXAMPLES:
+            break
+        if not exact and len(trace) >= max_length:
+            continue
+        for label in labels:
+            following = (one.step(a, label), two.step(b, label))
+            if not following[0] and not following[1]:
+                continue
+            if following not in seen:
+                if len(seen) >= max_pairs:
+                    result.exact = False
+                    result.max_length = len(trace)
+                    result.notes.append("The comparison stopped early: the nets have too "
+                                        "many states.")
+                    queue.clear()
+                    break
+                seen.add(following)
+                queue.append((following, trace + (label,)))
+    if one.overflow or two.overflow:
+        result.exact = False
+        result.max_length = result.max_length or max_length
+        result.notes.append("Long runs of silent steps were cut short.")
+    result.equivalent = not result.only_first and not result.only_second
+    return result
+
+
+def replayable_prefix(net: PetriNet, trace, mapping: dict[str, str] | None = None
+                      ) -> tuple[list[str], int]:
+    """Transition ids that replay as much of ``trace`` (labels) on ``net`` as
+    possible, τ-steps included, and how many labels they cover."""
+    automaton = _Automaton(net, mapping)
+    wanted = [normalise(x) for x in trace]
+    # Breadth-first over (marking, labels done), remembering how we got there.
+    start = (automaton.initial, 0)
+    parent: dict[tuple[Marking, int], tuple[tuple[Marking, int], str] | None] = {start: None}
+    queue = deque([start])
+    best = start
+    while queue and len(parent) < 50_000:
+        marking, done = queue.popleft()
+        if done > best[1] or (done == len(wanted) and best[1] == done and
+                              automaton.final is not None and marking == automaton.final):
+            best = (marking, done)
+        for transition in net.enabled(marking):
+            label = automaton.label[transition]
+            if label is None:
+                node = (net.fire(marking, transition), done)
+            elif done < len(wanted) and label == wanted[done]:
+                node = (net.fire(marking, transition), done + 1)
+            else:
+                continue
+            if node not in parent:
+                parent[node] = ((marking, done), transition)
+                queue.append(node)
+    path: list[str] = []
+    node = best
+    while parent[node] is not None:
+        previous, transition = parent[node]   # type: ignore[misc]
+        path.append(transition)
+        node = previous
+    return path[::-1], best[1]