taskflow-meter 1.0.0__py3-none-any.whl

taskflow_meter/diff.py

@@ -0,0 +1,223 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Turn a pair of snapshots into the events that explain the difference.
+
+This is how the read-only producer works: poll the flow, compare against
+what was seen last time, and synthesise exactly the events the in-process
+listener would have emitted.  One event vocabulary, two producers.
+
+What it cannot recover is ordering *within* a pair of snapshots.  If two
+atoms changed between polls, we did not see which moved first, so the rules
+below are chosen to be deterministic and defensible rather than to guess:
+
+* atoms are visited in name order, so replaying the same pair of snapshots
+  always produces the same events in the same order;
+* an atom's state event precedes its progress event, because a state change
+  is the coarser fact;
+* the flow's own event comes first when the flow is starting or continuing,
+  and last when the flow has reached a finish state -- a flow does not
+  finish before its atoms do.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Iterator
+from dataclasses import replace
+
+from taskflow_meter.events import Event
+from taskflow_meter.events import EventKind
+from taskflow_meter.events import SequenceAllocator
+from taskflow_meter.models import AtomSnapshot
+from taskflow_meter.models import FlowSnapshot
+
+LOG = logging.getLogger(__name__)
+
+_UNSTAMPED_SEQ = 0
+_UNSTAMPED_TS = 0.0
+
+
+def diff_flow(
+    old: FlowSnapshot | None,
+    new: FlowSnapshot,
+    *,
+    allocator: SequenceAllocator,
+    ts: float | None = None,
+) -> list[Event]:
+    """Return the events describing ``old`` -> ``new``.
+
+    ``old`` is ``None`` for a run's first observation, which yields one
+    event per known fact so that a client joining late still receives a
+    complete picture rather than only future changes.
+
+    Sequence numbers come from ``allocator`` in emission order.  Nothing is
+    allocated when there is nothing to report, so a quiet poll costs no
+    sequence numbers and a client's ``since_seq`` stays meaningful.
+    """
+    if old is not None and old.run_id != new.run_id:
+        msg = f"cannot diff across runs: {old.run_id!r} -> {new.run_id!r}"
+        raise ValueError(msg)
+
+    stamp = new.observed_at if ts is None else ts
+
+    flow_events = list(_flow_events(old, new))
+    atom_events = list(_atom_events(old, new))
+
+    ordered = (
+        atom_events + flow_events
+        if new.is_finished
+        else flow_events + atom_events
+    )
+    return [
+        replace(event, seq=allocator.allocate(new.run_id), ts=stamp)
+        for event in ordered
+    ]
+
+
+def _flow_events(
+    old: FlowSnapshot | None, new: FlowSnapshot
+) -> Iterator[Event]:
+    if old is not None and old.state == new.state:
+        return
+
+    details: dict[str, object] = {}
+    if old is None:
+        # First sight of this run: carry the identity that later events
+        # will not repeat, so a datasource can reconstruct the flow from
+        # the event stream alone.
+        details["flow_name"] = new.name
+        if new.book_name is not None:
+            details["book_name"] = new.book_name
+
+    yield _new_event(
+        new,
+        kind=EventKind.FLOW_STATE,
+        state=new.state,
+        old_state=old.state if old is not None else None,
+        details=details,
+    )
+
+
+def _atom_events(
+    old: FlowSnapshot | None, new: FlowSnapshot
+) -> Iterator[Event]:
+    previous = old.atoms if old is not None else {}
+
+    for name in new.atom_names:
+        current = new.atoms[name]
+        before = previous.get(name)
+
+        if before is None or before.state != current.state:
+            yield _atom_state_event(new, before, current)
+
+        if _progress_changed(before, current):
+            yield _atom_progress_event(new, current)
+
+    for name in sorted(set(previous) - set(new.atoms)):
+        # Atoms do not leave a flow while it runs.  If one vanishes the
+        # snapshot source is lying to us; say so rather than emit an event
+        # kind that means nothing downstream.
+        LOG.warning(
+            "atom %r disappeared from run %s between observations",
+            name,
+            new.run_id,
+        )
+
+
+def _progress_changed(
+    before: AtomSnapshot | None, current: AtomSnapshot
+) -> bool:
+    if before is None:
+        # Only worth an event if there is something to say; a brand new
+        # atom sitting at 0.0 is already fully described by its state.
+        return current.progress != 0.0 or current.progress_details is not None
+    return (
+        before.progress != current.progress
+        or before.progress_details != current.progress_details
+    )
+
+
+def _atom_state_event(
+    flow: FlowSnapshot,
+    before: AtomSnapshot | None,
+    current: AtomSnapshot,
+) -> Event:
+    details: dict[str, object] = {}
+    if current.failure is not None and (
+        before is None or before.failure != current.failure
+    ):
+        details["failure"] = current.failure
+    if current.revert_failure is not None and (
+        before is None or before.revert_failure != current.revert_failure
+    ):
+        details["revert_failure"] = current.revert_failure
+    if current.has_result:
+        details["has_result"] = True
+
+    return _new_event(
+        flow,
+        kind=EventKind.ATOM_STATE,
+        atom=current,
+        state=current.state,
+        old_state=before.state if before is not None else None,
+        progress=current.progress,
+        details=details,
+    )
+
+
+def _atom_progress_event(flow: FlowSnapshot, current: AtomSnapshot) -> Event:
+    details: dict[str, object] = {}
+    if current.progress_details is not None:
+        details["progress_details"] = current.progress_details
+    return _new_event(
+        flow,
+        kind=EventKind.ATOM_PROGRESS,
+        atom=current,
+        state=current.state,
+        progress=current.progress,
+        details=details,
+    )
+
+
+def _new_event(
+    flow: FlowSnapshot,
+    *,
+    kind: EventKind,
+    atom: AtomSnapshot | None = None,
+    state: str | None = None,
+    old_state: str | None = None,
+    progress: float | None = None,
+    details: dict[str, object] | None = None,
+) -> Event:
+    """Build an event with placeholder ``seq``/``ts``.
+
+    :func:`diff_flow` stamps both once emission order is settled.
+    """
+    return Event(
+        run_id=flow.run_id,
+        seq=_UNSTAMPED_SEQ,
+        ts=_UNSTAMPED_TS,
+        kind=kind,
+        book_id=flow.book_id,
+        atom_name=atom.name if atom is not None else None,
+        atom_uuid=atom.uuid if atom is not None else None,
+        atom_type=atom.atom_type if atom is not None else None,
+        state=state,
+        old_state=old_state,
+        intention=atom.intention if atom is not None else None,
+        progress=progress,
+        details=dict(details or {}),
+    )
+
+
+__all__ = ["diff_flow"]

taskflow_meter/events.py

@@ -0,0 +1,129 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""The single event shape every producer emits.
+
+The in-process listener and the persistence poller observe a flow in very
+different ways, but both express what they saw as one of these, so the
+datasources, the API and any transport only ever handle one vocabulary.
+"""
+
+from __future__ import annotations
+
+import threading
+from collections.abc import Mapping
+from dataclasses import asdict
+from dataclasses import dataclass
+from dataclasses import field
+from enum import StrEnum
+from typing import Any
+from typing import Self
+
+
+class EventKind(StrEnum):
+    """What an event is reporting."""
+
+    #: The flow itself moved between states.
+    FLOW_STATE = "flow_state"
+    #: An atom moved between states, or was seen for the first time.
+    ATOM_STATE = "atom_state"
+    #: An atom reported progress without necessarily changing state.
+    ATOM_PROGRESS = "atom_progress"
+    #: The flow's graph, emitted once by the in-process producer.  The
+    #: poller cannot produce this: taskflow persists atoms but not edges.
+    FLOW_STRUCTURE = "flow_structure"
+    #: The flow finished and a result or failure summary is available.
+    FLOW_RESULT = "flow_result"
+    #: Liveness only; carries no state change.
+    HEARTBEAT = "heartbeat"
+
+
+@dataclass(frozen=True, slots=True)
+class Event:
+    """One observation about one flow run.
+
+    ``seq`` is gap-free and monotonic per ``run_id``, which is what lets a
+    client reconnect with ``since_seq`` and know whether it missed
+    anything.  ``ts`` is observation time -- see :class:`.FlowSnapshot`.
+    """
+
+    run_id: str
+    seq: int
+    ts: float
+    kind: EventKind
+    book_id: str | None = None
+    atom_name: str | None = None
+    atom_uuid: str | None = None
+    atom_type: str | None = None
+    state: str | None = None
+    old_state: str | None = None
+    intention: str | None = None
+    progress: float | None = None
+    details: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Render to plain JSON-serialisable types."""
+        data = asdict(self)
+        data["kind"] = str(self.kind)
+        return data
+
+    @classmethod
+    def from_dict(cls, data: Mapping[str, Any]) -> Self:
+        """Rebuild from :meth:`to_dict` output.
+
+        Unknown keys are rejected rather than dropped -- a transport
+        speaking a newer dialect should fail loudly, not silently discard
+        the part we did not understand.
+        """
+        payload = dict(data)
+        payload["kind"] = EventKind(payload["kind"])
+        return cls(**payload)
+
+
+class SequenceAllocator:
+    """Hands out gap-free per-run sequence numbers, starting at 1.
+
+    Thread-safe by necessity: the parallel engine fires notifier callbacks
+    from executor threads, and the poller runs on its own thread.
+    """
+
+    __slots__ = ("_counters", "_lock")
+
+    def __init__(self, counters: Mapping[str, int] | None = None) -> None:
+        self._lock = threading.Lock()
+        self._counters: dict[str, int] = dict(counters or {})
+
+    def allocate(self, run_id: str) -> int:
+        """Return the next sequence number for ``run_id``."""
+        with self._lock:
+            nxt = self._counters.get(run_id, 0) + 1
+            self._counters[run_id] = nxt
+            return nxt
+
+    def peek(self, run_id: str) -> int:
+        """Return the last number handed out, or 0 if none has been."""
+        with self._lock:
+            return self._counters.get(run_id, 0)
+
+    def resume_from(self, run_id: str, seq: int) -> None:
+        """Continue numbering after ``seq``.
+
+        A restarted collector calls this with the highest sequence already
+        stored, so it does not renumber events a client has already seen.
+        """
+        with self._lock:
+            self._counters[run_id] = max(self._counters.get(run_id, 0), seq)
+
+    def forget(self, run_id: str) -> None:
+        """Drop the counter for a run that is finished and expired."""
+        with self._lock:
+            self._counters.pop(run_id, None)

taskflow_meter/fold.py

@@ -0,0 +1,137 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Folding events back into a snapshot.
+
+Shared by every writable datasource, so they cannot disagree about what
+an event means.  The property this exists to keep true: applying the
+events a diff produced reproduces the snapshots the diff came from --
+whichever store is doing the applying.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from taskflow_meter.events import Event
+from taskflow_meter.events import EventKind
+from taskflow_meter.models import AtomSnapshot
+from taskflow_meter.models import FlowSnapshot
+
+LOG = logging.getLogger(__name__)
+
+
+def flow_from_event(event: Event) -> FlowSnapshot:
+    """Seed a run from the first event seen for it.
+
+    The diff engine puts the flow's identity on the first event of a
+    run, but a client may join mid-stream, so everything here has to
+    tolerate absence.
+    """
+    details = event.details
+    return FlowSnapshot(
+        run_id=event.run_id,
+        name=str(details.get("flow_name", "")),
+        book_id=event.book_id,
+        book_name=_optional_str(details.get("book_name")),
+        observed_at=event.ts,
+    )
+
+
+def fold(flow: FlowSnapshot, event: Event) -> FlowSnapshot:
+    """Return ``flow`` updated by ``event``."""
+    from dataclasses import replace
+
+    flow = replace(flow, observed_at=max(flow.observed_at, event.ts))
+
+    if event.kind is EventKind.FLOW_STATE:
+        return replace(flow, state=event.state)
+
+    if event.kind in (EventKind.ATOM_STATE, EventKind.ATOM_PROGRESS):
+        if event.atom_name is None:
+            LOG.warning(
+                "ignoring %s event %d for run %s: no atom name",
+                event.kind,
+                event.seq,
+                event.run_id,
+            )
+            return flow
+        atoms = dict(flow.atoms)
+        atoms[event.atom_name] = fold_atom(
+            atoms.get(event.atom_name), event, event.atom_name
+        )
+        return replace(flow, atoms=atoms)
+
+    return flow
+
+
+def fold_atom(
+    atom: AtomSnapshot | None, event: Event, name: str
+) -> AtomSnapshot:
+    from dataclasses import replace
+
+    if atom is None:
+        atom = AtomSnapshot(name=name)
+
+    changes: dict[str, Any] = {}
+    if event.atom_uuid is not None:
+        changes["uuid"] = event.atom_uuid
+    if event.atom_type is not None:
+        changes["atom_type"] = event.atom_type
+    if event.intention is not None:
+        changes["intention"] = event.intention
+    if event.progress is not None:
+        changes["progress"] = event.progress
+
+    if event.kind is EventKind.ATOM_STATE:
+        changes["state"] = event.state
+        failure = event.details.get("failure")
+        if failure is not None:
+            changes["failure"] = failure
+        revert_failure = event.details.get("revert_failure")
+        if revert_failure is not None:
+            changes["revert_failure"] = revert_failure
+        if event.details.get("has_result"):
+            changes["has_result"] = True
+
+    if event.kind is EventKind.ATOM_PROGRESS:
+        changes["progress_details"] = event.details.get("progress_details")
+
+    return replace(atom, **changes)
+
+
+def contiguous_from(
+    events: list[Event], expected: int, limit: int
+) -> list[Event]:
+    """Take the unbroken run of events starting at ``expected``.
+
+    Concurrent producers do not arrive in sequence order: two threads
+    each allocate a number and then hand it on, and the hand-offs can
+    invert.  Returning event 8 while 7 is still in flight would advance
+    a caller past 7 for good, so anything beyond a gap waits.
+    """
+    taken: list[Event] = []
+    for event in events:
+        if event.seq < expected:
+            continue
+        if event.seq != expected:
+            break
+        taken.append(event)
+        expected += 1
+        if len(taken) >= limit:
+            break
+    return taken
+
+
+def _optional_str(value: Any) -> str | None:
+    return None if value is None else str(value)