kahn-queue 0.1.0__tar.gz

kahn_queue-0.1.0/PKG-INFO

@@ -0,0 +1,75 @@
+Metadata-Version: 2.4
+Name: kahn-queue
+Version: 0.1.0
+Summary: Kahn-style ready-queue for dependency scheduling
+License: MIT
+Project-URL: Homepage, https://github.com/Flashlock/kahn-queue
+Project-URL: Repository, https://github.com/Flashlock/kahn-queue
+Keywords: dag,scheduler,workflow,kahn
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# kahn-queue (Python)
+
+## Getting started
+
+- **Python:** **3.10+** recommended (venv + `requirements-dev.txt`; no version pinned in-repo).
+- **Tests:** `make test-python`, or `cd python`, create `.venv`, install `requirements-dev.txt`, then `pytest`.
+
+## Pieces
+
+| Piece | Role |
+|--------|------|
+| `Dag` / `Dag.builder()` | Immutable DAG: `add`, `connect`, `build`. |
+| `KahnScheduler` | Drives execution: `run`, `signal_complete` / `signal_failed`; `get_result()` returns `DagResult` (frozensets of ids). |
+| `KahnQueue` / `DefaultKahnQueue` / `ConcurrentKahnQueue` | **`DefaultKahnQueue`** for single-threaded updates; **`ConcurrentKahnQueue`** when `pop` / `prune` run from many threads. |
+| `IllegalGraphException` | Raised for invalid graphs (e.g. self-loop or cycle at `build()`). |
+| `NodeProgressTracker` | Optional per-node progress in `[0, 1]`; not required for scheduling. |
+
+## Examples
+
+### Single-threaded (Temporal-style)
+
+```python
+from dag import Dag
+from scheduler import KahnScheduler
+
+b = Dag.builder()
+lint = b.add("lint")
+comp = b.add("compile")
+test = b.add("test")
+b.connect(lint, comp).connect(comp, test)
+dag = b.build()
+
+def execute_node(node_id: int, sched: KahnScheduler[str]) -> None:
+    try:
+        run_step(dag[node_id])
+        sched.signal_complete(node_id)
+    except Exception:
+        sched.signal_failed(node_id)
+
+sched = KahnScheduler(dag, execute_node)
+sched.run()
+result = sched.get_result()
+```
+
+The two-arg `KahnScheduler(dag, execute_node)` uses `DefaultKahnQueue` when `queue` is omitted.
+
+### Concurrent (`ConcurrentKahnQueue`)
+
+```python
+from kahnQueue.concurrent_kahn_queue import ConcurrentKahnQueue
+from scheduler import KahnScheduler
+
+# … build dag, define execute_node as in the previous example …
+
+sched = KahnScheduler(
+    dag,
+    execute_node,
+    queue=ConcurrentKahnQueue(dag),
+)
+sched.run()
+result = sched.get_result()
+```
+
+Use a thread-safe queue when `execute_node` is invoked from many threads; keep any extra shared result structures thread-safe as well.

kahn_queue-0.1.0/README.md

@@ -0,0 +1,64 @@
+# kahn-queue (Python)
+
+## Getting started
+
+- **Python:** **3.10+** recommended (venv + `requirements-dev.txt`; no version pinned in-repo).
+- **Tests:** `make test-python`, or `cd python`, create `.venv`, install `requirements-dev.txt`, then `pytest`.
+
+## Pieces
+
+| Piece | Role |
+|--------|------|
+| `Dag` / `Dag.builder()` | Immutable DAG: `add`, `connect`, `build`. |
+| `KahnScheduler` | Drives execution: `run`, `signal_complete` / `signal_failed`; `get_result()` returns `DagResult` (frozensets of ids). |
+| `KahnQueue` / `DefaultKahnQueue` / `ConcurrentKahnQueue` | **`DefaultKahnQueue`** for single-threaded updates; **`ConcurrentKahnQueue`** when `pop` / `prune` run from many threads. |
+| `IllegalGraphException` | Raised for invalid graphs (e.g. self-loop or cycle at `build()`). |
+| `NodeProgressTracker` | Optional per-node progress in `[0, 1]`; not required for scheduling. |
+
+## Examples
+
+### Single-threaded (Temporal-style)
+
+```python
+from dag import Dag
+from scheduler import KahnScheduler
+
+b = Dag.builder()
+lint = b.add("lint")
+comp = b.add("compile")
+test = b.add("test")
+b.connect(lint, comp).connect(comp, test)
+dag = b.build()
+
+def execute_node(node_id: int, sched: KahnScheduler[str]) -> None:
+    try:
+        run_step(dag[node_id])
+        sched.signal_complete(node_id)
+    except Exception:
+        sched.signal_failed(node_id)
+
+sched = KahnScheduler(dag, execute_node)
+sched.run()
+result = sched.get_result()
+```
+
+The two-arg `KahnScheduler(dag, execute_node)` uses `DefaultKahnQueue` when `queue` is omitted.
+
+### Concurrent (`ConcurrentKahnQueue`)
+
+```python
+from kahnQueue.concurrent_kahn_queue import ConcurrentKahnQueue
+from scheduler import KahnScheduler
+
+# … build dag, define execute_node as in the previous example …
+
+sched = KahnScheduler(
+    dag,
+    execute_node,
+    queue=ConcurrentKahnQueue(dag),
+)
+sched.run()
+result = sched.get_result()
+```
+
+Use a thread-safe queue when `execute_node` is invoked from many threads; keep any extra shared result structures thread-safe as well.

kahn_queue-0.1.0/pyproject.toml

@@ -0,0 +1,26 @@
+[build-system]
+requires = ["setuptools>=61", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "kahn-queue"
+version = "0.1.0"
+description = "Kahn-style ready-queue for dependency scheduling"
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+keywords = ["dag", "scheduler", "workflow", "kahn"]
+
+[project.urls]
+Homepage = "https://github.com/Flashlock/kahn-queue"
+Repository = "https://github.com/Flashlock/kahn-queue"
+
+# Layout: packages under src/ plus top-level modules (dag, scheduler, …).
+[tool.setuptools.package-dir]
+"" = "src"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools]
+py-modules = ["dag", "scheduler", "exception", "tracker"]

kahn_queue-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

kahn_queue-0.1.0/src/dag.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass
+from typing import Deque, Generic, Iterable, Iterator, List, Set, Tuple, TypeVar
+
+from exception import IllegalGraphException
+
+T = TypeVar("T")
+
+
+def validate_node(node_id: int, size: int) -> None:
+    """Ensures ``node_id`` is valid for a graph of ``size`` nodes.
+
+    Raises:
+        IndexError: if out of range
+    """
+    if not 0 <= node_id < size:
+        raise IndexError(f"Invalid node id: {node_id}")
+
+
+@dataclass(frozen=True)
+class Dag(Generic[T], Iterable[T]):
+    """Immutable directed graph of nodes with integer ids and typed payloads.
+
+    Obtain instances via the static :meth:`builder` factory; edges go from source to target.
+    """
+
+    _nodes: Tuple[T, ...]
+    _adj: Tuple[Tuple[int, ...], ...]
+    _rev: Tuple[Tuple[int, ...], ...]
+
+    def __len__(self) -> int:
+        """Number of nodes (ids are ``0 .. size()-1``)."""
+        return len(self._nodes)
+
+    def size(self) -> int:
+        """Number of nodes."""
+        return len(self)
+
+    def __getitem__(self, node_id: int) -> T:
+        """Payload for node ``node_id``."""
+        validate_node(node_id, len(self))
+        return self._nodes[node_id]
+
+    def get(self, node_id: int) -> T:
+        """Payload for node ``node_id``."""
+        return self[node_id]
+
+    def in_degree(self, node_id: int) -> int:
+        """Count of incoming edges to ``node_id``."""
+        validate_node(node_id, len(self))
+        return len(self._rev[node_id])
+
+    def out_degree(self, node_id: int) -> int:
+        """Count of outgoing edges from ``node_id``."""
+        validate_node(node_id, len(self))
+        return len(self._adj[node_id])
+
+    def targets(self, node_id: int) -> Iterable[int]:
+        """Successor ids of ``node_id``."""
+        validate_node(node_id, len(self))
+        return self._adj[node_id]
+
+    def sources(self, node_id: int) -> Iterable[int]:
+        """Predecessor ids of ``node_id``."""
+        validate_node(node_id, len(self))
+        return self._rev[node_id]
+
+    def __iter__(self) -> Iterator[T]:
+        """Payloads in id order; use ``targets`` / ``sources`` for edge endpoints."""
+        return iter(self._nodes)
+
+    @staticmethod
+    def validate_node(node_id: int, size: int) -> None:
+        """Ensures ``node_id`` is valid for a graph of ``size`` nodes.
+
+        Raises:
+            IndexError: if out of range
+        """
+        validate_node(node_id, size)
+
+    @classmethod
+    def builder(cls) -> DagBuilder[T]:
+        """Returns a new mutable ``Builder``."""
+        return DagBuilder()
+
+
+class DagBuilder(Generic[T]):
+    """Mutable graph builder."""
+
+    def __init__(self) -> None:
+        self._nodes: List[T] = []
+        self._adj: List[Set[int]] = []
+        self._rev: List[Set[int]] = []
+
+    def add(self, data: T) -> int:
+        """Adds a node; returns its id for passing to ``connect``."""
+        node_id = len(self._nodes)
+        self._nodes.append(data)
+        self._adj.append(set())
+        self._rev.append(set())
+        return node_id
+
+    def connect(self, source: int, target: int) -> DagBuilder[T]:
+        """Directed edge ``source`` → ``target`` (ignored if duplicate).
+
+        Raises:
+            IllegalGraphException: if ``source == target``
+        """
+        validate_node(source, len(self._nodes))
+        validate_node(target, len(self._nodes))
+        
+        if source == target:
+            raise IllegalGraphException(f"Self-loop not allowed: {source}")
+            
+        if target not in self._adj[source]:
+            self._adj[source].add(target)
+            self._rev[target].add(source)
+        return self
+
+    def build(self) -> Dag[T]:
+        """Build the graph."""
+        self._cycle_check()
+        return Dag(
+            _nodes=tuple(self._nodes),
+            _adj=tuple(tuple(sorted(s)) for s in self._adj),
+            _rev=tuple(tuple(sorted(s)) for s in self._rev),
+        )
+
+    def _cycle_check(self) -> None:
+        n = len(self._nodes)
+        if n == 0:
+            return
+            
+        indeg = [len(self._rev[i]) for i in range(n)]
+        ready: Deque[int] = deque(i for i, d in enumerate(indeg) if d == 0)
+        processed = 0
+        
+        while ready:
+            u = ready.popleft()
+            processed += 1
+            for v in self._adj[u]:
+                indeg[v] -= 1
+                if indeg[v] == 0:
+                    ready.append(v)
+                    
+        if processed != n:
+            raise IllegalGraphException("Graph contains a directed cycle")

kahn_queue-0.1.0/src/exception.py

@@ -0,0 +1,6 @@
+"""Exceptions shared across the KahnQueue modules."""
+
+
+class IllegalGraphException(ValueError):
+    """Thrown when connecting a node to itself in ``Dag.Builder`` (self-loop)."""
+

kahn_queue-0.1.0/src/kahnQueue/__init__.py

@@ -0,0 +1,6 @@
+from .concurrent_kahn_queue import ConcurrentKahnQueue
+from .default_kahn_queue import DefaultKahnQueue
+from .kahn_queue import KahnQueue
+
+__all__ = ["ConcurrentKahnQueue", "DefaultKahnQueue", "KahnQueue"]
+

kahn_queue-0.1.0/src/kahnQueue/concurrent_kahn_queue.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from threading import RLock
+from typing import List, Set
+
+from dag import Dag, validate_node
+from kahnQueue.node_machine import NodeMachine
+from kahnQueue.node_state import NodeState
+from kahnQueue.kahn_queue import KahnQueue
+
+
+class ConcurrentKahnQueue(KahnQueue):
+    """``KahnQueue`` for concurrent ``pop`` and ``prune`` calls. ``ready_ids()`` may not
+    reflect a consistent snapshot if other threads update the queue at the same time; coordinate
+    externally if you need strict ordering or visibility. For single-threaded use, prefer
+    ``DefaultKahnQueue``.
+    """
+
+    def __init__(self, dag: Dag[object]) -> None:
+        """Builds a queue whose readiness matches ``dag``."""
+        self._dag = dag
+        # Each machine gets its own lock, mirroring the synchronized(nodeMachines[id]) pattern
+        self._node_machines: List[NodeMachine] = [
+            NodeMachine.create(i, dag.in_degree(i)) for i in range(dag.size())
+        ]
+        self._locks = [RLock() for _ in range(dag.size())]
+
+    def pop(self, id: int) -> Set[int]:
+        validate_node(id, self._dag.size())
+        
+        with self._locks[id]:
+            machine = self._node_machines[id]
+            if not machine.is_(NodeState.ACTIVE):
+                raise ValueError(f"Pop failed. Node {id} is not ACTIVE (state: {machine.state})")
+            
+            machine.transition(NodeState.COMPLETE)
+
+            promoted: Set[int] = set()
+            for cid in self._dag.targets(id):
+                with self._locks[cid]:
+                    child = self._node_machines[cid]
+                    child.decrement()
+                    
+                    if child.can_transition(NodeState.ACTIVE):
+                        child.transition(NodeState.ACTIVE)
+                        promoted.add(cid)
+            return promoted
+
+    def prune(self, id: int) -> Set[int]:
+        validate_node(id, self._dag.size())
+        
+        affected: Set[int] = set()
+        stack: List[int] = [id]
+
+        while stack:
+            curr = stack.pop()
+            
+            with self._locks[curr]:
+                machine = self._node_machines[curr]
+                # Avoid redundant transitions if another thread pruned this branch
+                if machine.is_(NodeState.PRUNED):
+                    continue
+                machine.transition(NodeState.PRUNED)
+                affected.add(curr)
+            
+            # Adding targets outside the lock is safe as the DAG structure is immutable
+            stack.extend(self._dag.targets(curr))
+            
+        return affected
+
+    def ready_ids(self) -> Set[int]:
+        return {
+            m.id for m in self._node_machines 
+            if m.is_(NodeState.READY)
+        }

kahn_queue-0.1.0/src/kahnQueue/default_kahn_queue.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from collections import deque
+from typing import List, Set
+
+from dag import Dag, validate_node
+from kahnQueue.node_machine import NodeMachine
+from kahnQueue.node_state import NodeState
+from kahnQueue.kahn_queue import KahnQueue
+
+class DefaultKahnQueue(KahnQueue):
+    """Single-threaded implementation of Kahn's algorithm state tracking."""
+
+    def __init__(self, dag: Dag[object]) -> None:
+        self._dag = dag
+        self._node_machines: List[NodeMachine] = [
+            NodeMachine.create(i, dag.in_degree(i)) for i in range(dag.size())
+        ]
+
+    def pop(self, id: int) -> Set[int]:
+        validate_node(id, self._dag.size())
+        machine = self._node_machines[id]
+        
+        if not machine.is_(NodeState.ACTIVE):
+            raise ValueError(f"Pop failed. Node {id} is in state {machine.state}, not ACTIVE")
+        
+        machine.transition(NodeState.COMPLETE)
+
+        promoted: Set[int] = set()
+        for cid in self._dag.targets(id):
+            child = self._node_machines[cid]
+            child.decrement()
+            # Encapsulated logic: if it's READY, move it to ACTIVE immediately
+            # in a single-threaded queue.
+            if child.is_(NodeState.READY):
+                child.transition(NodeState.ACTIVE)
+                promoted.add(cid)
+        return promoted
+
+    def prune(self, id: int) -> Set[int]:
+        validate_node(id, self._dag.size())
+        affected: Set[int] = set()
+        stack: List[int] = [id]
+        
+        while stack:
+            curr = stack.pop()
+            machine = self._node_machines[curr]
+            
+            # Skip if already pruned to avoid cycles/redundancy
+            if machine.is_(NodeState.PRUNED):
+                continue
+                
+            machine.transition(NodeState.PRUNED)
+            affected.add(curr)
+            stack.extend(self._dag.targets(curr))
+            
+        return affected
+
+    def ready_ids(self) -> Set[int]:
+        return {m.id for m in self._node_machines if m.is_(NodeState.READY)}

kahn_queue-0.1.0/src/kahnQueue/kahn_queue.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+from typing import Protocol, runtime_checkable, Iterable
+
+@runtime_checkable
+class KahnQueue(Protocol):
+    """Tracks which DAG nodes are runnable and applies completion or pruning."""
+
+    def pop(self, id: int) -> Iterable[int]:
+        """Marks ``id`` completed and returns ids of nodes that became runnable."""
+        ...
+
+    def prune(self, id: int) -> Iterable[int]:
+        """Marks ``id`` and its descendants pruned; returns every affected node id."""
+        ...
+
+    def ready_ids(self) -> Iterable[int]:
+        """Node ids currently runnable (not yet active)."""
+        ...

kahn_queue-0.1.0/src/kahnQueue/node_machine.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Final, Mapping, Set
+
+from kahnQueue.node_state import NodeState
+from utils.state_machine import StateMachine
+
+NODE_TRANSITIONS: Final[Mapping[NodeState, Set[NodeState]]] = {
+    NodeState.QUEUED: {NodeState.READY, NodeState.PRUNED},
+    NodeState.READY: {NodeState.ACTIVE, NodeState.PRUNED},
+    NodeState.ACTIVE: {NodeState.COMPLETE, NodeState.PRUNED},
+    NodeState.COMPLETE: set(),
+    NodeState.PRUNED: set(),
+}
+
+
+@dataclass
+class NodeMachine(StateMachine[NodeState]):
+    num_sources: int
+    id: int
+
+    @classmethod
+    def create(cls, id: int, num_sources: int) -> NodeMachine:
+        """Create a new NodeMachine and attempt initial transition to READY."""
+        m = cls(
+            _state=NodeState.QUEUED,
+            _transitions=NODE_TRANSITIONS,
+            num_sources=num_sources,
+            id=id,
+        )
+        m._try_ready()
+        return m
+
+    def can_transition(self, to: NodeState) -> bool:
+        if self.is_(NodeState.QUEUED) and self.num_sources > 0 and to == NodeState.READY:
+            return False
+        return super().can_transition(to)
+
+    def decrement(self) -> None:
+        """Decrement num_sources and transition to READY if possible."""
+        if self.num_sources <= 0:
+            raise RuntimeError("Attempting to decrement below zero")
+        self.num_sources -= 1
+        self._try_ready()
+
+    def _try_ready(self) -> None:
+        if self.can_transition(NodeState.READY):
+            self.transition(NodeState.READY)
+            

kahn_queue-0.1.0/src/kahnQueue/node_state.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from enum import StrEnum, auto
+
+class NodeState(StrEnum):
+    QUEUED = auto()
+    READY = auto()
+    ACTIVE = auto()
+    COMPLETE = auto()
+    PRUNED = auto()

kahn_queue-0.1.0/src/kahn_queue.egg-info/PKG-INFO

@@ -0,0 +1,75 @@
+Metadata-Version: 2.4
+Name: kahn-queue
+Version: 0.1.0
+Summary: Kahn-style ready-queue for dependency scheduling
+License: MIT
+Project-URL: Homepage, https://github.com/Flashlock/kahn-queue
+Project-URL: Repository, https://github.com/Flashlock/kahn-queue
+Keywords: dag,scheduler,workflow,kahn
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# kahn-queue (Python)
+
+## Getting started
+
+- **Python:** **3.10+** recommended (venv + `requirements-dev.txt`; no version pinned in-repo).
+- **Tests:** `make test-python`, or `cd python`, create `.venv`, install `requirements-dev.txt`, then `pytest`.
+
+## Pieces
+
+| Piece | Role |
+|--------|------|
+| `Dag` / `Dag.builder()` | Immutable DAG: `add`, `connect`, `build`. |
+| `KahnScheduler` | Drives execution: `run`, `signal_complete` / `signal_failed`; `get_result()` returns `DagResult` (frozensets of ids). |
+| `KahnQueue` / `DefaultKahnQueue` / `ConcurrentKahnQueue` | **`DefaultKahnQueue`** for single-threaded updates; **`ConcurrentKahnQueue`** when `pop` / `prune` run from many threads. |
+| `IllegalGraphException` | Raised for invalid graphs (e.g. self-loop or cycle at `build()`). |
+| `NodeProgressTracker` | Optional per-node progress in `[0, 1]`; not required for scheduling. |
+
+## Examples
+
+### Single-threaded (Temporal-style)
+
+```python
+from dag import Dag
+from scheduler import KahnScheduler
+
+b = Dag.builder()
+lint = b.add("lint")
+comp = b.add("compile")
+test = b.add("test")
+b.connect(lint, comp).connect(comp, test)
+dag = b.build()
+
+def execute_node(node_id: int, sched: KahnScheduler[str]) -> None:
+    try:
+        run_step(dag[node_id])
+        sched.signal_complete(node_id)
+    except Exception:
+        sched.signal_failed(node_id)
+
+sched = KahnScheduler(dag, execute_node)
+sched.run()
+result = sched.get_result()
+```
+
+The two-arg `KahnScheduler(dag, execute_node)` uses `DefaultKahnQueue` when `queue` is omitted.
+
+### Concurrent (`ConcurrentKahnQueue`)
+
+```python
+from kahnQueue.concurrent_kahn_queue import ConcurrentKahnQueue
+from scheduler import KahnScheduler
+
+# … build dag, define execute_node as in the previous example …
+
+sched = KahnScheduler(
+    dag,
+    execute_node,
+    queue=ConcurrentKahnQueue(dag),
+)
+sched.run()
+result = sched.get_result()
+```
+
+Use a thread-safe queue when `execute_node` is invoked from many threads; keep any extra shared result structures thread-safe as well.

kahn_queue-0.1.0/src/kahn_queue.egg-info/SOURCES.txt

@@ -0,0 +1,25 @@
+README.md
+pyproject.toml
+src/dag.py
+src/exception.py
+src/scheduler.py
+src/tracker.py
+src/kahnQueue/__init__.py
+src/kahnQueue/concurrent_kahn_queue.py
+src/kahnQueue/default_kahn_queue.py
+src/kahnQueue/kahn_queue.py
+src/kahnQueue/node_machine.py
+src/kahnQueue/node_state.py
+src/kahn_queue.egg-info/PKG-INFO
+src/kahn_queue.egg-info/SOURCES.txt
+src/kahn_queue.egg-info/dependency_links.txt
+src/kahn_queue.egg-info/top_level.txt
+src/utils/__init__.py
+src/utils/state_machine.py
+tests/test_concurrent_kahn_queue.py
+tests/test_dag.py
+tests/test_default_kahn_queue.py
+tests/test_kahn_scheduler.py
+tests/test_node_machine.py
+tests/test_node_progress_tracker.py
+tests/test_state_machine.py

kahn_queue-0.1.0/src/kahn_queue.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

kahn_queue-0.1.0/src/kahn_queue.egg-info/top_level.txt

@@ -0,0 +1,6 @@
+dag
+exception
+kahnQueue
+scheduler
+tracker
+utils