rivlet 0.1.0__tar.gz

rivlet-0.1.0/Cargo.lock

@@ -0,0 +1,182 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "action"
+version = "0.1.0"
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "libc"
+version = "0.2.189"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
+
+[[package]]
+name = "native-graph"
+version = "0.1.0"
+dependencies = [
+ "action",
+ "rivlet",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
+
+[[package]]
+name = "portable-atomic"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b"
+dependencies = [
+ "libc",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9"
+dependencies = [
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "python"
+version = "0.1.0"
+dependencies = [
+ "action",
+ "native-graph",
+ "pyo3",
+ "rivlet",
+ "tokio-runtime",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rivlet"
+version = "0.1.0"
+dependencies = [
+ "topology",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+
+[[package]]
+name = "tokio"
+version = "1.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
+dependencies = [
+ "pin-project-lite",
+]
+
+[[package]]
+name = "tokio-runtime"
+version = "0.1.0"
+dependencies = [
+ "rivlet",
+ "tokio",
+]
+
+[[package]]
+name = "topology"
+version = "0.1.0"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"

rivlet-0.1.0/Cargo.toml

@@ -0,0 +1,16 @@
+[workspace]
+members = ["crates/action", "crates/core", "crates/native-graph", "crates/python", "crates/topology", "crates/tokio-runtime"]
+resolver = "3"
+
+[workspace.package]
+version = "0.1.0"
+edition = "2024"
+rust-version = "1.85"
+license = "MIT"
+
+[workspace.lints.rust]
+unsafe_code = "forbid"
+
+[workspace.lints.clippy]
+all = "warn"
+pedantic = "warn"

rivlet-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ririv
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

rivlet-0.1.0/PKG-INFO

@@ -0,0 +1,97 @@
+Metadata-Version: 2.4
+Name: rivlet
+Version: 0.1.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Classifier: Programming Language :: Rust
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+License-File: LICENSE
+Summary: Explicit, composable workflows with a Python API and a Rust execution core
+Keywords: dag,workflow,task-graph,concurrency,rust
+Author: ririv
+License-Expression: MIT
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Documentation, https://github.com/ririv/rivlet/blob/main/docs/index.md
+Project-URL: Homepage, https://github.com/ririv/rivlet
+Project-URL: Issues, https://github.com/ririv/rivlet/issues
+Project-URL: Repository, https://github.com/ririv/rivlet
+
+# Rivlet
+
+English | [简体中文](./README.zh-CN.md)
+
+Rivlet is an explicit, composable DAG declaration and execution library
+with a Python API and a Rust execution core.
+
+It provides steps, subgraphs, conditional dependencies, thread-based and async
+execution, result access, execution state, and graph rendering. Logging, terminal
+integration, and request-level control belong to the host application and are
+connected through explicit interfaces.
+
+## When to use it
+
+- Compose build or automation tasks with explicit dependencies.
+- Run independent data-processing or validation tasks concurrently, then join them.
+- Reuse subgraphs inside a larger workflow while keeping execution observable.
+
+This is an in-process task graph library, not a distributed workflow service.
+Your application owns the actions and integration; the library handles dependency
+ordering and execution state.
+
+## Architecture
+
+On `main`, Rust implements declaration compilation, execution-graph validation,
+and the scheduling state machine. Python retains graph declarations, arbitrary
+callables, result objects, and host integration. Complete Rust-native subgraphs
+can join the same execution graph and concurrency budget.
+
+See [Rust architecture](./docs/rust-architecture.md) for implementation boundaries.
+
+## Local development
+
+Install the published package with `pip install rivlet`. The Python import name
+is also `rivlet`; there is no `rivetoo_graph` compatibility package.
+See [release policy](./docs/releasing.md) for wheel coverage and publishing.
+
+You need Python 3.13 or later, a Rust toolchain, and maturin. With a virtual
+environment activated, install the native extension into that environment:
+
+```bash
+maturin develop
+```
+
+The native extension is required. There is no pure-Python execution fallback.
+
+## Quick example
+
+```python
+from rivlet import GraphExecution, Step, ThreadGraphRunOptions
+
+prepare = Step(lambda: print("prepare"), label="prepare")
+validate = Step(lambda: print("validate"), label="validate")
+test = Step(lambda: print("test"), label="test")
+finish = Step(lambda: print("finish"), label="finish")
+
+checks = (validate | test).graph(label="checks")
+graph = (prepare >> checks >> finish).graph(label="example")
+GraphExecution(graph=graph).run(ThreadGraphRunOptions(workers=2))
+```
+
+After `prepare`, the `checks` subgraph can run `validate` and `test` concurrently.
+`finish` waits for both to succeed; their print order is not guaranteed.
+
+`>>` declares ordering; `|` composes independent branches. Run options determine
+actual concurrency. An ordinary action failure blocks dependent work while
+unrelated branches continue; the run ultimately reports the failures.
+
+See the [documentation](./docs/index.md) for the model, composition API, and execution rules.
+
+## License
+
+This project is licensed under the [MIT License](./LICENSE).
+

rivlet-0.1.0/README.md

@@ -0,0 +1,73 @@
+# Rivlet
+
+English | [简体中文](./README.zh-CN.md)
+
+Rivlet is an explicit, composable DAG declaration and execution library
+with a Python API and a Rust execution core.
+
+It provides steps, subgraphs, conditional dependencies, thread-based and async
+execution, result access, execution state, and graph rendering. Logging, terminal
+integration, and request-level control belong to the host application and are
+connected through explicit interfaces.
+
+## When to use it
+
+- Compose build or automation tasks with explicit dependencies.
+- Run independent data-processing or validation tasks concurrently, then join them.
+- Reuse subgraphs inside a larger workflow while keeping execution observable.
+
+This is an in-process task graph library, not a distributed workflow service.
+Your application owns the actions and integration; the library handles dependency
+ordering and execution state.
+
+## Architecture
+
+On `main`, Rust implements declaration compilation, execution-graph validation,
+and the scheduling state machine. Python retains graph declarations, arbitrary
+callables, result objects, and host integration. Complete Rust-native subgraphs
+can join the same execution graph and concurrency budget.
+
+See [Rust architecture](./docs/rust-architecture.md) for implementation boundaries.
+
+## Local development
+
+Install the published package with `pip install rivlet`. The Python import name
+is also `rivlet`; there is no `rivetoo_graph` compatibility package.
+See [release policy](./docs/releasing.md) for wheel coverage and publishing.
+
+You need Python 3.13 or later, a Rust toolchain, and maturin. With a virtual
+environment activated, install the native extension into that environment:
+
+```bash
+maturin develop
+```
+
+The native extension is required. There is no pure-Python execution fallback.
+
+## Quick example
+
+```python
+from rivlet import GraphExecution, Step, ThreadGraphRunOptions
+
+prepare = Step(lambda: print("prepare"), label="prepare")
+validate = Step(lambda: print("validate"), label="validate")
+test = Step(lambda: print("test"), label="test")
+finish = Step(lambda: print("finish"), label="finish")
+
+checks = (validate | test).graph(label="checks")
+graph = (prepare >> checks >> finish).graph(label="example")
+GraphExecution(graph=graph).run(ThreadGraphRunOptions(workers=2))
+```
+
+After `prepare`, the `checks` subgraph can run `validate` and `test` concurrently.
+`finish` waits for both to succeed; their print order is not guaranteed.
+
+`>>` declares ordering; `|` composes independent branches. Run options determine
+actual concurrency. An ordinary action failure blocks dependent work while
+unrelated branches continue; the run ultimately reports the failures.
+
+See the [documentation](./docs/index.md) for the model, composition API, and execution rules.
+
+## License
+
+This project is licensed under the [MIT License](./LICENSE).

rivlet-0.1.0/crates/action/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "action"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+
+[lints]
+workspace = true

rivlet-0.1.0/crates/action/src/executor.rs

@@ -0,0 +1,268 @@
+use std::collections::VecDeque;
+use std::panic::{AssertUnwindSafe, catch_unwind};
+use std::sync::{Arc, Condvar, Mutex, mpsc};
+use std::thread::{self, JoinHandle};
+
+use crate::{NativeActionContext, NativeActionError, NativeActionHandle};
+
+struct NativeActionJob {
+    node: usize,
+    action: NativeActionHandle,
+    context: NativeActionContext,
+}
+
+/// One native action completion retained until its graph session consumes it.
+#[derive(Debug)]
+pub struct NativeActionCompletion {
+    node: usize,
+    result: Result<(), NativeActionError>,
+}
+
+impl NativeActionCompletion {
+    #[must_use]
+    pub const fn node(&self) -> usize {
+        self.node
+    }
+
+    /// Returns the concrete action result.
+    ///
+    /// # Errors
+    /// Returns the error produced by the native action.
+    pub fn into_result(self) -> Result<(), NativeActionError> {
+        self.result
+    }
+}
+
+#[derive(Default)]
+struct CompletionState {
+    values: VecDeque<NativeActionCompletion>,
+    revision: u64,
+}
+
+#[derive(Default)]
+struct CompletionQueue {
+    state: Mutex<CompletionState>,
+    ready: Condvar,
+}
+
+enum WaitState {
+    Ready,
+    Pending(u64),
+}
+
+/// A wait bound to the completion queue and revision observed at preparation.
+pub struct NativeCompletionWait {
+    queue: Arc<CompletionQueue>,
+    state: WaitState,
+}
+
+impl NativeCompletionWait {
+    pub fn wait(self) {
+        match self.state {
+            WaitState::Ready => (),
+            WaitState::Pending(revision) => {
+                let mut state = self
+                    .queue
+                    .state
+                    .lock()
+                    .unwrap_or_else(std::sync::PoisonError::into_inner);
+                // 即使另一个 driver 已取走完成记录，序号变化也保留这次通知。
+                while state.values.is_empty() && state.revision == revision {
+                    state = self
+                        .queue
+                        .ready
+                        .wait(state)
+                        .unwrap_or_else(std::sync::PoisonError::into_inner);
+                }
+            }
+        }
+    }
+}
+
+struct ExecutorInner {
+    jobs: mpsc::Sender<Option<NativeActionJob>>,
+    completions: Arc<CompletionQueue>,
+    workers: Vec<JoinHandle<()>>,
+}
+
+impl Drop for ExecutorInner {
+    fn drop(&mut self) {
+        for _ in &self.workers {
+            let _ = self.jobs.send(None);
+        }
+        for worker in self.workers.drain(..) {
+            let _ = worker.join();
+        }
+    }
+}
+
+/// A bounded Rust worker set used only for Rust-owned actions.
+#[derive(Clone)]
+pub struct NativeActionExecutor {
+    inner: Arc<ExecutorInner>,
+}
+
+impl NativeActionExecutor {
+    /// Creates a native worker set with an explicit positive capacity.
+    ///
+    /// # Errors
+    /// Returns an error when capacity is zero or an operating-system thread
+    /// cannot be created.
+    pub fn new(workers: usize) -> Result<Self, NativeActionError> {
+        if workers == 0 {
+            return Err(NativeActionError::new(
+                "native action workers must be greater than zero",
+            ));
+        }
+        let (jobs, receiver) = mpsc::channel::<Option<NativeActionJob>>();
+        let receiver = Arc::new(Mutex::new(receiver));
+        let completions = Arc::new(CompletionQueue::default());
+        let mut worker_handles = Vec::with_capacity(workers);
+        for index in 0..workers {
+            let receiver = Arc::clone(&receiver);
+            let completions = Arc::clone(&completions);
+            let worker = thread::Builder::new()
+                .name(format!("NativeGraphWorker-{index}"))
+                .spawn(move || worker_loop(&receiver, &completions))
+                .map_err(|error| {
+                    NativeActionError::new(format!("failed to create native graph worker: {error}"))
+                })?;
+            worker_handles.push(worker);
+        }
+        Ok(Self {
+            inner: Arc::new(ExecutorInner {
+                jobs,
+                completions,
+                workers: worker_handles,
+            }),
+        })
+    }
+
+    /// Submits one action to the native worker set.
+    ///
+    /// # Errors
+    /// Returns an error when the executor has already stopped accepting work.
+    pub fn submit(
+        &self,
+        node: usize,
+        action: NativeActionHandle,
+        context: NativeActionContext,
+    ) -> Result<(), NativeActionError> {
+        self.inner
+            .jobs
+            .send(Some(NativeActionJob {
+                node,
+                action,
+                context,
+            }))
+            .map_err(|_| NativeActionError::new("native action executor is closed"))
+    }
+
+    /// Blocks until at least one completion can be drained.
+    pub fn wait_for_completion(&self) {
+        self.prepare_wait().wait();
+    }
+
+    #[must_use]
+    pub fn prepare_wait(&self) -> NativeCompletionWait {
+        let state = self
+            .inner
+            .completions
+            .state
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        NativeCompletionWait {
+            queue: Arc::clone(&self.inner.completions),
+            state: if state.values.is_empty() {
+                WaitState::Pending(state.revision)
+            } else {
+                WaitState::Ready
+            },
+        }
+    }
+
+    #[must_use]
+    pub fn drain_completions(&self) -> Vec<NativeActionCompletion> {
+        self.inner
+            .completions
+            .state
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .values
+            .drain(..)
+            .collect()
+    }
+}
+
+fn worker_loop(
+    jobs: &Mutex<mpsc::Receiver<Option<NativeActionJob>>>,
+    completions: &CompletionQueue,
+) {
+    loop {
+        let job = jobs
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .recv();
+        let Ok(Some(job)) = job else {
+            return;
+        };
+        let result = catch_unwind(AssertUnwindSafe(|| job.action.execute(&job.context)))
+            .unwrap_or_else(|_| Err(NativeActionError::new("native action panicked")));
+        let mut state = completions
+            .state
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        state.values.push_back(NativeActionCompletion {
+            node: job.node,
+            result,
+        });
+        state.revision = state
+            .revision
+            .checked_add(1)
+            .expect("native completion revision overflow");
+        drop(state);
+        completions.ready.notify_all();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::NativeAction;
+    use std::sync::mpsc;
+    use std::time::Duration;
+
+    struct Noop;
+
+    impl NativeAction for Noop {
+        fn execute(&self, _context: &NativeActionContext) -> Result<(), NativeActionError> {
+            Ok(())
+        }
+    }
+
+    #[test]
+    fn prepared_wait_survives_completion_drain() {
+        let executor = NativeActionExecutor::new(1).unwrap();
+        let pending = executor.prepare_wait();
+        executor
+            .submit(
+                0,
+                NativeActionHandle::new(Noop),
+                NativeActionContext::default(),
+            )
+            .unwrap();
+        executor.wait_for_completion();
+        let ready = executor.prepare_wait();
+        assert_eq!(executor.drain_completions().len(), 1);
+        let (sent, received) = mpsc::channel();
+        let waiter = thread::spawn(move || {
+            pending.wait();
+            ready.wait();
+            sent.send(()).unwrap();
+        });
+        received
+            .recv_timeout(Duration::from_secs(2))
+            .expect("completion notification was lost after drain");
+        waiter.join().unwrap();
+    }
+}