kazeflow 0.1.0a0__tar.gz

kazeflow-0.1.0a0/PKG-INFO

@@ -0,0 +1,116 @@
+Metadata-Version: 2.3
+Name: kazeflow
+Version: 0.1.0a0
+Summary: A lightweight, asset-based task flow engine
+Author: Kj
+Author-email: Kj <46800811+kj-9@users.noreply.github.com>
+Requires-Dist: rich>=14.1.0
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+
+# kazeflow
+
+`kazeflow` is a lightweight, asset-based task flow engine inspired by Dagster. It is designed to be simple, flexible, and easy to use.
+
+## Example
+
+When you run this script, `kazeflow` will execute the assets in the correct order, handle the failure of `failing_asset` gracefully, and provide a rich terminal UI to visualize the progress.
+
+
+Here is a simple example of how to use `kazeflow` to define and execute a data flow with dependencies, inputs/outputs, and logging.
+
+
+example.py:
+```python
+import asyncio
+
+from kazeflow.assets import asset
+from kazeflow.context import AssetContext
+from kazeflow.flow import Flow
+
+
+# A simple asset with no dependencies
+@asset()
+async def users() -> list[str]:
+    """This asset returns a list of user names."""
+    return ["Alice", "Bob", "Charlie"]
+
+
+# This asset depends on the `users` asset.
+# The output of `users` is automatically passed as an argument.
+@asset(deps=["users"])
+async def greetings(users: list[str], context: AssetContext) -> list[str]:
+    """This asset receives the list of users and a context object.
+
+    It uses the context to get a logger and log a message.
+    """
+    context.logger.info(f"Generating greetings for {len(users)} users.")
+    return [f"Hello, {user}!" for user in users]
+
+
+# This asset fails intentionally to demonstrate error handling.
+@asset(deps=["users"])
+async def failing_asset(users: list[str]):
+    """This asset always fails."""
+    raise ValueError("This asset is designed to fail.")
+
+
+if __name__ == "__main__":
+    # Define a flow that includes the final assets we want to generate.
+    # kazeflow automatically includes all upstream dependencies.
+    flow = Flow(asset_names=["greetings", "failing_asset"])
+
+    # Run the flow asynchronously.
+    # You can limit the number of concurrent assets with `max_concurrency`.
+    asyncio.run(flow.run_async(max_concurrency=2))
+
+```
+```bash
+❯ uv run example.py
+Task Flow (Execution Order)
+└── users
+    ├── failing_asset
+    └── greetings
+
+Execution Logs
+INFO     Executing asset: users                                         
+INFO     Finished executing asset: users in 0.00s                       
+INFO     Executing asset: greetings                                     
+INFO     Generating greetings for 3 users.                              
+INFO     Finished executing asset: greetings in 0.00s                   
+INFO     Executing asset: failing_asset                                 
+ERROR    Error executing asset failing_asset: This asset is designed to 
+         fail.                                                          
+         ╭───────────── Traceback (most recent call last) ─────────────╮
+         │ /Users/kh03/work/repos/myflow/src/kazeflow/flow.py:82 in    │
+         │ _execute_asset                                              │
+         │                                                             │
+         │    79 │   │   │   │   input_kwargs["context"] = context     │
+         │    80 │   │   │                                             │
+         │    81 │   │   │   if asyncio.iscoroutinefunction(asset_func │
+         │ ❱  82 │   │   │   │   output = await asset_func(**input_kwa │
+         │    83 │   │   │   else:                                     │
+         │    84 │   │   │   │   # Run sync function in a thread pool  │
+         │    85 │   │   │   │   loop = asyncio.get_running_loop()     │
+         │                                                             │
+         │ /Users/kh03/work/repos/myflow/example.py:31 in              │
+         │ failing_asset                                               │
+         │                                                             │
+         │   28 @asset(deps=["users"])                                 │
+         │   29 async def failing_asset(users: list[str]):             │
+         │   30 │   """This asset always fails."""                     │
+         │ ❱ 31 │   raise ValueError("This asset is designed to fail." │
+         │   32                                                        │
+         │   33                                                        │
+         │   34 if __name__ == "__main__":                             │
+         ╰─────────────────────────────────────────────────────────────╯
+         ValueError: This asset is designed to fail.                    
+╭─────────────────────────────── Assets ───────────────────────────────╮
+│ ✓ users                          (0.00s)                             │
+│ ✓ greetings                      (0.00s)                             │
+│ ✗ failing_asset                  (0.04s)                             │
+╰──────────────────────────────────────────────────────────────────────╯
+Overall Progress ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3/3 0:00:00
+
+```

kazeflow-0.1.0a0/README.md

@@ -0,0 +1,106 @@
+
+# kazeflow
+
+`kazeflow` is a lightweight, asset-based task flow engine inspired by Dagster. It is designed to be simple, flexible, and easy to use.
+
+## Example
+
+When you run this script, `kazeflow` will execute the assets in the correct order, handle the failure of `failing_asset` gracefully, and provide a rich terminal UI to visualize the progress.
+
+
+Here is a simple example of how to use `kazeflow` to define and execute a data flow with dependencies, inputs/outputs, and logging.
+
+
+example.py:
+```python
+import asyncio
+
+from kazeflow.assets import asset
+from kazeflow.context import AssetContext
+from kazeflow.flow import Flow
+
+
+# A simple asset with no dependencies
+@asset()
+async def users() -> list[str]:
+    """This asset returns a list of user names."""
+    return ["Alice", "Bob", "Charlie"]
+
+
+# This asset depends on the `users` asset.
+# The output of `users` is automatically passed as an argument.
+@asset(deps=["users"])
+async def greetings(users: list[str], context: AssetContext) -> list[str]:
+    """This asset receives the list of users and a context object.
+
+    It uses the context to get a logger and log a message.
+    """
+    context.logger.info(f"Generating greetings for {len(users)} users.")
+    return [f"Hello, {user}!" for user in users]
+
+
+# This asset fails intentionally to demonstrate error handling.
+@asset(deps=["users"])
+async def failing_asset(users: list[str]):
+    """This asset always fails."""
+    raise ValueError("This asset is designed to fail.")
+
+
+if __name__ == "__main__":
+    # Define a flow that includes the final assets we want to generate.
+    # kazeflow automatically includes all upstream dependencies.
+    flow = Flow(asset_names=["greetings", "failing_asset"])
+
+    # Run the flow asynchronously.
+    # You can limit the number of concurrent assets with `max_concurrency`.
+    asyncio.run(flow.run_async(max_concurrency=2))
+
+```
+```bash
+❯ uv run example.py
+Task Flow (Execution Order)
+└── users
+    ├── failing_asset
+    └── greetings
+
+Execution Logs
+INFO     Executing asset: users                                         
+INFO     Finished executing asset: users in 0.00s                       
+INFO     Executing asset: greetings                                     
+INFO     Generating greetings for 3 users.                              
+INFO     Finished executing asset: greetings in 0.00s                   
+INFO     Executing asset: failing_asset                                 
+ERROR    Error executing asset failing_asset: This asset is designed to 
+         fail.                                                          
+         ╭───────────── Traceback (most recent call last) ─────────────╮
+         │ /Users/kh03/work/repos/myflow/src/kazeflow/flow.py:82 in    │
+         │ _execute_asset                                              │
+         │                                                             │
+         │    79 │   │   │   │   input_kwargs["context"] = context     │
+         │    80 │   │   │                                             │
+         │    81 │   │   │   if asyncio.iscoroutinefunction(asset_func │
+         │ ❱  82 │   │   │   │   output = await asset_func(**input_kwa │
+         │    83 │   │   │   else:                                     │
+         │    84 │   │   │   │   # Run sync function in a thread pool  │
+         │    85 │   │   │   │   loop = asyncio.get_running_loop()     │
+         │                                                             │
+         │ /Users/kh03/work/repos/myflow/example.py:31 in              │
+         │ failing_asset                                               │
+         │                                                             │
+         │   28 @asset(deps=["users"])                                 │
+         │   29 async def failing_asset(users: list[str]):             │
+         │   30 │   """This asset always fails."""                     │
+         │ ❱ 31 │   raise ValueError("This asset is designed to fail." │
+         │   32                                                        │
+         │   33                                                        │
+         │   34 if __name__ == "__main__":                             │
+         ╰─────────────────────────────────────────────────────────────╯
+         ValueError: This asset is designed to fail.                    
+╭─────────────────────────────── Assets ───────────────────────────────╮
+│ ✓ users                          (0.00s)                             │
+│ ✓ greetings                      (0.00s)                             │
+│ ✗ failing_asset                  (0.04s)                             │
+╰──────────────────────────────────────────────────────────────────────╯
+Overall Progress ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3/3 0:00:00
+
+```

kazeflow-0.1.0a0/pyproject.toml

@@ -0,0 +1,28 @@
+[project]
+name = "kazeflow"
+version = "0.1.0a0"
+description = "A lightweight, asset-based task flow engine"
+readme = "README.md"
+authors = [
+    { name = "Kj", email = "46800811+kj-9@users.noreply.github.com" }
+]
+requires-python = ">=3.10"
+dependencies = [
+    "rich>=14.1.0",
+]
+
+[build-system]
+requires = ["uv_build>=0.8.17,<0.9.0"]
+build-backend = "uv_build"
+
+
+[tool.ruff.format]
+quote-style = "double"
+
+[dependency-groups]
+dev = [
+    "ruff>=0.13.0",
+    "pytest>=8.4.2",
+    "ty>=0.0.1a20",
+    "pytest-asyncio>=0.24.0",
+]

kazeflow-0.1.0a0/src/kazeflow/assets.py

@@ -0,0 +1,48 @@
+from typing import Any, Optional, Protocol, TypedDict
+
+
+class NamedCallable(Protocol):
+    __name__: str
+
+    def __call__(self, *args: Any, **kwargs: Any) -> Any: ...
+
+
+class AssetMeta(TypedDict):
+    func: NamedCallable
+    deps: list[str]
+
+
+_assets: dict[str, AssetMeta] = {}
+
+
+def asset(
+    deps: Optional[list[str]] = None,
+):
+    """
+    A decorator to define an asset, its dependencies, and its configuration schema.
+    """
+
+    def decorator(func: NamedCallable) -> NamedCallable:
+        _assets[func.__name__] = {
+            "func": func,
+            "deps": deps or [],
+        }
+        return func
+
+    return decorator
+
+
+def get_asset(name: str) -> AssetMeta:
+    """Retrieves an asset's metadata including its function, dependencies,
+    and config schema."""
+    if name not in _assets:
+        raise ValueError(f"Asset '{name}' not found.")
+    return _assets[name]
+
+
+def clear_assets() -> None:
+    """Clears all registered assets.
+
+    This is useful for testing purposes.
+    """
+    _assets.clear()

kazeflow-0.1.0a0/src/kazeflow/context.py

@@ -0,0 +1,10 @@
+import logging
+from dataclasses import dataclass
+
+
+@dataclass
+class AssetContext:
+    """Holds contextual information for an asset's execution."""
+
+    logger: logging.Logger
+    asset_name: str

kazeflow-0.1.0a0/src/kazeflow/flow.py

@@ -0,0 +1,156 @@
+import asyncio
+import inspect
+import logging
+import time
+from graphlib import TopologicalSorter
+from typing import Any, Optional
+
+from .assets import get_asset
+from .context import AssetContext
+from .result import AssetResult
+from .tui import FlowTUIRenderer, show_flow_tree
+
+
+class Flow:
+    """A class representing a workflow of assets."""
+
+    def __init__(self, asset_names: list[str]):
+        self.asset_names = asset_names
+        self.asset_outputs: dict[str, Any] = {}
+
+        self.graph = self._get_graph()
+
+        ts = self._get_ts()
+        self.static_order = list(ts.static_order())
+
+    def _get_graph(self) -> dict[str, set[str]]:
+        """Sets up the graph based on asset dependencies."""
+        graph: dict[str, set[str]] = {}
+
+        queue = list(self.asset_names)
+        visited = set()
+
+        while queue:
+            asset_name = queue.pop(0)
+            if asset_name in visited:
+                continue
+            visited.add(asset_name)
+
+            asset = get_asset(asset_name)
+            deps = set(asset["deps"])
+            graph[asset_name] = deps
+
+            for dep in deps:
+                queue.append(dep)
+
+        return graph
+
+    def _get_ts(self) -> TopologicalSorter:
+        """Sets up the topological sorter based on asset dependencies."""
+
+        return TopologicalSorter(self.graph)
+
+    async def _execute_asset(
+        self, asset_name: str, logger: logging.Logger
+    ) -> AssetResult:
+        """Executes an asset and returns a result object."""
+        start_time = time.monotonic()
+        output = None
+        exception = None
+        success = False
+        try:
+            logger.info(f"Executing asset: {asset_name}")
+            asset = get_asset(asset_name)
+
+            asset_func = asset["func"]
+            deps = asset["deps"]
+
+            # Only pass outputs that are actual parameters of the asset function
+            sig = inspect.signature(asset_func)
+            params = sig.parameters
+            input_kwargs = {
+                dep: self.asset_outputs[dep]
+                for dep in deps
+                if dep in self.asset_outputs and dep in params
+            }
+
+            if "context" in params:
+                context = AssetContext(logger=logger, asset_name=asset_name)
+                input_kwargs["context"] = context
+
+            if asyncio.iscoroutinefunction(asset_func):
+                output = await asset_func(**input_kwargs)
+            else:
+                # Run sync function in a thread pool executor
+                loop = asyncio.get_running_loop()
+                import functools
+
+                p = functools.partial(asset_func, **input_kwargs)
+                output = await loop.run_in_executor(None, p)
+            success = True
+
+        except Exception as e:
+            exception = e
+            logger.exception(f"Error executing asset {asset_name}: {e}")
+
+        duration = time.monotonic() - start_time
+        if success:
+            logger.info(f"Finished executing asset: {asset_name} in {duration:.2f}s")
+
+        return AssetResult(
+            name=asset_name,
+            success=success,
+            duration=duration,
+            start_time=start_time,
+            output=output,
+            exception=exception,
+        )
+
+    async def run_async(
+        self,
+        config: Optional[dict[str, Any]] = None,
+        max_concurrency: Optional[int] = None,
+    ) -> None:
+        """Executes the assets in the flow asynchronously with a concurrency limit."""
+        if max_concurrency is not None and max_concurrency <= 0:
+            raise ValueError("max_concurrency must be a positive integer or None.")
+
+        show_flow_tree(self.graph)
+        ts = self._get_ts()
+        ts.prepare()
+
+        tui = FlowTUIRenderer(total_assets=len(self.static_order))
+        running_tasks_map: dict[asyncio.Task, tuple[str, int]] = {}
+
+        with tui:
+            while ts.is_active():
+                ready_to_run = list(ts.get_ready())
+
+                limit = max_concurrency if max_concurrency is not None else float("inf")
+
+                # Start new tasks if we have capacity and there are tasks ready to run
+                while len(running_tasks_map) < limit and ready_to_run:
+                    asset_name = ready_to_run.pop(0)
+                    progress_task_id = tui.add_running_task(asset_name)
+                    async_task = asyncio.create_task(
+                        self._execute_asset(asset_name, tui.logger)
+                    )
+                    running_tasks_map[async_task] = (asset_name, progress_task_id)
+
+                if not running_tasks_map:
+                    break  # Nothing running, nothing new to start
+
+                done, _ = await asyncio.wait(
+                    running_tasks_map.keys(), return_when=asyncio.FIRST_COMPLETED
+                )
+
+                for task in done:
+                    asset_name, progress_task_id = running_tasks_map.pop(task)
+
+                    asset_result: AssetResult = task.result()
+
+                    tui.complete_running_task(progress_task_id, asset_result)
+
+                    if asset_result.success:
+                        self.asset_outputs[asset_name] = asset_result.output
+                        ts.done(asset_name)

kazeflow-0.1.0a0/src/kazeflow/graph.py

@@ -0,0 +1,29 @@
+from rich.console import Console
+from rich.panel import Panel
+
+console = Console()
+
+
+def draw_simple_dag():
+    # ノード
+    a = Panel("A", style="green", expand=False)
+    b = Panel("B", style="yellow", expand=False)
+    c = Panel("C", style="yellow", expand=False)
+    d = Panel("D", style="grey50", expand=False)
+
+    lines = [
+        f"    {a}",
+        "      │",
+        "      ▼",
+        f"{b}   {c}",
+        "   │     │",
+        "   └──┬──┘",
+        "      ▼",
+        f"    {d}",
+    ]
+
+    for line in lines:
+        console.print(line)
+
+
+draw_simple_dag()

kazeflow-0.1.0a0/src/kazeflow/logger.py

@@ -0,0 +1,23 @@
+import logging
+from typing import Optional
+
+from rich.console import Console
+from rich.logging import RichHandler
+
+
+def get_logger(
+    name: str = __name__, console: Optional[Console] = None
+) -> logging.Logger:
+    """Get a logger that uses rich for pretty printing."""
+    logger = logging.getLogger(name)
+    # Clear existing handlers to allow for reconfiguration
+    if logger.hasHandlers():
+        logger.handlers.clear()
+
+    logger.setLevel(logging.INFO)
+    handler = RichHandler(
+        show_path=False, rich_tracebacks=True, console=console, show_time=False
+    )
+    logger.addHandler(handler)
+    logger.propagate = False
+    return logger

kazeflow-0.1.0a0/src/kazeflow/play.py

@@ -0,0 +1,89 @@
+import asyncio
+import time
+from graphlib import TopologicalSorter
+
+
+async def task(name, delay=1, records: list = []):
+    start = time.time()
+    print(f"start {name}")
+    await asyncio.sleep(delay)
+    end = time.time()
+    print(f"done {name}")
+
+    records.append((name, start, end))
+    return name
+
+
+dag = {
+    "A": [],
+    "B": ["A"],
+    "C": ["A"],
+    "D": ["B", "C"],
+    "E": ["C"],
+    "F": ["C"],
+}
+
+
+async def run_tasks():
+    ts = TopologicalSorter(dag)
+    ts.prepare()
+
+    records = []
+    running = {
+        asyncio.create_task(task(n, delay=1 + len(n), records=records)): n
+        for n in ts.get_ready()
+    }
+
+    while running:
+        done, _ = await asyncio.wait(
+            running.keys(), return_when=asyncio.FIRST_COMPLETED
+        )
+
+        for d in done:
+            name = running.pop(d)
+            ts.done(name)
+            for new in ts.get_ready():
+                running[
+                    asyncio.create_task(task(new, delay=1 + len(new), records=records))
+                ] = new
+
+    return records
+
+
+def text_gantt(records):
+    base = min(start for _, start, _ in records)
+    end_time = max(end for _, _, end in records)
+    length = int(end_time - base) + 1  # 秒単位
+
+    tasks = sorted(set(name for name, _, _ in records))
+
+    lines = []
+    for name in tasks:
+        line = [" "] * length
+        for n, start, end in records:
+            if n == name:
+                s = int(start - base)
+                e = int(end - base)
+                for i in range(s, e):
+                    line[i] = "#"
+        lines.append(f"{name:>3} |{''.join(line)}|")
+    return "\n".join(lines)
+
+
+def text_graph(dag):
+    lines = []
+    for node, deps in dag.items():
+        if deps:
+            for dep in deps:
+                lines.append(f"{dep} --> {node}")
+        else:
+            lines.append(f"{node}")
+    return "\n".join(lines)
+
+
+if __name__ == "__main__":
+    print("\nDependency Graph (text-based):")
+    print(text_graph(dag))
+    records = asyncio.run(run_tasks())
+    print("\nGantt Chart (text-based):")
+    print(text_gantt(records))

kazeflow-0.1.0a0/src/kazeflow/result.py

@@ -0,0 +1,14 @@
+from dataclasses import dataclass
+from typing import Any, Optional
+
+
+@dataclass
+class AssetResult:
+    """Holds the result of a single asset's execution."""
+
+    name: str
+    success: bool
+    duration: float
+    start_time: float
+    output: Optional[Any] = None
+    exception: Optional[Exception] = None

kazeflow-0.1.0a0/src/kazeflow/tui.py

@@ -0,0 +1,116 @@
+from typing import Any, Dict, Set
+
+from rich.console import Console, Group
+from rich.live import Live
+from rich.panel import Panel
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    Progress,
+    SpinnerColumn,
+    TextColumn,
+    TimeElapsedColumn,
+)
+from rich.tree import Tree
+
+from .logger import get_logger
+from .result import AssetResult
+
+
+def show_flow_tree(graph: Dict[str, Set[str]]) -> None:
+    """Displays the task flow as a rich tree, in execution order."""
+
+    # Reverse the graph to show data flow from dependencies to dependents
+    reversed_graph = {node: set() for node in graph}
+    for node, deps in graph.items():
+        for dep in deps:
+            if dep in reversed_graph:
+                reversed_graph[dep].add(node)
+
+    # Find root nodes (assets with no dependencies)
+    root_nodes = [node for node, deps in graph.items() if not deps]
+
+    tree = Tree("[bold green]Task Flow (Execution Order)[/bold green]")
+    added_nodes = set()
+
+    def add_to_tree(parent_tree: Tree, node_name: str) -> None:
+        if node_name in added_nodes:
+            return
+        added_nodes.add(node_name)
+        node_tree = parent_tree.add(node_name)
+        # Use reversed_graph to find nodes that depend on the current one
+        for dependent_node in sorted(list(reversed_graph.get(node_name, []))):
+            add_to_tree(node_tree, dependent_node)
+
+    for root in sorted(root_nodes):
+        add_to_tree(tree, root)
+
+    Console().print(tree)
+
+
+class FlowTUIRenderer:
+    """Manages the Text-based User Interface for flow execution using rich."""
+
+    def __init__(self, total_assets: int):
+        self.completed_progress = Progress(TextColumn("✓ [green]{task.description}"))
+        self.failed_progress = Progress(TextColumn("✗ [red]{task.description}"))
+        self.running_progress = Progress(
+            TextColumn("  [purple]Running: {task.description}"),
+            SpinnerColumn("simpleDots"),
+            TimeElapsedColumn(),
+        )
+        self.overall_progress = Progress(
+            TextColumn("[bold blue]Overall Progress"),
+            BarColumn(),
+            MofNCompleteColumn(),
+            TimeElapsedColumn(),
+        )
+        self.progress_group = Group(
+            Panel(
+                Group(
+                    self.completed_progress,
+                    self.failed_progress,
+                    self.running_progress,
+                ),
+                title="Assets",
+            ),
+            self.overall_progress,
+        )
+        self.overall_task_id = self.overall_progress.add_task(
+            "Assets", total=total_assets
+        )
+        self.live = Live(self.progress_group)
+        self.logger = get_logger(__name__, console=self.live.console)
+        self.results: list[AssetResult] = []
+
+    def __enter__(self) -> Live:
+        Console().print("\n[bold underline green]Execution Logs[/bold underline green]")
+        return self.live.__enter__()
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        # Sort results by start time before final render
+        self.results.sort(key=lambda r: r.start_time)
+
+        # Now, populate the final progress bars
+        for result in self.results:
+            description = f"{result.name:<30} ({result.duration:.2f}s)"
+            if result.success:
+                self.completed_progress.add_task(description)
+            else:
+                self.failed_progress.add_task(description)
+
+        return self.live.__exit__(exc_type, exc_val, exc_tb)
+
+    def add_running_task(self, name: str) -> int:
+        """Adds a task to the running progress bar."""
+        return self.running_progress.add_task(name, total=1)
+
+    def complete_running_task(self, task_id: int, result: AssetResult) -> None:
+        """Moves a task from running to a temporary list for later sorting."""
+        self.running_progress.stop_task(task_id)
+        self.running_progress.update(task_id, visible=False)
+
+        # Store the result for final sorting
+        self.results.append(result)
+
+        self.overall_progress.update(self.overall_task_id, advance=1)