pyoco 0.1.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

pyoco/server/store.py

@@ -0,0 +1,82 @@
+import uuid
+import time
+from typing import Dict, List, Optional
+from ..core.models import RunContext, RunStatus
+
+class StateStore:
+    def __init__(self):
+        self.runs: Dict[str, RunContext] = {}
+        self.queue: List[str] = []
+        
+    def create_run(self, flow_name: str, params: Dict) -> RunContext:
+        run_id = str(uuid.uuid4())
+        run_ctx = RunContext(
+            run_id=run_id,
+            status=RunStatus.PENDING,
+            start_time=time.time()
+        )
+        # Store extra metadata if needed (flow_name, params)
+        # For now, RunContext doesn't have flow_name/params fields in core.models.
+        # We might need to extend RunContext or store them separately.
+        # Let's attach them dynamically for now or assume the worker knows.
+        # Actually, the worker needs flow_name and params to run.
+        # We should store them in the store alongside the context.
+        run_ctx.flow_name = flow_name
+        run_ctx.params = params
+        
+        self.runs[run_id] = run_ctx
+        self.queue.append(run_id)
+        return run_ctx
+
+    def get_run(self, run_id: str) -> Optional[RunContext]:
+        return self.runs.get(run_id)
+
+    def list_runs(self) -> List[RunContext]:
+        return list(self.runs.values())
+
+    def dequeue(self, tags: List[str] = None) -> Optional[RunContext]:
+        # Simple FIFO queue for now. Tags ignored in v0.3.0 MVP.
+        if not self.queue:
+            return None
+            
+        # Find first PENDING run
+        # Note: queue might contain cancelled runs?
+        # We should check status.
+        
+        # Pop from front
+        # We iterate to find a valid candidate
+        for i, run_id in enumerate(self.queue):
+            run = self.runs.get(run_id)
+            if run and run.status == RunStatus.PENDING:
+                self.queue.pop(i)
+                return run
+                
+        return None
+
+    def update_run(self, run_id: str, status: RunStatus = None, task_states: Dict = None):
+        run = self.runs.get(run_id)
+        if not run:
+            return
+            
+        if status:
+            # State transition check
+            # If server has CANCELLING, ignore RUNNING from worker
+            if run.status == RunStatus.CANCELLING and status == RunStatus.RUNNING:
+                pass
+            else:
+                run.status = status
+                
+            if status in [RunStatus.COMPLETED, RunStatus.FAILED, RunStatus.CANCELLED]:
+                if not run.end_time:
+                    run.end_time = time.time()
+                    
+        if task_states:
+            run.tasks.update(task_states)
+
+    def cancel_run(self, run_id: str):
+        run = self.runs.get(run_id)
+        if not run:
+            return
+            
+        if run.status in [RunStatus.PENDING, RunStatus.RUNNING]:
+            run.status = RunStatus.CANCELLING

pyoco/trace/backend.py

@@ -3,7 +3,7 @@ from typing import Any, Optional
 
 class TraceBackend(ABC):
     @abstractmethod
-    def on_flow_start(self, flow_name: str):
+    def on_flow_start(self, flow_name: str, run_id: str = None):
         pass
 
     @abstractmethod

pyoco/trace/console.py

@@ -2,14 +2,22 @@ import time
 from .backend import TraceBackend
 
 class ConsoleTraceBackend(TraceBackend):
-    def __init__(self, style: str = "cute"):
+    def __init__(self, style: str = None):
+        if style is None:
+            import os
+            env_cute = os.environ.get("PYOCO_CUTE", "true").lower()
+            if env_cute in ["0", "false", "no", "off"]:
+                style = "plain"
+            else:
+                style = "cute"
         self.style = style
 
-    def on_flow_start(self, flow_name: str):
+    def on_flow_start(self, flow_name: str, run_id: str = None):
+        rid_str = f" run_id={run_id}" if run_id else ""
         if self.style == "cute":
-            print(f"🐇 pyoco > start flow={flow_name}")
+            print(f"🐇 pyoco > start flow={flow_name}{rid_str}")
         else:
-            print(f"INFO pyoco start flow={flow_name}")
+            print(f"INFO pyoco start flow={flow_name}{rid_str}")
 
     def on_flow_end(self, flow_name: str):
         if self.style == "cute":

pyoco/worker/client.py

@@ -0,0 +1,43 @@
+import httpx
+from typing import Dict, List, Optional, Any
+from ..core.models import RunStatus, TaskState
+
+class WorkerClient:
+    def __init__(self, server_url: str, worker_id: str):
+        self.server_url = server_url.rstrip("/")
+        self.worker_id = worker_id
+        self.client = httpx.Client(base_url=self.server_url)
+
+    def poll(self, tags: List[str] = []) -> Optional[Dict[str, Any]]:
+        try:
+            resp = self.client.post("/workers/poll", json={
+                "worker_id": self.worker_id,
+                "tags": tags
+            })
+            resp.raise_for_status()
+            data = resp.json()
+            if data.get("run_id"):
+                return data
+            return None
+        except Exception as e:
+            print(f"Poll failed: {e}")
+            return None
+
+    def heartbeat(self, run_id: str, task_states: Dict[str, TaskState], run_status: RunStatus) -> bool:
+        """
+        Sends heartbeat. Returns True if cancellation is requested.
+        """
+        try:
+            # Convert Enums to values
+            states_json = {k: v.value for k, v in task_states.items()}
+            status_value = run_status.value
+            
+            resp = self.client.post(f"/runs/{run_id}/heartbeat", json={
+                "task_states": states_json,
+                "run_status": status_value
+            })
+            resp.raise_for_status()
+            return resp.json().get("cancel_requested", False)
+        except Exception as e:
+            print(f"Heartbeat failed: {e}")
+            return False

pyoco/worker/runner.py

@@ -0,0 +1,171 @@
+import time
+import uuid
+from typing import List, Optional
+from ..core.engine import Engine
+from ..core.models import RunContext, RunStatus, Flow
+from ..trace.backend import TraceBackend
+from ..discovery.loader import TaskLoader
+from ..schemas.config import PyocoConfig
+from ..client import Client
+
+from ..trace.console import ConsoleTraceBackend
+
+class RemoteTraceBackend(TraceBackend):
+    def __init__(self, client: Client, run_ctx: RunContext, cute: bool = True):
+        self.client = client
+        self.run_ctx = run_ctx
+        self.last_heartbeat = 0
+        self.heartbeat_interval = 1.0 # sec
+        self.console = ConsoleTraceBackend(style="cute" if cute else "plain")
+
+    def _send_heartbeat(self, force=False):
+        now = time.time()
+        if force or (now - self.last_heartbeat > self.heartbeat_interval):
+            cancel = self.client.heartbeat(
+                self.run_ctx.run_id,
+                self.run_ctx.tasks,
+                self.run_ctx.status
+            )
+            if cancel and self.run_ctx.status not in [RunStatus.CANCELLING, RunStatus.CANCELLED]:
+                print(f"🛑 Cancellation requested from server for run {self.run_ctx.run_id}")
+                self.run_ctx.status = RunStatus.CANCELLING
+            self.last_heartbeat = now
+
+    def on_flow_start(self, name: str, run_id: Optional[str] = None):
+        self.console.on_flow_start(name, run_id)
+        self._send_heartbeat(force=True)
+
+    def on_flow_end(self, name: str):
+        self.console.on_flow_end(name)
+        self._send_heartbeat(force=True)
+
+    def on_node_start(self, node_name: str):
+        self.console.on_node_start(node_name)
+        self._send_heartbeat()
+
+    def on_node_end(self, node_name: str, duration: float):
+        self.console.on_node_end(node_name, duration)
+        self._send_heartbeat(force=True)
+
+    def on_node_error(self, node_name: str, error: Exception):
+        self.console.on_node_error(node_name, error)
+        self._send_heartbeat(force=True)
+
+
+class Worker:
+    def __init__(self, server_url: str, config: PyocoConfig, tags: List[str] = []):
+        self.server_url = server_url
+        self.config = config
+        self.tags = tags
+        self.worker_id = f"w-{uuid.uuid4().hex[:8]}"
+        self.client = Client(server_url, self.worker_id)
+        self.loader = TaskLoader(config)
+        self.loader.load() # Load tasks/flows once
+
+    def start(self):
+        print(f"🐜 Worker {self.worker_id} started. Connected to {self.server_url}")
+        try:
+            while True:
+                job = self.client.poll(self.tags)
+                if job:
+                    self._execute_job(job)
+                else:
+                    time.sleep(2.0)
+        except KeyboardInterrupt:
+            print("\n🛑 Worker stopping...")
+
+    def _execute_job(self, job):
+        run_id = job["run_id"]
+        flow_name = job["flow_name"]
+        params = job["params"] or {}
+        
+        print(f"🚀 Received job: {run_id} (Flow: {flow_name})")
+        
+        # Find flow
+        flow_def = self.config.flows.get(flow_name)
+        if not flow_def:
+            print(f"❌ Flow '{flow_name}' not found in local config.")
+            return
+
+        # Build Flow object using exec (same as main.py)
+        from ..core.models import Flow as FlowModel
+        from ..dsl.syntax import TaskWrapper
+        
+        eval_context = {name: TaskWrapper(task) for name, task in self.loader.tasks.items()}
+        
+        try:
+            flow = FlowModel(name=flow_name)
+            for t in self.loader.tasks.values():
+                flow.add_task(t)
+            
+            # Evaluate graph
+            exec(flow_def.graph, {}, eval_context)
+            
+        except Exception as e:
+            print(f"❌ Error building flow: {e}")
+            return
+        
+        # Execute
+        engine = Engine()
+        
+        # We need to inject run_id into Engine.
+        # Engine.run generates run_id if not provided.
+        # We need to pass it.
+        # Engine.run doesn't accept run_id argument currently.
+        # It creates RunContext inside.
+        # I need to modify Engine.run to accept optional run_id or RunContext.
+        
+        # Wait, I modified Engine.run in v0.2.0.
+        # Let's check Engine.run signature.
+        pass 
+        
+        # I will modify Engine.run to accept run_id.
+        # For now, let's assume I will do that.
+        
+        # Create RemoteTraceBackend
+        # But we need run_ctx to create backend.
+        # And Engine creates run_ctx.
+        # Chicken and egg.
+        
+        # Solution: Engine.run should accept an existing RunContext or run_id.
+        # If I pass run_id, Engine creates RunContext with that ID.
+        # Then I can access it?
+        # Or I pass a callback to get it?
+        
+        # Better: Pass run_id to Engine.run.
+        # Engine creates RunContext.
+        # Engine calls trace.on_flow_start(run_id=...).
+        # RemoteTraceBackend receives run_id.
+        # But RemoteTraceBackend needs to know which run_id to report to (it knows from constructor).
+        # Actually, RemoteTraceBackend needs access to the RunContext object that Engine creates.
+        # Because it reads `run_ctx.tasks` and `run_ctx.status`.
+        
+        # If Engine creates RunContext internally, we can't pass it to Backend beforehand.
+        # Unless Engine exposes it.
+        
+        # Alternative:
+        # Modify Engine to accept `run_context` argument.
+        # If provided, use it.
+        
+        # I will modify Engine.run to accept `run_context`.
+        
+        run_ctx = RunContext(run_id=run_id, status=RunStatus.RUNNING)
+        # Initialize tasks as PENDING? Engine does that.
+        
+        backend = RemoteTraceBackend(self.client, run_ctx)
+        engine.trace = backend # Replace default console trace? Or chain?
+        # Maybe chain if we want local logs too.
+        # For now, just replace or use MultiBackend (not implemented).
+        # Let's just use RemoteBackend.
+        
+        try:
+            engine.run(flow, params=params, run_context=run_ctx)
+            print(f"✅ Job {run_id} completed: {run_ctx.status}")
+            # Send final heartbeat
+            self.client.heartbeat(run_id, run_ctx.tasks, run_ctx.status)
+        except Exception as e:
+            print(f"💥 Job {run_id} failed: {e}")
+            # Heartbeat one last time
+            run_ctx.status = RunStatus.FAILED
+            self.client.heartbeat(run_id, run_ctx.tasks, run_ctx.status)
+

pyoco-0.3.0.dist-info/METADATA

@@ -0,0 +1,146 @@
+Metadata-Version: 2.4
+Name: pyoco
+Version: 0.3.0
+Summary: A workflow engine with sugar syntax
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: fastapi>=0.100.0
+Requires-Dist: uvicorn>=0.20.0
+Requires-Dist: httpx>=0.24.0
+
+# 🐇 Pyoco
+
+**pyoco is a minimal, pure-Python DAG engine for defining and running simple task-based workflows.**
+
+## Overview
+
+Pyoco is designed to be significantly smaller, lighter, and have fewer dependencies than full-scale workflow engines like Airflow. It is optimized for local development and single-machine execution.
+
+You can define tasks and their dependencies entirely in Python code using decorators and a simple API. There is no need for complex configuration files or external databases.
+
+It is ideal for small jobs, development environments, and personal projects where a full-stack workflow engine would be overkill.
+
+## ✨ Features
+
+- **Pure Python**: No external services or heavy dependencies required.
+- **Minimal DAG model**: Tasks and dependencies are defined directly in code.
+- **Task-oriented**: Focus on "small workflows" that should be easy to read and maintain.
+- **Friendly trace logs**: Runs can be traced step by step from the terminal with cute (or plain) logs.
+- **Parallel Execution**: Automatically runs independent tasks in parallel.
+- **Artifact Management**: Easily save and manage task outputs and files.
+- **Observability**: Track execution with unique Run IDs and detailed state transitions.
+- **Control**: Cancel running workflows gracefully with `Ctrl+C`.
+
+## 📦 Installation
+
+```bash
+pip install pyoco
+```
+
+## 🚀 Usage
+
+Here is a minimal example of a pure-Python workflow.
+
+```python
+from pyoco import task
+from pyoco.core.models import Flow
+from pyoco.core.engine import Engine
+
+@task
+def fetch_data(ctx):
+    print("🐰 Fetching data...")
+    return {"id": 1, "value": "carrot"}
+
+@task
+def process_data(ctx, data):
+    print(f"🥕 Processing: {data['value']}")
+    return data['value'].upper()
+
+@task
+def save_result(ctx, result):
+    print(f"✨ Saved: {result}")
+
+# Define the flow
+flow = Flow(name="hello_pyoco")
+flow >> fetch_data >> process_data >> save_result
+
+# Wire inputs (explicitly for this example)
+process_data.task.inputs = {"data": "$node.fetch_data.output"}
+save_result.task.inputs = {"result": "$node.process_data.output"}
+
+if __name__ == "__main__":
+    engine = Engine()
+    engine.run(flow)
+```
+
+Run it:
+
+```bash
+python examples/hello_pyoco.py
+```
+
+Output:
+
+```
+🐇 pyoco > start flow=hello_pyoco
+🏃 start node=fetch_data
+🐰 Fetching data...
+✅ done node=fetch_data (0.30 ms)
+🏃 start node=process_data
+🥕 Processing: carrot
+✅ done node=process_data (0.23 ms)
+🏃 start node=save_result
+✨ Saved: CARROT
+✅ done node=save_result (0.30 ms)
+🥕 done flow=hello_pyoco
+```
+
+See [examples/hello_pyoco.py](examples/hello_pyoco.py) for the full code.
+
+## 🏗️ Architecture
+
+Pyoco is designed with a simple flow:
+
+```
++-----------+        +------------------+        +-----------------+
+| User Code |  --->  | pyoco.core.Flow  |  --->  | trace/logger    |
+| (Tasks)   |        | (Engine)         |        | (Console/File)  |
++-----------+        +------------------+        +-----------------+
+```
+
+1. **User Code**: You define tasks and flows using Python decorators.
+2. **Core Engine**: The engine resolves dependencies and executes tasks (in parallel where possible).
+3. **Trace**: Execution events are sent to the trace backend for logging (cute or plain).
+
+## 🎭 Modes
+
+Pyoco has two output modes:
+
+- **Cute Mode** (Default): Uses emojis and friendly messages. Best for local development and learning.
+- **Non-Cute Mode**: Plain text logs. Best for CI/CD and production monitoring.
+
+You can switch modes using an environment variable:
+
+```bash
+export PYOCO_CUTE=0  # Disable cute mode
+```
+
+Or via CLI flag:
+
+```bash
+pyoco run --non-cute ...
+```
+
+## 📚 Documentation
+
+- [Tutorials](docs/tutorial/index.md)
+- [Roadmap](docs/roadmap.md)
+
+## 💖 Contributing
+
+We love contributions! Please feel free to submit a Pull Request.
+
+---
+
+*Made with 🥕 by the Pyoco Team.*

pyoco-0.3.0.dist-info/RECORD

@@ -0,0 +1,25 @@
+pyoco/__init__.py,sha256=E2pgDGvGRSVon7dSqIM4UD55LgVpf4jiZZA-70kOcuw,409
+pyoco/client.py,sha256=0PFC-1WaIhKHLLVMUXx9hCADsNklBjXRrQS7b9IrFnc,2498
+pyoco/cli/entry.py,sha256=zPIG0Gx-cFO8Cf1Z3wD3Ifz_2sHaryHZ6mCRri2WEqE,93
+pyoco/cli/main.py,sha256=kGrgxh7s4I7JBJzC1CUxxt528ZtCy_Z0PfpLvN4axpM,11315
+pyoco/core/base_task.py,sha256=z7hOFntAPv4yCADapS-fhtLe5eWqaO8k3T1r05YEEUE,2106
+pyoco/core/context.py,sha256=khAXRLhkwYtZVxgJIFMuQshFDyDF-wqAKi2-dLELw20,4173
+pyoco/core/engine.py,sha256=OlQtpbdEESHGoaVHbe0TY0ScxvQ15KrkLHylkEyb0fo,17099
+pyoco/core/models.py,sha256=SXsv13DxrfUUB-ChLyMXVIqw-CmGqacPgxVWybPcCzo,5708
+pyoco/discovery/loader.py,sha256=HzTs4zf2UO1zsPQ2lU5vfGXZx5hwZ_ti50xGPVfeCuI,4931
+pyoco/dsl/__init__.py,sha256=xWdb60pSRL8lNFk4GHF3EJ4hon0uiWqpv264g6-4gdg,45
+pyoco/dsl/syntax.py,sha256=AkFcD5gLlbJLFN0KkMIyttpHUV3v21pjz_ZqwreZkdM,4312
+pyoco/schemas/config.py,sha256=KkGZK3GxTHoIHEGb4f4k8GE2W-aBN4iPzmc_HrwuROU,1735
+pyoco/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pyoco/server/api.py,sha256=v66kRzFHxE874ZaAtOjFtJCFLBkQzE9EjfQpfI2aDuI,2592
+pyoco/server/models.py,sha256=tsdF8ShDkW6ajVV_oq9w8rUM3SbH6nbkAT0khELyZoI,712
+pyoco/server/store.py,sha256=S02LbMN5sQzSJuvAb4nN7rqqs4G1BwipAdNQGELEd50,2888
+pyoco/trace/backend.py,sha256=a1css94_lhO4SGSPHZ1f59HJqFQtZ5Sjx09Kw7v5bsk,617
+pyoco/trace/console.py,sha256=I-BcF405OGLWoacJWeke8vTT9M5JxSBpJL-NazVyxb4,1742
+pyoco/worker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pyoco/worker/client.py,sha256=862KccXRtfG7zd9ZSLqrpVSV6ev8zeuEHHdtAfLghiM,1557
+pyoco/worker/runner.py,sha256=8Z__t54cFZmbouLFtUBNmItC_w6FBCNF7Ey-OgafdTg,6538
+pyoco-0.3.0.dist-info/METADATA,sha256=u5afgdDSCKgbK6LEbgZGFM1wafznJRHVOoesZ4ZR_Us,4140
+pyoco-0.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pyoco-0.3.0.dist-info/top_level.txt,sha256=2JRVocfaWRbX1VJ3zq1c5wQaOK6fMARS6ptVFWyvRF4,6
+pyoco-0.3.0.dist-info/RECORD,,

pyoco-0.1.0.dist-info/METADATA

@@ -1,7 +0,0 @@
-Metadata-Version: 2.4
-Name: pyoco
-Version: 0.1.0
-Summary: A workflow engine with sugar syntax
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-Requires-Dist: pyyaml>=6.0.3

pyoco-0.1.0.dist-info/RECORD

@@ -1,17 +0,0 @@
-pyoco/__init__.py,sha256=E2pgDGvGRSVon7dSqIM4UD55LgVpf4jiZZA-70kOcuw,409
-pyoco/cli/entry.py,sha256=zPIG0Gx-cFO8Cf1Z3wD3Ifz_2sHaryHZ6mCRri2WEqE,93
-pyoco/cli/main.py,sha256=uRc6CzUTVRYF4JbehlbrprT7GvWQ-WyBZ8k12NrSxO8,6502
-pyoco/core/base_task.py,sha256=z7hOFntAPv4yCADapS-fhtLe5eWqaO8k3T1r05YEEUE,2106
-pyoco/core/context.py,sha256=SnoTz3vRghO1A-FNOrw2NEjbx1HySDqrBnQU5-KWGbk,3696
-pyoco/core/engine.py,sha256=m5LrEsXcpUAran5DxULtWbvhsMNj5mv17wE6lDFkFmQ,11416
-pyoco/core/models.py,sha256=zTt5HTSBChwRpOuw3qY2pvjRGZVsq4OQ-ZBHE3ujMWA,4548
-pyoco/discovery/loader.py,sha256=XzZzOAyFYrdA8K6APuEGWgjSIyp4Bgwlr834MyJc8vk,4950
-pyoco/dsl/__init__.py,sha256=xWdb60pSRL8lNFk4GHF3EJ4hon0uiWqpv264g6-4gdg,45
-pyoco/dsl/syntax.py,sha256=AkFcD5gLlbJLFN0KkMIyttpHUV3v21pjz_ZqwreZkdM,4312
-pyoco/schemas/config.py,sha256=KkGZK3GxTHoIHEGb4f4k8GE2W-aBN4iPzmc_HrwuROU,1735
-pyoco/trace/backend.py,sha256=h7l1PU8zuCSOo_VA5T1ax4znN_Az3Xuvx-KXibg3e-U,597
-pyoco/trace/console.py,sha256=Kf2-vma98ojhVQZHFzCUYfD_46Lr1WfAfI56smZkSZM,1397
-pyoco-0.1.0.dist-info/METADATA,sha256=bA_qJXUkIiC7TIOSo8CEzJ6PXp01pLQ1Q1LoMOrIw_k,187
-pyoco-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pyoco-0.1.0.dist-info/top_level.txt,sha256=2JRVocfaWRbX1VJ3zq1c5wQaOK6fMARS6ptVFWyvRF4,6
-pyoco-0.1.0.dist-info/RECORD,,