pyoco 0.3.0__tar.gz → 0.5.1__tar.gz

{pyoco-0.3.0 → pyoco-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyoco
-Version: 0.3.0
+Version: 0.5.1
 Summary: A workflow engine with sugar syntax
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -8,6 +8,7 @@ Requires-Dist: pyyaml>=6.0.3
 Requires-Dist: fastapi>=0.100.0
 Requires-Dist: uvicorn>=0.20.0
 Requires-Dist: httpx>=0.24.0
+Requires-Dist: prometheus-client>=0.20.0
 
 # 🐇 Pyoco
 
@@ -132,6 +133,20 @@ Or via CLI flag:
 pyoco run --non-cute ...
 ```
 
+## 🔭 Observability Bridge (v0.5)
+
+- `/metrics` exposes Prometheus counters (`pyoco_runs_total`, `pyoco_runs_in_progress`) and histograms (`pyoco_task_duration_seconds`, `pyoco_run_duration_seconds`). Point Grafana/Prometheus at it to watch pipelines without opening sockets.
+- `/runs` now accepts `status`, `flow`, `limit` query params; `/runs/{id}/logs?tail=100` fetches only the latest snippets for dashboards.
+- Webhook notifications fire when runs COMPLETE/FAIL—configure via `PYOCO_WEBHOOK_*` env vars and forward to Slack or your alerting stack.
+- Import `docs/grafana_pyoco_cute.json` for a lavender/orange starter dashboard (3 panels: in-progress count, completion trend, per-flow latency).
+- 詳細な手順は [docs/observability.md](docs/observability.md) を参照してください。
+
+## 🧩 Plug-ins
+
+Need to share domain-specific tasks? Publish an entry point under `pyoco.tasks` and pyoco will auto-load it. In v0.5.1 we recommend **Task subclasses first** (callables still work with warnings). See [docs/plugins.md](docs/plugins.md) for examples, quickstart, and `pyoco plugins list` / `pyoco plugins lint`.
+
+**Big data note:** pass handles, not copies. For large tensors/images, stash paths or handles in `ctx.artifacts`/`ctx.scratch` and let downstream tasks materialize only when needed. For lazy pipelines (e.g., DataPipe), log the pipeline when you actually iterate (typically the training task) instead of materializing upstream.
+
 ## 📚 Documentation
 
 - [Tutorials](docs/tutorial/index.md)

{pyoco-0.3.0 → pyoco-0.5.1}/README.md

@@ -121,6 +121,20 @@ Or via CLI flag:
 pyoco run --non-cute ...
 ```
 
+## 🔭 Observability Bridge (v0.5)
+
+- `/metrics` exposes Prometheus counters (`pyoco_runs_total`, `pyoco_runs_in_progress`) and histograms (`pyoco_task_duration_seconds`, `pyoco_run_duration_seconds`). Point Grafana/Prometheus at it to watch pipelines without opening sockets.
+- `/runs` now accepts `status`, `flow`, `limit` query params; `/runs/{id}/logs?tail=100` fetches only the latest snippets for dashboards.
+- Webhook notifications fire when runs COMPLETE/FAIL—configure via `PYOCO_WEBHOOK_*` env vars and forward to Slack or your alerting stack.
+- Import `docs/grafana_pyoco_cute.json` for a lavender/orange starter dashboard (3 panels: in-progress count, completion trend, per-flow latency).
+- 詳細な手順は [docs/observability.md](docs/observability.md) を参照してください。
+
+## 🧩 Plug-ins
+
+Need to share domain-specific tasks? Publish an entry point under `pyoco.tasks` and pyoco will auto-load it. In v0.5.1 we recommend **Task subclasses first** (callables still work with warnings). See [docs/plugins.md](docs/plugins.md) for examples, quickstart, and `pyoco plugins list` / `pyoco plugins lint`.
+
+**Big data note:** pass handles, not copies. For large tensors/images, stash paths or handles in `ctx.artifacts`/`ctx.scratch` and let downstream tasks materialize only when needed. For lazy pipelines (e.g., DataPipe), log the pipeline when you actually iterate (typically the training task) instead of materializing upstream.
+
 ## 📚 Documentation
 
 - [Tutorials](docs/tutorial/index.md)

{pyoco-0.3.0 → pyoco-0.5.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pyoco"
-version = "0.3.0"
+version = "0.5.1"
 description = "A workflow engine with sugar syntax"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -9,4 +9,12 @@ dependencies = [
     "fastapi>=0.100.0",
     "uvicorn>=0.20.0",
     "httpx>=0.24.0",
+    "prometheus-client>=0.20.0",
+]
+
+[dependency-groups]
+dev = [
+    "pytest>=9.0.1",
+    "socketless-http>=0.1.0",
+    "pytest-asyncio>=0.23.0",
 ]

{pyoco-0.3.0 → pyoco-0.5.1}/src/pyoco/cli/main.py

@@ -1,7 +1,10 @@
 import argparse
+import json
 import sys
 import os
 import signal
+import time
+from types import SimpleNamespace
 from ..schemas.config import PyocoConfig
 from ..discovery.loader import TaskLoader
 from ..core.models import Flow
@@ -28,6 +31,8 @@ def main():
     check_parser = subparsers.add_parser("check", help="Verify a workflow")
     check_parser.add_argument("--config", required=True, help="Path to flow.yaml")
     check_parser.add_argument("--flow", default="main", help="Flow name to check")
+    check_parser.add_argument("--dry-run", action="store_true", help="Traverse flow without executing tasks")
+    check_parser.add_argument("--json", action="store_true", help="Output report as JSON")
 
     # List tasks command
     list_parser = subparsers.add_parser("list-tasks", help="List available tasks")
@@ -55,6 +60,8 @@ def main():
     runs_list = runs_subparsers.add_parser("list", help="List runs")
     runs_list.add_argument("--server", default="http://localhost:8000", help="Server URL")
     runs_list.add_argument("--status", help="Filter by status")
+    runs_list.add_argument("--flow", help="Filter by flow name")
+    runs_list.add_argument("--limit", type=int, help="Maximum number of runs to show")
     
     runs_show = runs_subparsers.add_parser("show", help="Show run details")
     runs_show.add_argument("run_id", help="Run ID")
@@ -63,6 +70,26 @@ def main():
     runs_cancel = runs_subparsers.add_parser("cancel", help="Cancel a run")
     runs_cancel.add_argument("run_id", help="Run ID")
     runs_cancel.add_argument("--server", default="http://localhost:8000", help="Server URL")
+    
+    runs_inspect = runs_subparsers.add_parser("inspect", help="Inspect run details")
+    runs_inspect.add_argument("run_id", help="Run ID")
+    runs_inspect.add_argument("--server", default="http://localhost:8000", help="Server URL")
+    runs_inspect.add_argument("--json", action="store_true", help="Output JSON payload")
+    
+    runs_logs = runs_subparsers.add_parser("logs", help="Show run logs")
+    runs_logs.add_argument("run_id", help="Run ID")
+    runs_logs.add_argument("--server", default="http://localhost:8000", help="Server URL")
+    runs_logs.add_argument("--task", help="Filter logs by task")
+    runs_logs.add_argument("--tail", type=int, help="Show last N log entries")
+    runs_logs.add_argument("--follow", action="store_true", help="Stream logs until completion")
+    runs_logs.add_argument("--allow-failure", action="store_true", help="Don't exit non-zero when run failed")
+
+    plugins_parser = subparsers.add_parser("plugins", help="Inspect plug-in entry points")
+    plugins_sub = plugins_parser.add_subparsers(dest="plugins_command")
+    plugins_list = plugins_sub.add_parser("list", help="List discovered plug-ins")
+    plugins_list.add_argument("--json", action="store_true", help="Output JSON payload")
+    plugins_lint = plugins_sub.add_parser("lint", help="Validate plug-ins for upcoming requirements")
+    plugins_lint.add_argument("--json", action="store_true", help="Output JSON payload")
 
     args = parser.parse_args()
 
@@ -94,6 +121,55 @@ def main():
             print(f" - {name}")
         return
 
+    if args.command == "plugins":
+        reports = _collect_plugin_reports()
+        if args.plugins_command == "list":
+            if getattr(args, "json", False):
+                print(json.dumps(reports, indent=2))
+            else:
+                if not reports:
+                    print("No plug-ins registered under group 'pyoco.tasks'.")
+                else:
+                    print("Discovered plug-ins:")
+                    for info in reports:
+                        mod = info.get("module") or info.get("value")
+                        print(f" - {info.get('name')} ({mod})")
+                        if info.get("error"):
+                            print(f"     ⚠️  error: {info['error']}")
+                            continue
+                        for task in info.get("tasks", []):
+                            warn_msg = "; ".join(task.get("warnings", [])) or "ok"
+                            print(f"     • {task['name']} [{task['origin']}] ({warn_msg})")
+                        for warn in info.get("warnings", []):
+                            print(f"     ⚠️  {warn}")
+        elif args.plugins_command == "lint":
+            issues = []
+            for info in reports:
+                prefix = info["name"]
+                if info.get("error"):
+                    issues.append(f"{prefix}: {info['error']}")
+                    continue
+                for warn in info.get("warnings", []):
+                    issues.append(f"{prefix}: {warn}")
+                for task in info.get("tasks", []):
+                    for warn in task.get("warnings", []):
+                        issues.append(f"{prefix}.{task['name']}: {warn}")
+            payload = {"issues": issues, "reports": reports}
+            if getattr(args, "json", False):
+                print(json.dumps(payload, indent=2))
+            else:
+                if not issues:
+                    print("✅ All plug-ins look good.")
+                else:
+                    print("⚠️  Plug-in issues found:")
+                    for issue in issues:
+                        print(f" - {issue}")
+            if issues:
+                sys.exit(1)
+        else:
+            plugins_parser.print_help()
+        return
+
     if args.command == "server":
         if args.server_command == "start":
             import uvicorn
@@ -113,7 +189,7 @@ def main():
         client = Client(args.server)
         try:
             if args.runs_command == "list":
-                runs = client.list_runs(status=args.status)
+                runs = client.list_runs(status=args.status, flow=args.flow, limit=args.limit)
                 print(f"🐇 Active Runs ({len(runs)}):")
                 print(f"{'ID':<36} | {'Status':<12} | {'Flow':<15}")
                 print("-" * 70)
@@ -134,6 +210,31 @@ def main():
             elif args.runs_command == "cancel":
                 client.cancel_run(args.run_id)
                 print(f"🛑 Cancellation requested for run {args.run_id}")
+            elif args.runs_command == "inspect":
+                run = client.get_run(args.run_id)
+                if args.json:
+                    print(json.dumps(run, indent=2))
+                else:
+                    print(f"🐇 Run: {run['run_id']} ({run.get('flow_name', 'n/a')})")
+                    print(f"Status: {run['status']}")
+                    if run.get("start_time"):
+                        print(f"Started: {run['start_time']}")
+                    if run.get("end_time"):
+                        print(f"Ended: {run['end_time']}")
+                    print("Tasks:")
+                    records = run.get("task_records", {})
+                    for name, info in records.items():
+                        state = info.get("state", run["tasks"].get(name))
+                        duration = info.get("duration_ms")
+                        duration_str = f"{duration:.2f} ms" if duration else "-"
+                        print(f"  - {name}: {state} ({duration_str})")
+                        if info.get("error"):
+                            print(f"      error: {info['error']}")
+                    if not records:
+                        for t_name, t_state in run.get("tasks", {}).items():
+                            print(f"  - {t_name}: {t_state}")
+            elif args.runs_command == "logs":
+                _stream_logs(client, args)
         except Exception as e:
             print(f"Error: {e}")
         return
@@ -164,14 +265,16 @@ def main():
                 sys.exit(1)
             return
         # Build Flow from graph string
-        from ..dsl.syntax import TaskWrapper
+        from ..dsl.syntax import TaskWrapper, switch
         eval_context = {name: TaskWrapper(task) for name, task in loader.tasks.items()}
+        eval_context["switch"] = switch
         
         try:
             # Create Flow and add all loaded tasks
             flow = Flow(name=args.flow)
             for t in loader.tasks.values():
                 flow.add_task(t)
+            eval_context["flow"] = flow
 
             # Evaluate graph to set up dependencies
             exec(flow_conf.graph, {}, eval_context)
@@ -210,36 +313,36 @@ def main():
 
         # 1. Check imports (already done by loader.load(), but we can check for missing tasks in graph)
         # 2. Build flow to check graph
-        from ..dsl.syntax import TaskWrapper
+        from ..dsl.syntax import TaskWrapper, switch
         eval_context = {name: TaskWrapper(task) for name, task in loader.tasks.items()}
+        eval_context["switch"] = switch
         
         try:
             flow = Flow(name=args.flow)
             for t in loader.tasks.values():
                 flow.add_task(t)
+            eval_context["flow"] = flow
             
             eval(flow_conf.graph, {}, eval_context)
             
             # 3. Reachability / Orphans
-            # Nodes with no deps and no dependents (except if single node flow)
             if len(flow.tasks) > 1:
                 for t in flow.tasks:
                     if not t.dependencies and not t.dependents:
                         warnings.append(f"Task '{t.name}' is orphaned (no dependencies or dependents).")
 
             # 4. Cycles
-            # Simple DFS for cycle detection
             visited = set()
             path = set()
+
             def visit(node):
                 if node in path:
-                    return True # Cycle
+                    return True
                 if node in visited:
                     return False
-                
                 visited.add(node)
                 path.add(node)
-                for dep in node.dependencies: # Check upstream
+                for dep in node.dependencies:
                     if visit(dep):
                         return True
                 path.remove(node)
@@ -255,29 +358,85 @@ def main():
             for t in flow.tasks:
                 sig = inspect.signature(t.func)
                 for name, param in sig.parameters.items():
-                    if name == 'ctx': continue
-                    # Check if input provided in task config or defaults
-                    # This is hard because inputs are resolved at runtime.
-                    # But we can check if 'inputs' mapping exists for it.
+                    if name == 'ctx': 
+                        continue
                     if name not in t.inputs and name not in flow_conf.defaults:
-                        # Warning: might be missing input
                         warnings.append(f"Task '{t.name}' argument '{name}' might be missing input (not in inputs or defaults).")
 
         except Exception as e:
             errors.append(f"Graph evaluation failed: {e}")
 
-        # Report
-        print("\n--- Check Report ---")
-        if not errors and not warnings:
-            print("✅ All checks passed!")
+        if args.dry_run:
+            from ..dsl.validator import FlowValidator
+            try:
+                validator = FlowValidator(flow)
+                dr_report = validator.validate()
+                warnings.extend(dr_report.warnings)
+                errors.extend(dr_report.errors)
+            except Exception as exc:
+                print(f"❌ Dry run internal error: {exc}")
+                import traceback
+                traceback.print_exc()
+                sys.exit(3)
+
+        status = "ok"
+        if errors:
+            status = "error"
+        elif warnings:
+            status = "warning"
+
+        report = {"status": status, "warnings": warnings, "errors": errors}
+
+        if args.json:
+            print(json.dumps(report, indent=2))
         else:
-            for w in warnings:
-                print(f"⚠️  {w}")
-            for e in errors:
-                print(f"❌ {e}")
-            
-            if errors:
+            print("\n--- Check Report ---")
+            print(f"Status: {status}")
+            if not errors and not warnings:
+                print("✅ All checks passed!")
+            else:
+                for w in warnings:
+                    print(f"⚠️  {w}")
+                for e in errors:
+                    print(f"❌ {e}")
+        
+        if errors:
+            sys.exit(2 if args.dry_run else 1)
+        return
+
+def _collect_plugin_reports():
+    dummy = SimpleNamespace(
+        tasks={},
+        discovery=SimpleNamespace(entry_points=[], packages=[], glob_modules=[]),
+    )
+    loader = TaskLoader(dummy)
+    loader.load()
+    return loader.plugin_reports
+
+
+def _stream_logs(client, args):
+    seen_seq = -1
+    follow = args.follow
+    while True:
+        tail = args.tail if (args.tail and seen_seq == -1 and not follow) else None
+        data = client.get_run_logs(args.run_id, task=args.task, tail=tail)
+        logs = data.get("logs", [])
+        logs.sort(key=lambda entry: entry.get("seq", 0))
+        for entry in logs:
+            seq = entry.get("seq", 0)
+            if seq <= seen_seq:
+                continue
+            line = entry.get("text", "")
+            line = line.rstrip("\n")
+            print(f"[{entry.get('task', 'unknown')}][{entry.get('stream', '')}] {line}")
+            seen_seq = seq
+        status = data.get("run_status", "UNKNOWN")
+        if not follow or status in ("COMPLETED", "FAILED", "CANCELLED"):
+            if status == "FAILED" and not args.allow_failure:
                 sys.exit(1)
+            break
+        time.sleep(1)
+
 
 if __name__ == "__main__":
     main()

{pyoco-0.3.0 → pyoco-0.5.1}/src/pyoco/client.py

@@ -1,6 +1,6 @@
 import httpx
 from typing import Dict, List, Optional, Any
-from .core.models import RunStatus, TaskState
+from .core.models import RunStatus, TaskState, RunContext
 
 class Client:
     def __init__(self, server_url: str, client_id: str = "cli"):
@@ -17,10 +17,19 @@ class Client:
         resp.raise_for_status()
         return resp.json()["run_id"]
 
-    def list_runs(self, status: Optional[str] = None) -> List[Dict]:
+    def list_runs(
+        self,
+        status: Optional[str] = None,
+        flow: Optional[str] = None,
+        limit: Optional[int] = None,
+    ) -> List[Dict]:
         params = {}
         if status:
             params["status"] = status
+        if flow:
+            params["flow"] = flow
+        if limit:
+            params["limit"] = limit
         resp = self.client.get("/runs", params=params)
         resp.raise_for_status()
         return resp.json()
@@ -49,21 +58,32 @@ class Client:
             # print(f"Poll failed: {e}")
             return None
 
-    def heartbeat(self, run_id: str, task_states: Dict[str, TaskState], run_status: RunStatus) -> bool:
+    def heartbeat(self, run_ctx: RunContext) -> bool:
         """
         Sends heartbeat. Returns True if cancellation is requested.
         """
         try:
-            # Convert Enums to values
-            states_json = {k: v.value if hasattr(v, 'value') else v for k, v in task_states.items()}
-            status_value = run_status.value if hasattr(run_status, 'value') else run_status
-            
-            resp = self.client.post(f"/runs/{run_id}/heartbeat", json={
+            states_json = {k: v.value if hasattr(v, 'value') else v for k, v in run_ctx.tasks.items()}
+            status_value = run_ctx.status.value if hasattr(run_ctx.status, 'value') else run_ctx.status
+            payload = {
                 "task_states": states_json,
+                "task_records": run_ctx.serialize_task_records(),
+                "logs": run_ctx.drain_logs(),
                 "run_status": status_value
-            })
+            }
+            resp = self.client.post(f"/runs/{run_ctx.run_id}/heartbeat", json=payload)
             resp.raise_for_status()
             return resp.json().get("cancel_requested", False)
         except Exception as e:
             print(f"Heartbeat failed: {e}")
             return False
+
+    def get_run_logs(self, run_id: str, task: Optional[str] = None, tail: Optional[int] = None) -> Dict[str, Any]:
+        params = {}
+        if task:
+            params["task"] = task
+        if tail:
+            params["tail"] = tail
+        resp = self.client.get(f"/runs/{run_id}/logs", params=params)
+        resp.raise_for_status()
+        return resp.json()

{pyoco-0.3.0 → pyoco-0.5.1}/src/pyoco/core/context.py

@@ -1,8 +1,46 @@
 import threading
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Sequence
 from dataclasses import dataclass, field
 from .models import RunContext
 
+
+@dataclass
+class LoopFrame:
+    name: str
+    type: str
+    index: Optional[int] = None
+    iteration: Optional[int] = None
+    count: Optional[int] = None
+    item: Any = None
+    condition: Optional[bool] = None
+    path: Optional[str] = None
+
+
+class LoopStack:
+    def __init__(self):
+        self._frames: List[LoopFrame] = []
+
+    def push(self, frame: LoopFrame) -> LoopFrame:
+        parent_path = self._frames[-1].path if self._frames else ""
+        segment = frame.name
+        if frame.index is not None:
+            segment = f"{segment}[{frame.index}]"
+        frame.path = f"{parent_path}.{segment}" if parent_path else segment
+        self._frames.append(frame)
+        return frame
+
+    def pop(self) -> LoopFrame:
+        if not self._frames:
+            raise RuntimeError("Loop stack underflow")
+        return self._frames.pop()
+
+    @property
+    def current(self) -> Optional[LoopFrame]:
+        return self._frames[-1] if self._frames else None
+
+    def snapshot(self) -> Sequence[LoopFrame]:
+        return tuple(self._frames)
+
 @dataclass
 class Context:
     """
@@ -14,11 +52,13 @@ class Context:
     artifacts: Dict[str, Any] = field(default_factory=dict)
     env: Dict[str, str] = field(default_factory=dict)
     artifact_dir: Optional[str] = None
+    _vars: Dict[str, Any] = field(default_factory=dict, repr=False)
     
     # Reference to the parent run context (v0.2.0+)
     run_context: Optional[RunContext] = None
     
     _lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
+    _loop_stack: LoopStack = field(default_factory=LoopStack, repr=False)
 
     @property
     def is_cancelled(self) -> bool:
@@ -27,6 +67,29 @@ class Context:
             return self.run_context.status in [RunStatus.CANCELLING, RunStatus.CANCELLED]
         return False
 
+    @property
+    def loop(self) -> Optional[LoopFrame]:
+        return self._loop_stack.current
+
+    @property
+    def loops(self) -> Sequence[LoopFrame]:
+        return self._loop_stack.snapshot()
+
+    def push_loop(self, frame: LoopFrame) -> LoopFrame:
+        return self._loop_stack.push(frame)
+
+    def pop_loop(self) -> LoopFrame:
+        return self._loop_stack.pop()
+
+    def set_var(self, name: str, value: Any):
+        self._vars[name] = value
+
+    def get_var(self, name: str, default=None):
+        return self._vars.get(name, default)
+
+    def clear_var(self, name: str):
+        self._vars.pop(name, None)
+
     def __post_init__(self):
         # Ensure artifact directory exists
         if self.artifact_dir is None:
@@ -124,3 +187,20 @@ class Context:
 
         return value
 
+    def expression_data(self) -> Dict[str, Any]:
+        data: Dict[str, Any] = {}
+        data.update(self._vars)
+        data["params"] = self.params
+        data["results"] = self.results
+        data["scratch"] = self.scratch
+        data["artifacts"] = self.artifacts
+        data["loop"] = self.loop
+        data["loops"] = list(self.loops)
+        return data
+
+    def env_data(self) -> Dict[str, str]:
+        import os
+
+        env_data = dict(os.environ)
+        env_data.update(self.env)
+        return env_data