pyoco 0.5.0__tar.gz → 0.5.1__tar.gz

{pyoco-0.5.0 → pyoco-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyoco
-Version: 0.5.0
+Version: 0.5.1
 Summary: A workflow engine with sugar syntax
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -143,7 +143,9 @@ pyoco run --non-cute ...
 
 ## 🧩 Plug-ins
 
-Need to share domain-specific tasks? Publish an entry point under `pyoco.tasks` and pyoco will auto-load it. See [docs/plugins.md](docs/plugins.md) for the `PluginRegistry` decorator, example `pyproject.toml`, and `pyoco plugins list` CLI helper.
+Need to share domain-specific tasks? Publish an entry point under `pyoco.tasks` and pyoco will auto-load it. In v0.5.1 we recommend **Task subclasses first** (callables still work with warnings). See [docs/plugins.md](docs/plugins.md) for examples, quickstart, and `pyoco plugins list` / `pyoco plugins lint`.
+
+**Big data note:** pass handles, not copies. For large tensors/images, stash paths or handles in `ctx.artifacts`/`ctx.scratch` and let downstream tasks materialize only when needed. For lazy pipelines (e.g., DataPipe), log the pipeline when you actually iterate (typically the training task) instead of materializing upstream.
 
 ## 📚 Documentation
 

{pyoco-0.5.0 → pyoco-0.5.1}/README.md

@@ -131,7 +131,9 @@ pyoco run --non-cute ...
 
 ## 🧩 Plug-ins
 
-Need to share domain-specific tasks? Publish an entry point under `pyoco.tasks` and pyoco will auto-load it. See [docs/plugins.md](docs/plugins.md) for the `PluginRegistry` decorator, example `pyproject.toml`, and `pyoco plugins list` CLI helper.
+Need to share domain-specific tasks? Publish an entry point under `pyoco.tasks` and pyoco will auto-load it. In v0.5.1 we recommend **Task subclasses first** (callables still work with warnings). See [docs/plugins.md](docs/plugins.md) for examples, quickstart, and `pyoco plugins list` / `pyoco plugins lint`.
+
+**Big data note:** pass handles, not copies. For large tensors/images, stash paths or handles in `ctx.artifacts`/`ctx.scratch` and let downstream tasks materialize only when needed. For lazy pipelines (e.g., DataPipe), log the pipeline when you actually iterate (typically the training task) instead of materializing upstream.
 
 ## 📚 Documentation
 

{pyoco-0.5.0 → pyoco-0.5.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pyoco"
-version = "0.5.0"
+version = "0.5.1"
 description = "A workflow engine with sugar syntax"
 readme = "README.md"
 requires-python = ">=3.10"

{pyoco-0.5.0 → pyoco-0.5.1}/src/pyoco/cli/main.py

@@ -4,13 +4,13 @@ import sys
 import os
 import signal
 import time
+from types import SimpleNamespace
 from ..schemas.config import PyocoConfig
 from ..discovery.loader import TaskLoader
 from ..core.models import Flow
 from ..core.engine import Engine
 from ..trace.console import ConsoleTraceBackend
 from ..client import Client
-from ..discovery.plugins import list_available_plugins
 
 def main():
     parser = argparse.ArgumentParser(description="Pyoco Workflow Engine")
@@ -88,6 +88,8 @@ def main():
     plugins_sub = plugins_parser.add_subparsers(dest="plugins_command")
     plugins_list = plugins_sub.add_parser("list", help="List discovered plug-ins")
     plugins_list.add_argument("--json", action="store_true", help="Output JSON payload")
+    plugins_lint = plugins_sub.add_parser("lint", help="Validate plug-ins for upcoming requirements")
+    plugins_lint.add_argument("--json", action="store_true", help="Output JSON payload")
 
     args = parser.parse_args()
 
@@ -120,18 +122,52 @@ def main():
         return
 
     if args.command == "plugins":
-        infos = list_available_plugins()
+        reports = _collect_plugin_reports()
         if args.plugins_command == "list":
             if getattr(args, "json", False):
-                print(json.dumps(infos, indent=2))
+                print(json.dumps(reports, indent=2))
             else:
-                if not infos:
+                if not reports:
                     print("No plug-ins registered under group 'pyoco.tasks'.")
                 else:
                     print("Discovered plug-ins:")
-                    for info in infos:
+                    for info in reports:
                         mod = info.get("module") or info.get("value")
                         print(f" - {info.get('name')} ({mod})")
+                        if info.get("error"):
+                            print(f"     ⚠️  error: {info['error']}")
+                            continue
+                        for task in info.get("tasks", []):
+                            warn_msg = "; ".join(task.get("warnings", [])) or "ok"
+                            print(f"     • {task['name']} [{task['origin']}] ({warn_msg})")
+                        for warn in info.get("warnings", []):
+                            print(f"     ⚠️  {warn}")
+        elif args.plugins_command == "lint":
+            issues = []
+            for info in reports:
+                prefix = info["name"]
+                if info.get("error"):
+                    issues.append(f"{prefix}: {info['error']}")
+                    continue
+                for warn in info.get("warnings", []):
+                    issues.append(f"{prefix}: {warn}")
+                for task in info.get("tasks", []):
+                    for warn in task.get("warnings", []):
+                        issues.append(f"{prefix}.{task['name']}: {warn}")
+            payload = {"issues": issues, "reports": reports}
+            if getattr(args, "json", False):
+                print(json.dumps(payload, indent=2))
+            else:
+                if not issues:
+                    print("✅ All plug-ins look good.")
+                else:
+                    print("⚠️  Plug-in issues found:")
+                    for issue in issues:
+                        print(f" - {issue}")
+            if issues:
+                sys.exit(1)
+        else:
+            plugins_parser.print_help()
         return
 
     if args.command == "server":
@@ -368,6 +404,16 @@ def main():
             sys.exit(2 if args.dry_run else 1)
         return
 
+def _collect_plugin_reports():
+    dummy = SimpleNamespace(
+        tasks={},
+        discovery=SimpleNamespace(entry_points=[], packages=[], glob_modules=[]),
+    )
+    loader = TaskLoader(dummy)
+    loader.load()
+    return loader.plugin_reports
+
+
 def _stream_logs(client, args):
     seen_seq = -1
     follow = args.follow

{pyoco-0.5.0 → pyoco-0.5.1}/src/pyoco/discovery/loader.py

@@ -109,6 +109,7 @@ class TaskLoader:
                 "value": ep.value,
                 "module": getattr(ep, "module", ""),
                 "tasks": [],
+                "warnings": [],
             }
             registry = PluginRegistry(self, ep.name)
             try:
@@ -116,7 +117,10 @@ class TaskLoader:
                 if not callable(hook):
                     raise TypeError("Entry point must be callable")
                 hook(registry)
-                info["tasks"] = list(registry.registered_names)
+                info["tasks"] = list(registry.records)
+                info["warnings"] = list(registry.warnings)
+                if not registry.records:
+                    info["warnings"].append("no tasks registered")
             except Exception as exc:
                 info["error"] = str(exc)
                 if self.strict:

pyoco-0.5.1/src/pyoco/discovery/plugins.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+from importlib import metadata as importlib_metadata
+from typing import Any, Callable, Dict, List, Optional, Type
+
+from ..core.models import Task
+from ..dsl.syntax import TaskWrapper
+
+
+class CallablePluginTask(Task):
+    """Lightweight subclass so callable registrations still appear as Task-derived."""
+
+    def __init__(self, func: Callable, name: str):
+        super().__init__(func=func, name=name)
+
+
+def iter_entry_points(group: str = "pyoco.tasks"):
+    eps = importlib_metadata.entry_points()
+    if hasattr(eps, "select"):
+        return list(eps.select(group=group))
+    return list(eps.get(group, []))
+
+
+def list_available_plugins() -> List[Dict[str, Any]]:
+    plugins = []
+    for ep in iter_entry_points():
+        plugins.append(
+            {
+                "name": ep.name,
+                "module": getattr(ep, "module", ""),
+                "value": ep.value,
+            }
+        )
+    return plugins
+
+
+class PluginRegistry:
+    def __init__(self, loader: Any, provider_name: str) -> None:
+        self.loader = loader
+        self.provider_name = provider_name
+        self.registered_names: List[str] = []
+        self.records: List[Dict[str, Any]] = []
+        self.warnings: List[str] = []
+
+    def task(
+        self,
+        func: Optional[Callable] = None,
+        *,
+        name: Optional[str] = None,
+        inputs: Optional[Dict[str, Any]] = None,
+        outputs: Optional[List[str]] = None,
+    ):
+        if func is not None:
+            self.register_callable(
+                func,
+                name=name,
+                inputs=inputs or {},
+                outputs=outputs or [],
+            )
+            return func
+
+        def decorator(inner: Callable):
+            self.register_callable(
+                inner,
+                name=name,
+                inputs=inputs or {},
+                outputs=outputs or [],
+            )
+            return inner
+
+        return decorator
+
+    def register_callable(
+        self,
+        func: Callable,
+        *,
+        name: Optional[str] = None,
+        inputs: Optional[Dict[str, Any]] = None,
+        outputs: Optional[List[str]] = None,
+    ) -> Task:
+        task_name = name or getattr(func, "__name__", f"{self.provider_name}_task")
+        task = CallablePluginTask(func=func, name=task_name)
+        if inputs:
+            task.inputs.update(inputs)
+        if outputs:
+            task.outputs.extend(outputs)
+        self._finalize_task(task, origin="callable")
+        return task
+
+    def task_class(
+        self,
+        task_cls: Type[Task],
+        *args: Any,
+        name: Optional[str] = None,
+        **kwargs: Any,
+    ) -> Task:
+        if not issubclass(task_cls, Task):
+            raise TypeError(f"{task_cls} is not a Task subclass")
+        task = task_cls(*args, **kwargs)
+        if name:
+            task.name = name
+        self._finalize_task(task, origin="task_class")
+        return task
+
+    def add(self, obj: Any, *, name: Optional[str] = None) -> None:
+        if isinstance(obj, TaskWrapper):
+            task = obj.task
+            if name:
+                task.name = name
+            self._finalize_task(task, origin="wrapper")
+        elif isinstance(obj, Task):
+            if name:
+                obj.name = name
+            origin = "task_class" if obj.__class__ is not Task else "task"
+            self._finalize_task(obj, origin=origin)
+        elif callable(obj):
+            self.register_callable(obj, name=name)
+        else:
+            raise TypeError(f"Unsupported task object: {obj!r}")
+
+    def _finalize_task(self, task: Task, origin: str) -> None:
+        warnings = self._validate_task(task, origin)
+        self.loader._register_task(task.name, task)
+        self.registered_names.append(task.name)
+        self.records.append(
+            {
+                "name": task.name,
+                "origin": origin,
+                "class": task.__class__.__name__,
+                "warnings": warnings,
+            }
+        )
+        for msg in warnings:
+            self.warnings.append(f"{task.name}: {msg}")
+
+    def _validate_task(self, task: Task, origin: str) -> List[str]:
+        warnings: List[str] = []
+        if not getattr(task, "name", None):
+            generated = f"{self.provider_name}_{len(self.registered_names) + 1}"
+            task.name = generated
+            warnings.append(f"name missing; auto-assigned '{generated}'")
+        if not callable(getattr(task, "func", None)):
+            warnings.append("task.func is not callable")
+        if origin == "callable":
+            warnings.append("registered via callable; prefer Task subclass for extensibility")
+        if task.__class__ is Task and origin not in ("callable", "wrapper"):
+            warnings.append("plain Task instance detected; subclass Task for metadata support")
+        return warnings

{pyoco-0.5.0 → pyoco-0.5.1}/src/pyoco.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyoco
-Version: 0.5.0
+Version: 0.5.1
 Summary: A workflow engine with sugar syntax
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -143,7 +143,9 @@ pyoco run --non-cute ...
 
 ## 🧩 Plug-ins
 
-Need to share domain-specific tasks? Publish an entry point under `pyoco.tasks` and pyoco will auto-load it. See [docs/plugins.md](docs/plugins.md) for the `PluginRegistry` decorator, example `pyproject.toml`, and `pyoco plugins list` CLI helper.
+Need to share domain-specific tasks? Publish an entry point under `pyoco.tasks` and pyoco will auto-load it. In v0.5.1 we recommend **Task subclasses first** (callables still work with warnings). See [docs/plugins.md](docs/plugins.md) for examples, quickstart, and `pyoco plugins list` / `pyoco plugins lint`.
+
+**Big data note:** pass handles, not copies. For large tensors/images, stash paths or handles in `ctx.artifacts`/`ctx.scratch` and let downstream tasks materialize only when needed. For lazy pipelines (e.g., DataPipe), log the pipeline when you actually iterate (typically the training task) instead of materializing upstream.
 
 ## 📚 Documentation
 

pyoco-0.5.0/src/pyoco/discovery/plugins.py

@@ -1,92 +0,0 @@
-from __future__ import annotations
-
-from importlib import metadata as importlib_metadata
-from typing import Any, Callable, Dict, List, Optional
-
-from ..core.models import Task
-from ..dsl.syntax import TaskWrapper
-
-
-def iter_entry_points(group: str = "pyoco.tasks"):
-    eps = importlib_metadata.entry_points()
-    if hasattr(eps, "select"):
-        return list(eps.select(group=group))
-    return list(eps.get(group, []))
-
-
-def list_available_plugins() -> List[Dict[str, Any]]:
-    plugins = []
-    for ep in iter_entry_points():
-        plugins.append(
-            {
-                "name": ep.name,
-                "module": getattr(ep, "module", ""),
-                "value": ep.value,
-            }
-        )
-    return plugins
-
-
-class PluginRegistry:
-    def __init__(self, loader: Any, provider_name: str) -> None:
-        self.loader = loader
-        self.provider_name = provider_name
-        self.registered_names: List[str] = []
-
-    def task(
-        self,
-        func: Optional[Callable] = None,
-        *,
-        name: Optional[str] = None,
-        inputs: Optional[Dict[str, Any]] = None,
-        outputs: Optional[List[str]] = None,
-    ):
-        if func is not None:
-            self.register_callable(
-                func,
-                name=name,
-                inputs=inputs or {},
-                outputs=outputs or [],
-            )
-            return func
-
-        def decorator(inner: Callable):
-            self.register_callable(
-                inner,
-                name=name,
-                inputs=inputs or {},
-                outputs=outputs or [],
-            )
-            return inner
-
-        return decorator
-
-    def register_callable(
-        self,
-        func: Callable,
-        *,
-        name: Optional[str] = None,
-        inputs: Optional[Dict[str, Any]] = None,
-        outputs: Optional[List[str]] = None,
-    ) -> Task:
-        task_name = name or getattr(func, "__name__", f"{self.provider_name}_task")
-        task = Task(func=func, name=task_name)
-        if inputs:
-            task.inputs.update(inputs)
-        if outputs:
-            task.outputs.extend(outputs)
-        self.loader._register_task(task_name, task)
-        self.registered_names.append(task_name)
-        return task
-
-    def add(self, obj: Any, *, name: Optional[str] = None) -> None:
-        if isinstance(obj, TaskWrapper):
-            self.loader._register_task(name or obj.task.name, obj.task)
-            self.registered_names.append(name or obj.task.name)
-        elif isinstance(obj, Task):
-            self.loader._register_task(name or obj.name, obj)
-            self.registered_names.append(name or obj.name)
-        elif callable(obj):
-            self.register_callable(obj, name=name)
-        else:
-            raise TypeError(f"Unsupported task object: {obj!r}")