pyoco 0.5.1__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyoco-0.5.1 → pyoco-0.6.0}/PKG-INFO +14 -9
- {pyoco-0.5.1 → pyoco-0.6.0}/README.md +13 -8
- {pyoco-0.5.1 → pyoco-0.6.0}/pyproject.toml +1 -1
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/__init__.py +2 -1
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/cli/main.py +86 -4
- pyoco-0.6.0/src/pyoco/core/exceptions.py +51 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/core/models.py +44 -3
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/discovery/loader.py +41 -60
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/discovery/plugins.py +63 -1
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/schemas/config.py +10 -13
- pyoco-0.6.0/src/pyoco/support/__init__.py +21 -0
- pyoco-0.6.0/src/pyoco/support/collector.py +56 -0
- pyoco-0.6.0/src/pyoco/support/filters.py +56 -0
- pyoco-0.6.0/src/pyoco/support/renderer.py +188 -0
- pyoco-0.6.0/src/pyoco/support/service.py +42 -0
- pyoco-0.6.0/src/pyoco/support/writer.py +15 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco.egg-info/PKG-INFO +14 -9
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco.egg-info/SOURCES.txt +7 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/tests/test_e2e_socketless.py +0 -1
- pyoco-0.6.0/tests/test_e2e_support_info.py +77 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/tests/test_integration_v030.py +0 -1
- {pyoco-0.5.1 → pyoco-0.6.0}/tests/test_socketless_basic.py +0 -1
- pyoco-0.5.1/src/pyoco/core/exceptions.py +0 -15
- {pyoco-0.5.1 → pyoco-0.6.0}/setup.cfg +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/cli/entry.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/client.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/core/base_task.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/core/context.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/core/engine.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/dsl/__init__.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/dsl/expressions.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/dsl/nodes.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/dsl/syntax.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/dsl/validator.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/server/__init__.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/server/api.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/server/metrics.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/server/models.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/server/store.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/server/webhook.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/socketless_reset.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/trace/backend.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/trace/console.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/worker/__init__.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/worker/client.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco/worker/runner.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco.egg-info/dependency_links.txt +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco.egg-info/requires.txt +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/src/pyoco.egg-info/top_level.txt +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/tests/test_cancellation.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/tests/test_cli_cancellation.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/tests/test_dsl.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/tests/test_engine.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/tests/test_engine_state.py +0 -0
- {pyoco-0.5.1 → pyoco-0.6.0}/tests/test_state_models.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pyoco
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: A workflow engine with sugar syntax
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -133,24 +133,29 @@ Or via CLI flag:
|
|
|
133
133
|
pyoco run --non-cute ...
|
|
134
134
|
```
|
|
135
135
|
|
|
136
|
-
## 🔭 Observability
|
|
136
|
+
## 🔭 Observability / Server (Archived)
|
|
137
137
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
- Webhook notifications fire when runs COMPLETE/FAIL—configure via `PYOCO_WEBHOOK_*` env vars and forward to Slack or your alerting stack.
|
|
141
|
-
- Import `docs/grafana_pyoco_cute.json` for a lavender/orange starter dashboard (3 panels: in-progress count, completion trend, per-flow latency).
|
|
142
|
-
- 詳細な手順は [docs/observability.md](docs/observability.md) を参照してください。
|
|
138
|
+
Observability and server-related docs are archived and out of scope for the current requirements.
|
|
139
|
+
See `docs/archive/observability.md` and `docs/archive/roadmap.md`.
|
|
143
140
|
|
|
144
141
|
## 🧩 Plug-ins
|
|
145
142
|
|
|
146
|
-
Need to share domain-specific tasks? Publish an entry point under `pyoco.tasks` and pyoco will auto-load it.
|
|
143
|
+
Need to share domain-specific tasks? Publish an entry point under `pyoco.tasks` and pyoco will auto-load it. We recommend **Task subclasses first** (callables still work with warnings). See [docs/plugins.md](docs/plugins.md) for examples, quickstart, and `pyoco plugins list` / `pyoco plugins lint`.
|
|
147
144
|
|
|
148
145
|
**Big data note:** pass handles, not copies. For large tensors/images, stash paths or handles in `ctx.artifacts`/`ctx.scratch` and let downstream tasks materialize only when needed. For lazy pipelines (e.g., DataPipe), log the pipeline when you actually iterate (typically the training task) instead of materializing upstream.
|
|
149
146
|
|
|
147
|
+
## 🧭 Task Discovery (Security)
|
|
148
|
+
|
|
149
|
+
Pyoco does not allow configuring discovery scope in `flow.yaml` (the `discovery:` key is rejected) to reduce the risk of importing unexpected code.
|
|
150
|
+
|
|
151
|
+
- **Entry point plug-ins**: auto-loaded from `importlib.metadata.entry_points(group="pyoco.tasks")`
|
|
152
|
+
- **Extra imports (ops-controlled)**: set `PYOCO_DISCOVERY_MODULES` (comma/space-separated module names), e.g. `PYOCO_DISCOVERY_MODULES=tasks,myapp.extra_tasks`
|
|
153
|
+
- **Explicit tasks**: prefer `tasks.<name>.callable` in `flow.yaml` (see tutorials)
|
|
154
|
+
|
|
150
155
|
## 📚 Documentation
|
|
151
156
|
|
|
152
157
|
- [Tutorials](docs/tutorial/index.md)
|
|
153
|
-
- [Roadmap](docs/roadmap.md)
|
|
158
|
+
- [Roadmap (Archived)](docs/archive/roadmap.md)
|
|
154
159
|
|
|
155
160
|
## 💖 Contributing
|
|
156
161
|
|
|
@@ -121,24 +121,29 @@ Or via CLI flag:
|
|
|
121
121
|
pyoco run --non-cute ...
|
|
122
122
|
```
|
|
123
123
|
|
|
124
|
-
## 🔭 Observability
|
|
124
|
+
## 🔭 Observability / Server (Archived)
|
|
125
125
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
- Webhook notifications fire when runs COMPLETE/FAIL—configure via `PYOCO_WEBHOOK_*` env vars and forward to Slack or your alerting stack.
|
|
129
|
-
- Import `docs/grafana_pyoco_cute.json` for a lavender/orange starter dashboard (3 panels: in-progress count, completion trend, per-flow latency).
|
|
130
|
-
- 詳細な手順は [docs/observability.md](docs/observability.md) を参照してください。
|
|
126
|
+
Observability and server-related docs are archived and out of scope for the current requirements.
|
|
127
|
+
See `docs/archive/observability.md` and `docs/archive/roadmap.md`.
|
|
131
128
|
|
|
132
129
|
## 🧩 Plug-ins
|
|
133
130
|
|
|
134
|
-
Need to share domain-specific tasks? Publish an entry point under `pyoco.tasks` and pyoco will auto-load it.
|
|
131
|
+
Need to share domain-specific tasks? Publish an entry point under `pyoco.tasks` and pyoco will auto-load it. We recommend **Task subclasses first** (callables still work with warnings). See [docs/plugins.md](docs/plugins.md) for examples, quickstart, and `pyoco plugins list` / `pyoco plugins lint`.
|
|
135
132
|
|
|
136
133
|
**Big data note:** pass handles, not copies. For large tensors/images, stash paths or handles in `ctx.artifacts`/`ctx.scratch` and let downstream tasks materialize only when needed. For lazy pipelines (e.g., DataPipe), log the pipeline when you actually iterate (typically the training task) instead of materializing upstream.
|
|
137
134
|
|
|
135
|
+
## 🧭 Task Discovery (Security)
|
|
136
|
+
|
|
137
|
+
Pyoco does not allow configuring discovery scope in `flow.yaml` (the `discovery:` key is rejected) to reduce the risk of importing unexpected code.
|
|
138
|
+
|
|
139
|
+
- **Entry point plug-ins**: auto-loaded from `importlib.metadata.entry_points(group="pyoco.tasks")`
|
|
140
|
+
- **Extra imports (ops-controlled)**: set `PYOCO_DISCOVERY_MODULES` (comma/space-separated module names), e.g. `PYOCO_DISCOVERY_MODULES=tasks,myapp.extra_tasks`
|
|
141
|
+
- **Explicit tasks**: prefer `tasks.<name>.callable` in `flow.yaml` (see tutorials)
|
|
142
|
+
|
|
138
143
|
## 📚 Documentation
|
|
139
144
|
|
|
140
145
|
- [Tutorials](docs/tutorial/index.md)
|
|
141
|
-
- [Roadmap](docs/roadmap.md)
|
|
146
|
+
- [Roadmap (Archived)](docs/archive/roadmap.md)
|
|
142
147
|
|
|
143
148
|
## 💖 Contributing
|
|
144
149
|
|
|
@@ -2,10 +2,11 @@ from .core.models import Flow, Task
|
|
|
2
2
|
from .core.engine import Engine
|
|
3
3
|
from .dsl.syntax import task
|
|
4
4
|
from .trace.console import ConsoleTraceBackend
|
|
5
|
+
from . import support
|
|
5
6
|
|
|
6
7
|
def run(flow: Flow, params: dict = None, trace: bool = True, cute: bool = True):
|
|
7
8
|
backend = ConsoleTraceBackend(style="cute" if cute else "plain")
|
|
8
9
|
engine = Engine(trace_backend=backend)
|
|
9
10
|
return engine.run(flow, params)
|
|
10
11
|
|
|
11
|
-
__all__ = ["task", "Flow", "run"]
|
|
12
|
+
__all__ = ["task", "Flow", "run", "support"]
|
|
@@ -11,6 +11,15 @@ from ..core.models import Flow
|
|
|
11
11
|
from ..core.engine import Engine
|
|
12
12
|
from ..trace.console import ConsoleTraceBackend
|
|
13
13
|
from ..client import Client
|
|
14
|
+
from ..support.service import SupportInfoService
|
|
15
|
+
from ..core.exceptions import (
|
|
16
|
+
SupportInfoError,
|
|
17
|
+
InvalidFormatError,
|
|
18
|
+
TaskNotFoundError,
|
|
19
|
+
InvalidFilterError,
|
|
20
|
+
OutputWriteError,
|
|
21
|
+
MissingTaskMetadataError,
|
|
22
|
+
)
|
|
14
23
|
|
|
15
24
|
def main():
|
|
16
25
|
parser = argparse.ArgumentParser(description="Pyoco Workflow Engine")
|
|
@@ -91,6 +100,30 @@ def main():
|
|
|
91
100
|
plugins_lint = plugins_sub.add_parser("lint", help="Validate plug-ins for upcoming requirements")
|
|
92
101
|
plugins_lint.add_argument("--json", action="store_true", help="Output JSON payload")
|
|
93
102
|
|
|
103
|
+
support_parser = subparsers.add_parser("support", help="Generate support info")
|
|
104
|
+
support_subparsers = support_parser.add_subparsers(dest="support_command")
|
|
105
|
+
|
|
106
|
+
support_tasks = support_subparsers.add_parser("tasks", help="List tasks for LLM support")
|
|
107
|
+
support_tasks.add_argument("--config", required=True, help="Path to flow.yaml")
|
|
108
|
+
support_tasks.add_argument("--format", default="prompt", choices=["prompt", "json", "md"])
|
|
109
|
+
support_tasks.add_argument("--output", help="Write output to file")
|
|
110
|
+
support_tasks.add_argument("--name", action="append", help="Filter by task name (repeatable)")
|
|
111
|
+
support_tasks.add_argument("--origin", action="append", help="Filter by origin (repeatable)")
|
|
112
|
+
support_tasks.add_argument("--tag", action="append", help="Filter by tag (repeatable)")
|
|
113
|
+
|
|
114
|
+
support_task = support_subparsers.add_parser("task", help="Show task detail for LLM support")
|
|
115
|
+
support_task.add_argument("--config", required=True, help="Path to flow.yaml")
|
|
116
|
+
support_task.add_argument("--name", required=True, help="Task name")
|
|
117
|
+
support_task.add_argument("--format", default="prompt", choices=["prompt", "json", "md"])
|
|
118
|
+
support_task.add_argument("--output", help="Write output to file")
|
|
119
|
+
support_task.add_argument("--origin", action="append", help="Filter by origin (repeatable)")
|
|
120
|
+
support_task.add_argument("--tag", action="append", help="Filter by tag (repeatable)")
|
|
121
|
+
|
|
122
|
+
support_guide = support_subparsers.add_parser("guide", help="Show flow.yaml guide for LLM support")
|
|
123
|
+
support_guide.add_argument("--config", required=True, help="Path to flow.yaml")
|
|
124
|
+
support_guide.add_argument("--format", default="prompt", choices=["prompt", "json", "md"])
|
|
125
|
+
support_guide.add_argument("--output", help="Write output to file")
|
|
126
|
+
|
|
94
127
|
args = parser.parse_args()
|
|
95
128
|
|
|
96
129
|
if not args.command:
|
|
@@ -170,6 +203,26 @@ def main():
|
|
|
170
203
|
plugins_parser.print_help()
|
|
171
204
|
return
|
|
172
205
|
|
|
206
|
+
if args.command == "support":
|
|
207
|
+
if not args.support_command:
|
|
208
|
+
support_parser.print_help()
|
|
209
|
+
sys.exit(1)
|
|
210
|
+
filters = _build_support_filters(args)
|
|
211
|
+
try:
|
|
212
|
+
content = SupportInfoService().build(
|
|
213
|
+
kind=args.support_command,
|
|
214
|
+
config_path=args.config,
|
|
215
|
+
format=args.format,
|
|
216
|
+
filters=filters or None,
|
|
217
|
+
output_path=args.output,
|
|
218
|
+
)
|
|
219
|
+
except SupportInfoError as exc:
|
|
220
|
+
_print_support_error(exc)
|
|
221
|
+
sys.exit(1)
|
|
222
|
+
if not args.output:
|
|
223
|
+
print(content)
|
|
224
|
+
return
|
|
225
|
+
|
|
173
226
|
if args.command == "server":
|
|
174
227
|
if args.server_command == "start":
|
|
175
228
|
import uvicorn
|
|
@@ -405,15 +458,44 @@ def main():
|
|
|
405
458
|
return
|
|
406
459
|
|
|
407
460
|
def _collect_plugin_reports():
|
|
408
|
-
dummy = SimpleNamespace(
|
|
409
|
-
tasks={},
|
|
410
|
-
discovery=SimpleNamespace(entry_points=[], packages=[], glob_modules=[]),
|
|
411
|
-
)
|
|
461
|
+
dummy = SimpleNamespace(tasks={})
|
|
412
462
|
loader = TaskLoader(dummy)
|
|
413
463
|
loader.load()
|
|
414
464
|
return loader.plugin_reports
|
|
415
465
|
|
|
416
466
|
|
|
467
|
+
def _build_support_filters(args):
|
|
468
|
+
filters = {}
|
|
469
|
+
if getattr(args, "name", None):
|
|
470
|
+
value = args.name
|
|
471
|
+
filters["name"] = value if isinstance(value, list) else [value]
|
|
472
|
+
if getattr(args, "origin", None):
|
|
473
|
+
filters["origin"] = args.origin
|
|
474
|
+
if getattr(args, "tag", None):
|
|
475
|
+
filters["tag"] = args.tag
|
|
476
|
+
return filters
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def _print_support_error(exc: SupportInfoError) -> None:
|
|
480
|
+
if isinstance(exc, InvalidFormatError):
|
|
481
|
+
print(f"Invalid format: {exc.format}")
|
|
482
|
+
return
|
|
483
|
+
if isinstance(exc, TaskNotFoundError):
|
|
484
|
+
print(f"Task not found: {exc.name}")
|
|
485
|
+
return
|
|
486
|
+
if isinstance(exc, OutputWriteError):
|
|
487
|
+
print(f"Failed to write output: {exc.path}")
|
|
488
|
+
return
|
|
489
|
+
if isinstance(exc, InvalidFilterError):
|
|
490
|
+
print(f"Invalid filter: {exc.filter_value}")
|
|
491
|
+
return
|
|
492
|
+
if isinstance(exc, MissingTaskMetadataError):
|
|
493
|
+
fields = ",".join(exc.fields)
|
|
494
|
+
print(f"Missing task metadata: {exc.name} fields={fields}")
|
|
495
|
+
return
|
|
496
|
+
print(f"Error: {exc}")
|
|
497
|
+
|
|
498
|
+
|
|
417
499
|
def _stream_logs(client, args):
|
|
418
500
|
seen_seq = -1
|
|
419
501
|
follow = args.follow
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
class ControlFlowError(Exception):
|
|
2
|
+
"""Base error for control flow execution issues."""
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class UntilMaxIterationsExceeded(ControlFlowError):
|
|
6
|
+
def __init__(self, expression: str, max_iter: int):
|
|
7
|
+
super().__init__(f"Until condition '{expression}' exceeded max_iter={max_iter}")
|
|
8
|
+
self.expression = expression
|
|
9
|
+
self.max_iter = max_iter
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class SwitchNoMatch(ControlFlowError):
|
|
13
|
+
def __init__(self, expression: str):
|
|
14
|
+
super().__init__(f"Switch expression '{expression}' did not match any case.")
|
|
15
|
+
self.expression = expression
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class SupportInfoError(Exception):
|
|
19
|
+
"""Base error for support info generation."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class InvalidFormatError(SupportInfoError):
|
|
23
|
+
def __init__(self, format: str):
|
|
24
|
+
self.format = format
|
|
25
|
+
super().__init__(f"Invalid format: {format}")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class TaskNotFoundError(SupportInfoError):
|
|
29
|
+
def __init__(self, name: str):
|
|
30
|
+
self.name = name
|
|
31
|
+
super().__init__(f"Task not found: {name}")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class InvalidFilterError(SupportInfoError):
|
|
35
|
+
def __init__(self, filter_value: str):
|
|
36
|
+
self.filter_value = filter_value
|
|
37
|
+
super().__init__(f"Invalid filter: {filter_value}")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class OutputWriteError(SupportInfoError):
|
|
41
|
+
def __init__(self, path: str):
|
|
42
|
+
self.path = path
|
|
43
|
+
super().__init__(f"Failed to write output: {path}")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class MissingTaskMetadataError(SupportInfoError):
|
|
47
|
+
def __init__(self, name: str, fields: list[str]):
|
|
48
|
+
self.name = name
|
|
49
|
+
self.fields = fields
|
|
50
|
+
field_list = ",".join(fields)
|
|
51
|
+
super().__init__(f"Missing task metadata: {name} fields={field_list}")
|
|
@@ -238,11 +238,10 @@ class Flow:
|
|
|
238
238
|
# So `flow >> (A | B)` just adds A and B.
|
|
239
239
|
# Then `(A | B) >> C` is handled by Branch.
|
|
240
240
|
pass
|
|
241
|
-
|
|
242
|
-
# Update tail
|
|
241
|
+
|
|
243
242
|
if new_tasks:
|
|
244
243
|
self._tail = set(new_tasks)
|
|
245
|
-
|
|
244
|
+
|
|
246
245
|
return self
|
|
247
246
|
|
|
248
247
|
def add_task(self, task: Task):
|
|
@@ -284,3 +283,45 @@ class Flow:
|
|
|
284
283
|
tail_task.dependents.add(task)
|
|
285
284
|
task.dependencies.add(tail_task)
|
|
286
285
|
self._tail = {task}
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
@dataclass
|
|
289
|
+
class TaskIO:
|
|
290
|
+
name: str
|
|
291
|
+
type: str
|
|
292
|
+
required: bool
|
|
293
|
+
constraints: Optional[List[str]] = None
|
|
294
|
+
|
|
295
|
+
@classmethod
|
|
296
|
+
def from_dict(cls, data: Dict[str, Any]) -> "TaskIO":
|
|
297
|
+
return cls(
|
|
298
|
+
name=data.get("name"),
|
|
299
|
+
type=data.get("type"),
|
|
300
|
+
required=data.get("required"),
|
|
301
|
+
constraints=data.get("constraints"),
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
@dataclass
|
|
306
|
+
class TaskInfo:
|
|
307
|
+
name: str
|
|
308
|
+
summary: str
|
|
309
|
+
inputs: List[TaskIO]
|
|
310
|
+
outputs: List[TaskIO]
|
|
311
|
+
origin: Optional[str] = None
|
|
312
|
+
tags: Optional[List[str]] = None
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
@dataclass
|
|
316
|
+
class SupportFilters:
|
|
317
|
+
name: Optional[List[str]] = None
|
|
318
|
+
origin: Optional[List[str]] = None
|
|
319
|
+
tag: Optional[List[str]] = None
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
@dataclass
|
|
323
|
+
class SupportInfo:
|
|
324
|
+
kind: str
|
|
325
|
+
format: str
|
|
326
|
+
content: str
|
|
327
|
+
filters: SupportFilters
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import importlib
|
|
2
|
-
import
|
|
3
|
-
import sys
|
|
2
|
+
import os
|
|
4
3
|
from typing import Dict, List, Any, Set
|
|
5
4
|
from ..core.models import Task
|
|
6
5
|
from ..dsl.syntax import TaskWrapper
|
|
@@ -11,30 +10,28 @@ class TaskLoader:
|
|
|
11
10
|
self.config = config
|
|
12
11
|
self.strict = strict
|
|
13
12
|
self.tasks: Dict[str, Task] = {}
|
|
13
|
+
self.task_infos: Dict[str, Any] = {}
|
|
14
14
|
self._explicit_tasks: Set[str] = set()
|
|
15
15
|
self.plugin_reports: List[Dict[str, Any]] = []
|
|
16
16
|
|
|
17
17
|
def load(self):
|
|
18
18
|
# Load explicitly defined tasks in config FIRST (Higher priority)
|
|
19
19
|
for task_name, task_conf in self.config.tasks.items():
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
callable_path = self._conf_get(task_conf, "callable")
|
|
21
|
+
if callable_path:
|
|
22
|
+
self._load_explicit_task(task_name, task_conf, callable_path)
|
|
22
23
|
self._explicit_tasks.add(task_name)
|
|
23
24
|
|
|
24
|
-
|
|
25
|
-
for package in self.config.discovery.packages:
|
|
26
|
-
self._load_package(package)
|
|
27
|
-
|
|
28
|
-
# Load from entry points (simplified)
|
|
29
|
-
for ep in self.config.discovery.entry_points:
|
|
30
|
-
self._load_module(ep)
|
|
31
|
-
|
|
32
|
-
# Load from glob modules
|
|
33
|
-
for pattern in self.config.discovery.glob_modules:
|
|
34
|
-
self._load_glob_modules(pattern)
|
|
25
|
+
self._load_env_modules()
|
|
35
26
|
|
|
36
27
|
self._load_entry_point_plugins()
|
|
37
28
|
|
|
29
|
+
def _load_env_modules(self) -> None:
|
|
30
|
+
raw = os.getenv("PYOCO_DISCOVERY_MODULES", "")
|
|
31
|
+
modules = [item.strip() for item in raw.replace(",", " ").split() if item.strip()]
|
|
32
|
+
for module_name in modules:
|
|
33
|
+
self._load_module(module_name)
|
|
34
|
+
|
|
38
35
|
def _register_task(self, name: str, task: Task):
|
|
39
36
|
if name in self.tasks:
|
|
40
37
|
if name in self._explicit_tasks:
|
|
@@ -51,24 +48,27 @@ class TaskLoader:
|
|
|
51
48
|
# Apply config overlay if exists
|
|
52
49
|
if self.config and name in self.config.tasks:
|
|
53
50
|
conf = self.config.tasks[name]
|
|
54
|
-
if not conf
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
if
|
|
58
|
-
task.
|
|
51
|
+
if not self._conf_get(conf, "callable"):
|
|
52
|
+
inputs = self._conf_get(conf, "inputs") or {}
|
|
53
|
+
outputs = self._conf_get(conf, "outputs") or []
|
|
54
|
+
if inputs:
|
|
55
|
+
task.inputs.update(inputs)
|
|
56
|
+
if outputs:
|
|
57
|
+
task.outputs.extend(outputs)
|
|
59
58
|
|
|
60
59
|
self.tasks[name] = task
|
|
61
60
|
|
|
62
|
-
def
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
61
|
+
def _register_task_info(self, info: Any):
|
|
62
|
+
name = getattr(info, "name", None)
|
|
63
|
+
if not name:
|
|
64
|
+
return
|
|
65
|
+
if name in self.task_infos:
|
|
66
|
+
msg = f"Task metadata '{name}' already defined."
|
|
67
|
+
if self.strict:
|
|
68
|
+
raise ValueError(f"{msg} (Strict mode enabled)")
|
|
68
69
|
else:
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
print(f"Warning: Could not import package {package_name}: {e}")
|
|
70
|
+
print(f"Warning: {msg} Overwriting.")
|
|
71
|
+
self.task_infos[name] = info
|
|
72
72
|
|
|
73
73
|
def _load_module(self, module_name: str):
|
|
74
74
|
try:
|
|
@@ -76,30 +76,6 @@ class TaskLoader:
|
|
|
76
76
|
self._scan_module(mod)
|
|
77
77
|
except ImportError as e:
|
|
78
78
|
print(f"Warning: Could not import module {module_name}: {e}")
|
|
79
|
-
|
|
80
|
-
def _load_glob_modules(self, pattern: str):
|
|
81
|
-
import glob
|
|
82
|
-
import os
|
|
83
|
-
|
|
84
|
-
# Pattern is likely a file path glob, e.g. "jobs/*.py"
|
|
85
|
-
# We need to convert file paths to module paths
|
|
86
|
-
files = glob.glob(pattern, recursive=True)
|
|
87
|
-
for file_path in files:
|
|
88
|
-
if not file_path.endswith(".py"):
|
|
89
|
-
continue
|
|
90
|
-
|
|
91
|
-
# Convert path to module
|
|
92
|
-
# This is tricky without knowing the root.
|
|
93
|
-
# Assumption: running from root, and file path is relative to root.
|
|
94
|
-
# e.g. "myproject/tasks/foo.py" -> "myproject.tasks.foo"
|
|
95
|
-
|
|
96
|
-
rel_path = os.path.relpath(file_path)
|
|
97
|
-
if rel_path.startswith(".."):
|
|
98
|
-
# Out of tree, skip or warn
|
|
99
|
-
continue
|
|
100
|
-
|
|
101
|
-
module_name = rel_path.replace(os.sep, ".")[:-3] # strip .py
|
|
102
|
-
self._load_module(module_name)
|
|
103
79
|
|
|
104
80
|
def _load_entry_point_plugins(self):
|
|
105
81
|
entries = iter_entry_points()
|
|
@@ -118,6 +94,7 @@ class TaskLoader:
|
|
|
118
94
|
raise TypeError("Entry point must be callable")
|
|
119
95
|
hook(registry)
|
|
120
96
|
info["tasks"] = list(registry.records)
|
|
97
|
+
info["task_infos"] = list(registry.task_infos.values())
|
|
121
98
|
info["warnings"] = list(registry.warnings)
|
|
122
99
|
if not registry.records:
|
|
123
100
|
info["warnings"].append("no tasks registered")
|
|
@@ -134,13 +111,17 @@ class TaskLoader:
|
|
|
134
111
|
self._register_task(name, obj.task)
|
|
135
112
|
elif isinstance(obj, Task):
|
|
136
113
|
self._register_task(name, obj)
|
|
137
|
-
elif callable(obj) and getattr(obj, '__pyoco_task__', False):
|
|
138
|
-
# Convert to Task if not already
|
|
139
|
-
pass
|
|
140
114
|
|
|
141
|
-
def
|
|
115
|
+
def _conf_get(self, conf: Any, key: str):
|
|
116
|
+
if hasattr(conf, key):
|
|
117
|
+
return getattr(conf, key)
|
|
118
|
+
if isinstance(conf, dict):
|
|
119
|
+
return conf.get(key)
|
|
120
|
+
return None
|
|
121
|
+
|
|
122
|
+
def _load_explicit_task(self, name: str, conf: Any, callable_path: str):
|
|
142
123
|
# Load callable
|
|
143
|
-
module_path, func_name =
|
|
124
|
+
module_path, func_name = callable_path.split(':')
|
|
144
125
|
try:
|
|
145
126
|
mod = importlib.import_module(module_path)
|
|
146
127
|
obj = getattr(mod, func_name)
|
|
@@ -154,8 +135,8 @@ class TaskLoader:
|
|
|
154
135
|
|
|
155
136
|
# Create a Task wrapper
|
|
156
137
|
t = Task(func=real_func, name=name)
|
|
157
|
-
t.inputs = conf
|
|
158
|
-
t.outputs = conf
|
|
138
|
+
t.inputs = self._conf_get(conf, "inputs") or {}
|
|
139
|
+
t.outputs = self._conf_get(conf, "outputs") or []
|
|
159
140
|
self.tasks[name] = t
|
|
160
141
|
except (ImportError, AttributeError) as e:
|
|
161
142
|
print(f"Error loading task {name}: {e}")
|
|
@@ -3,7 +3,8 @@ from __future__ import annotations
|
|
|
3
3
|
from importlib import metadata as importlib_metadata
|
|
4
4
|
from typing import Any, Callable, Dict, List, Optional, Type
|
|
5
5
|
|
|
6
|
-
from ..core.models import Task
|
|
6
|
+
from ..core.models import Task, TaskInfo, TaskIO
|
|
7
|
+
from ..core.exceptions import MissingTaskMetadataError
|
|
7
8
|
from ..dsl.syntax import TaskWrapper
|
|
8
9
|
|
|
9
10
|
|
|
@@ -41,6 +42,45 @@ class PluginRegistry:
|
|
|
41
42
|
self.registered_names: List[str] = []
|
|
42
43
|
self.records: List[Dict[str, Any]] = []
|
|
43
44
|
self.warnings: List[str] = []
|
|
45
|
+
self.task_infos: Dict[str, TaskInfo] = {}
|
|
46
|
+
|
|
47
|
+
def task_info(
|
|
48
|
+
self,
|
|
49
|
+
*,
|
|
50
|
+
name: Optional[str] = None,
|
|
51
|
+
summary: Optional[str] = None,
|
|
52
|
+
inputs: Optional[List[Any]] = None,
|
|
53
|
+
outputs: Optional[List[Any]] = None,
|
|
54
|
+
tags: Optional[List[str]] = None,
|
|
55
|
+
origin: Optional[str] = None,
|
|
56
|
+
) -> None:
|
|
57
|
+
missing = []
|
|
58
|
+
if not name:
|
|
59
|
+
missing.append("name")
|
|
60
|
+
if not summary:
|
|
61
|
+
missing.append("summary")
|
|
62
|
+
if inputs is None:
|
|
63
|
+
missing.append("inputs")
|
|
64
|
+
if outputs is None:
|
|
65
|
+
missing.append("outputs")
|
|
66
|
+
if missing:
|
|
67
|
+
raise MissingTaskMetadataError(name or "<unknown>", missing)
|
|
68
|
+
|
|
69
|
+
if not isinstance(inputs, list) or not isinstance(outputs, list):
|
|
70
|
+
raise MissingTaskMetadataError(name, ["inputs", "outputs"])
|
|
71
|
+
task_inputs = self._normalize_taskio_list(inputs, "inputs", name)
|
|
72
|
+
task_outputs = self._normalize_taskio_list(outputs, "outputs", name)
|
|
73
|
+
info = TaskInfo(
|
|
74
|
+
name=name,
|
|
75
|
+
summary=summary,
|
|
76
|
+
inputs=task_inputs,
|
|
77
|
+
outputs=task_outputs,
|
|
78
|
+
origin=origin or self.provider_name,
|
|
79
|
+
tags=tags or [],
|
|
80
|
+
)
|
|
81
|
+
self.task_infos[name] = info
|
|
82
|
+
if hasattr(self.loader, "_register_task_info"):
|
|
83
|
+
self.loader._register_task_info(info)
|
|
44
84
|
|
|
45
85
|
def task(
|
|
46
86
|
self,
|
|
@@ -146,3 +186,25 @@ class PluginRegistry:
|
|
|
146
186
|
if task.__class__ is Task and origin not in ("callable", "wrapper"):
|
|
147
187
|
warnings.append("plain Task instance detected; subclass Task for metadata support")
|
|
148
188
|
return warnings
|
|
189
|
+
|
|
190
|
+
def _normalize_taskio_list(self, items: List[Any], label: str, task_name: str) -> List[TaskIO]:
|
|
191
|
+
normalized: List[TaskIO] = []
|
|
192
|
+
for idx, item in enumerate(items):
|
|
193
|
+
if isinstance(item, TaskIO):
|
|
194
|
+
normalized.append(item)
|
|
195
|
+
continue
|
|
196
|
+
if isinstance(item, dict):
|
|
197
|
+
taskio = TaskIO.from_dict(item)
|
|
198
|
+
missing = []
|
|
199
|
+
if not taskio.name:
|
|
200
|
+
missing.append(f"{label}[{idx}].name")
|
|
201
|
+
if not taskio.type:
|
|
202
|
+
missing.append(f"{label}[{idx}].type")
|
|
203
|
+
if taskio.required is None:
|
|
204
|
+
missing.append(f"{label}[{idx}].required")
|
|
205
|
+
if missing:
|
|
206
|
+
raise MissingTaskMetadataError(task_name, missing)
|
|
207
|
+
normalized.append(taskio)
|
|
208
|
+
continue
|
|
209
|
+
raise MissingTaskMetadataError(task_name, [f"{label}[{idx}]"])
|
|
210
|
+
return normalized
|
|
@@ -13,12 +13,6 @@ class FlowConfig:
|
|
|
13
13
|
graph: str
|
|
14
14
|
defaults: Dict[str, Any] = field(default_factory=dict)
|
|
15
15
|
|
|
16
|
-
@dataclass
|
|
17
|
-
class DiscoveryConfig:
|
|
18
|
-
entry_points: List[str] = field(default_factory=list)
|
|
19
|
-
packages: List[str] = field(default_factory=list)
|
|
20
|
-
glob_modules: List[str] = field(default_factory=list)
|
|
21
|
-
|
|
22
16
|
@dataclass
|
|
23
17
|
class RuntimeConfig:
|
|
24
18
|
expose_env: List[str] = field(default_factory=list)
|
|
@@ -28,23 +22,27 @@ class PyocoConfig:
|
|
|
28
22
|
version: int
|
|
29
23
|
flows: Dict[str, FlowConfig]
|
|
30
24
|
tasks: Dict[str, TaskConfig]
|
|
31
|
-
discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
|
|
32
25
|
runtime: RuntimeConfig = field(default_factory=RuntimeConfig)
|
|
33
26
|
|
|
34
27
|
@classmethod
|
|
35
28
|
def from_yaml(cls, path: str) -> 'PyocoConfig':
|
|
36
29
|
with open(path, 'r') as f:
|
|
37
|
-
data = yaml.safe_load(f)
|
|
30
|
+
data = yaml.safe_load(f) or {}
|
|
38
31
|
|
|
39
32
|
# Simple manual parsing/validation for MVP
|
|
40
33
|
# In a real app, use pydantic or similar
|
|
41
34
|
|
|
42
35
|
flows = {k: FlowConfig(**v) for k, v in data.get('flows', {}).items()}
|
|
43
36
|
tasks = {k: TaskConfig(**v) for k, v in data.get('tasks', {}).items()}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
37
|
+
|
|
38
|
+
if "discovery" in data:
|
|
39
|
+
raise ValueError(
|
|
40
|
+
"Unsupported config key 'discovery'.\n"
|
|
41
|
+
"For safety, discovery scope is not configurable in flow.yaml.\n"
|
|
42
|
+
"Remove 'discovery' and use PYOCO_DISCOVERY_MODULES to import extra modules, "
|
|
43
|
+
"or define tasks explicitly via tasks.<name>.callable."
|
|
44
|
+
)
|
|
45
|
+
|
|
48
46
|
run_data = data.get('runtime', {})
|
|
49
47
|
runtime = RuntimeConfig(**run_data)
|
|
50
48
|
|
|
@@ -52,6 +50,5 @@ class PyocoConfig:
|
|
|
52
50
|
version=data.get('version', 1),
|
|
53
51
|
flows=flows,
|
|
54
52
|
tasks=tasks,
|
|
55
|
-
discovery=discovery,
|
|
56
53
|
runtime=runtime
|
|
57
54
|
)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from .service import SupportInfoService
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def build(
|
|
5
|
+
kind: str,
|
|
6
|
+
config_path: str,
|
|
7
|
+
format: str = "prompt",
|
|
8
|
+
filters=None,
|
|
9
|
+
output_path: str | None = None,
|
|
10
|
+
):
|
|
11
|
+
service = SupportInfoService()
|
|
12
|
+
return service.build(
|
|
13
|
+
kind=kind,
|
|
14
|
+
config_path=config_path,
|
|
15
|
+
format=format,
|
|
16
|
+
filters=filters,
|
|
17
|
+
output_path=output_path,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
__all__ = ["SupportInfoService", "build"]
|