jpipe-runner 3.3.0__tar.gz → 3.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/PKG-INFO +3 -5
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/README.md +1 -4
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/pyproject.toml +4 -2
- jpipe_runner-3.4.1/src/jpipe_runner/framework/decorators/__init__.py +3 -0
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/src/jpipe_runner/framework/decorators/jpipe_decorator.py +37 -5
- jpipe_runner-3.4.1/src/jpipe_runner/framework/decorators/link_decorator.py +33 -0
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/src/jpipe_runner/framework/engine.py +74 -99
- jpipe_runner-3.4.1/src/jpipe_runner/framework/graphviz_exporter.py +106 -0
- jpipe_runner-3.4.1/src/jpipe_runner/framework/validators/__init__.py +21 -0
- jpipe_runner-3.4.1/src/jpipe_runner/framework/validators/base.py +41 -0
- jpipe_runner-3.4.1/src/jpipe_runner/framework/validators/duplicate_producer.py +47 -0
- jpipe_runner-3.4.1/src/jpipe_runner/framework/validators/evidence_dependency.py +186 -0
- jpipe_runner-3.4.1/src/jpipe_runner/framework/validators/missing_variable.py +54 -0
- jpipe_runner-3.4.1/src/jpipe_runner/framework/validators/order.py +77 -0
- jpipe_runner-3.4.1/src/jpipe_runner/framework/validators/produced_not_consumed.py +46 -0
- jpipe_runner-3.4.1/src/jpipe_runner/framework/validators/schema.py +92 -0
- jpipe_runner-3.4.1/src/jpipe_runner/framework/validators/self_dependency.py +47 -0
- jpipe_runner-3.4.1/src/jpipe_runner/framework/validators/unbound_element.py +52 -0
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/src/jpipe_runner/runner.py +16 -10
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/src/jpipe_runner/runtime.py +27 -1
- jpipe_runner-3.4.1/src/jpipe_runner/schema/justification.schema.json +36 -0
- jpipe_runner-3.3.0/src/jpipe_runner/framework/validators.py +0 -613
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/LICENSE +0 -0
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/src/jpipe_runner/__init__.py +0 -0
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/src/jpipe_runner/__main__.py +0 -0
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/src/jpipe_runner/enums.py +0 -0
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/src/jpipe_runner/exceptions.py +0 -0
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/src/jpipe_runner/framework/__init__.py +0 -0
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/src/jpipe_runner/framework/context.py +0 -0
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/src/jpipe_runner/framework/decorators/contribution_decorator.py +0 -0
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/src/jpipe_runner/framework/decorators/skip_decorator.py +0 -0
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/src/jpipe_runner/framework/logger.py +0 -0
- {jpipe_runner-3.3.0/src/jpipe_runner/framework/decorators → jpipe_runner-3.4.1/src/jpipe_runner/schema}/__init__.py +0 -0
- {jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/src/jpipe_runner/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: jpipe-runner
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.4.1
|
|
4
4
|
Summary: A Justification Runner designed for jPipe.
|
|
5
5
|
License: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -18,6 +18,7 @@ Classifier: Programming Language :: Python :: 3.14
|
|
|
18
18
|
Provides-Extra: docs
|
|
19
19
|
Provides-Extra: full
|
|
20
20
|
Requires-Dist: graphviz (>=0.20)
|
|
21
|
+
Requires-Dist: jsonschema (>=4.0)
|
|
21
22
|
Requires-Dist: networkx (==3.5)
|
|
22
23
|
Requires-Dist: pyyaml (==6.0.2)
|
|
23
24
|
Requires-Dist: sphinx (==8.1.3) ; extra == "docs" or extra == "full"
|
|
@@ -44,7 +45,7 @@ A Justification Runner designed for jPipe.
|
|
|
44
45
|
```bash
|
|
45
46
|
poetry run jpipe-runner [-h] [--variable NAME:VALUE] [--library LIB] \
|
|
46
47
|
[--diagram PATTERN] [--output FILE] [--dry-run] \
|
|
47
|
-
[--verbose] [--config-file PATH]
|
|
48
|
+
[--verbose] [--config-file PATH] jd_file
|
|
48
49
|
```
|
|
49
50
|
|
|
50
51
|
**Key options:**
|
|
@@ -56,8 +57,6 @@ poetry run jpipe-runner [-h] [--variable NAME:VALUE] [--library LIB] \
|
|
|
56
57
|
* `--dry-run`: Validate workflow without executing.
|
|
57
58
|
* `--verbose`, `-V`: Enable debug logging.
|
|
58
59
|
* `--config-file`: Load workflow config from a YAML file.
|
|
59
|
-
* `--gui`: Launch the Tkinter-based `GraphWorkflowVisualizer`
|
|
60
|
-
|
|
61
60
|
Example:
|
|
62
61
|
|
|
63
62
|
```bash
|
|
@@ -74,7 +73,6 @@ For detailed instructions on how to execute the project, including descriptions
|
|
|
74
73
|
* Python 3.10+
|
|
75
74
|
* [Poetry](https://python-poetry.org)
|
|
76
75
|
* [Graphviz](https://graphviz.org/) (`libgraphviz-dev`, `pkg-config`)
|
|
77
|
-
* Optional for GUI version: [Tkinter](https://docs.python.org/3/library/tkinter.html)
|
|
78
76
|
|
|
79
77
|
### From Source
|
|
80
78
|
|
|
@@ -18,7 +18,7 @@ A Justification Runner designed for jPipe.
|
|
|
18
18
|
```bash
|
|
19
19
|
poetry run jpipe-runner [-h] [--variable NAME:VALUE] [--library LIB] \
|
|
20
20
|
[--diagram PATTERN] [--output FILE] [--dry-run] \
|
|
21
|
-
[--verbose] [--config-file PATH]
|
|
21
|
+
[--verbose] [--config-file PATH] jd_file
|
|
22
22
|
```
|
|
23
23
|
|
|
24
24
|
**Key options:**
|
|
@@ -30,8 +30,6 @@ poetry run jpipe-runner [-h] [--variable NAME:VALUE] [--library LIB] \
|
|
|
30
30
|
* `--dry-run`: Validate workflow without executing.
|
|
31
31
|
* `--verbose`, `-V`: Enable debug logging.
|
|
32
32
|
* `--config-file`: Load workflow config from a YAML file.
|
|
33
|
-
* `--gui`: Launch the Tkinter-based `GraphWorkflowVisualizer`
|
|
34
|
-
|
|
35
33
|
Example:
|
|
36
34
|
|
|
37
35
|
```bash
|
|
@@ -48,7 +46,6 @@ For detailed instructions on how to execute the project, including descriptions
|
|
|
48
46
|
* Python 3.10+
|
|
49
47
|
* [Poetry](https://python-poetry.org)
|
|
50
48
|
* [Graphviz](https://graphviz.org/) (`libgraphviz-dev`, `pkg-config`)
|
|
51
|
-
* Optional for GUI version: [Tkinter](https://docs.python.org/3/library/tkinter.html)
|
|
52
49
|
|
|
53
50
|
### From Source
|
|
54
51
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "jpipe-runner"
|
|
3
|
-
version = "3.
|
|
3
|
+
version = "3.4.1"
|
|
4
4
|
description = "A Justification Runner designed for jPipe."
|
|
5
5
|
authors = [
|
|
6
6
|
"Jason Lyu <xjasonlyu@gmail.com>",
|
|
@@ -16,7 +16,8 @@ classifiers = [
|
|
|
16
16
|
"Operating System :: OS Independent"
|
|
17
17
|
]
|
|
18
18
|
packages = [
|
|
19
|
-
{ include = "jpipe_runner", from = "src" }
|
|
19
|
+
{ include = "jpipe_runner", from = "src" },
|
|
20
|
+
{ include = "jpipe_runner/schema", from = "src" }
|
|
20
21
|
]
|
|
21
22
|
|
|
22
23
|
[tool.poetry.dependencies]
|
|
@@ -24,6 +25,7 @@ python = ">=3.11"
|
|
|
24
25
|
networkx = "3.5"
|
|
25
26
|
graphviz = ">=0.20"
|
|
26
27
|
pyyaml = "6.0.2"
|
|
28
|
+
jsonschema = ">=4.0"
|
|
27
29
|
|
|
28
30
|
sphinx = { version = "8.1.3", optional = true }
|
|
29
31
|
|
{jpipe_runner-3.3.0 → jpipe_runner-3.4.1}/src/jpipe_runner/framework/decorators/jpipe_decorator.py
RENAMED
|
@@ -5,6 +5,7 @@ from functools import wraps
|
|
|
5
5
|
from typing import Any, Callable, List, Optional
|
|
6
6
|
|
|
7
7
|
from jpipe_runner.framework.context import RuntimeContext, ctx
|
|
8
|
+
from jpipe_runner.framework.decorators.link_decorator import jpipe_link # noqa: F401
|
|
8
9
|
from jpipe_runner.framework.logger import GLOBAL_LOGGER
|
|
9
10
|
|
|
10
11
|
|
|
@@ -20,8 +21,11 @@ def jpipe(consume: Optional[List[str]] = None, produce: Optional[List[str]] = No
|
|
|
20
21
|
produce = produce or []
|
|
21
22
|
|
|
22
23
|
def decorator(func: Callable) -> Callable:
|
|
24
|
+
_check_produce_param(func, produce)
|
|
23
25
|
consume_checker = _init_checker(ConsumedVariableChecker, func, consume)
|
|
24
26
|
produce_checker = _init_checker(ProducedVariableChecker, func, produce)
|
|
27
|
+
params = list(inspect.signature(func).parameters)
|
|
28
|
+
_has_produce_slot = bool(params) and params[-1] == "produce"
|
|
25
29
|
|
|
26
30
|
@wraps(func)
|
|
27
31
|
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
@@ -29,6 +33,8 @@ def jpipe(consume: Optional[List[str]] = None, produce: Optional[List[str]] = No
|
|
|
29
33
|
kwargs = consume_checker.inject_arguments(kwargs)
|
|
30
34
|
if produce_checker:
|
|
31
35
|
kwargs["produce"] = produce_checker.produce
|
|
36
|
+
elif _has_produce_slot:
|
|
37
|
+
kwargs["produce"] = _noop_produce
|
|
32
38
|
|
|
33
39
|
result = func(*args, **kwargs)
|
|
34
40
|
|
|
@@ -42,6 +48,26 @@ def jpipe(consume: Optional[List[str]] = None, produce: Optional[List[str]] = No
|
|
|
42
48
|
return decorator
|
|
43
49
|
|
|
44
50
|
|
|
51
|
+
def _check_produce_param(func: Callable, produce: List[str]) -> None:
|
|
52
|
+
if not produce:
|
|
53
|
+
return
|
|
54
|
+
params = list(inspect.signature(func).parameters)
|
|
55
|
+
if not params or params[-1] != "produce":
|
|
56
|
+
raise ValueError(
|
|
57
|
+
f"[jpipe] Function '{func.__name__}' declares produce={produce!r} "
|
|
58
|
+
f"but its last parameter is not 'produce'.\n"
|
|
59
|
+
f" • Actual signature: {func.__name__}({', '.join(params)})\n"
|
|
60
|
+
f" • Fix: add 'produce' as the final parameter, "
|
|
61
|
+
f"typed as Callable[[str, Any], None]."
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _noop_produce(name: str, value: Any) -> None:
|
|
66
|
+
# Intentional no-op: allows functions to keep a produce param for consistency
|
|
67
|
+
# when produce=[] — calls are accepted and silently discarded.
|
|
68
|
+
pass
|
|
69
|
+
|
|
70
|
+
|
|
45
71
|
def _init_checker(
|
|
46
72
|
CheckerClass: type, func: Callable, params: List[str]
|
|
47
73
|
) -> "ConsumedVariableChecker | ProducedVariableChecker | None":
|
|
@@ -92,6 +118,15 @@ class ConsumedVariableChecker:
|
|
|
92
118
|
ctx._set(self.func_name, param, None, RuntimeContext.CONSUME)
|
|
93
119
|
GLOBAL_LOGGER.debug(f"[{self.func_name}] Variable '{param}' registered as CONSUME")
|
|
94
120
|
|
|
121
|
+
unused = set(self.declared_params) - self.used_params
|
|
122
|
+
if unused:
|
|
123
|
+
raise ValueError(
|
|
124
|
+
f"[jpipe] Function '{self.func_name}' declares consumed variable(s) "
|
|
125
|
+
f"{sorted(unused)!r} that are never referenced in the function body.\n"
|
|
126
|
+
f" • Fix: either reference each variable in the function body, "
|
|
127
|
+
f"or remove it from consume=[]."
|
|
128
|
+
)
|
|
129
|
+
|
|
95
130
|
def inject_arguments(self, kwargs: dict) -> dict:
|
|
96
131
|
"""
|
|
97
132
|
Injects variable values from the context into the function’s keyword arguments.
|
|
@@ -101,10 +136,6 @@ class ConsumedVariableChecker:
|
|
|
101
136
|
:raises ValueError: If a declared variable is missing from the context.
|
|
102
137
|
"""
|
|
103
138
|
for param in self.declared_params:
|
|
104
|
-
if param not in self.used_params:
|
|
105
|
-
GLOBAL_LOGGER.warning(
|
|
106
|
-
f"Consumed variable '{param}' is declared but not used in function '{self.func_name}'."
|
|
107
|
-
)
|
|
108
139
|
value = ctx.get(param)
|
|
109
140
|
GLOBAL_LOGGER.debug(
|
|
110
141
|
f"[{self.func_name}] Injecting consumed variable '{param}' = {value}"
|
|
@@ -135,7 +166,8 @@ class ConsumedVariableChecker:
|
|
|
135
166
|
self.used_vars: set[str] = set()
|
|
136
167
|
|
|
137
168
|
def visit_Name(self, node: ast.Name) -> None:
|
|
138
|
-
|
|
169
|
+
if isinstance(node.ctx, ast.Load):
|
|
170
|
+
self.used_vars.add(node.id)
|
|
139
171
|
|
|
140
172
|
visitor = VarVisitor()
|
|
141
173
|
visitor.visit(tree)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
from typing import Callable, TypeVar
|
|
2
|
+
|
|
3
|
+
F = TypeVar("F", bound=Callable)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def jpipe_link(element_id: str) -> Callable[[F], F]:
|
|
7
|
+
"""
|
|
8
|
+
Decorator to explicitly bind a Python function to a JSON pipeline element by its id.
|
|
9
|
+
|
|
10
|
+
This removes the fragile label-to-function-name mapping: instead of requiring
|
|
11
|
+
the function name to match the sanitized form of the JSON label, the decorator
|
|
12
|
+
records the element id directly on the function.
|
|
13
|
+
|
|
14
|
+
Compatible with @jpipe, @skip, and @contribution in any stacking order, because
|
|
15
|
+
all three use @wraps which propagates __dict__ attributes to their wrappers.
|
|
16
|
+
|
|
17
|
+
:param element_id: The id of the JSON element this function implements (e.g. "E1").
|
|
18
|
+
:type element_id: str
|
|
19
|
+
|
|
20
|
+
Example::
|
|
21
|
+
|
|
22
|
+
@jpipe_link("E1")
|
|
23
|
+
@jpipe(consume=["file_path"], produce=["file_exists"])
|
|
24
|
+
def check_file(file_path, produce):
|
|
25
|
+
produce("file_exists", os.path.isfile(file_path))
|
|
26
|
+
return True
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
def decorator(func: F) -> F:
|
|
30
|
+
func.__jpipe_link_id__ = element_id
|
|
31
|
+
return func
|
|
32
|
+
|
|
33
|
+
return decorator
|
|
@@ -1,6 +1,4 @@
|
|
|
1
1
|
import json
|
|
2
|
-
import logging
|
|
3
|
-
from pathlib import Path
|
|
4
2
|
from typing import Any, Iterable, Iterator, Optional, Tuple
|
|
5
3
|
|
|
6
4
|
import networkx as nx
|
|
@@ -11,6 +9,7 @@ from ..exceptions import FunctionException
|
|
|
11
9
|
from ..runtime import PythonRuntime
|
|
12
10
|
from ..utils import normalize_structure, parse_value, sanitize_string
|
|
13
11
|
from .context import RuntimeContext, ctx
|
|
12
|
+
from .graphviz_exporter import GraphvizExporter
|
|
14
13
|
from .logger import GLOBAL_LOGGER
|
|
15
14
|
from .validators import (
|
|
16
15
|
DuplicateProducerValidator,
|
|
@@ -20,6 +19,7 @@ from .validators import (
|
|
|
20
19
|
OrderValidator,
|
|
21
20
|
ProducedButNotConsumedValidator,
|
|
22
21
|
SelfDependencyValidator,
|
|
22
|
+
UnboundElementValidator,
|
|
23
23
|
)
|
|
24
24
|
|
|
25
25
|
|
|
@@ -58,6 +58,7 @@ class PipelineEngine:
|
|
|
58
58
|
"""
|
|
59
59
|
GLOBAL_LOGGER.info("Initializing PipelineEngine...")
|
|
60
60
|
self.justification_name = "Unknown Justification"
|
|
61
|
+
self._link_registry: dict[str, str] = {}
|
|
61
62
|
self.load_config(config_path, variables)
|
|
62
63
|
if justification_path:
|
|
63
64
|
self.graph = self.parse_justification(justification_path)
|
|
@@ -270,6 +271,7 @@ class PipelineEngine:
|
|
|
270
271
|
ProducedButNotConsumedValidator(self, ctx),
|
|
271
272
|
DuplicateProducerValidator(self, ctx),
|
|
272
273
|
EvidenceDependencyValidator(self, ctx, self.graph),
|
|
274
|
+
UnboundElementValidator(self, ctx, self._link_registry),
|
|
273
275
|
]
|
|
274
276
|
|
|
275
277
|
all_passed = True
|
|
@@ -312,7 +314,12 @@ class PipelineEngine:
|
|
|
312
314
|
cycle = None
|
|
313
315
|
|
|
314
316
|
if cycle:
|
|
315
|
-
cycle_labels = [
|
|
317
|
+
cycle_labels = [
|
|
318
|
+
f"{label} ({self._qualified_id(n)})"
|
|
319
|
+
if (label := self.graph.nodes[n].get("label"))
|
|
320
|
+
else self._qualified_id(n)
|
|
321
|
+
for n in cycle
|
|
322
|
+
]
|
|
316
323
|
error_msg = (
|
|
317
324
|
"[ExecutionOrder]\n"
|
|
318
325
|
"Pipeline validation error: cycle detected in justification graph.\n"
|
|
@@ -358,6 +365,8 @@ class PipelineEngine:
|
|
|
358
365
|
"""
|
|
359
366
|
GLOBAL_LOGGER.info("Running pipeline...")
|
|
360
367
|
|
|
368
|
+
self._apply_link_registry(runtime.build_link_registry())
|
|
369
|
+
|
|
361
370
|
if not self._validate_pipeline():
|
|
362
371
|
return
|
|
363
372
|
|
|
@@ -368,6 +377,53 @@ class PipelineEngine:
|
|
|
368
377
|
for node in execution_order:
|
|
369
378
|
yield self._process_node(node, runtime, dry_run)
|
|
370
379
|
|
|
380
|
+
def _apply_link_registry(self, registry: dict[str, str]) -> None:
|
|
381
|
+
"""
|
|
382
|
+
Override the stored function_name for nodes that have an explicit @jpipe_link binding.
|
|
383
|
+
|
|
384
|
+
Called before validation so that all downstream lookups (skip checks, contribution
|
|
385
|
+
lookups, function dispatch) use the correct function name instead of the sanitized label.
|
|
386
|
+
|
|
387
|
+
Supports two id formats:
|
|
388
|
+
- Plain element id: ``"e1"``
|
|
389
|
+
- Qualified id with justification name: ``"performant:e1"``
|
|
390
|
+
|
|
391
|
+
:param registry: Mapping of link_id → attr_name produced by runtime.build_link_registry().
|
|
392
|
+
:type registry: dict[str, str]
|
|
393
|
+
"""
|
|
394
|
+
self._link_registry = registry
|
|
395
|
+
for link_id, attr_name in registry.items():
|
|
396
|
+
node_id = self._resolve_node_id(link_id)
|
|
397
|
+
if node_id is not None:
|
|
398
|
+
GLOBAL_LOGGER.debug(
|
|
399
|
+
"Linking node '%s' to function '%s' via @jpipe_link.", node_id, attr_name
|
|
400
|
+
)
|
|
401
|
+
self.graph.nodes[node_id]["function_name"] = attr_name
|
|
402
|
+
|
|
403
|
+
def _qualified_id(self, node_id: str) -> str:
|
|
404
|
+
"""Return the fully qualified node id (``justification_name:node_id``)."""
|
|
405
|
+
return f"{self.justification_name}:{node_id}"
|
|
406
|
+
|
|
407
|
+
def _resolve_node_id(self, link_id: str) -> str | None:
|
|
408
|
+
"""
|
|
409
|
+
Resolve a @jpipe_link id to a graph node id.
|
|
410
|
+
|
|
411
|
+
Accepts plain ids (``"e1"``) or qualified ids (``"justification_name:e1"``).
|
|
412
|
+
For qualified ids the justification-name prefix must match this pipeline's name.
|
|
413
|
+
|
|
414
|
+
:param link_id: The id value passed to @jpipe_link.
|
|
415
|
+
:type link_id: str
|
|
416
|
+
:return: The matching graph node id, or None if not found.
|
|
417
|
+
:rtype: str | None
|
|
418
|
+
"""
|
|
419
|
+
if link_id in self.graph.nodes:
|
|
420
|
+
return link_id
|
|
421
|
+
if ":" in link_id:
|
|
422
|
+
qualifier, element_id = link_id.rsplit(":", 1)
|
|
423
|
+
if qualifier == self.justification_name and element_id in self.graph.nodes:
|
|
424
|
+
return element_id
|
|
425
|
+
return None
|
|
426
|
+
|
|
371
427
|
def _validate_pipeline(self) -> bool:
|
|
372
428
|
"""
|
|
373
429
|
Validates the justification graph and updates visualization markers accordingly.
|
|
@@ -417,7 +473,7 @@ class PipelineEngine:
|
|
|
417
473
|
node_data = self.graph.nodes[node]
|
|
418
474
|
node_type = node_data.get("type")
|
|
419
475
|
label = node_data.get("label")
|
|
420
|
-
fn_name =
|
|
476
|
+
fn_name = node_data.get("function_name")
|
|
421
477
|
exception = None
|
|
422
478
|
|
|
423
479
|
GLOBAL_LOGGER.debug("Processing node: %s", node)
|
|
@@ -428,7 +484,7 @@ class PipelineEngine:
|
|
|
428
484
|
status = StatusType.SKIP
|
|
429
485
|
exception = skip_config.get("reason", "Skipped by context")
|
|
430
486
|
GLOBAL_LOGGER.info(
|
|
431
|
-
f"Skipping function '{fn_name}' for node '{node}' due to context: {exception}"
|
|
487
|
+
f"Skipping function '{fn_name}' for node '{self._qualified_id(node)}' due to context: {exception}"
|
|
432
488
|
)
|
|
433
489
|
|
|
434
490
|
# --- Check if predecessor failure or implicit skip should block execution ---
|
|
@@ -436,12 +492,17 @@ class PipelineEngine:
|
|
|
436
492
|
status = StatusType.SKIP
|
|
437
493
|
|
|
438
494
|
# --- Attempt function execution (or dry-run) ---
|
|
439
|
-
elif node_type in {"evidence", "strategy"}
|
|
495
|
+
elif node_type in {"evidence", "strategy"} or (
|
|
496
|
+
node_type == "sub-conclusion" and (
|
|
497
|
+
node in self._link_registry
|
|
498
|
+
or f"{self.justification_name}:{node}" in self._link_registry
|
|
499
|
+
)
|
|
500
|
+
):
|
|
440
501
|
status, exception = self._execute_justification_fn(
|
|
441
502
|
label, fn_name, runtime, dry_run, node
|
|
442
503
|
)
|
|
443
504
|
|
|
444
|
-
# --- Default handling for conclusion nodes ---
|
|
505
|
+
# --- Default handling for conclusion and unbound sub-conclusion nodes ---
|
|
445
506
|
else:
|
|
446
507
|
status = StatusType.PASS
|
|
447
508
|
|
|
@@ -483,8 +544,7 @@ class PipelineEngine:
|
|
|
483
544
|
for pred in self.graph.predecessors(node):
|
|
484
545
|
pred_data = self.graph.nodes[pred]
|
|
485
546
|
status = pred_data.get("status")
|
|
486
|
-
|
|
487
|
-
fn_name = sanitize_string(pred_label)
|
|
547
|
+
fn_name = pred_data.get("function_name")
|
|
488
548
|
|
|
489
549
|
if status is None or status == StatusType.FAIL:
|
|
490
550
|
return True
|
|
@@ -524,14 +584,14 @@ class PipelineEngine:
|
|
|
524
584
|
if not isinstance(result, bool):
|
|
525
585
|
raise FunctionException(
|
|
526
586
|
f"Function '{fn_name}' returned an unexpected type: {type(result).__name__}.\n"
|
|
527
|
-
f" - The function associated with node '{node}' (label: '{label}') must return either True or False.\n"
|
|
587
|
+
f" - The function associated with node '{self._qualified_id(node)}' (label: '{label}') must return either True or False.\n"
|
|
528
588
|
f" - Received: {result!r} ({type(result).__name__})\n"
|
|
529
589
|
f" - Please ensure the function implementation returns a boolean to indicate pass/fail status correctly."
|
|
530
590
|
)
|
|
531
591
|
if not result:
|
|
532
592
|
raise FunctionException(
|
|
533
593
|
f"\nFunction '{fn_name}' returned False, indicating failure.\n"
|
|
534
|
-
f" - The function associated with node '{node}' (label: '{label}') executed but did not pass its check.\n"
|
|
594
|
+
f" - The function associated with node '{self._qualified_id(node)}' (label: '{label}') executed but did not pass its check.\n"
|
|
535
595
|
f" - Please review the implementation and input data for this function.\n"
|
|
536
596
|
f" - Returned value: {result!r}\n"
|
|
537
597
|
f" - The function must return True to indicate a successful check."
|
|
@@ -584,96 +644,11 @@ class PipelineEngine:
|
|
|
584
644
|
self, status_dict: dict[str, str], output_path: str, filename: str, format: str
|
|
585
645
|
) -> None:
|
|
586
646
|
"""
|
|
587
|
-
Export the justification graph to
|
|
588
|
-
styling nodes by VariableType and edges by status.
|
|
647
|
+
Export the justification graph to an image file.
|
|
589
648
|
|
|
590
649
|
:param status_dict: Mapping node id -> status ("PASS", "FAIL", "SKIP")
|
|
591
650
|
:param output_path: Directory path to save the exported graph image.
|
|
592
651
|
:param filename: Output filename (without extension).
|
|
593
|
-
:param format: Image format string (e.g. "png", "svg", "
|
|
652
|
+
:param format: Image format string (e.g. "png", "svg", "dot").
|
|
594
653
|
"""
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
resolved_path = self._resolve_output_path(output_path, filename)
|
|
598
|
-
|
|
599
|
-
G = self.graph.copy()
|
|
600
|
-
dot = gv.Digraph()
|
|
601
|
-
dot.attr(rankdir="BT")
|
|
602
|
-
|
|
603
|
-
self._style_nodes(dot, G, status_dict)
|
|
604
|
-
self._style_edges(dot, G, status_dict)
|
|
605
|
-
|
|
606
|
-
dot.render(
|
|
607
|
-
str(resolved_path),
|
|
608
|
-
format=format,
|
|
609
|
-
engine="dot",
|
|
610
|
-
cleanup=True,
|
|
611
|
-
outfile=str(resolved_path.with_suffix(f".{format}")),
|
|
612
|
-
)
|
|
613
|
-
|
|
614
|
-
@staticmethod
|
|
615
|
-
def _resolve_output_path(output_path: str, filename: str) -> Path:
|
|
616
|
-
"""
|
|
617
|
-
Resolve and prepare the output file path, creating the directory if needed.
|
|
618
|
-
|
|
619
|
-
:param output_path: Directory path for the output file.
|
|
620
|
-
:param filename: Output filename (without extension).
|
|
621
|
-
:return: Full Path object pointing to the output file location.
|
|
622
|
-
"""
|
|
623
|
-
path = Path(output_path)
|
|
624
|
-
if not path.exists():
|
|
625
|
-
path.mkdir(parents=True, exist_ok=True)
|
|
626
|
-
return path / filename
|
|
627
|
-
|
|
628
|
-
@staticmethod
|
|
629
|
-
def _style_nodes(dot: Any, G: nx.DiGraph, status_dict: dict[str, str]) -> None:
|
|
630
|
-
"""
|
|
631
|
-
Apply visual styles to graph nodes based on their type and execution status.
|
|
632
|
-
|
|
633
|
-
:param dot: graphviz.Digraph instance to add nodes to.
|
|
634
|
-
:param G: NetworkX DiGraph with node attribute data.
|
|
635
|
-
:param status_dict: Mapping of node id -> status string ("PASS", "FAIL", "SKIP").
|
|
636
|
-
"""
|
|
637
|
-
node_attr_map = {
|
|
638
|
-
"conclusion": {"shape": "rect", "style": "filled,rounded", "fillcolor": "lightgrey"},
|
|
639
|
-
"sub-conclusion": {"shape": "rect", "color": "#0072B2"},
|
|
640
|
-
"strategy": {"shape": "hexagon", "style": "filled", "fillcolor": "#F0C27F"},
|
|
641
|
-
"evidence": {"shape": "note", "style": "filled", "fillcolor": "#9ECAE1"},
|
|
642
|
-
"support": {"shape": "rect", "style": "dotted"},
|
|
643
|
-
}
|
|
644
|
-
for node_id, attrs in G.nodes(data=True):
|
|
645
|
-
var_type = attrs.get("type", "").lower()
|
|
646
|
-
style = dict(node_attr_map.get(
|
|
647
|
-
var_type, dict(fillcolor="white", shape="ellipse", style="filled")
|
|
648
|
-
))
|
|
649
|
-
|
|
650
|
-
status = status_dict.get(node_id, "UNKNOWN")
|
|
651
|
-
logging.info("Setting node color for %s with status %s", node_id, status)
|
|
652
|
-
if status == StatusType.FAIL.name:
|
|
653
|
-
style.update(style="filled", fillcolor="red", fontcolor="white", fontname="Helvetica-Bold")
|
|
654
|
-
elif status == StatusType.SKIP.name:
|
|
655
|
-
style.update(style="filled", fillcolor="#cccccc", opacity="1", fontcolor="white", fontname="Helvetica-Bold")
|
|
656
|
-
|
|
657
|
-
label = attrs.get("label", node_id)
|
|
658
|
-
dot.node(node_id, label=label, **style)
|
|
659
|
-
|
|
660
|
-
@staticmethod
|
|
661
|
-
def _style_edges(dot: Any, G: nx.DiGraph, status_dict: dict[str, str]) -> None:
|
|
662
|
-
"""
|
|
663
|
-
Apply visual styles to graph edges based on the execution status of their source node.
|
|
664
|
-
|
|
665
|
-
:param dot: graphviz.Digraph instance to add edges to.
|
|
666
|
-
:param G: NetworkX DiGraph with edge data.
|
|
667
|
-
:param status_dict: Mapping of node id -> status string ("PASS", "FAIL", "SKIP").
|
|
668
|
-
"""
|
|
669
|
-
for source, target in G.edges():
|
|
670
|
-
status = status_dict.get(source, "UNKNOWN")
|
|
671
|
-
logging.info("Setting edge color for %s -> %s with status %s", source, target, status)
|
|
672
|
-
if status == StatusType.PASS.name:
|
|
673
|
-
dot.edge(source, target, color="black")
|
|
674
|
-
elif status == StatusType.FAIL.name:
|
|
675
|
-
dot.edge(source, target, color="red")
|
|
676
|
-
elif status == StatusType.SKIP.name:
|
|
677
|
-
dot.edge(source, target, color="#cccccc", opacity="1")
|
|
678
|
-
else:
|
|
679
|
-
dot.edge(source, target, color="gray")
|
|
654
|
+
GraphvizExporter(self.graph).export(status_dict, output_path, filename, format)
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""
|
|
2
|
+
jpipe_runner.framework.graphviz_exporter
|
|
3
|
+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
4
|
+
|
|
5
|
+
Graphviz rendering for justification pipeline graphs.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import networkx as nx
|
|
12
|
+
|
|
13
|
+
from ..enums import StatusType
|
|
14
|
+
from .logger import GLOBAL_LOGGER
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class GraphvizExporter:
|
|
18
|
+
"""
|
|
19
|
+
Renders a justification pipeline graph to an image file via Graphviz.
|
|
20
|
+
|
|
21
|
+
Handles DOT-safe node ID sanitization: element IDs containing `:`
|
|
22
|
+
(e.g. ``final:hook``) are replaced with ``_`` in the DOT graph while
|
|
23
|
+
the human-readable label is preserved for display.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
_NODE_STYLES: dict[str, dict[str, str]] = {
|
|
27
|
+
"conclusion": {"shape": "rect", "style": "filled,rounded", "fillcolor": "lightgrey"},
|
|
28
|
+
"sub-conclusion": {"shape": "rect", "color": "#0072B2"},
|
|
29
|
+
"strategy": {"shape": "hexagon", "style": "filled", "fillcolor": "#F0C27F"},
|
|
30
|
+
"evidence": {"shape": "note", "style": "filled", "fillcolor": "#9ECAE1"},
|
|
31
|
+
"support": {"shape": "rect", "style": "dotted"},
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
def __init__(self, graph: nx.DiGraph) -> None:
|
|
35
|
+
self.graph = graph
|
|
36
|
+
|
|
37
|
+
def export(
|
|
38
|
+
self, status_dict: dict[str, str], output_path: str, filename: str, fmt: str
|
|
39
|
+
) -> None:
|
|
40
|
+
"""
|
|
41
|
+
Render the pipeline graph to an image file.
|
|
42
|
+
|
|
43
|
+
:param status_dict: Mapping of node id → status string ("PASS", "FAIL", "SKIP").
|
|
44
|
+
:param output_path: Directory path to save the exported image.
|
|
45
|
+
:param filename: Output filename (without extension).
|
|
46
|
+
:param fmt: Image format string (e.g. "png", "svg", "dot").
|
|
47
|
+
"""
|
|
48
|
+
import graphviz as gv
|
|
49
|
+
|
|
50
|
+
resolved_path = self._resolve_output_path(output_path, filename)
|
|
51
|
+
dot = gv.Digraph()
|
|
52
|
+
dot.attr(rankdir="BT")
|
|
53
|
+
self._style_nodes(dot, status_dict)
|
|
54
|
+
self._style_edges(dot, status_dict)
|
|
55
|
+
dot.render(
|
|
56
|
+
str(resolved_path),
|
|
57
|
+
format=fmt,
|
|
58
|
+
engine="dot",
|
|
59
|
+
cleanup=True,
|
|
60
|
+
outfile=str(resolved_path.with_suffix(f".{fmt}")),
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
@staticmethod
|
|
64
|
+
def _dot_id(node_id: str) -> str:
|
|
65
|
+
"""Return a DOT-safe identifier by replacing `:` with `_`."""
|
|
66
|
+
return node_id.replace(":", "_")
|
|
67
|
+
|
|
68
|
+
def _style_nodes(self, dot: Any, status_dict: dict[str, str]) -> None:
|
|
69
|
+
"""Add nodes to *dot* styled by element type and execution status."""
|
|
70
|
+
for node_id, attrs in self.graph.nodes(data=True):
|
|
71
|
+
var_type = attrs.get("type", "").lower()
|
|
72
|
+
style = dict(self._NODE_STYLES.get(
|
|
73
|
+
var_type, {"fillcolor": "white", "shape": "ellipse", "style": "filled"}
|
|
74
|
+
))
|
|
75
|
+
|
|
76
|
+
status = status_dict.get(node_id, "UNKNOWN")
|
|
77
|
+
GLOBAL_LOGGER.info("Setting node color for %s with status %s", node_id, status)
|
|
78
|
+
if status == StatusType.FAIL.name:
|
|
79
|
+
style.update(style="filled", fillcolor="red", fontcolor="white", fontname="Helvetica-Bold")
|
|
80
|
+
elif status == StatusType.SKIP.name:
|
|
81
|
+
style.update(style="filled", fillcolor="#cccccc", opacity="1", fontcolor="white", fontname="Helvetica-Bold")
|
|
82
|
+
|
|
83
|
+
label = attrs.get("label", node_id)
|
|
84
|
+
dot.node(self._dot_id(node_id), label=label, **style)
|
|
85
|
+
|
|
86
|
+
def _style_edges(self, dot: Any, status_dict: dict[str, str]) -> None:
|
|
87
|
+
"""Add edges to *dot* colored by the source node's execution status."""
|
|
88
|
+
for source, target in self.graph.edges():
|
|
89
|
+
status = status_dict.get(source, "UNKNOWN")
|
|
90
|
+
GLOBAL_LOGGER.info("Setting edge color for %s -> %s with status %s", source, target, status)
|
|
91
|
+
if status == StatusType.PASS.name:
|
|
92
|
+
dot.edge(self._dot_id(source), self._dot_id(target), color="black")
|
|
93
|
+
elif status == StatusType.FAIL.name:
|
|
94
|
+
dot.edge(self._dot_id(source), self._dot_id(target), color="red")
|
|
95
|
+
elif status == StatusType.SKIP.name:
|
|
96
|
+
dot.edge(self._dot_id(source), self._dot_id(target), color="#cccccc", opacity="1")
|
|
97
|
+
else:
|
|
98
|
+
dot.edge(self._dot_id(source), self._dot_id(target), color="gray")
|
|
99
|
+
|
|
100
|
+
@staticmethod
|
|
101
|
+
def _resolve_output_path(output_path: str, filename: str) -> Path:
|
|
102
|
+
"""Resolve and prepare the output directory, returning the full file path."""
|
|
103
|
+
path = Path(output_path)
|
|
104
|
+
if not path.exists():
|
|
105
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
106
|
+
return path / filename
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from .base import BaseValidator
|
|
2
|
+
from .duplicate_producer import DuplicateProducerValidator
|
|
3
|
+
from .evidence_dependency import EvidenceDependencyValidator
|
|
4
|
+
from .missing_variable import MissingVariableValidator
|
|
5
|
+
from .order import OrderValidator
|
|
6
|
+
from .produced_not_consumed import ProducedButNotConsumedValidator
|
|
7
|
+
from .schema import JustificationSchemaValidator
|
|
8
|
+
from .self_dependency import SelfDependencyValidator
|
|
9
|
+
from .unbound_element import UnboundElementValidator
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"BaseValidator",
|
|
13
|
+
"DuplicateProducerValidator",
|
|
14
|
+
"EvidenceDependencyValidator",
|
|
15
|
+
"JustificationSchemaValidator",
|
|
16
|
+
"MissingVariableValidator",
|
|
17
|
+
"OrderValidator",
|
|
18
|
+
"ProducedButNotConsumedValidator",
|
|
19
|
+
"SelfDependencyValidator",
|
|
20
|
+
"UnboundElementValidator",
|
|
21
|
+
]
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
from jpipe_runner.framework.context import RuntimeContext
|
|
2
|
+
|
|
3
|
+
from ..logger import GLOBAL_LOGGER # noqa: F401 — re-exported for submodule use
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class BaseValidator:
|
|
7
|
+
"""
|
|
8
|
+
Abstract base class for all pipeline validation checks.
|
|
9
|
+
|
|
10
|
+
Subclasses must implement the `validate()` method and append any errors
|
|
11
|
+
encountered during validation to `self.errors`.
|
|
12
|
+
|
|
13
|
+
:param pipeline: The pipeline engine to validate.
|
|
14
|
+
:type pipeline: PipelineEngine
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def __init__(self, pipeline: "PipelineEngine", ctx: "RuntimeContext") -> None:
|
|
18
|
+
self.pipeline = pipeline
|
|
19
|
+
self.ctx = ctx
|
|
20
|
+
self.errors: list[str] = []
|
|
21
|
+
self.warnings: list[str] = []
|
|
22
|
+
|
|
23
|
+
def validate(self) -> tuple[list[str], list[str]]:
|
|
24
|
+
"""
|
|
25
|
+
Abstract method for performing validation.
|
|
26
|
+
|
|
27
|
+
Subclasses must override this method to implement specific validation logic. This method
|
|
28
|
+
should populate `self.errors` with detailed error messages and return them.
|
|
29
|
+
|
|
30
|
+
:raises NotImplementedError: If called on the abstract base class.
|
|
31
|
+
:return: A list of error messages (if any).
|
|
32
|
+
:rtype: list[str]
|
|
33
|
+
"""
|
|
34
|
+
raise NotImplementedError("Subclasses must implement the `validate()` method.")
|
|
35
|
+
|
|
36
|
+
def _find_element_context(self, fn_name: str) -> tuple[str, str, str] | None:
|
|
37
|
+
"""Return (node_id, label, element_type) for the graph node bound to fn_name, or None."""
|
|
38
|
+
for node_id, data in self.pipeline.graph.nodes(data=True):
|
|
39
|
+
if data.get("function_name") == fn_name:
|
|
40
|
+
return node_id, data.get("label", node_id), data.get("type", "unknown")
|
|
41
|
+
return None
|