appligator 0.1.1__tar.gz → 0.2.0.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {appligator-0.1.1 → appligator-0.2.0.dev1}/.gitignore +1 -1
- {appligator-0.1.1 → appligator-0.2.0.dev1}/PKG-INFO +4 -4
- {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/gen_workflow_dag.py +10 -1
- {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/handlers/k8s_handler.py +8 -1
- {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/ir.py +7 -0
- {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/models.py +12 -0
- {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/renderer.py +52 -2
- appligator-0.2.0.dev1/appligator/airflow/run_step.py +232 -0
- {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/cli.py +64 -3
- {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/config.py +10 -1
- {appligator-0.1.1 → appligator-0.2.0.dev1}/pyproject.toml +4 -4
- appligator-0.1.1/appligator/airflow/run_step.py +0 -129
- {appligator-0.1.1 → appligator-0.2.0.dev1}/LICENSE +0 -0
- {appligator-0.1.1 → appligator-0.2.0.dev1}/README.md +0 -0
- {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/__init__.py +0 -0
- {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/__init__.py +0 -0
- {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/gen_dockerfile.py +0 -0
- {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/gen_image.py +0 -0
- {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/handlers/__init__.py +0 -0
- {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/handlers/base_handler.py +0 -0
- {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/handlers/syn_python_handler.py +0 -0
- {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/templates/Dockerfile.pixi.j2 +0 -0
- {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: appligator
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0.dev1
|
|
4
4
|
Summary: An application package bundler
|
|
5
5
|
Project-URL: Documentation, https://eo-tools.github.io/eozilla
|
|
6
6
|
Project-URL: Repository, https://github.com/eo-tools/eozilla
|
|
@@ -27,14 +27,14 @@ Classifier: Topic :: Scientific/Engineering
|
|
|
27
27
|
Classifier: Topic :: Software Development
|
|
28
28
|
Classifier: Typing :: Typed
|
|
29
29
|
Requires-Python: >=3.11
|
|
30
|
-
Requires-Dist: gavicore>=0.
|
|
30
|
+
Requires-Dist: gavicore>=0.2.0.dev1
|
|
31
31
|
Requires-Dist: jinja2
|
|
32
|
-
Requires-Dist: procodile>=0.
|
|
32
|
+
Requires-Dist: procodile>=0.2.0.dev1
|
|
33
33
|
Requires-Dist: pydantic
|
|
34
34
|
Requires-Dist: pyyaml
|
|
35
35
|
Requires-Dist: tomli-w
|
|
36
36
|
Requires-Dist: tomli; python_version < '3.11'
|
|
37
|
-
Requires-Dist: typer
|
|
37
|
+
Requires-Dist: typer<1,>=0.26.0
|
|
38
38
|
Description-Content-Type: text/markdown
|
|
39
39
|
|
|
40
40
|
[](https://github.com/eo-tools/eozilla/actions/workflows/ci.yml)
|
|
@@ -3,7 +3,12 @@
|
|
|
3
3
|
# https://opensource.org/license/apache-2-0.
|
|
4
4
|
|
|
5
5
|
from appligator.airflow.ir import workflow_to_ir
|
|
6
|
-
from appligator.airflow.models import
|
|
6
|
+
from appligator.airflow.models import (
|
|
7
|
+
ConfigMapMount,
|
|
8
|
+
PvcMount,
|
|
9
|
+
ResourceRequirements,
|
|
10
|
+
Toleration,
|
|
11
|
+
)
|
|
7
12
|
from appligator.airflow.renderer import AirflowRenderer
|
|
8
13
|
from procodile.workflow import WorkflowStepRegistry
|
|
9
14
|
|
|
@@ -16,6 +21,8 @@ def gen_workflow_dag(
|
|
|
16
21
|
resources: ResourceRequirements | None = None,
|
|
17
22
|
pvc_mounts: list[PvcMount] | None = None,
|
|
18
23
|
config_map_mounts: list[ConfigMapMount] | None = None,
|
|
24
|
+
node_selector: dict[str, str] | None = None,
|
|
25
|
+
tolerations: list[Toleration] | None = None,
|
|
19
26
|
) -> str:
|
|
20
27
|
"""Generates a fully-formed Airflow DAG Python file."""
|
|
21
28
|
|
|
@@ -34,6 +41,8 @@ def gen_workflow_dag(
|
|
|
34
41
|
resources=resources,
|
|
35
42
|
pvc_mounts=pvc_mounts,
|
|
36
43
|
config_map_mounts=config_map_mounts,
|
|
44
|
+
node_selector=node_selector,
|
|
45
|
+
tolerations=tolerations,
|
|
37
46
|
)
|
|
38
47
|
|
|
39
48
|
dag_code = AirflowRenderer().render(ir)
|
|
@@ -76,6 +76,13 @@ class KubernetesOperatorHandler(OperatorHandler):
|
|
|
76
76
|
f"\n volume_mounts=[{', '.join(mount_entries)}],"
|
|
77
77
|
)
|
|
78
78
|
|
|
79
|
+
node_selector_block = (
|
|
80
|
+
"\n node_selector=_node_selector," if task.node_selector else ""
|
|
81
|
+
)
|
|
82
|
+
tolerations_block = (
|
|
83
|
+
"\n tolerations=_tolerations," if task.tolerations else ""
|
|
84
|
+
)
|
|
85
|
+
|
|
79
86
|
return f"""
|
|
80
87
|
tasks["{task.id}"] = KubernetesPodOperator(
|
|
81
88
|
task_id="{task.id}",
|
|
@@ -86,7 +93,7 @@ class KubernetesOperatorHandler(OperatorHandler):
|
|
|
86
93
|
"func_qualname": "{task.func_qualname}",
|
|
87
94
|
"inputs": {{{inputs}}},
|
|
88
95
|
"output_keys": {task.outputs},
|
|
89
|
-
}})],{env_from_block}{resources_block}{volumes_block}{volume_mounts_block}
|
|
96
|
+
}})],{env_from_block}{resources_block}{volumes_block}{volume_mounts_block}{node_selector_block}{tolerations_block}
|
|
90
97
|
do_xcom_push=True,
|
|
91
98
|
)
|
|
92
99
|
"""
|
|
@@ -9,6 +9,7 @@ from appligator.airflow.models import (
|
|
|
9
9
|
PvcMount,
|
|
10
10
|
ResourceRequirements,
|
|
11
11
|
TaskIR,
|
|
12
|
+
Toleration,
|
|
12
13
|
WorkflowIR,
|
|
13
14
|
)
|
|
14
15
|
from gavicore.models import InputDescription
|
|
@@ -24,6 +25,8 @@ def workflow_to_ir(
|
|
|
24
25
|
resources: ResourceRequirements | None = None,
|
|
25
26
|
pvc_mounts: list[PvcMount] | None = None,
|
|
26
27
|
config_map_mounts: list[ConfigMapMount] | None = None,
|
|
28
|
+
node_selector: dict[str, str] | None = None,
|
|
29
|
+
tolerations: list[Toleration] | None = None,
|
|
27
30
|
) -> WorkflowIR:
|
|
28
31
|
"""
|
|
29
32
|
Convert a WorkflowStepRegistry into a fully normalized WorkflowIR (Workflow
|
|
@@ -79,6 +82,8 @@ def workflow_to_ir(
|
|
|
79
82
|
resources=resources,
|
|
80
83
|
pvc_mounts=pvc_mounts or [],
|
|
81
84
|
config_map_mounts=config_map_mounts or [],
|
|
85
|
+
node_selector=node_selector,
|
|
86
|
+
tolerations=tolerations,
|
|
82
87
|
inputs={name: f"param:{name}" for name in params},
|
|
83
88
|
outputs=list((main_step.description.outputs or {}).keys()),
|
|
84
89
|
depends_on=[],
|
|
@@ -115,6 +120,8 @@ def workflow_to_ir(
|
|
|
115
120
|
resources=resources,
|
|
116
121
|
pvc_mounts=pvc_mounts or [],
|
|
117
122
|
config_map_mounts=config_map_mounts or [],
|
|
123
|
+
node_selector=node_selector,
|
|
124
|
+
tolerations=tolerations,
|
|
118
125
|
inputs=inputs,
|
|
119
126
|
outputs=list((step.description.outputs or {}).keys()),
|
|
120
127
|
depends_on=sorted(set(depends_on)),
|
|
@@ -29,6 +29,16 @@ class ConfigMapMount(BaseModel):
|
|
|
29
29
|
sub_path: str | None = None
|
|
30
30
|
|
|
31
31
|
|
|
32
|
+
class Toleration(BaseModel):
|
|
33
|
+
"""A Kubernetes pod toleration."""
|
|
34
|
+
|
|
35
|
+
key: str
|
|
36
|
+
operator: str = "Equal"
|
|
37
|
+
value: str | None = None
|
|
38
|
+
effect: str | None = None
|
|
39
|
+
toleration_seconds: int | None = None
|
|
40
|
+
|
|
41
|
+
|
|
32
42
|
class ResourceRequirements(BaseModel):
|
|
33
43
|
"""
|
|
34
44
|
CPU and memory resource requests and limits for a container.
|
|
@@ -80,6 +90,8 @@ class TaskIR(BaseModel):
|
|
|
80
90
|
resources: ResourceRequirements | None = None
|
|
81
91
|
pvc_mounts: list[PvcMount] = Field(default_factory=list)
|
|
82
92
|
config_map_mounts: list[ConfigMapMount] = Field(default_factory=list)
|
|
93
|
+
node_selector: dict[str, str] | None = None
|
|
94
|
+
tolerations: list[Toleration] | None = None
|
|
83
95
|
|
|
84
96
|
# Data flow
|
|
85
97
|
inputs: dict[str, str] = Field(default_factory=dict)
|
|
@@ -8,7 +8,7 @@ from appligator.airflow.handlers.k8s_handler import KubernetesOperatorHandler
|
|
|
8
8
|
from appligator.airflow.handlers.syn_python_handler import (
|
|
9
9
|
SyntheticPythonsOperatorHandler,
|
|
10
10
|
)
|
|
11
|
-
from appligator.airflow.models import TaskIR, WorkflowIR
|
|
11
|
+
from appligator.airflow.models import TaskIR, Toleration, WorkflowIR
|
|
12
12
|
|
|
13
13
|
INDENT = " "
|
|
14
14
|
TAB = " "
|
|
@@ -41,11 +41,20 @@ class AirflowRenderer:
|
|
|
41
41
|
lines: list[str] = []
|
|
42
42
|
|
|
43
43
|
needs_k8s = any(
|
|
44
|
-
t.env_from_secrets
|
|
44
|
+
t.env_from_secrets
|
|
45
|
+
or t.resources
|
|
46
|
+
or t.pvc_mounts
|
|
47
|
+
or t.config_map_mounts
|
|
48
|
+
or t.node_selector
|
|
49
|
+
or t.tolerations
|
|
45
50
|
for t in workflow.tasks
|
|
46
51
|
)
|
|
47
52
|
lines.extend(render_header(needs_k8s=needs_k8s))
|
|
48
53
|
|
|
54
|
+
shared_config = render_k8s_shared_config(workflow)
|
|
55
|
+
if shared_config:
|
|
56
|
+
lines.append(shared_config)
|
|
57
|
+
|
|
49
58
|
lines.append(render_dag_open(workflow))
|
|
50
59
|
|
|
51
60
|
for task in workflow.tasks:
|
|
@@ -66,6 +75,47 @@ class AirflowRenderer:
|
|
|
66
75
|
raise ValueError(f"No operator adapter for task {task.id}")
|
|
67
76
|
|
|
68
77
|
|
|
78
|
+
def render_k8s_shared_config(workflow: WorkflowIR) -> str | None:
|
|
79
|
+
node_selector = next(
|
|
80
|
+
(t.node_selector for t in workflow.tasks if t.node_selector), None
|
|
81
|
+
)
|
|
82
|
+
tolerations = next((t.tolerations for t in workflow.tasks if t.tolerations), None)
|
|
83
|
+
|
|
84
|
+
if not node_selector and not tolerations:
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
lines = ["# ── Shared Kubernetes config ───────────────────────────────────"]
|
|
88
|
+
|
|
89
|
+
if node_selector:
|
|
90
|
+
lines.append(f"_node_selector = {node_selector!r}")
|
|
91
|
+
lines.append("")
|
|
92
|
+
|
|
93
|
+
if tolerations:
|
|
94
|
+
tol_items = [_render_toleration(t) for t in tolerations]
|
|
95
|
+
lines.append("_tolerations = [")
|
|
96
|
+
for i, item in enumerate(tol_items):
|
|
97
|
+
sep = "," if i < len(tol_items) - 1 else ""
|
|
98
|
+
lines.append(f"{item}{sep}")
|
|
99
|
+
lines.append("]")
|
|
100
|
+
lines.append("")
|
|
101
|
+
|
|
102
|
+
return "\n".join(lines)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _render_toleration(t: Toleration) -> str:
|
|
106
|
+
args = [" k8s.V1Toleration("]
|
|
107
|
+
args.append(f" key={t.key!r},")
|
|
108
|
+
args.append(f" operator={t.operator!r},")
|
|
109
|
+
if t.value is not None:
|
|
110
|
+
args.append(f" value={t.value!r},")
|
|
111
|
+
if t.effect is not None:
|
|
112
|
+
args.append(f" effect={t.effect!r},")
|
|
113
|
+
if t.toleration_seconds is not None:
|
|
114
|
+
args.append(f" toleration_seconds={t.toleration_seconds},")
|
|
115
|
+
args.append(" )")
|
|
116
|
+
return "\n".join(args)
|
|
117
|
+
|
|
118
|
+
|
|
69
119
|
def render_header(needs_k8s: bool = False) -> list[str]:
|
|
70
120
|
lines = [
|
|
71
121
|
"import json",
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
# Copyright (c) 2026 by the Eozilla team and contributors
|
|
2
|
+
# Permissions are hereby granted under the terms of the Apache 2.0 License:
|
|
3
|
+
# https://opensource.org/license/apache-2-0.
|
|
4
|
+
|
|
5
|
+
# run_step.py
|
|
6
|
+
# (this is run inside the image that is used by the KPO in Airflow)
|
|
7
|
+
|
|
8
|
+
import importlib
|
|
9
|
+
import json
|
|
10
|
+
import logging
|
|
11
|
+
import os
|
|
12
|
+
import typing
|
|
13
|
+
from typing import Any, Callable
|
|
14
|
+
|
|
15
|
+
from pydantic import BaseModel, ValidationError
|
|
16
|
+
|
|
17
|
+
from gavicore.util.dynimp import import_value
|
|
18
|
+
|
|
19
|
+
logger = logging.getLogger("appligator.airflow")
|
|
20
|
+
|
|
21
|
+
# Prefix used to identify step input environment variables.
|
|
22
|
+
# Reserved for future use if inputs are passed via env vars instead of argv.
|
|
23
|
+
INPUT_PREFIX = "STEP_INPUT_"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def coerce_inputs(func: Callable[..., Any], inputs: dict[str, Any]) -> dict[str, Any]:
|
|
27
|
+
"""Cast Airflow XCom inputs to the types declared in the signature of `func`.
|
|
28
|
+
|
|
29
|
+
Airflow renders all Jinja {{ params.* }} as strings, so numeric params
|
|
30
|
+
arrive as str even when declared as float/int in the process function.
|
|
31
|
+
Pydantic models (e.g. PathRef) arrive as dicts after XCom round-trip.
|
|
32
|
+
We use the function's type hints (with Annotated stripped) to coerce them.
|
|
33
|
+
|
|
34
|
+
Coercion is best-effort: if introspection fails for any reason (e.g. a
|
|
35
|
+
forward reference that can't be resolved at runtime) we fall back to
|
|
36
|
+
JSON-parsing string values so that "5.0" → 5.0, "true" → True, etc.
|
|
37
|
+
"""
|
|
38
|
+
try:
|
|
39
|
+
# include_extras=False strips Annotated[X, ...] → X so we get the
|
|
40
|
+
# bare type (e.g. float) rather than Annotated[float, Field(...)].
|
|
41
|
+
hints = typing.get_type_hints(func, include_extras=False)
|
|
42
|
+
except Exception:
|
|
43
|
+
hints = {} # fall through to json.loads fallback below
|
|
44
|
+
|
|
45
|
+
coerced: dict[str, Any] = {}
|
|
46
|
+
for key, value in inputs.items():
|
|
47
|
+
if value is None:
|
|
48
|
+
coerced[key] = value
|
|
49
|
+
continue
|
|
50
|
+
hint = hints.get(key)
|
|
51
|
+
if hint in _SCALAR_TYPES and isinstance(value, str):
|
|
52
|
+
coerced[key] = hint(value)
|
|
53
|
+
elif (model_cls := _pydantic_type(hint)) and not isinstance(value, model_cls):
|
|
54
|
+
if isinstance(value, str):
|
|
55
|
+
# Airflow Jinja renders dict XCom/param values as Python repr
|
|
56
|
+
# strings (single-quoted, None not null). Try JSON first, then
|
|
57
|
+
# ast.literal_eval to recover the dict before model construction.
|
|
58
|
+
try:
|
|
59
|
+
value = json.loads(value)
|
|
60
|
+
except (json.JSONDecodeError, ValueError):
|
|
61
|
+
try:
|
|
62
|
+
import ast
|
|
63
|
+
|
|
64
|
+
value = ast.literal_eval(value)
|
|
65
|
+
except (ValueError, SyntaxError): # pragma: no cover
|
|
66
|
+
pass
|
|
67
|
+
try:
|
|
68
|
+
coerced[key] = model_cls.model_validate(value)
|
|
69
|
+
except ValidationError:
|
|
70
|
+
# OGC job results are a mapping of output name -> value
|
|
71
|
+
# (Link | QualifiedValue | InlineValue); a procodile Workflow
|
|
72
|
+
# called directly returns that same shape for its single
|
|
73
|
+
# output. The output name is arbitrary per OGC (procodile
|
|
74
|
+
# only defaults it to "return_value" when none is declared),
|
|
75
|
+
# so unwrap by shape -- a single-entry dict -- rather than
|
|
76
|
+
# matching a specific key.
|
|
77
|
+
if not isinstance(value, dict) or len(value) != 1:
|
|
78
|
+
raise
|
|
79
|
+
(inner_value,) = value.values()
|
|
80
|
+
coerced[key] = model_cls.model_validate(inner_value)
|
|
81
|
+
elif isinstance(value, str):
|
|
82
|
+
# Fallback: JSON-parse strings that look like non-string scalars.
|
|
83
|
+
# "5.0" → 5.0, "true" → True, but "dummy" stays "dummy".
|
|
84
|
+
try:
|
|
85
|
+
parsed = json.loads(value)
|
|
86
|
+
coerced[key] = parsed if not isinstance(parsed, str) else value
|
|
87
|
+
except (json.JSONDecodeError, ValueError):
|
|
88
|
+
coerced[key] = value
|
|
89
|
+
else:
|
|
90
|
+
coerced[key] = value
|
|
91
|
+
return coerced
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def resolve_function(module_name: str, qualname: str):
|
|
95
|
+
"""Import *module_name* and return the object at *qualname*.
|
|
96
|
+
|
|
97
|
+
*qualname* may be a dotted path to a nested attribute, e.g.
|
|
98
|
+
``"MyClass.my_method"``, resolved via gavicore's
|
|
99
|
+
``import_value("module:qualname", ...)``.
|
|
100
|
+
|
|
101
|
+
If the final segment of *qualname* is ``"function"`` and it can't be
|
|
102
|
+
resolved directly but the parent object is a procodile Workflow (which
|
|
103
|
+
stores its main callable in registry.main), the main function is
|
|
104
|
+
extracted from the registry instead. This handles both old procodile
|
|
105
|
+
builds (no .function property) and new ones transparently.
|
|
106
|
+
"""
|
|
107
|
+
importlib.import_module(module_name) # let ImportError propagate as-is
|
|
108
|
+
try:
|
|
109
|
+
obj = import_value(f"{module_name}:{qualname}", type=object, name="step target")
|
|
110
|
+
except ValueError as e:
|
|
111
|
+
attr = qualname.rsplit(".", 1)[-1]
|
|
112
|
+
if attr != "function":
|
|
113
|
+
raise AttributeError(str(e)) from e
|
|
114
|
+
|
|
115
|
+
parent_qualname = qualname.rsplit(".", 1)[0] if "." in qualname else None
|
|
116
|
+
try:
|
|
117
|
+
parent = (
|
|
118
|
+
import_value(
|
|
119
|
+
f"{module_name}:{parent_qualname}", type=object, name="step target"
|
|
120
|
+
)
|
|
121
|
+
if parent_qualname
|
|
122
|
+
else importlib.import_module(module_name)
|
|
123
|
+
)
|
|
124
|
+
except ValueError as pe:
|
|
125
|
+
raise AttributeError(str(pe)) from pe
|
|
126
|
+
|
|
127
|
+
if not (hasattr(parent, "registry") and hasattr(parent.registry, "main")):
|
|
128
|
+
raise AttributeError(str(e)) from e
|
|
129
|
+
|
|
130
|
+
try:
|
|
131
|
+
main_id = next(iter(parent.registry.main))
|
|
132
|
+
except StopIteration:
|
|
133
|
+
raise AttributeError(
|
|
134
|
+
f"Workflow registry has no main step (attr={attr!r})"
|
|
135
|
+
) from None
|
|
136
|
+
obj = parent.registry.main[main_id].function
|
|
137
|
+
|
|
138
|
+
# If the resolved object is a procodile Step (has a .function attribute
|
|
139
|
+
# holding the original callable), use the bare function so that calling it
|
|
140
|
+
# returns the raw result rather than a procodile-wrapped output dict.
|
|
141
|
+
if hasattr(obj, "function") and callable(obj.function):
|
|
142
|
+
obj = obj.function
|
|
143
|
+
|
|
144
|
+
return obj
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def main(
|
|
148
|
+
*,
|
|
149
|
+
func_module: str,
|
|
150
|
+
func_qualname: str,
|
|
151
|
+
inputs: dict[str, Any],
|
|
152
|
+
output_keys: list[str] | None = None,
|
|
153
|
+
):
|
|
154
|
+
"""Entry point invoked by the Airflow KubernetesPodOperator (KPO).
|
|
155
|
+
|
|
156
|
+
The KPO launches this script inside the task container, passing a single
|
|
157
|
+
JSON-encoded argument via ``sys.argv[1]`` with the keys:
|
|
158
|
+
|
|
159
|
+
- ``func_module``: dotted module path of the step function.
|
|
160
|
+
- ``func_qualname``: qualified name of the function within that module.
|
|
161
|
+
- ``inputs``: keyword arguments forwarded to the function.
|
|
162
|
+
- ``output_keys``: names to assign to the function's return value(s).
|
|
163
|
+
If the function returns a tuple each element is paired with the
|
|
164
|
+
corresponding key; a scalar return is stored under the first key.
|
|
165
|
+
Omit to use the default key ``"return_value"``.
|
|
166
|
+
|
|
167
|
+
Outputs are written as JSON to the XCom file so Airflow can pass them to
|
|
168
|
+
downstream tasks. The XCom directory defaults to ``/airflow/xcom`` but
|
|
169
|
+
can be overridden via the ``AIRFLOW_XCOM_DIR`` environment variable for
|
|
170
|
+
local testing.
|
|
171
|
+
"""
|
|
172
|
+
func = resolve_function(func_module, func_qualname)
|
|
173
|
+
logger.debug("RAW inputs: %s", inputs)
|
|
174
|
+
inputs = coerce_inputs(func, inputs)
|
|
175
|
+
logger.debug("COERCED inputs: %s", inputs)
|
|
176
|
+
|
|
177
|
+
result = func(**inputs)
|
|
178
|
+
|
|
179
|
+
if output_keys:
|
|
180
|
+
if isinstance(result, tuple):
|
|
181
|
+
output = dict(zip(output_keys, result, strict=False))
|
|
182
|
+
else:
|
|
183
|
+
output = {output_keys[0]: result}
|
|
184
|
+
else:
|
|
185
|
+
output = {"return_value": result}
|
|
186
|
+
|
|
187
|
+
XCOM_DIR = os.environ.get("AIRFLOW_XCOM_DIR", "/airflow/xcom")
|
|
188
|
+
XCOM_FILE = os.path.join(XCOM_DIR, "return.json")
|
|
189
|
+
|
|
190
|
+
os.makedirs(XCOM_DIR, exist_ok=True)
|
|
191
|
+
with open(XCOM_FILE, "w") as f:
|
|
192
|
+
json.dump(output, f, cls=_XComEncoder)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
class _XComEncoder(json.JSONEncoder):
|
|
196
|
+
"""Serialize types that json.dump can't handle natively.
|
|
197
|
+
|
|
198
|
+
Airflow reads step outputs from a JSON file (XCom). Standard json.dump
|
|
199
|
+
fails on Pydantic models and other domain objects, so we serialise them
|
|
200
|
+
here rather than requiring every step function to return plain dicts.
|
|
201
|
+
"""
|
|
202
|
+
|
|
203
|
+
def default(self, obj):
|
|
204
|
+
if hasattr(obj, "model_dump"): # Pydantic models (e.g. PathRef)
|
|
205
|
+
return obj.model_dump()
|
|
206
|
+
return str(obj)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
_SCALAR_TYPES = (int, float, bool)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _pydantic_type(hint) -> type[BaseModel] | None:
|
|
213
|
+
"""Return the Pydantic model class from a hint, including Optional[Model]."""
|
|
214
|
+
if isinstance(hint, type) and issubclass(hint, BaseModel):
|
|
215
|
+
return hint
|
|
216
|
+
args = typing.get_args(hint)
|
|
217
|
+
if args:
|
|
218
|
+
non_none = [a for a in args if a is not type(None)]
|
|
219
|
+
if (
|
|
220
|
+
len(non_none) == 1
|
|
221
|
+
and isinstance(non_none[0], type)
|
|
222
|
+
and issubclass(non_none[0], BaseModel)
|
|
223
|
+
):
|
|
224
|
+
return non_none[0]
|
|
225
|
+
return None
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
if __name__ == "__main__": # pragma: no cover
|
|
229
|
+
import sys
|
|
230
|
+
|
|
231
|
+
payload = json.loads(sys.argv[1])
|
|
232
|
+
main(**payload)
|
|
@@ -17,7 +17,7 @@ CLI_NAME = "appligator"
|
|
|
17
17
|
cli = typer.Typer(name=CLI_NAME)
|
|
18
18
|
|
|
19
19
|
|
|
20
|
-
@cli.command()
|
|
20
|
+
@cli.command(name=CLI_NAME)
|
|
21
21
|
def main(
|
|
22
22
|
process_registry_spec: Annotated[
|
|
23
23
|
str | None,
|
|
@@ -122,6 +122,27 @@ def main(
|
|
|
122
122
|
),
|
|
123
123
|
),
|
|
124
124
|
] = None,
|
|
125
|
+
node_selector: Annotated[
|
|
126
|
+
list[str] | None,
|
|
127
|
+
typer.Option(
|
|
128
|
+
"--node-selector",
|
|
129
|
+
help=(
|
|
130
|
+
"Node selector label for every pod. "
|
|
131
|
+
"Format: key=value (e.g. --node-selector pool=airflow-workers-big). Repeatable."
|
|
132
|
+
),
|
|
133
|
+
),
|
|
134
|
+
] = None,
|
|
135
|
+
tolerations: Annotated[
|
|
136
|
+
list[str] | None,
|
|
137
|
+
typer.Option(
|
|
138
|
+
"--toleration",
|
|
139
|
+
help=(
|
|
140
|
+
"Toleration for every pod. "
|
|
141
|
+
"Format: key:operator[:value[:effect]] "
|
|
142
|
+
"(e.g. --toleration airflow/component:Equal:worker:NoSchedule). Repeatable."
|
|
143
|
+
),
|
|
144
|
+
),
|
|
145
|
+
] = None,
|
|
125
146
|
):
|
|
126
147
|
"""
|
|
127
148
|
Generate various application formats from your processes.
|
|
@@ -143,7 +164,12 @@ def main(
|
|
|
143
164
|
from appligator import __version__
|
|
144
165
|
from appligator.airflow.gen_image import gen_image
|
|
145
166
|
from appligator.airflow.gen_workflow_dag import gen_workflow_dag
|
|
146
|
-
from appligator.airflow.models import
|
|
167
|
+
from appligator.airflow.models import (
|
|
168
|
+
ConfigMapMount,
|
|
169
|
+
PvcMount,
|
|
170
|
+
ResourceRequirements,
|
|
171
|
+
Toleration,
|
|
172
|
+
)
|
|
147
173
|
from appligator.config import AppligatorConfig, load_config
|
|
148
174
|
from gavicore.util.dynimp import import_value
|
|
149
175
|
from procodile import ProcessRegistry
|
|
@@ -225,6 +251,39 @@ def main(
|
|
|
225
251
|
else cfg.config_map_mounts
|
|
226
252
|
)
|
|
227
253
|
|
|
254
|
+
parsed_node_selector: dict[str, str] = {}
|
|
255
|
+
for spec in node_selector or []:
|
|
256
|
+
if "=" not in spec:
|
|
257
|
+
typer.echo(f"Error: --node-selector must be key=value, got: {spec!r}")
|
|
258
|
+
raise typer.Exit(1)
|
|
259
|
+
k, v = spec.split("=", 1)
|
|
260
|
+
parsed_node_selector[k] = v
|
|
261
|
+
effective_node_selector = (
|
|
262
|
+
parsed_node_selector
|
|
263
|
+
if node_selector is not None
|
|
264
|
+
else (cfg.node_selector or None)
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
parsed_tolerations: list[Toleration] = []
|
|
268
|
+
for spec in tolerations or []:
|
|
269
|
+
parts = spec.split(":", 3)
|
|
270
|
+
if len(parts) < 2: # noqa: PLR2004
|
|
271
|
+
typer.echo(
|
|
272
|
+
f"Error: --toleration must be key:operator[:value[:effect]], got: {spec!r}"
|
|
273
|
+
)
|
|
274
|
+
raise typer.Exit(1)
|
|
275
|
+
parsed_tolerations.append(
|
|
276
|
+
Toleration(
|
|
277
|
+
key=parts[0],
|
|
278
|
+
operator=parts[1],
|
|
279
|
+
value=parts[2] if len(parts) > 2 and parts[2] else None, # noqa: PLR2004
|
|
280
|
+
effect=parts[3] if len(parts) > 3 and parts[3] else None, # noqa: PLR2004
|
|
281
|
+
)
|
|
282
|
+
)
|
|
283
|
+
effective_tolerations = (
|
|
284
|
+
parsed_tolerations if tolerations is not None else (cfg.tolerations or None)
|
|
285
|
+
)
|
|
286
|
+
|
|
228
287
|
process_registry: ProcessRegistry = import_value(
|
|
229
288
|
process_registry_spec,
|
|
230
289
|
type=ProcessRegistry,
|
|
@@ -259,9 +318,11 @@ def main(
|
|
|
259
318
|
resources=resources,
|
|
260
319
|
pvc_mounts=effective_pvc_mounts or None,
|
|
261
320
|
config_map_mounts=effective_config_map_mounts or None,
|
|
321
|
+
node_selector=effective_node_selector or None,
|
|
322
|
+
tolerations=effective_tolerations or None,
|
|
262
323
|
)
|
|
263
324
|
dag_file = dags_folder / f"{file_stem}.py"
|
|
264
|
-
with dag_file.open("w") as stream:
|
|
325
|
+
with dag_file.open("w", encoding="utf-8") as stream:
|
|
265
326
|
stream.write(
|
|
266
327
|
f"# WARNING - THIS IS GENERATED CODE\n"
|
|
267
328
|
f"# Generator: Eozilla Appligator v{__version__}\n"
|
|
@@ -6,7 +6,7 @@ from pathlib import Path
|
|
|
6
6
|
|
|
7
7
|
from pydantic import BaseModel, ConfigDict, Field
|
|
8
8
|
|
|
9
|
-
from appligator.airflow.models import ConfigMapMount, PvcMount
|
|
9
|
+
from appligator.airflow.models import ConfigMapMount, PvcMount, Toleration
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class AppligatorConfig(BaseModel):
|
|
@@ -37,6 +37,13 @@ class AppligatorConfig(BaseModel):
|
|
|
37
37
|
config_map_name: s2gos-settings
|
|
38
38
|
mount_path: /opt/pixi/s2gos_settings.yaml
|
|
39
39
|
sub_path: s2gos_settings.yaml
|
|
40
|
+
node_selector:
|
|
41
|
+
pool: airflow-workers-big
|
|
42
|
+
tolerations:
|
|
43
|
+
- key: airflow/component
|
|
44
|
+
operator: Equal
|
|
45
|
+
value: worker
|
|
46
|
+
effect: NoSchedule
|
|
40
47
|
"""
|
|
41
48
|
|
|
42
49
|
image_name: str | None = None
|
|
@@ -48,6 +55,8 @@ class AppligatorConfig(BaseModel):
|
|
|
48
55
|
memory_limit: str | None = None
|
|
49
56
|
pvc_mounts: list[PvcMount] = Field(default_factory=list)
|
|
50
57
|
config_map_mounts: list[ConfigMapMount] = Field(default_factory=list)
|
|
58
|
+
node_selector: dict[str, str] = Field(default_factory=dict)
|
|
59
|
+
tolerations: list[Toleration] = Field(default_factory=list)
|
|
51
60
|
|
|
52
61
|
|
|
53
62
|
def load_config(path: Path) -> AppligatorConfig:
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "appligator"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0.dev1"
|
|
8
8
|
description = "An application package bundler"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = [
|
|
@@ -45,10 +45,10 @@ dependencies = [
|
|
|
45
45
|
"pyyaml",
|
|
46
46
|
"tomli-w",
|
|
47
47
|
"tomli; python_version < '3.11'",
|
|
48
|
-
"typer",
|
|
48
|
+
"typer >=0.26.0,<1",
|
|
49
49
|
# local dependency
|
|
50
|
-
"gavicore >=0.
|
|
51
|
-
"procodile >=0.
|
|
50
|
+
"gavicore >=0.2.0.dev1",
|
|
51
|
+
"procodile >=0.2.0.dev1",
|
|
52
52
|
]
|
|
53
53
|
|
|
54
54
|
[project.urls]
|
|
@@ -1,129 +0,0 @@
|
|
|
1
|
-
# Copyright (c) 2026 by the Eozilla team and contributors
|
|
2
|
-
# Permissions are hereby granted under the terms of the Apache 2.0 License:
|
|
3
|
-
# https://opensource.org/license/apache-2-0.
|
|
4
|
-
|
|
5
|
-
# run_step.py
|
|
6
|
-
# (this is run inside the image that is used by the KPO in Airflow)
|
|
7
|
-
|
|
8
|
-
import importlib
|
|
9
|
-
import json
|
|
10
|
-
import os
|
|
11
|
-
import typing
|
|
12
|
-
from typing import Any
|
|
13
|
-
|
|
14
|
-
# Prefix used to identify step input environment variables.
|
|
15
|
-
# Reserved for future use if inputs are passed via env vars instead of argv.
|
|
16
|
-
INPUT_PREFIX = "STEP_INPUT_"
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
class _XComEncoder(json.JSONEncoder):
|
|
20
|
-
"""Serialize types that json.dump can't handle natively.
|
|
21
|
-
|
|
22
|
-
Airflow reads step outputs from a JSON file (XCom). Standard json.dump
|
|
23
|
-
fails on Pydantic models and other domain objects, so we serialise them
|
|
24
|
-
here rather than requiring every step function to return plain dicts.
|
|
25
|
-
"""
|
|
26
|
-
|
|
27
|
-
def default(self, obj):
|
|
28
|
-
if hasattr(obj, "model_dump"): # Pydantic models (e.g. PathRef)
|
|
29
|
-
return obj.model_dump()
|
|
30
|
-
return str(obj)
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
_SCALAR_TYPES = (int, float, bool)
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def coerce_inputs(func, inputs: dict[str, Any]) -> dict[str, Any]:
|
|
37
|
-
"""Cast string inputs to the types declared in func's signature.
|
|
38
|
-
|
|
39
|
-
Airflow renders all Jinja {{ params.* }} as strings, so numeric params
|
|
40
|
-
arrive as str even when declared as float/int in the process function.
|
|
41
|
-
We use the function's type hints (with Annotated stripped) to coerce them.
|
|
42
|
-
|
|
43
|
-
Coercion is best-effort: if introspection fails for any reason (e.g. a
|
|
44
|
-
forward reference that can't be resolved at runtime) the original inputs
|
|
45
|
-
are returned unchanged so the step function can handle or report the error
|
|
46
|
-
itself.
|
|
47
|
-
"""
|
|
48
|
-
try:
|
|
49
|
-
# include_extras=False strips Annotated[X, ...] → X so we get the
|
|
50
|
-
# bare type (e.g. float) rather than Annotated[float, Field(...)].
|
|
51
|
-
hints = typing.get_type_hints(func, include_extras=False)
|
|
52
|
-
except Exception:
|
|
53
|
-
return inputs
|
|
54
|
-
|
|
55
|
-
coerced = {}
|
|
56
|
-
for key, value in inputs.items():
|
|
57
|
-
hint = hints.get(key)
|
|
58
|
-
if hint in _SCALAR_TYPES and isinstance(value, str):
|
|
59
|
-
coerced[key] = hint(value)
|
|
60
|
-
else:
|
|
61
|
-
coerced[key] = value
|
|
62
|
-
return coerced
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def resolve_function(module_name: str, qualname: str):
|
|
66
|
-
"""Import *module_name* and return the object at *qualname*.
|
|
67
|
-
|
|
68
|
-
*qualname* may be a dotted path to a nested attribute, e.g.
|
|
69
|
-
``"MyClass.my_method"``, which is walked attribute-by-attribute after the
|
|
70
|
-
module is imported.
|
|
71
|
-
"""
|
|
72
|
-
module = importlib.import_module(module_name)
|
|
73
|
-
obj = module
|
|
74
|
-
for attr in qualname.split("."):
|
|
75
|
-
obj = getattr(obj, attr)
|
|
76
|
-
return obj
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
def main(
|
|
80
|
-
*,
|
|
81
|
-
func_module: str,
|
|
82
|
-
func_qualname: str,
|
|
83
|
-
inputs: dict[str, Any],
|
|
84
|
-
output_keys: list[str] | None = None,
|
|
85
|
-
):
|
|
86
|
-
"""Entry point invoked by the Airflow KubernetesPodOperator (KPO).
|
|
87
|
-
|
|
88
|
-
The KPO launches this script inside the task container, passing a single
|
|
89
|
-
JSON-encoded argument via ``sys.argv[1]`` with the keys:
|
|
90
|
-
|
|
91
|
-
- ``func_module``: dotted module path of the step function.
|
|
92
|
-
- ``func_qualname``: qualified name of the function within that module.
|
|
93
|
-
- ``inputs``: keyword arguments forwarded to the function.
|
|
94
|
-
- ``output_keys``: names to assign to the function's return value(s).
|
|
95
|
-
If the function returns a tuple each element is paired with the
|
|
96
|
-
corresponding key; a scalar return is stored under the first key.
|
|
97
|
-
Omit to use the default key ``"return_value"``.
|
|
98
|
-
|
|
99
|
-
Outputs are written as JSON to the XCom file so Airflow can pass them to
|
|
100
|
-
downstream tasks. The XCom directory defaults to ``/airflow/xcom`` but
|
|
101
|
-
can be overridden via the ``AIRFLOW_XCOM_DIR`` environment variable for
|
|
102
|
-
local testing.
|
|
103
|
-
"""
|
|
104
|
-
func = resolve_function(func_module, func_qualname)
|
|
105
|
-
inputs = coerce_inputs(func, inputs)
|
|
106
|
-
|
|
107
|
-
result = func(**inputs)
|
|
108
|
-
|
|
109
|
-
if output_keys:
|
|
110
|
-
if isinstance(result, tuple):
|
|
111
|
-
output = dict(zip(output_keys, result, strict=False))
|
|
112
|
-
else:
|
|
113
|
-
output = {output_keys[0]: result}
|
|
114
|
-
else:
|
|
115
|
-
output = {"return_value": result}
|
|
116
|
-
|
|
117
|
-
XCOM_DIR = os.environ.get("AIRFLOW_XCOM_DIR", "/airflow/xcom")
|
|
118
|
-
XCOM_FILE = os.path.join(XCOM_DIR, "return.json")
|
|
119
|
-
|
|
120
|
-
os.makedirs(XCOM_DIR, exist_ok=True)
|
|
121
|
-
with open(XCOM_FILE, "w") as f:
|
|
122
|
-
json.dump(output, f, cls=_XComEncoder)
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
if __name__ == "__main__": # pragma: no cover
|
|
126
|
-
import sys
|
|
127
|
-
|
|
128
|
-
payload = json.loads(sys.argv[1])
|
|
129
|
-
main(**payload)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/handlers/syn_python_handler.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|