appligator 0.1.1__tar.gz → 0.2.0.dev1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. {appligator-0.1.1 → appligator-0.2.0.dev1}/.gitignore +1 -1
  2. {appligator-0.1.1 → appligator-0.2.0.dev1}/PKG-INFO +4 -4
  3. {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/gen_workflow_dag.py +10 -1
  4. {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/handlers/k8s_handler.py +8 -1
  5. {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/ir.py +7 -0
  6. {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/models.py +12 -0
  7. {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/renderer.py +52 -2
  8. appligator-0.2.0.dev1/appligator/airflow/run_step.py +232 -0
  9. {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/cli.py +64 -3
  10. {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/config.py +10 -1
  11. {appligator-0.1.1 → appligator-0.2.0.dev1}/pyproject.toml +4 -4
  12. appligator-0.1.1/appligator/airflow/run_step.py +0 -129
  13. {appligator-0.1.1 → appligator-0.2.0.dev1}/LICENSE +0 -0
  14. {appligator-0.1.1 → appligator-0.2.0.dev1}/README.md +0 -0
  15. {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/__init__.py +0 -0
  16. {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/__init__.py +0 -0
  17. {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/gen_dockerfile.py +0 -0
  18. {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/gen_image.py +0 -0
  19. {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/handlers/__init__.py +0 -0
  20. {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/handlers/base_handler.py +0 -0
  21. {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/handlers/syn_python_handler.py +0 -0
  22. {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/airflow/templates/Dockerfile.pixi.j2 +0 -0
  23. {appligator-0.1.1 → appligator-0.2.0.dev1}/appligator/py.typed +0 -0
@@ -17,7 +17,7 @@ __pycache__/
17
17
  .Python
18
18
  build/
19
19
  develop-eggs/
20
- dist/
20
+ /*/dist/
21
21
  downloads/
22
22
  eggs/
23
23
  .eggs/
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: appligator
3
- Version: 0.1.1
3
+ Version: 0.2.0.dev1
4
4
  Summary: An application package bundler
5
5
  Project-URL: Documentation, https://eo-tools.github.io/eozilla
6
6
  Project-URL: Repository, https://github.com/eo-tools/eozilla
@@ -27,14 +27,14 @@ Classifier: Topic :: Scientific/Engineering
27
27
  Classifier: Topic :: Software Development
28
28
  Classifier: Typing :: Typed
29
29
  Requires-Python: >=3.11
30
- Requires-Dist: gavicore>=0.1.1
30
+ Requires-Dist: gavicore>=0.2.0.dev1
31
31
  Requires-Dist: jinja2
32
- Requires-Dist: procodile>=0.1.1
32
+ Requires-Dist: procodile>=0.2.0.dev1
33
33
  Requires-Dist: pydantic
34
34
  Requires-Dist: pyyaml
35
35
  Requires-Dist: tomli-w
36
36
  Requires-Dist: tomli; python_version < '3.11'
37
- Requires-Dist: typer
37
+ Requires-Dist: typer<1,>=0.26.0
38
38
  Description-Content-Type: text/markdown
39
39
 
40
40
  [![CI](https://github.com/eo-tools/eozilla/actions/workflows/ci.yml/badge.svg)](https://github.com/eo-tools/eozilla/actions/workflows/ci.yml)
@@ -3,7 +3,12 @@
3
3
  # https://opensource.org/license/apache-2-0.
4
4
 
5
5
  from appligator.airflow.ir import workflow_to_ir
6
- from appligator.airflow.models import ConfigMapMount, PvcMount, ResourceRequirements
6
+ from appligator.airflow.models import (
7
+ ConfigMapMount,
8
+ PvcMount,
9
+ ResourceRequirements,
10
+ Toleration,
11
+ )
7
12
  from appligator.airflow.renderer import AirflowRenderer
8
13
  from procodile.workflow import WorkflowStepRegistry
9
14
 
@@ -16,6 +21,8 @@ def gen_workflow_dag(
16
21
  resources: ResourceRequirements | None = None,
17
22
  pvc_mounts: list[PvcMount] | None = None,
18
23
  config_map_mounts: list[ConfigMapMount] | None = None,
24
+ node_selector: dict[str, str] | None = None,
25
+ tolerations: list[Toleration] | None = None,
19
26
  ) -> str:
20
27
  """Generates a fully-formed Airflow DAG Python file."""
21
28
 
@@ -34,6 +41,8 @@ def gen_workflow_dag(
34
41
  resources=resources,
35
42
  pvc_mounts=pvc_mounts,
36
43
  config_map_mounts=config_map_mounts,
44
+ node_selector=node_selector,
45
+ tolerations=tolerations,
37
46
  )
38
47
 
39
48
  dag_code = AirflowRenderer().render(ir)
@@ -76,6 +76,13 @@ class KubernetesOperatorHandler(OperatorHandler):
76
76
  f"\n volume_mounts=[{', '.join(mount_entries)}],"
77
77
  )
78
78
 
79
+ node_selector_block = (
80
+ "\n node_selector=_node_selector," if task.node_selector else ""
81
+ )
82
+ tolerations_block = (
83
+ "\n tolerations=_tolerations," if task.tolerations else ""
84
+ )
85
+
79
86
  return f"""
80
87
  tasks["{task.id}"] = KubernetesPodOperator(
81
88
  task_id="{task.id}",
@@ -86,7 +93,7 @@ class KubernetesOperatorHandler(OperatorHandler):
86
93
  "func_qualname": "{task.func_qualname}",
87
94
  "inputs": {{{inputs}}},
88
95
  "output_keys": {task.outputs},
89
- }})],{env_from_block}{resources_block}{volumes_block}{volume_mounts_block}
96
+ }})],{env_from_block}{resources_block}{volumes_block}{volume_mounts_block}{node_selector_block}{tolerations_block}
90
97
  do_xcom_push=True,
91
98
  )
92
99
  """
@@ -9,6 +9,7 @@ from appligator.airflow.models import (
9
9
  PvcMount,
10
10
  ResourceRequirements,
11
11
  TaskIR,
12
+ Toleration,
12
13
  WorkflowIR,
13
14
  )
14
15
  from gavicore.models import InputDescription
@@ -24,6 +25,8 @@ def workflow_to_ir(
24
25
  resources: ResourceRequirements | None = None,
25
26
  pvc_mounts: list[PvcMount] | None = None,
26
27
  config_map_mounts: list[ConfigMapMount] | None = None,
28
+ node_selector: dict[str, str] | None = None,
29
+ tolerations: list[Toleration] | None = None,
27
30
  ) -> WorkflowIR:
28
31
  """
29
32
  Convert a WorkflowStepRegistry into a fully normalized WorkflowIR (Workflow
@@ -79,6 +82,8 @@ def workflow_to_ir(
79
82
  resources=resources,
80
83
  pvc_mounts=pvc_mounts or [],
81
84
  config_map_mounts=config_map_mounts or [],
85
+ node_selector=node_selector,
86
+ tolerations=tolerations,
82
87
  inputs={name: f"param:{name}" for name in params},
83
88
  outputs=list((main_step.description.outputs or {}).keys()),
84
89
  depends_on=[],
@@ -115,6 +120,8 @@ def workflow_to_ir(
115
120
  resources=resources,
116
121
  pvc_mounts=pvc_mounts or [],
117
122
  config_map_mounts=config_map_mounts or [],
123
+ node_selector=node_selector,
124
+ tolerations=tolerations,
118
125
  inputs=inputs,
119
126
  outputs=list((step.description.outputs or {}).keys()),
120
127
  depends_on=sorted(set(depends_on)),
@@ -29,6 +29,16 @@ class ConfigMapMount(BaseModel):
29
29
  sub_path: str | None = None
30
30
 
31
31
 
32
+ class Toleration(BaseModel):
33
+ """A Kubernetes pod toleration."""
34
+
35
+ key: str
36
+ operator: str = "Equal"
37
+ value: str | None = None
38
+ effect: str | None = None
39
+ toleration_seconds: int | None = None
40
+
41
+
32
42
  class ResourceRequirements(BaseModel):
33
43
  """
34
44
  CPU and memory resource requests and limits for a container.
@@ -80,6 +90,8 @@ class TaskIR(BaseModel):
80
90
  resources: ResourceRequirements | None = None
81
91
  pvc_mounts: list[PvcMount] = Field(default_factory=list)
82
92
  config_map_mounts: list[ConfigMapMount] = Field(default_factory=list)
93
+ node_selector: dict[str, str] | None = None
94
+ tolerations: list[Toleration] | None = None
83
95
 
84
96
  # Data flow
85
97
  inputs: dict[str, str] = Field(default_factory=dict)
@@ -8,7 +8,7 @@ from appligator.airflow.handlers.k8s_handler import KubernetesOperatorHandler
8
8
  from appligator.airflow.handlers.syn_python_handler import (
9
9
  SyntheticPythonsOperatorHandler,
10
10
  )
11
- from appligator.airflow.models import TaskIR, WorkflowIR
11
+ from appligator.airflow.models import TaskIR, Toleration, WorkflowIR
12
12
 
13
13
  INDENT = " "
14
14
  TAB = " "
@@ -41,11 +41,20 @@ class AirflowRenderer:
41
41
  lines: list[str] = []
42
42
 
43
43
  needs_k8s = any(
44
- t.env_from_secrets or t.resources or t.pvc_mounts or t.config_map_mounts
44
+ t.env_from_secrets
45
+ or t.resources
46
+ or t.pvc_mounts
47
+ or t.config_map_mounts
48
+ or t.node_selector
49
+ or t.tolerations
45
50
  for t in workflow.tasks
46
51
  )
47
52
  lines.extend(render_header(needs_k8s=needs_k8s))
48
53
 
54
+ shared_config = render_k8s_shared_config(workflow)
55
+ if shared_config:
56
+ lines.append(shared_config)
57
+
49
58
  lines.append(render_dag_open(workflow))
50
59
 
51
60
  for task in workflow.tasks:
@@ -66,6 +75,47 @@ class AirflowRenderer:
66
75
  raise ValueError(f"No operator adapter for task {task.id}")
67
76
 
68
77
 
78
+ def render_k8s_shared_config(workflow: WorkflowIR) -> str | None:
79
+ node_selector = next(
80
+ (t.node_selector for t in workflow.tasks if t.node_selector), None
81
+ )
82
+ tolerations = next((t.tolerations for t in workflow.tasks if t.tolerations), None)
83
+
84
+ if not node_selector and not tolerations:
85
+ return None
86
+
87
+ lines = ["# ── Shared Kubernetes config ───────────────────────────────────"]
88
+
89
+ if node_selector:
90
+ lines.append(f"_node_selector = {node_selector!r}")
91
+ lines.append("")
92
+
93
+ if tolerations:
94
+ tol_items = [_render_toleration(t) for t in tolerations]
95
+ lines.append("_tolerations = [")
96
+ for i, item in enumerate(tol_items):
97
+ sep = "," if i < len(tol_items) - 1 else ""
98
+ lines.append(f"{item}{sep}")
99
+ lines.append("]")
100
+ lines.append("")
101
+
102
+ return "\n".join(lines)
103
+
104
+
105
+ def _render_toleration(t: Toleration) -> str:
106
+ args = [" k8s.V1Toleration("]
107
+ args.append(f" key={t.key!r},")
108
+ args.append(f" operator={t.operator!r},")
109
+ if t.value is not None:
110
+ args.append(f" value={t.value!r},")
111
+ if t.effect is not None:
112
+ args.append(f" effect={t.effect!r},")
113
+ if t.toleration_seconds is not None:
114
+ args.append(f" toleration_seconds={t.toleration_seconds},")
115
+ args.append(" )")
116
+ return "\n".join(args)
117
+
118
+
69
119
  def render_header(needs_k8s: bool = False) -> list[str]:
70
120
  lines = [
71
121
  "import json",
@@ -0,0 +1,232 @@
1
+ # Copyright (c) 2026 by the Eozilla team and contributors
2
+ # Permissions are hereby granted under the terms of the Apache 2.0 License:
3
+ # https://opensource.org/license/apache-2-0.
4
+
5
+ # run_step.py
6
+ # (this is run inside the image that is used by the KPO in Airflow)
7
+
8
+ import importlib
9
+ import json
10
+ import logging
11
+ import os
12
+ import typing
13
+ from typing import Any, Callable
14
+
15
+ from pydantic import BaseModel, ValidationError
16
+
17
+ from gavicore.util.dynimp import import_value
18
+
19
+ logger = logging.getLogger("appligator.airflow")
20
+
21
+ # Prefix used to identify step input environment variables.
22
+ # Reserved for future use if inputs are passed via env vars instead of argv.
23
+ INPUT_PREFIX = "STEP_INPUT_"
24
+
25
+
26
+ def coerce_inputs(func: Callable[..., Any], inputs: dict[str, Any]) -> dict[str, Any]:
27
+ """Cast Airflow XCom inputs to the types declared in the signature of `func`.
28
+
29
+ Airflow renders all Jinja {{ params.* }} as strings, so numeric params
30
+ arrive as str even when declared as float/int in the process function.
31
+ Pydantic models (e.g. PathRef) arrive as dicts after XCom round-trip.
32
+ We use the function's type hints (with Annotated stripped) to coerce them.
33
+
34
+ Coercion is best-effort: if introspection fails for any reason (e.g. a
35
+ forward reference that can't be resolved at runtime) we fall back to
36
+ JSON-parsing string values so that "5.0" → 5.0, "true" → True, etc.
37
+ """
38
+ try:
39
+ # include_extras=False strips Annotated[X, ...] → X so we get the
40
+ # bare type (e.g. float) rather than Annotated[float, Field(...)].
41
+ hints = typing.get_type_hints(func, include_extras=False)
42
+ except Exception:
43
+ hints = {} # fall through to json.loads fallback below
44
+
45
+ coerced: dict[str, Any] = {}
46
+ for key, value in inputs.items():
47
+ if value is None:
48
+ coerced[key] = value
49
+ continue
50
+ hint = hints.get(key)
51
+ if hint in _SCALAR_TYPES and isinstance(value, str):
52
+ coerced[key] = hint(value)
53
+ elif (model_cls := _pydantic_type(hint)) and not isinstance(value, model_cls):
54
+ if isinstance(value, str):
55
+ # Airflow Jinja renders dict XCom/param values as Python repr
56
+ # strings (single-quoted, None not null). Try JSON first, then
57
+ # ast.literal_eval to recover the dict before model construction.
58
+ try:
59
+ value = json.loads(value)
60
+ except (json.JSONDecodeError, ValueError):
61
+ try:
62
+ import ast
63
+
64
+ value = ast.literal_eval(value)
65
+ except (ValueError, SyntaxError): # pragma: no cover
66
+ pass
67
+ try:
68
+ coerced[key] = model_cls.model_validate(value)
69
+ except ValidationError:
70
+ # OGC job results are a mapping of output name -> value
71
+ # (Link | QualifiedValue | InlineValue); a procodile Workflow
72
+ # called directly returns that same shape for its single
73
+ # output. The output name is arbitrary per OGC (procodile
74
+ # only defaults it to "return_value" when none is declared),
75
+ # so unwrap by shape -- a single-entry dict -- rather than
76
+ # matching a specific key.
77
+ if not isinstance(value, dict) or len(value) != 1:
78
+ raise
79
+ (inner_value,) = value.values()
80
+ coerced[key] = model_cls.model_validate(inner_value)
81
+ elif isinstance(value, str):
82
+ # Fallback: JSON-parse strings that look like non-string scalars.
83
+ # "5.0" → 5.0, "true" → True, but "dummy" stays "dummy".
84
+ try:
85
+ parsed = json.loads(value)
86
+ coerced[key] = parsed if not isinstance(parsed, str) else value
87
+ except (json.JSONDecodeError, ValueError):
88
+ coerced[key] = value
89
+ else:
90
+ coerced[key] = value
91
+ return coerced
92
+
93
+
94
+ def resolve_function(module_name: str, qualname: str):
95
+ """Import *module_name* and return the object at *qualname*.
96
+
97
+ *qualname* may be a dotted path to a nested attribute, e.g.
98
+ ``"MyClass.my_method"``, resolved via gavicore's
99
+ ``import_value("module:qualname", ...)``.
100
+
101
+ If the final segment of *qualname* is ``"function"`` and it can't be
102
+ resolved directly but the parent object is a procodile Workflow (which
103
+ stores its main callable in registry.main), the main function is
104
+ extracted from the registry instead. This handles both old procodile
105
+ builds (no .function property) and new ones transparently.
106
+ """
107
+ importlib.import_module(module_name) # let ImportError propagate as-is
108
+ try:
109
+ obj = import_value(f"{module_name}:{qualname}", type=object, name="step target")
110
+ except ValueError as e:
111
+ attr = qualname.rsplit(".", 1)[-1]
112
+ if attr != "function":
113
+ raise AttributeError(str(e)) from e
114
+
115
+ parent_qualname = qualname.rsplit(".", 1)[0] if "." in qualname else None
116
+ try:
117
+ parent = (
118
+ import_value(
119
+ f"{module_name}:{parent_qualname}", type=object, name="step target"
120
+ )
121
+ if parent_qualname
122
+ else importlib.import_module(module_name)
123
+ )
124
+ except ValueError as pe:
125
+ raise AttributeError(str(pe)) from pe
126
+
127
+ if not (hasattr(parent, "registry") and hasattr(parent.registry, "main")):
128
+ raise AttributeError(str(e)) from e
129
+
130
+ try:
131
+ main_id = next(iter(parent.registry.main))
132
+ except StopIteration:
133
+ raise AttributeError(
134
+ f"Workflow registry has no main step (attr={attr!r})"
135
+ ) from None
136
+ obj = parent.registry.main[main_id].function
137
+
138
+ # If the resolved object is a procodile Step (has a .function attribute
139
+ # holding the original callable), use the bare function so that calling it
140
+ # returns the raw result rather than a procodile-wrapped output dict.
141
+ if hasattr(obj, "function") and callable(obj.function):
142
+ obj = obj.function
143
+
144
+ return obj
145
+
146
+
147
+ def main(
148
+ *,
149
+ func_module: str,
150
+ func_qualname: str,
151
+ inputs: dict[str, Any],
152
+ output_keys: list[str] | None = None,
153
+ ):
154
+ """Entry point invoked by the Airflow KubernetesPodOperator (KPO).
155
+
156
+ The KPO launches this script inside the task container, passing a single
157
+ JSON-encoded argument via ``sys.argv[1]`` with the keys:
158
+
159
+ - ``func_module``: dotted module path of the step function.
160
+ - ``func_qualname``: qualified name of the function within that module.
161
+ - ``inputs``: keyword arguments forwarded to the function.
162
+ - ``output_keys``: names to assign to the function's return value(s).
163
+ If the function returns a tuple each element is paired with the
164
+ corresponding key; a scalar return is stored under the first key.
165
+ Omit to use the default key ``"return_value"``.
166
+
167
+ Outputs are written as JSON to the XCom file so Airflow can pass them to
168
+ downstream tasks. The XCom directory defaults to ``/airflow/xcom`` but
169
+ can be overridden via the ``AIRFLOW_XCOM_DIR`` environment variable for
170
+ local testing.
171
+ """
172
+ func = resolve_function(func_module, func_qualname)
173
+ logger.debug("RAW inputs: %s", inputs)
174
+ inputs = coerce_inputs(func, inputs)
175
+ logger.debug("COERCED inputs: %s", inputs)
176
+
177
+ result = func(**inputs)
178
+
179
+ if output_keys:
180
+ if isinstance(result, tuple):
181
+ output = dict(zip(output_keys, result, strict=False))
182
+ else:
183
+ output = {output_keys[0]: result}
184
+ else:
185
+ output = {"return_value": result}
186
+
187
+ XCOM_DIR = os.environ.get("AIRFLOW_XCOM_DIR", "/airflow/xcom")
188
+ XCOM_FILE = os.path.join(XCOM_DIR, "return.json")
189
+
190
+ os.makedirs(XCOM_DIR, exist_ok=True)
191
+ with open(XCOM_FILE, "w") as f:
192
+ json.dump(output, f, cls=_XComEncoder)
193
+
194
+
195
+ class _XComEncoder(json.JSONEncoder):
196
+ """Serialize types that json.dump can't handle natively.
197
+
198
+ Airflow reads step outputs from a JSON file (XCom). Standard json.dump
199
+ fails on Pydantic models and other domain objects, so we serialise them
200
+ here rather than requiring every step function to return plain dicts.
201
+ """
202
+
203
+ def default(self, obj):
204
+ if hasattr(obj, "model_dump"): # Pydantic models (e.g. PathRef)
205
+ return obj.model_dump()
206
+ return str(obj)
207
+
208
+
209
+ _SCALAR_TYPES = (int, float, bool)
210
+
211
+
212
+ def _pydantic_type(hint) -> type[BaseModel] | None:
213
+ """Return the Pydantic model class from a hint, including Optional[Model]."""
214
+ if isinstance(hint, type) and issubclass(hint, BaseModel):
215
+ return hint
216
+ args = typing.get_args(hint)
217
+ if args:
218
+ non_none = [a for a in args if a is not type(None)]
219
+ if (
220
+ len(non_none) == 1
221
+ and isinstance(non_none[0], type)
222
+ and issubclass(non_none[0], BaseModel)
223
+ ):
224
+ return non_none[0]
225
+ return None
226
+
227
+
228
+ if __name__ == "__main__": # pragma: no cover
229
+ import sys
230
+
231
+ payload = json.loads(sys.argv[1])
232
+ main(**payload)
@@ -17,7 +17,7 @@ CLI_NAME = "appligator"
17
17
  cli = typer.Typer(name=CLI_NAME)
18
18
 
19
19
 
20
- @cli.command()
20
+ @cli.command(name=CLI_NAME)
21
21
  def main(
22
22
  process_registry_spec: Annotated[
23
23
  str | None,
@@ -122,6 +122,27 @@ def main(
122
122
  ),
123
123
  ),
124
124
  ] = None,
125
+ node_selector: Annotated[
126
+ list[str] | None,
127
+ typer.Option(
128
+ "--node-selector",
129
+ help=(
130
+ "Node selector label for every pod. "
131
+ "Format: key=value (e.g. --node-selector pool=airflow-workers-big). Repeatable."
132
+ ),
133
+ ),
134
+ ] = None,
135
+ tolerations: Annotated[
136
+ list[str] | None,
137
+ typer.Option(
138
+ "--toleration",
139
+ help=(
140
+ "Toleration for every pod. "
141
+ "Format: key:operator[:value[:effect]] "
142
+ "(e.g. --toleration airflow/component:Equal:worker:NoSchedule). Repeatable."
143
+ ),
144
+ ),
145
+ ] = None,
125
146
  ):
126
147
  """
127
148
  Generate various application formats from your processes.
@@ -143,7 +164,12 @@ def main(
143
164
  from appligator import __version__
144
165
  from appligator.airflow.gen_image import gen_image
145
166
  from appligator.airflow.gen_workflow_dag import gen_workflow_dag
146
- from appligator.airflow.models import ConfigMapMount, PvcMount, ResourceRequirements
167
+ from appligator.airflow.models import (
168
+ ConfigMapMount,
169
+ PvcMount,
170
+ ResourceRequirements,
171
+ Toleration,
172
+ )
147
173
  from appligator.config import AppligatorConfig, load_config
148
174
  from gavicore.util.dynimp import import_value
149
175
  from procodile import ProcessRegistry
@@ -225,6 +251,39 @@ def main(
225
251
  else cfg.config_map_mounts
226
252
  )
227
253
 
254
+ parsed_node_selector: dict[str, str] = {}
255
+ for spec in node_selector or []:
256
+ if "=" not in spec:
257
+ typer.echo(f"Error: --node-selector must be key=value, got: {spec!r}")
258
+ raise typer.Exit(1)
259
+ k, v = spec.split("=", 1)
260
+ parsed_node_selector[k] = v
261
+ effective_node_selector = (
262
+ parsed_node_selector
263
+ if node_selector is not None
264
+ else (cfg.node_selector or None)
265
+ )
266
+
267
+ parsed_tolerations: list[Toleration] = []
268
+ for spec in tolerations or []:
269
+ parts = spec.split(":", 3)
270
+ if len(parts) < 2: # noqa: PLR2004
271
+ typer.echo(
272
+ f"Error: --toleration must be key:operator[:value[:effect]], got: {spec!r}"
273
+ )
274
+ raise typer.Exit(1)
275
+ parsed_tolerations.append(
276
+ Toleration(
277
+ key=parts[0],
278
+ operator=parts[1],
279
+ value=parts[2] if len(parts) > 2 and parts[2] else None, # noqa: PLR2004
280
+ effect=parts[3] if len(parts) > 3 and parts[3] else None, # noqa: PLR2004
281
+ )
282
+ )
283
+ effective_tolerations = (
284
+ parsed_tolerations if tolerations is not None else (cfg.tolerations or None)
285
+ )
286
+
228
287
  process_registry: ProcessRegistry = import_value(
229
288
  process_registry_spec,
230
289
  type=ProcessRegistry,
@@ -259,9 +318,11 @@ def main(
259
318
  resources=resources,
260
319
  pvc_mounts=effective_pvc_mounts or None,
261
320
  config_map_mounts=effective_config_map_mounts or None,
321
+ node_selector=effective_node_selector or None,
322
+ tolerations=effective_tolerations or None,
262
323
  )
263
324
  dag_file = dags_folder / f"{file_stem}.py"
264
- with dag_file.open("w") as stream:
325
+ with dag_file.open("w", encoding="utf-8") as stream:
265
326
  stream.write(
266
327
  f"# WARNING - THIS IS GENERATED CODE\n"
267
328
  f"# Generator: Eozilla Appligator v{__version__}\n"
@@ -6,7 +6,7 @@ from pathlib import Path
6
6
 
7
7
  from pydantic import BaseModel, ConfigDict, Field
8
8
 
9
- from appligator.airflow.models import ConfigMapMount, PvcMount
9
+ from appligator.airflow.models import ConfigMapMount, PvcMount, Toleration
10
10
 
11
11
 
12
12
  class AppligatorConfig(BaseModel):
@@ -37,6 +37,13 @@ class AppligatorConfig(BaseModel):
37
37
  config_map_name: s2gos-settings
38
38
  mount_path: /opt/pixi/s2gos_settings.yaml
39
39
  sub_path: s2gos_settings.yaml
40
+ node_selector:
41
+ pool: airflow-workers-big
42
+ tolerations:
43
+ - key: airflow/component
44
+ operator: Equal
45
+ value: worker
46
+ effect: NoSchedule
40
47
  """
41
48
 
42
49
  image_name: str | None = None
@@ -48,6 +55,8 @@ class AppligatorConfig(BaseModel):
48
55
  memory_limit: str | None = None
49
56
  pvc_mounts: list[PvcMount] = Field(default_factory=list)
50
57
  config_map_mounts: list[ConfigMapMount] = Field(default_factory=list)
58
+ node_selector: dict[str, str] = Field(default_factory=dict)
59
+ tolerations: list[Toleration] = Field(default_factory=list)
51
60
 
52
61
 
53
62
  def load_config(path: Path) -> AppligatorConfig:
@@ -4,7 +4,7 @@
4
4
 
5
5
  [project]
6
6
  name = "appligator"
7
- version = "0.1.1"
7
+ version = "0.2.0.dev1"
8
8
  description = "An application package bundler"
9
9
  readme = "README.md"
10
10
  authors = [
@@ -45,10 +45,10 @@ dependencies = [
45
45
  "pyyaml",
46
46
  "tomli-w",
47
47
  "tomli; python_version < '3.11'",
48
- "typer",
48
+ "typer >=0.26.0,<1",
49
49
  # local dependency
50
- "gavicore >=0.1.1",
51
- "procodile >=0.1.1",
50
+ "gavicore >=0.2.0.dev1",
51
+ "procodile >=0.2.0.dev1",
52
52
  ]
53
53
 
54
54
  [project.urls]
@@ -1,129 +0,0 @@
1
- # Copyright (c) 2026 by the Eozilla team and contributors
2
- # Permissions are hereby granted under the terms of the Apache 2.0 License:
3
- # https://opensource.org/license/apache-2-0.
4
-
5
- # run_step.py
6
- # (this is run inside the image that is used by the KPO in Airflow)
7
-
8
- import importlib
9
- import json
10
- import os
11
- import typing
12
- from typing import Any
13
-
14
- # Prefix used to identify step input environment variables.
15
- # Reserved for future use if inputs are passed via env vars instead of argv.
16
- INPUT_PREFIX = "STEP_INPUT_"
17
-
18
-
19
- class _XComEncoder(json.JSONEncoder):
20
- """Serialize types that json.dump can't handle natively.
21
-
22
- Airflow reads step outputs from a JSON file (XCom). Standard json.dump
23
- fails on Pydantic models and other domain objects, so we serialise them
24
- here rather than requiring every step function to return plain dicts.
25
- """
26
-
27
- def default(self, obj):
28
- if hasattr(obj, "model_dump"): # Pydantic models (e.g. PathRef)
29
- return obj.model_dump()
30
- return str(obj)
31
-
32
-
33
- _SCALAR_TYPES = (int, float, bool)
34
-
35
-
36
- def coerce_inputs(func, inputs: dict[str, Any]) -> dict[str, Any]:
37
- """Cast string inputs to the types declared in func's signature.
38
-
39
- Airflow renders all Jinja {{ params.* }} as strings, so numeric params
40
- arrive as str even when declared as float/int in the process function.
41
- We use the function's type hints (with Annotated stripped) to coerce them.
42
-
43
- Coercion is best-effort: if introspection fails for any reason (e.g. a
44
- forward reference that can't be resolved at runtime) the original inputs
45
- are returned unchanged so the step function can handle or report the error
46
- itself.
47
- """
48
- try:
49
- # include_extras=False strips Annotated[X, ...] → X so we get the
50
- # bare type (e.g. float) rather than Annotated[float, Field(...)].
51
- hints = typing.get_type_hints(func, include_extras=False)
52
- except Exception:
53
- return inputs
54
-
55
- coerced = {}
56
- for key, value in inputs.items():
57
- hint = hints.get(key)
58
- if hint in _SCALAR_TYPES and isinstance(value, str):
59
- coerced[key] = hint(value)
60
- else:
61
- coerced[key] = value
62
- return coerced
63
-
64
-
65
- def resolve_function(module_name: str, qualname: str):
66
- """Import *module_name* and return the object at *qualname*.
67
-
68
- *qualname* may be a dotted path to a nested attribute, e.g.
69
- ``"MyClass.my_method"``, which is walked attribute-by-attribute after the
70
- module is imported.
71
- """
72
- module = importlib.import_module(module_name)
73
- obj = module
74
- for attr in qualname.split("."):
75
- obj = getattr(obj, attr)
76
- return obj
77
-
78
-
79
- def main(
80
- *,
81
- func_module: str,
82
- func_qualname: str,
83
- inputs: dict[str, Any],
84
- output_keys: list[str] | None = None,
85
- ):
86
- """Entry point invoked by the Airflow KubernetesPodOperator (KPO).
87
-
88
- The KPO launches this script inside the task container, passing a single
89
- JSON-encoded argument via ``sys.argv[1]`` with the keys:
90
-
91
- - ``func_module``: dotted module path of the step function.
92
- - ``func_qualname``: qualified name of the function within that module.
93
- - ``inputs``: keyword arguments forwarded to the function.
94
- - ``output_keys``: names to assign to the function's return value(s).
95
- If the function returns a tuple each element is paired with the
96
- corresponding key; a scalar return is stored under the first key.
97
- Omit to use the default key ``"return_value"``.
98
-
99
- Outputs are written as JSON to the XCom file so Airflow can pass them to
100
- downstream tasks. The XCom directory defaults to ``/airflow/xcom`` but
101
- can be overridden via the ``AIRFLOW_XCOM_DIR`` environment variable for
102
- local testing.
103
- """
104
- func = resolve_function(func_module, func_qualname)
105
- inputs = coerce_inputs(func, inputs)
106
-
107
- result = func(**inputs)
108
-
109
- if output_keys:
110
- if isinstance(result, tuple):
111
- output = dict(zip(output_keys, result, strict=False))
112
- else:
113
- output = {output_keys[0]: result}
114
- else:
115
- output = {"return_value": result}
116
-
117
- XCOM_DIR = os.environ.get("AIRFLOW_XCOM_DIR", "/airflow/xcom")
118
- XCOM_FILE = os.path.join(XCOM_DIR, "return.json")
119
-
120
- os.makedirs(XCOM_DIR, exist_ok=True)
121
- with open(XCOM_FILE, "w") as f:
122
- json.dump(output, f, cls=_XComEncoder)
123
-
124
-
125
- if __name__ == "__main__": # pragma: no cover
126
- import sys
127
-
128
- payload = json.loads(sys.argv[1])
129
- main(**payload)
File without changes
File without changes