tasktree 0.0.19__py3-none-any.whl → 0.0.21__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasktree/__init__.py +6 -1
- tasktree/cli.py +119 -33
- tasktree/docker.py +87 -53
- tasktree/executor.py +208 -131
- tasktree/graph.py +176 -64
- tasktree/hasher.py +68 -19
- tasktree/parser.py +408 -233
- tasktree/state.py +46 -17
- tasktree/substitution.py +193 -61
- tasktree/types.py +50 -12
- {tasktree-0.0.19.dist-info → tasktree-0.0.21.dist-info}/METADATA +364 -1
- tasktree-0.0.21.dist-info/RECORD +14 -0
- tasktree-0.0.19.dist-info/RECORD +0 -14
- {tasktree-0.0.19.dist-info → tasktree-0.0.21.dist-info}/WHEEL +0 -0
- {tasktree-0.0.19.dist-info → tasktree-0.0.21.dist-info}/entry_points.txt +0 -0
tasktree/parser.py
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""
|
|
2
|
+
Parse recipe YAML files and handle imports.
|
|
3
|
+
@athena: ec0087b16df9
|
|
4
|
+
"""
|
|
2
5
|
|
|
3
6
|
from __future__ import annotations
|
|
4
7
|
|
|
@@ -16,17 +19,22 @@ from tasktree.types import get_click_type
|
|
|
16
19
|
|
|
17
20
|
|
|
18
21
|
class CircularImportError(Exception):
|
|
19
|
-
"""
|
|
22
|
+
"""
|
|
23
|
+
Raised when a circular import is detected.
|
|
24
|
+
@athena: 935d53bc7d05
|
|
25
|
+
"""
|
|
20
26
|
pass
|
|
21
27
|
|
|
22
28
|
|
|
23
29
|
@dataclass
|
|
24
30
|
class Environment:
|
|
25
|
-
"""
|
|
31
|
+
"""
|
|
32
|
+
Represents an execution environment configuration.
|
|
26
33
|
|
|
27
34
|
Can be either a shell environment or a Docker environment:
|
|
28
35
|
- Shell environment: has 'shell' field, executes directly on host
|
|
29
36
|
- Docker environment: has 'dockerfile' field, executes in container
|
|
37
|
+
@athena: cfe3f8754968
|
|
30
38
|
"""
|
|
31
39
|
|
|
32
40
|
name: str
|
|
@@ -44,20 +52,26 @@ class Environment:
|
|
|
44
52
|
run_as_root: bool = False # If True, skip user mapping (run as root in container)
|
|
45
53
|
|
|
46
54
|
def __post_init__(self):
|
|
47
|
-
"""
|
|
55
|
+
"""
|
|
56
|
+
Ensure args is in the correct format.
|
|
57
|
+
@athena: a4292f3f4150
|
|
58
|
+
"""
|
|
48
59
|
if isinstance(self.args, str):
|
|
49
60
|
self.args = [self.args]
|
|
50
61
|
|
|
51
62
|
|
|
52
63
|
@dataclass
|
|
53
64
|
class Task:
|
|
54
|
-
"""
|
|
65
|
+
"""
|
|
66
|
+
Represents a task definition.
|
|
67
|
+
@athena: f516b5ae61c5
|
|
68
|
+
"""
|
|
55
69
|
|
|
56
70
|
name: str
|
|
57
71
|
cmd: str
|
|
58
72
|
desc: str = ""
|
|
59
73
|
deps: list[str | dict[str, Any]] = field(default_factory=list) # Can be strings or dicts with args
|
|
60
|
-
inputs: list[str] = field(default_factory=list)
|
|
74
|
+
inputs: list[str | dict[str, str]] = field(default_factory=list) # Can be strings or dicts with named inputs
|
|
61
75
|
outputs: list[str | dict[str, str]] = field(default_factory=list) # Can be strings or dicts with named outputs
|
|
62
76
|
working_dir: str = ""
|
|
63
77
|
args: list[str | dict[str, Any]] = field(default_factory=list) # Can be strings or dicts (each dict has single key: arg name)
|
|
@@ -69,8 +83,19 @@ class Task:
|
|
|
69
83
|
_output_map: dict[str, str] = field(init=False, default_factory=dict, repr=False) # name → path mapping
|
|
70
84
|
_anonymous_outputs: list[str] = field(init=False, default_factory=list, repr=False) # unnamed outputs
|
|
71
85
|
|
|
86
|
+
# Internal fields for efficient input lookup (built in __post_init__)
|
|
87
|
+
_input_map: dict[str, str] = field(init=False, default_factory=dict, repr=False) # name → path mapping
|
|
88
|
+
_anonymous_inputs: list[str] = field(init=False, default_factory=list, repr=False) # unnamed inputs
|
|
89
|
+
|
|
90
|
+
# Internal fields for positional input/output access (built in __post_init__)
|
|
91
|
+
_indexed_inputs: list[str] = field(init=False, default_factory=list, repr=False) # all inputs in YAML order
|
|
92
|
+
_indexed_outputs: list[str] = field(init=False, default_factory=list, repr=False) # all outputs in YAML order
|
|
93
|
+
|
|
72
94
|
def __post_init__(self):
|
|
73
|
-
"""
|
|
95
|
+
"""
|
|
96
|
+
Ensure lists are always lists and build input/output maps and indexed lists.
|
|
97
|
+
@athena: a48b1eba81cd
|
|
98
|
+
"""
|
|
74
99
|
if isinstance(self.deps, str):
|
|
75
100
|
self.deps = [self.deps]
|
|
76
101
|
if isinstance(self.inputs, str):
|
|
@@ -96,6 +121,7 @@ class Task:
|
|
|
96
121
|
# Build output maps for efficient lookup
|
|
97
122
|
self._output_map = {}
|
|
98
123
|
self._anonymous_outputs = []
|
|
124
|
+
self._indexed_outputs = []
|
|
99
125
|
|
|
100
126
|
for idx, output in enumerate(self.outputs):
|
|
101
127
|
if isinstance(output, dict):
|
|
@@ -124,34 +150,83 @@ class Task:
|
|
|
124
150
|
)
|
|
125
151
|
|
|
126
152
|
self._output_map[name] = path
|
|
153
|
+
self._indexed_outputs.append(path)
|
|
127
154
|
elif isinstance(output, str):
|
|
128
155
|
# Anonymous output: just store
|
|
129
156
|
self._anonymous_outputs.append(output)
|
|
157
|
+
self._indexed_outputs.append(output)
|
|
130
158
|
else:
|
|
131
159
|
raise ValueError(
|
|
132
160
|
f"Task '{self.name}': Output at index {idx} must be a string or dict, got {type(output).__name__}: {output}"
|
|
133
161
|
)
|
|
134
162
|
|
|
163
|
+
# Build input maps for efficient lookup
|
|
164
|
+
self._input_map = {}
|
|
165
|
+
self._anonymous_inputs = []
|
|
166
|
+
self._indexed_inputs = []
|
|
167
|
+
|
|
168
|
+
for idx, input_item in enumerate(self.inputs):
|
|
169
|
+
if isinstance(input_item, dict):
|
|
170
|
+
# Named input: validate and store
|
|
171
|
+
if len(input_item) != 1:
|
|
172
|
+
raise ValueError(
|
|
173
|
+
f"Task '{self.name}': Named input at index {idx} must have exactly one key-value pair, got {len(input_item)}: {input_item}"
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
name, path = next(iter(input_item.items()))
|
|
177
|
+
|
|
178
|
+
if not isinstance(path, str):
|
|
179
|
+
raise ValueError(
|
|
180
|
+
f"Task '{self.name}': Named input '{name}' must have a string path, got {type(path).__name__}: {path}"
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', name):
|
|
184
|
+
raise ValueError(
|
|
185
|
+
f"Task '{self.name}': Named input '{name}' must be a valid identifier "
|
|
186
|
+
f"(letters, numbers, underscores, cannot start with number)"
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
if name in self._input_map:
|
|
190
|
+
raise ValueError(
|
|
191
|
+
f"Task '{self.name}': Duplicate input name '{name}' at index {idx}"
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
self._input_map[name] = path
|
|
195
|
+
self._indexed_inputs.append(path)
|
|
196
|
+
elif isinstance(input_item, str):
|
|
197
|
+
# Anonymous input: just store
|
|
198
|
+
self._anonymous_inputs.append(input_item)
|
|
199
|
+
self._indexed_inputs.append(input_item)
|
|
200
|
+
else:
|
|
201
|
+
raise ValueError(
|
|
202
|
+
f"Task '{self.name}': Input at index {idx} must be a string or dict, got {type(input_item).__name__}: {input_item}"
|
|
203
|
+
)
|
|
204
|
+
|
|
135
205
|
|
|
136
206
|
@dataclass
|
|
137
207
|
class DependencySpec:
|
|
138
|
-
"""
|
|
208
|
+
"""
|
|
209
|
+
Parsed dependency specification with potential template placeholders.
|
|
139
210
|
|
|
140
211
|
This represents a dependency as defined in the recipe file, before template
|
|
141
212
|
substitution. Argument values may contain {{ arg.* }} templates that will be
|
|
142
213
|
substituted with parent task's argument values during graph construction.
|
|
143
214
|
|
|
144
215
|
Attributes:
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
216
|
+
task_name: Name of the dependency task
|
|
217
|
+
arg_templates: Dictionary mapping argument names to string templates
|
|
218
|
+
(None if no args specified). All values are strings, even
|
|
219
|
+
for numeric types, to preserve template placeholders.
|
|
220
|
+
@athena: 7b2f8a15d312
|
|
149
221
|
"""
|
|
150
222
|
task_name: str
|
|
151
223
|
arg_templates: dict[str, str] | None = None
|
|
152
224
|
|
|
153
225
|
def __str__(self) -> str:
|
|
154
|
-
"""
|
|
226
|
+
"""
|
|
227
|
+
String representation for display.
|
|
228
|
+
@athena: e5669be6329b
|
|
229
|
+
"""
|
|
155
230
|
if not self.arg_templates:
|
|
156
231
|
return self.task_name
|
|
157
232
|
args_str = ", ".join(f"{k}={v}" for k, v in self.arg_templates.items())
|
|
@@ -160,17 +235,22 @@ class DependencySpec:
|
|
|
160
235
|
|
|
161
236
|
@dataclass
|
|
162
237
|
class DependencyInvocation:
|
|
163
|
-
"""
|
|
238
|
+
"""
|
|
239
|
+
Represents a task dependency invocation with optional arguments.
|
|
164
240
|
|
|
165
241
|
Attributes:
|
|
166
|
-
|
|
167
|
-
|
|
242
|
+
task_name: Name of the dependency task
|
|
243
|
+
args: Dictionary of argument names to values (None if no args specified)
|
|
244
|
+
@athena: 0c023366160b
|
|
168
245
|
"""
|
|
169
246
|
task_name: str
|
|
170
247
|
args: dict[str, Any] | None = None
|
|
171
248
|
|
|
172
249
|
def __str__(self) -> str:
|
|
173
|
-
"""
|
|
250
|
+
"""
|
|
251
|
+
String representation for display.
|
|
252
|
+
@athena: 22fc0502192b
|
|
253
|
+
"""
|
|
174
254
|
if not self.args:
|
|
175
255
|
return self.task_name
|
|
176
256
|
args_str = ", ".join(f"{k}={v}" for k, v in self.args.items())
|
|
@@ -179,16 +259,18 @@ class DependencyInvocation:
|
|
|
179
259
|
|
|
180
260
|
@dataclass
|
|
181
261
|
class ArgSpec:
|
|
182
|
-
"""
|
|
262
|
+
"""
|
|
263
|
+
Represents a parsed argument specification.
|
|
183
264
|
|
|
184
265
|
Attributes:
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
266
|
+
name: Argument name
|
|
267
|
+
arg_type: Type of the argument (str, int, float, bool, path)
|
|
268
|
+
default: Default value as a string (None if no default)
|
|
269
|
+
is_exported: Whether the argument is exported as an environment variable
|
|
270
|
+
min_val: Minimum value for numeric arguments (None if not specified)
|
|
271
|
+
max_val: Maximum value for numeric arguments (None if not specified)
|
|
272
|
+
choices: List of valid choices for the argument (None if not specified)
|
|
273
|
+
@athena: fcaf20fb1ca2
|
|
192
274
|
"""
|
|
193
275
|
name: str
|
|
194
276
|
arg_type: str
|
|
@@ -201,7 +283,10 @@ class ArgSpec:
|
|
|
201
283
|
|
|
202
284
|
@dataclass
|
|
203
285
|
class Recipe:
|
|
204
|
-
"""
|
|
286
|
+
"""
|
|
287
|
+
Represents a parsed recipe file with all tasks.
|
|
288
|
+
@athena: 47f568c77013
|
|
289
|
+
"""
|
|
205
290
|
|
|
206
291
|
tasks: dict[str, Task]
|
|
207
292
|
project_root: Path
|
|
@@ -216,33 +301,41 @@ class Recipe:
|
|
|
216
301
|
_original_yaml_data: dict[str, Any] = field(default_factory=dict) # Store original YAML data for lazy evaluation context
|
|
217
302
|
|
|
218
303
|
def get_task(self, name: str) -> Task | None:
|
|
219
|
-
"""
|
|
304
|
+
"""
|
|
305
|
+
Get task by name.
|
|
220
306
|
|
|
221
307
|
Args:
|
|
222
|
-
|
|
308
|
+
name: Task name (may be namespaced like 'build.compile')
|
|
223
309
|
|
|
224
310
|
Returns:
|
|
225
|
-
|
|
311
|
+
Task if found, None otherwise
|
|
312
|
+
@athena: 3f8137d71757
|
|
226
313
|
"""
|
|
227
314
|
return self.tasks.get(name)
|
|
228
315
|
|
|
229
316
|
def task_names(self) -> list[str]:
|
|
230
|
-
"""
|
|
317
|
+
"""
|
|
318
|
+
Get all task names.
|
|
319
|
+
@athena: 1df54563a7b6
|
|
320
|
+
"""
|
|
231
321
|
return list(self.tasks.keys())
|
|
232
322
|
|
|
233
323
|
def get_environment(self, name: str) -> Environment | None:
|
|
234
|
-
"""
|
|
324
|
+
"""
|
|
325
|
+
Get environment by name.
|
|
235
326
|
|
|
236
327
|
Args:
|
|
237
|
-
|
|
328
|
+
name: Environment name
|
|
238
329
|
|
|
239
330
|
Returns:
|
|
240
|
-
|
|
331
|
+
Environment if found, None otherwise
|
|
332
|
+
@athena: 098227ca38a2
|
|
241
333
|
"""
|
|
242
334
|
return self.environments.get(name)
|
|
243
335
|
|
|
244
336
|
def evaluate_variables(self, root_task: str | None = None) -> None:
|
|
245
|
-
"""
|
|
337
|
+
"""
|
|
338
|
+
Evaluate variables lazily based on task reachability.
|
|
246
339
|
|
|
247
340
|
This method implements lazy variable evaluation, which only evaluates
|
|
248
341
|
variables that are actually reachable from the target task. This provides:
|
|
@@ -256,15 +349,16 @@ class Recipe:
|
|
|
256
349
|
This method is idempotent - calling it multiple times is safe (uses caching).
|
|
257
350
|
|
|
258
351
|
Args:
|
|
259
|
-
|
|
352
|
+
root_task: Optional task name to determine reachability (None = evaluate all)
|
|
260
353
|
|
|
261
354
|
Raises:
|
|
262
|
-
|
|
355
|
+
ValueError: If variable evaluation or substitution fails
|
|
263
356
|
|
|
264
357
|
Example:
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
358
|
+
>>> recipe = parse_recipe(path) # Variables not yet evaluated
|
|
359
|
+
>>> recipe.evaluate_variables("build") # Evaluate only reachable variables
|
|
360
|
+
>>> # Now recipe.evaluated_variables contains only vars used by "build" task
|
|
361
|
+
@athena: d8de7b5f42b6
|
|
268
362
|
"""
|
|
269
363
|
if self._variables_evaluated:
|
|
270
364
|
return # Already evaluated, skip (idempotent)
|
|
@@ -309,7 +403,21 @@ class Recipe:
|
|
|
309
403
|
task.cmd = substitute_variables(task.cmd, self.evaluated_variables)
|
|
310
404
|
task.desc = substitute_variables(task.desc, self.evaluated_variables)
|
|
311
405
|
task.working_dir = substitute_variables(task.working_dir, self.evaluated_variables)
|
|
312
|
-
|
|
406
|
+
|
|
407
|
+
# Substitute variables in inputs (handle both string and dict inputs)
|
|
408
|
+
resolved_inputs = []
|
|
409
|
+
for inp in task.inputs:
|
|
410
|
+
if isinstance(inp, str):
|
|
411
|
+
resolved_inputs.append(substitute_variables(inp, self.evaluated_variables))
|
|
412
|
+
elif isinstance(inp, dict):
|
|
413
|
+
# Named input: substitute the path value
|
|
414
|
+
resolved_dict = {}
|
|
415
|
+
for name, path in inp.items():
|
|
416
|
+
resolved_dict[name] = substitute_variables(path, self.evaluated_variables)
|
|
417
|
+
resolved_inputs.append(resolved_dict)
|
|
418
|
+
else:
|
|
419
|
+
resolved_inputs.append(inp)
|
|
420
|
+
task.inputs = resolved_inputs
|
|
313
421
|
|
|
314
422
|
# Substitute variables in outputs (handle both string and dict outputs)
|
|
315
423
|
resolved_outputs = []
|
|
@@ -397,7 +505,8 @@ class Recipe:
|
|
|
397
505
|
|
|
398
506
|
|
|
399
507
|
def find_recipe_file(start_dir: Path | None = None) -> Path | None:
|
|
400
|
-
"""
|
|
508
|
+
"""
|
|
509
|
+
Find recipe file in current or parent directories.
|
|
401
510
|
|
|
402
511
|
Looks for recipe files matching these patterns (in order of preference):
|
|
403
512
|
- tasktree.yaml
|
|
@@ -409,13 +518,14 @@ def find_recipe_file(start_dir: Path | None = None) -> Path | None:
|
|
|
409
518
|
with instructions to use --tasks option.
|
|
410
519
|
|
|
411
520
|
Args:
|
|
412
|
-
|
|
521
|
+
start_dir: Directory to start searching from (defaults to cwd)
|
|
413
522
|
|
|
414
523
|
Returns:
|
|
415
|
-
|
|
524
|
+
Path to recipe file if found, None otherwise
|
|
416
525
|
|
|
417
526
|
Raises:
|
|
418
|
-
|
|
527
|
+
ValueError: If multiple recipe files found in the same directory
|
|
528
|
+
@athena: 38ccb0c1bb86
|
|
419
529
|
"""
|
|
420
530
|
if start_dir is None:
|
|
421
531
|
start_dir = Path.cwd()
|
|
@@ -475,13 +585,15 @@ def find_recipe_file(start_dir: Path | None = None) -> Path | None:
|
|
|
475
585
|
|
|
476
586
|
|
|
477
587
|
def _validate_variable_name(name: str) -> None:
|
|
478
|
-
"""
|
|
588
|
+
"""
|
|
589
|
+
Validate that a variable name is a valid identifier.
|
|
479
590
|
|
|
480
591
|
Args:
|
|
481
|
-
|
|
592
|
+
name: Variable name to validate
|
|
482
593
|
|
|
483
594
|
Raises:
|
|
484
|
-
|
|
595
|
+
ValueError: If name is not a valid identifier
|
|
596
|
+
@athena: 61f92f7ad278
|
|
485
597
|
"""
|
|
486
598
|
if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', name):
|
|
487
599
|
raise ValueError(
|
|
@@ -491,16 +603,18 @@ def _validate_variable_name(name: str) -> None:
|
|
|
491
603
|
|
|
492
604
|
|
|
493
605
|
def _infer_variable_type(value: Any) -> str:
|
|
494
|
-
"""
|
|
606
|
+
"""
|
|
607
|
+
Infer type name from Python value.
|
|
495
608
|
|
|
496
609
|
Args:
|
|
497
|
-
|
|
610
|
+
value: Python value from YAML
|
|
498
611
|
|
|
499
612
|
Returns:
|
|
500
|
-
|
|
613
|
+
Type name string (str, int, float, bool)
|
|
501
614
|
|
|
502
615
|
Raises:
|
|
503
|
-
|
|
616
|
+
ValueError: If value type is not supported
|
|
617
|
+
@athena: 335ae24e1504
|
|
504
618
|
"""
|
|
505
619
|
type_map = {
|
|
506
620
|
str: "str",
|
|
@@ -518,29 +632,33 @@ def _infer_variable_type(value: Any) -> str:
|
|
|
518
632
|
|
|
519
633
|
|
|
520
634
|
def _is_env_variable_reference(value: Any) -> bool:
|
|
521
|
-
"""
|
|
635
|
+
"""
|
|
636
|
+
Check if value is an environment variable reference.
|
|
522
637
|
|
|
523
638
|
Args:
|
|
524
|
-
|
|
639
|
+
value: Raw value from YAML
|
|
525
640
|
|
|
526
641
|
Returns:
|
|
527
|
-
|
|
642
|
+
True if value is { env: VAR_NAME } dict
|
|
643
|
+
@athena: c01927ec19ef
|
|
528
644
|
"""
|
|
529
645
|
return isinstance(value, dict) and "env" in value
|
|
530
646
|
|
|
531
647
|
|
|
532
648
|
def _validate_env_variable_reference(var_name: str, value: dict) -> tuple[str, str | None]:
|
|
533
|
-
"""
|
|
649
|
+
"""
|
|
650
|
+
Validate and extract environment variable name and optional default from reference.
|
|
534
651
|
|
|
535
652
|
Args:
|
|
536
|
-
|
|
537
|
-
|
|
653
|
+
var_name: Name of the variable being defined
|
|
654
|
+
value: Dict that should be { env: ENV_VAR_NAME } or { env: ENV_VAR_NAME, default: "value" }
|
|
538
655
|
|
|
539
656
|
Returns:
|
|
540
|
-
|
|
657
|
+
Tuple of (environment variable name, default value or None)
|
|
541
658
|
|
|
542
659
|
Raises:
|
|
543
|
-
|
|
660
|
+
ValueError: If reference is invalid
|
|
661
|
+
@athena: 9fc8b2333b54
|
|
544
662
|
"""
|
|
545
663
|
# Validate dict structure - allow 'env' and optionally 'default'
|
|
546
664
|
valid_keys = {"env", "default"}
|
|
@@ -594,18 +712,20 @@ def _validate_env_variable_reference(var_name: str, value: dict) -> tuple[str, s
|
|
|
594
712
|
|
|
595
713
|
|
|
596
714
|
def _resolve_env_variable(var_name: str, env_var_name: str, default: str | None = None) -> str:
|
|
597
|
-
"""
|
|
715
|
+
"""
|
|
716
|
+
Resolve environment variable value.
|
|
598
717
|
|
|
599
718
|
Args:
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
719
|
+
var_name: Name of the variable being defined
|
|
720
|
+
env_var_name: Name of environment variable to read
|
|
721
|
+
default: Optional default value to use if environment variable is not set
|
|
603
722
|
|
|
604
723
|
Returns:
|
|
605
|
-
|
|
724
|
+
Environment variable value as string, or default if not set and default provided
|
|
606
725
|
|
|
607
726
|
Raises:
|
|
608
|
-
|
|
727
|
+
ValueError: If environment variable is not set and no default provided
|
|
728
|
+
@athena: c00d3a241a99
|
|
609
729
|
"""
|
|
610
730
|
value = os.environ.get(env_var_name, default)
|
|
611
731
|
|
|
@@ -623,29 +743,33 @@ def _resolve_env_variable(var_name: str, env_var_name: str, default: str | None
|
|
|
623
743
|
|
|
624
744
|
|
|
625
745
|
def _is_file_read_reference(value: Any) -> bool:
|
|
626
|
-
"""
|
|
746
|
+
"""
|
|
747
|
+
Check if value is a file read reference.
|
|
627
748
|
|
|
628
749
|
Args:
|
|
629
|
-
|
|
750
|
+
value: Raw value from YAML
|
|
630
751
|
|
|
631
752
|
Returns:
|
|
632
|
-
|
|
753
|
+
True if value is { read: filepath } dict
|
|
754
|
+
@athena: da129db1b17b
|
|
633
755
|
"""
|
|
634
756
|
return isinstance(value, dict) and "read" in value
|
|
635
757
|
|
|
636
758
|
|
|
637
759
|
def _validate_file_read_reference(var_name: str, value: dict) -> str:
|
|
638
|
-
"""
|
|
760
|
+
"""
|
|
761
|
+
Validate and extract filepath from file read reference.
|
|
639
762
|
|
|
640
763
|
Args:
|
|
641
|
-
|
|
642
|
-
|
|
764
|
+
var_name: Name of the variable being defined
|
|
765
|
+
value: Dict that should be { read: filepath }
|
|
643
766
|
|
|
644
767
|
Returns:
|
|
645
|
-
|
|
768
|
+
Filepath string
|
|
646
769
|
|
|
647
770
|
Raises:
|
|
648
|
-
|
|
771
|
+
ValueError: If reference is invalid
|
|
772
|
+
@athena: 2615951372fc
|
|
649
773
|
"""
|
|
650
774
|
# Validate dict structure (only "read" key allowed)
|
|
651
775
|
if len(value) != 1:
|
|
@@ -671,7 +795,8 @@ def _validate_file_read_reference(var_name: str, value: dict) -> str:
|
|
|
671
795
|
|
|
672
796
|
|
|
673
797
|
def _resolve_file_path(filepath: str, recipe_file_path: Path) -> Path:
|
|
674
|
-
"""
|
|
798
|
+
"""
|
|
799
|
+
Resolve file path relative to recipe file location.
|
|
675
800
|
|
|
676
801
|
Handles three path types:
|
|
677
802
|
1. Tilde paths (~): Expand to user home directory
|
|
@@ -679,11 +804,12 @@ def _resolve_file_path(filepath: str, recipe_file_path: Path) -> Path:
|
|
|
679
804
|
3. Relative paths: Resolve relative to recipe file's directory
|
|
680
805
|
|
|
681
806
|
Args:
|
|
682
|
-
|
|
683
|
-
|
|
807
|
+
filepath: Path string from YAML (may be relative, absolute, or tilde)
|
|
808
|
+
recipe_file_path: Path to the recipe file containing the variable
|
|
684
809
|
|
|
685
810
|
Returns:
|
|
686
|
-
|
|
811
|
+
Resolved absolute Path object
|
|
812
|
+
@athena: e80470e9c7d6
|
|
687
813
|
"""
|
|
688
814
|
# Expand tilde to home directory
|
|
689
815
|
if filepath.startswith("~"):
|
|
@@ -701,18 +827,20 @@ def _resolve_file_path(filepath: str, recipe_file_path: Path) -> Path:
|
|
|
701
827
|
|
|
702
828
|
|
|
703
829
|
def _resolve_file_variable(var_name: str, filepath: str, resolved_path: Path) -> str:
|
|
704
|
-
"""
|
|
830
|
+
"""
|
|
831
|
+
Read file contents for variable value.
|
|
705
832
|
|
|
706
833
|
Args:
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
834
|
+
var_name: Name of the variable being defined
|
|
835
|
+
filepath: Original filepath string (for error messages)
|
|
836
|
+
resolved_path: Resolved absolute path to the file
|
|
710
837
|
|
|
711
838
|
Returns:
|
|
712
|
-
|
|
839
|
+
File contents as string (with trailing newline stripped)
|
|
713
840
|
|
|
714
841
|
Raises:
|
|
715
|
-
|
|
842
|
+
ValueError: If file doesn't exist, can't be read, or contains invalid UTF-8
|
|
843
|
+
@athena: cab84337f145
|
|
716
844
|
"""
|
|
717
845
|
# Check file exists
|
|
718
846
|
if not resolved_path.exists():
|
|
@@ -754,29 +882,33 @@ def _resolve_file_variable(var_name: str, filepath: str, resolved_path: Path) ->
|
|
|
754
882
|
|
|
755
883
|
|
|
756
884
|
def _is_eval_reference(value: Any) -> bool:
|
|
757
|
-
"""
|
|
885
|
+
"""
|
|
886
|
+
Check if value is an eval command reference.
|
|
758
887
|
|
|
759
888
|
Args:
|
|
760
|
-
|
|
889
|
+
value: Raw value from YAML
|
|
761
890
|
|
|
762
891
|
Returns:
|
|
763
|
-
|
|
892
|
+
True if value is { eval: command } dict
|
|
893
|
+
@athena: 121784f6d4ab
|
|
764
894
|
"""
|
|
765
895
|
return isinstance(value, dict) and "eval" in value
|
|
766
896
|
|
|
767
897
|
|
|
768
898
|
def _validate_eval_reference(var_name: str, value: dict) -> str:
|
|
769
|
-
"""
|
|
899
|
+
"""
|
|
900
|
+
Validate and extract command from eval reference.
|
|
770
901
|
|
|
771
902
|
Args:
|
|
772
|
-
|
|
773
|
-
|
|
903
|
+
var_name: Name of the variable being defined
|
|
904
|
+
value: Dict that should be { eval: command }
|
|
774
905
|
|
|
775
906
|
Returns:
|
|
776
|
-
|
|
907
|
+
Command string
|
|
777
908
|
|
|
778
909
|
Raises:
|
|
779
|
-
|
|
910
|
+
ValueError: If reference is invalid
|
|
911
|
+
@athena: f3cde1011d2d
|
|
780
912
|
"""
|
|
781
913
|
# Validate dict structure (only "eval" key allowed)
|
|
782
914
|
if len(value) != 1:
|
|
@@ -802,10 +934,12 @@ def _validate_eval_reference(var_name: str, value: dict) -> str:
|
|
|
802
934
|
|
|
803
935
|
|
|
804
936
|
def _get_default_shell_and_args() -> tuple[str, list[str]]:
|
|
805
|
-
"""
|
|
937
|
+
"""
|
|
938
|
+
Get default shell and args for current platform.
|
|
806
939
|
|
|
807
940
|
Returns:
|
|
808
|
-
|
|
941
|
+
Tuple of (shell, args) for platform default
|
|
942
|
+
@athena: 68a19449a035
|
|
809
943
|
"""
|
|
810
944
|
is_windows = platform.system() == "Windows"
|
|
811
945
|
if is_windows:
|
|
@@ -820,19 +954,21 @@ def _resolve_eval_variable(
|
|
|
820
954
|
recipe_file_path: Path,
|
|
821
955
|
recipe_data: dict
|
|
822
956
|
) -> str:
|
|
823
|
-
"""
|
|
957
|
+
"""
|
|
958
|
+
Execute command and capture output for variable value.
|
|
824
959
|
|
|
825
960
|
Args:
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
961
|
+
var_name: Name of the variable being defined
|
|
962
|
+
command: Command to execute
|
|
963
|
+
recipe_file_path: Path to recipe file (for working directory)
|
|
964
|
+
recipe_data: Parsed YAML data (for accessing default_env)
|
|
830
965
|
|
|
831
966
|
Returns:
|
|
832
|
-
|
|
967
|
+
Command stdout as string (with trailing newline stripped)
|
|
833
968
|
|
|
834
969
|
Raises:
|
|
835
|
-
|
|
970
|
+
ValueError: If command fails or cannot be executed
|
|
971
|
+
@athena: 647d3a310c77
|
|
836
972
|
"""
|
|
837
973
|
# Determine shell to use
|
|
838
974
|
shell = None
|
|
@@ -911,21 +1047,23 @@ def _resolve_variable_value(
|
|
|
911
1047
|
file_path: Path,
|
|
912
1048
|
recipe_data: dict | None = None
|
|
913
1049
|
) -> str:
|
|
914
|
-
"""
|
|
1050
|
+
"""
|
|
1051
|
+
Resolve a single variable value with circular reference detection.
|
|
915
1052
|
|
|
916
1053
|
Args:
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
1054
|
+
name: Variable name being resolved
|
|
1055
|
+
raw_value: Raw value from YAML (int, str, bool, float, dict with env/read/eval)
|
|
1056
|
+
resolved: Dictionary of already-resolved variables
|
|
1057
|
+
resolution_stack: Stack of variables currently being resolved (for circular detection)
|
|
1058
|
+
file_path: Path to recipe file (for resolving relative file paths in { read: ... })
|
|
1059
|
+
recipe_data: Parsed YAML data (for accessing default_env in { eval: ... })
|
|
923
1060
|
|
|
924
1061
|
Returns:
|
|
925
|
-
|
|
1062
|
+
Resolved string value
|
|
926
1063
|
|
|
927
1064
|
Raises:
|
|
928
|
-
|
|
1065
|
+
ValueError: If circular reference detected or validation fails
|
|
1066
|
+
@athena: da94de106756
|
|
929
1067
|
"""
|
|
930
1068
|
# Check for circular reference
|
|
931
1069
|
if name in resolution_stack:
|
|
@@ -1057,20 +1195,22 @@ def _resolve_variable_value(
|
|
|
1057
1195
|
|
|
1058
1196
|
|
|
1059
1197
|
def _parse_variables_section(data: dict, file_path: Path) -> dict[str, str]:
|
|
1060
|
-
"""
|
|
1198
|
+
"""
|
|
1199
|
+
Parse and resolve the variables section from YAML data.
|
|
1061
1200
|
|
|
1062
1201
|
Variables are resolved in order, allowing variables to reference
|
|
1063
1202
|
previously-defined variables using {{ var.name }} syntax.
|
|
1064
1203
|
|
|
1065
1204
|
Args:
|
|
1066
|
-
|
|
1067
|
-
|
|
1205
|
+
data: Parsed YAML data (root level)
|
|
1206
|
+
file_path: Path to the recipe file (for resolving relative file paths)
|
|
1068
1207
|
|
|
1069
1208
|
Returns:
|
|
1070
|
-
|
|
1209
|
+
Dictionary mapping variable names to resolved string values
|
|
1071
1210
|
|
|
1072
1211
|
Raises:
|
|
1073
|
-
|
|
1212
|
+
ValueError: For validation errors, undefined refs, or circular refs
|
|
1213
|
+
@athena: aa45e860a958
|
|
1074
1214
|
"""
|
|
1075
1215
|
if "variables" not in data:
|
|
1076
1216
|
return {}
|
|
@@ -1095,26 +1235,28 @@ def _expand_variable_dependencies(
|
|
|
1095
1235
|
variable_names: set[str],
|
|
1096
1236
|
raw_variables: dict[str, Any]
|
|
1097
1237
|
) -> set[str]:
|
|
1098
|
-
"""
|
|
1238
|
+
"""
|
|
1239
|
+
Expand variable set to include all transitively referenced variables.
|
|
1099
1240
|
|
|
1100
1241
|
If variable A references variable B, and B references C, then requesting A
|
|
1101
1242
|
should also evaluate B and C.
|
|
1102
1243
|
|
|
1103
1244
|
Args:
|
|
1104
|
-
|
|
1105
|
-
|
|
1245
|
+
variable_names: Initial set of variable names
|
|
1246
|
+
raw_variables: Raw variable definitions from YAML
|
|
1106
1247
|
|
|
1107
1248
|
Returns:
|
|
1108
|
-
|
|
1249
|
+
Expanded set including all transitively referenced variables
|
|
1109
1250
|
|
|
1110
1251
|
Example:
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1252
|
+
>>> raw_vars = {
|
|
1253
|
+
... "a": "{{ var.b }}",
|
|
1254
|
+
... "b": "{{ var.c }}",
|
|
1255
|
+
... "c": "value"
|
|
1256
|
+
... }
|
|
1257
|
+
>>> _expand_variable_dependencies({"a"}, raw_vars)
|
|
1258
|
+
{"a", "b", "c"}
|
|
1259
|
+
@athena: e7ec4a4a6db3
|
|
1118
1260
|
"""
|
|
1119
1261
|
expanded = set(variable_names)
|
|
1120
1262
|
to_process = list(variable_names)
|
|
@@ -1178,7 +1320,8 @@ def _evaluate_variable_subset(
|
|
|
1178
1320
|
file_path: Path,
|
|
1179
1321
|
data: dict
|
|
1180
1322
|
) -> dict[str, str]:
|
|
1181
|
-
"""
|
|
1323
|
+
"""
|
|
1324
|
+
Evaluate only specified variables from raw specs (for lazy evaluation).
|
|
1182
1325
|
|
|
1183
1326
|
This function is similar to _parse_variables_section but only evaluates
|
|
1184
1327
|
a subset of variables. This enables lazy evaluation where only reachable
|
|
@@ -1188,21 +1331,22 @@ def _evaluate_variable_subset(
|
|
|
1188
1331
|
variable B, both will be evaluated even if only A was explicitly requested.
|
|
1189
1332
|
|
|
1190
1333
|
Args:
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1334
|
+
raw_variables: Raw variable definitions from YAML (not yet evaluated)
|
|
1335
|
+
variable_names: Set of variable names to evaluate
|
|
1336
|
+
file_path: Recipe file path (for relative file resolution)
|
|
1337
|
+
data: Full YAML data (for context in _resolve_variable_value)
|
|
1195
1338
|
|
|
1196
1339
|
Returns:
|
|
1197
|
-
|
|
1340
|
+
Dictionary of evaluated variable values (for specified variables and their dependencies)
|
|
1198
1341
|
|
|
1199
1342
|
Raises:
|
|
1200
|
-
|
|
1343
|
+
ValueError: For validation errors, undefined refs, or circular refs
|
|
1201
1344
|
|
|
1202
1345
|
Example:
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1346
|
+
>>> raw_vars = {"a": "{{ var.b }}", "b": "value", "c": "unused"}
|
|
1347
|
+
>>> _evaluate_variable_subset(raw_vars, {"a"}, path, data)
|
|
1348
|
+
{"a": "value", "b": "value"} # "a" and its dependency "b", but not "c"
|
|
1349
|
+
@athena: 1e9d491d7404
|
|
1206
1350
|
"""
|
|
1207
1351
|
if not isinstance(raw_variables, dict):
|
|
1208
1352
|
raise ValueError("'variables' must be a dictionary")
|
|
@@ -1230,17 +1374,19 @@ def _parse_file_with_env(
|
|
|
1230
1374
|
project_root: Path,
|
|
1231
1375
|
import_stack: list[Path] | None = None,
|
|
1232
1376
|
) -> tuple[dict[str, Task], dict[str, Environment], str, dict[str, Any], dict[str, Any]]:
|
|
1233
|
-
"""
|
|
1377
|
+
"""
|
|
1378
|
+
Parse file and extract tasks, environments, and variables.
|
|
1234
1379
|
|
|
1235
1380
|
Args:
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1381
|
+
file_path: Path to YAML file
|
|
1382
|
+
namespace: Optional namespace prefix for tasks
|
|
1383
|
+
project_root: Root directory of the project
|
|
1384
|
+
import_stack: Stack of files being imported (for circular detection)
|
|
1240
1385
|
|
|
1241
1386
|
Returns:
|
|
1242
|
-
|
|
1243
|
-
|
|
1387
|
+
Tuple of (tasks, environments, default_env_name, raw_variables, yaml_data)
|
|
1388
|
+
Note: Variables are NOT evaluated here - they're stored as raw specs for lazy evaluation
|
|
1389
|
+
@athena: b2dced506787
|
|
1244
1390
|
"""
|
|
1245
1391
|
# Parse tasks normally
|
|
1246
1392
|
tasks = _parse_file(file_path, namespace, project_root, import_stack)
|
|
@@ -1363,25 +1509,27 @@ def _parse_file_with_env(
|
|
|
1363
1509
|
|
|
1364
1510
|
|
|
1365
1511
|
def collect_reachable_tasks(tasks: dict[str, Task], root_task: str) -> set[str]:
|
|
1366
|
-
"""
|
|
1512
|
+
"""
|
|
1513
|
+
Collect all tasks reachable from the root task via dependencies.
|
|
1367
1514
|
|
|
1368
1515
|
Uses BFS to traverse the dependency graph and collect all task names
|
|
1369
1516
|
that could potentially be executed when running the root task.
|
|
1370
1517
|
|
|
1371
1518
|
Args:
|
|
1372
|
-
|
|
1373
|
-
|
|
1519
|
+
tasks: Dictionary mapping task names to Task objects
|
|
1520
|
+
root_task: Name of the root task to start traversal from
|
|
1374
1521
|
|
|
1375
1522
|
Returns:
|
|
1376
|
-
|
|
1523
|
+
Set of task names reachable from root_task (includes root_task itself)
|
|
1377
1524
|
|
|
1378
1525
|
Raises:
|
|
1379
|
-
|
|
1526
|
+
ValueError: If root_task doesn't exist
|
|
1380
1527
|
|
|
1381
1528
|
Example:
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1529
|
+
>>> tasks = {"a": Task("a", deps=["b"]), "b": Task("b", deps=[]), "c": Task("c", deps=[])}
|
|
1530
|
+
>>> collect_reachable_tasks(tasks, "a")
|
|
1531
|
+
{"a", "b"}
|
|
1532
|
+
@athena: fe29d8558be3
|
|
1385
1533
|
"""
|
|
1386
1534
|
if root_task not in tasks:
|
|
1387
1535
|
raise ValueError(f"Root task '{root_task}' not found in recipe")
|
|
@@ -1425,23 +1573,25 @@ def collect_reachable_variables(
|
|
|
1425
1573
|
environments: dict[str, Environment],
|
|
1426
1574
|
reachable_task_names: set[str]
|
|
1427
1575
|
) -> set[str]:
|
|
1428
|
-
"""
|
|
1576
|
+
"""
|
|
1577
|
+
Extract variable names used by reachable tasks.
|
|
1429
1578
|
|
|
1430
1579
|
Searches for {{ var.* }} placeholders in task and environment definitions to determine
|
|
1431
1580
|
which variables are actually needed for execution.
|
|
1432
1581
|
|
|
1433
1582
|
Args:
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1583
|
+
tasks: Dictionary mapping task names to Task objects
|
|
1584
|
+
environments: Dictionary mapping environment names to Environment objects
|
|
1585
|
+
reachable_task_names: Set of task names that will be executed
|
|
1437
1586
|
|
|
1438
1587
|
Returns:
|
|
1439
|
-
|
|
1588
|
+
Set of variable names referenced by reachable tasks
|
|
1440
1589
|
|
|
1441
1590
|
Example:
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1591
|
+
>>> task = Task("build", cmd="echo {{ var.version }}")
|
|
1592
|
+
>>> collect_reachable_variables({"build": task}, {"build"})
|
|
1593
|
+
{"version"}
|
|
1594
|
+
@athena: fc2a03927486
|
|
1445
1595
|
"""
|
|
1446
1596
|
import re
|
|
1447
1597
|
|
|
@@ -1473,8 +1623,15 @@ def collect_reachable_variables(
|
|
|
1473
1623
|
# Search in inputs
|
|
1474
1624
|
if task.inputs:
|
|
1475
1625
|
for input_pattern in task.inputs:
|
|
1476
|
-
|
|
1477
|
-
|
|
1626
|
+
if isinstance(input_pattern, str):
|
|
1627
|
+
for match in var_pattern.finditer(input_pattern):
|
|
1628
|
+
variables.add(match.group(1))
|
|
1629
|
+
elif isinstance(input_pattern, dict):
|
|
1630
|
+
# Named input - check the path value
|
|
1631
|
+
for input_path in input_pattern.values():
|
|
1632
|
+
if isinstance(input_path, str):
|
|
1633
|
+
for match in var_pattern.finditer(input_path):
|
|
1634
|
+
variables.add(match.group(1))
|
|
1478
1635
|
|
|
1479
1636
|
# Search in outputs
|
|
1480
1637
|
if task.outputs:
|
|
@@ -1562,28 +1719,30 @@ def parse_recipe(
|
|
|
1562
1719
|
project_root: Path | None = None,
|
|
1563
1720
|
root_task: str | None = None
|
|
1564
1721
|
) -> Recipe:
|
|
1565
|
-
"""
|
|
1722
|
+
"""
|
|
1723
|
+
Parse a recipe file and handle imports recursively.
|
|
1566
1724
|
|
|
1567
1725
|
This function now implements lazy variable evaluation: if root_task is provided,
|
|
1568
1726
|
only variables reachable from that task will be evaluated. This provides significant
|
|
1569
1727
|
performance and security benefits for recipes with many variables.
|
|
1570
1728
|
|
|
1571
1729
|
Args:
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1730
|
+
recipe_path: Path to the main recipe file
|
|
1731
|
+
project_root: Optional project root directory. If not provided, uses recipe file's parent directory.
|
|
1732
|
+
When using --tasks option, this should be the current working directory.
|
|
1733
|
+
root_task: Optional root task for lazy variable evaluation. If provided, only variables
|
|
1734
|
+
used by tasks reachable from root_task will be evaluated (optimization).
|
|
1735
|
+
If None, all variables will be evaluated (for --list command compatibility).
|
|
1578
1736
|
|
|
1579
1737
|
Returns:
|
|
1580
|
-
|
|
1738
|
+
Recipe object with all tasks (including recursively imported tasks) and evaluated variables
|
|
1581
1739
|
|
|
1582
1740
|
Raises:
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1741
|
+
FileNotFoundError: If recipe file doesn't exist
|
|
1742
|
+
CircularImportError: If circular imports are detected
|
|
1743
|
+
yaml.YAMLError: If YAML is invalid
|
|
1744
|
+
ValueError: If recipe structure is invalid
|
|
1745
|
+
@athena: 27326e37d5f3
|
|
1587
1746
|
"""
|
|
1588
1747
|
if not recipe_path.exists():
|
|
1589
1748
|
raise FileNotFoundError(f"Recipe file not found: {recipe_path}")
|
|
@@ -1626,21 +1785,23 @@ def _parse_file(
|
|
|
1626
1785
|
project_root: Path,
|
|
1627
1786
|
import_stack: list[Path] | None = None,
|
|
1628
1787
|
) -> dict[str, Task]:
|
|
1629
|
-
"""
|
|
1788
|
+
"""
|
|
1789
|
+
Parse a single YAML file and return tasks, recursively processing imports.
|
|
1630
1790
|
|
|
1631
1791
|
Args:
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1792
|
+
file_path: Path to YAML file
|
|
1793
|
+
namespace: Optional namespace prefix for tasks
|
|
1794
|
+
project_root: Root directory of the project
|
|
1795
|
+
import_stack: Stack of files being imported (for circular detection)
|
|
1636
1796
|
|
|
1637
1797
|
Returns:
|
|
1638
|
-
|
|
1798
|
+
Dictionary of task name to Task objects
|
|
1639
1799
|
|
|
1640
1800
|
Raises:
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1801
|
+
CircularImportError: If a circular import is detected
|
|
1802
|
+
FileNotFoundError: If an imported file doesn't exist
|
|
1803
|
+
ValueError: If task structure is invalid
|
|
1804
|
+
@athena: f2a1f2c70240
|
|
1644
1805
|
"""
|
|
1645
1806
|
# Initialize import stack if not provided
|
|
1646
1807
|
if import_stack is None:
|
|
@@ -1829,15 +1990,17 @@ def _parse_file(
|
|
|
1829
1990
|
|
|
1830
1991
|
|
|
1831
1992
|
def _check_case_sensitive_arg_collisions(args: list[str], task_name: str) -> None:
|
|
1832
|
-
"""
|
|
1993
|
+
"""
|
|
1994
|
+
Check for exported arguments that differ only in case.
|
|
1833
1995
|
|
|
1834
1996
|
On Unix systems, environment variables are case-sensitive, but having
|
|
1835
1997
|
args that differ only in case (e.g., $Server and $server) can be confusing.
|
|
1836
1998
|
This function emits a warning if such collisions are detected.
|
|
1837
1999
|
|
|
1838
2000
|
Args:
|
|
1839
|
-
|
|
1840
|
-
|
|
2001
|
+
args: List of argument specifications
|
|
2002
|
+
task_name: Name of the task (for warning message)
|
|
2003
|
+
@athena: a3f0f3b184a8
|
|
1841
2004
|
"""
|
|
1842
2005
|
import sys
|
|
1843
2006
|
|
|
@@ -1867,41 +2030,43 @@ def _check_case_sensitive_arg_collisions(args: list[str], task_name: str) -> Non
|
|
|
1867
2030
|
|
|
1868
2031
|
|
|
1869
2032
|
def parse_arg_spec(arg_spec: str | dict) -> ArgSpec:
|
|
1870
|
-
"""
|
|
2033
|
+
"""
|
|
2034
|
+
Parse argument specification from YAML.
|
|
1871
2035
|
|
|
1872
2036
|
Supports both string format and dictionary format:
|
|
1873
2037
|
|
|
1874
2038
|
String format (simple names only):
|
|
1875
|
-
|
|
1876
|
-
|
|
2039
|
+
- Simple name: "argname"
|
|
2040
|
+
- Exported (becomes env var): "$argname"
|
|
1877
2041
|
|
|
1878
2042
|
Dictionary format:
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
2043
|
+
- argname: { default: "value" }
|
|
2044
|
+
- argname: { type: int, default: 42 }
|
|
2045
|
+
- argname: { type: int, min: 1, max: 100 }
|
|
2046
|
+
- argname: { type: str, choices: ["dev", "staging", "prod"] }
|
|
2047
|
+
- $argname: { default: "value" } # Exported (type not allowed)
|
|
1884
2048
|
|
|
1885
2049
|
Args:
|
|
1886
|
-
|
|
2050
|
+
arg_spec: Argument specification (string or dict with single key)
|
|
1887
2051
|
|
|
1888
2052
|
Returns:
|
|
1889
|
-
|
|
2053
|
+
ArgSpec object containing parsed argument information
|
|
1890
2054
|
|
|
1891
2055
|
Examples:
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
2056
|
+
>>> parse_arg_spec("environment")
|
|
2057
|
+
ArgSpec(name='environment', arg_type='str', default=None, is_exported=False, min_val=None, max_val=None, choices=None)
|
|
2058
|
+
>>> parse_arg_spec({"key2": {"default": "foo"}})
|
|
2059
|
+
ArgSpec(name='key2', arg_type='str', default='foo', is_exported=False, min_val=None, max_val=None, choices=None)
|
|
2060
|
+
>>> parse_arg_spec({"key3": {"type": "int", "default": 42}})
|
|
2061
|
+
ArgSpec(name='key3', arg_type='int', default='42', is_exported=False, min_val=None, max_val=None, choices=None)
|
|
2062
|
+
>>> parse_arg_spec({"replicas": {"type": "int", "min": 1, "max": 100}})
|
|
2063
|
+
ArgSpec(name='replicas', arg_type='int', default=None, is_exported=False, min_val=1, max_val=100, choices=None)
|
|
2064
|
+
>>> parse_arg_spec({"env": {"type": "str", "choices": ["dev", "prod"]}})
|
|
2065
|
+
ArgSpec(name='env', arg_type='str', default=None, is_exported=False, min_val=None, max_val=None, choices=['dev', 'prod'])
|
|
1902
2066
|
|
|
1903
2067
|
Raises:
|
|
1904
|
-
|
|
2068
|
+
ValueError: If argument specification is invalid
|
|
2069
|
+
@athena: 2a4c7e804622
|
|
1905
2070
|
"""
|
|
1906
2071
|
# Handle dictionary format: { argname: { type: ..., default: ... } }
|
|
1907
2072
|
if isinstance(arg_spec, dict):
|
|
@@ -1964,18 +2129,20 @@ def parse_arg_spec(arg_spec: str | dict) -> ArgSpec:
|
|
|
1964
2129
|
|
|
1965
2130
|
|
|
1966
2131
|
def _parse_arg_dict(arg_name: str, config: dict, is_exported: bool) -> ArgSpec:
|
|
1967
|
-
"""
|
|
2132
|
+
"""
|
|
2133
|
+
Parse argument specification from dictionary format.
|
|
1968
2134
|
|
|
1969
2135
|
Args:
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
2136
|
+
arg_name: Name of the argument
|
|
2137
|
+
config: Dictionary with optional keys: type, default, min, max, choices
|
|
2138
|
+
is_exported: Whether argument should be exported to environment
|
|
1973
2139
|
|
|
1974
2140
|
Returns:
|
|
1975
|
-
|
|
2141
|
+
ArgSpec object containing the parsed argument specification
|
|
1976
2142
|
|
|
1977
2143
|
Raises:
|
|
1978
|
-
|
|
2144
|
+
ValueError: If dictionary format is invalid
|
|
2145
|
+
@athena: 5b6b93a3612a
|
|
1979
2146
|
"""
|
|
1980
2147
|
# Validate dictionary keys
|
|
1981
2148
|
valid_keys = {"type", "default", "min", "max", "choices"}
|
|
@@ -2211,7 +2378,8 @@ def _parse_arg_dict(arg_name: str, config: dict, is_exported: bool) -> ArgSpec:
|
|
|
2211
2378
|
|
|
2212
2379
|
|
|
2213
2380
|
def parse_dependency_spec(dep_spec: str | dict[str, Any], recipe: Recipe) -> DependencyInvocation:
|
|
2214
|
-
"""
|
|
2381
|
+
"""
|
|
2382
|
+
Parse a dependency specification into a DependencyInvocation.
|
|
2215
2383
|
|
|
2216
2384
|
Supports three forms:
|
|
2217
2385
|
1. Simple string: "task_name" -> DependencyInvocation(task_name, None)
|
|
@@ -2219,14 +2387,15 @@ def parse_dependency_spec(dep_spec: str | dict[str, Any], recipe: Recipe) -> Dep
|
|
|
2219
2387
|
3. Named args: {"task_name": {arg1: val1}} -> DependencyInvocation(task_name, {arg1: val1})
|
|
2220
2388
|
|
|
2221
2389
|
Args:
|
|
2222
|
-
|
|
2223
|
-
|
|
2390
|
+
dep_spec: Dependency specification (string or dict)
|
|
2391
|
+
recipe: Recipe containing task definitions (for arg normalization)
|
|
2224
2392
|
|
|
2225
2393
|
Returns:
|
|
2226
|
-
|
|
2394
|
+
DependencyInvocation object with normalized args
|
|
2227
2395
|
|
|
2228
2396
|
Raises:
|
|
2229
|
-
|
|
2397
|
+
ValueError: If dependency specification is invalid
|
|
2398
|
+
@athena: d30ff06259c2
|
|
2230
2399
|
"""
|
|
2231
2400
|
# Simple string case
|
|
2232
2401
|
if isinstance(dep_spec, str):
|
|
@@ -2275,17 +2444,19 @@ def parse_dependency_spec(dep_spec: str | dict[str, Any], recipe: Recipe) -> Dep
|
|
|
2275
2444
|
|
|
2276
2445
|
|
|
2277
2446
|
def _get_validated_task(task_name: str, recipe: Recipe) -> Task:
|
|
2278
|
-
"""
|
|
2447
|
+
"""
|
|
2448
|
+
Get and validate that a task exists in the recipe.
|
|
2279
2449
|
|
|
2280
2450
|
Args:
|
|
2281
|
-
|
|
2282
|
-
|
|
2451
|
+
task_name: Name of the task to retrieve
|
|
2452
|
+
recipe: Recipe containing task definitions
|
|
2283
2453
|
|
|
2284
2454
|
Returns:
|
|
2285
|
-
|
|
2455
|
+
The validated Task object
|
|
2286
2456
|
|
|
2287
2457
|
Raises:
|
|
2288
|
-
|
|
2458
|
+
ValueError: If task is not found
|
|
2459
|
+
@athena: 674f077e3977
|
|
2289
2460
|
"""
|
|
2290
2461
|
task = recipe.get_task(task_name)
|
|
2291
2462
|
if task is None:
|
|
@@ -2296,18 +2467,20 @@ def _get_validated_task(task_name: str, recipe: Recipe) -> Task:
|
|
|
2296
2467
|
def _parse_positional_dependency_args(
|
|
2297
2468
|
task_name: str, args_list: list[Any], recipe: Recipe
|
|
2298
2469
|
) -> DependencyInvocation:
|
|
2299
|
-
"""
|
|
2470
|
+
"""
|
|
2471
|
+
Parse positional dependency arguments.
|
|
2300
2472
|
|
|
2301
2473
|
Args:
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2474
|
+
task_name: Name of the dependency task
|
|
2475
|
+
args_list: List of positional argument values
|
|
2476
|
+
recipe: Recipe containing task definitions
|
|
2305
2477
|
|
|
2306
2478
|
Returns:
|
|
2307
|
-
|
|
2479
|
+
DependencyInvocation with normalized named args
|
|
2308
2480
|
|
|
2309
2481
|
Raises:
|
|
2310
|
-
|
|
2482
|
+
ValueError: If validation fails
|
|
2483
|
+
@athena: 4d1c7957e2dd
|
|
2311
2484
|
"""
|
|
2312
2485
|
# Get the task to validate against
|
|
2313
2486
|
task = _get_validated_task(task_name, recipe)
|
|
@@ -2356,18 +2529,20 @@ def _parse_positional_dependency_args(
|
|
|
2356
2529
|
def _parse_named_dependency_args(
|
|
2357
2530
|
task_name: str, args_dict: dict[str, Any], recipe: Recipe
|
|
2358
2531
|
) -> DependencyInvocation:
|
|
2359
|
-
"""
|
|
2532
|
+
"""
|
|
2533
|
+
Parse named dependency arguments.
|
|
2360
2534
|
|
|
2361
2535
|
Args:
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2536
|
+
task_name: Name of the dependency task
|
|
2537
|
+
args_dict: Dictionary of argument names to values
|
|
2538
|
+
recipe: Recipe containing task definitions
|
|
2365
2539
|
|
|
2366
2540
|
Returns:
|
|
2367
|
-
|
|
2541
|
+
DependencyInvocation with normalized args (defaults filled)
|
|
2368
2542
|
|
|
2369
2543
|
Raises:
|
|
2370
|
-
|
|
2544
|
+
ValueError: If validation fails
|
|
2545
|
+
@athena: c522211de525
|
|
2371
2546
|
"""
|
|
2372
2547
|
# Get the task to validate against
|
|
2373
2548
|
task = _get_validated_task(task_name, recipe)
|