tasktree 0.0.15__py3-none-any.whl → 0.0.16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tasktree/cli.py +10 -5
- tasktree/executor.py +61 -0
- tasktree/parser.py +281 -39
- tasktree/substitution.py +5 -0
- {tasktree-0.0.15.dist-info → tasktree-0.0.16.dist-info}/METADATA +1 -1
- tasktree-0.0.16.dist-info/RECORD +14 -0
- tasktree-0.0.15.dist-info/RECORD +0 -14
- {tasktree-0.0.15.dist-info → tasktree-0.0.16.dist-info}/WHEEL +0 -0
- {tasktree-0.0.15.dist-info → tasktree-0.0.16.dist-info}/entry_points.txt +0 -0
tasktree/cli.py
CHANGED
|
@@ -137,7 +137,8 @@ def _list_tasks(tasks_file: Optional[str] = None):
|
|
|
137
137
|
|
|
138
138
|
def _show_task(task_name: str, tasks_file: Optional[str] = None):
|
|
139
139
|
"""Show task definition with syntax highlighting."""
|
|
140
|
-
|
|
140
|
+
# Pass task_name as root_task for lazy variable evaluation
|
|
141
|
+
recipe = _get_recipe(tasks_file, root_task=task_name)
|
|
141
142
|
if recipe is None:
|
|
142
143
|
console.print("[red]No recipe file found (tasktree.yaml, tasktree.yml, tt.yaml, or *.tasks)[/red]")
|
|
143
144
|
raise typer.Exit(1)
|
|
@@ -186,7 +187,8 @@ def _show_task(task_name: str, tasks_file: Optional[str] = None):
|
|
|
186
187
|
|
|
187
188
|
def _show_tree(task_name: str, tasks_file: Optional[str] = None):
|
|
188
189
|
"""Show dependency tree structure."""
|
|
189
|
-
|
|
190
|
+
# Pass task_name as root_task for lazy variable evaluation
|
|
191
|
+
recipe = _get_recipe(tasks_file, root_task=task_name)
|
|
190
192
|
if recipe is None:
|
|
191
193
|
console.print("[red]No recipe file found (tasktree.yaml, tasktree.yml, tt.yaml, or *.tasks)[/red]")
|
|
192
194
|
raise typer.Exit(1)
|
|
@@ -372,11 +374,13 @@ def _clean_state(tasks_file: Optional[str] = None) -> None:
|
|
|
372
374
|
console.print(f"[yellow]No state file found at {state_path}[/yellow]")
|
|
373
375
|
|
|
374
376
|
|
|
375
|
-
def _get_recipe(recipe_file: Optional[str] = None) -> Optional[Recipe]:
|
|
377
|
+
def _get_recipe(recipe_file: Optional[str] = None, root_task: Optional[str] = None) -> Optional[Recipe]:
|
|
376
378
|
"""Get parsed recipe or None if not found.
|
|
377
379
|
|
|
378
380
|
Args:
|
|
379
381
|
recipe_file: Optional path to recipe file. If not provided, searches for recipe file.
|
|
382
|
+
root_task: Optional root task for lazy variable evaluation. If provided, only variables
|
|
383
|
+
reachable from this task will be evaluated (performance optimization).
|
|
380
384
|
"""
|
|
381
385
|
if recipe_file:
|
|
382
386
|
recipe_path = Path(recipe_file)
|
|
@@ -398,7 +402,7 @@ def _get_recipe(recipe_file: Optional[str] = None) -> Optional[Recipe]:
|
|
|
398
402
|
project_root = None
|
|
399
403
|
|
|
400
404
|
try:
|
|
401
|
-
return parse_recipe(recipe_path, project_root)
|
|
405
|
+
return parse_recipe(recipe_path, project_root, root_task)
|
|
402
406
|
except Exception as e:
|
|
403
407
|
console.print(f"[red]Error parsing recipe: {e}[/red]")
|
|
404
408
|
raise typer.Exit(1)
|
|
@@ -411,7 +415,8 @@ def _execute_dynamic_task(args: list[str], force: bool = False, only: bool = Fal
|
|
|
411
415
|
task_name = args[0]
|
|
412
416
|
task_args = args[1:]
|
|
413
417
|
|
|
414
|
-
|
|
418
|
+
# Pass task_name as root_task for lazy variable evaluation
|
|
419
|
+
recipe = _get_recipe(tasks_file, root_task=task_name)
|
|
415
420
|
if recipe is None:
|
|
416
421
|
console.print("[red]No recipe file found (tasktree.yaml, tasktree.yml, tt.yaml, or *.tasks)[/red]")
|
|
417
422
|
raise typer.Exit(1)
|
tasktree/executor.py
CHANGED
|
@@ -662,6 +662,60 @@ class Executor:
|
|
|
662
662
|
except OSError:
|
|
663
663
|
pass # Ignore cleanup errors
|
|
664
664
|
|
|
665
|
+
def _substitute_builtin_in_environment(self, env: Environment, builtin_vars: dict[str, str]) -> Environment:
|
|
666
|
+
"""Substitute builtin and environment variables in environment fields.
|
|
667
|
+
|
|
668
|
+
Args:
|
|
669
|
+
env: Environment to process
|
|
670
|
+
builtin_vars: Built-in variable values
|
|
671
|
+
|
|
672
|
+
Returns:
|
|
673
|
+
New Environment with builtin and environment variables substituted
|
|
674
|
+
|
|
675
|
+
Raises:
|
|
676
|
+
ValueError: If builtin variable or environment variable is not defined
|
|
677
|
+
"""
|
|
678
|
+
from dataclasses import replace
|
|
679
|
+
|
|
680
|
+
# Substitute in volumes (builtin vars first, then env vars)
|
|
681
|
+
substituted_volumes = [
|
|
682
|
+
self._substitute_env(self._substitute_builtin(vol, builtin_vars)) for vol in env.volumes
|
|
683
|
+
] if env.volumes else []
|
|
684
|
+
|
|
685
|
+
# Substitute in env_vars values (builtin vars first, then env vars)
|
|
686
|
+
substituted_env_vars = {
|
|
687
|
+
key: self._substitute_env(self._substitute_builtin(value, builtin_vars))
|
|
688
|
+
for key, value in env.env_vars.items()
|
|
689
|
+
} if env.env_vars else {}
|
|
690
|
+
|
|
691
|
+
# Substitute in ports (builtin vars first, then env vars)
|
|
692
|
+
substituted_ports = [
|
|
693
|
+
self._substitute_env(self._substitute_builtin(port, builtin_vars)) for port in env.ports
|
|
694
|
+
] if env.ports else []
|
|
695
|
+
|
|
696
|
+
# Substitute in working_dir (builtin vars first, then env vars)
|
|
697
|
+
substituted_working_dir = self._substitute_env(self._substitute_builtin(env.working_dir, builtin_vars)) if env.working_dir else ""
|
|
698
|
+
|
|
699
|
+
# Substitute in build args (for Docker environments, args is a dict)
|
|
700
|
+
# Apply builtin vars first, then env vars
|
|
701
|
+
if isinstance(env.args, dict):
|
|
702
|
+
substituted_args = {
|
|
703
|
+
key: self._substitute_env(self._substitute_builtin(str(value), builtin_vars))
|
|
704
|
+
for key, value in env.args.items()
|
|
705
|
+
}
|
|
706
|
+
else:
|
|
707
|
+
substituted_args = env.args
|
|
708
|
+
|
|
709
|
+
# Create new environment with substituted values
|
|
710
|
+
return replace(
|
|
711
|
+
env,
|
|
712
|
+
volumes=substituted_volumes,
|
|
713
|
+
env_vars=substituted_env_vars,
|
|
714
|
+
ports=substituted_ports,
|
|
715
|
+
working_dir=substituted_working_dir,
|
|
716
|
+
args=substituted_args
|
|
717
|
+
)
|
|
718
|
+
|
|
665
719
|
def _run_task_in_docker(
|
|
666
720
|
self, task: Task, env: Any, cmd: str, working_dir: Path,
|
|
667
721
|
exported_env_vars: dict[str, str] | None = None
|
|
@@ -678,6 +732,13 @@ class Executor:
|
|
|
678
732
|
Raises:
|
|
679
733
|
ExecutionError: If Docker execution fails
|
|
680
734
|
"""
|
|
735
|
+
# Get builtin variables for substitution in environment fields
|
|
736
|
+
task_start_time = datetime.now(timezone.utc)
|
|
737
|
+
builtin_vars = self._collect_builtin_variables(task, working_dir, task_start_time)
|
|
738
|
+
|
|
739
|
+
# Substitute builtin variables in environment fields (volumes, env_vars, etc.)
|
|
740
|
+
env = self._substitute_builtin_in_environment(env, builtin_vars)
|
|
741
|
+
|
|
681
742
|
# Resolve container working directory
|
|
682
743
|
container_working_dir = docker_module.resolve_container_working_dir(
|
|
683
744
|
env.working_dir, task.working_dir
|
tasktree/parser.py
CHANGED
|
@@ -165,7 +165,11 @@ class Recipe:
|
|
|
165
165
|
environments: dict[str, Environment] = field(default_factory=dict)
|
|
166
166
|
default_env: str = "" # Name of default environment
|
|
167
167
|
global_env_override: str = "" # Global environment override (set via CLI --env)
|
|
168
|
-
variables: dict[str, str] = field(default_factory=dict) # Global variables (resolved at parse time)
|
|
168
|
+
variables: dict[str, str] = field(default_factory=dict) # Global variables (resolved at parse time) - DEPRECATED, use evaluated_variables
|
|
169
|
+
raw_variables: dict[str, Any] = field(default_factory=dict) # Raw variable specs from YAML (not yet evaluated)
|
|
170
|
+
evaluated_variables: dict[str, str] = field(default_factory=dict) # Evaluated variable values (cached after evaluation)
|
|
171
|
+
_variables_evaluated: bool = False # Track if variables have been evaluated
|
|
172
|
+
_original_yaml_data: dict[str, Any] = field(default_factory=dict) # Store original YAML data for lazy evaluation context
|
|
169
173
|
|
|
170
174
|
def get_task(self, name: str) -> Task | None:
|
|
171
175
|
"""Get task by name.
|
|
@@ -193,6 +197,108 @@ class Recipe:
|
|
|
193
197
|
"""
|
|
194
198
|
return self.environments.get(name)
|
|
195
199
|
|
|
200
|
+
def evaluate_variables(self, root_task: str | None = None) -> None:
|
|
201
|
+
"""Evaluate variables lazily based on task reachability.
|
|
202
|
+
|
|
203
|
+
This method implements lazy variable evaluation, which only evaluates
|
|
204
|
+
variables that are actually reachable from the target task. This provides:
|
|
205
|
+
- Performance improvement: expensive { eval: ... } commands only run when needed
|
|
206
|
+
- Security improvement: sensitive { read: ... } files only accessed when needed
|
|
207
|
+
- Side-effect control: commands with side effects only execute when necessary
|
|
208
|
+
|
|
209
|
+
If root_task is provided, only variables used by reachable tasks are evaluated.
|
|
210
|
+
If root_task is None, all variables are evaluated (for --list command compatibility).
|
|
211
|
+
|
|
212
|
+
This method is idempotent - calling it multiple times is safe (uses caching).
|
|
213
|
+
|
|
214
|
+
Args:
|
|
215
|
+
root_task: Optional task name to determine reachability (None = evaluate all)
|
|
216
|
+
|
|
217
|
+
Raises:
|
|
218
|
+
ValueError: If variable evaluation or substitution fails
|
|
219
|
+
|
|
220
|
+
Example:
|
|
221
|
+
>>> recipe = parse_recipe(path) # Variables not yet evaluated
|
|
222
|
+
>>> recipe.evaluate_variables("build") # Evaluate only reachable variables
|
|
223
|
+
>>> # Now recipe.evaluated_variables contains only vars used by "build" task
|
|
224
|
+
"""
|
|
225
|
+
if self._variables_evaluated:
|
|
226
|
+
return # Already evaluated, skip (idempotent)
|
|
227
|
+
|
|
228
|
+
# Determine which variables to evaluate
|
|
229
|
+
if root_task:
|
|
230
|
+
# Lazy path: only evaluate reachable variables
|
|
231
|
+
# If root_task doesn't exist, fall back to eager evaluation
|
|
232
|
+
# (CLI will provide its own "Task not found" error)
|
|
233
|
+
try:
|
|
234
|
+
reachable_tasks = collect_reachable_tasks(self.tasks, root_task)
|
|
235
|
+
variables_to_eval = collect_reachable_variables(self.tasks, reachable_tasks)
|
|
236
|
+
except ValueError:
|
|
237
|
+
# Root task not found - fall back to eager evaluation
|
|
238
|
+
# This allows the recipe to be parsed even with invalid task names
|
|
239
|
+
# so the CLI can provide its own error message
|
|
240
|
+
variables_to_eval = set(self.raw_variables.keys())
|
|
241
|
+
else:
|
|
242
|
+
# Eager path: evaluate all variables (for --list command)
|
|
243
|
+
variables_to_eval = set(self.raw_variables.keys())
|
|
244
|
+
|
|
245
|
+
# Evaluate the selected variables using helper function
|
|
246
|
+
self.evaluated_variables = _evaluate_variable_subset(
|
|
247
|
+
self.raw_variables,
|
|
248
|
+
variables_to_eval,
|
|
249
|
+
self.recipe_path,
|
|
250
|
+
self._original_yaml_data
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
# Also update the deprecated 'variables' field for backward compatibility
|
|
254
|
+
self.variables = self.evaluated_variables
|
|
255
|
+
|
|
256
|
+
# Substitute evaluated variables into all tasks
|
|
257
|
+
from tasktree.substitution import substitute_variables
|
|
258
|
+
|
|
259
|
+
for task in self.tasks.values():
|
|
260
|
+
task.cmd = substitute_variables(task.cmd, self.evaluated_variables)
|
|
261
|
+
task.desc = substitute_variables(task.desc, self.evaluated_variables)
|
|
262
|
+
task.working_dir = substitute_variables(task.working_dir, self.evaluated_variables)
|
|
263
|
+
task.inputs = [substitute_variables(inp, self.evaluated_variables) for inp in task.inputs]
|
|
264
|
+
task.outputs = [substitute_variables(out, self.evaluated_variables) for out in task.outputs]
|
|
265
|
+
# Substitute in argument default values (in arg spec strings)
|
|
266
|
+
task.args = [substitute_variables(arg, self.evaluated_variables) for arg in task.args]
|
|
267
|
+
|
|
268
|
+
# Substitute evaluated variables into all environments
|
|
269
|
+
for env in self.environments.values():
|
|
270
|
+
if env.preamble:
|
|
271
|
+
env.preamble = substitute_variables(env.preamble, self.evaluated_variables)
|
|
272
|
+
|
|
273
|
+
# Substitute in volumes
|
|
274
|
+
if env.volumes:
|
|
275
|
+
env.volumes = [substitute_variables(vol, self.evaluated_variables) for vol in env.volumes]
|
|
276
|
+
|
|
277
|
+
# Substitute in ports
|
|
278
|
+
if env.ports:
|
|
279
|
+
env.ports = [substitute_variables(port, self.evaluated_variables) for port in env.ports]
|
|
280
|
+
|
|
281
|
+
# Substitute in env_vars values
|
|
282
|
+
if env.env_vars:
|
|
283
|
+
env.env_vars = {
|
|
284
|
+
key: substitute_variables(value, self.evaluated_variables)
|
|
285
|
+
for key, value in env.env_vars.items()
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
# Substitute in working_dir
|
|
289
|
+
if env.working_dir:
|
|
290
|
+
env.working_dir = substitute_variables(env.working_dir, self.evaluated_variables)
|
|
291
|
+
|
|
292
|
+
# Substitute in build args (dict for Docker environments)
|
|
293
|
+
if env.args and isinstance(env.args, dict):
|
|
294
|
+
env.args = {
|
|
295
|
+
key: substitute_variables(str(value), self.evaluated_variables)
|
|
296
|
+
for key, value in env.args.items()
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
# Mark as evaluated
|
|
300
|
+
self._variables_evaluated = True
|
|
301
|
+
|
|
196
302
|
|
|
197
303
|
def find_recipe_file(start_dir: Path | None = None) -> Path | None:
|
|
198
304
|
"""Find recipe file in current or parent directories.
|
|
@@ -825,7 +931,6 @@ def _resolve_variable_value(
|
|
|
825
931
|
error_msg = str(e)
|
|
826
932
|
if "not defined" in error_msg:
|
|
827
933
|
# Extract the variable name from the error message
|
|
828
|
-
import re
|
|
829
934
|
match = re.search(r"Variable '(\w+)' is not defined", error_msg)
|
|
830
935
|
if match:
|
|
831
936
|
undefined_var = match.group(1)
|
|
@@ -875,12 +980,145 @@ def _parse_variables_section(data: dict, file_path: Path) -> dict[str, str]:
|
|
|
875
980
|
return resolved
|
|
876
981
|
|
|
877
982
|
|
|
983
|
+
def _expand_variable_dependencies(
|
|
984
|
+
variable_names: set[str],
|
|
985
|
+
raw_variables: dict[str, Any]
|
|
986
|
+
) -> set[str]:
|
|
987
|
+
"""Expand variable set to include all transitively referenced variables.
|
|
988
|
+
|
|
989
|
+
If variable A references variable B, and B references C, then requesting A
|
|
990
|
+
should also evaluate B and C.
|
|
991
|
+
|
|
992
|
+
Args:
|
|
993
|
+
variable_names: Initial set of variable names
|
|
994
|
+
raw_variables: Raw variable definitions from YAML
|
|
995
|
+
|
|
996
|
+
Returns:
|
|
997
|
+
Expanded set including all transitively referenced variables
|
|
998
|
+
|
|
999
|
+
Example:
|
|
1000
|
+
>>> raw_vars = {
|
|
1001
|
+
... "a": "{{ var.b }}",
|
|
1002
|
+
... "b": "{{ var.c }}",
|
|
1003
|
+
... "c": "value"
|
|
1004
|
+
... }
|
|
1005
|
+
>>> _expand_variable_dependencies({"a"}, raw_vars)
|
|
1006
|
+
{"a", "b", "c"}
|
|
1007
|
+
"""
|
|
1008
|
+
expanded = set(variable_names)
|
|
1009
|
+
to_process = list(variable_names)
|
|
1010
|
+
pattern = re.compile(r'\{\{\s*var\.(\w+)\s*\}\}')
|
|
1011
|
+
|
|
1012
|
+
while to_process:
|
|
1013
|
+
var_name = to_process.pop(0)
|
|
1014
|
+
|
|
1015
|
+
if var_name not in raw_variables:
|
|
1016
|
+
continue
|
|
1017
|
+
|
|
1018
|
+
raw_value = raw_variables[var_name]
|
|
1019
|
+
|
|
1020
|
+
# Extract referenced variables from the raw value
|
|
1021
|
+
# Handle string values with {{ var.* }} patterns
|
|
1022
|
+
if isinstance(raw_value, str):
|
|
1023
|
+
for match in pattern.finditer(raw_value):
|
|
1024
|
+
referenced_var = match.group(1)
|
|
1025
|
+
if referenced_var not in expanded:
|
|
1026
|
+
expanded.add(referenced_var)
|
|
1027
|
+
to_process.append(referenced_var)
|
|
1028
|
+
# Handle { read: filepath } variables - check file contents for variable references
|
|
1029
|
+
elif isinstance(raw_value, dict) and 'read' in raw_value:
|
|
1030
|
+
filepath = raw_value['read']
|
|
1031
|
+
# For dependency expansion, we speculatively read files to find variable references
|
|
1032
|
+
# This is acceptable because file reads are relatively cheap compared to eval commands
|
|
1033
|
+
try:
|
|
1034
|
+
# Try to read the file (may not exist yet, which is fine for dependency tracking)
|
|
1035
|
+
# Skip if filepath is None or empty (validation error will be caught during evaluation)
|
|
1036
|
+
if filepath and isinstance(filepath, str):
|
|
1037
|
+
from pathlib import Path
|
|
1038
|
+
if Path(filepath).exists():
|
|
1039
|
+
file_content = Path(filepath).read_text()
|
|
1040
|
+
# Extract variable references from file content
|
|
1041
|
+
for match in pattern.finditer(file_content):
|
|
1042
|
+
referenced_var = match.group(1)
|
|
1043
|
+
if referenced_var not in expanded:
|
|
1044
|
+
expanded.add(referenced_var)
|
|
1045
|
+
to_process.append(referenced_var)
|
|
1046
|
+
except (IOError, OSError, TypeError):
|
|
1047
|
+
# If file can't be read during expansion, that's okay
|
|
1048
|
+
# The error will be caught during actual evaluation
|
|
1049
|
+
pass
|
|
1050
|
+
# Handle { env: VAR, default: ... } variables - check default value for variable references
|
|
1051
|
+
elif isinstance(raw_value, dict) and 'env' in raw_value and 'default' in raw_value:
|
|
1052
|
+
default_value = raw_value['default']
|
|
1053
|
+
# Check if default value contains variable references
|
|
1054
|
+
if isinstance(default_value, str):
|
|
1055
|
+
for match in pattern.finditer(default_value):
|
|
1056
|
+
referenced_var = match.group(1)
|
|
1057
|
+
if referenced_var not in expanded:
|
|
1058
|
+
expanded.add(referenced_var)
|
|
1059
|
+
to_process.append(referenced_var)
|
|
1060
|
+
|
|
1061
|
+
return expanded
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
def _evaluate_variable_subset(
|
|
1065
|
+
raw_variables: dict[str, Any],
|
|
1066
|
+
variable_names: set[str],
|
|
1067
|
+
file_path: Path,
|
|
1068
|
+
data: dict
|
|
1069
|
+
) -> dict[str, str]:
|
|
1070
|
+
"""Evaluate only specified variables from raw specs (for lazy evaluation).
|
|
1071
|
+
|
|
1072
|
+
This function is similar to _parse_variables_section but only evaluates
|
|
1073
|
+
a subset of variables. This enables lazy evaluation where only reachable
|
|
1074
|
+
variables are evaluated, improving performance and security.
|
|
1075
|
+
|
|
1076
|
+
Transitive dependencies are automatically included: if variable A references
|
|
1077
|
+
variable B, both will be evaluated even if only A was explicitly requested.
|
|
1078
|
+
|
|
1079
|
+
Args:
|
|
1080
|
+
raw_variables: Raw variable definitions from YAML (not yet evaluated)
|
|
1081
|
+
variable_names: Set of variable names to evaluate
|
|
1082
|
+
file_path: Recipe file path (for relative file resolution)
|
|
1083
|
+
data: Full YAML data (for context in _resolve_variable_value)
|
|
1084
|
+
|
|
1085
|
+
Returns:
|
|
1086
|
+
Dictionary of evaluated variable values (for specified variables and their dependencies)
|
|
1087
|
+
|
|
1088
|
+
Raises:
|
|
1089
|
+
ValueError: For validation errors, undefined refs, or circular refs
|
|
1090
|
+
|
|
1091
|
+
Example:
|
|
1092
|
+
>>> raw_vars = {"a": "{{ var.b }}", "b": "value", "c": "unused"}
|
|
1093
|
+
>>> _evaluate_variable_subset(raw_vars, {"a"}, path, data)
|
|
1094
|
+
{"a": "value", "b": "value"} # "a" and its dependency "b", but not "c"
|
|
1095
|
+
"""
|
|
1096
|
+
if not isinstance(raw_variables, dict):
|
|
1097
|
+
raise ValueError("'variables' must be a dictionary")
|
|
1098
|
+
|
|
1099
|
+
# Expand variable set to include transitive dependencies
|
|
1100
|
+
variables_to_eval = _expand_variable_dependencies(variable_names, raw_variables)
|
|
1101
|
+
|
|
1102
|
+
resolved = {} # name -> resolved string value
|
|
1103
|
+
resolution_stack = [] # For circular detection
|
|
1104
|
+
|
|
1105
|
+
# Evaluate variables in order (to handle references between variables)
|
|
1106
|
+
for var_name, raw_value in raw_variables.items():
|
|
1107
|
+
if var_name in variables_to_eval:
|
|
1108
|
+
_validate_variable_name(var_name)
|
|
1109
|
+
resolved[var_name] = _resolve_variable_value(
|
|
1110
|
+
var_name, raw_value, resolved, resolution_stack, file_path, data
|
|
1111
|
+
)
|
|
1112
|
+
|
|
1113
|
+
return resolved
|
|
1114
|
+
|
|
1115
|
+
|
|
878
1116
|
def _parse_file_with_env(
|
|
879
1117
|
file_path: Path,
|
|
880
1118
|
namespace: str | None,
|
|
881
1119
|
project_root: Path,
|
|
882
1120
|
import_stack: list[Path] | None = None,
|
|
883
|
-
) -> tuple[dict[str, Task], dict[str, Environment], str, dict[str, str]]:
|
|
1121
|
+
) -> tuple[dict[str, Task], dict[str, Environment], str, dict[str, Any], dict[str, Any]]:
|
|
884
1122
|
"""Parse file and extract tasks, environments, and variables.
|
|
885
1123
|
|
|
886
1124
|
Args:
|
|
@@ -890,7 +1128,8 @@ def _parse_file_with_env(
|
|
|
890
1128
|
import_stack: Stack of files being imported (for circular detection)
|
|
891
1129
|
|
|
892
1130
|
Returns:
|
|
893
|
-
Tuple of (tasks, environments, default_env_name,
|
|
1131
|
+
Tuple of (tasks, environments, default_env_name, raw_variables, yaml_data)
|
|
1132
|
+
Note: Variables are NOT evaluated here - they're stored as raw specs for lazy evaluation
|
|
894
1133
|
"""
|
|
895
1134
|
# Parse tasks normally
|
|
896
1135
|
tasks = _parse_file(file_path, namespace, project_root, import_stack)
|
|
@@ -898,29 +1137,23 @@ def _parse_file_with_env(
|
|
|
898
1137
|
# Load YAML again to extract environments and variables (only from root file)
|
|
899
1138
|
environments: dict[str, Environment] = {}
|
|
900
1139
|
default_env = ""
|
|
901
|
-
|
|
1140
|
+
raw_variables: dict[str, Any] = {}
|
|
1141
|
+
yaml_data: dict[str, Any] = {}
|
|
902
1142
|
|
|
903
1143
|
# Only parse environments and variables from the root file (namespace is None)
|
|
904
1144
|
if namespace is None:
|
|
905
1145
|
with open(file_path, "r") as f:
|
|
906
1146
|
data = yaml.safe_load(f)
|
|
1147
|
+
yaml_data = data or {}
|
|
907
1148
|
|
|
908
|
-
#
|
|
909
|
-
|
|
910
|
-
|
|
1149
|
+
# Store raw variable specs WITHOUT evaluating them (lazy evaluation)
|
|
1150
|
+
# Variable evaluation will happen later in Recipe.evaluate_variables()
|
|
1151
|
+
if data and "variables" in data:
|
|
1152
|
+
raw_variables = data["variables"]
|
|
911
1153
|
|
|
912
|
-
#
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
for task in tasks.values():
|
|
917
|
-
task.cmd = substitute_variables(task.cmd, variables)
|
|
918
|
-
task.desc = substitute_variables(task.desc, variables)
|
|
919
|
-
task.working_dir = substitute_variables(task.working_dir, variables)
|
|
920
|
-
task.inputs = [substitute_variables(inp, variables) for inp in task.inputs]
|
|
921
|
-
task.outputs = [substitute_variables(out, variables) for out in task.outputs]
|
|
922
|
-
# Substitute in argument default values (in arg spec strings)
|
|
923
|
-
task.args = [substitute_variables(arg, variables) for arg in task.args]
|
|
1154
|
+
# SKIP variable substitution here - defer to lazy evaluation phase
|
|
1155
|
+
# Tasks and environments will contain {{ var.* }} placeholders until evaluation
|
|
1156
|
+
# This allows us to only evaluate variables that are actually reachable from the target task
|
|
924
1157
|
|
|
925
1158
|
if data and "environments" in data:
|
|
926
1159
|
env_data = data["environments"]
|
|
@@ -944,10 +1177,8 @@ def _parse_file_with_env(
|
|
|
944
1177
|
preamble = env_config.get("preamble", "")
|
|
945
1178
|
working_dir = env_config.get("working_dir", "")
|
|
946
1179
|
|
|
947
|
-
#
|
|
948
|
-
|
|
949
|
-
from tasktree.substitution import substitute_variables
|
|
950
|
-
preamble = substitute_variables(preamble, variables)
|
|
1180
|
+
# SKIP variable substitution in preamble - defer to lazy evaluation
|
|
1181
|
+
# preamble may contain {{ var.* }} placeholders
|
|
951
1182
|
|
|
952
1183
|
# Parse Docker-specific fields
|
|
953
1184
|
dockerfile = env_config.get("dockerfile", "")
|
|
@@ -958,6 +1189,9 @@ def _parse_file_with_env(
|
|
|
958
1189
|
extra_args = env_config.get("extra_args", [])
|
|
959
1190
|
run_as_root = env_config.get("run_as_root", False)
|
|
960
1191
|
|
|
1192
|
+
# SKIP variable substitution in environment fields - defer to lazy evaluation
|
|
1193
|
+
# Environment fields may contain {{ var.* }} placeholders
|
|
1194
|
+
|
|
961
1195
|
# Validate environment type
|
|
962
1196
|
if not shell and not dockerfile:
|
|
963
1197
|
raise ValueError(
|
|
@@ -1014,7 +1248,7 @@ def _parse_file_with_env(
|
|
|
1014
1248
|
run_as_root=run_as_root
|
|
1015
1249
|
)
|
|
1016
1250
|
|
|
1017
|
-
return tasks, environments, default_env,
|
|
1251
|
+
return tasks, environments, default_env, raw_variables, yaml_data
|
|
1018
1252
|
|
|
1019
1253
|
|
|
1020
1254
|
def collect_reachable_tasks(tasks: dict[str, Task], root_task: str) -> set[str]:
|
|
@@ -1174,17 +1408,20 @@ def parse_recipe(
|
|
|
1174
1408
|
) -> Recipe:
|
|
1175
1409
|
"""Parse a recipe file and handle imports recursively.
|
|
1176
1410
|
|
|
1411
|
+
This function now implements lazy variable evaluation: if root_task is provided,
|
|
1412
|
+
only variables reachable from that task will be evaluated. This provides significant
|
|
1413
|
+
performance and security benefits for recipes with many variables.
|
|
1414
|
+
|
|
1177
1415
|
Args:
|
|
1178
1416
|
recipe_path: Path to the main recipe file
|
|
1179
1417
|
project_root: Optional project root directory. If not provided, uses recipe file's parent directory.
|
|
1180
1418
|
When using --tasks option, this should be the current working directory.
|
|
1181
1419
|
root_task: Optional root task for lazy variable evaluation. If provided, only variables
|
|
1182
1420
|
used by tasks reachable from root_task will be evaluated (optimization).
|
|
1183
|
-
|
|
1184
|
-
implemented - all variables are still evaluated for backward compatibility.
|
|
1421
|
+
If None, all variables will be evaluated (for --list command compatibility).
|
|
1185
1422
|
|
|
1186
1423
|
Returns:
|
|
1187
|
-
Recipe object with all tasks (including recursively imported tasks)
|
|
1424
|
+
Recipe object with all tasks (including recursively imported tasks) and evaluated variables
|
|
1188
1425
|
|
|
1189
1426
|
Raises:
|
|
1190
1427
|
FileNotFoundError: If recipe file doesn't exist
|
|
@@ -1200,27 +1437,32 @@ def parse_recipe(
|
|
|
1200
1437
|
project_root = recipe_path.parent
|
|
1201
1438
|
|
|
1202
1439
|
# Parse main file - it will recursively handle all imports
|
|
1203
|
-
|
|
1440
|
+
# Variables are NOT evaluated here (lazy evaluation)
|
|
1441
|
+
tasks, environments, default_env, raw_variables, yaml_data = _parse_file_with_env(
|
|
1204
1442
|
recipe_path, namespace=None, project_root=project_root
|
|
1205
1443
|
)
|
|
1206
1444
|
|
|
1207
|
-
#
|
|
1208
|
-
|
|
1209
|
-
# 1. Deferring variable evaluation until after task parsing
|
|
1210
|
-
# 2. Collecting reachable tasks and variables
|
|
1211
|
-
# 3. Evaluating only reachable variables
|
|
1212
|
-
# 4. Re-substituting variables into task definitions
|
|
1213
|
-
# For now, we evaluate all variables eagerly (current behavior)
|
|
1214
|
-
|
|
1215
|
-
return Recipe(
|
|
1445
|
+
# Create recipe with raw (unevaluated) variables
|
|
1446
|
+
recipe = Recipe(
|
|
1216
1447
|
tasks=tasks,
|
|
1217
1448
|
project_root=project_root,
|
|
1218
1449
|
recipe_path=recipe_path,
|
|
1219
1450
|
environments=environments,
|
|
1220
1451
|
default_env=default_env,
|
|
1221
|
-
variables=
|
|
1452
|
+
variables={}, # Empty initially (deprecated field)
|
|
1453
|
+
raw_variables=raw_variables,
|
|
1454
|
+
evaluated_variables={}, # Empty initially
|
|
1455
|
+
_variables_evaluated=False,
|
|
1456
|
+
_original_yaml_data=yaml_data
|
|
1222
1457
|
)
|
|
1223
1458
|
|
|
1459
|
+
# Trigger lazy variable evaluation
|
|
1460
|
+
# If root_task is provided: evaluate only reachable variables
|
|
1461
|
+
# If root_task is None: evaluate all variables (for --list)
|
|
1462
|
+
recipe.evaluate_variables(root_task)
|
|
1463
|
+
|
|
1464
|
+
return recipe
|
|
1465
|
+
|
|
1224
1466
|
|
|
1225
1467
|
def _parse_file(
|
|
1226
1468
|
file_path: Path,
|
tasktree/substitution.py
CHANGED
|
@@ -48,6 +48,11 @@ def substitute_variables(text: str | dict[str, Any], variables: dict[str, str])
|
|
|
48
48
|
else:
|
|
49
49
|
raise ValueError("Empty arg dictionary")
|
|
50
50
|
else:
|
|
51
|
+
# If not a string (e.g., int, float, bool, None), return unchanged
|
|
52
|
+
# This handles cases like: default: 5, min: 0, choices: [1, 2, 3]
|
|
53
|
+
if not isinstance(text, str):
|
|
54
|
+
return text
|
|
55
|
+
|
|
51
56
|
def replace_match(match: re.Match) -> str:
|
|
52
57
|
prefix = match.group(1)
|
|
53
58
|
name = match.group(2)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
tasktree/__init__.py,sha256=MVmdvKb3JdqLlo0x2_TPGMfgFC0HsDnP79HAzGnFnjI,1081
|
|
2
|
+
tasktree/cli.py,sha256=Uhv0RNFrogjvqxBYKYIfxEPd0SdYAIpXH7SPGIxQnmk,20136
|
|
3
|
+
tasktree/docker.py,sha256=qvja8G63uAcC73YMVY739egda1_CcBtoqzm0qIJU_Q8,14443
|
|
4
|
+
tasktree/executor.py,sha256=QQcABThmof0MLTtwpJKpyqh80hr3YRIqqs3NZ-Ry-Bk,44873
|
|
5
|
+
tasktree/graph.py,sha256=yITp71RfhJ7sdC-2zRf89SHYZqQyF3XVAnaqX-XnMdE,15821
|
|
6
|
+
tasktree/hasher.py,sha256=0GrnCfwAXnwq_kpnHFFb12B5_2VFNXx6Ng7hTdcCyXo,4415
|
|
7
|
+
tasktree/parser.py,sha256=yOe6G1Fg2FqmEjaPD73FD2aiYP9sZisTx1DZPofx9wc,87049
|
|
8
|
+
tasktree/state.py,sha256=Cktl4D8iDZVd55aO2LqVyPrc-BnljkesxxkcMcdcfOY,3541
|
|
9
|
+
tasktree/substitution.py,sha256=qG7SyEHn1PAKteWA0AgA1dUNbJfwQTupCLRq9FvOBD0,10724
|
|
10
|
+
tasktree/types.py,sha256=R_YAyO5bMLB6XZnkMRT7VAtlkA_Xx6xu0aIpzQjrBXs,4357
|
|
11
|
+
tasktree-0.0.16.dist-info/METADATA,sha256=ssKlllByyOeVMGDuHa11XLDhcGoxPfepRRoO776A_Ig,37234
|
|
12
|
+
tasktree-0.0.16.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
13
|
+
tasktree-0.0.16.dist-info/entry_points.txt,sha256=lQINlvRYnimvteBbnhH84A9clTg8NnpEjCWqWkqg8KE,40
|
|
14
|
+
tasktree-0.0.16.dist-info/RECORD,,
|
tasktree-0.0.15.dist-info/RECORD
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
tasktree/__init__.py,sha256=MVmdvKb3JdqLlo0x2_TPGMfgFC0HsDnP79HAzGnFnjI,1081
|
|
2
|
-
tasktree/cli.py,sha256=2RisQ4LSU9hTOj7h1vYcIflMAKCjJ9ohJUH33NC3Jx0,19655
|
|
3
|
-
tasktree/docker.py,sha256=qvja8G63uAcC73YMVY739egda1_CcBtoqzm0qIJU_Q8,14443
|
|
4
|
-
tasktree/executor.py,sha256=Q7Bks5B88i-IyZDpxGSps9MM3uflz0U3yn4Rtq_uHMM,42266
|
|
5
|
-
tasktree/graph.py,sha256=yITp71RfhJ7sdC-2zRf89SHYZqQyF3XVAnaqX-XnMdE,15821
|
|
6
|
-
tasktree/hasher.py,sha256=0GrnCfwAXnwq_kpnHFFb12B5_2VFNXx6Ng7hTdcCyXo,4415
|
|
7
|
-
tasktree/parser.py,sha256=DUILTXWGoCxwB6m69yrlmPeqZc2_nNLsUMNKYo2GwIQ,75827
|
|
8
|
-
tasktree/state.py,sha256=Cktl4D8iDZVd55aO2LqVyPrc-BnljkesxxkcMcdcfOY,3541
|
|
9
|
-
tasktree/substitution.py,sha256=ZS_MmPKVB0iUafXyxjEqxSo1Yn0F9LP-yY8msknyBQk,10512
|
|
10
|
-
tasktree/types.py,sha256=R_YAyO5bMLB6XZnkMRT7VAtlkA_Xx6xu0aIpzQjrBXs,4357
|
|
11
|
-
tasktree-0.0.15.dist-info/METADATA,sha256=oDvrc54QieGWzxtmCzeMfERUqxGBs86VOZAGehsVY6k,37234
|
|
12
|
-
tasktree-0.0.15.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
13
|
-
tasktree-0.0.15.dist-info/entry_points.txt,sha256=lQINlvRYnimvteBbnhH84A9clTg8NnpEjCWqWkqg8KE,40
|
|
14
|
-
tasktree-0.0.15.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|