cook-build 0.6.3__py3-none-any.whl → 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cook/__init__.py +1 -2
- cook/__main__.py +18 -7
- cook/actions.py +71 -50
- cook/contexts.py +5 -5
- cook/controller.py +112 -174
- cook/manager.py +12 -9
- cook/task.py +30 -7
- cook/util.py +2 -13
- {cook_build-0.6.3.dist-info → cook_build-0.7.0.dist-info}/METADATA +1 -1
- cook_build-0.7.0.dist-info/RECORD +14 -0
- cook_build-0.6.3.dist-info/RECORD +0 -14
- {cook_build-0.6.3.dist-info → cook_build-0.7.0.dist-info}/WHEEL +0 -0
- {cook_build-0.6.3.dist-info → cook_build-0.7.0.dist-info}/entry_points.txt +0 -0
- {cook_build-0.6.3.dist-info → cook_build-0.7.0.dist-info}/licenses/LICENSE +0 -0
- {cook_build-0.6.3.dist-info → cook_build-0.7.0.dist-info}/top_level.txt +0 -0
cook/__init__.py
CHANGED
cook/__main__.py
CHANGED
|
@@ -1,29 +1,30 @@
|
|
|
1
1
|
import argparse
|
|
2
|
-
import
|
|
3
|
-
from contextlib import closing
|
|
4
|
-
from datetime import datetime
|
|
2
|
+
import asyncio
|
|
5
3
|
import fnmatch
|
|
6
4
|
import importlib.util
|
|
7
5
|
import logging
|
|
8
6
|
import os
|
|
9
|
-
from pathlib import Path
|
|
10
7
|
import re
|
|
11
8
|
import sqlite3
|
|
12
9
|
import sys
|
|
13
10
|
import textwrap
|
|
11
|
+
from contextlib import closing
|
|
12
|
+
from datetime import datetime
|
|
13
|
+
from pathlib import Path
|
|
14
14
|
from typing import Iterable
|
|
15
15
|
|
|
16
|
+
import colorama
|
|
17
|
+
|
|
16
18
|
from .contexts import (
|
|
17
19
|
create_target_directories,
|
|
18
20
|
normalize_action,
|
|
19
21
|
normalize_dependencies,
|
|
20
22
|
)
|
|
21
|
-
from .controller import
|
|
23
|
+
from .controller import QUERIES, Controller
|
|
22
24
|
from .manager import Manager
|
|
23
25
|
from .task import Task
|
|
24
26
|
from .util import FailedTaskError, format_datetime, format_timedelta
|
|
25
27
|
|
|
26
|
-
|
|
27
28
|
LOGGER = logging.getLogger("cook")
|
|
28
29
|
|
|
29
30
|
|
|
@@ -122,6 +123,7 @@ class Command:
|
|
|
122
123
|
|
|
123
124
|
class ExecArgs(Args):
|
|
124
125
|
jobs: int
|
|
126
|
+
dry_run: bool
|
|
125
127
|
|
|
126
128
|
|
|
127
129
|
class ExecCommand(Command):
|
|
@@ -135,11 +137,20 @@ class ExecCommand(Command):
|
|
|
135
137
|
parser.add_argument(
|
|
136
138
|
"--jobs", "-j", help="number of concurrent jobs", type=int, default=1
|
|
137
139
|
)
|
|
140
|
+
parser.add_argument(
|
|
141
|
+
"--dry-run",
|
|
142
|
+
"-n",
|
|
143
|
+
help="show what \033[1mwould\033[0m be executed without running tasks",
|
|
144
|
+
action="store_true",
|
|
145
|
+
dest="dry_run",
|
|
146
|
+
)
|
|
138
147
|
super().configure_parser(parser)
|
|
139
148
|
|
|
140
149
|
def execute(self, controller: Controller, args: ExecArgs) -> None: # pyright: ignore[reportIncompatibleMethodOverride]
|
|
141
150
|
tasks = self.discover_tasks(controller, args)
|
|
142
|
-
|
|
151
|
+
asyncio.run(
|
|
152
|
+
controller.execute(tasks, num_concurrent=args.jobs, dry_run=args.dry_run)
|
|
153
|
+
)
|
|
143
154
|
|
|
144
155
|
|
|
145
156
|
class LsArgs(Args):
|
cook/actions.py
CHANGED
|
@@ -8,51 +8,58 @@ multiple actions using :class:`.CompositeAction`, and executing modules as scrip
|
|
|
8
8
|
:class:`.ModuleAction`.
|
|
9
9
|
|
|
10
10
|
Custom actions can be implemented by inheriting from :class:`.Action` and implementing the
|
|
11
|
-
:meth:`~.Action.execute` method which receives a :class:`~.task.Task`. The method should
|
|
12
|
-
|
|
11
|
+
:meth:`~.Action.execute` method which receives a :class:`~.task.Task`. The method should be async
|
|
12
|
+
and its return value is ignored. For example, the following action waits for a specified time.
|
|
13
13
|
|
|
14
14
|
.. doctest::
|
|
15
15
|
|
|
16
16
|
>>> from cook.actions import Action
|
|
17
17
|
>>> from cook.task import Task
|
|
18
|
-
>>>
|
|
18
|
+
>>> import asyncio
|
|
19
|
+
>>> from time import time
|
|
19
20
|
|
|
20
21
|
>>> class SleepAction(Action):
|
|
21
22
|
... def __init__(self, delay: float) -> None:
|
|
22
23
|
... self.delay = delay
|
|
23
24
|
...
|
|
24
|
-
... def execute(self, task: Task) -> None:
|
|
25
|
+
... async def execute(self, task: Task) -> None:
|
|
25
26
|
... start = time()
|
|
26
|
-
... sleep(self.delay)
|
|
27
|
+
... await asyncio.sleep(self.delay)
|
|
27
28
|
... print(f"time: {time() - start:.3f}")
|
|
28
29
|
|
|
29
30
|
>>> action = SleepAction(0.1)
|
|
30
|
-
>>> action.execute(None)
|
|
31
|
+
>>> asyncio.run(action.execute(None))
|
|
31
32
|
time: 0.1...
|
|
33
|
+
|
|
34
|
+
For backwards compatibility, synchronous execute methods are also supported but will run in an
|
|
35
|
+
executor with a deprecation warning.
|
|
32
36
|
"""
|
|
33
37
|
|
|
38
|
+
import asyncio
|
|
34
39
|
import hashlib
|
|
40
|
+
import logging
|
|
35
41
|
import os
|
|
36
42
|
import shlex
|
|
37
43
|
import subprocess
|
|
38
44
|
import sys
|
|
39
45
|
from types import ModuleType
|
|
40
|
-
from typing import
|
|
41
|
-
|
|
46
|
+
from typing import TYPE_CHECKING, Callable
|
|
42
47
|
|
|
43
48
|
if TYPE_CHECKING:
|
|
44
49
|
from .task import Task
|
|
45
|
-
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
LOGGER = logging.getLogger(__name__)
|
|
46
53
|
|
|
47
54
|
|
|
48
55
|
class Action:
|
|
49
56
|
"""
|
|
50
|
-
Action to perform when a task is executed
|
|
57
|
+
Action to perform when a task is executed.
|
|
51
58
|
"""
|
|
52
59
|
|
|
53
|
-
def execute(self, task: "Task"
|
|
60
|
+
async def execute(self, task: "Task") -> None:
|
|
54
61
|
"""
|
|
55
|
-
Execute the action.
|
|
62
|
+
Execute the action asynchronously.
|
|
56
63
|
"""
|
|
57
64
|
raise NotImplementedError
|
|
58
65
|
|
|
@@ -80,17 +87,23 @@ class FunctionAction(Action):
|
|
|
80
87
|
self.args = args
|
|
81
88
|
self.kwargs = kwargs
|
|
82
89
|
|
|
83
|
-
def execute(self, task: "Task"
|
|
84
|
-
|
|
90
|
+
async def execute(self, task: "Task") -> None:
|
|
91
|
+
# Check if the function is already async
|
|
92
|
+
if asyncio.iscoroutinefunction(self.func):
|
|
93
|
+
await self.func(task, *self.args, **self.kwargs)
|
|
94
|
+
else:
|
|
95
|
+
# Run sync function in executor
|
|
96
|
+
loop = asyncio.get_running_loop()
|
|
97
|
+
await loop.run_in_executor(None, self.func, task, *self.args, **self.kwargs)
|
|
85
98
|
|
|
86
99
|
|
|
87
100
|
class SubprocessAction(Action):
|
|
88
101
|
"""
|
|
89
|
-
Run a subprocess.
|
|
102
|
+
Run a subprocess asynchronously.
|
|
90
103
|
|
|
91
104
|
Args:
|
|
92
|
-
*args: Positional arguments for
|
|
93
|
-
**kwargs: Keyword arguments for
|
|
105
|
+
*args: Positional arguments for subprocess execution.
|
|
106
|
+
**kwargs: Keyword arguments for subprocess execution.
|
|
94
107
|
|
|
95
108
|
Example:
|
|
96
109
|
|
|
@@ -98,47 +111,55 @@ class SubprocessAction(Action):
|
|
|
98
111
|
|
|
99
112
|
>>> from cook.actions import SubprocessAction
|
|
100
113
|
>>> from pathlib import Path
|
|
114
|
+
>>> import asyncio
|
|
101
115
|
|
|
102
116
|
>>> action = SubprocessAction(["touch", "hello.txt"])
|
|
103
|
-
>>> action.execute(None)
|
|
117
|
+
>>> asyncio.run(action.execute(None))
|
|
104
118
|
>>> Path("hello.txt").is_file()
|
|
105
119
|
True
|
|
106
120
|
"""
|
|
107
121
|
|
|
108
122
|
def __init__(self, *args, **kwargs) -> None:
|
|
123
|
+
# Validate shell argument early
|
|
124
|
+
if kwargs.get("shell", False) and args and not isinstance(args[0], str):
|
|
125
|
+
raise ValueError("shell=True requires string args")
|
|
109
126
|
self.args = args
|
|
110
127
|
self.kwargs = kwargs
|
|
111
128
|
|
|
112
|
-
def execute(self, task: "Task"
|
|
113
|
-
#
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
129
|
+
async def execute(self, task: "Task") -> None:
|
|
130
|
+
# Get the command arguments
|
|
131
|
+
(args,) = self.args
|
|
132
|
+
shell = self.kwargs.get("shell", False)
|
|
133
|
+
other_kwargs = {k: v for k, v in self.kwargs.items() if k != "shell"}
|
|
134
|
+
|
|
135
|
+
# Create the subprocess
|
|
136
|
+
if shell:
|
|
137
|
+
process = await asyncio.create_subprocess_shell(args, **other_kwargs)
|
|
138
|
+
else:
|
|
139
|
+
# Exec mode: args can be a string (single command) or list
|
|
140
|
+
if isinstance(args, str):
|
|
141
|
+
# Single command string - treat as program name with no arguments
|
|
142
|
+
process = await asyncio.create_subprocess_exec(args, **other_kwargs)
|
|
143
|
+
else:
|
|
144
|
+
# List of arguments
|
|
145
|
+
process = await asyncio.create_subprocess_exec(*args, **other_kwargs)
|
|
146
|
+
|
|
147
|
+
try:
|
|
148
|
+
# Wait for the process to complete
|
|
149
|
+
returncode = await process.wait()
|
|
150
|
+
if returncode:
|
|
151
|
+
raise subprocess.CalledProcessError(returncode, args)
|
|
152
|
+
|
|
153
|
+
except asyncio.CancelledError:
|
|
154
|
+
# Task was cancelled - terminate the subprocess
|
|
155
|
+
if process.returncode is None:
|
|
156
|
+
process.terminate()
|
|
157
|
+
try:
|
|
158
|
+
await asyncio.wait_for(process.wait(), timeout=3)
|
|
159
|
+
except asyncio.TimeoutError:
|
|
160
|
+
process.kill()
|
|
161
|
+
await process.wait()
|
|
162
|
+
raise
|
|
142
163
|
|
|
143
164
|
@property
|
|
144
165
|
def hexdigest(self) -> str:
|
|
@@ -169,9 +190,9 @@ class CompositeAction(Action):
|
|
|
169
190
|
def __init__(self, *actions: Action) -> None:
|
|
170
191
|
self.actions = actions
|
|
171
192
|
|
|
172
|
-
def execute(self, task: "Task"
|
|
193
|
+
async def execute(self, task: "Task") -> None:
|
|
173
194
|
for action in self.actions:
|
|
174
|
-
action.execute(task
|
|
195
|
+
await action.execute(task)
|
|
175
196
|
|
|
176
197
|
@property
|
|
177
198
|
def hexdigest(self) -> str | None:
|
cook/contexts.py
CHANGED
|
@@ -31,15 +31,15 @@ Custom contexts can be implemented by inheriting from :class:`.Context` and impl
|
|
|
31
31
|
"""
|
|
32
32
|
|
|
33
33
|
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import warnings
|
|
34
36
|
from pathlib import Path
|
|
35
37
|
from types import ModuleType
|
|
36
|
-
from typing import
|
|
37
|
-
|
|
38
|
-
from . import actions
|
|
38
|
+
from typing import TYPE_CHECKING, Callable, TypeVar
|
|
39
|
+
|
|
40
|
+
from . import actions, util
|
|
39
41
|
from . import manager as manager_
|
|
40
42
|
from . import task as task_
|
|
41
|
-
from . import util
|
|
42
|
-
|
|
43
43
|
|
|
44
44
|
if TYPE_CHECKING:
|
|
45
45
|
from .manager import Manager
|
cook/controller.py
CHANGED
|
@@ -1,22 +1,19 @@
|
|
|
1
|
-
|
|
2
|
-
from datetime import datetime
|
|
1
|
+
import asyncio
|
|
3
2
|
import hashlib
|
|
4
3
|
import logging
|
|
5
|
-
import
|
|
4
|
+
import warnings
|
|
5
|
+
from datetime import datetime
|
|
6
6
|
from pathlib import Path
|
|
7
|
-
from queue import Empty, Queue
|
|
8
7
|
from sqlite3 import Connection
|
|
9
|
-
import sys
|
|
10
|
-
import threading
|
|
11
|
-
from types import TracebackType
|
|
12
8
|
from typing import (
|
|
13
|
-
cast,
|
|
14
|
-
Iterable,
|
|
15
|
-
Literal,
|
|
16
|
-
Sequence,
|
|
17
9
|
TYPE_CHECKING,
|
|
10
|
+
Sequence,
|
|
11
|
+
cast,
|
|
18
12
|
overload,
|
|
19
13
|
)
|
|
14
|
+
|
|
15
|
+
import networkx as nx
|
|
16
|
+
|
|
20
17
|
from . import util
|
|
21
18
|
|
|
22
19
|
if TYPE_CHECKING:
|
|
@@ -75,18 +72,6 @@ QUERIES = {
|
|
|
75
72
|
}
|
|
76
73
|
|
|
77
74
|
|
|
78
|
-
@dataclass
|
|
79
|
-
class Event:
|
|
80
|
-
kind: Literal["start", "complete", "fail"]
|
|
81
|
-
task: "Task"
|
|
82
|
-
timestamp: datetime
|
|
83
|
-
exc_info: (
|
|
84
|
-
tuple[type[BaseException], BaseException, TracebackType]
|
|
85
|
-
| tuple[None, None, None]
|
|
86
|
-
)
|
|
87
|
-
digest: str | None
|
|
88
|
-
|
|
89
|
-
|
|
90
75
|
class Controller:
|
|
91
76
|
"""
|
|
92
77
|
Controller to manage dependencies and execute tasks.
|
|
@@ -232,36 +217,33 @@ class Controller:
|
|
|
232
217
|
LOGGER.debug("%s is up to date", task)
|
|
233
218
|
return False
|
|
234
219
|
|
|
235
|
-
def execute(
|
|
236
|
-
self,
|
|
220
|
+
async def execute(
|
|
221
|
+
self,
|
|
222
|
+
tasks: "Task | list[Task]",
|
|
223
|
+
num_concurrent: int = 1,
|
|
224
|
+
interval: float | None = None,
|
|
225
|
+
dry_run: bool = False,
|
|
237
226
|
) -> None:
|
|
238
227
|
"""
|
|
239
|
-
Execute one or more tasks.
|
|
228
|
+
Execute one or more tasks asynchronously.
|
|
240
229
|
|
|
241
230
|
Args:
|
|
242
231
|
tasks: Tasks to execute.
|
|
243
|
-
num_concurrent: Number of concurrent
|
|
232
|
+
num_concurrent: Number of concurrent tasks to run.
|
|
233
|
+
interval: Deprecated, kept for backward compatibility.
|
|
234
|
+
dry_run: If True, show what would execute without running tasks.
|
|
244
235
|
"""
|
|
236
|
+
if interval is not None: # pragma: no cover
|
|
237
|
+
warnings.warn(
|
|
238
|
+
"The 'interval' parameter is deprecated and has no effect",
|
|
239
|
+
DeprecationWarning,
|
|
240
|
+
stacklevel=2,
|
|
241
|
+
)
|
|
245
242
|
if not isinstance(tasks, Sequence):
|
|
246
243
|
tasks = [tasks]
|
|
247
244
|
if not any(self.is_stale(tasks)):
|
|
248
245
|
return
|
|
249
246
|
|
|
250
|
-
# Start the worker threads.
|
|
251
|
-
threads: list[threading.Thread] = []
|
|
252
|
-
input_queue = Queue()
|
|
253
|
-
output_queue = Queue[Event]()
|
|
254
|
-
stop = util.StopEvent(interval)
|
|
255
|
-
for i in range(num_concurrent):
|
|
256
|
-
thread = threading.Thread(
|
|
257
|
-
target=self._target,
|
|
258
|
-
name=f"cook-thread-{i}",
|
|
259
|
-
args=(stop, input_queue, output_queue),
|
|
260
|
-
daemon=True,
|
|
261
|
-
)
|
|
262
|
-
thread.start()
|
|
263
|
-
threads.append(thread)
|
|
264
|
-
|
|
265
247
|
# Get the subgraph of stale nodes.
|
|
266
248
|
stale_nodes = [
|
|
267
249
|
node
|
|
@@ -270,119 +252,80 @@ class Controller:
|
|
|
270
252
|
]
|
|
271
253
|
dependencies = cast(nx.DiGraph, self.dependencies.subgraph(stale_nodes).copy())
|
|
272
254
|
|
|
273
|
-
#
|
|
274
|
-
|
|
275
|
-
if out_degree == 0:
|
|
276
|
-
input_queue.put((node, self._evaluate_task_hexdigest(node)))
|
|
255
|
+
# Create semaphore for concurrency control
|
|
256
|
+
semaphore = asyncio.Semaphore(num_concurrent)
|
|
277
257
|
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
except Empty: # pragma: no cover
|
|
284
|
-
continue
|
|
285
|
-
|
|
286
|
-
assert event is not None, "output queue returned `None`; this is a bug"
|
|
287
|
-
|
|
288
|
-
# Unpack the results.
|
|
289
|
-
if event.kind == "fail":
|
|
290
|
-
# Update the status in the database.
|
|
291
|
-
params = {
|
|
292
|
-
"name": event.task.name,
|
|
293
|
-
"last_failed": event.timestamp,
|
|
294
|
-
}
|
|
295
|
-
self.connection.execute(QUERIES["upsert_task_failed"], params)
|
|
296
|
-
self.connection.commit()
|
|
297
|
-
ex = event.exc_info[1]
|
|
298
|
-
raise util.FailedTaskError(ex, task=event.task) from ex
|
|
299
|
-
elif event.kind == "complete":
|
|
300
|
-
# Update the status in the database.
|
|
301
|
-
params = {
|
|
302
|
-
"name": event.task.name,
|
|
303
|
-
"digest": event.digest,
|
|
304
|
-
"last_completed": event.timestamp,
|
|
305
|
-
}
|
|
306
|
-
self.connection.execute(QUERIES["upsert_task_completed"], params)
|
|
307
|
-
self.connection.commit()
|
|
308
|
-
elif event.kind == "start":
|
|
309
|
-
params = {
|
|
310
|
-
"name": event.task.name,
|
|
311
|
-
"last_started": event.timestamp,
|
|
312
|
-
}
|
|
313
|
-
self.connection.execute(QUERIES["upsert_task_started"], params)
|
|
314
|
-
self.connection.commit()
|
|
315
|
-
continue
|
|
316
|
-
else:
|
|
317
|
-
raise ValueError(event) # pragma: no cover
|
|
318
|
-
|
|
319
|
-
# Check if the stop event is set and abort if so.
|
|
320
|
-
if stop.is_set():
|
|
321
|
-
break
|
|
322
|
-
|
|
323
|
-
# Add tasks that are now leaf nodes to the tree.
|
|
324
|
-
predecessors = list(dependencies.predecessors(event.task))
|
|
325
|
-
dependencies.remove_node(event.task)
|
|
326
|
-
self.dependencies.add_node(event.task, is_stale=False)
|
|
327
|
-
for node, out_degree in cast(
|
|
328
|
-
Iterable, dependencies.out_degree(predecessors)
|
|
329
|
-
):
|
|
330
|
-
if out_degree == 0:
|
|
331
|
-
input_queue.put((node, self._evaluate_task_hexdigest(node)))
|
|
332
|
-
finally:
|
|
333
|
-
# Set the stop event and add "None" to the queue so the workers stop waiting.
|
|
334
|
-
LOGGER.debug(
|
|
335
|
-
"set stop event for threads: %s", [thread.name for thread in threads]
|
|
258
|
+
# Create futures for all stale tasks
|
|
259
|
+
task_futures: dict["Task", asyncio.Task] = {}
|
|
260
|
+
for task in dependencies:
|
|
261
|
+
task_futures[task] = asyncio.create_task(
|
|
262
|
+
self._execute_task(task, task_futures, dependencies, semaphore, dry_run)
|
|
336
263
|
)
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
264
|
+
|
|
265
|
+
# Wait for requested tasks
|
|
266
|
+
requested_futures = [task_futures[t] for t in tasks if t in task_futures]
|
|
267
|
+
try:
|
|
268
|
+
await asyncio.gather(*requested_futures)
|
|
269
|
+
except Exception:
|
|
270
|
+
# Cancel all pending tasks
|
|
271
|
+
for future in task_futures.values():
|
|
272
|
+
if not future.done():
|
|
273
|
+
future.cancel()
|
|
274
|
+
# Wait for all cancellations to complete
|
|
275
|
+
await asyncio.gather(*task_futures.values(), return_exceptions=True)
|
|
276
|
+
raise
|
|
277
|
+
|
|
278
|
+
async def _execute_task(
|
|
279
|
+
self,
|
|
280
|
+
task: "Task",
|
|
281
|
+
task_futures: dict["Task", asyncio.Task],
|
|
282
|
+
dependencies: nx.DiGraph,
|
|
283
|
+
semaphore: asyncio.Semaphore,
|
|
284
|
+
dry_run: bool,
|
|
349
285
|
) -> None:
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
assert task is not None, "input queue returned `None`; this is a bug"
|
|
365
|
-
|
|
366
|
-
start = datetime.now()
|
|
367
|
-
try:
|
|
368
|
-
# Execute the task.
|
|
286
|
+
"""Execute a single task after waiting for its dependencies."""
|
|
287
|
+
# Wait for all dependencies to complete
|
|
288
|
+
dep_tasks = list(dependencies.successors(task))
|
|
289
|
+
if dep_tasks:
|
|
290
|
+
dep_futures = [task_futures[dep] for dep in dep_tasks]
|
|
291
|
+
await asyncio.gather(*dep_futures)
|
|
292
|
+
|
|
293
|
+
start = datetime.now()
|
|
294
|
+
digest = self._evaluate_task_hexdigest(task)
|
|
295
|
+
|
|
296
|
+
try:
|
|
297
|
+
# Log what we're doing
|
|
298
|
+
if dry_run:
|
|
369
299
|
LOGGER.log(
|
|
370
300
|
logging.DEBUG if task.name.startswith("_") else logging.INFO,
|
|
371
|
-
"
|
|
301
|
+
"would execute %s",
|
|
372
302
|
task,
|
|
373
303
|
)
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
timestamp=start,
|
|
380
|
-
exc_info=(None, None, None),
|
|
304
|
+
if task.action:
|
|
305
|
+
LOGGER.log(
|
|
306
|
+
logging.DEBUG if task.name.startswith("_") else logging.INFO,
|
|
307
|
+
" action: %s",
|
|
308
|
+
task.action,
|
|
381
309
|
)
|
|
310
|
+
else:
|
|
311
|
+
LOGGER.log(
|
|
312
|
+
logging.DEBUG if task.name.startswith("_") else logging.INFO,
|
|
313
|
+
"executing %s ...",
|
|
314
|
+
task,
|
|
382
315
|
)
|
|
383
|
-
task.execute(stop)
|
|
384
316
|
|
|
385
|
-
|
|
317
|
+
# Update DB for start
|
|
318
|
+
if not dry_run:
|
|
319
|
+
params = {"name": task.name, "last_started": start}
|
|
320
|
+
self.connection.execute(QUERIES["upsert_task_started"], params)
|
|
321
|
+
self.connection.commit()
|
|
322
|
+
|
|
323
|
+
# Execute the task
|
|
324
|
+
if not dry_run:
|
|
325
|
+
async with semaphore:
|
|
326
|
+
await task.execute()
|
|
327
|
+
|
|
328
|
+
# Check that all targets were created
|
|
386
329
|
for target in task.targets:
|
|
387
330
|
if not target.is_file():
|
|
388
331
|
raise FileNotFoundError(
|
|
@@ -390,16 +333,17 @@ class Controller:
|
|
|
390
333
|
)
|
|
391
334
|
LOGGER.debug("%s created `%s`", task, target)
|
|
392
335
|
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
336
|
+
# Update DB for completion
|
|
337
|
+
if not dry_run:
|
|
338
|
+
params = {
|
|
339
|
+
"name": task.name,
|
|
340
|
+
"digest": digest,
|
|
341
|
+
"last_completed": datetime.now(),
|
|
342
|
+
}
|
|
343
|
+
self.connection.execute(QUERIES["upsert_task_completed"], params)
|
|
344
|
+
self.connection.commit()
|
|
345
|
+
|
|
346
|
+
# Log completion
|
|
403
347
|
delta = util.format_timedelta(datetime.now() - start)
|
|
404
348
|
LOGGER.log(
|
|
405
349
|
logging.DEBUG if task.name.startswith("_") else logging.INFO,
|
|
@@ -407,26 +351,20 @@ class Controller:
|
|
|
407
351
|
task,
|
|
408
352
|
delta,
|
|
409
353
|
)
|
|
410
|
-
except: # noqa: E722
|
|
411
|
-
exc_info = sys.exc_info()
|
|
412
|
-
delta = util.format_timedelta(datetime.now() - start)
|
|
413
|
-
LOGGER.exception(
|
|
414
|
-
"failed to execute %s after %s", task, delta, exc_info=exc_info
|
|
415
|
-
)
|
|
416
|
-
stop.set()
|
|
417
|
-
output_queue.put(
|
|
418
|
-
Event(
|
|
419
|
-
kind="fail",
|
|
420
|
-
task=task,
|
|
421
|
-
digest=digest,
|
|
422
|
-
timestamp=datetime.now(),
|
|
423
|
-
exc_info=sys.exc_info(),
|
|
424
|
-
)
|
|
425
|
-
)
|
|
426
354
|
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
355
|
+
# Mark task as no longer stale
|
|
356
|
+
self.dependencies.nodes[task]["is_stale"] = False
|
|
357
|
+
|
|
358
|
+
except Exception as ex:
|
|
359
|
+
# Update DB for failure
|
|
360
|
+
if not dry_run:
|
|
361
|
+
params = {"name": task.name, "last_failed": datetime.now()}
|
|
362
|
+
self.connection.execute(QUERIES["upsert_task_failed"], params)
|
|
363
|
+
self.connection.commit()
|
|
364
|
+
|
|
365
|
+
delta = util.format_timedelta(datetime.now() - start)
|
|
366
|
+
LOGGER.exception("failed to execute %s after %s", task, delta)
|
|
367
|
+
raise util.FailedTaskError(ex, task=task) from ex
|
|
430
368
|
|
|
431
369
|
def reset(self, *tasks: "Task") -> None:
|
|
432
370
|
# TODO: add tests for resetting.
|
cook/manager.py
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
+
|
|
2
3
|
import logging
|
|
3
|
-
import networkx as nx
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
import networkx as nx
|
|
8
|
+
|
|
6
9
|
from . import task as task_
|
|
7
10
|
from . import util
|
|
8
11
|
|
|
9
|
-
|
|
10
12
|
if TYPE_CHECKING:
|
|
11
13
|
from .actions import Action
|
|
12
14
|
from .contexts import Context
|
|
@@ -47,21 +49,21 @@ class Manager:
|
|
|
47
49
|
raise ValueError("no manager is active")
|
|
48
50
|
return Manager._INSTANCE
|
|
49
51
|
|
|
50
|
-
def create_task(self, name: str, **kwargs):
|
|
52
|
+
def create_task(self, name: str | None = None, **kwargs):
|
|
51
53
|
"""
|
|
52
54
|
Create a task. See :func:`.create_task` for details.
|
|
53
55
|
"""
|
|
54
56
|
try:
|
|
55
|
-
if name in self.tasks:
|
|
56
|
-
raise ValueError(f"task with name '{name}' already exists")
|
|
57
57
|
task = task_.Task(name, **kwargs)
|
|
58
|
+
if task.name in self.tasks:
|
|
59
|
+
raise ValueError(f"task with name '{task.name}' already exists")
|
|
58
60
|
for context in reversed(self.contexts):
|
|
59
61
|
task = context.apply(task)
|
|
60
62
|
if task is None:
|
|
61
63
|
raise ValueError(f"{context} did not return a task")
|
|
62
|
-
self.tasks[name] = task
|
|
64
|
+
self.tasks[task.name] = task
|
|
63
65
|
return task
|
|
64
|
-
except:
|
|
66
|
+
except:
|
|
65
67
|
filename, lineno = util.get_location()
|
|
66
68
|
LOGGER.exception(
|
|
67
69
|
"failed to create task with name '%s' at %s:%d", name, filename, lineno
|
|
@@ -135,7 +137,7 @@ class Manager:
|
|
|
135
137
|
|
|
136
138
|
|
|
137
139
|
def create_task(
|
|
138
|
-
name: str,
|
|
140
|
+
name: str | None = None,
|
|
139
141
|
*,
|
|
140
142
|
action: "Action | str | None" = None,
|
|
141
143
|
targets: list["Path | str"] | None = None,
|
|
@@ -147,7 +149,8 @@ def create_task(
|
|
|
147
149
|
Create a new task.
|
|
148
150
|
|
|
149
151
|
Args:
|
|
150
|
-
name: Name of the new task.
|
|
152
|
+
name: Name of the new task. Defaults to the string representation of the first
|
|
153
|
+
dependency if not provided.
|
|
151
154
|
action: Action to execute or a string for shell commands.
|
|
152
155
|
targets: Paths for files to be generated.
|
|
153
156
|
dependencies: Paths to files on which this task depends.
|
cook/task.py
CHANGED
|
@@ -1,13 +1,21 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
-
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import inspect
|
|
5
|
+
import logging
|
|
3
6
|
from pathlib import Path
|
|
4
7
|
from typing import TYPE_CHECKING
|
|
5
|
-
from . import util
|
|
6
8
|
|
|
9
|
+
import colorama
|
|
10
|
+
|
|
11
|
+
from . import util
|
|
7
12
|
|
|
8
13
|
if TYPE_CHECKING:
|
|
9
|
-
from .util import PathOrStr
|
|
10
14
|
from .actions import Action
|
|
15
|
+
from .util import PathOrStr
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
LOGGER = logging.getLogger(__name__)
|
|
11
19
|
|
|
12
20
|
|
|
13
21
|
class Task:
|
|
@@ -17,7 +25,7 @@ class Task:
|
|
|
17
25
|
|
|
18
26
|
def __init__(
|
|
19
27
|
self,
|
|
20
|
-
name: str,
|
|
28
|
+
name: str | None = None,
|
|
21
29
|
*,
|
|
22
30
|
dependencies: list["PathOrStr | Task"] | None = None,
|
|
23
31
|
targets: list["PathOrStr"] | None = None,
|
|
@@ -25,16 +33,31 @@ class Task:
|
|
|
25
33
|
task_dependencies: list[Task] | None = None,
|
|
26
34
|
location: tuple[str, int] | None = None,
|
|
27
35
|
) -> None:
|
|
28
|
-
self.name = name
|
|
29
36
|
self.dependencies = dependencies or []
|
|
37
|
+
if name is None:
|
|
38
|
+
if not self.dependencies:
|
|
39
|
+
raise ValueError("name is required if there are no dependencies")
|
|
40
|
+
name = str(self.dependencies[0])
|
|
41
|
+
self.name = name
|
|
30
42
|
self.targets = [Path(path) for path in (targets or [])]
|
|
31
43
|
self.action = action
|
|
32
44
|
self.task_dependencies = task_dependencies or []
|
|
33
45
|
self.location = location or util.get_location()
|
|
34
46
|
|
|
35
|
-
def execute(self
|
|
47
|
+
async def execute(self) -> None:
|
|
36
48
|
if self.action:
|
|
37
|
-
|
|
49
|
+
# Check if the action's execute method is actually async
|
|
50
|
+
# This handles custom actions that may have implemented sync execute()
|
|
51
|
+
if inspect.iscoroutinefunction(self.action.execute):
|
|
52
|
+
await self.action.execute(self)
|
|
53
|
+
else:
|
|
54
|
+
# User implemented old-style sync execute() - run in executor with warning
|
|
55
|
+
LOGGER.warning(
|
|
56
|
+
f"{self.action.__class__.__name__} implements sync execute(); "
|
|
57
|
+
"please update to async def execute() for better performance"
|
|
58
|
+
)
|
|
59
|
+
loop = asyncio.get_running_loop()
|
|
60
|
+
await loop.run_in_executor(None, self.action.execute, self)
|
|
38
61
|
|
|
39
62
|
def __hash__(self) -> int:
|
|
40
63
|
return hash(self.name)
|
cook/util.py
CHANGED
|
@@ -1,15 +1,14 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
|
+
|
|
2
3
|
import contextlib
|
|
3
|
-
from datetime import datetime, timedelta
|
|
4
4
|
import hashlib
|
|
5
5
|
import inspect
|
|
6
6
|
import os
|
|
7
|
+
from datetime import datetime, timedelta
|
|
7
8
|
from pathlib import Path
|
|
8
|
-
import threading
|
|
9
9
|
from time import time
|
|
10
10
|
from typing import TYPE_CHECKING, Generator
|
|
11
11
|
|
|
12
|
-
|
|
13
12
|
if TYPE_CHECKING:
|
|
14
13
|
from .task import Task
|
|
15
14
|
|
|
@@ -84,16 +83,6 @@ def get_location() -> tuple[Path, int]:
|
|
|
84
83
|
return Path(frame.f_code.co_filename).resolve(), frame.f_lineno
|
|
85
84
|
|
|
86
85
|
|
|
87
|
-
class StopEvent(threading.Event):
|
|
88
|
-
"""
|
|
89
|
-
Event used for stopping execution with a polling interval.
|
|
90
|
-
"""
|
|
91
|
-
|
|
92
|
-
def __init__(self, interval: float = 1) -> None:
|
|
93
|
-
super().__init__()
|
|
94
|
-
self.interval = interval
|
|
95
|
-
|
|
96
|
-
|
|
97
86
|
def format_timedelta(delta: timedelta) -> str:
|
|
98
87
|
"""
|
|
99
88
|
Format a time difference.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
cook/__init__.py,sha256=SCa9_i6B84IzSAwq0wnSQqvycyL4dvTO7dRIysJXZj4,179
|
|
2
|
+
cook/__main__.py,sha256=4sO22TsNTt3oirT71dJjZJwmynsRyUGT-CHJNlDgCkk,13242
|
|
3
|
+
cook/actions.py,sha256=1VLyWL-pACuWpAjIyNWPBeu6wlKkoqi6t3lHr6hV0EQ,7399
|
|
4
|
+
cook/contexts.py,sha256=AMKO7Uz-nI52OsdPQ_zCLXjOf77V4pgzvDyj6-N8Ods,10705
|
|
5
|
+
cook/controller.py,sha256=vpF3QhiM3HImaI0rHJ58T8i5AQi5_P4Z2p42qFvG1po,13426
|
|
6
|
+
cook/manager.py,sha256=Y-QGVw9x8ZpSBMRALJIHgcMmIZulAV9KxwtBJz35Gpw,6241
|
|
7
|
+
cook/task.py,sha256=2UsXJiPe7htnlFqAKYm3Ot3bz9jXGHm3qfeqh6l5alM,2320
|
|
8
|
+
cook/util.py,sha256=15MMG07CYZZ-YdFE_2jzRRTaqHMsw83UFg0s7e72MhI,2435
|
|
9
|
+
cook_build-0.7.0.dist-info/licenses/LICENSE,sha256=3Nuj_WTTcz7JDg4-9EzNf6vHlKRWpdLUccg-pvoZ3WE,1500
|
|
10
|
+
cook_build-0.7.0.dist-info/METADATA,sha256=j_WuF101Muusbbdrw_w8lv-pB_p0bEJbBuUZLUh8V30,4586
|
|
11
|
+
cook_build-0.7.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
12
|
+
cook_build-0.7.0.dist-info/entry_points.txt,sha256=5UP0ZmmxSNKevTVISUJxmdXEQsKrI4n54OQYkjrdX2c,48
|
|
13
|
+
cook_build-0.7.0.dist-info/top_level.txt,sha256=ewNQIn2oRSYV98vAsUnw88u2Q8XHKhAz70ed2PEdR2c,5
|
|
14
|
+
cook_build-0.7.0.dist-info/RECORD,,
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
cook/__init__.py,sha256=uNRvyxT6-XuoAMdDujWYa_4ZW0ppAiom6tsxMovZ-A0,180
|
|
2
|
-
cook/__main__.py,sha256=jcXYDWbMkRU24ZrrAZGrsT2wQ7nhimpigrPjfrMsRtk,12925
|
|
3
|
-
cook/actions.py,sha256=erjjDKC45kQpv_Qb2V3OCGvrrphka4Ytj78RPEvs8Uc,6718
|
|
4
|
-
cook/contexts.py,sha256=e3bddOaR8OLhPh38-U5b-u9D83NXlbQU0Hn7UvYaITI,10717
|
|
5
|
-
cook/controller.py,sha256=z20sKMzCG-wJziMIn3XeDmmvu7IUogx-iDI4sGSrwzs,16132
|
|
6
|
-
cook/manager.py,sha256=FAy8CKIMOx5liYNnaDTgnC5YZm1aqWy3pCIGiP8Hv2w,6118
|
|
7
|
-
cook/task.py,sha256=ioPuQ8jp09BY9VhKn2cC6Q8G17W-1e4q4kj2GD4Zs8k,1388
|
|
8
|
-
cook/util.py,sha256=hpMxxHnmJRfCUUAzkwMQU4kF_JAcEBFVpvPsHxOOY2U,2681
|
|
9
|
-
cook_build-0.6.3.dist-info/licenses/LICENSE,sha256=3Nuj_WTTcz7JDg4-9EzNf6vHlKRWpdLUccg-pvoZ3WE,1500
|
|
10
|
-
cook_build-0.6.3.dist-info/METADATA,sha256=RMt42OtLjrg1Z4iUvFvD1oRv2TMj-_mpQM_kD9LczMk,4586
|
|
11
|
-
cook_build-0.6.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
12
|
-
cook_build-0.6.3.dist-info/entry_points.txt,sha256=5UP0ZmmxSNKevTVISUJxmdXEQsKrI4n54OQYkjrdX2c,48
|
|
13
|
-
cook_build-0.6.3.dist-info/top_level.txt,sha256=ewNQIn2oRSYV98vAsUnw88u2Q8XHKhAz70ed2PEdR2c,5
|
|
14
|
-
cook_build-0.6.3.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|