PyProd 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyprod/__init__.py +1 -0
- pyprod/main.py +28 -8
- pyprod/prod.py +248 -135
- pyprod/utils.py +5 -3
- pyprod/venv.py +13 -4
- {pyprod-0.3.0.dist-info → pyprod-0.5.0.dist-info}/METADATA +13 -6
- pyprod-0.5.0.dist-info/RECORD +11 -0
- pyprod-0.3.0.dist-info/RECORD +0 -11
- {pyprod-0.3.0.dist-info → pyprod-0.5.0.dist-info}/WHEEL +0 -0
- {pyprod-0.3.0.dist-info → pyprod-0.5.0.dist-info}/entry_points.txt +0 -0
- {pyprod-0.3.0.dist-info → pyprod-0.5.0.dist-info}/licenses/LICENSE +0 -0
pyprod/__init__.py
CHANGED
@@ -0,0 +1 @@
|
|
1
|
+
__version__ = "0.5.0"
|
pyprod/main.py
CHANGED
@@ -15,7 +15,14 @@ parser = argparse.ArgumentParser(
|
|
15
15
|
)
|
16
16
|
|
17
17
|
parser.add_argument(
|
18
|
-
"-
|
18
|
+
"-C",
|
19
|
+
"--directory",
|
20
|
+
dest="directory",
|
21
|
+
help="Change to DIRECTORY before performing any operations",
|
22
|
+
)
|
23
|
+
|
24
|
+
parser.add_argument(
|
25
|
+
"-f", "--file", help="Use FILE as the Prodfile (default: 'Prodfile.py')"
|
19
26
|
)
|
20
27
|
|
21
28
|
parser.add_argument(
|
@@ -27,10 +34,15 @@ parser.add_argument(
|
|
27
34
|
)
|
28
35
|
|
29
36
|
parser.add_argument(
|
30
|
-
"-
|
31
|
-
|
32
|
-
|
33
|
-
|
37
|
+
"-r", "--rebuild", dest="rebuild", action="store_true", help="Rebuild all"
|
38
|
+
)
|
39
|
+
|
40
|
+
parser.add_argument(
|
41
|
+
"-g",
|
42
|
+
"--use-git",
|
43
|
+
dest="use_git",
|
44
|
+
action="store_true",
|
45
|
+
help="Get file timestamps from Git",
|
34
46
|
)
|
35
47
|
|
36
48
|
parser.add_argument(
|
@@ -55,8 +67,13 @@ def print_exc(e):
|
|
55
67
|
logger.exception("Terminated by exception")
|
56
68
|
|
57
69
|
|
70
|
+
def init_args(args=None):
|
71
|
+
args = pyprod.args = parser.parse_args(args)
|
72
|
+
return args
|
73
|
+
|
74
|
+
|
58
75
|
def main():
|
59
|
-
args =
|
76
|
+
args = init_args()
|
60
77
|
pyprod.verbose = args.verbose
|
61
78
|
chdir = args.directory
|
62
79
|
if chdir:
|
@@ -89,6 +106,7 @@ def main():
|
|
89
106
|
|
90
107
|
params = {}
|
91
108
|
targets = []
|
109
|
+
|
92
110
|
for target in args.targets:
|
93
111
|
if "=" in target:
|
94
112
|
name, value = target.split("=", 1)
|
@@ -99,7 +117,6 @@ def main():
|
|
99
117
|
try:
|
100
118
|
# load module
|
101
119
|
prod = pyprod.prod.Prod(mod, args.job, params)
|
102
|
-
|
103
120
|
# select targets
|
104
121
|
if not targets:
|
105
122
|
target = prod.get_default_target()
|
@@ -107,7 +124,10 @@ def main():
|
|
107
124
|
sys.exit("No default target")
|
108
125
|
targets = [target]
|
109
126
|
|
110
|
-
ret =
|
127
|
+
ret = 0
|
128
|
+
for target in targets:
|
129
|
+
ret += asyncio.run(prod.start([target]))
|
130
|
+
|
111
131
|
if not ret:
|
112
132
|
print(f"Nothing to be done for {targets}")
|
113
133
|
|
pyprod/prod.py
CHANGED
@@ -15,8 +15,9 @@ from collections import defaultdict
|
|
15
15
|
from collections.abc import Collection
|
16
16
|
from dataclasses import dataclass, field
|
17
17
|
from fnmatch import fnmatch, translate
|
18
|
+
from functools import wraps
|
18
19
|
from pathlib import Path
|
19
|
-
|
20
|
+
import dateutil.parser
|
20
21
|
import pyprod
|
21
22
|
|
22
23
|
from .utils import flatten, unique_list
|
@@ -37,6 +38,10 @@ class RuleError(Exception):
|
|
37
38
|
pass
|
38
39
|
|
39
40
|
|
41
|
+
class TargetError(Exception):
|
42
|
+
pass
|
43
|
+
|
44
|
+
|
40
45
|
class HandledExceptionError(Exception):
|
41
46
|
pass
|
42
47
|
|
@@ -99,9 +104,10 @@ def capture(*args, echo=True, cwd=None, check=True, text=True, shell=None):
|
|
99
104
|
|
100
105
|
def glob(path, dir="."):
|
101
106
|
ret = []
|
102
|
-
|
107
|
+
root = Path(dir)
|
108
|
+
for c in root.glob(path):
|
103
109
|
# ignore dot files
|
104
|
-
if any(p.startswith(".") for p in c.parts):
|
110
|
+
if any((p not in (".", "..")) and p.startswith(".") for p in c.parts):
|
105
111
|
continue
|
106
112
|
ret.append(c)
|
107
113
|
return ret
|
@@ -109,7 +115,7 @@ def glob(path, dir="."):
|
|
109
115
|
|
110
116
|
def rule_to_re(rule):
|
111
117
|
if not isinstance(rule, (str, Path)):
|
112
|
-
raise
|
118
|
+
raise TypeError(f"str or Path required: {rule}")
|
113
119
|
|
114
120
|
srule = str(rule)
|
115
121
|
srule = translate(srule)
|
@@ -156,47 +162,51 @@ def _check_pattern(pattern):
|
|
156
162
|
raise RuleError(f"{pattern}: Pattern should contain a '%'.")
|
157
163
|
|
158
164
|
|
159
|
-
def _strip_dot(path):
|
160
|
-
if not path:
|
161
|
-
return path
|
162
|
-
path = Path(path) # ./aaa/ -> aaa
|
163
|
-
parts = path.parts
|
164
|
-
if ".." in parts:
|
165
|
-
raise RuleError(f"{path}: '..' directory is not allowed")
|
166
|
-
return str(path)
|
167
|
-
|
168
|
-
|
169
165
|
def _check_wildcard(path):
|
170
166
|
if "*" in path:
|
171
167
|
raise RuleError(f"{path}: '*' directory is not allowed")
|
172
168
|
|
173
169
|
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
170
|
+
def _name_to_str(name):
|
171
|
+
match name:
|
172
|
+
case Task():
|
173
|
+
return name.name
|
174
|
+
case _TaskFunc():
|
175
|
+
return name.name
|
176
|
+
case Path():
|
177
|
+
return str(name)
|
178
|
+
case str():
|
179
|
+
return name
|
180
|
+
case _:
|
181
|
+
raise ValueError(f"Invalid dependency name: {name}")
|
183
182
|
|
184
|
-
|
185
|
-
for target in flatten(targets or ()):
|
186
|
-
target = str(target)
|
187
|
-
if not target:
|
188
|
-
continue
|
183
|
+
return name
|
189
184
|
|
190
|
-
if "*" in target:
|
191
|
-
continue
|
192
185
|
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
186
|
+
class Rule:
|
187
|
+
def __init__(self, targets, pattern=None, depends=(), uses=(), builder=None):
|
188
|
+
self.targets = []
|
189
|
+
self.default = False
|
190
|
+
self.first_target = None
|
191
|
+
if targets:
|
192
|
+
for target in flatten(targets):
|
193
|
+
if not target:
|
194
|
+
continue
|
195
|
+
target = str(target)
|
196
|
+
if not target:
|
197
|
+
continue
|
198
|
+
|
199
|
+
if not self.first_target:
|
200
|
+
if "*" not in target:
|
201
|
+
if _check_pattern_count(target) == 0:
|
202
|
+
# not contain one %
|
203
|
+
self.first_target = target
|
204
|
+
|
205
|
+
target = rule_to_re(target)
|
206
|
+
self.targets.append(target)
|
197
207
|
|
198
208
|
if pattern:
|
199
|
-
pattern =
|
209
|
+
pattern = str(pattern)
|
200
210
|
if _check_pattern_count(pattern) != 1:
|
201
211
|
raise RuleError(f"{pattern}: Pattern should contain a '%'")
|
202
212
|
|
@@ -206,18 +216,16 @@ class Rule:
|
|
206
216
|
|
207
217
|
self.depends = []
|
208
218
|
for depend in flatten(depends or ()):
|
209
|
-
depend =
|
219
|
+
depend = _name_to_str(depend)
|
210
220
|
_check_pattern_count(depend)
|
211
221
|
_check_wildcard(depend)
|
212
|
-
depend = _strip_dot(depend)
|
213
222
|
self.depends.append(depend)
|
214
223
|
|
215
224
|
self.uses = []
|
216
225
|
for use in flatten(uses or ()):
|
217
|
-
use =
|
226
|
+
use = _name_to_str(use)
|
218
227
|
_check_pattern_count(use)
|
219
228
|
_check_wildcard(use)
|
220
|
-
use = _strip_dot(use)
|
221
229
|
self.uses.append(use)
|
222
230
|
|
223
231
|
self.builder = builder
|
@@ -227,6 +235,54 @@ class Rule:
|
|
227
235
|
return f
|
228
236
|
|
229
237
|
|
238
|
+
class _TaskFunc:
|
239
|
+
def __init__(self, f, name):
|
240
|
+
self.f = f
|
241
|
+
self.name = name
|
242
|
+
|
243
|
+
def __call__(self, *args, **kwargs):
|
244
|
+
return self.f(*args, **kwargs)
|
245
|
+
|
246
|
+
|
247
|
+
def default_builder(*args, **kwargs):
|
248
|
+
# default builder
|
249
|
+
pass
|
250
|
+
|
251
|
+
|
252
|
+
class Task(Rule):
|
253
|
+
def __init__(self, name, uses, default, func=None):
|
254
|
+
super().__init__((), pattern=None, depends=(), uses=uses, builder=func)
|
255
|
+
if name:
|
256
|
+
self.name = _name_to_str(name)
|
257
|
+
if name:
|
258
|
+
self.targets = [name]
|
259
|
+
self.first_target = self.name
|
260
|
+
else:
|
261
|
+
self.name = None
|
262
|
+
|
263
|
+
self.default = default
|
264
|
+
if func:
|
265
|
+
self._set_funcname(func)
|
266
|
+
if not self.builder:
|
267
|
+
self.builder = default_builder
|
268
|
+
|
269
|
+
def _set_funcname(self, f):
|
270
|
+
if not self.name:
|
271
|
+
if not f.__name__ or f.__name__ == "<lambda>":
|
272
|
+
raise RuleError(
|
273
|
+
"Task function should have a name. Use @task(name='name')"
|
274
|
+
)
|
275
|
+
self.name = f.__name__
|
276
|
+
self.targets = [f.__name__]
|
277
|
+
|
278
|
+
self.first_target = self.name
|
279
|
+
|
280
|
+
def __call__(self, f):
|
281
|
+
self.builder = f
|
282
|
+
self._set_funcname(f)
|
283
|
+
return _TaskFunc(f, self.name)
|
284
|
+
|
285
|
+
|
230
286
|
class Rules:
|
231
287
|
def __init__(self):
|
232
288
|
self.rules = []
|
@@ -234,40 +290,52 @@ class Rules:
|
|
234
290
|
self._detect_loop = set()
|
235
291
|
self.frozen = False
|
236
292
|
|
237
|
-
def add_rule(self, targets, pattern, depends, uses, builder):
|
293
|
+
def add_rule(self, targets, pattern=None, depends=(), uses=(), builder=None):
|
294
|
+
if builder:
|
295
|
+
if not callable(builder):
|
296
|
+
raise ValueError(f"{builder} is not callable")
|
297
|
+
|
238
298
|
if self.frozen:
|
239
299
|
raise RuntimeError("No new rule can be added after initialization")
|
240
300
|
|
241
|
-
dep = Rule(
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
301
|
+
dep = Rule(targets, pattern, depends, uses, builder)
|
302
|
+
self.rules.append(dep)
|
303
|
+
return dep
|
304
|
+
|
305
|
+
def add_task(self, name=None, uses=(), default=False, func=None):
|
306
|
+
if self.frozen:
|
307
|
+
raise RuntimeError("No new rule can be added after initialization")
|
308
|
+
dep = Task(name, uses, default, func)
|
248
309
|
self.rules.append(dep)
|
249
310
|
return dep
|
250
311
|
|
251
|
-
def rule(self,
|
252
|
-
if
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
dep
|
312
|
+
def rule(self, targets, *, pattern=None, depends=(), uses=()):
|
313
|
+
if not targets:
|
314
|
+
raise ValueError("No target specified")
|
315
|
+
|
316
|
+
dep = self.add_rule([targets], pattern, depends, uses, None)
|
317
|
+
return dep
|
318
|
+
|
319
|
+
def task(self, func=None, *, name=None, uses=(), default=False):
|
320
|
+
if func:
|
321
|
+
if not callable(func):
|
322
|
+
raise ValueError(f"{func} is not callable")
|
323
|
+
|
324
|
+
dep = self.add_task(name, uses, default, func)
|
257
325
|
return dep
|
258
326
|
|
259
327
|
def iter_rule(self, name):
|
260
|
-
name =
|
328
|
+
name = _name_to_str(name)
|
261
329
|
for dep in self.rules:
|
262
330
|
for target in dep.targets:
|
263
|
-
m = re.fullmatch(
|
331
|
+
m = re.fullmatch(target, name)
|
264
332
|
if m:
|
265
333
|
stem = None
|
266
334
|
d = m.groupdict().get("stem", None)
|
267
335
|
if d is not None:
|
268
336
|
stem = d
|
269
337
|
elif dep.pattern:
|
270
|
-
m = re.fullmatch(
|
338
|
+
m = re.fullmatch(dep.pattern, name)
|
271
339
|
if m:
|
272
340
|
stem = m.groupdict().get("stem", None)
|
273
341
|
|
@@ -282,7 +350,6 @@ class Rules:
|
|
282
350
|
break
|
283
351
|
|
284
352
|
def get_dep_names(self, name):
|
285
|
-
assert name
|
286
353
|
ret_depends = []
|
287
354
|
ret_uses = []
|
288
355
|
|
@@ -296,10 +363,16 @@ class Rules:
|
|
296
363
|
return unique_list(ret_depends), unique_list(ret_uses)
|
297
364
|
|
298
365
|
def select_first_target(self):
|
366
|
+
first = None
|
299
367
|
for dep in self.rules:
|
368
|
+
if dep.default and (not first):
|
369
|
+
first = dep.name
|
370
|
+
|
300
371
|
if dep.first_target:
|
301
372
|
return dep.first_target
|
302
373
|
|
374
|
+
return first
|
375
|
+
|
303
376
|
def select_builder(self, name):
|
304
377
|
for depends, uses, dep in self.iter_rule(name):
|
305
378
|
if not dep.builder:
|
@@ -307,11 +380,13 @@ class Rules:
|
|
307
380
|
return depends, uses, dep
|
308
381
|
|
309
382
|
def build_tree(self, name, lv=1):
|
383
|
+
assert name
|
310
384
|
self.frozen = True
|
311
385
|
|
312
|
-
name =
|
386
|
+
name = _name_to_str(name)
|
313
387
|
if name in self._detect_loop:
|
314
|
-
raise CircularReferenceError(name)
|
388
|
+
raise CircularReferenceError(f"Circular reference detected: {name}")
|
389
|
+
|
315
390
|
self._detect_loop.add(name)
|
316
391
|
try:
|
317
392
|
if name in self.tree:
|
@@ -326,7 +401,7 @@ class Rules:
|
|
326
401
|
depends.extend(build_uses)
|
327
402
|
|
328
403
|
depends = unique_list(depends)
|
329
|
-
self.tree[name].update(
|
404
|
+
self.tree[name].update(depends)
|
330
405
|
for dep in depends:
|
331
406
|
self.build_tree(dep, lv=lv + 1)
|
332
407
|
|
@@ -339,17 +414,19 @@ class Checkers:
|
|
339
414
|
self.checkers = []
|
340
415
|
|
341
416
|
def get_checker(self, name):
|
417
|
+
name = _name_to_str(name)
|
342
418
|
for targets, f in self.checkers:
|
343
419
|
for target in targets:
|
344
|
-
if fnmatch(name,
|
420
|
+
if fnmatch(name, target):
|
345
421
|
return f
|
346
422
|
|
347
|
-
def add_check(self,
|
348
|
-
|
423
|
+
def add_check(self, targets, f):
|
424
|
+
targets = list(map(_name_to_str, flatten(targets or ())))
|
425
|
+
self.checkers.append((targets, f))
|
349
426
|
|
350
|
-
def check(self,
|
427
|
+
def check(self, targets):
|
351
428
|
def deco(f):
|
352
|
-
self.add_check(
|
429
|
+
self.add_check(targets, f)
|
353
430
|
return f
|
354
431
|
|
355
432
|
return deco
|
@@ -358,10 +435,6 @@ class Checkers:
|
|
358
435
|
MAX_TS = 1 << 63
|
359
436
|
|
360
437
|
|
361
|
-
def is_file_exists(name):
|
362
|
-
return os.path.getmtime(name)
|
363
|
-
|
364
|
-
|
365
438
|
class Exists:
|
366
439
|
def __init__(self, name, exists, ts=None):
|
367
440
|
self.name = name
|
@@ -421,35 +494,45 @@ def write(filename, s, append=False):
|
|
421
494
|
f.write(s)
|
422
495
|
|
423
496
|
|
424
|
-
def quote(s):
|
425
|
-
return shlex.quote(str(s))
|
426
|
-
|
427
|
-
|
428
|
-
def squote(*s):
|
497
|
+
def quote(*s):
|
429
498
|
ret = [shlex.quote(str(x)) for x in flatten(s)]
|
430
499
|
return ret
|
431
500
|
|
432
501
|
|
502
|
+
def squote(s):
|
503
|
+
s = " ".join(str(e) for e in flatten(s))
|
504
|
+
return shlex.quote(s)
|
505
|
+
|
506
|
+
|
433
507
|
def makedirs(path):
|
434
508
|
os.makedirs(path, exist_ok=True)
|
435
509
|
|
436
510
|
|
437
511
|
class Prod:
|
438
512
|
def __init__(self, modulefile, njobs=1, params=None):
|
439
|
-
|
513
|
+
if modulefile:
|
514
|
+
self.modulefile = Path(modulefile)
|
515
|
+
else:
|
516
|
+
self.modulefile = None
|
517
|
+
|
440
518
|
self.rules = Rules()
|
441
519
|
self.checkers = Checkers()
|
442
520
|
if njobs > 1:
|
443
|
-
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=
|
521
|
+
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=100)
|
444
522
|
else:
|
445
523
|
self.executor = None
|
446
524
|
self.params = Params(params)
|
525
|
+
self.use_git_timestamp = pyprod.args.use_git
|
526
|
+
|
447
527
|
self.buildings = {}
|
448
|
-
self.module =
|
528
|
+
self.module = None
|
529
|
+
if self.modulefile:
|
530
|
+
self.module = self.load_pyprodfile(self.modulefile)
|
449
531
|
self.built = 0 # number of build execused
|
450
532
|
|
451
533
|
def get_module_globals(self):
|
452
534
|
globals = {
|
535
|
+
"build": self.build,
|
453
536
|
"capture": capture,
|
454
537
|
"check": self.checkers.check,
|
455
538
|
"environ": Envs(),
|
@@ -466,6 +549,8 @@ class Prod:
|
|
466
549
|
"rule": self.rules.rule,
|
467
550
|
"run": run,
|
468
551
|
"shutil": shutil,
|
552
|
+
"task": self.rules.task,
|
553
|
+
"use_git": self.use_git,
|
469
554
|
"write": write,
|
470
555
|
"MAX_TS": MAX_TS,
|
471
556
|
"Path": Path,
|
@@ -489,65 +574,94 @@ class Prod:
|
|
489
574
|
async def run_in_executor(self, func, *args, **kwargs):
|
490
575
|
if self.executor:
|
491
576
|
loop = asyncio.get_running_loop()
|
492
|
-
|
577
|
+
ret = await loop.run_in_executor(
|
493
578
|
self.executor, lambda: func(*args, **kwargs)
|
494
579
|
)
|
495
580
|
else:
|
496
|
-
|
581
|
+
ret = func(*args, **kwargs)
|
582
|
+
|
583
|
+
return ret
|
584
|
+
|
585
|
+
def get_file_mtime(self, name):
|
586
|
+
return os.path.getmtime(name)
|
587
|
+
|
588
|
+
def get_file_mtime_git(self, name):
|
589
|
+
ret = subprocess.check_output(
|
590
|
+
["git", "log", "-1", "--format=%ai", "--", name], text=True
|
591
|
+
).strip()
|
592
|
+
if not ret:
|
593
|
+
raise FileNotFoundError(f"{name} did not match any file in git")
|
594
|
+
|
595
|
+
# 2025-01-17 00:05:48 +0900
|
596
|
+
return dateutil.parser.parse(ret)
|
597
|
+
|
598
|
+
async def is_exists(self, name):
|
599
|
+
checker = self.checkers.get_checker(name)
|
600
|
+
try:
|
601
|
+
if checker:
|
602
|
+
ret = await self.run_in_executor(checker, name)
|
603
|
+
elif self.use_git_timestamp:
|
604
|
+
ret = await self.run_in_executor(self.get_file_mtime_git, name)
|
605
|
+
else:
|
606
|
+
ret = await self.run_in_executor(self.get_file_mtime, name)
|
607
|
+
except FileNotFoundError:
|
608
|
+
ret = False
|
609
|
+
|
610
|
+
if isinstance(ret, FileNotFoundError):
|
611
|
+
ret = False
|
612
|
+
|
613
|
+
if not ret:
|
614
|
+
return Exists(name, False)
|
615
|
+
if isinstance(ret, datetime.datetime):
|
616
|
+
ret = ret.timestamp()
|
617
|
+
if ret < 0:
|
618
|
+
ret = MAX_TS
|
619
|
+
return Exists(name, True, ret)
|
620
|
+
|
621
|
+
def build(self, *deps):
|
622
|
+
children = []
|
623
|
+
for elem in deps:
|
624
|
+
child = [_name_to_str(name) for name in flatten(elem)]
|
625
|
+
children.append(child)
|
626
|
+
self.deps[0:0] = children
|
627
|
+
|
628
|
+
def use_git(self, use):
|
629
|
+
self.use_git_timestamp = use
|
497
630
|
|
498
631
|
def get_default_target(self):
|
499
632
|
return self.rules.select_first_target()
|
500
633
|
|
501
634
|
async def start(self, deps):
|
635
|
+
self.loop = asyncio.get_running_loop()
|
502
636
|
self.built = 0
|
503
|
-
|
504
|
-
|
505
|
-
|
506
|
-
|
507
|
-
if value:
|
508
|
-
names.append(value)
|
509
|
-
else:
|
510
|
-
names.append(name)
|
511
|
-
else:
|
512
|
-
names.append(name)
|
513
|
-
|
514
|
-
builds = []
|
515
|
-
waitings = []
|
516
|
-
for obj in flatten(names):
|
517
|
-
if isinstance(obj, str | Path):
|
518
|
-
builds.append(obj)
|
519
|
-
elif isinstance(obj, Rule):
|
520
|
-
raise RuleError(obj)
|
521
|
-
elif callable(obj):
|
522
|
-
self.built += 1
|
523
|
-
task = asyncio.create_task(self.run_in_executor(obj))
|
524
|
-
waitings.append(task)
|
525
|
-
else:
|
526
|
-
raise RuleError(obj)
|
637
|
+
self.deps = deps[:]
|
638
|
+
while self.deps:
|
639
|
+
dep = self.deps.pop(0)
|
640
|
+
await self.schedule([dep])
|
527
641
|
|
528
|
-
await self.build(builds)
|
529
|
-
await asyncio.gather(*waitings)
|
530
642
|
return self.built
|
531
643
|
|
532
|
-
async def
|
644
|
+
async def schedule(self, deps):
|
645
|
+
deps = list(flatten(deps))
|
533
646
|
tasks = []
|
534
647
|
waits = []
|
535
648
|
for dep in deps:
|
536
649
|
if dep not in self.buildings:
|
537
650
|
ev = asyncio.Event()
|
538
651
|
self.buildings[dep] = ev
|
539
|
-
|
540
|
-
tasks.append((dep,
|
652
|
+
coro = self.run(dep)
|
653
|
+
tasks.append((dep, coro))
|
541
654
|
waits.append(ev)
|
542
655
|
else:
|
543
656
|
obj = self.buildings[dep]
|
544
657
|
if isinstance(obj, asyncio.Event):
|
545
658
|
waits.append(obj)
|
546
659
|
|
547
|
-
for
|
660
|
+
results = await asyncio.gather(*(coro for _, coro in tasks))
|
661
|
+
for ret, (dep, _) in zip(results, tasks):
|
548
662
|
ev = self.buildings[dep]
|
549
663
|
try:
|
550
|
-
self.buildings[dep] =
|
664
|
+
self.buildings[dep] = ret
|
551
665
|
finally:
|
552
666
|
ev.set()
|
553
667
|
|
@@ -563,27 +677,8 @@ class Prod:
|
|
563
677
|
return max(ts)
|
564
678
|
return 0
|
565
679
|
|
566
|
-
async def is_exists(self, name):
|
567
|
-
checker = self.checkers.get_checker(name)
|
568
|
-
try:
|
569
|
-
if checker:
|
570
|
-
ret = await self.run_in_executor(checker, name)
|
571
|
-
else:
|
572
|
-
ret = await self.run_in_executor(is_file_exists, name)
|
573
|
-
except FileNotFoundError:
|
574
|
-
ret = False
|
575
|
-
|
576
|
-
if not ret:
|
577
|
-
return Exists(name, False)
|
578
|
-
if isinstance(ret, datetime.datetime):
|
579
|
-
ret = ret.timestamp()
|
580
|
-
if ret < 0:
|
581
|
-
ret = MAX_TS
|
582
|
-
return Exists(name, True, ret)
|
583
|
-
|
584
680
|
async def run(self, name): # -> Any | int:
|
585
|
-
name =
|
586
|
-
|
681
|
+
name = _name_to_str(name)
|
587
682
|
self.rules.build_tree(name)
|
588
683
|
deps, uses = self.rules.get_dep_names(name)
|
589
684
|
selected = self.rules.select_builder(name)
|
@@ -592,11 +687,23 @@ class Prod:
|
|
592
687
|
deps = deps + build_deps
|
593
688
|
uses = uses + build_uses
|
594
689
|
|
595
|
-
|
690
|
+
tasks = []
|
596
691
|
if deps:
|
597
|
-
|
692
|
+
deps_task = asyncio.create_task(self.schedule(deps))
|
693
|
+
tasks.append(deps_task)
|
598
694
|
if uses:
|
599
|
-
|
695
|
+
uses_task = self.schedule(uses)
|
696
|
+
tasks.append(uses_task)
|
697
|
+
|
698
|
+
await asyncio.gather(*tasks)
|
699
|
+
ts = 0
|
700
|
+
if deps:
|
701
|
+
ts = deps_task.result()
|
702
|
+
|
703
|
+
if selected and isinstance(builder, Task):
|
704
|
+
self.built += 1
|
705
|
+
await self.run_in_executor(builder.builder, *build_deps)
|
706
|
+
return MAX_TS
|
600
707
|
|
601
708
|
exists = await self.is_exists(name)
|
602
709
|
|
@@ -608,8 +715,14 @@ class Prod:
|
|
608
715
|
logger.debug("%r already exists", name)
|
609
716
|
|
610
717
|
if not exists.exists and not selected:
|
611
|
-
raise NoRuleToMakeTargetError(name)
|
612
|
-
|
718
|
+
raise NoRuleToMakeTargetError(f"No rule to make target: {name}")
|
719
|
+
|
720
|
+
elif selected and (
|
721
|
+
(not exists.exists)
|
722
|
+
or (ts >= MAX_TS)
|
723
|
+
or (exists.ts < ts)
|
724
|
+
or pyprod.args.rebuild
|
725
|
+
):
|
613
726
|
logger.warning("building: %r", name)
|
614
727
|
await self.run_in_executor(builder.builder, name, *build_deps)
|
615
728
|
self.built += 1
|
pyprod/utils.py
CHANGED
@@ -1,13 +1,15 @@
|
|
1
|
-
from collections.abc import Iterable
|
1
|
+
from collections.abc import Iterable
|
2
2
|
|
3
3
|
|
4
|
-
def flatten(seq):
|
5
|
-
if isinstance(seq, str) or (not isinstance(seq,
|
4
|
+
def flatten(seq, ignore_none=True):
|
5
|
+
if isinstance(seq, str) or (not isinstance(seq, Iterable)):
|
6
6
|
yield seq
|
7
7
|
return
|
8
8
|
|
9
9
|
for item in seq:
|
10
10
|
if isinstance(item, str) or (not isinstance(item, Iterable)):
|
11
|
+
if ignore_none and (item is None):
|
12
|
+
continue
|
11
13
|
yield item
|
12
14
|
else:
|
13
15
|
yield from flatten(item)
|
pyprod/venv.py
CHANGED
@@ -11,15 +11,15 @@ import pyprod
|
|
11
11
|
from .utils import flatten
|
12
12
|
|
13
13
|
THREADID = threading.get_ident()
|
14
|
-
pyprodenv = ".pyprod"
|
15
14
|
venvdir = None
|
16
15
|
|
17
|
-
PYPRODVENV = "
|
16
|
+
PYPRODVENV = "pyprod"
|
18
17
|
|
19
18
|
|
20
19
|
def makevenv(conffile):
|
21
20
|
global venvdir
|
22
|
-
|
21
|
+
major, minor = sys.version_info[:2]
|
22
|
+
venvdir = Path(".") / f".{conffile.name}.{major}.{minor}.{PYPRODVENV}"
|
23
23
|
if not venvdir.is_dir():
|
24
24
|
venv.main([str(venvdir)])
|
25
25
|
|
@@ -42,6 +42,15 @@ def pip(*args):
|
|
42
42
|
makevenv(pyprod.modulefile)
|
43
43
|
args = flatten(args)
|
44
44
|
subprocess.run(
|
45
|
-
[
|
45
|
+
[
|
46
|
+
venvdir / "bin/python",
|
47
|
+
"-m",
|
48
|
+
"pip",
|
49
|
+
"--disable-pip-version-check",
|
50
|
+
"--no-input",
|
51
|
+
"install",
|
52
|
+
"-q",
|
53
|
+
*args,
|
54
|
+
],
|
46
55
|
check=True,
|
47
56
|
)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: PyProd
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.5.0
|
4
4
|
Summary: PyProd: More Makeable than Make
|
5
5
|
Project-URL: Homepage, https://github.com/atsuoishimoto/pyprod
|
6
6
|
Project-URL: Documentation, https://pyprod.readthedocs.io/en/latest/
|
@@ -12,12 +12,13 @@ Classifier: Operating System :: OS Independent
|
|
12
12
|
Classifier: Programming Language :: Python :: 3
|
13
13
|
Classifier: Topic :: Software Development :: Build Tools
|
14
14
|
Requires-Python: >=3.10
|
15
|
+
Requires-Dist: python-dateutil
|
15
16
|
Description-Content-Type: text/x-rst
|
16
17
|
|
17
18
|
PyProd - More Makeable than Make
|
18
19
|
=================================
|
19
20
|
|
20
|
-
PyProd is a Python script that can be used as an alternative to Makefile. By leveraging Python's versatility, it enables you to define build rules and dependencies programmatically, allowing for dynamic configurations, integration with existing Python libraries, and custom build logic not easily achievable with traditional Makefiles. For detailed documentation, please refer to the `official documentation <https://pyprod.readthedocs.io/en/
|
21
|
+
PyProd is a Python script that can be used as an alternative to Makefile. By leveraging Python's versatility, it enables you to define build rules and dependencies programmatically, allowing for dynamic configurations, integration with existing Python libraries, and custom build logic not easily achievable with traditional Makefiles. For detailed documentation, please refer to the `official documentation <https://pyprod.readthedocs.io/en/stable/>`_.
|
21
22
|
|
22
23
|
|
23
24
|
Features
|
@@ -42,22 +43,26 @@ With PyProd, a traditional Makefile for C can be expressed as a Python script li
|
|
42
43
|
.. code-block:: python
|
43
44
|
|
44
45
|
CC = "gcc"
|
45
|
-
CFLAGS = "-I."
|
46
|
+
CFLAGS = "-c -I."
|
46
47
|
DEPS = "hello.h"
|
47
48
|
OBJS = "hello.o main.o".split()
|
49
|
+
EXE = "hello.exe"
|
48
50
|
|
49
51
|
@rule("%.o", depends=("%.c", DEPS))
|
50
52
|
def compile(target, src, *deps):
|
51
|
-
run(CC, "-
|
53
|
+
run(CC, "-o", target, src, CFLAGS)
|
52
54
|
|
53
|
-
@rule(
|
55
|
+
@rule(EXE, depends=OBJS)
|
54
56
|
def link(target, *objs):
|
55
57
|
run(CC, "-o", target, objs)
|
56
58
|
|
59
|
+
@task
|
57
60
|
def clean():
|
58
61
|
run("rm -f", OBJS, "hello.exe")
|
59
62
|
|
60
|
-
|
63
|
+
@task
|
64
|
+
def rebuild():
|
65
|
+
build(clean, EXE)
|
61
66
|
|
62
67
|
|
63
68
|
To run the build script, simply execute:
|
@@ -67,6 +72,8 @@ To run the build script, simply execute:
|
|
67
72
|
$ cd project
|
68
73
|
$ pyprod
|
69
74
|
|
75
|
+
Other examples can be found in the `samples <https://github.com/atsuoishimoto/pyprod/tree/main/samples>`_ directory.
|
76
|
+
|
70
77
|
License
|
71
78
|
-------
|
72
79
|
PyProd is licensed under the MIT License. See the `LICENSE <LICENSE>`_ file for more details.
|
@@ -0,0 +1,11 @@
|
|
1
|
+
pyprod/__init__.py,sha256=LBK46heutvn3KmsCrKIYu8RQikbfnjZaj2xFrXaeCzQ,22
|
2
|
+
pyprod/__main__.py,sha256=Vdhw8YA1K3wPMlbJQYL5WqvRzAKVeZ16mZQFO9VRmCo,62
|
3
|
+
pyprod/main.py,sha256=O1tkuZzZ7nuoI2DjFyJlKn-2YYn-UL-jsgrxb5EvmVQ,2945
|
4
|
+
pyprod/prod.py,sha256=Tdhr3cTnJv3m61157H48WNuFhvQGJ8lZuaSD6R9XspM,20000
|
5
|
+
pyprod/utils.py,sha256=6bA06MtxvzcEArAozeJVMgCvoTT185OPEGypM1jjoG0,481
|
6
|
+
pyprod/venv.py,sha256=ZNMtHDBdC-eNFJE0-GxDlh6tlGy5Y-2m1r86SqxJJR0,1229
|
7
|
+
pyprod-0.5.0.dist-info/METADATA,sha256=q28RfZ10Oodv3EHGo9Fa1x5U42cihFb9XTV-J9uaNFc,2683
|
8
|
+
pyprod-0.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
9
|
+
pyprod-0.5.0.dist-info/entry_points.txt,sha256=zFycf8BYSMRDTiI0jftmcvtkf9XM4MZ4BL3JaIer_ZM,44
|
10
|
+
pyprod-0.5.0.dist-info/licenses/LICENSE,sha256=OtPgwnlLrsVEYPnTraun5AqftAT5vUv4rIan-qYj7nE,1071
|
11
|
+
pyprod-0.5.0.dist-info/RECORD,,
|
pyprod-0.3.0.dist-info/RECORD
DELETED
@@ -1,11 +0,0 @@
|
|
1
|
-
pyprod/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
2
|
-
pyprod/__main__.py,sha256=Vdhw8YA1K3wPMlbJQYL5WqvRzAKVeZ16mZQFO9VRmCo,62
|
3
|
-
pyprod/main.py,sha256=nS0mcLy3Ix7ydnX1Pu5g7ceLzhh6DndwJWO5sjO_qWg,2580
|
4
|
-
pyprod/prod.py,sha256=r2e-JSHiTw2SOcv-tYFx4aRpVkQbOb5j9vEVrjT0meE,16694
|
5
|
-
pyprod/utils.py,sha256=oiiUkSbeqTazbtJ6gz7ZKqG1OvAeV-nV9u_3Y0DCOOM,401
|
6
|
-
pyprod/venv.py,sha256=_riw56YQvUOSd55u_1m9ElsqPdjM5qVvIZP6dr9Fzt4,1051
|
7
|
-
pyprod-0.3.0.dist-info/METADATA,sha256=t9TMYeBveYE1LFZNpcYJAtOdUo7gPARFHmIZ9p0bnag,2477
|
8
|
-
pyprod-0.3.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
9
|
-
pyprod-0.3.0.dist-info/entry_points.txt,sha256=zFycf8BYSMRDTiI0jftmcvtkf9XM4MZ4BL3JaIer_ZM,44
|
10
|
-
pyprod-0.3.0.dist-info/licenses/LICENSE,sha256=OtPgwnlLrsVEYPnTraun5AqftAT5vUv4rIan-qYj7nE,1071
|
11
|
-
pyprod-0.3.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|