ScriptCollection 4.3.64__py3-none-any.whl → 4.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/AnionBuildPlatform.py +205 -205
- ScriptCollection/CertificateUpdater.py +157 -157
- ScriptCollection/Executables.py +4 -2
- ScriptCollection/GeneralUtilities.py +1696 -1696
- ScriptCollection/OCIImages/OCIImageManager.py +278 -278
- ScriptCollection/ProcessesRunner.py +43 -43
- ScriptCollection/ProgramRunnerSudo.py +110 -110
- ScriptCollection/Resources/CultureChooser/CultureChooser.js +0 -0
- ScriptCollection/Resources/CultureChooser/index.html +0 -0
- ScriptCollection/Resources/Dependencies/JRE/Version.txt +0 -0
- ScriptCollection/Resources/Dependencies/OpenAPIGenerator/Version.txt +0 -0
- ScriptCollection/Resources/Dependencies/PlantUML/Version.txt +0 -0
- ScriptCollection/Resources/Fonts/DejaVuSans-Bold.ttf +0 -0
- ScriptCollection/Resources/Fonts/DejaVuSans-BoldOblique.ttf +0 -0
- ScriptCollection/Resources/Fonts/DejaVuSans-Oblique.ttf +0 -0
- ScriptCollection/Resources/Fonts/DejaVuSans.ttf +0 -0
- ScriptCollection/Resources/Fonts/LICENSE.txt +0 -0
- ScriptCollection/Resources/MaintenanceSite/MaintenanceSite.html +0 -0
- ScriptCollection/SCLog.py +115 -115
- ScriptCollection/ScriptCollectionCore.py +4372 -4351
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +716 -709
- ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +235 -235
- ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +258 -258
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +524 -524
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +157 -155
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +691 -681
- ScriptCollection/TFCPS/TFCPS_CreateRelease.py +98 -98
- ScriptCollection/TFCPS/TFCPS_Generic.py +44 -44
- ScriptCollection/TFCPS/TFCPS_MergeToMain.py +132 -132
- ScriptCollection/TFCPS/TFCPS_MergeToStable.py +358 -358
- ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +48 -48
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +30 -4
- {scriptcollection-4.3.64.dist-info → scriptcollection-4.4.0.dist-info}/METADATA +1 -1
- {scriptcollection-4.3.64.dist-info → scriptcollection-4.4.0.dist-info}/RECORD +26 -26
- {scriptcollection-4.3.64.dist-info → scriptcollection-4.4.0.dist-info}/WHEEL +0 -0
- {scriptcollection-4.3.64.dist-info → scriptcollection-4.4.0.dist-info}/entry_points.txt +0 -0
- {scriptcollection-4.3.64.dist-info → scriptcollection-4.4.0.dist-info}/top_level.txt +0 -0
|
@@ -1,681 +1,691 @@
|
|
|
1
|
-
import os
|
|
2
|
-
import json
|
|
3
|
-
import re
|
|
4
|
-
import socket
|
|
5
|
-
from datetime import datetime, timedelta,timezone
|
|
6
|
-
import xmlschema
|
|
7
|
-
import yaml
|
|
8
|
-
from packaging.version import Version
|
|
9
|
-
from ..GeneralUtilities import GeneralUtilities
|
|
10
|
-
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
11
|
-
from ..SCLog import LogLevel
|
|
12
|
-
from .TFCPS_BuildCodeUnitsHook import TFCPS_BuildCodeUnitsHook
|
|
13
|
-
from .TFCPS_CodeUnit_BuildCodeUnit import TFCPS_CodeUnit_BuildCodeUnit
|
|
14
|
-
from .TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
|
|
15
|
-
from .TFCPS_Tools_General import TFCPS_Tools_General
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
class TFCPS_UpdateDependenciesHook(TFCPS_BuildCodeUnitsHook):
|
|
19
|
-
"""Updates the dependencies of the repository and of its codeunits while the codeunits are built.
|
|
20
|
-
The dependency-update is implemented as a hook of the regular codeunit-build (and not as an own reduced build-process, which is how
|
|
21
|
-
it was implemented before) so that it runs with exactly the same preparation as a normal build: the required environment-variables
|
|
22
|
-
are set, the custom pre-codeunit-build-script runs and therefore the package-sources which are needed to resolve the dependencies of
|
|
23
|
-
a codeunit are available. Without that preparation an update fails as soon as a codeunit has a dependency which is not available on a
|
|
24
|
-
public package-source - which was the case for an update inside a build-container."""
|
|
25
|
-
|
|
26
|
-
__sc: ScriptCollectionCore = None
|
|
27
|
-
__tfcps_tools_general: TFCPS_Tools_General = None
|
|
28
|
-
|
|
29
|
-
def __init__(self, sc: ScriptCollectionCore, tfcps_tools_general: TFCPS_Tools_General):
|
|
30
|
-
self.__sc = sc
|
|
31
|
-
self.__tfcps_tools_general = tfcps_tools_general
|
|
32
|
-
|
|
33
|
-
@GeneralUtilities.check_arguments
|
|
34
|
-
def run_after_preparation(self, repository: str) -> None:
|
|
35
|
-
update_dependencies_script_folder: str = os.path.join(repository, "Other", "Scripts")
|
|
36
|
-
if os.path.isfile(os.path.join(update_dependencies_script_folder, "UpdateDependencies.py")):
|
|
37
|
-
self.__sc.log.log("Update dependencies of the repository...")
|
|
38
|
-
self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", update_dependencies_script_folder)
|
|
39
|
-
|
|
40
|
-
@GeneralUtilities.check_arguments
|
|
41
|
-
def run_after_codeunit_was_built(self, codeunit_build: TFCPS_CodeUnit_BuildCodeUnit) -> None:
|
|
42
|
-
codeunit_name: str = codeunit_build.codeunit_name
|
|
43
|
-
codeunit_folder: str = codeunit_build.codeunit_folder
|
|
44
|
-
repository: str = codeunit_build.repository_folder
|
|
45
|
-
if not self.__tfcps_tools_general.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml")):
|
|
46
|
-
return
|
|
47
|
-
self.__sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
|
|
48
|
-
self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", os.path.join(codeunit_folder, "Other"))
|
|
49
|
-
if self.__sc.git_repository_has_uncommitted_changes(repository):
|
|
50
|
-
#the update changed something, so it has to be verified that the codeunit is still buildable with the updated dependencies.
|
|
51
|
-
self.__sc.log.log(f"Build codeunit {codeunit_name} again to verify it is still buildable with the updated dependencies...")
|
|
52
|
-
codeunit_build.build_codeunit()
|
|
53
|
-
|
|
54
|
-
class TFCPS_CodeUnit_BuildCodeUnits:
|
|
55
|
-
repository:str=None
|
|
56
|
-
tfcps_tools_general:TFCPS_Tools_General=None
|
|
57
|
-
sc:ScriptCollectionCore=None
|
|
58
|
-
target_environment_type:str=None
|
|
59
|
-
additionalargumentsfile:str=None
|
|
60
|
-
__use_cache:bool = None
|
|
61
|
-
__is_pre_merge:bool = None
|
|
62
|
-
__assert_no_new_changes:bool = None
|
|
63
|
-
__add_ready_to_merge_flag:bool = None
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
self.sc
|
|
68
|
-
self.
|
|
69
|
-
self.
|
|
70
|
-
self.repository
|
|
71
|
-
self.
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
self.
|
|
76
|
-
self.
|
|
77
|
-
self.
|
|
78
|
-
self.
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
GeneralUtilities.assert_condition(
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
GeneralUtilities.assert_condition(not
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
self.
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
if
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
GeneralUtilities.
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
self.
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
GeneralUtilities.
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
GeneralUtilities.
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
for
|
|
239
|
-
self.sc.
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
(
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
self.
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
@GeneralUtilities.check_arguments
|
|
299
|
-
def
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
#
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
if
|
|
331
|
-
scbuildcodeunits_arguments
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
#
|
|
350
|
-
#
|
|
351
|
-
#
|
|
352
|
-
#
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
]
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
# the
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
GeneralUtilities.
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
GeneralUtilities.
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
GeneralUtilities.ensure_file_exists(
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
GeneralUtilities.
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
"
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
"
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
"
|
|
485
|
-
"
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
"
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
#
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
#
|
|
571
|
-
#
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
#
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
raise ValueError(f"
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
#
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
self.
|
|
651
|
-
|
|
652
|
-
self.sc.
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
1
|
+
import os
|
|
2
|
+
import json
|
|
3
|
+
import re
|
|
4
|
+
import socket
|
|
5
|
+
from datetime import datetime, timedelta,timezone
|
|
6
|
+
import xmlschema
|
|
7
|
+
import yaml
|
|
8
|
+
from packaging.version import Version
|
|
9
|
+
from ..GeneralUtilities import GeneralUtilities
|
|
10
|
+
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
11
|
+
from ..SCLog import LogLevel
|
|
12
|
+
from .TFCPS_BuildCodeUnitsHook import TFCPS_BuildCodeUnitsHook
|
|
13
|
+
from .TFCPS_CodeUnit_BuildCodeUnit import TFCPS_CodeUnit_BuildCodeUnit
|
|
14
|
+
from .TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
|
|
15
|
+
from .TFCPS_Tools_General import TFCPS_Tools_General
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class TFCPS_UpdateDependenciesHook(TFCPS_BuildCodeUnitsHook):
|
|
19
|
+
"""Updates the dependencies of the repository and of its codeunits while the codeunits are built.
|
|
20
|
+
The dependency-update is implemented as a hook of the regular codeunit-build (and not as an own reduced build-process, which is how
|
|
21
|
+
it was implemented before) so that it runs with exactly the same preparation as a normal build: the required environment-variables
|
|
22
|
+
are set, the custom pre-codeunit-build-script runs and therefore the package-sources which are needed to resolve the dependencies of
|
|
23
|
+
a codeunit are available. Without that preparation an update fails as soon as a codeunit has a dependency which is not available on a
|
|
24
|
+
public package-source - which was the case for an update inside a build-container."""
|
|
25
|
+
|
|
26
|
+
__sc: ScriptCollectionCore = None
|
|
27
|
+
__tfcps_tools_general: TFCPS_Tools_General = None
|
|
28
|
+
|
|
29
|
+
def __init__(self, sc: ScriptCollectionCore, tfcps_tools_general: TFCPS_Tools_General):
|
|
30
|
+
self.__sc = sc
|
|
31
|
+
self.__tfcps_tools_general = tfcps_tools_general
|
|
32
|
+
|
|
33
|
+
@GeneralUtilities.check_arguments
|
|
34
|
+
def run_after_preparation(self, repository: str) -> None:
|
|
35
|
+
update_dependencies_script_folder: str = os.path.join(repository, "Other", "Scripts")
|
|
36
|
+
if os.path.isfile(os.path.join(update_dependencies_script_folder, "UpdateDependencies.py")):
|
|
37
|
+
self.__sc.log.log("Update dependencies of the repository...")
|
|
38
|
+
self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", update_dependencies_script_folder)
|
|
39
|
+
|
|
40
|
+
@GeneralUtilities.check_arguments
|
|
41
|
+
def run_after_codeunit_was_built(self, codeunit_build: TFCPS_CodeUnit_BuildCodeUnit) -> None:
|
|
42
|
+
codeunit_name: str = codeunit_build.codeunit_name
|
|
43
|
+
codeunit_folder: str = codeunit_build.codeunit_folder
|
|
44
|
+
repository: str = codeunit_build.repository_folder
|
|
45
|
+
if not self.__tfcps_tools_general.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml")):
|
|
46
|
+
return
|
|
47
|
+
self.__sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
|
|
48
|
+
self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", os.path.join(codeunit_folder, "Other"))
|
|
49
|
+
if self.__sc.git_repository_has_uncommitted_changes(repository):
|
|
50
|
+
#the update changed something, so it has to be verified that the codeunit is still buildable with the updated dependencies.
|
|
51
|
+
self.__sc.log.log(f"Build codeunit {codeunit_name} again to verify it is still buildable with the updated dependencies...")
|
|
52
|
+
codeunit_build.build_codeunit()
|
|
53
|
+
|
|
54
|
+
class TFCPS_CodeUnit_BuildCodeUnits:
|
|
55
|
+
repository:str=None
|
|
56
|
+
tfcps_tools_general:TFCPS_Tools_General=None
|
|
57
|
+
sc:ScriptCollectionCore=None
|
|
58
|
+
target_environment_type:str=None
|
|
59
|
+
additionalargumentsfile:str=None
|
|
60
|
+
__use_cache:bool = None
|
|
61
|
+
__is_pre_merge:bool = None
|
|
62
|
+
__assert_no_new_changes:bool = None
|
|
63
|
+
__add_ready_to_merge_flag:bool = None
|
|
64
|
+
__fast_lane:bool = None
|
|
65
|
+
|
|
66
|
+
def __init__(self,repository:str,loglevel:LogLevel,target_environment_type:str,additionalargumentsfile:str,use_cache:bool,is_pre_merge:bool,assertnonewchanges:bool,add_ready_to_merge_flag:bool=False,fast_lane:bool=False):
|
|
67
|
+
self.sc=ScriptCollectionCore()
|
|
68
|
+
self.sc.log.loglevel=loglevel
|
|
69
|
+
self.__use_cache=use_cache
|
|
70
|
+
self.sc.assert_is_git_repository(repository)
|
|
71
|
+
self.repository=repository
|
|
72
|
+
self.tfcps_tools_general:TFCPS_Tools_General=TFCPS_Tools_General(self.sc)
|
|
73
|
+
allowed_target_environment_types=["Development","QualityCheck","Productive"]
|
|
74
|
+
GeneralUtilities.assert_condition(target_environment_type in allowed_target_environment_types,"Unknown target-environment-type. Allowed values are: "+", ".join(allowed_target_environment_types))
|
|
75
|
+
self.target_environment_type=target_environment_type
|
|
76
|
+
self.additionalargumentsfile=additionalargumentsfile
|
|
77
|
+
self.__is_pre_merge=is_pre_merge
|
|
78
|
+
self.__assert_no_new_changes=assertnonewchanges
|
|
79
|
+
self.__add_ready_to_merge_flag=add_ready_to_merge_flag
|
|
80
|
+
self.__fast_lane=fast_lane
|
|
81
|
+
|
|
82
|
+
@GeneralUtilities.check_arguments
|
|
83
|
+
def build_codeunits(self, hook: TFCPS_BuildCodeUnitsHook = None) -> None:
|
|
84
|
+
"""Builds all codeunits of the repository. The optional hook allows a caller inside ScriptCollection to run additional actions
|
|
85
|
+
between the regular build-steps (see TFCPS_BuildCodeUnitsHook); a caller which only wants to build passes nothing."""
|
|
86
|
+
if hook is None:
|
|
87
|
+
hook = TFCPS_BuildCodeUnitsHook()
|
|
88
|
+
self.sc.log.log(GeneralUtilities.get_line())
|
|
89
|
+
start_time:datetime=GeneralUtilities.get_now()
|
|
90
|
+
ready_to_merge_file=os.path.join(self.repository,".ScriptCollection",".IsReadyToMerge")
|
|
91
|
+
error_occurred=False
|
|
92
|
+
try:
|
|
93
|
+
if self.__fast_lane:
|
|
94
|
+
current_branch_name = self.sc.git_get_current_branch_name(self.repository)
|
|
95
|
+
GeneralUtilities.assert_condition(self.sc.is_fix_branch(current_branch_name), f"Fastlane-builds are only allowed on branches whose name starts with 'fix/', but the current branch is '{current_branch_name}'.")
|
|
96
|
+
|
|
97
|
+
#assert that the product-information-file exists
|
|
98
|
+
product_information_file = os.path.join(self.repository, ".ScriptCollection", "ProductInformation.xml")
|
|
99
|
+
GeneralUtilities.assert_file_exists(product_information_file, f"The file '{product_information_file}' does not exist.")
|
|
100
|
+
|
|
101
|
+
#when the build runs inside a container, ensure the used SCBuilder-image is at least the version required by this repository (defined in .ScriptCollection/OCIImages/ImageDefinition.csv)
|
|
102
|
+
if self.sc.is_runnning_in_container():
|
|
103
|
+
scbuilder_version_environment_value = os.environ.get("SCBuilderVersion")
|
|
104
|
+
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(scbuilder_version_environment_value), "The environment-variable 'SCBuilderVersion' is not set although the build runs inside a container (environment-variable 'ISRUNNINGINCONTAINER' is 'true').")
|
|
105
|
+
required_scbuilder_version = Version(self.tfcps_tools_general.oci_image_manager.get_tag_for_image(self.repository, "SCBuilder"))
|
|
106
|
+
version_match = re.search(r"\d+(\.\d+)+", scbuilder_version_environment_value)
|
|
107
|
+
GeneralUtilities.assert_condition(version_match is not None, f"The environment-variable 'SCBuilderVersion' (value: '{scbuilder_version_environment_value}') does not contain a version-string.")
|
|
108
|
+
actual_scbuilder_version = Version(version_match.group(0))
|
|
109
|
+
GeneralUtilities.assert_condition(actual_scbuilder_version >= required_scbuilder_version, f"The used SCBuilder-version {actual_scbuilder_version} is older than the version {required_scbuilder_version} required by '{self.tfcps_tools_general.get_product_name(self.repository)}' (defined in .ScriptCollection/OCIImages/ImageDefinition.csv). Please update to a newer SCBuilder-image.")
|
|
110
|
+
|
|
111
|
+
if self.is_pre_merge():
|
|
112
|
+
GeneralUtilities.assert_condition(not self.__assert_no_new_changes,f"A pre-merge build can not be done with the assert-no-new-changes-option.")
|
|
113
|
+
|
|
114
|
+
self.__update_openspec()
|
|
115
|
+
|
|
116
|
+
self.sc.git_set_local_configuration_value(self.repository, "core.autocrlf", "false")
|
|
117
|
+
|
|
118
|
+
#ensure the artifacts-folder of the repository is git-ignored so build-results never show up as uncommitted changes
|
|
119
|
+
self.sc.ensure_line_is_in_gitignore(self.repository, "Other/Resources/Artifacts")
|
|
120
|
+
|
|
121
|
+
#ensure <repo>/.ScriptCollection/.gitignore is set up (ignores the cache-folder so cache-files never show up as uncommitted changes)
|
|
122
|
+
self.sc.ensure_scriptcollection_gitignore_is_setup(self.repository)
|
|
123
|
+
|
|
124
|
+
self.sc.log.log(f"Start building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(start_time,False)}. (Target environment-type: {self.target_environment_type})")
|
|
125
|
+
|
|
126
|
+
self.tfcps_tools_general.ensure_required_environment_variables_are_set(self.repository)
|
|
127
|
+
|
|
128
|
+
self.__run_custom_pre_codeunit_build_script()
|
|
129
|
+
|
|
130
|
+
try:
|
|
131
|
+
xmlschema.validate(product_information_file, "https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/raw/main/Conventions/RepositoryStructure/CommonProjectStructure/productinformation.xsd")
|
|
132
|
+
except Exception as exception:
|
|
133
|
+
self.sc.log.log_exception(f"'{product_information_file}' could not be validated against the XSD:", exception, LogLevel.Warning)
|
|
134
|
+
|
|
135
|
+
if self.__assert_no_new_changes:
|
|
136
|
+
self.sc.assert_no_uncommitted_changes(self.repository,"Can not build codeunit: There are uncommitted changes in the repository.")
|
|
137
|
+
|
|
138
|
+
#run prepare-script
|
|
139
|
+
self.run_prepare_script()
|
|
140
|
+
|
|
141
|
+
hook.run_after_preparation(self.repository)
|
|
142
|
+
|
|
143
|
+
#check if changelog exists
|
|
144
|
+
changelog_file=os.path.join(self.repository,"Other","Resources","Changelog",f"v{self.tfcps_tools_general.get_version_of_project(self.repository)}.md")
|
|
145
|
+
GeneralUtilities.assert_file_exists(changelog_file,f"Changelogfile \"{changelog_file}\" does not exist. Try to create it for example using \"sccreatechangelogentry -m ...\".")
|
|
146
|
+
|
|
147
|
+
#mark current version as supported
|
|
148
|
+
now = GeneralUtilities.get_now()
|
|
149
|
+
project_version:str=self.tfcps_tools_general.get_version_of_project(self.repository)
|
|
150
|
+
if not self.tfcps_tools_general.suport_information_exists(self.repository, project_version):
|
|
151
|
+
amount_of_years_for_support:int=1
|
|
152
|
+
support_time = timedelta(days=365*amount_of_years_for_support+30*3+1)
|
|
153
|
+
until = now + support_time
|
|
154
|
+
until_day = datetime(until.year, until.month, until.day, 0, 0, 0)
|
|
155
|
+
from_day = datetime(now.year, now.month, now.day, 0, 0, 0)
|
|
156
|
+
self.tfcps_tools_general.mark_current_version_as_supported(self.repository,project_version,from_day,until_day)
|
|
157
|
+
|
|
158
|
+
codeunits:list[str]=self.tfcps_tools_general.get_codeunits(self.repository)
|
|
159
|
+
GeneralUtilities.assert_condition(0<len(codeunits),f"No codeunits found in repository {self.repository}.")
|
|
160
|
+
self.sc.log.log("Codeunits will be built in the following order:")
|
|
161
|
+
for codeunit_name in codeunits:
|
|
162
|
+
self.sc.log.log(f" - {codeunit_name}")
|
|
163
|
+
for codeunit_name in codeunits:
|
|
164
|
+
tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(os.path.join(self.repository,codeunit_name),self.sc.log.loglevel,self.target_environment_type,self.additionalargumentsfile,self.use_cache(),self.is_pre_merge(),self.__fast_lane)
|
|
165
|
+
self.sc.log.log(GeneralUtilities.get_line())
|
|
166
|
+
tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()
|
|
167
|
+
hook.run_after_codeunit_was_built(tFCPS_CodeUnit_BuildCodeUnit)
|
|
168
|
+
self.sc.log.log(GeneralUtilities.get_line())
|
|
169
|
+
|
|
170
|
+
self.search_for_secrets()
|
|
171
|
+
self.tfcps_tools_general.generate_svg_files_from_plantuml_files_for_repository(self.repository, self.use_cache())
|
|
172
|
+
self.__normalize_line_endings_of_common_files()
|
|
173
|
+
|
|
174
|
+
self.sc.log.log(GeneralUtilities.get_line())
|
|
175
|
+
|
|
176
|
+
if self.is_pre_merge():
|
|
177
|
+
self.__translate()
|
|
178
|
+
self.__collect_metrics()
|
|
179
|
+
self.__generate_loc_diagram()
|
|
180
|
+
self.sc.log.log(GeneralUtilities.get_line())
|
|
181
|
+
|
|
182
|
+
except Exception:
|
|
183
|
+
error_occurred=True
|
|
184
|
+
raise
|
|
185
|
+
finally:
|
|
186
|
+
if error_occurred:
|
|
187
|
+
GeneralUtilities.ensure_file_does_not_exist(ready_to_merge_file)
|
|
188
|
+
else:
|
|
189
|
+
if self.is_pre_merge():
|
|
190
|
+
GeneralUtilities.ensure_file_does_not_exist(ready_to_merge_file)#ensure it does not exist because the flag is not supposed to be on the main branch
|
|
191
|
+
else:
|
|
192
|
+
if self.is_working_branch():
|
|
193
|
+
if self.add_ready_to_merge_flag():
|
|
194
|
+
GeneralUtilities.ensure_file_exists(ready_to_merge_file)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
if self.__assert_no_new_changes:
|
|
198
|
+
self.sc.assert_no_uncommitted_changes(self.repository,"There are new uncommitted changes in the repository.")
|
|
199
|
+
|
|
200
|
+
self.__create_artifacts_archive(codeunits)
|
|
201
|
+
|
|
202
|
+
end_time:datetime=GeneralUtilities.get_now()
|
|
203
|
+
duration=end_time-start_time
|
|
204
|
+
self.sc.log.log(f"Finished building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(end_time,False)}. (Duration: {GeneralUtilities.timedelta_to_simple_string(duration)})")
|
|
205
|
+
self.sc.log.log(GeneralUtilities.get_line())
|
|
206
|
+
|
|
207
|
+
@GeneralUtilities.check_arguments
|
|
208
|
+
def __create_artifacts_archive(self, codeunits:list[str]) -> str:
|
|
209
|
+
product_name:str=self.tfcps_tools_general.get_product_name(self.repository)
|
|
210
|
+
product_version:str=self.tfcps_tools_general.get_version_of_project(self.repository)
|
|
211
|
+
target_folder:str=os.path.join(self.repository,"Other","Resources","Artifacts")
|
|
212
|
+
target_file_zip:str=os.path.join(target_folder,f"{product_name}_Artifacts_v{product_version}.zip")
|
|
213
|
+
target_file_infotxt:str=os.path.join(target_folder,f"{product_name}_Artifacts_v{product_version}.information.txt")
|
|
214
|
+
|
|
215
|
+
artifacts_folders:dict[str,str]={}
|
|
216
|
+
for codeunit_name in codeunits:
|
|
217
|
+
artifacts_folder = os.path.join(self.repository, codeunit_name, "Other", "Artifacts")
|
|
218
|
+
GeneralUtilities.assert_folder_exists(artifacts_folder,f"The artifacts-folder '{artifacts_folder}' of codeunit '{codeunit_name}' does not exist.")
|
|
219
|
+
artifacts_folders[codeunit_name]=artifacts_folder
|
|
220
|
+
|
|
221
|
+
GeneralUtilities.ensure_directory_exists(target_folder)
|
|
222
|
+
GeneralUtilities.ensure_file_does_not_exist(target_file_zip)#ensure the archive of a previous build gets replaced
|
|
223
|
+
self.sc.create_zip_archive_of_folders(artifacts_folders,target_file_zip)
|
|
224
|
+
|
|
225
|
+
hash_of_zip_file:str=GeneralUtilities.get_sha256_of_file(target_file_zip)
|
|
226
|
+
creation_timestamp:datetime=GeneralUtilities.get_now()
|
|
227
|
+
GeneralUtilities.ensure_file_does_not_exist(target_file_infotxt)
|
|
228
|
+
GeneralUtilities.ensure_file_exists(target_file_infotxt)
|
|
229
|
+
information_lines:list[str]=[
|
|
230
|
+
f"Product: {product_name}",
|
|
231
|
+
f"Version: {product_version}",
|
|
232
|
+
f"Artifacts-archive-hash: {hash_of_zip_file}",
|
|
233
|
+
f"Creation-timestamp: {GeneralUtilities.datetime_to_string_for_readable_entry(creation_timestamp,False)}",
|
|
234
|
+
]
|
|
235
|
+
GeneralUtilities.write_lines_to_file(target_file_infotxt,information_lines)
|
|
236
|
+
normalized_relative_path=GeneralUtilities.normalize_path(os.path.relpath(target_file_zip, self.repository).replace("\\","/"))
|
|
237
|
+
self.sc.log.log(f"Created artifacts-archive \"{normalized_relative_path}\":")
|
|
238
|
+
for information_line in information_lines:
|
|
239
|
+
self.sc.log.log(f" {information_line}")
|
|
240
|
+
|
|
241
|
+
@GeneralUtilities.check_arguments
|
|
242
|
+
def __normalize_line_endings_of_common_files(self) -> None:
|
|
243
|
+
self.sc.format_xml_file(os.path.join(self.repository, ".ScriptCollection", "ProductInformation.xml"))
|
|
244
|
+
|
|
245
|
+
for codeunit in self.tfcps_tools_general.get_codeunits(self.repository):
|
|
246
|
+
self.sc.format_xml_file(os.path.join(self.repository, codeunit, f"{codeunit}.codeunit.xml"))
|
|
247
|
+
|
|
248
|
+
workspace_file = os.path.join(self.repository, f"{self.tfcps_tools_general.get_product_name(self.repository)}.code-workspace")
|
|
249
|
+
if os.path.isfile(workspace_file):
|
|
250
|
+
self.sc.format_json_file(workspace_file)
|
|
251
|
+
|
|
252
|
+
#TODO add option to define exceptions (means: files which should not be normalized).
|
|
253
|
+
self.sc.normalize_line_endings_of_files_in_folder(self.repository, ["txt", "md", "py", "json", "xml", "svg", "csv", "yml", "yaml", "toml","gitignore", "gitattributes", "code-workspace"])
|
|
254
|
+
|
|
255
|
+
@GeneralUtilities.check_arguments
|
|
256
|
+
def is_working_branch(self)->bool:
|
|
257
|
+
if self.sc.git_repository_has_uncommitted_changes(self.repository):
|
|
258
|
+
return True
|
|
259
|
+
if self.sc.git_get_current_branch_name(self.repository) != "main":
|
|
260
|
+
return True
|
|
261
|
+
return False
|
|
262
|
+
|
|
263
|
+
@GeneralUtilities.check_arguments
|
|
264
|
+
def __get_build_arguments(self, repository: str) -> list[str]:
|
|
265
|
+
"""Returns the arguments which describe the current build. They are passed to every script which is called by this build
|
|
266
|
+
(the prepare-script of the repository as well as the optional user-specific custom scripts), so such a script can behave
|
|
267
|
+
differently depending on the repository and the target-environment-type it is called for."""
|
|
268
|
+
result: list[str] = ["--repository", repository, "--targetenvironmenttype", self.target_environment_type, "--verbosity", str(int(self.sc.log.loglevel))]
|
|
269
|
+
if GeneralUtilities.string_has_content(self.additionalargumentsfile):
|
|
270
|
+
result = result+["--additionalargumentsfile", self.additionalargumentsfile]
|
|
271
|
+
if not self.__use_cache:
|
|
272
|
+
if self.sc.git_repository_has_uncommitted_changes(repository):
|
|
273
|
+
self.sc.log.log("No-cache-option can not be applied because there are uncommited changes in the repository.", LogLevel.Warning)
|
|
274
|
+
else:
|
|
275
|
+
result = result+["--nocache"]
|
|
276
|
+
return result
|
|
277
|
+
|
|
278
|
+
@GeneralUtilities.check_arguments
|
|
279
|
+
def __run_custom_pre_codeunit_build_script(self) -> None:
|
|
280
|
+
"""Runs the optional user-specific script which prepares this machine for a codeunit-build. The script is located in
|
|
281
|
+
'~/.ScriptCollection/TFCPS', so it is outside of any repository and is not part of a product's sourcecode.
|
|
282
|
+
Which script is used depends on where the build runs, because both cases need different preparation-steps:
|
|
283
|
+
- build directly on the host ('scbuildcodeunits'): 'CustomPreCodeUnitBuildScript.py'
|
|
284
|
+
- build inside the container ('scbuildcodeunitsc' or a build-pipeline): 'CustomPreCodeUnitBuildScriptInContainer.py', located in
|
|
285
|
+
the folder returned by TFCPS_Tools_General.get_custom_scripts_folder_for_container(). It is searched in the folder into which
|
|
286
|
+
'scbuildcodeunitsc' mounts that whole folder and - if it is not there - directly in the configuration-folder, because a
|
|
287
|
+
build-runner usually gets the whole configuration-folder mounted instead (see the article 'Build-runner-configuration'). The
|
|
288
|
+
script which the host itself runs before it starts the container is 'CustomPreCodeUnitBuildScriptForContainer.py' and is
|
|
289
|
+
therefore run there and not here."""
|
|
290
|
+
if self.sc.is_runnning_in_container():
|
|
291
|
+
script_file: str = os.path.join(self.tfcps_tools_general.get_folder_of_custom_scripts_in_container(), "CustomPreCodeUnitBuildScriptInContainer.py")
|
|
292
|
+
if not os.path.isfile(script_file):
|
|
293
|
+
script_file = os.path.join(self.tfcps_tools_general.get_custom_scripts_folder_for_container(), "CustomPreCodeUnitBuildScriptInContainer.py")
|
|
294
|
+
else:
|
|
295
|
+
script_file: str = self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScript.py")
|
|
296
|
+
self.tfcps_tools_general.run_custom_script_if_available(script_file, self.__get_build_arguments(self.repository))
|
|
297
|
+
|
|
298
|
+
@GeneralUtilities.check_arguments
|
|
299
|
+
def run_prepare_script(self):
|
|
300
|
+
args = self.__get_build_arguments(self.repository)
|
|
301
|
+
if os.path.isfile( os.path.join(self.repository,"Other","Scripts","PrepareBuildCodeunits.py")):
|
|
302
|
+
self.sc.log.log("Prepare build codeunits...")
|
|
303
|
+
self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["PrepareBuildCodeunits.py"]+args, os.path.join(self.repository,"Other","Scripts"),print_live_output=True)
|
|
304
|
+
|
|
305
|
+
@GeneralUtilities.check_arguments
|
|
306
|
+
def __update_openspec(self) -> None:
|
|
307
|
+
openspec_configuration_file: str = os.path.join(self.repository, "openspec", "config.yaml")
|
|
308
|
+
if os.path.isfile(openspec_configuration_file):
|
|
309
|
+
self.sc.log.log("Update openspec-instruction-files...")
|
|
310
|
+
self.sc.run_program_argsasarray("openspec", ["update", "--force"], self.repository)
|
|
311
|
+
|
|
312
|
+
@GeneralUtilities.check_arguments
|
|
313
|
+
def build_codeunits_in_container(self,base_mount_folder:str) -> tuple[bool, str]:
|
|
314
|
+
#base_mount_folder is assumed to be an absolute path set correctly by the caller (see BuildCodeUnitsC in Executables.py, which defaults it to the repository itself).
|
|
315
|
+
#it may be the repository itself or any parent-folder of it, which allows the caller to mount not only the repository but the whole surrounding folder-structure into the container.
|
|
316
|
+
normalized_base_mount_folder = os.path.normpath(base_mount_folder)
|
|
317
|
+
normalized_repository = os.path.normpath(self.repository)
|
|
318
|
+
relative_repository_path = os.path.relpath(normalized_repository, normalized_base_mount_folder).replace(os.sep, "/")
|
|
319
|
+
GeneralUtilities.assert_condition(relative_repository_path == "." or not relative_repository_path.startswith(".."), f"The repository '{self.repository}' is not located inside the base-mount-folder '{base_mount_folder}'.")
|
|
320
|
+
container_base_mount_folder = f"/Workspace/Project/{os.path.basename(normalized_base_mount_folder)}"
|
|
321
|
+
container_repository_folder = container_base_mount_folder if relative_repository_path == "." else f"{container_base_mount_folder}/{relative_repository_path}"
|
|
322
|
+
image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "SCBuilder")
|
|
323
|
+
|
|
324
|
+
#build the scbuildcodeunits-arguments based on the current state (analogous to the arguments accepted by the scbuildcodeunits-executable). each token must be a separate argument because run_program_argsasarray passes every list-element verbatim and does not split on spaces.
|
|
325
|
+
scbuildcodeunits_arguments = ["scbuildcodeunits", "-r", container_repository_folder, "-v", "4"]
|
|
326
|
+
if not self.__use_cache:
|
|
327
|
+
scbuildcodeunits_arguments.append("-c")
|
|
328
|
+
if self.__is_pre_merge:
|
|
329
|
+
scbuildcodeunits_arguments.append("-p")
|
|
330
|
+
if self.__assert_no_new_changes:
|
|
331
|
+
scbuildcodeunits_arguments.append("-u")
|
|
332
|
+
if self.__add_ready_to_merge_flag:
|
|
333
|
+
scbuildcodeunits_arguments.append("-m")
|
|
334
|
+
if self.__fast_lane:
|
|
335
|
+
scbuildcodeunits_arguments.append("-f")
|
|
336
|
+
if GeneralUtilities.string_has_content(self.additionalargumentsfile):
|
|
337
|
+
scbuildcodeunits_arguments += ["-a", self.__translate_path_into_container(self.additionalargumentsfile, container_repository_folder)]
|
|
338
|
+
|
|
339
|
+
update_scriptcollection=True
|
|
340
|
+
update_argument:str=""
|
|
341
|
+
if update_scriptcollection:
|
|
342
|
+
update_argument="pip3 install scriptcollection --upgrade && "
|
|
343
|
+
scbuildcodeunits_arguments=["bash","-c", f"{update_argument}scshowversion && "+" ".join(scbuildcodeunits_arguments)]
|
|
344
|
+
|
|
345
|
+
#run the optional user-specific script which prepares this host for a container-build (for example to log in to the registry the image is pulled from).
|
|
346
|
+
#it runs before the environment-variables are resolved, so it can also create the files their values are read from.
|
|
347
|
+
self.tfcps_tools_general.run_custom_script_if_available(self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScriptForContainer.py"), self.__get_build_arguments(self.repository))
|
|
348
|
+
|
|
349
|
+
#mount the optional user-specific scripts which prepare the container itself. the whole folder (not only the pre-codeunit-build-hook
|
|
350
|
+
#itself) is mounted, so the hook can start a sibling script placed next to it (for example to keep the hook independent of a
|
|
351
|
+
#package it only installs, while the actual preparation-logic which needs that package lives in the sibling script; see
|
|
352
|
+
#get_custom_scripts_folder_for_container). The mount is writable (not read-only) because the scripts in that folder also use it as
|
|
353
|
+
#their own download-cache-folder. The build inside the container runs the hook (see __run_custom_pre_codeunit_build_script); if the
|
|
354
|
+
#folder does not exist on the host nothing is mounted and nothing is run.
|
|
355
|
+
mount_arguments: list[str] = []
|
|
356
|
+
custom_scripts_folder_for_inside_the_container: str = self.tfcps_tools_general.get_custom_scripts_folder_for_container()
|
|
357
|
+
if os.path.isdir(custom_scripts_folder_for_inside_the_container):
|
|
358
|
+
mount_arguments += ["-v", f"{custom_scripts_folder_for_inside_the_container}:{self.tfcps_tools_general.get_folder_of_custom_scripts_in_container()}"]
|
|
359
|
+
|
|
360
|
+
#pass the environment-variables which are declared as required in <repository>/.ScriptCollection/ProductInformation.xml into the container
|
|
361
|
+
#so they do not have to be specified explicitly on every scbuildcodeunits-call. Their values are resolved from the user-specific
|
|
362
|
+
#configuration-file (see TFCPS_Tools_General.get_environment_variables_configuration_file), which only exists on the host.
|
|
363
|
+
#only the names are passed as arguments; the values are given to the docker-client through its own environment, because arguments
|
|
364
|
+
#are written to the log and are visible in the process-list of this host, which must not happen for a value which is typically a secret.
|
|
365
|
+
required_environment_variables: dict[str, str] = self.tfcps_tools_general.get_required_environment_variables(self.repository)
|
|
366
|
+
env_arguments: list[str] = []
|
|
367
|
+
for env_variable_name in required_environment_variables:
|
|
368
|
+
env_arguments += ["-e", env_variable_name]
|
|
369
|
+
|
|
370
|
+
#run scbuildcodeunits inside the SCBuilder-image. base_mount_folder is mounted into the container (covering the repository and, for submodules, its real gitdir) and the docker-socket is forwarded because codeunit-builds often start containers (for example local test-services).
|
|
371
|
+
docker_arguments = [
|
|
372
|
+
"run", "--rm",
|
|
373
|
+
"-v", f"{base_mount_folder}:{container_base_mount_folder}",
|
|
374
|
+
"-v", "/var/run/docker.sock:/var/run/docker.sock",
|
|
375
|
+
"-w", container_repository_folder,
|
|
376
|
+
] + mount_arguments + env_arguments + [
|
|
377
|
+
image,
|
|
378
|
+
] + scbuildcodeunits_arguments
|
|
379
|
+
self.sc.log.log(f"Build codeunits in container using image \"{image}\"...")
|
|
380
|
+
image_address, image_tag = ScriptCollectionCore.split_image_address_and_tag(image)
|
|
381
|
+
self.sc.docker_pull(image_address, image_tag)
|
|
382
|
+
# the exitcode is evaluated by the caller (returned as part of the result-tuple), so the program-runner must not raise on a non-zero exitcode here.
|
|
383
|
+
result=self.sc.run_program_argsasarray("docker", docker_arguments, throw_exception_if_exitcode_is_not_zero=False, print_live_output=True, env_vars=required_environment_variables)
|
|
384
|
+
exit_code:int=result[0]
|
|
385
|
+
stdout:str=result[1] or GeneralUtilities.empty_string
|
|
386
|
+
stderr:str=result[2] or GeneralUtilities.empty_string
|
|
387
|
+
return (exit_code==0,f"{stdout}\n{stderr}")
|
|
388
|
+
|
|
389
|
+
@GeneralUtilities.check_arguments
|
|
390
|
+
def __translate_path_into_container(self, host_path: str, container_repository_folder: str) -> str:
|
|
391
|
+
normalized_repository = os.path.normpath(self.repository)
|
|
392
|
+
normalized_path = os.path.normpath(host_path)
|
|
393
|
+
if normalized_path.startswith(normalized_repository):
|
|
394
|
+
relative_path = os.path.relpath(normalized_path, normalized_repository).replace(os.sep, "/")
|
|
395
|
+
return f"{container_repository_folder}/{relative_path}"
|
|
396
|
+
return host_path
|
|
397
|
+
|
|
398
|
+
@GeneralUtilities.check_arguments
|
|
399
|
+
def __translate(self) -> None:
|
|
400
|
+
for taskfile_name in ("Taskfile.yml", "Taskfile.yaml"):
|
|
401
|
+
taskfile = os.path.join(self.repository, taskfile_name)
|
|
402
|
+
if os.path.isfile(taskfile):
|
|
403
|
+
with open(taskfile, "r", encoding="utf-8") as f:
|
|
404
|
+
taskfile_content = yaml.safe_load(f)
|
|
405
|
+
if isinstance(taskfile_content.get("tasks"), dict) and "Translate" in taskfile_content["tasks"]:
|
|
406
|
+
self.sc.run_program("task", "Translate", self.repository, print_live_output=self.sc.log.loglevel == LogLevel.Debug)
|
|
407
|
+
break
|
|
408
|
+
|
|
409
|
+
@GeneralUtilities.check_arguments
|
|
410
|
+
def __collect_metrics(self) -> None:
|
|
411
|
+
project_version: str=self.tfcps_tools_general.get_version_of_project(self.repository)
|
|
412
|
+
self.sc.log.log("Collect metrics...")
|
|
413
|
+
loc = self.sc.get_lines_of_code_with_default_excluded_patterns(self.repository)
|
|
414
|
+
loc_metric_folder = os.path.join(self.repository, "Other", "Metrics")
|
|
415
|
+
GeneralUtilities.ensure_directory_exists(loc_metric_folder)
|
|
416
|
+
loc_metric_file = os.path.join(loc_metric_folder, "RepositoryStatisticsPerCommit.csv")
|
|
417
|
+
GeneralUtilities.ensure_file_exists(loc_metric_file)
|
|
418
|
+
|
|
419
|
+
#remove legacy metrics-file. the following 2 lines should be removed after 2026-12-31
|
|
420
|
+
legacy_metrics_file = os.path.join(loc_metric_folder, "LinesOfCode.csv")
|
|
421
|
+
GeneralUtilities.ensure_file_does_not_exist(legacy_metrics_file)
|
|
422
|
+
|
|
423
|
+
old_lines = GeneralUtilities.read_nonempty_lines_from_file(loc_metric_file)
|
|
424
|
+
header_line="Version;Timestamp;LinesOfCode"
|
|
425
|
+
new_lines = [header_line]
|
|
426
|
+
current_version_string=f"v{project_version}"
|
|
427
|
+
for old_line in old_lines:
|
|
428
|
+
if not old_line.startswith(current_version_string+";") and old_line!=header_line:
|
|
429
|
+
new_lines.append(old_line)
|
|
430
|
+
c_date:datetime=GeneralUtilities.get_now().astimezone(timezone.utc)
|
|
431
|
+
commit_date=GeneralUtilities.datetime_to_string_for_logfile_entry(c_date,False)
|
|
432
|
+
new_lines.append(f"{current_version_string};{commit_date};{loc}")
|
|
433
|
+
GeneralUtilities.write_lines_to_file(loc_metric_file, new_lines)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
@GeneralUtilities.check_arguments
|
|
437
|
+
def __generate_loc_diagram(self):
|
|
438
|
+
self.sc.log.log("Generate LoC-diagram...")
|
|
439
|
+
loc_metric_folder = os.path.join(self.repository, "Other", "Metrics")
|
|
440
|
+
GeneralUtilities.ensure_directory_exists(loc_metric_folder)
|
|
441
|
+
loc_metric_file = os.path.join(loc_metric_folder, "RepositoryStatisticsPerCommit.csv")
|
|
442
|
+
GeneralUtilities.ensure_file_exists(loc_metric_file)
|
|
443
|
+
|
|
444
|
+
filenamebase="LoC-Diagram"
|
|
445
|
+
|
|
446
|
+
diagram_definition_folder=os.path.join(self.repository, "Other", "Reference","Technical","Diagrams")
|
|
447
|
+
GeneralUtilities.ensure_directory_exists(diagram_definition_folder)
|
|
448
|
+
|
|
449
|
+
diagram_definition_file=os.path.join(diagram_definition_folder,f"{filenamebase}.json")
|
|
450
|
+
GeneralUtilities.ensure_file_exists(diagram_definition_file)
|
|
451
|
+
GeneralUtilities.write_text_to_file(diagram_definition_file,GeneralUtilities.empty_string)
|
|
452
|
+
|
|
453
|
+
loc_data_file=os.path.join(diagram_definition_folder,f"{filenamebase}.csv")
|
|
454
|
+
GeneralUtilities.ensure_file_exists(loc_data_file)
|
|
455
|
+
csv_lines=[]
|
|
456
|
+
for line in GeneralUtilities.read_lines_from_file(loc_metric_file):
|
|
457
|
+
if GeneralUtilities.string_has_content(line):
|
|
458
|
+
splitted=line.split(";")
|
|
459
|
+
v=splitted[0]
|
|
460
|
+
t=splitted[1]
|
|
461
|
+
loc=splitted[2]
|
|
462
|
+
csv_lines.append(f"{v},{t},{loc}")
|
|
463
|
+
GeneralUtilities.write_lines_to_file(loc_data_file,csv_lines)
|
|
464
|
+
self.sc.normalize_line_endings(loc_data_file) # ensure the generated LoC-diagram-csv always uses LF line-endings
|
|
465
|
+
diagram_json = {
|
|
466
|
+
"$schema": "https://vega.github.io/schema/vega-lite/v5.json",
|
|
467
|
+
"description": "Lines of Code over time",
|
|
468
|
+
"width": 800,
|
|
469
|
+
"height": 400,
|
|
470
|
+
"data": {
|
|
471
|
+
"url": f"./{filenamebase}.csv",
|
|
472
|
+
"format": {
|
|
473
|
+
"type": "csv"
|
|
474
|
+
}
|
|
475
|
+
},
|
|
476
|
+
"mark": {
|
|
477
|
+
"type": "line",
|
|
478
|
+
"point": True
|
|
479
|
+
},
|
|
480
|
+
"encoding": {
|
|
481
|
+
"x": {
|
|
482
|
+
"field": "Timestamp",
|
|
483
|
+
"type": "temporal",
|
|
484
|
+
"title": "Date",
|
|
485
|
+
"scale": {
|
|
486
|
+
"type": "utc"#render the ticks in utc so the diagram looks the same independent of the timezone of the machine which builds it
|
|
487
|
+
},
|
|
488
|
+
"axis": {
|
|
489
|
+
"format": "%Y-%m-%d %H:%M:%S (UTC)",#everything which is not a format-specifier starting with a percent-sign is taken literally
|
|
490
|
+
"labelAngle": -45,#the timestamps are too long to be displayed horizontally next to each other
|
|
491
|
+
"labelLimit": 250,#without this the labels would be truncated with an ellipsis because they are longer than the default-limit of 180 pixels
|
|
492
|
+
"tickCount": 8
|
|
493
|
+
}
|
|
494
|
+
},
|
|
495
|
+
"y": {
|
|
496
|
+
"field": "LinesOfCode",
|
|
497
|
+
"type": "quantitative",
|
|
498
|
+
"title": "Lines of Code"
|
|
499
|
+
},
|
|
500
|
+
#the timestamp and the lines of code are not listed here although they are part of the tooltip: the fields which are encoded
|
|
501
|
+
#as x respectively as y are always part of the generated tooltip anyway, so listing them again would show them twice.
|
|
502
|
+
"tooltip": [
|
|
503
|
+
{
|
|
504
|
+
"field": "Version",
|
|
505
|
+
"type": "ordinal"
|
|
506
|
+
}
|
|
507
|
+
]
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
with open(diagram_definition_file, "w", encoding="utf-8") as f:
|
|
512
|
+
json.dump(
|
|
513
|
+
diagram_json,
|
|
514
|
+
f,
|
|
515
|
+
indent=2,
|
|
516
|
+
sort_keys=False,
|
|
517
|
+
ensure_ascii=False
|
|
518
|
+
)
|
|
519
|
+
diagram_svg_file=os.path.join(self.repository,"Other","Reference","Technical","Diagrams",f"{filenamebase}.svg")
|
|
520
|
+
GeneralUtilities.ensure_file_exists(diagram_svg_file)
|
|
521
|
+
GeneralUtilities.assert_condition(not self.sc.file_is_git_ignored(f"Other/Reference/Technical/Diagrams/{filenamebase}.svg",self.repository),f"Other/Reference/Technical/Diagrams/{filenamebase}.svg must not be git-ignored")#because it should be referencable in markdown-files and viewable without building the codeunits.
|
|
522
|
+
self.sc.generate_chart_diagram(diagram_definition_file,os.path.basename(diagram_svg_file))
|
|
523
|
+
self.sc.add_tooltips_to_chart_diagram(diagram_svg_file)#so the exact values of a data-point are visible when the svg-file is opened directly in a browser
|
|
524
|
+
self.sc.format_xml_file(diagram_svg_file)
|
|
525
|
+
|
|
526
|
+
@GeneralUtilities.check_arguments
|
|
527
|
+
def search_for_secrets(self) -> None:
|
|
528
|
+
self.sc.log.log("Search for secrets...")
|
|
529
|
+
self.__search_for_secrets_in_repository()
|
|
530
|
+
self.__search_for_secrets_in_oci_images()
|
|
531
|
+
|
|
532
|
+
@GeneralUtilities.check_arguments
|
|
533
|
+
def __search_for_secrets_in_oci_images(self) -> None:
|
|
534
|
+
#the repository-scan above can not find secrets which are only contained in a built image (build-arguments recorded in the
|
|
535
|
+
#image-history, environment-variables of the image and files which are generated during the image-build), and betterleaks
|
|
536
|
+
#does not look into the "*.tar"-files the image-builds produce. Since these images get published, they are scanned separately.
|
|
537
|
+
scan: TFCPS_OCIImageSecretScan = TFCPS_OCIImageSecretScan(self.sc)
|
|
538
|
+
findings: list[str] = []
|
|
539
|
+
for image in self.__get_oci_images_of_repository():
|
|
540
|
+
findings = findings+scan.search_for_secrets_in_image(image, self.repository)
|
|
541
|
+
if 0 < len(findings):
|
|
542
|
+
for finding in findings:
|
|
543
|
+
self.sc.log.log(finding, LogLevel.Error)
|
|
544
|
+
raise ValueError(f"Found {len(findings)} secret-finding(s) in the built OCI-image(s). A secret which is part of an image is readable by everybody who is allowed to pull that image. See {os.path.join(self.repository, '.betterleaks.toml')} to ignore known false positives.")
|
|
545
|
+
|
|
546
|
+
@GeneralUtilities.check_arguments
|
|
547
|
+
def __get_oci_images_of_repository(self) -> list[str]:
|
|
548
|
+
"""Returns the references of the images which were built by the codeunits of this repository. A codeunit builds an image
|
|
549
|
+
exactly if it has an OCI-image-artifacts-folder; the built image is loaded into the local docker-instance by the
|
|
550
|
+
codeunit-build, tagged with the lowercase codeunit-name and the codeunit-version."""
|
|
551
|
+
result: list[str] = []
|
|
552
|
+
for codeunit_name in self.tfcps_tools_general.get_codeunits(self.repository):
|
|
553
|
+
codeunit_folder: str = os.path.join(self.repository, codeunit_name)
|
|
554
|
+
if not os.path.isdir(os.path.join(codeunit_folder, "Other", "Artifacts", "BuildResult_OCIImage")):
|
|
555
|
+
continue
|
|
556
|
+
codeunit_version: str = self.tfcps_tools_general.get_version_of_codeunit(os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml"))
|
|
557
|
+
result.append(f"{codeunit_name}:{codeunit_version}".lower())
|
|
558
|
+
return result
|
|
559
|
+
|
|
560
|
+
@GeneralUtilities.check_arguments
|
|
561
|
+
def __search_for_secrets_in_repository(self) -> None:
|
|
562
|
+
image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "Betterleaks")
|
|
563
|
+
|
|
564
|
+
config_file = os.path.join(self.repository, ".betterleaks.toml")
|
|
565
|
+
#the filesystem-marker is checked in addition to the convention-based environment-variable (which the rest of this class uses as well),
|
|
566
|
+
#because a wrong result here does not only change a message but makes the scan analyse the wrong folder.
|
|
567
|
+
running_in_container = os.path.exists("/.dockerenv") or self.sc.is_runnning_in_container()
|
|
568
|
+
if running_in_container:
|
|
569
|
+
# We run inside the build-container with the docker-socket forwarded to the host-daemon.
|
|
570
|
+
# A bind-mount of our in-container repository-path (e.g. "/__w/<repo>/<repo>" on a
|
|
571
|
+
# GitHub-runner or "/Workspace/Repository" when the pipeline is run locally) would be
|
|
572
|
+
# resolved by the host-daemon, where that path does not exist or points to unrelated
|
|
573
|
+
# data (for example test-service-volumes written there by other sibling-containers).
|
|
574
|
+
# The sibling betterleaks-container would then scan the wrong directory - without the
|
|
575
|
+
# repository-content and without ".betterleaks.toml", which causes false positives.
|
|
576
|
+
# Sharing our own volumes instead exposes the repository to betterleaks at the same path.
|
|
577
|
+
mount_arguments = ["--volumes-from", self.__get_own_container_id()]
|
|
578
|
+
repository_in_scan_container = self.repository
|
|
579
|
+
else:
|
|
580
|
+
# Running directly on the host: a normal bind-mount works because the path is resolved
|
|
581
|
+
# on the same filesystem the docker-daemon uses.
|
|
582
|
+
mount_arguments = ["-v", f"{self.repository}:/repo"]
|
|
583
|
+
repository_in_scan_container = "/repo"
|
|
584
|
+
scan_args = ["dir", repository_in_scan_container, "-v"]
|
|
585
|
+
if os.path.isfile(config_file):
|
|
586
|
+
# Pass the config explicitly instead of relying on auto-detection, because betterleaks
|
|
587
|
+
# silently falls back to the default ruleset when it does not find the config at the
|
|
588
|
+
# scan-root, which results in false positives.
|
|
589
|
+
scan_args = scan_args + ["-c", f"{repository_in_scan_container}/.betterleaks.toml"]
|
|
590
|
+
else:
|
|
591
|
+
self.sc.log.log(f"No betterleaks-config found at '{config_file}'; scanning with default ruleset only.", LogLevel.Warning)
|
|
592
|
+
# Verify that the docker-daemon is reachable before running the scan. The scan runs betterleaks via "docker run",
|
|
593
|
+
# and "docker run" returns exit-code 1 both when the daemon is unreachable and when betterleaks actually finds
|
|
594
|
+
# secrets. Without this pre-check an unreachable daemon (e.g. the socket is not forwarded into a build-container)
|
|
595
|
+
# would be misreported below as "found secrets", which is a misleading infrastructure-error. "docker version
|
|
596
|
+
# --format {{.Server.Version}}" queries the daemon (the server-part) and exits non-zero exactly when it can not
|
|
597
|
+
# be reached, so it distinguishes "docker not available" from a real scan-result.
|
|
598
|
+
daemon_check = self.sc.run_program_argsasarray("docker", ["version", "--format", "{{.Server.Version}}"], throw_exception_if_exitcode_is_not_zero=False, print_live_output=False)
|
|
599
|
+
if daemon_check[0] != 0:
|
|
600
|
+
raise ValueError(f"The secret-scan can not be performed because the docker-daemon is not reachable (exit-code {daemon_check[0]}: {daemon_check[2].strip()}). The scan runs betterleaks via 'docker run', which requires a running docker-daemon (inside a build-container its socket must be forwarded to the host-daemon). This is an infrastructure-problem, not a secret-finding in the repository.")
|
|
601
|
+
args = ["run", "--rm"] + mount_arguments + [image] + scan_args
|
|
602
|
+
image_address, image_tag = ScriptCollectionCore.split_image_address_and_tag(image)
|
|
603
|
+
self.sc.docker_pull(image_address, image_tag)
|
|
604
|
+
result = self.sc.run_program_argsasarray("docker", args, throw_exception_if_exitcode_is_not_zero=False, print_live_output=self.sc.log.loglevel==False, print_errors_as_information=True)
|
|
605
|
+
if result[0] != 0:
|
|
606
|
+
for line in GeneralUtilities.string_to_lines(result[1]):
|
|
607
|
+
self.sc.log.log(line, LogLevel.Information)
|
|
608
|
+
for line in GeneralUtilities.string_to_lines(result[2]):
|
|
609
|
+
self.sc.log.log(line, LogLevel.Error)
|
|
610
|
+
raise ValueError(f"Found unignored secret findings (exit code {result[0]}). See {os.path.join(self.repository, '.betterleaks.toml')} to ignore known false positives.")
|
|
611
|
+
|
|
612
|
+
@GeneralUtilities.check_arguments
|
|
613
|
+
def __get_own_container_id(self) -> str:
|
|
614
|
+
# Determine the id of the container this process runs in so its volumes can be shared with
|
|
615
|
+
# sibling-containers via "docker run --volumes-from".
|
|
616
|
+
# In mountinfo the own container-id only appears reliably in the source-path of the
|
|
617
|
+
# "/etc/hostname"/"/etc/hosts"/"/etc/resolv.conf"-mounts (".../containers/<id>/..."). A plain
|
|
618
|
+
# 64-hex-match there would also hit overlay-layer-hashes (which are not containers), so the
|
|
619
|
+
# "containers/"-prefix must be matched explicitly.
|
|
620
|
+
try:
|
|
621
|
+
with open("/proc/self/mountinfo", "r", encoding="utf-8") as file_handle:
|
|
622
|
+
match = re.search(r"/containers/([0-9a-f]{64})/", file_handle.read())
|
|
623
|
+
if match is not None:
|
|
624
|
+
return match.group(1)
|
|
625
|
+
except Exception:
|
|
626
|
+
pass
|
|
627
|
+
# cgroup (v1): the container-id is part of the cgroup-path; here a plain 64-hex-match is safe.
|
|
628
|
+
try:
|
|
629
|
+
with open("/proc/self/cgroup", "r", encoding="utf-8") as file_handle:
|
|
630
|
+
match = re.search(r"[0-9a-f]{64}", file_handle.read())
|
|
631
|
+
if match is not None:
|
|
632
|
+
return match.group(0)
|
|
633
|
+
except Exception:
|
|
634
|
+
pass
|
|
635
|
+
# Fallback: the hostname equals the short container-id for containers started without an
|
|
636
|
+
# explicit hostname.
|
|
637
|
+
return socket.gethostname()
|
|
638
|
+
|
|
639
|
+
@GeneralUtilities.check_arguments
|
|
640
|
+
def use_cache(self) -> bool:
|
|
641
|
+
return self.__use_cache
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
@GeneralUtilities.check_arguments
|
|
645
|
+
def is_pre_merge(self) -> bool:
|
|
646
|
+
return self.__is_pre_merge
|
|
647
|
+
|
|
648
|
+
@GeneralUtilities.check_arguments
|
|
649
|
+
def add_ready_to_merge_flag(self) -> bool:
|
|
650
|
+
if self.__add_ready_to_merge_flag:
|
|
651
|
+
return True
|
|
652
|
+
if self.sc.git_get_current_branch_name(self.repository)=="other/maintenance":
|
|
653
|
+
return True
|
|
654
|
+
return False
|
|
655
|
+
|
|
656
|
+
@GeneralUtilities.check_arguments
|
|
657
|
+
def update_dependencies(self) -> None:
|
|
658
|
+
repository=self.repository
|
|
659
|
+
self.sc.log.log("Update dependencies for product...")
|
|
660
|
+
self.update_year_in_license_file()
|
|
661
|
+
self.sc.assert_is_git_repository(repository)
|
|
662
|
+
self.sc.assert_no_uncommitted_changes(repository)
|
|
663
|
+
#the update is done while the codeunits are built regularly (see TFCPS_UpdateDependenciesHook): a codeunit is built before its
|
|
664
|
+
#dependencies are updated (some programming-languages need that) and again afterwards if the update changed something. Using the
|
|
665
|
+
#regular build means the update also gets all preparation-steps of a build - especially the package-sources which are required to
|
|
666
|
+
#resolve dependencies which are not available on a public package-source.
|
|
667
|
+
self.build_codeunits(TFCPS_UpdateDependenciesHook(self.sc, self.tfcps_tools_general))
|
|
668
|
+
if self.sc.git_repository_has_uncommitted_changes(repository):
|
|
669
|
+
changelog_folder = os.path.join(repository, "Other", "Resources", "Changelog")
|
|
670
|
+
project_version:str=self.tfcps_tools_general.get_version_of_project(repository)
|
|
671
|
+
changelog_file = os.path.join(changelog_folder, f"v{project_version}.md")
|
|
672
|
+
if not os.path.isfile(changelog_file):
|
|
673
|
+
self.__ensure_changelog_file_is_added(repository, project_version)
|
|
674
|
+
self.build_codeunits()#check the codeunits are buildable at all with all updates together
|
|
675
|
+
self.sc.git_commit(repository, "Updated dependencies", stage_all_changes=True)
|
|
676
|
+
|
|
677
|
+
@GeneralUtilities.check_arguments
|
|
678
|
+
def __ensure_changelog_file_is_added(self, repository_folder: str, version_of_project: str):
|
|
679
|
+
changelog_file = os.path.join(repository_folder, "Other", "Resources", "Changelog", f"v{version_of_project}.md")
|
|
680
|
+
if not os.path.isfile(changelog_file):
|
|
681
|
+
GeneralUtilities.ensure_file_exists(changelog_file)
|
|
682
|
+
GeneralUtilities.write_text_to_file(changelog_file, """# Release notes
|
|
683
|
+
|
|
684
|
+
## Changes
|
|
685
|
+
|
|
686
|
+
- Updated dependencies.
|
|
687
|
+
""")
|
|
688
|
+
|
|
689
|
+
@GeneralUtilities.check_arguments
|
|
690
|
+
def update_year_in_license_file(self) -> None:
|
|
691
|
+
self.sc.update_year_in_first_line_of_file(os.path.join(self.repository, "License.txt"))
|