ScriptCollection 3.4.56__py3-none-any.whl → 3.5.45__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/Executables.py +389 -353
- ScriptCollection/GeneralUtilities.py +54 -14
- ScriptCollection/ProcessesRunner.py +41 -0
- ScriptCollection/ProgramRunnerBase.py +47 -42
- ScriptCollection/ProgramRunnerEpew.py +122 -122
- ScriptCollection/ProgramRunnerPopen.py +51 -52
- ScriptCollection/RPStream.py +42 -0
- ScriptCollection/ScriptCollectionCore.py +1930 -1710
- ScriptCollection/TasksForCommonProjectStructure.py +2990 -2426
- ScriptCollection/UpdateCertificates.py +126 -128
- {ScriptCollection-3.4.56.dist-info → ScriptCollection-3.5.45.dist-info}/METADATA +22 -21
- ScriptCollection-3.5.45.dist-info/RECORD +16 -0
- {ScriptCollection-3.4.56.dist-info → ScriptCollection-3.5.45.dist-info}/WHEEL +1 -1
- {ScriptCollection-3.4.56.dist-info → ScriptCollection-3.5.45.dist-info}/entry_points.txt +3 -0
- ScriptCollection-3.4.56.dist-info/RECORD +0 -14
- {ScriptCollection-3.4.56.dist-info → ScriptCollection-3.5.45.dist-info}/top_level.txt +0 -0
|
@@ -1,1710 +1,1930 @@
|
|
|
1
|
-
|
|
2
|
-
import
|
|
3
|
-
import
|
|
4
|
-
import
|
|
5
|
-
|
|
6
|
-
import
|
|
7
|
-
import
|
|
8
|
-
import
|
|
9
|
-
from
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
import
|
|
13
|
-
import
|
|
14
|
-
import
|
|
15
|
-
import
|
|
16
|
-
import
|
|
17
|
-
import
|
|
18
|
-
import
|
|
19
|
-
import
|
|
20
|
-
import
|
|
21
|
-
import
|
|
22
|
-
import
|
|
23
|
-
import
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
def
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
@GeneralUtilities.check_arguments
|
|
88
|
-
def
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
self.run_program("dotnet", f"
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
if
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
@GeneralUtilities.check_arguments
|
|
211
|
-
def
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
return
|
|
217
|
-
|
|
218
|
-
@GeneralUtilities.check_arguments
|
|
219
|
-
def
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
return
|
|
225
|
-
|
|
226
|
-
@GeneralUtilities.check_arguments
|
|
227
|
-
def
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
if (
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
if
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
self.
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
@GeneralUtilities.check_arguments
|
|
489
|
-
def
|
|
490
|
-
result = self.run_program_argsasarray("git", ["
|
|
491
|
-
return result[
|
|
492
|
-
|
|
493
|
-
@GeneralUtilities.check_arguments
|
|
494
|
-
def
|
|
495
|
-
|
|
496
|
-
result =
|
|
497
|
-
return result
|
|
498
|
-
|
|
499
|
-
@GeneralUtilities.check_arguments
|
|
500
|
-
def
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
return
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
lines =
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
@GeneralUtilities.check_arguments
|
|
777
|
-
def
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
if
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
if
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
if (
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
@GeneralUtilities.check_arguments
|
|
1034
|
-
def
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
if
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
if
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
if
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
@GeneralUtilities.check_arguments
|
|
1384
|
-
def
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
if
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
@GeneralUtilities.check_arguments
|
|
1479
|
-
def
|
|
1480
|
-
return
|
|
1481
|
-
|
|
1482
|
-
@GeneralUtilities.check_arguments
|
|
1483
|
-
def
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
@GeneralUtilities.check_arguments
|
|
1492
|
-
def
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
@GeneralUtilities.check_arguments
|
|
1544
|
-
def
|
|
1545
|
-
if days_until_expire is None:
|
|
1546
|
-
days_until_expire =
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
self.run_program("openssl", f'
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
"
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
GeneralUtilities.
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
GeneralUtilities.
|
|
1
|
+
import sys
|
|
2
|
+
from datetime import timedelta, datetime
|
|
3
|
+
import json
|
|
4
|
+
import binascii
|
|
5
|
+
import filecmp
|
|
6
|
+
import hashlib
|
|
7
|
+
import multiprocessing
|
|
8
|
+
import time
|
|
9
|
+
from io import BytesIO
|
|
10
|
+
import itertools
|
|
11
|
+
import math
|
|
12
|
+
import os
|
|
13
|
+
from queue import Queue, Empty
|
|
14
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from subprocess import Popen
|
|
17
|
+
import re
|
|
18
|
+
import shutil
|
|
19
|
+
import uuid
|
|
20
|
+
import tempfile
|
|
21
|
+
import io
|
|
22
|
+
import requests
|
|
23
|
+
import ntplib
|
|
24
|
+
import yaml
|
|
25
|
+
import qrcode
|
|
26
|
+
import pycdlib
|
|
27
|
+
import send2trash
|
|
28
|
+
import PyPDF2
|
|
29
|
+
from .GeneralUtilities import GeneralUtilities
|
|
30
|
+
from .ProgramRunnerBase import ProgramRunnerBase
|
|
31
|
+
from .ProgramRunnerPopen import ProgramRunnerPopen
|
|
32
|
+
from .ProgramRunnerEpew import ProgramRunnerEpew, CustomEpewArgument
|
|
33
|
+
|
|
34
|
+
version = "3.5.45"
|
|
35
|
+
__version__ = version
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class ScriptCollectionCore:
|
|
39
|
+
|
|
40
|
+
# The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
|
|
41
|
+
# Do not change this value for productive environments.
|
|
42
|
+
mock_program_calls: bool = False
|
|
43
|
+
# The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
|
|
44
|
+
execute_program_really_if_no_mock_call_is_defined: bool = False
|
|
45
|
+
__mocked_program_calls: list = None
|
|
46
|
+
program_runner: ProgramRunnerBase = None
|
|
47
|
+
call_program_runner_directly: bool = None
|
|
48
|
+
|
|
49
|
+
def __init__(self):
|
|
50
|
+
self.program_runner = ProgramRunnerPopen()
|
|
51
|
+
self.call_program_runner_directly = None
|
|
52
|
+
self.__mocked_program_calls = list[ScriptCollectionCore.__MockProgramCall]()
|
|
53
|
+
|
|
54
|
+
@staticmethod
|
|
55
|
+
@GeneralUtilities.check_arguments
|
|
56
|
+
def get_scriptcollection_version() -> str:
|
|
57
|
+
return __version__
|
|
58
|
+
|
|
59
|
+
@GeneralUtilities.check_arguments
|
|
60
|
+
def python_file_has_errors(self, file: str, working_directory: str, treat_warnings_as_errors: bool = True) -> tuple[bool, list[str]]:
|
|
61
|
+
errors = list()
|
|
62
|
+
filename = os.path.relpath(file, working_directory)
|
|
63
|
+
if treat_warnings_as_errors:
|
|
64
|
+
errorsonly_argument = ""
|
|
65
|
+
else:
|
|
66
|
+
errorsonly_argument = " --errors-only"
|
|
67
|
+
(exit_code, stdout, stderr, _) = self.run_program("pylint", filename + errorsonly_argument, working_directory, throw_exception_if_exitcode_is_not_zero=False)
|
|
68
|
+
if (exit_code != 0):
|
|
69
|
+
errors.append(f"Linting-issues of {file}:")
|
|
70
|
+
errors.append(f"Pylint-exitcode: {exit_code}")
|
|
71
|
+
for line in GeneralUtilities.string_to_lines(stdout):
|
|
72
|
+
errors.append(line)
|
|
73
|
+
for line in GeneralUtilities.string_to_lines(stderr):
|
|
74
|
+
errors.append(line)
|
|
75
|
+
return (True, errors)
|
|
76
|
+
|
|
77
|
+
return (False, errors)
|
|
78
|
+
|
|
79
|
+
@GeneralUtilities.check_arguments
|
|
80
|
+
def replace_version_in_dockerfile_file(self, dockerfile: str, new_version_value: str) -> None:
|
|
81
|
+
GeneralUtilities.write_text_to_file(dockerfile, re.sub("ARG Version=\"\\d+\\.\\d+\\.\\d+\"", f"ARG Version=\"{new_version_value}\"", GeneralUtilities.read_text_from_file(dockerfile)))
|
|
82
|
+
|
|
83
|
+
@GeneralUtilities.check_arguments
|
|
84
|
+
def replace_version_in_python_file(self, file: str, new_version_value: str):
|
|
85
|
+
GeneralUtilities.write_text_to_file(file, re.sub("version = \"\\d+\\.\\d+\\.\\d+\"", f"version = \"{new_version_value}\"", GeneralUtilities.read_text_from_file(file)))
|
|
86
|
+
|
|
87
|
+
@GeneralUtilities.check_arguments
|
|
88
|
+
def replace_version_in_ini_file(self, file: str, new_version_value: str):
|
|
89
|
+
GeneralUtilities.write_text_to_file(file, re.sub("version = \\d+\\.\\d+\\.\\d+", f"version = {new_version_value}", GeneralUtilities.read_text_from_file(file)))
|
|
90
|
+
|
|
91
|
+
@GeneralUtilities.check_arguments
|
|
92
|
+
def replace_version_in_nuspec_file(self, nuspec_file: str, new_version: str) -> None:
|
|
93
|
+
# TODO use XSLT instead
|
|
94
|
+
versionregex = "\\d+\\.\\d+\\.\\d+"
|
|
95
|
+
versiononlyregex = f"^{versionregex}$"
|
|
96
|
+
pattern = re.compile(versiononlyregex)
|
|
97
|
+
if pattern.match(new_version):
|
|
98
|
+
GeneralUtilities.write_text_to_file(nuspec_file, re.sub(f"<version>{versionregex}<\\/version>", f"<version>{new_version}</version>", GeneralUtilities.read_text_from_file(nuspec_file)))
|
|
99
|
+
else:
|
|
100
|
+
raise ValueError(f"Version '{new_version}' does not match version-regex '{versiononlyregex}'")
|
|
101
|
+
|
|
102
|
+
@GeneralUtilities.check_arguments
|
|
103
|
+
def replace_version_in_csproj_file(self, csproj_file: str, current_version: str):
|
|
104
|
+
versionregex = "\\d+\\.\\d+\\.\\d+"
|
|
105
|
+
versiononlyregex = f"^{versionregex}$"
|
|
106
|
+
pattern = re.compile(versiononlyregex)
|
|
107
|
+
if pattern.match(current_version):
|
|
108
|
+
for tag in ["Version", "AssemblyVersion", "FileVersion"]:
|
|
109
|
+
GeneralUtilities.write_text_to_file(csproj_file, re.sub(f"<{tag}>{versionregex}(.\\d+)?<\\/{tag}>", f"<{tag}>{current_version}</{tag}>", GeneralUtilities.read_text_from_file(csproj_file)))
|
|
110
|
+
else:
|
|
111
|
+
raise ValueError(f"Version '{current_version}' does not match version-regex '{versiononlyregex}'")
|
|
112
|
+
|
|
113
|
+
@GeneralUtilities.check_arguments
|
|
114
|
+
def push_nuget_build_artifact(self, nupkg_file: str, registry_address: str, api_key: str, verbosity: int = 1):
|
|
115
|
+
nupkg_file_name = os.path.basename(nupkg_file)
|
|
116
|
+
nupkg_file_folder = os.path.dirname(nupkg_file)
|
|
117
|
+
self.run_program("dotnet", f"nuget push {nupkg_file_name} --force-english-output --source {registry_address} --api-key {api_key}", nupkg_file_folder, verbosity)
|
|
118
|
+
|
|
119
|
+
@GeneralUtilities.check_arguments
|
|
120
|
+
def dotnet_build(self, repository_folder: str, projectname: str, configuration: str):
|
|
121
|
+
self.run_program("dotnet", f"clean -c {configuration}", repository_folder)
|
|
122
|
+
self.run_program("dotnet", f"build {projectname}/{projectname}.csproj -c {configuration}", repository_folder)
|
|
123
|
+
|
|
124
|
+
@GeneralUtilities.check_arguments
|
|
125
|
+
def find_file_by_extension(self, folder: str, extension: str):
|
|
126
|
+
result = [file for file in GeneralUtilities.get_direct_files_of_folder(folder) if file.endswith(f".{extension}")]
|
|
127
|
+
result_length = len(result)
|
|
128
|
+
if result_length == 0:
|
|
129
|
+
raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension}'.")
|
|
130
|
+
if result_length == 1:
|
|
131
|
+
return result[0]
|
|
132
|
+
else:
|
|
133
|
+
raise ValueError(f"Multiple values available in folder '{folder}' with extension '{extension}'.")
|
|
134
|
+
|
|
135
|
+
@GeneralUtilities.check_arguments
|
|
136
|
+
def find_latest_file_by_extension(self, folder: str, extension: str) -> str:
|
|
137
|
+
files: list[str] = GeneralUtilities.get_direct_files_of_folder(folder)
|
|
138
|
+
possible_results: list[str] = []
|
|
139
|
+
for file in files:
|
|
140
|
+
if file.endswith(f".{extension}"):
|
|
141
|
+
possible_results.append(file)
|
|
142
|
+
result_length = len(possible_results)
|
|
143
|
+
if result_length == 0:
|
|
144
|
+
raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension}'.")
|
|
145
|
+
else:
|
|
146
|
+
return possible_results[-1]
|
|
147
|
+
|
|
148
|
+
@GeneralUtilities.check_arguments
|
|
149
|
+
def commit_is_signed_by_key(self, repository_folder: str, revision_identifier: str, key: str) -> bool:
|
|
150
|
+
result = self.run_program(
|
|
151
|
+
"git", f"verify-commit {revision_identifier}", repository_folder, throw_exception_if_exitcode_is_not_zero=False)
|
|
152
|
+
if (result[0] != 0):
|
|
153
|
+
return False
|
|
154
|
+
if (not GeneralUtilities.contains_line(result[1].splitlines(), f"gpg\\:\\ using\\ [A-Za-z0-9]+\\ key\\ [A-Za-z0-9]+{key}")):
|
|
155
|
+
# TODO check whether this works on machines where gpg is installed in another langauge than english
|
|
156
|
+
return False
|
|
157
|
+
if (not GeneralUtilities.contains_line(result[1].splitlines(), "gpg\\:\\ Good\\ signature\\ from")):
|
|
158
|
+
# TODO check whether this works on machines where gpg is installed in another langauge than english
|
|
159
|
+
return False
|
|
160
|
+
return True
|
|
161
|
+
|
|
162
|
+
@GeneralUtilities.check_arguments
|
|
163
|
+
def get_parent_commit_ids_of_commit(self, repository_folder: str, commit_id: str) -> str:
|
|
164
|
+
return self.run_program("git", f'log --pretty=%P -n 1 "{commit_id}"', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].replace("\r", "").replace("\n", "").split(" ")
|
|
165
|
+
|
|
166
|
+
@GeneralUtilities.check_arguments
|
|
167
|
+
def get_all_authors_and_committers_of_repository(self, repository_folder: str, subfolder: str = None, verbosity: int = 1) -> list[tuple[str, str]]:
|
|
168
|
+
space_character = "_"
|
|
169
|
+
if subfolder is None:
|
|
170
|
+
subfolder_argument = ""
|
|
171
|
+
else:
|
|
172
|
+
subfolder_argument = f" -- {subfolder}"
|
|
173
|
+
log_result = self.run_program("git", f'log --pretty=%aN{space_character}%aE%n%cN{space_character}%cE HEAD{subfolder_argument}', repository_folder, verbosity=0)
|
|
174
|
+
plain_content: list[str] = list(
|
|
175
|
+
set([line for line in log_result[1].split("\n") if len(line) > 0]))
|
|
176
|
+
result: list[tuple[str, str]] = []
|
|
177
|
+
for item in plain_content:
|
|
178
|
+
if len(re.findall(space_character, item)) == 1:
|
|
179
|
+
splitted = item.split(space_character)
|
|
180
|
+
result.append((splitted[0], splitted[1]))
|
|
181
|
+
else:
|
|
182
|
+
raise ValueError(f'Unexpected author: "{item}"')
|
|
183
|
+
return result
|
|
184
|
+
|
|
185
|
+
@GeneralUtilities.check_arguments
|
|
186
|
+
def get_commit_ids_between_dates(self, repository_folder: str, since: datetime, until: datetime, ignore_commits_which_are_not_in_history_of_head: bool = True) -> None:
|
|
187
|
+
since_as_string = self.__datetime_to_string_for_git(since)
|
|
188
|
+
until_as_string = self.__datetime_to_string_for_git(until)
|
|
189
|
+
result = filter(lambda line: not GeneralUtilities.string_is_none_or_whitespace(line), self.run_program("git", f'log --since "{since_as_string}" --until "{until_as_string}" --pretty=format:"%H" --no-patch', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].split("\n").replace("\r", ""))
|
|
190
|
+
if ignore_commits_which_are_not_in_history_of_head:
|
|
191
|
+
result = [commit_id for commit_id in result if self.git_commit_is_ancestor(
|
|
192
|
+
repository_folder, commit_id)]
|
|
193
|
+
return result
|
|
194
|
+
|
|
195
|
+
@GeneralUtilities.check_arguments
|
|
196
|
+
def __datetime_to_string_for_git(self, datetime_object: datetime) -> str:
|
|
197
|
+
return datetime_object.strftime('%Y-%m-%d %H:%M:%S')
|
|
198
|
+
|
|
199
|
+
@GeneralUtilities.check_arguments
|
|
200
|
+
def git_commit_is_ancestor(self, repository_folder: str, ancestor: str, descendant: str = "HEAD") -> bool:
|
|
201
|
+
result = self.run_program_argsasarray("git", ["merge-base", "--is-ancestor", ancestor, descendant], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
|
|
202
|
+
exit_code = result[0]
|
|
203
|
+
if exit_code == 0:
|
|
204
|
+
return True
|
|
205
|
+
elif exit_code == 1:
|
|
206
|
+
return False
|
|
207
|
+
else:
|
|
208
|
+
raise ValueError(f'Can not calculate if {ancestor} is an ancestor of {descendant} in repository {repository_folder}. Outout of "{repository_folder}> git merge-base --is-ancestor {ancestor} {descendant}": Exitcode: {exit_code}; StdOut: {result[1]}; StdErr: {result[2]}.')
|
|
209
|
+
|
|
210
|
+
@GeneralUtilities.check_arguments
|
|
211
|
+
def __git_changes_helper(self, repository_folder: str, arguments_as_array: list[str]) -> bool:
|
|
212
|
+
lines = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", arguments_as_array, repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
|
|
213
|
+
for line in lines:
|
|
214
|
+
if GeneralUtilities.string_has_content(line):
|
|
215
|
+
return True
|
|
216
|
+
return False
|
|
217
|
+
|
|
218
|
+
@GeneralUtilities.check_arguments
|
|
219
|
+
def git_repository_has_new_untracked_files(self, repositoryFolder: str):
|
|
220
|
+
return self.__git_changes_helper(repositoryFolder, ["ls-files", "--exclude-standard", "--others"])
|
|
221
|
+
|
|
222
|
+
@GeneralUtilities.check_arguments
|
|
223
|
+
def git_repository_has_unstaged_changes_of_tracked_files(self, repositoryFolder: str):
|
|
224
|
+
return self.__git_changes_helper(repositoryFolder, ["--no-pager", "diff"])
|
|
225
|
+
|
|
226
|
+
@GeneralUtilities.check_arguments
|
|
227
|
+
def git_repository_has_staged_changes(self, repositoryFolder: str):
|
|
228
|
+
return self.__git_changes_helper(repositoryFolder, ["--no-pager", "diff", "--cached"])
|
|
229
|
+
|
|
230
|
+
@GeneralUtilities.check_arguments
|
|
231
|
+
def git_repository_has_uncommitted_changes(self, repositoryFolder: str) -> bool:
|
|
232
|
+
if (self.git_repository_has_unstaged_changes(repositoryFolder)):
|
|
233
|
+
return True
|
|
234
|
+
if (self.git_repository_has_staged_changes(repositoryFolder)):
|
|
235
|
+
return True
|
|
236
|
+
return False
|
|
237
|
+
|
|
238
|
+
@GeneralUtilities.check_arguments
|
|
239
|
+
def git_repository_has_unstaged_changes(self, repository_folder: str) -> bool:
|
|
240
|
+
if (self.git_repository_has_unstaged_changes_of_tracked_files(repository_folder)):
|
|
241
|
+
return True
|
|
242
|
+
if (self.git_repository_has_new_untracked_files(repository_folder)):
|
|
243
|
+
return True
|
|
244
|
+
return False
|
|
245
|
+
|
|
246
|
+
@GeneralUtilities.check_arguments
|
|
247
|
+
def git_get_commit_id(self, repository_folder: str, commit: str = "HEAD") -> str:
|
|
248
|
+
result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["rev-parse", "--verify", commit], repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
249
|
+
return result[1].replace('\n', '')
|
|
250
|
+
|
|
251
|
+
@GeneralUtilities.check_arguments
|
|
252
|
+
def git_get_commit_date(self, repository_folder: str, commit: str = "HEAD") -> datetime:
|
|
253
|
+
result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["show", "-s", "--format=%ci", commit], repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
254
|
+
date_as_string = result[1].replace('\n', '')
|
|
255
|
+
result = datetime.strptime(date_as_string, '%Y-%m-%d %H:%M:%S %z')
|
|
256
|
+
return result
|
|
257
|
+
|
|
258
|
+
@GeneralUtilities.check_arguments
|
|
259
|
+
def git_fetch(self, folder: str, remotename: str = "--all") -> None:
|
|
260
|
+
self.run_program_argsasarray("git", ["fetch", remotename, "--tags", "--prune"], folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
261
|
+
|
|
262
|
+
@GeneralUtilities.check_arguments
|
|
263
|
+
def git_fetch_in_bare_repository(self, folder: str, remotename, localbranch: str, remotebranch: str) -> None:
|
|
264
|
+
self.run_program_argsasarray("git", ["fetch", remotename, f"{remotebranch}:{localbranch}"], folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
265
|
+
|
|
266
|
+
@GeneralUtilities.check_arguments
|
|
267
|
+
def git_remove_branch(self, folder: str, branchname: str) -> None:
|
|
268
|
+
self.run_program("git", f"branch -D {branchname}", folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
269
|
+
|
|
270
|
+
@GeneralUtilities.check_arguments
|
|
271
|
+
def git_push(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: int = 0) -> None:
|
|
272
|
+
argument = ["push", "--recurse-submodules=on-demand", remotename, f"{localbranchname}:{remotebranchname}"]
|
|
273
|
+
if (forcepush):
|
|
274
|
+
argument.append("--force")
|
|
275
|
+
if (pushalltags):
|
|
276
|
+
argument.append("--tags")
|
|
277
|
+
result: tuple[int, str, str, int] = self.run_program_argsasarray("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=verbosity, print_errors_as_information=True)
|
|
278
|
+
return result[1].replace('\r', '').replace('\n', '')
|
|
279
|
+
|
|
280
|
+
@GeneralUtilities.check_arguments
|
|
281
|
+
def git_pull(self, folder: str, remote: str, localbranchname: str, remotebranchname: str, force: bool = False) -> None:
|
|
282
|
+
argument = f"pull {remote} {remotebranchname}:{localbranchname}"
|
|
283
|
+
if force:
|
|
284
|
+
argument = f"{argument} --force"
|
|
285
|
+
self.run_program("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
286
|
+
|
|
287
|
+
@GeneralUtilities.check_arguments
|
|
288
|
+
def git_list_remote_branches(self, folder: str, remote: str, fetch: bool) -> list[str]:
|
|
289
|
+
if fetch:
|
|
290
|
+
self.git_fetch(folder, remote)
|
|
291
|
+
run_program_result = self.run_program("git", f"branch -rl {remote}/*", folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
292
|
+
output = GeneralUtilities.string_to_lines(run_program_result[1])
|
|
293
|
+
result = list[str]()
|
|
294
|
+
for item in output:
|
|
295
|
+
striped_item = item.strip()
|
|
296
|
+
if GeneralUtilities.string_has_content(striped_item):
|
|
297
|
+
branch: str = None
|
|
298
|
+
if " " in striped_item:
|
|
299
|
+
branch = striped_item.split(" ")[0]
|
|
300
|
+
else:
|
|
301
|
+
branch = striped_item
|
|
302
|
+
branchname = branch[len(remote)+1:]
|
|
303
|
+
if branchname != "HEAD":
|
|
304
|
+
result.append(branchname)
|
|
305
|
+
return result
|
|
306
|
+
|
|
307
|
+
@GeneralUtilities.check_arguments
|
|
308
|
+
def git_clone(self, clone_target_folder: str, remote_repository_path: str, include_submodules: bool = True, mirror: bool = False) -> None:
|
|
309
|
+
if (os.path.isdir(clone_target_folder)):
|
|
310
|
+
pass # TODO throw error
|
|
311
|
+
else:
|
|
312
|
+
args = ["clone", remote_repository_path, clone_target_folder]
|
|
313
|
+
if include_submodules:
|
|
314
|
+
args.append("--recurse-submodules")
|
|
315
|
+
args.append("--remote-submodules")
|
|
316
|
+
if mirror:
|
|
317
|
+
args.append("--mirror")
|
|
318
|
+
self.run_program_argsasarray("git", args, os.getcwd(), throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
319
|
+
|
|
320
|
+
@GeneralUtilities.check_arguments
|
|
321
|
+
def git_get_all_remote_names(self, directory: str) -> list[str]:
|
|
322
|
+
result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
|
|
323
|
+
return result
|
|
324
|
+
|
|
325
|
+
@GeneralUtilities.check_arguments
|
|
326
|
+
def git_get_remote_url(self, directory: str, remote_name: str) -> str:
|
|
327
|
+
result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote", "get-url", remote_name], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
|
|
328
|
+
return result[0].replace('\n', '')
|
|
329
|
+
|
|
330
|
+
@GeneralUtilities.check_arguments
|
|
331
|
+
def repository_has_remote_with_specific_name(self, directory: str, remote_name: str) -> bool:
|
|
332
|
+
return remote_name in self.git_get_all_remote_names(directory)
|
|
333
|
+
|
|
334
|
+
@GeneralUtilities.check_arguments
|
|
335
|
+
def git_add_or_set_remote_address(self, directory: str, remote_name: str, remote_address: str) -> None:
|
|
336
|
+
if (self.repository_has_remote_with_specific_name(directory, remote_name)):
|
|
337
|
+
self.run_program_argsasarray("git", ['remote', 'set-url', 'remote_name', remote_address], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
338
|
+
else:
|
|
339
|
+
self.run_program_argsasarray("git", ['remote', 'add', remote_name, remote_address], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
340
|
+
|
|
341
|
+
@GeneralUtilities.check_arguments
|
|
342
|
+
def git_stage_all_changes(self, directory: str) -> None:
|
|
343
|
+
self.run_program_argsasarray("git", ["add", "-A"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
344
|
+
|
|
345
|
+
@GeneralUtilities.check_arguments
|
|
346
|
+
def git_unstage_all_changes(self, directory: str) -> None:
|
|
347
|
+
self.run_program_argsasarray("git", ["reset"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
348
|
+
|
|
349
|
+
@GeneralUtilities.check_arguments
|
|
350
|
+
def git_stage_file(self, directory: str, file: str) -> None:
|
|
351
|
+
self.run_program_argsasarray("git", ['stage', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
352
|
+
|
|
353
|
+
@GeneralUtilities.check_arguments
|
|
354
|
+
def git_unstage_file(self, directory: str, file: str) -> None:
|
|
355
|
+
self.run_program_argsasarray("git", ['reset', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
356
|
+
|
|
357
|
+
@GeneralUtilities.check_arguments
|
|
358
|
+
def git_discard_unstaged_changes_of_file(self, directory: str, file: str) -> None:
|
|
359
|
+
"""Caution: This method works really only for 'changed' files yet. So this method does not work properly for new or renamed files."""
|
|
360
|
+
self.run_program_argsasarray("git", ['checkout', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
361
|
+
|
|
362
|
+
@GeneralUtilities.check_arguments
|
|
363
|
+
def git_discard_all_unstaged_changes(self, directory: str) -> None:
|
|
364
|
+
"""Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
|
|
365
|
+
self.run_program_argsasarray("git", ['clean', '-df'], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
366
|
+
self.run_program_argsasarray("git", ['checkout', '.'], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
367
|
+
|
|
368
|
+
@GeneralUtilities.check_arguments
|
|
369
|
+
def git_commit(self, directory: str, message: str, author_name: str = None, author_email: str = None, stage_all_changes: bool = True, no_changes_behavior: int = 0) -> str:
|
|
370
|
+
# no_changes_behavior=0 => No commit
|
|
371
|
+
# no_changes_behavior=1 => Commit anyway
|
|
372
|
+
# no_changes_behavior=2 => Exception
|
|
373
|
+
author_name = GeneralUtilities.str_none_safe(author_name).strip()
|
|
374
|
+
author_email = GeneralUtilities.str_none_safe(author_email).strip()
|
|
375
|
+
argument = ['commit', '--quiet', '--allow-empty', '--message', message]
|
|
376
|
+
if (GeneralUtilities.string_has_content(author_name)):
|
|
377
|
+
argument.append(f'--author="{author_name} <{author_email}>"')
|
|
378
|
+
git_repository_has_uncommitted_changes = self.git_repository_has_uncommitted_changes(directory)
|
|
379
|
+
|
|
380
|
+
if git_repository_has_uncommitted_changes:
|
|
381
|
+
do_commit = True
|
|
382
|
+
if stage_all_changes:
|
|
383
|
+
self.git_stage_all_changes(directory)
|
|
384
|
+
else:
|
|
385
|
+
if no_changes_behavior == 0:
|
|
386
|
+
GeneralUtilities.write_message_to_stdout(f"Commit '{message}' will not be done because there are no changes to commit in repository '{directory}'")
|
|
387
|
+
do_commit = False
|
|
388
|
+
elif no_changes_behavior == 1:
|
|
389
|
+
GeneralUtilities.write_message_to_stdout(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will be done anyway.")
|
|
390
|
+
do_commit = True
|
|
391
|
+
elif no_changes_behavior == 2:
|
|
392
|
+
raise RuntimeError(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will not be done.")
|
|
393
|
+
else:
|
|
394
|
+
raise ValueError(f"Unknown value for no_changes_behavior: {GeneralUtilities.str_none_safe(no_changes_behavior)}")
|
|
395
|
+
|
|
396
|
+
if do_commit:
|
|
397
|
+
GeneralUtilities.write_message_to_stdout(f"Commit changes in '{directory}'")
|
|
398
|
+
self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
399
|
+
|
|
400
|
+
return self.git_get_commit_id(directory)
|
|
401
|
+
|
|
402
|
+
@GeneralUtilities.check_arguments
|
|
403
|
+
def git_create_tag(self, directory: str, target_for_tag: str, tag: str, sign: bool = False, message: str = None) -> None:
|
|
404
|
+
argument = ["tag", tag, target_for_tag]
|
|
405
|
+
if sign:
|
|
406
|
+
if message is None:
|
|
407
|
+
message = f"Created {target_for_tag}"
|
|
408
|
+
argument.extend(["-s", '-m', message])
|
|
409
|
+
self.run_program_argsasarray(
|
|
410
|
+
"git", argument, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
411
|
+
|
|
412
|
+
@GeneralUtilities.check_arguments
|
|
413
|
+
def git_delete_tag(self, directory: str, tag: str) -> None:
|
|
414
|
+
self.run_program_argsasarray("git", ["tag", "--delete", tag], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
415
|
+
|
|
416
|
+
@GeneralUtilities.check_arguments
|
|
417
|
+
def git_checkout(self, directory: str, branch: str) -> None:
|
|
418
|
+
self.run_program_argsasarray("git", ["checkout", branch], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
419
|
+
self.run_program_argsasarray("git", ["submodule", "update", "--recursive"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
420
|
+
|
|
421
|
+
@GeneralUtilities.check_arguments
|
|
422
|
+
def git_merge_abort(self, directory: str) -> None:
|
|
423
|
+
self.run_program_argsasarray("git", ["merge", "--abort"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
424
|
+
|
|
425
|
+
@GeneralUtilities.check_arguments
|
|
426
|
+
def git_merge(self, directory: str, sourcebranch: str, targetbranch: str, fastforward: bool = True, commit: bool = True, commit_message: str = None) -> str:
|
|
427
|
+
self.git_checkout(directory, targetbranch)
|
|
428
|
+
args = ["merge"]
|
|
429
|
+
if not commit:
|
|
430
|
+
args.append("--no-commit")
|
|
431
|
+
if not fastforward:
|
|
432
|
+
args.append("--no-ff")
|
|
433
|
+
if commit_message is not None:
|
|
434
|
+
args.append("-m")
|
|
435
|
+
args.append(commit_message)
|
|
436
|
+
args.append(sourcebranch)
|
|
437
|
+
self.run_program_argsasarray("git", args, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
438
|
+
self.run_program_argsasarray("git", ["submodule", "update"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
439
|
+
return self.git_get_commit_id(directory)
|
|
440
|
+
|
|
441
|
+
@GeneralUtilities.check_arguments
|
|
442
|
+
def git_undo_all_changes(self, directory: str) -> None:
|
|
443
|
+
"""Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
|
|
444
|
+
self.git_unstage_all_changes(directory)
|
|
445
|
+
self.git_discard_all_unstaged_changes(directory)
|
|
446
|
+
|
|
447
|
+
@GeneralUtilities.check_arguments
|
|
448
|
+
def git_fetch_or_clone_all_in_directory(self, source_directory: str, target_directory: str) -> None:
|
|
449
|
+
for subfolder in GeneralUtilities.get_direct_folders_of_folder(source_directory):
|
|
450
|
+
foldername = os.path.basename(subfolder)
|
|
451
|
+
if self.is_git_repository(subfolder):
|
|
452
|
+
source_repository = subfolder
|
|
453
|
+
target_repository = os.path.join(target_directory, foldername)
|
|
454
|
+
if os.path.isdir(target_directory):
|
|
455
|
+
# fetch
|
|
456
|
+
self.git_fetch(target_directory)
|
|
457
|
+
else:
|
|
458
|
+
# clone
|
|
459
|
+
self.git_clone(target_repository, source_repository, include_submodules=True, mirror=True)
|
|
460
|
+
|
|
461
|
+
def get_git_submodules(self, folder: str) -> list[str]:
|
|
462
|
+
e = self.run_program("git", "submodule status", folder)
|
|
463
|
+
result = []
|
|
464
|
+
for submodule_line in GeneralUtilities.string_to_lines(e[1], False, True):
|
|
465
|
+
result.append(submodule_line.split(' ')[1])
|
|
466
|
+
return result
|
|
467
|
+
|
|
468
|
+
@GeneralUtilities.check_arguments
|
|
469
|
+
def is_git_repository(self, folder: str) -> bool:
|
|
470
|
+
combined = os.path.join(folder, ".git")
|
|
471
|
+
# TODO consider check for bare-repositories
|
|
472
|
+
return os.path.isdir(combined) or os.path.isfile(combined)
|
|
473
|
+
|
|
474
|
+
@GeneralUtilities.check_arguments
|
|
475
|
+
def file_is_git_ignored(self, file_in_repository: str, repositorybasefolder: str) -> None:
|
|
476
|
+
exit_code = self.run_program_argsasarray("git", ['check-ignore', file_in_repository], repositorybasefolder, throw_exception_if_exitcode_is_not_zero=False, verbosity=0)[0]
|
|
477
|
+
if (exit_code == 0):
|
|
478
|
+
return True
|
|
479
|
+
if (exit_code == 1):
|
|
480
|
+
return False
|
|
481
|
+
raise ValueError(f"Unable to calculate whether '{file_in_repository}' in repository '{repositorybasefolder}' is ignored due to git-exitcode {exit_code}.")
|
|
482
|
+
|
|
483
|
+
@GeneralUtilities.check_arguments
|
|
484
|
+
def git_discard_all_changes(self, repository: str) -> None:
|
|
485
|
+
self.run_program_argsasarray("git", ["reset", "HEAD", "."], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
486
|
+
self.run_program_argsasarray("git", ["checkout", "."], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
487
|
+
|
|
488
|
+
@GeneralUtilities.check_arguments
|
|
489
|
+
def git_get_current_branch_name(self, repository: str) -> str:
|
|
490
|
+
result = self.run_program_argsasarray("git", ["rev-parse", "--abbrev-ref", "HEAD"], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
491
|
+
return result[1].replace("\r", "").replace("\n", "")
|
|
492
|
+
|
|
493
|
+
@GeneralUtilities.check_arguments
|
|
494
|
+
def git_get_commitid_of_tag(self, repository: str, tag: str) -> str:
|
|
495
|
+
stdout = self.run_program_argsasarray("git", ["rev-list", "-n", "1", tag], repository, verbosity=0)
|
|
496
|
+
result = stdout[1].replace("\r", "").replace("\n", "")
|
|
497
|
+
return result
|
|
498
|
+
|
|
499
|
+
@GeneralUtilities.check_arguments
|
|
500
|
+
def git_get_tags(self, repository: str) -> list[str]:
|
|
501
|
+
tags = [line.replace("\r", "") for line in self.run_program_argsasarray(
|
|
502
|
+
"git", ["tag"], repository)[1].split("\n") if len(line) > 0]
|
|
503
|
+
return tags
|
|
504
|
+
|
|
505
|
+
@GeneralUtilities.check_arguments
|
|
506
|
+
def git_move_tags_to_another_branch(self, repository: str, tag_source_branch: str, tag_target_branch: str, sign: bool = False, message: str = None) -> None:
|
|
507
|
+
tags = self.git_get_tags(repository)
|
|
508
|
+
tags_count = len(tags)
|
|
509
|
+
counter = 0
|
|
510
|
+
for tag in tags:
|
|
511
|
+
counter = counter+1
|
|
512
|
+
GeneralUtilities.write_message_to_stdout(f"Process tag {counter}/{tags_count}.")
|
|
513
|
+
# tag is on source-branch
|
|
514
|
+
if self.git_commit_is_ancestor(repository, tag, tag_source_branch):
|
|
515
|
+
commit_id_old = self.git_get_commitid_of_tag(repository, tag)
|
|
516
|
+
commit_date: datetime = self.git_get_commit_date(repository, commit_id_old)
|
|
517
|
+
date_as_string = self.__datetime_to_string_for_git(commit_date)
|
|
518
|
+
search_commit_result = self.run_program_argsasarray("git", ["log", f'--after="{date_as_string}"', f'--before="{date_as_string}"', "--pretty=format:%H", tag_target_branch], repository, throw_exception_if_exitcode_is_not_zero=False)
|
|
519
|
+
if search_commit_result[0] != 0 or not GeneralUtilities.string_has_nonwhitespace_content(search_commit_result[1]):
|
|
520
|
+
raise ValueError(f"Can not calculate corresponding commit for tag '{tag}'.")
|
|
521
|
+
commit_id_new = search_commit_result[1]
|
|
522
|
+
self.git_delete_tag(repository, tag)
|
|
523
|
+
self.git_create_tag(repository, commit_id_new, tag, sign, message)
|
|
524
|
+
|
|
525
|
+
@GeneralUtilities.check_arguments
|
|
526
|
+
def get_current_git_branch_has_tag(self, repository_folder: str) -> bool:
|
|
527
|
+
result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder, verbosity=0, throw_exception_if_exitcode_is_not_zero=False)
|
|
528
|
+
return result[0] == 0
|
|
529
|
+
|
|
530
|
+
@GeneralUtilities.check_arguments
|
|
531
|
+
def get_latest_git_tag(self, repository_folder: str) -> str:
|
|
532
|
+
result = self.run_program_argsasarray(
|
|
533
|
+
"git", ["describe", "--tags", "--abbrev=0"], repository_folder, verbosity=0)
|
|
534
|
+
result = result[1].replace("\r", "").replace("\n", "")
|
|
535
|
+
return result
|
|
536
|
+
|
|
537
|
+
@GeneralUtilities.check_arguments
|
|
538
|
+
def get_staged_or_committed_git_ignored_files(self, repository_folder: str) -> list[str]:
|
|
539
|
+
tresult = self.run_program_argsasarray("git", ["ls-files", "-i", "-c", "--exclude-standard"], repository_folder, verbosity=0)
|
|
540
|
+
tresult = tresult[1].replace("\r", "")
|
|
541
|
+
result = [line for line in tresult.split("\n") if len(line) > 0]
|
|
542
|
+
return result
|
|
543
|
+
|
|
544
|
+
@GeneralUtilities.check_arguments
|
|
545
|
+
def git_repository_has_commits(self, repository_folder: str) -> bool:
|
|
546
|
+
return self.run_program_argsasarray("git", ["rev-parse", "--verify", "HEAD"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)[0] == 0
|
|
547
|
+
|
|
548
|
+
@GeneralUtilities.check_arguments
|
|
549
|
+
def export_filemetadata(self, folder: str, target_file: str, encoding: str = "utf-8", filter_function=None) -> None:
|
|
550
|
+
folder = GeneralUtilities.resolve_relative_path_from_current_working_directory(folder)
|
|
551
|
+
lines = list()
|
|
552
|
+
path_prefix = len(folder)+1
|
|
553
|
+
items = dict()
|
|
554
|
+
for item in GeneralUtilities.get_all_folders_of_folder(folder):
|
|
555
|
+
items[item] = "d"
|
|
556
|
+
for item in GeneralUtilities.get_all_files_of_folder(folder):
|
|
557
|
+
items[item] = "f"
|
|
558
|
+
for file_or_folder, item_type in items.items():
|
|
559
|
+
truncated_file = file_or_folder[path_prefix:]
|
|
560
|
+
if (filter_function is None or filter_function(folder, truncated_file)):
|
|
561
|
+
owner_and_permisssion = self.get_file_owner_and_file_permission(file_or_folder)
|
|
562
|
+
user = owner_and_permisssion[0]
|
|
563
|
+
permissions = owner_and_permisssion[1]
|
|
564
|
+
lines.append(f"{truncated_file};{item_type};{user};{permissions}")
|
|
565
|
+
lines = sorted(lines, key=str.casefold)
|
|
566
|
+
with open(target_file, "w", encoding=encoding) as file_object:
|
|
567
|
+
file_object.write("\n".join(lines))
|
|
568
|
+
|
|
569
|
+
@GeneralUtilities.check_arguments
|
|
570
|
+
def escape_git_repositories_in_folder(self, folder: str) -> dict[str, str]:
|
|
571
|
+
return self.__escape_git_repositories_in_folder_internal(folder, dict[str, str]())
|
|
572
|
+
|
|
573
|
+
@GeneralUtilities.check_arguments
|
|
574
|
+
def __escape_git_repositories_in_folder_internal(self, folder: str, renamed_items: dict[str, str]) -> dict[str, str]:
|
|
575
|
+
for file in GeneralUtilities.get_direct_files_of_folder(folder):
|
|
576
|
+
filename = os.path.basename(file)
|
|
577
|
+
if ".git" in filename:
|
|
578
|
+
new_name = filename.replace(".git", ".gitx")
|
|
579
|
+
target = os.path.join(folder, new_name)
|
|
580
|
+
os.rename(file, target)
|
|
581
|
+
renamed_items[target] = file
|
|
582
|
+
for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
|
|
583
|
+
foldername = os.path.basename(subfolder)
|
|
584
|
+
if ".git" in foldername:
|
|
585
|
+
new_name = foldername.replace(".git", ".gitx")
|
|
586
|
+
subfolder2 = os.path.join(str(Path(subfolder).parent), new_name)
|
|
587
|
+
os.rename(subfolder, subfolder2)
|
|
588
|
+
renamed_items[subfolder2] = subfolder
|
|
589
|
+
else:
|
|
590
|
+
subfolder2 = subfolder
|
|
591
|
+
self.__escape_git_repositories_in_folder_internal(subfolder2, renamed_items)
|
|
592
|
+
return renamed_items
|
|
593
|
+
|
|
594
|
+
@GeneralUtilities.check_arguments
|
|
595
|
+
def deescape_git_repositories_in_folder(self, renamed_items: dict[str, str]):
|
|
596
|
+
for renamed_item, original_name in renamed_items.items():
|
|
597
|
+
os.rename(renamed_item, original_name)
|
|
598
|
+
|
|
599
|
+
@GeneralUtilities.check_arguments
|
|
600
|
+
def __sort_fmd(self, line: str):
|
|
601
|
+
splitted: list = line.split(";")
|
|
602
|
+
filetype: str = splitted[1]
|
|
603
|
+
if filetype == "d":
|
|
604
|
+
return -1
|
|
605
|
+
if filetype == "f":
|
|
606
|
+
return 1
|
|
607
|
+
return 0
|
|
608
|
+
|
|
609
|
+
@GeneralUtilities.check_arguments
|
|
610
|
+
def restore_filemetadata(self, folder: str, source_file: str, strict=False, encoding: str = "utf-8", create_folder_is_not_exist: bool = True) -> None:
|
|
611
|
+
lines = GeneralUtilities.read_lines_from_file(source_file, encoding)
|
|
612
|
+
lines.sort(key=self.__sort_fmd)
|
|
613
|
+
for line in lines:
|
|
614
|
+
splitted: list = line.split(";")
|
|
615
|
+
full_path_of_file_or_folder: str = os.path.join(folder, splitted[0])
|
|
616
|
+
filetype: str = splitted[1]
|
|
617
|
+
user: str = splitted[2]
|
|
618
|
+
permissions: str = splitted[3]
|
|
619
|
+
if filetype == "d" and create_folder_is_not_exist and not os.path.isdir(full_path_of_file_or_folder):
|
|
620
|
+
GeneralUtilities.ensure_directory_exists(full_path_of_file_or_folder)
|
|
621
|
+
if (filetype == "f" and os.path.isfile(full_path_of_file_or_folder)) or (filetype == "d" and os.path.isdir(full_path_of_file_or_folder)):
|
|
622
|
+
self.set_owner(full_path_of_file_or_folder, user, os.name != 'nt')
|
|
623
|
+
self.set_permission(full_path_of_file_or_folder, permissions)
|
|
624
|
+
else:
|
|
625
|
+
if strict:
|
|
626
|
+
if filetype == "f":
|
|
627
|
+
filetype_full = "File"
|
|
628
|
+
elif filetype == "d":
|
|
629
|
+
filetype_full = "Directory"
|
|
630
|
+
else:
|
|
631
|
+
raise ValueError(f"Unknown filetype: {GeneralUtilities.str_none_safe(filetype)}")
|
|
632
|
+
raise ValueError(f"{filetype_full} '{full_path_of_file_or_folder}' does not exist")
|
|
633
|
+
|
|
634
|
+
@GeneralUtilities.check_arguments
|
|
635
|
+
def __calculate_lengh_in_seconds(self, filename: str, folder: str) -> float:
|
|
636
|
+
argument = ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', filename]
|
|
637
|
+
result = self.run_program_argsasarray("ffprobe", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
638
|
+
return float(result[1].replace('\n', ''))
|
|
639
|
+
|
|
640
|
+
@GeneralUtilities.check_arguments
|
|
641
|
+
def __create_thumbnails(self, filename: str, fps: str, folder: str, tempname_for_thumbnails: str) -> list[str]:
|
|
642
|
+
argument = ['-i', filename, '-r', str(fps), '-vf', 'scale=-1:120', '-vcodec', 'png', f'{tempname_for_thumbnails}-%002d.png']
|
|
643
|
+
self.run_program_argsasarray("ffmpeg", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
644
|
+
files = GeneralUtilities.get_direct_files_of_folder(folder)
|
|
645
|
+
result: list[str] = []
|
|
646
|
+
regex = "^"+re.escape(tempname_for_thumbnails)+"\\-\\d+\\.png$"
|
|
647
|
+
regex_for_files = re.compile(regex)
|
|
648
|
+
for file in files:
|
|
649
|
+
filename = os.path.basename(file)
|
|
650
|
+
if regex_for_files.match(filename):
|
|
651
|
+
result.append(file)
|
|
652
|
+
GeneralUtilities.assert_condition(0 < len(result), "No thumbnail-files found.")
|
|
653
|
+
return result
|
|
654
|
+
|
|
655
|
+
@GeneralUtilities.check_arguments
|
|
656
|
+
def __create_thumbnail(self, outputfilename: str, folder: str, length_in_seconds: float, tempname_for_thumbnails: str, amount_of_images: int) -> None:
|
|
657
|
+
duration = timedelta(seconds=length_in_seconds)
|
|
658
|
+
info = GeneralUtilities.timedelta_to_simple_string(duration)
|
|
659
|
+
rows: int = 5
|
|
660
|
+
columns: int = math.ceil(amount_of_images/rows)
|
|
661
|
+
argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{rows}x{columns}', f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
|
|
662
|
+
self.run_program_argsasarray("montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
663
|
+
|
|
664
|
+
@GeneralUtilities.check_arguments
|
|
665
|
+
def __roundup(self, x: float, places: int) -> int:
|
|
666
|
+
d = 10 ** places
|
|
667
|
+
if x < 0:
|
|
668
|
+
return math.floor(x * d) / d
|
|
669
|
+
else:
|
|
670
|
+
return math.ceil(x * d) / d
|
|
671
|
+
|
|
672
|
+
@GeneralUtilities.check_arguments
|
|
673
|
+
def generate_thumbnail(self, file: str, frames_per_second: str, tempname_for_thumbnails: str = None, hook=None) -> None:
|
|
674
|
+
if tempname_for_thumbnails is None:
|
|
675
|
+
tempname_for_thumbnails = "t_"+str(uuid.uuid4())
|
|
676
|
+
|
|
677
|
+
file = GeneralUtilities.resolve_relative_path_from_current_working_directory(file)
|
|
678
|
+
filename = os.path.basename(file)
|
|
679
|
+
folder = os.path.dirname(file)
|
|
680
|
+
filename_without_extension = Path(file).stem
|
|
681
|
+
preview_files: list[str] = []
|
|
682
|
+
try:
|
|
683
|
+
length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
|
|
684
|
+
if (frames_per_second.endswith("fps")):
|
|
685
|
+
# frames per second, example: frames_per_second="20fps" => 20 frames per second
|
|
686
|
+
frames_per_second = self.__roundup(float(frames_per_second[:-3]), 2)
|
|
687
|
+
frames_per_second_as_string = str(frames_per_second)
|
|
688
|
+
amounf_of_previewframes = int(math.floor(length_in_seconds*frames_per_second))
|
|
689
|
+
else:
|
|
690
|
+
# concrete amount of frame, examples: frames_per_second="16" => 16 frames for entire video
|
|
691
|
+
amounf_of_previewframes = int(float(frames_per_second))
|
|
692
|
+
# self.roundup((amounf_of_previewframes-2)/length_in_seconds, 2)
|
|
693
|
+
frames_per_second_as_string = f"{amounf_of_previewframes-2}/{length_in_seconds}"
|
|
694
|
+
preview_files = self.__create_thumbnails(filename, frames_per_second_as_string, folder, tempname_for_thumbnails)
|
|
695
|
+
if hook is not None:
|
|
696
|
+
hook(file, preview_files)
|
|
697
|
+
actual_amounf_of_previewframes = len(preview_files)
|
|
698
|
+
self.__create_thumbnail(filename_without_extension, folder, length_in_seconds, tempname_for_thumbnails, actual_amounf_of_previewframes)
|
|
699
|
+
finally:
|
|
700
|
+
for thumbnail_to_delete in preview_files:
|
|
701
|
+
os.remove(thumbnail_to_delete)
|
|
702
|
+
|
|
703
|
+
@GeneralUtilities.check_arguments
|
|
704
|
+
def extract_pdf_pages(self, file: str, from_page: int, to_page: int, outputfile: str) -> None:
|
|
705
|
+
pdf_reader = PyPDF2.PdfReader(file)
|
|
706
|
+
pdf_writer = PyPDF2.PdfWriter()
|
|
707
|
+
start = from_page
|
|
708
|
+
end = to_page
|
|
709
|
+
while start <= end:
|
|
710
|
+
pdf_writer.add_page(pdf_reader.pages[start-1])
|
|
711
|
+
start += 1
|
|
712
|
+
with open(outputfile, 'wb') as out:
|
|
713
|
+
pdf_writer.write(out)
|
|
714
|
+
|
|
715
|
+
@GeneralUtilities.check_arguments
|
|
716
|
+
def merge_pdf_files(self, files: list[str], outputfile: str) -> None:
|
|
717
|
+
# TODO add wildcard-option
|
|
718
|
+
pdfFileMerger = PyPDF2.PdfFileMerger()
|
|
719
|
+
for file in files:
|
|
720
|
+
pdfFileMerger.append(file.strip())
|
|
721
|
+
pdfFileMerger.write(outputfile)
|
|
722
|
+
pdfFileMerger.close()
|
|
723
|
+
|
|
724
|
+
@GeneralUtilities.check_arguments
|
|
725
|
+
def pdf_to_image(self, file: str, outputfilename_without_extension: str) -> None:
|
|
726
|
+
raise ValueError("Function currently not available")
|
|
727
|
+
# PyMuPDF can be used for that but sometimes it throws
|
|
728
|
+
# "ImportError: DLL load failed while importing _fitz: Das angegebene Modul wurde nicht gefunden."
|
|
729
|
+
|
|
730
|
+
# doc = None # fitz.open(file)
|
|
731
|
+
# for i, page in enumerate(doc):
|
|
732
|
+
# pix = page.get_pixmap()
|
|
733
|
+
# img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
|
|
734
|
+
# img.save(f"{outputfilename_without_extension}_{i}.png", "PNG")
|
|
735
|
+
|
|
736
|
+
@GeneralUtilities.check_arguments
|
|
737
|
+
def show_missing_files(self, folderA: str, folderB: str):
|
|
738
|
+
for file in GeneralUtilities.get_missing_files(folderA, folderB):
|
|
739
|
+
GeneralUtilities.write_message_to_stdout(file)
|
|
740
|
+
|
|
741
|
+
@GeneralUtilities.check_arguments
|
|
742
|
+
def SCCreateEmptyFileWithSpecificSize(self, name: str, size_string: str) -> int:
|
|
743
|
+
if size_string.isdigit():
|
|
744
|
+
size = int(size_string)
|
|
745
|
+
else:
|
|
746
|
+
if len(size_string) >= 3:
|
|
747
|
+
if (size_string.endswith("kb")):
|
|
748
|
+
size = int(size_string[:-2]) * pow(10, 3)
|
|
749
|
+
elif (size_string.endswith("mb")):
|
|
750
|
+
size = int(size_string[:-2]) * pow(10, 6)
|
|
751
|
+
elif (size_string.endswith("gb")):
|
|
752
|
+
size = int(size_string[:-2]) * pow(10, 9)
|
|
753
|
+
elif (size_string.endswith("kib")):
|
|
754
|
+
size = int(size_string[:-3]) * pow(2, 10)
|
|
755
|
+
elif (size_string.endswith("mib")):
|
|
756
|
+
size = int(size_string[:-3]) * pow(2, 20)
|
|
757
|
+
elif (size_string.endswith("gib")):
|
|
758
|
+
size = int(size_string[:-3]) * pow(2, 30)
|
|
759
|
+
else:
|
|
760
|
+
GeneralUtilities.write_message_to_stderr("Wrong format")
|
|
761
|
+
return 1
|
|
762
|
+
else:
|
|
763
|
+
GeneralUtilities.write_message_to_stderr("Wrong format")
|
|
764
|
+
return 1
|
|
765
|
+
with open(name, "wb") as f:
|
|
766
|
+
f.seek(size-1)
|
|
767
|
+
f.write(b"\0")
|
|
768
|
+
return 0
|
|
769
|
+
|
|
770
|
+
@GeneralUtilities.check_arguments
|
|
771
|
+
def SCCreateHashOfAllFiles(self, folder: str) -> None:
|
|
772
|
+
for file in GeneralUtilities.absolute_file_paths(folder):
|
|
773
|
+
with open(file+".sha256", "w+", encoding="utf-8") as f:
|
|
774
|
+
f.write(GeneralUtilities.get_sha256_of_file(file))
|
|
775
|
+
|
|
776
|
+
@GeneralUtilities.check_arguments
|
|
777
|
+
def SCCreateSimpleMergeWithoutRelease(self, repository: str, sourcebranch: str, targetbranch: str, remotename: str, remove_source_branch: bool) -> None:
|
|
778
|
+
commitid = self.git_merge(repository, sourcebranch, targetbranch, False, True)
|
|
779
|
+
self.git_merge(repository, targetbranch, sourcebranch, True, True)
|
|
780
|
+
created_version = self.get_semver_version_from_gitversion(repository)
|
|
781
|
+
self.git_create_tag(repository, commitid, f"v{created_version}", True)
|
|
782
|
+
self.git_push(repository, remotename, targetbranch, targetbranch, False, True)
|
|
783
|
+
if (GeneralUtilities.string_has_nonwhitespace_content(remotename)):
|
|
784
|
+
self.git_push(repository, remotename, sourcebranch, sourcebranch, False, True)
|
|
785
|
+
if (remove_source_branch):
|
|
786
|
+
self.git_remove_branch(repository, sourcebranch)
|
|
787
|
+
|
|
788
|
+
@GeneralUtilities.check_arguments
|
|
789
|
+
def sc_organize_lines_in_file(self, file: str, encoding: str, sort: bool = False, remove_duplicated_lines: bool = False, ignore_first_line: bool = False, remove_empty_lines: bool = True, ignored_start_character: list = list()) -> int:
|
|
790
|
+
if os.path.isfile(file):
|
|
791
|
+
|
|
792
|
+
# read file
|
|
793
|
+
lines = GeneralUtilities.read_lines_from_file(file, encoding)
|
|
794
|
+
if (len(lines) == 0):
|
|
795
|
+
return 0
|
|
796
|
+
|
|
797
|
+
# store first line if desiredpopd
|
|
798
|
+
|
|
799
|
+
if (ignore_first_line):
|
|
800
|
+
first_line = lines.pop(0)
|
|
801
|
+
|
|
802
|
+
# remove empty lines if desired
|
|
803
|
+
if remove_empty_lines:
|
|
804
|
+
temp = lines
|
|
805
|
+
lines = []
|
|
806
|
+
for line in temp:
|
|
807
|
+
if (not (GeneralUtilities.string_is_none_or_whitespace(line))):
|
|
808
|
+
lines.append(line)
|
|
809
|
+
|
|
810
|
+
# remove duplicated lines if desired
|
|
811
|
+
if remove_duplicated_lines:
|
|
812
|
+
lines = GeneralUtilities.remove_duplicates(lines)
|
|
813
|
+
|
|
814
|
+
# sort lines if desired
|
|
815
|
+
if sort:
|
|
816
|
+
lines = sorted(lines, key=lambda singleline: self.__adapt_line_for_sorting(singleline, ignored_start_character))
|
|
817
|
+
|
|
818
|
+
# reinsert first line
|
|
819
|
+
if ignore_first_line:
|
|
820
|
+
lines.insert(0, first_line)
|
|
821
|
+
|
|
822
|
+
# write result to file
|
|
823
|
+
GeneralUtilities.write_lines_to_file(file, lines, encoding)
|
|
824
|
+
|
|
825
|
+
return 0
|
|
826
|
+
else:
|
|
827
|
+
GeneralUtilities.write_message_to_stdout(f"File '{file}' does not exist")
|
|
828
|
+
return 1
|
|
829
|
+
|
|
830
|
+
@GeneralUtilities.check_arguments
|
|
831
|
+
def __adapt_line_for_sorting(self, line: str, ignored_start_characters: list):
|
|
832
|
+
result = line.lower()
|
|
833
|
+
while len(result) > 0 and result[0] in ignored_start_characters:
|
|
834
|
+
result = result[1:]
|
|
835
|
+
return result
|
|
836
|
+
|
|
837
|
+
@GeneralUtilities.check_arguments
|
|
838
|
+
def SCGenerateSnkFiles(self, outputfolder, keysize=4096, amountofkeys=10) -> int:
|
|
839
|
+
GeneralUtilities.ensure_directory_exists(outputfolder)
|
|
840
|
+
for _ in range(amountofkeys):
|
|
841
|
+
file = os.path.join(outputfolder, str(uuid.uuid4())+".snk")
|
|
842
|
+
argument = f"-k {keysize} {file}"
|
|
843
|
+
self.run_program("sn", argument, outputfolder)
|
|
844
|
+
|
|
845
|
+
@GeneralUtilities.check_arguments
|
|
846
|
+
def __merge_files(self, sourcefile: str, targetfile: str) -> None:
|
|
847
|
+
with open(sourcefile, "rb") as f:
|
|
848
|
+
source_data = f.read()
|
|
849
|
+
with open(targetfile, "ab") as fout:
|
|
850
|
+
merge_separator = [0x0A]
|
|
851
|
+
fout.write(bytes(merge_separator))
|
|
852
|
+
fout.write(source_data)
|
|
853
|
+
|
|
854
|
+
@GeneralUtilities.check_arguments
|
|
855
|
+
def __process_file(self, file: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
|
|
856
|
+
new_filename = os.path.join(os.path.dirname(file), os.path.basename(file).replace(substringInFilename, newSubstringInFilename))
|
|
857
|
+
if file != new_filename:
|
|
858
|
+
if os.path.isfile(new_filename):
|
|
859
|
+
if filecmp.cmp(file, new_filename):
|
|
860
|
+
send2trash.send2trash(file)
|
|
861
|
+
else:
|
|
862
|
+
if conflictResolveMode == "ignore":
|
|
863
|
+
pass
|
|
864
|
+
elif conflictResolveMode == "preservenewest":
|
|
865
|
+
if (os.path.getmtime(file) - os.path.getmtime(new_filename) > 0):
|
|
866
|
+
send2trash.send2trash(file)
|
|
867
|
+
else:
|
|
868
|
+
send2trash.send2trash(new_filename)
|
|
869
|
+
os.rename(file, new_filename)
|
|
870
|
+
elif (conflictResolveMode == "merge"):
|
|
871
|
+
self.__merge_files(file, new_filename)
|
|
872
|
+
send2trash.send2trash(file)
|
|
873
|
+
else:
|
|
874
|
+
raise ValueError('Unknown conflict resolve mode')
|
|
875
|
+
else:
|
|
876
|
+
os.rename(file, new_filename)
|
|
877
|
+
|
|
878
|
+
@GeneralUtilities.check_arguments
|
|
879
|
+
def SCReplaceSubstringsInFilenames(self, folder: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
|
|
880
|
+
for file in GeneralUtilities.absolute_file_paths(folder):
|
|
881
|
+
self.__process_file(file, substringInFilename, newSubstringInFilename, conflictResolveMode)
|
|
882
|
+
|
|
883
|
+
@GeneralUtilities.check_arguments
|
|
884
|
+
def __check_file(self, file: str, searchstring: str) -> None:
|
|
885
|
+
bytes_ascii = bytes(searchstring, "ascii")
|
|
886
|
+
# often called "unicode-encoding"
|
|
887
|
+
bytes_utf16 = bytes(searchstring, "utf-16")
|
|
888
|
+
bytes_utf8 = bytes(searchstring, "utf-8")
|
|
889
|
+
with open(file, mode='rb') as file_object:
|
|
890
|
+
content = file_object.read()
|
|
891
|
+
if bytes_ascii in content:
|
|
892
|
+
GeneralUtilities.write_message_to_stdout(file)
|
|
893
|
+
elif bytes_utf16 in content:
|
|
894
|
+
GeneralUtilities.write_message_to_stdout(file)
|
|
895
|
+
elif bytes_utf8 in content:
|
|
896
|
+
GeneralUtilities.write_message_to_stdout(file)
|
|
897
|
+
|
|
898
|
+
@GeneralUtilities.check_arguments
|
|
899
|
+
def SCSearchInFiles(self, folder: str, searchstring: str) -> None:
|
|
900
|
+
for file in GeneralUtilities.absolute_file_paths(folder):
|
|
901
|
+
self.__check_file(file, searchstring)
|
|
902
|
+
|
|
903
|
+
@GeneralUtilities.check_arguments
|
|
904
|
+
def __print_qr_code_by_csv_line(self, displayname: str, website: str, emailaddress: str, key: str, period: str) -> None:
|
|
905
|
+
qrcode_content = f"otpauth://totp/{website}:{emailaddress}?secret={key}&issuer={displayname}&period={period}"
|
|
906
|
+
GeneralUtilities.write_message_to_stdout(
|
|
907
|
+
f"{displayname} ({emailaddress}):")
|
|
908
|
+
GeneralUtilities.write_message_to_stdout(qrcode_content)
|
|
909
|
+
qr = qrcode.QRCode()
|
|
910
|
+
qr.add_data(qrcode_content)
|
|
911
|
+
f = io.StringIO()
|
|
912
|
+
qr.print_ascii(out=f)
|
|
913
|
+
f.seek(0)
|
|
914
|
+
GeneralUtilities.write_message_to_stdout(f.read())
|
|
915
|
+
|
|
916
|
+
@GeneralUtilities.check_arguments
|
|
917
|
+
def SCShow2FAAsQRCode(self, csvfile: str) -> None:
|
|
918
|
+
separator_line = "--------------------------------------------------------"
|
|
919
|
+
lines = GeneralUtilities.read_csv_file(csvfile, True)
|
|
920
|
+
lines.sort(key=lambda items: ''.join(items).lower())
|
|
921
|
+
for line in lines:
|
|
922
|
+
GeneralUtilities.write_message_to_stdout(separator_line)
|
|
923
|
+
self.__print_qr_code_by_csv_line(
|
|
924
|
+
line[0], line[1], line[2], line[3], line[4])
|
|
925
|
+
GeneralUtilities.write_message_to_stdout(separator_line)
|
|
926
|
+
|
|
927
|
+
@GeneralUtilities.check_arguments
|
|
928
|
+
def SCCalculateBitcoinBlockHash(self, block_version_number: str, previousblockhash: str, transactionsmerkleroot: str, timestamp: str, target: str, nonce: str) -> str:
|
|
929
|
+
# Example-values:
|
|
930
|
+
# block_version_number: "00000020"
|
|
931
|
+
# previousblockhash: "66720b99e07d284bd4fe67ff8c49a5db1dd8514fcdab61000000000000000000"
|
|
932
|
+
# transactionsmerkleroot: "7829844f4c3a41a537b3131ca992643eaa9d093b2383e4cdc060ad7dc5481187"
|
|
933
|
+
# timestamp: "51eb505a"
|
|
934
|
+
# target: "c1910018"
|
|
935
|
+
# nonce: "de19b302"
|
|
936
|
+
header = str(block_version_number + previousblockhash + transactionsmerkleroot + timestamp + target + nonce)
|
|
937
|
+
return binascii.hexlify(hashlib.sha256(hashlib.sha256(binascii.unhexlify(header)).digest()).digest()[::-1]).decode('utf-8')
|
|
938
|
+
|
|
939
|
+
@GeneralUtilities.check_arguments
|
|
940
|
+
def SCChangeHashOfProgram(self, inputfile: str) -> None:
|
|
941
|
+
valuetoappend = str(uuid.uuid4())
|
|
942
|
+
|
|
943
|
+
outputfile = inputfile + '.modified'
|
|
944
|
+
|
|
945
|
+
shutil.copy2(inputfile, outputfile)
|
|
946
|
+
with open(outputfile, 'a', encoding="utf-8") as file:
|
|
947
|
+
# TODO use rcedit for .exe-files instead of appending valuetoappend ( https://github.com/electron/rcedit/ )
|
|
948
|
+
# background: you can retrieve the "original-filename" from the .exe-file like discussed here:
|
|
949
|
+
# https://security.stackexchange.com/questions/210843/ is-it-possible-to-change-original-filename-of-an-exe
|
|
950
|
+
# so removing the original filename with rcedit is probably a better way to make it more difficult to detect the programname.
|
|
951
|
+
# this would obviously also change the hashvalue of the program so appending a whitespace is not required anymore.
|
|
952
|
+
file.write(valuetoappend)
|
|
953
|
+
|
|
954
|
+
@GeneralUtilities.check_arguments
|
|
955
|
+
def __adjust_folder_name(self, folder: str) -> str:
|
|
956
|
+
result = os.path.dirname(folder).replace("\\", "/")
|
|
957
|
+
if result == "/":
|
|
958
|
+
return ""
|
|
959
|
+
else:
|
|
960
|
+
return result
|
|
961
|
+
|
|
962
|
+
@GeneralUtilities.check_arguments
|
|
963
|
+
def __create_iso(self, folder, iso_file) -> None:
|
|
964
|
+
created_directories = []
|
|
965
|
+
files_directory = "FILES"
|
|
966
|
+
iso = pycdlib.PyCdlib()
|
|
967
|
+
iso.new()
|
|
968
|
+
files_directory = files_directory.upper()
|
|
969
|
+
iso.add_directory("/" + files_directory)
|
|
970
|
+
created_directories.append("/" + files_directory)
|
|
971
|
+
for root, _, files in os.walk(folder):
|
|
972
|
+
for file in files:
|
|
973
|
+
full_path = os.path.join(root, file)
|
|
974
|
+
with (open(full_path, "rb").read()) as text_io_wrapper:
|
|
975
|
+
content = text_io_wrapper
|
|
976
|
+
path_in_iso = '/' + files_directory + \
|
|
977
|
+
self.__adjust_folder_name(full_path[len(folder)::1]).upper()
|
|
978
|
+
if path_in_iso not in created_directories:
|
|
979
|
+
iso.add_directory(path_in_iso)
|
|
980
|
+
created_directories.append(path_in_iso)
|
|
981
|
+
iso.add_fp(BytesIO(content), len(content), path_in_iso + '/' + file.upper() + ';1')
|
|
982
|
+
iso.write(iso_file)
|
|
983
|
+
iso.close()
|
|
984
|
+
|
|
985
|
+
@GeneralUtilities.check_arguments
|
|
986
|
+
def SCCreateISOFileWithObfuscatedFiles(self, inputfolder: str, outputfile: str, printtableheadline, createisofile, extensions) -> None:
|
|
987
|
+
if (os.path.isdir(inputfolder)):
|
|
988
|
+
namemappingfile = "name_map.csv"
|
|
989
|
+
files_directory = inputfolder
|
|
990
|
+
files_directory_obf = f"{files_directory}_Obfuscated"
|
|
991
|
+
self.SCObfuscateFilesFolder(
|
|
992
|
+
inputfolder, printtableheadline, namemappingfile, extensions)
|
|
993
|
+
os.rename(namemappingfile, os.path.join(
|
|
994
|
+
files_directory_obf, namemappingfile))
|
|
995
|
+
if createisofile:
|
|
996
|
+
self.__create_iso(files_directory_obf, outputfile)
|
|
997
|
+
shutil.rmtree(files_directory_obf)
|
|
998
|
+
else:
|
|
999
|
+
raise ValueError(f"Directory not found: '{inputfolder}'")
|
|
1000
|
+
|
|
1001
|
+
@GeneralUtilities.check_arguments
|
|
1002
|
+
def SCFilenameObfuscator(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
|
|
1003
|
+
obfuscate_all_files = extensions == "*"
|
|
1004
|
+
if (obfuscate_all_files):
|
|
1005
|
+
obfuscate_file_extensions = None
|
|
1006
|
+
else:
|
|
1007
|
+
obfuscate_file_extensions = extensions.split(",")
|
|
1008
|
+
if (os.path.isdir(inputfolder)):
|
|
1009
|
+
printtableheadline = GeneralUtilities.string_to_boolean(
|
|
1010
|
+
printtableheadline)
|
|
1011
|
+
files = []
|
|
1012
|
+
if not os.path.isfile(namemappingfile):
|
|
1013
|
+
with open(namemappingfile, "a", encoding="utf-8"):
|
|
1014
|
+
pass
|
|
1015
|
+
if printtableheadline:
|
|
1016
|
+
GeneralUtilities.append_line_to_file(
|
|
1017
|
+
namemappingfile, "Original filename;new filename;SHA2-hash of file")
|
|
1018
|
+
for file in GeneralUtilities.absolute_file_paths(inputfolder):
|
|
1019
|
+
if os.path.isfile(os.path.join(inputfolder, file)):
|
|
1020
|
+
if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
|
|
1021
|
+
files.append(file)
|
|
1022
|
+
for file in files:
|
|
1023
|
+
hash_value = GeneralUtilities.get_sha256_of_file(file)
|
|
1024
|
+
extension = Path(file).suffix
|
|
1025
|
+
new_file_name_without_path = str(uuid.uuid4())[0:8] + extension
|
|
1026
|
+
new_file_name = os.path.join(
|
|
1027
|
+
os.path.dirname(file), new_file_name_without_path)
|
|
1028
|
+
os.rename(file, new_file_name)
|
|
1029
|
+
GeneralUtilities.append_line_to_file(namemappingfile, os.path.basename(file) + ";" + new_file_name_without_path + ";" + hash_value)
|
|
1030
|
+
else:
|
|
1031
|
+
raise ValueError(f"Directory not found: '{inputfolder}'")
|
|
1032
|
+
|
|
1033
|
+
@GeneralUtilities.check_arguments
|
|
1034
|
+
def __extension_matchs(self, file: str, obfuscate_file_extensions) -> bool:
|
|
1035
|
+
for extension in obfuscate_file_extensions:
|
|
1036
|
+
if file.lower().endswith("."+extension.lower()):
|
|
1037
|
+
return True
|
|
1038
|
+
return False
|
|
1039
|
+
|
|
1040
|
+
@GeneralUtilities.check_arguments
|
|
1041
|
+
def SCHealthcheck(self, file: str) -> int:
|
|
1042
|
+
lines = GeneralUtilities.read_lines_from_file(file)
|
|
1043
|
+
for line in reversed(lines):
|
|
1044
|
+
if not GeneralUtilities.string_is_none_or_whitespace(line):
|
|
1045
|
+
if "RunningHealthy (" in line: # TODO use regex
|
|
1046
|
+
GeneralUtilities.write_message_to_stderr(f"Healthy running due to line '{line}' in file '{file}'.")
|
|
1047
|
+
return 0
|
|
1048
|
+
else:
|
|
1049
|
+
GeneralUtilities.write_message_to_stderr(f"Not healthy running due to line '{line}' in file '{file}'.")
|
|
1050
|
+
return 1
|
|
1051
|
+
GeneralUtilities.write_message_to_stderr(f"No valid line found for healthycheck in file '{file}'.")
|
|
1052
|
+
return 2
|
|
1053
|
+
|
|
1054
|
+
@GeneralUtilities.check_arguments
|
|
1055
|
+
def SCObfuscateFilesFolder(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
|
|
1056
|
+
obfuscate_all_files = extensions == "*"
|
|
1057
|
+
if (obfuscate_all_files):
|
|
1058
|
+
obfuscate_file_extensions = None
|
|
1059
|
+
else:
|
|
1060
|
+
if "," in extensions:
|
|
1061
|
+
obfuscate_file_extensions = extensions.split(",")
|
|
1062
|
+
else:
|
|
1063
|
+
obfuscate_file_extensions = [extensions]
|
|
1064
|
+
newd = inputfolder+"_Obfuscated"
|
|
1065
|
+
shutil.copytree(inputfolder, newd)
|
|
1066
|
+
inputfolder = newd
|
|
1067
|
+
if (os.path.isdir(inputfolder)):
|
|
1068
|
+
for file in GeneralUtilities.absolute_file_paths(inputfolder):
|
|
1069
|
+
if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
|
|
1070
|
+
self.SCChangeHashOfProgram(file)
|
|
1071
|
+
os.remove(file)
|
|
1072
|
+
os.rename(file + ".modified", file)
|
|
1073
|
+
self.SCFilenameObfuscator(inputfolder, printtableheadline, namemappingfile, extensions)
|
|
1074
|
+
else:
|
|
1075
|
+
raise ValueError(f"Directory not found: '{inputfolder}'")
|
|
1076
|
+
|
|
1077
|
+
@GeneralUtilities.check_arguments
|
|
1078
|
+
def get_services_from_yaml_file(self, yaml_file: str) -> list[str]:
|
|
1079
|
+
with open(yaml_file, encoding="utf-8") as stream:
|
|
1080
|
+
loaded = yaml.safe_load(stream)
|
|
1081
|
+
services = loaded["services"]
|
|
1082
|
+
result = list(services.keys())
|
|
1083
|
+
return result
|
|
1084
|
+
|
|
1085
|
+
@GeneralUtilities.check_arguments
|
|
1086
|
+
def kill_docker_container(self, container_name: str) -> None:
|
|
1087
|
+
self.run_program("docker", f"container rm -f {container_name}")
|
|
1088
|
+
|
|
1089
|
+
@GeneralUtilities.check_arguments
|
|
1090
|
+
def get_docker_debian_version(self, image_tag: str) -> str:
|
|
1091
|
+
result = ScriptCollectionCore().run_program_argsasarray("docker", ['run', f'debian:{image_tag}', 'bash', '-c', 'apt-get -y update && apt-get -y install lsb-release && lsb_release -cs'])
|
|
1092
|
+
result_line = GeneralUtilities.string_to_lines(result[1])[-1]
|
|
1093
|
+
return result_line
|
|
1094
|
+
|
|
1095
|
+
@GeneralUtilities.check_arguments
|
|
1096
|
+
def get_latest_tor_version_of_debian_repository(self, debian_version: str) -> str:
|
|
1097
|
+
package_url: str = f"https://deb.torproject.org/torproject.org/dists/{debian_version}/main/binary-amd64/Packages"
|
|
1098
|
+
headers = {'Cache-Control': 'no-cache'}
|
|
1099
|
+
r = requests.get(package_url, timeout=5, headers=headers)
|
|
1100
|
+
if r.status_code != 200:
|
|
1101
|
+
raise ValueError(f"Checking for latest tor package resulted in HTTP-response-code {r.status_code}.")
|
|
1102
|
+
lines = GeneralUtilities.string_to_lines(GeneralUtilities.bytes_to_string(r.content))
|
|
1103
|
+
version_line_prefix = "Version: "
|
|
1104
|
+
version_content_line = [line for line in lines if line.startswith(version_line_prefix)][1]
|
|
1105
|
+
version_with_overhead = version_content_line[len(version_line_prefix):]
|
|
1106
|
+
tor_version = version_with_overhead.split("~")[0]
|
|
1107
|
+
return tor_version
|
|
1108
|
+
|
|
1109
|
+
def run_testcases_for_python_project(self, repository_folder: str):
|
|
1110
|
+
self.run_program("coverage", "run -m pytest", repository_folder)
|
|
1111
|
+
self.run_program("coverage", "xml", repository_folder)
|
|
1112
|
+
GeneralUtilities.ensure_directory_exists(os.path.join(repository_folder, "Other/TestCoverage"))
|
|
1113
|
+
coveragefile = os.path.join(repository_folder, "Other/TestCoverage/TestCoverage.xml")
|
|
1114
|
+
GeneralUtilities.ensure_file_does_not_exist(coveragefile)
|
|
1115
|
+
os.rename(os.path.join(repository_folder, "coverage.xml"), coveragefile)
|
|
1116
|
+
|
|
1117
|
+
@GeneralUtilities.check_arguments
|
|
1118
|
+
def get_file_permission(self, file: str) -> str:
|
|
1119
|
+
"""This function returns an usual octet-triple, for example "700"."""
|
|
1120
|
+
ls_output: str = self.run_ls_for_folder(file)
|
|
1121
|
+
return self.__get_file_permission_helper(ls_output)
|
|
1122
|
+
|
|
1123
|
+
@GeneralUtilities.check_arguments
|
|
1124
|
+
def __get_file_permission_helper(self, permissions: str) -> str:
|
|
1125
|
+
return str(self.__to_octet(permissions[0:3])) + str(self.__to_octet(permissions[3:6]))+str(self.__to_octet(permissions[6:9]))
|
|
1126
|
+
|
|
1127
|
+
@GeneralUtilities.check_arguments
|
|
1128
|
+
def __to_octet(self, string: str) -> int:
|
|
1129
|
+
return int(self.__to_octet_helper(string[0]) + self.__to_octet_helper(string[1])+self.__to_octet_helper(string[2]), 2)
|
|
1130
|
+
|
|
1131
|
+
@GeneralUtilities.check_arguments
|
|
1132
|
+
def __to_octet_helper(self, string: str) -> str:
|
|
1133
|
+
if (string == "-"):
|
|
1134
|
+
return "0"
|
|
1135
|
+
else:
|
|
1136
|
+
return "1"
|
|
1137
|
+
|
|
1138
|
+
@GeneralUtilities.check_arguments
|
|
1139
|
+
def get_file_owner(self, file: str) -> str:
|
|
1140
|
+
"""This function returns the user and the group in the format "user:group"."""
|
|
1141
|
+
ls_output: str = self.run_ls_for_folder(file)
|
|
1142
|
+
return self.__get_file_owner_helper(ls_output)
|
|
1143
|
+
|
|
1144
|
+
@GeneralUtilities.check_arguments
|
|
1145
|
+
def __get_file_owner_helper(self, ls_output: str) -> str:
|
|
1146
|
+
splitted = ls_output.split()
|
|
1147
|
+
return f"{splitted[2]}:{splitted[3]}"
|
|
1148
|
+
|
|
1149
|
+
@GeneralUtilities.check_arguments
|
|
1150
|
+
def get_file_owner_and_file_permission(self, file: str) -> str:
|
|
1151
|
+
ls_output: str = self.run_ls_for_folder(file)
|
|
1152
|
+
return [self.__get_file_owner_helper(ls_output), self.__get_file_permission_helper(ls_output)]
|
|
1153
|
+
|
|
1154
|
+
@GeneralUtilities.check_arguments
|
|
1155
|
+
def run_ls_for_folder(self, file_or_folder: str) -> str:
|
|
1156
|
+
file_or_folder = file_or_folder.replace("\\", "/")
|
|
1157
|
+
GeneralUtilities.assert_condition(os.path.isfile(file_or_folder) or os.path.isdir(file_or_folder), f"Can not execute 'ls -ld' because '{file_or_folder}' does not exist.")
|
|
1158
|
+
ls_result = self.run_program_argsasarray("ls", ["-ld", file_or_folder])
|
|
1159
|
+
GeneralUtilities.assert_condition(ls_result[0] == 0, f"'ls -ld {file_or_folder}' resulted in exitcode {str(ls_result[0])}. StdErr: {ls_result[2]}")
|
|
1160
|
+
GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(ls_result[1]), f"'ls -ld' of '{file_or_folder}' had an empty output. StdErr: '{ls_result[2]}'")
|
|
1161
|
+
GeneralUtilities.write_message_to_stdout(ls_result[1])
|
|
1162
|
+
output = ls_result[1]
|
|
1163
|
+
result = output.replace("\n", "")
|
|
1164
|
+
result = ' '.join(result.split()) # reduce multiple whitespaces to one
|
|
1165
|
+
return result
|
|
1166
|
+
|
|
1167
|
+
@GeneralUtilities.check_arguments
|
|
1168
|
+
def run_ls_for_folder_content(self, file_or_folder: str) -> list[str]:
|
|
1169
|
+
file_or_folder = file_or_folder.replace("\\", "/")
|
|
1170
|
+
GeneralUtilities.assert_condition(os.path.isfile(file_or_folder) or os.path.isdir(file_or_folder), f"Can not execute 'ls -la' because '{file_or_folder}' does not exist.")
|
|
1171
|
+
ls_result = self.run_program_argsasarray("ls", ["-la", file_or_folder])
|
|
1172
|
+
GeneralUtilities.assert_condition(ls_result[0] == 0, f"'ls -la {file_or_folder}' resulted in exitcode {str(ls_result[0])}. StdErr: {ls_result[2]}")
|
|
1173
|
+
GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(ls_result[1]), f"'ls -la' of '{file_or_folder}' had an empty output. StdErr: '{ls_result[2]}'")
|
|
1174
|
+
GeneralUtilities.write_message_to_stdout(ls_result[1])
|
|
1175
|
+
output = ls_result[1]
|
|
1176
|
+
result = output.split("\n")[3:] # skip the lines with "Total", "." and ".."
|
|
1177
|
+
result = [' '.join(line.split()) for line in result] # reduce multiple whitespaces to one
|
|
1178
|
+
return result
|
|
1179
|
+
|
|
1180
|
+
@GeneralUtilities.check_arguments
|
|
1181
|
+
def set_permission(self, file_or_folder: str, permissions: str, recursive: bool = False) -> None:
|
|
1182
|
+
"""This function expects an usual octet-triple, for example "700"."""
|
|
1183
|
+
args = []
|
|
1184
|
+
if recursive:
|
|
1185
|
+
args.append("--recursive")
|
|
1186
|
+
args.append(permissions)
|
|
1187
|
+
args.append(file_or_folder)
|
|
1188
|
+
self.run_program_argsasarray("chmod", args)
|
|
1189
|
+
|
|
1190
|
+
@GeneralUtilities.check_arguments
|
|
1191
|
+
def set_owner(self, file_or_folder: str, owner: str, recursive: bool = False, follow_symlinks: bool = False) -> None:
|
|
1192
|
+
"""This function expects the user and the group in the format "user:group"."""
|
|
1193
|
+
args = []
|
|
1194
|
+
if recursive:
|
|
1195
|
+
args.append("--recursive")
|
|
1196
|
+
if follow_symlinks:
|
|
1197
|
+
args.append("--no-dereference")
|
|
1198
|
+
args.append(owner)
|
|
1199
|
+
args.append(file_or_folder)
|
|
1200
|
+
self.run_program_argsasarray("chown", args)
|
|
1201
|
+
|
|
1202
|
+
# <run programs>
|
|
1203
|
+
|
|
1204
|
+
@GeneralUtilities.check_arguments
|
|
1205
|
+
def __run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> Popen:
|
|
1206
|
+
# Verbosity:
|
|
1207
|
+
# 0=Quiet (No output will be printed.)
|
|
1208
|
+
# 1=Normal (If the exitcode of the executed program is not 0 then the StdErr will be printed.)
|
|
1209
|
+
# 2=Full (Prints StdOut and StdErr of the executed program.)
|
|
1210
|
+
# 3=Verbose (Same as "Full" but with some more information.)
|
|
1211
|
+
|
|
1212
|
+
if isinstance(self.program_runner, ProgramRunnerEpew):
|
|
1213
|
+
custom_argument = CustomEpewArgument(print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, verbosity, arguments_for_log)
|
|
1214
|
+
popen: Popen = self.program_runner.run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, custom_argument, interactive)
|
|
1215
|
+
return popen
|
|
1216
|
+
|
|
1217
|
+
@staticmethod
|
|
1218
|
+
def __enqueue_output(file, queue):
|
|
1219
|
+
for line in iter(file.readline, ''):
|
|
1220
|
+
queue.put(line)
|
|
1221
|
+
file.close()
|
|
1222
|
+
|
|
1223
|
+
@staticmethod
|
|
1224
|
+
def __continue_process_reading(pid: int, p: Popen, q_stdout: Queue, q_stderr: Queue, reading_stdout_last_time_resulted_in_exception: bool, reading_stderr_last_time_resulted_in_exception: bool):
|
|
1225
|
+
if p.poll() is None:
|
|
1226
|
+
return True
|
|
1227
|
+
|
|
1228
|
+
# if reading_stdout_last_time_resulted_in_exception and reading_stderr_last_time_resulted_in_exception:
|
|
1229
|
+
# return False
|
|
1230
|
+
|
|
1231
|
+
if not q_stdout.empty():
|
|
1232
|
+
return True
|
|
1233
|
+
|
|
1234
|
+
if not q_stderr.empty():
|
|
1235
|
+
return True
|
|
1236
|
+
|
|
1237
|
+
return False
|
|
1238
|
+
|
|
1239
|
+
@staticmethod
|
|
1240
|
+
def __read_popen_pipes(p: Popen):
|
|
1241
|
+
p_id = p.pid
|
|
1242
|
+
with ThreadPoolExecutor(2) as pool:
|
|
1243
|
+
q_stdout = Queue()
|
|
1244
|
+
q_stderr = Queue()
|
|
1245
|
+
|
|
1246
|
+
pool.submit(ScriptCollectionCore.__enqueue_output, p.stdout, q_stdout)
|
|
1247
|
+
pool.submit(ScriptCollectionCore.__enqueue_output, p.stderr, q_stderr)
|
|
1248
|
+
reading_stdout_last_time_resulted_in_exception: bool = False
|
|
1249
|
+
reading_stderr_last_time_resulted_in_exception: bool = False
|
|
1250
|
+
while (ScriptCollectionCore.__continue_process_reading(p_id, p, q_stdout, q_stderr, reading_stdout_last_time_resulted_in_exception, reading_stderr_last_time_resulted_in_exception)):
|
|
1251
|
+
out_line = None
|
|
1252
|
+
err_line = None
|
|
1253
|
+
try:
|
|
1254
|
+
out_line = q_stdout.get_nowait() # TODO read all avaliable lines
|
|
1255
|
+
reading_stdout_last_time_resulted_in_exception = False
|
|
1256
|
+
except Empty:
|
|
1257
|
+
reading_stdout_last_time_resulted_in_exception = True
|
|
1258
|
+
|
|
1259
|
+
try:
|
|
1260
|
+
err_line = q_stderr.get_nowait() # TODO read all avaliable lines
|
|
1261
|
+
reading_stderr_last_time_resulted_in_exception = False
|
|
1262
|
+
except Empty:
|
|
1263
|
+
reading_stderr_last_time_resulted_in_exception = True
|
|
1264
|
+
|
|
1265
|
+
time.sleep(0.01) # this is required to not finish too early
|
|
1266
|
+
|
|
1267
|
+
yield (out_line, err_line)
|
|
1268
|
+
|
|
1269
|
+
# Return-values program_runner: Exitcode, StdOut, StdErr, Pid
|
|
1270
|
+
@GeneralUtilities.check_arguments
|
|
1271
|
+
def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False) -> tuple[int, str, str, int]:
|
|
1272
|
+
# verbosity 1: No output will be logged.
|
|
1273
|
+
# verbosity 2: If the exitcode of the executed program is not 0 then the StdErr will be logged. This is supposed to be the default verbosity-level.
|
|
1274
|
+
# verbosity 3: Logs and prints StdOut and StdErr of the executed program in realtime.
|
|
1275
|
+
# verbosity 4: Same as loglevel 3 but with some more overhead-information.
|
|
1276
|
+
if self.call_program_runner_directly:
|
|
1277
|
+
return self.program_runner.run_program_argsasarray(program, arguments_as_array, working_directory, custom_argument, interactive)
|
|
1278
|
+
try:
|
|
1279
|
+
arguments_as_str = ' '.join(arguments_as_array)
|
|
1280
|
+
mock_loader_result = self.__try_load_mock(program, arguments_as_str, working_directory)
|
|
1281
|
+
if mock_loader_result[0]:
|
|
1282
|
+
return mock_loader_result[1]
|
|
1283
|
+
|
|
1284
|
+
working_directory = self.__adapt_workingdirectory(working_directory)
|
|
1285
|
+
|
|
1286
|
+
if arguments_for_log is None:
|
|
1287
|
+
arguments_for_log = arguments_as_array
|
|
1288
|
+
|
|
1289
|
+
arguments_for_log_as_string: str = ' '.join(arguments_for_log)
|
|
1290
|
+
cmd = f'{working_directory}>{program} {arguments_for_log_as_string}'
|
|
1291
|
+
|
|
1292
|
+
if GeneralUtilities.string_is_none_or_whitespace(title):
|
|
1293
|
+
info_for_log = cmd
|
|
1294
|
+
else:
|
|
1295
|
+
info_for_log = title
|
|
1296
|
+
|
|
1297
|
+
if verbosity >= 3:
|
|
1298
|
+
GeneralUtilities.write_message_to_stdout(f"Run '{info_for_log}'.")
|
|
1299
|
+
|
|
1300
|
+
print_live_output = 1 < verbosity
|
|
1301
|
+
|
|
1302
|
+
exit_code: int = None
|
|
1303
|
+
stdout: str = ""
|
|
1304
|
+
stderr: str = ""
|
|
1305
|
+
pid: int = None
|
|
1306
|
+
|
|
1307
|
+
with self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive) as process:
|
|
1308
|
+
|
|
1309
|
+
if log_file is not None:
|
|
1310
|
+
GeneralUtilities.ensure_file_exists(log_file)
|
|
1311
|
+
pid = process.pid
|
|
1312
|
+
for out_line_plain, err_line_plain in ScriptCollectionCore.__read_popen_pipes(process): # see https://stackoverflow.com/a/57084403/3905529
|
|
1313
|
+
|
|
1314
|
+
if out_line_plain is not None:
|
|
1315
|
+
out_line: str = None
|
|
1316
|
+
if isinstance(out_line_plain, str):
|
|
1317
|
+
out_line = out_line_plain
|
|
1318
|
+
elif isinstance(out_line_plain, bytes):
|
|
1319
|
+
out_line = GeneralUtilities.bytes_to_string(out_line_plain)
|
|
1320
|
+
else:
|
|
1321
|
+
raise ValueError(f"Unknown type of output: {str(type(out_line_plain))}")
|
|
1322
|
+
|
|
1323
|
+
if out_line is not None and GeneralUtilities.string_has_content(out_line):
|
|
1324
|
+
if out_line.endswith("\n"):
|
|
1325
|
+
out_line = out_line[:-1]
|
|
1326
|
+
if print_live_output:
|
|
1327
|
+
print(out_line, end='\n', file=sys.stdout, flush=True)
|
|
1328
|
+
if 0 < len(stdout):
|
|
1329
|
+
stdout = stdout+"\n"
|
|
1330
|
+
stdout = stdout+out_line
|
|
1331
|
+
if log_file is not None:
|
|
1332
|
+
GeneralUtilities.append_line_to_file(log_file, out_line)
|
|
1333
|
+
|
|
1334
|
+
if err_line_plain is not None:
|
|
1335
|
+
err_line: str = None
|
|
1336
|
+
if isinstance(err_line_plain, str):
|
|
1337
|
+
err_line = err_line_plain
|
|
1338
|
+
elif isinstance(err_line_plain, bytes):
|
|
1339
|
+
err_line = GeneralUtilities.bytes_to_string(err_line_plain)
|
|
1340
|
+
else:
|
|
1341
|
+
raise ValueError(f"Unknown type of output: {str(type(err_line_plain))}")
|
|
1342
|
+
if err_line is not None and GeneralUtilities.string_has_content(err_line):
|
|
1343
|
+
if err_line.endswith("\n"):
|
|
1344
|
+
err_line = err_line[:-1]
|
|
1345
|
+
if print_live_output:
|
|
1346
|
+
print(err_line, end='\n', file=sys.stderr, flush=True)
|
|
1347
|
+
if 0 < len(stderr):
|
|
1348
|
+
stderr = stderr+"\n"
|
|
1349
|
+
stderr = stderr+err_line
|
|
1350
|
+
if log_file is not None:
|
|
1351
|
+
GeneralUtilities.append_line_to_file(log_file, err_line)
|
|
1352
|
+
|
|
1353
|
+
exit_code = process.returncode
|
|
1354
|
+
|
|
1355
|
+
if throw_exception_if_exitcode_is_not_zero and exit_code != 0:
|
|
1356
|
+
raise ValueError(f"Program '{working_directory}>{program} {arguments_for_log_as_string}' resulted in exitcode {exit_code}. (StdOut: '{stdout}', StdErr: '{stderr}')")
|
|
1357
|
+
|
|
1358
|
+
GeneralUtilities.assert_condition(exit_code is not None, f"Exitcode of program-run of '{info_for_log}' is None.")
|
|
1359
|
+
result = (exit_code, stdout, stderr, pid)
|
|
1360
|
+
return result
|
|
1361
|
+
except Exception as e:
|
|
1362
|
+
raise e
|
|
1363
|
+
|
|
1364
|
+
# Return-values program_runner: Exitcode, StdOut, StdErr, Pid
|
|
1365
|
+
@GeneralUtilities.check_arguments
|
|
1366
|
+
def run_program(self, program: str, arguments: str = "", working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False) -> tuple[int, str, str, int]:
|
|
1367
|
+
if self.call_program_runner_directly:
|
|
1368
|
+
return self.program_runner.run_program(program, arguments, working_directory, custom_argument, interactive)
|
|
1369
|
+
return self.run_program_argsasarray(program, GeneralUtilities.arguments_to_array(arguments), working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive)
|
|
1370
|
+
|
|
1371
|
+
# Return-values program_runner: Pid
|
|
1372
|
+
@GeneralUtilities.check_arguments
|
|
1373
|
+
def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
|
|
1374
|
+
if self.call_program_runner_directly:
|
|
1375
|
+
return self.program_runner.run_program_argsasarray_async(program, arguments_as_array, working_directory, custom_argument, interactive)
|
|
1376
|
+
mock_loader_result = self.__try_load_mock(program, ' '.join(arguments_as_array), working_directory)
|
|
1377
|
+
if mock_loader_result[0]:
|
|
1378
|
+
return mock_loader_result[1]
|
|
1379
|
+
process: Popen = self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
|
|
1380
|
+
return process.pid
|
|
1381
|
+
|
|
1382
|
+
# Return-values program_runner: Pid
|
|
1383
|
+
@GeneralUtilities.check_arguments
|
|
1384
|
+
def run_program_async(self, program: str, arguments: str = "", working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
|
|
1385
|
+
if self.call_program_runner_directly:
|
|
1386
|
+
return self.program_runner.run_program_argsasarray_async(program, arguments, working_directory, custom_argument, interactive)
|
|
1387
|
+
return self.run_program_argsasarray_async(program, GeneralUtilities.arguments_to_array(arguments), working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
|
|
1388
|
+
|
|
1389
|
+
@GeneralUtilities.check_arguments
|
|
1390
|
+
def __try_load_mock(self, program: str, arguments: str, working_directory: str) -> tuple[bool, tuple[int, str, str, int]]:
|
|
1391
|
+
if self.mock_program_calls:
|
|
1392
|
+
try:
|
|
1393
|
+
return [True, self.__get_mock_program_call(program, arguments, working_directory)]
|
|
1394
|
+
except LookupError:
|
|
1395
|
+
if not self.execute_program_really_if_no_mock_call_is_defined:
|
|
1396
|
+
raise
|
|
1397
|
+
return [False, None]
|
|
1398
|
+
|
|
1399
|
+
@GeneralUtilities.check_arguments
|
|
1400
|
+
def __adapt_workingdirectory(self, workingdirectory: str) -> str:
|
|
1401
|
+
if workingdirectory is None:
|
|
1402
|
+
return os.getcwd()
|
|
1403
|
+
else:
|
|
1404
|
+
return GeneralUtilities.resolve_relative_path_from_current_working_directory(workingdirectory)
|
|
1405
|
+
|
|
1406
|
+
@GeneralUtilities.check_arguments
|
|
1407
|
+
def verify_no_pending_mock_program_calls(self):
|
|
1408
|
+
if (len(self.__mocked_program_calls) > 0):
|
|
1409
|
+
raise AssertionError("The following mock-calls were not called:\n"+",\n ".join([self.__format_mock_program_call(r) for r in self.__mocked_program_calls]))
|
|
1410
|
+
|
|
1411
|
+
@GeneralUtilities.check_arguments
|
|
1412
|
+
def __format_mock_program_call(self, r) -> str:
|
|
1413
|
+
r: ScriptCollectionCore.__MockProgramCall = r
|
|
1414
|
+
return f"'{r.workingdirectory}>{r.program} {r.argument}' (" \
|
|
1415
|
+
f"exitcode: {GeneralUtilities.str_none_safe(str(r.exit_code))}, " \
|
|
1416
|
+
f"pid: {GeneralUtilities.str_none_safe(str(r.pid))}, "\
|
|
1417
|
+
f"stdout: {GeneralUtilities.str_none_safe(str(r.stdout))}, " \
|
|
1418
|
+
f"stderr: {GeneralUtilities.str_none_safe(str(r.stderr))})"
|
|
1419
|
+
|
|
1420
|
+
@GeneralUtilities.check_arguments
|
|
1421
|
+
def register_mock_program_call(self, program: str, argument: str, workingdirectory: str, result_exit_code: int, result_stdout: str, result_stderr: str, result_pid: int, amount_of_expected_calls=1):
|
|
1422
|
+
"This function is for test-purposes only"
|
|
1423
|
+
for _ in itertools.repeat(None, amount_of_expected_calls):
|
|
1424
|
+
mock_call = ScriptCollectionCore.__MockProgramCall()
|
|
1425
|
+
mock_call.program = program
|
|
1426
|
+
mock_call.argument = argument
|
|
1427
|
+
mock_call.workingdirectory = workingdirectory
|
|
1428
|
+
mock_call.exit_code = result_exit_code
|
|
1429
|
+
mock_call.stdout = result_stdout
|
|
1430
|
+
mock_call.stderr = result_stderr
|
|
1431
|
+
mock_call.pid = result_pid
|
|
1432
|
+
self.__mocked_program_calls.append(mock_call)
|
|
1433
|
+
|
|
1434
|
+
@GeneralUtilities.check_arguments
|
|
1435
|
+
def __get_mock_program_call(self, program: str, argument: str, workingdirectory: str):
|
|
1436
|
+
result: ScriptCollectionCore.__MockProgramCall = None
|
|
1437
|
+
for mock_call in self.__mocked_program_calls:
|
|
1438
|
+
if ((re.match(mock_call.program, program) is not None)
|
|
1439
|
+
and (re.match(mock_call.argument, argument) is not None)
|
|
1440
|
+
and (re.match(mock_call.workingdirectory, workingdirectory) is not None)):
|
|
1441
|
+
result = mock_call
|
|
1442
|
+
break
|
|
1443
|
+
if result is None:
|
|
1444
|
+
raise LookupError(f"Tried to execute mock-call '{workingdirectory}>{program} {argument}' but no mock-call was defined for that execution")
|
|
1445
|
+
else:
|
|
1446
|
+
self.__mocked_program_calls.remove(result)
|
|
1447
|
+
return (result.exit_code, result.stdout, result.stderr, result.pid)
|
|
1448
|
+
|
|
1449
|
+
@GeneralUtilities.check_arguments
|
|
1450
|
+
class __MockProgramCall:
|
|
1451
|
+
program: str
|
|
1452
|
+
argument: str
|
|
1453
|
+
workingdirectory: str
|
|
1454
|
+
exit_code: int
|
|
1455
|
+
stdout: str
|
|
1456
|
+
stderr: str
|
|
1457
|
+
pid: int
|
|
1458
|
+
|
|
1459
|
+
# </run programs>
|
|
1460
|
+
|
|
1461
|
+
@GeneralUtilities.check_arguments
|
|
1462
|
+
def extract_archive_with_7z(self, unzip_program_file: str, zipfile: str, password: str, output_directory: str) -> None:
|
|
1463
|
+
password_set = not password is None
|
|
1464
|
+
file_name = Path(zipfile).name
|
|
1465
|
+
file_folder = os.path.dirname(zipfile)
|
|
1466
|
+
argument = "x"
|
|
1467
|
+
if password_set:
|
|
1468
|
+
argument = f"{argument} -p\"{password}\""
|
|
1469
|
+
argument = f"{argument} -o {output_directory}"
|
|
1470
|
+
argument = f"{argument} {file_name}"
|
|
1471
|
+
return self.run_program(unzip_program_file, argument, file_folder)
|
|
1472
|
+
|
|
1473
|
+
@GeneralUtilities.check_arguments
|
|
1474
|
+
def get_internet_time(self) -> datetime:
|
|
1475
|
+
response = ntplib.NTPClient().request('pool.ntp.org')
|
|
1476
|
+
return datetime.fromtimestamp(response.tx_time)
|
|
1477
|
+
|
|
1478
|
+
@GeneralUtilities.check_arguments
|
|
1479
|
+
def system_time_equals_internet_time(self, maximal_tolerance_difference: timedelta) -> bool:
|
|
1480
|
+
return abs(datetime.now() - self.get_internet_time()) < maximal_tolerance_difference
|
|
1481
|
+
|
|
1482
|
+
@GeneralUtilities.check_arguments
|
|
1483
|
+
def system_time_equals_internet_time_with_default_tolerance(self) -> bool:
|
|
1484
|
+
return self.system_time_equals_internet_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
|
|
1485
|
+
|
|
1486
|
+
@GeneralUtilities.check_arguments
|
|
1487
|
+
def check_system_time(self, maximal_tolerance_difference: timedelta):
|
|
1488
|
+
if not self.system_time_equals_internet_time(maximal_tolerance_difference):
|
|
1489
|
+
raise ValueError("System time may be wrong")
|
|
1490
|
+
|
|
1491
|
+
@GeneralUtilities.check_arguments
|
|
1492
|
+
def check_system_time_with_default_tolerance(self) -> None:
|
|
1493
|
+
self.check_system_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
|
|
1494
|
+
|
|
1495
|
+
@GeneralUtilities.check_arguments
|
|
1496
|
+
def __get_default_tolerance_for_system_time_equals_internet_time(self) -> timedelta:
|
|
1497
|
+
return timedelta(hours=0, minutes=0, seconds=3)
|
|
1498
|
+
|
|
1499
|
+
@GeneralUtilities.check_arguments
|
|
1500
|
+
def increment_version(self, input_version: str, increment_major: bool, increment_minor: bool, increment_patch: bool) -> str:
|
|
1501
|
+
splitted = input_version.split(".")
|
|
1502
|
+
GeneralUtilities.assert_condition(len(splitted) == 3, f"Version '{input_version}' does not have the 'major.minor.patch'-pattern.")
|
|
1503
|
+
major = int(splitted[0])
|
|
1504
|
+
minor = int(splitted[1])
|
|
1505
|
+
patch = int(splitted[2])
|
|
1506
|
+
if increment_major:
|
|
1507
|
+
major = major+1
|
|
1508
|
+
if increment_minor:
|
|
1509
|
+
minor = minor+1
|
|
1510
|
+
if increment_patch:
|
|
1511
|
+
patch = patch+1
|
|
1512
|
+
return f"{major}.{minor}.{patch}"
|
|
1513
|
+
|
|
1514
|
+
@GeneralUtilities.check_arguments
|
|
1515
|
+
def get_semver_version_from_gitversion(self, repository_folder: str) -> str:
|
|
1516
|
+
if (self.git_repository_has_commits(repository_folder)):
|
|
1517
|
+
result = self.get_version_from_gitversion(repository_folder, "MajorMinorPatch")
|
|
1518
|
+
if self.git_repository_has_uncommitted_changes(repository_folder):
|
|
1519
|
+
if self.get_current_git_branch_has_tag(repository_folder):
|
|
1520
|
+
id_of_latest_tag = self.git_get_commitid_of_tag(repository_folder, self.get_latest_git_tag(repository_folder))
|
|
1521
|
+
current_commit = self.git_get_commit_id(repository_folder)
|
|
1522
|
+
current_commit_is_on_latest_tag = id_of_latest_tag == current_commit
|
|
1523
|
+
if current_commit_is_on_latest_tag:
|
|
1524
|
+
result = self.increment_version(result, False, False, True)
|
|
1525
|
+
else:
|
|
1526
|
+
result = "0.1.0"
|
|
1527
|
+
return result
|
|
1528
|
+
|
|
1529
|
+
@staticmethod
|
|
1530
|
+
@GeneralUtilities.check_arguments
|
|
1531
|
+
def is_patch_version(version_string: str) -> bool:
|
|
1532
|
+
return not version_string.endswith(".0")
|
|
1533
|
+
|
|
1534
|
+
@GeneralUtilities.check_arguments
|
|
1535
|
+
def get_version_from_gitversion(self, folder: str, variable: str) -> str:
|
|
1536
|
+
# called twice as workaround for issue 1877 in gitversion ( https://github.com/GitTools/GitVersion/issues/1877 )
|
|
1537
|
+
result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder, verbosity=0)
|
|
1538
|
+
result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder, verbosity=0)
|
|
1539
|
+
result = GeneralUtilities.strip_new_line_character(result[1])
|
|
1540
|
+
|
|
1541
|
+
return result
|
|
1542
|
+
|
|
1543
|
+
@GeneralUtilities.check_arguments
|
|
1544
|
+
def generate_certificate_authority(self, folder: str, name: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
|
|
1545
|
+
if days_until_expire is None:
|
|
1546
|
+
days_until_expire = 1825
|
|
1547
|
+
if password is None:
|
|
1548
|
+
password = GeneralUtilities.generate_password()
|
|
1549
|
+
GeneralUtilities.ensure_directory_exists(folder)
|
|
1550
|
+
self.run_program("openssl", f'req -new -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -days {days_until_expire} -nodes -x509 -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={name}/OU={subj_ou} -passout pass:{password} -keyout {name}.key -out {name}.crt', folder)
|
|
1551
|
+
|
|
1552
|
+
@GeneralUtilities.check_arguments
|
|
1553
|
+
def generate_certificate(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
|
|
1554
|
+
if days_until_expire is None:
|
|
1555
|
+
days_until_expire = 397
|
|
1556
|
+
if password is None:
|
|
1557
|
+
password = GeneralUtilities.generate_password()
|
|
1558
|
+
rsa_key_length = 4096
|
|
1559
|
+
self.run_program("openssl", f'genrsa -out {filename}.key {rsa_key_length}', folder)
|
|
1560
|
+
self.run_program("openssl", f'req -new -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou} -x509 -key {filename}.key -out {filename}.unsigned.crt -days {days_until_expire}', folder)
|
|
1561
|
+
self.run_program("openssl", f'pkcs12 -export -out {filename}.selfsigned.pfx -password pass:{password} -inkey {filename}.key -in {filename}.unsigned.crt', folder)
|
|
1562
|
+
GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.password"), password)
|
|
1563
|
+
GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.san.conf"), f"""[ req ]
|
|
1564
|
+
default_bits = {rsa_key_length}
|
|
1565
|
+
distinguished_name = req_distinguished_name
|
|
1566
|
+
req_extensions = v3_req
|
|
1567
|
+
default_md = sha256
|
|
1568
|
+
dirstring_type = nombstr
|
|
1569
|
+
prompt = no
|
|
1570
|
+
|
|
1571
|
+
[ req_distinguished_name ]
|
|
1572
|
+
countryName = {subj_c}
|
|
1573
|
+
stateOrProvinceName = {subj_st}
|
|
1574
|
+
localityName = {subj_l}
|
|
1575
|
+
organizationName = {subj_o}
|
|
1576
|
+
organizationUnit = {subj_ou}
|
|
1577
|
+
commonName = {domain}
|
|
1578
|
+
|
|
1579
|
+
[v3_req]
|
|
1580
|
+
subjectAltName = @subject_alt_name
|
|
1581
|
+
|
|
1582
|
+
[ subject_alt_name ]
|
|
1583
|
+
DNS = {domain}
|
|
1584
|
+
""")
|
|
1585
|
+
|
|
1586
|
+
@GeneralUtilities.check_arguments
|
|
1587
|
+
def generate_certificate_sign_request(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str) -> None:
|
|
1588
|
+
self.run_program("openssl", f'req -new -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou} -key {filename}.key -out {filename}.csr -config {filename}.san.conf', folder)
|
|
1589
|
+
|
|
1590
|
+
@GeneralUtilities.check_arguments
|
|
1591
|
+
def sign_certificate(self, folder: str, ca_folder: str, ca_name: str, domain: str, filename: str, days_until_expire: int = None) -> None:
|
|
1592
|
+
if days_until_expire is None:
|
|
1593
|
+
days_until_expire = 397
|
|
1594
|
+
ca = os.path.join(ca_folder, ca_name)
|
|
1595
|
+
password_file = os.path.join(folder, f"{filename}.password")
|
|
1596
|
+
password = GeneralUtilities.read_text_from_file(password_file)
|
|
1597
|
+
self.run_program("openssl", f'x509 -req -in {filename}.csr -CA {ca}.crt -CAkey {ca}.key -CAcreateserial -CAserial {ca}.srl -out {filename}.crt -days {days_until_expire} -sha256 -extensions v3_req -extfile {filename}.san.conf', folder)
|
|
1598
|
+
self.run_program("openssl", f'pkcs12 -export -out {filename}.pfx -inkey {filename}.key -in {filename}.crt -password pass:{password}', folder)
|
|
1599
|
+
|
|
1600
|
+
@GeneralUtilities.check_arguments
|
|
1601
|
+
def update_dependencies_of_python_in_requirementstxt_file(self, file: str, verbosity: int):
|
|
1602
|
+
lines = GeneralUtilities.read_lines_from_file(file)
|
|
1603
|
+
new_lines = []
|
|
1604
|
+
for line in lines:
|
|
1605
|
+
if GeneralUtilities.string_has_content(line):
|
|
1606
|
+
new_lines.append(self.__get_updated_line_for_python_requirements(line.strip()))
|
|
1607
|
+
GeneralUtilities.write_lines_to_file(file, new_lines)
|
|
1608
|
+
|
|
1609
|
+
@GeneralUtilities.check_arguments
|
|
1610
|
+
def __get_updated_line_for_python_requirements(self, line: str) -> str:
|
|
1611
|
+
if "==" in line or "<" in line:
|
|
1612
|
+
return line
|
|
1613
|
+
elif ">" in line:
|
|
1614
|
+
try:
|
|
1615
|
+
# line is something like "cyclonedx-bom>=2.0.2" and the function must return with the updated version
|
|
1616
|
+
# (something like "cyclonedx-bom>=2.11.0" for example)
|
|
1617
|
+
package = line.split(">")[0]
|
|
1618
|
+
operator = ">=" if ">=" in line else ">"
|
|
1619
|
+
headers = {'Cache-Control': 'no-cache'}
|
|
1620
|
+
response = requests.get(f'https://pypi.org/pypi/{package}/json', timeout=5, headers=headers)
|
|
1621
|
+
latest_version = response.json()['info']['version']
|
|
1622
|
+
# TODO update only minor- and patch-version
|
|
1623
|
+
# TODO print info if there is a new major-version
|
|
1624
|
+
return package+operator+latest_version
|
|
1625
|
+
except:
|
|
1626
|
+
return line
|
|
1627
|
+
else:
|
|
1628
|
+
raise ValueError(f'Unexpected line in requirements-file: "{line}"')
|
|
1629
|
+
|
|
1630
|
+
@GeneralUtilities.check_arguments
|
|
1631
|
+
def update_dependencies_of_python_in_setupcfg_file(self, setup_cfg_file: str, verbosity: int):
|
|
1632
|
+
lines = GeneralUtilities.read_lines_from_file(setup_cfg_file)
|
|
1633
|
+
new_lines = []
|
|
1634
|
+
requirement_parsing_mode = False
|
|
1635
|
+
for line in lines:
|
|
1636
|
+
new_line = line
|
|
1637
|
+
if (requirement_parsing_mode):
|
|
1638
|
+
if ("<" in line or "=" in line or ">" in line):
|
|
1639
|
+
updated_line = f" {self.__get_updated_line_for_python_requirements(line.strip())}"
|
|
1640
|
+
new_line = updated_line
|
|
1641
|
+
else:
|
|
1642
|
+
requirement_parsing_mode = False
|
|
1643
|
+
else:
|
|
1644
|
+
if line.startswith("install_requires ="):
|
|
1645
|
+
requirement_parsing_mode = True
|
|
1646
|
+
new_lines.append(new_line)
|
|
1647
|
+
GeneralUtilities.write_lines_to_file(setup_cfg_file, new_lines)
|
|
1648
|
+
|
|
1649
|
+
@GeneralUtilities.check_arguments
|
|
1650
|
+
def update_dependencies_of_dotnet_project(self, csproj_file: str, verbosity: int, ignored_dependencies: list[str]):
|
|
1651
|
+
folder = os.path.dirname(csproj_file)
|
|
1652
|
+
csproj_filename = os.path.basename(csproj_file)
|
|
1653
|
+
GeneralUtilities.write_message_to_stderr(f"Check for updates in {csproj_filename}")
|
|
1654
|
+
result = self.run_program("dotnet", f"list {csproj_filename} package --outdated", folder)
|
|
1655
|
+
for line in result[1].replace("\r", "").split("\n"):
|
|
1656
|
+
# Relevant output-lines are something like " > NJsonSchema 10.7.0 10.7.0 10.9.0"
|
|
1657
|
+
if ">" in line:
|
|
1658
|
+
package_name = line.replace(">", "").strip().split(" ")[0]
|
|
1659
|
+
if not (package_name in ignored_dependencies):
|
|
1660
|
+
GeneralUtilities.write_message_to_stderr(f"Update package {package_name}")
|
|
1661
|
+
self.run_program("dotnet", f"add {csproj_filename} package {package_name}", folder)
|
|
1662
|
+
|
|
1663
|
+
@GeneralUtilities.check_arguments
|
|
1664
|
+
def create_deb_package(self, toolname: str, binary_folder: str, control_file_content: str, deb_output_folder: str, verbosity: int, permission_of_executable_file_as_octet_triple: int) -> None:
|
|
1665
|
+
|
|
1666
|
+
# prepare
|
|
1667
|
+
GeneralUtilities.ensure_directory_exists(deb_output_folder)
|
|
1668
|
+
temp_folder = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
|
|
1669
|
+
GeneralUtilities.ensure_directory_exists(temp_folder)
|
|
1670
|
+
bin_folder = binary_folder
|
|
1671
|
+
tool_content_folder_name = toolname+"Content"
|
|
1672
|
+
|
|
1673
|
+
# create folder
|
|
1674
|
+
GeneralUtilities.ensure_directory_exists(temp_folder)
|
|
1675
|
+
control_content_folder_name = "controlcontent"
|
|
1676
|
+
packagecontent_control_folder = os.path.join(temp_folder, control_content_folder_name)
|
|
1677
|
+
GeneralUtilities.ensure_directory_exists(packagecontent_control_folder)
|
|
1678
|
+
data_content_folder_name = "datacontent"
|
|
1679
|
+
packagecontent_data_folder = os.path.join(temp_folder, data_content_folder_name)
|
|
1680
|
+
GeneralUtilities.ensure_directory_exists(packagecontent_data_folder)
|
|
1681
|
+
entireresult_content_folder_name = "entireresultcontent"
|
|
1682
|
+
packagecontent_entireresult_folder = os.path.join(temp_folder, entireresult_content_folder_name)
|
|
1683
|
+
GeneralUtilities.ensure_directory_exists(packagecontent_entireresult_folder)
|
|
1684
|
+
|
|
1685
|
+
# create "debian-binary"-file
|
|
1686
|
+
debianbinary_file = os.path.join(packagecontent_entireresult_folder, "debian-binary")
|
|
1687
|
+
GeneralUtilities.ensure_file_exists(debianbinary_file)
|
|
1688
|
+
GeneralUtilities.write_text_to_file(debianbinary_file, "2.0\n")
|
|
1689
|
+
|
|
1690
|
+
# create control-content
|
|
1691
|
+
|
|
1692
|
+
# conffiles
|
|
1693
|
+
conffiles_file = os.path.join(packagecontent_control_folder, "conffiles")
|
|
1694
|
+
GeneralUtilities.ensure_file_exists(conffiles_file)
|
|
1695
|
+
|
|
1696
|
+
# postinst-script
|
|
1697
|
+
postinst_file = os.path.join(packagecontent_control_folder, "postinst")
|
|
1698
|
+
GeneralUtilities.ensure_file_exists(postinst_file)
|
|
1699
|
+
exe_file = f"/usr/bin/{tool_content_folder_name}/{toolname}"
|
|
1700
|
+
link_file = f"/usr/bin/{toolname.lower()}"
|
|
1701
|
+
permission = str(permission_of_executable_file_as_octet_triple)
|
|
1702
|
+
GeneralUtilities.write_text_to_file(postinst_file, f"""#!/bin/sh
|
|
1703
|
+
ln -s {exe_file} {link_file}
|
|
1704
|
+
chmod {permission} {exe_file}
|
|
1705
|
+
chmod {permission} {link_file}
|
|
1706
|
+
""")
|
|
1707
|
+
|
|
1708
|
+
# control
|
|
1709
|
+
control_file = os.path.join(packagecontent_control_folder, "control")
|
|
1710
|
+
GeneralUtilities.ensure_file_exists(control_file)
|
|
1711
|
+
GeneralUtilities.write_text_to_file(control_file, control_file_content)
|
|
1712
|
+
|
|
1713
|
+
# md5sums
|
|
1714
|
+
md5sums_file = os.path.join(packagecontent_control_folder, "md5sums")
|
|
1715
|
+
GeneralUtilities.ensure_file_exists(md5sums_file)
|
|
1716
|
+
|
|
1717
|
+
# create data-content
|
|
1718
|
+
|
|
1719
|
+
# copy binaries
|
|
1720
|
+
usr_bin_folder = os.path.join(packagecontent_data_folder, "usr/bin")
|
|
1721
|
+
GeneralUtilities.ensure_directory_exists(usr_bin_folder)
|
|
1722
|
+
usr_bin_content_folder = os.path.join(usr_bin_folder, tool_content_folder_name)
|
|
1723
|
+
GeneralUtilities.copy_content_of_folder(bin_folder, usr_bin_content_folder)
|
|
1724
|
+
|
|
1725
|
+
# create debfile
|
|
1726
|
+
deb_filename = f"{toolname}.deb"
|
|
1727
|
+
self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/control.tar.gz", "*"], packagecontent_control_folder, verbosity=verbosity)
|
|
1728
|
+
self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/data.tar.gz", "*"], packagecontent_data_folder, verbosity=verbosity)
|
|
1729
|
+
self.run_program_argsasarray("ar", ["r", deb_filename, "debian-binary", "control.tar.gz", "data.tar.gz"], packagecontent_entireresult_folder, verbosity=verbosity)
|
|
1730
|
+
result_file = os.path.join(packagecontent_entireresult_folder, deb_filename)
|
|
1731
|
+
shutil.copy(result_file, os.path.join(deb_output_folder, deb_filename))
|
|
1732
|
+
|
|
1733
|
+
# cleanup
|
|
1734
|
+
GeneralUtilities.ensure_directory_does_not_exist(temp_folder)
|
|
1735
|
+
|
|
1736
|
+
@GeneralUtilities.check_arguments
|
|
1737
|
+
def update_year_in_copyright_tags(self, file: str) -> None:
|
|
1738
|
+
current_year = str(datetime.now().year)
|
|
1739
|
+
lines = GeneralUtilities.read_lines_from_file(file)
|
|
1740
|
+
lines_result = []
|
|
1741
|
+
for line in lines:
|
|
1742
|
+
if match := re.search("(.*<[Cc]opyright>.*)\\d\\d\\d\\d(.*<\\/[Cc]opyright>.*)", line):
|
|
1743
|
+
part1 = match.group(1)
|
|
1744
|
+
part2 = match.group(2)
|
|
1745
|
+
adapted = part1+current_year+part2
|
|
1746
|
+
else:
|
|
1747
|
+
adapted = line
|
|
1748
|
+
lines_result.append(adapted)
|
|
1749
|
+
GeneralUtilities.write_lines_to_file(file, lines_result)
|
|
1750
|
+
|
|
1751
|
+
@GeneralUtilities.check_arguments
|
|
1752
|
+
def update_year_in_first_line_of_file(self, file: str) -> None:
|
|
1753
|
+
current_year = str(datetime.now().year)
|
|
1754
|
+
lines = GeneralUtilities.read_lines_from_file(file)
|
|
1755
|
+
lines[0] = re.sub("\\d\\d\\d\\d", current_year, lines[0])
|
|
1756
|
+
GeneralUtilities.write_lines_to_file(file, lines)
|
|
1757
|
+
|
|
1758
|
+
@GeneralUtilities.check_arguments
|
|
1759
|
+
def get_external_ip(self) -> str:
|
|
1760
|
+
information = self.get_externalnetworkinformation_as_json_string()
|
|
1761
|
+
parsed = json.loads(information)
|
|
1762
|
+
return parsed.IPAddress
|
|
1763
|
+
|
|
1764
|
+
@GeneralUtilities.check_arguments
|
|
1765
|
+
def get_country_of_external_ip(self) -> str:
|
|
1766
|
+
information = self.get_externalnetworkinformation_as_json_string()
|
|
1767
|
+
parsed = json.loads(information)
|
|
1768
|
+
return parsed.Country
|
|
1769
|
+
|
|
1770
|
+
@GeneralUtilities.check_arguments
|
|
1771
|
+
def get_externalnetworkinformation_as_json_string(self) -> str:
|
|
1772
|
+
headers = {'Cache-Control': 'no-cache'}
|
|
1773
|
+
response = requests.get('https://clientinformation.anion327.de/API/v1/ClientInformationBackendController/Information', timeout=5, headers=headers)
|
|
1774
|
+
network_information_as_json_string = GeneralUtilities.bytes_to_string(response.content)
|
|
1775
|
+
return network_information_as_json_string
|
|
1776
|
+
|
|
1777
|
+
@GeneralUtilities.check_arguments
|
|
1778
|
+
def change_file_extensions(self, folder: str, from_extension: str, to_extension: str, recursive: bool, ignore_case: bool) -> None:
|
|
1779
|
+
extension_to_compare: str = None
|
|
1780
|
+
if ignore_case:
|
|
1781
|
+
extension_to_compare = from_extension.lower()
|
|
1782
|
+
else:
|
|
1783
|
+
extension_to_compare = from_extension
|
|
1784
|
+
for file in GeneralUtilities.get_direct_files_of_folder(folder):
|
|
1785
|
+
if (ignore_case and file.lower().endswith(f".{extension_to_compare}") or not ignore_case and file.endswith(f".{extension_to_compare}")):
|
|
1786
|
+
p = Path(file)
|
|
1787
|
+
p.rename(p.with_suffix('.'+to_extension))
|
|
1788
|
+
if recursive:
|
|
1789
|
+
for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
|
|
1790
|
+
self.change_file_extensions(subfolder, from_extension, to_extension, recursive, ignore_case)
|
|
1791
|
+
|
|
1792
|
+
@GeneralUtilities.check_arguments
|
|
1793
|
+
def __add_chapter(self, main_reference_file, reference_content_folder, number: int, chaptertitle: str, content: str = None):
|
|
1794
|
+
if content is None:
|
|
1795
|
+
content = "TXDX add content here"
|
|
1796
|
+
filename = str(number).zfill(2)+"_"+chaptertitle.replace(' ', '-')
|
|
1797
|
+
file = f"{reference_content_folder}/{filename}.md"
|
|
1798
|
+
full_title = f"{number}. {chaptertitle}"
|
|
1799
|
+
|
|
1800
|
+
GeneralUtilities.append_line_to_file(main_reference_file, f"- [{full_title}](./{filename}.md)")
|
|
1801
|
+
|
|
1802
|
+
GeneralUtilities.ensure_file_exists(file)
|
|
1803
|
+
GeneralUtilities.write_text_to_file(file, f"""# {full_title}
|
|
1804
|
+
|
|
1805
|
+
{content}
|
|
1806
|
+
""".replace("XDX", "ODO"))
|
|
1807
|
+
|
|
1808
|
+
@GeneralUtilities.check_arguments
|
|
1809
|
+
def generate_arc42_reference_template(self, repository: str, productname: str = None, subfolder: str = None):
|
|
1810
|
+
productname: str
|
|
1811
|
+
if productname is None:
|
|
1812
|
+
productname = os.path.basename(repository)
|
|
1813
|
+
if subfolder is None:
|
|
1814
|
+
subfolder = "Other/Resources/Reference"
|
|
1815
|
+
reference_root_folder = f"{repository}/{subfolder}"
|
|
1816
|
+
reference_content_folder = reference_root_folder + "/Technical"
|
|
1817
|
+
if os.path.isdir(reference_root_folder):
|
|
1818
|
+
raise ValueError(f"The folder '{reference_root_folder}' does already exist.")
|
|
1819
|
+
GeneralUtilities.ensure_directory_exists(reference_root_folder)
|
|
1820
|
+
GeneralUtilities.ensure_directory_exists(reference_content_folder)
|
|
1821
|
+
main_reference_file = f"{reference_root_folder}/Reference.md"
|
|
1822
|
+
GeneralUtilities.ensure_file_exists(main_reference_file)
|
|
1823
|
+
GeneralUtilities.write_text_to_file(main_reference_file, f"""# {productname}
|
|
1824
|
+
|
|
1825
|
+
TXDX add minimal service-description here.
|
|
1826
|
+
|
|
1827
|
+
## Technical documentation
|
|
1828
|
+
|
|
1829
|
+
""".replace("XDX", "ODO"))
|
|
1830
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 1, 'Introduction and Goals', """## Overview
|
|
1831
|
+
|
|
1832
|
+
TXDX
|
|
1833
|
+
|
|
1834
|
+
## Quality goals
|
|
1835
|
+
|
|
1836
|
+
TXDX
|
|
1837
|
+
|
|
1838
|
+
## Stakeholder
|
|
1839
|
+
|
|
1840
|
+
| Name | How to contact | Reason |
|
|
1841
|
+
| ---- | -------------- | ------ |""")
|
|
1842
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 2, 'Constraints', """## Technical constraints
|
|
1843
|
+
|
|
1844
|
+
| Constraint-identifier | Constraint | Reason |
|
|
1845
|
+
| --------------------- | ---------- | ------ |
|
|
1846
|
+
|
|
1847
|
+
## Organizational constraints
|
|
1848
|
+
|
|
1849
|
+
| Constraint-identifier | Constraint | Reason |
|
|
1850
|
+
| --------------------- | ---------- | ------ |""")
|
|
1851
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 3, 'Context and Scope', """## Context
|
|
1852
|
+
|
|
1853
|
+
TXDX
|
|
1854
|
+
|
|
1855
|
+
## Scope
|
|
1856
|
+
|
|
1857
|
+
TXDX""")
|
|
1858
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 4, 'Solution Strategy', """TXDX""")
|
|
1859
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 5, 'Building Block View', """TXDX""")
|
|
1860
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 6, 'Runtime View', """TXDX""")
|
|
1861
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 7, 'Deployment View', """## Infrastructure-overview
|
|
1862
|
+
|
|
1863
|
+
TXDX
|
|
1864
|
+
|
|
1865
|
+
## Infrastructure-requirements
|
|
1866
|
+
|
|
1867
|
+
TXDX
|
|
1868
|
+
|
|
1869
|
+
## Deployment-proecsses
|
|
1870
|
+
|
|
1871
|
+
TXDX
|
|
1872
|
+
""")
|
|
1873
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 8, 'Crosscutting Concepts', """TXDX""")
|
|
1874
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 9, 'Architectural Decisions', """## Decision-board
|
|
1875
|
+
|
|
1876
|
+
| Decision-identifier | Date | Decision | Reason and notes |
|
|
1877
|
+
| ------------------- | ---- | -------- | ---------------- |""") # empty because there are no decsions yet
|
|
1878
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 10, 'Quality Requirements', """TXDX""")
|
|
1879
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 11, 'Risks and Technical Debt', """## Risks
|
|
1880
|
+
|
|
1881
|
+
Currently there are no known risks.
|
|
1882
|
+
|
|
1883
|
+
## Technical debts
|
|
1884
|
+
|
|
1885
|
+
Currently there are no technical depts.""")
|
|
1886
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 12, 'Glossary', """## Terms
|
|
1887
|
+
|
|
1888
|
+
| Term | Meaning |
|
|
1889
|
+
| ---- | ------- |
|
|
1890
|
+
|
|
1891
|
+
## Abbreviations
|
|
1892
|
+
|
|
1893
|
+
| Abbreviation | Meaning |
|
|
1894
|
+
| ------------ | ------- |""")
|
|
1895
|
+
|
|
1896
|
+
GeneralUtilities.append_to_file(main_reference_file, """
|
|
1897
|
+
|
|
1898
|
+
## Responsibilities
|
|
1899
|
+
|
|
1900
|
+
| Responsibility | Name and contact-information |
|
|
1901
|
+
| --------------- | ---------------------------- |
|
|
1902
|
+
| Pdocut-owner | TXDX |
|
|
1903
|
+
| Product-manager | TXDX |
|
|
1904
|
+
| Support | TXDX |
|
|
1905
|
+
|
|
1906
|
+
## License & Pricing
|
|
1907
|
+
|
|
1908
|
+
TXDX
|
|
1909
|
+
|
|
1910
|
+
## External resources
|
|
1911
|
+
|
|
1912
|
+
- [Repository](TXDX)
|
|
1913
|
+
- [Productive-System](TXDX)
|
|
1914
|
+
- [QualityCheck-system](TXDX)
|
|
1915
|
+
|
|
1916
|
+
""".replace("XDX", "ODO"))
|
|
1917
|
+
|
|
1918
|
+
@GeneralUtilities.check_arguments
|
|
1919
|
+
def run_with_timeout(self, method, timeout_in_seconds: float) -> bool:
|
|
1920
|
+
# Returns true if the method was terminated due to a timeout
|
|
1921
|
+
# Returns false if the method terminates in the given time
|
|
1922
|
+
p = multiprocessing.Process(target=method)
|
|
1923
|
+
p.start()
|
|
1924
|
+
p.join(timeout_in_seconds)
|
|
1925
|
+
if p.is_alive():
|
|
1926
|
+
p.kill()
|
|
1927
|
+
p.join()
|
|
1928
|
+
return True
|
|
1929
|
+
else:
|
|
1930
|
+
return False
|