ScriptCollection 3.5.16__py3-none-any.whl → 4.0.78__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/AnionBuildPlatform.py +206 -0
- ScriptCollection/{UpdateCertificates.py → CertificateUpdater.py} +69 -46
- ScriptCollection/Executables.py +515 -18
- ScriptCollection/GeneralUtilities.py +1272 -873
- ScriptCollection/ImageUpdater.py +648 -0
- ScriptCollection/ProgramRunnerBase.py +10 -10
- ScriptCollection/ProgramRunnerMock.py +2 -0
- ScriptCollection/ProgramRunnerPopen.py +7 -1
- ScriptCollection/ProgramRunnerSudo.py +108 -0
- ScriptCollection/SCLog.py +115 -0
- ScriptCollection/ScriptCollectionCore.py +942 -266
- ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +95 -0
- ScriptCollection/TFCPS/Docker/__init__.py +0 -0
- ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py +8 -0
- ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py +6 -0
- ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py +7 -0
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +485 -0
- ScriptCollection/TFCPS/DotNet/__init__.py +0 -0
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +130 -0
- ScriptCollection/TFCPS/Flutter/__init__.py +0 -0
- ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py +74 -0
- ScriptCollection/TFCPS/Go/__init__.py +0 -0
- ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +131 -0
- ScriptCollection/TFCPS/NodeJS/__init__.py +0 -0
- ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +227 -0
- ScriptCollection/TFCPS/Python/__init__.py +0 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +418 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +128 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +136 -0
- ScriptCollection/TFCPS/TFCPS_CreateRelease.py +95 -0
- ScriptCollection/TFCPS/TFCPS_Generic.py +43 -0
- ScriptCollection/TFCPS/TFCPS_MergeToMain.py +122 -0
- ScriptCollection/TFCPS/TFCPS_MergeToStable.py +350 -0
- ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +47 -0
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +1356 -0
- ScriptCollection/TFCPS/__init__.py +0 -0
- {ScriptCollection-3.5.16.dist-info → scriptcollection-4.0.78.dist-info}/METADATA +23 -22
- scriptcollection-4.0.78.dist-info/RECORD +43 -0
- {ScriptCollection-3.5.16.dist-info → scriptcollection-4.0.78.dist-info}/WHEEL +1 -1
- {ScriptCollection-3.5.16.dist-info → scriptcollection-4.0.78.dist-info}/entry_points.txt +32 -0
- ScriptCollection/ProgramRunnerEpew.py +0 -122
- ScriptCollection/RPStream.py +0 -42
- ScriptCollection/TasksForCommonProjectStructure.py +0 -2625
- ScriptCollection-3.5.16.dist-info/RECORD +0 -16
- {ScriptCollection-3.5.16.dist-info → scriptcollection-4.0.78.dist-info}/top_level.txt +0 -0
|
@@ -1,20 +1,25 @@
|
|
|
1
|
-
import sys
|
|
2
1
|
from datetime import timedelta, datetime
|
|
3
2
|
import json
|
|
4
3
|
import binascii
|
|
5
4
|
import filecmp
|
|
6
5
|
import hashlib
|
|
6
|
+
import multiprocessing
|
|
7
7
|
import time
|
|
8
8
|
from io import BytesIO
|
|
9
9
|
import itertools
|
|
10
|
+
import zipfile
|
|
10
11
|
import math
|
|
12
|
+
import base64
|
|
11
13
|
import os
|
|
12
14
|
from queue import Queue, Empty
|
|
13
15
|
from concurrent.futures import ThreadPoolExecutor
|
|
16
|
+
import xml.etree.ElementTree as ET
|
|
14
17
|
from pathlib import Path
|
|
15
18
|
from subprocess import Popen
|
|
16
19
|
import re
|
|
17
20
|
import shutil
|
|
21
|
+
from typing import IO
|
|
22
|
+
import fnmatch
|
|
18
23
|
import uuid
|
|
19
24
|
import tempfile
|
|
20
25
|
import io
|
|
@@ -24,13 +29,13 @@ import yaml
|
|
|
24
29
|
import qrcode
|
|
25
30
|
import pycdlib
|
|
26
31
|
import send2trash
|
|
27
|
-
import
|
|
32
|
+
from pypdf import PdfReader, PdfWriter
|
|
28
33
|
from .GeneralUtilities import GeneralUtilities
|
|
29
34
|
from .ProgramRunnerBase import ProgramRunnerBase
|
|
30
35
|
from .ProgramRunnerPopen import ProgramRunnerPopen
|
|
31
|
-
from .
|
|
36
|
+
from .SCLog import SCLog, LogLevel
|
|
32
37
|
|
|
33
|
-
version = "
|
|
38
|
+
version = "4.0.78"
|
|
34
39
|
__version__ = version
|
|
35
40
|
|
|
36
41
|
|
|
@@ -38,15 +43,19 @@ class ScriptCollectionCore:
|
|
|
38
43
|
|
|
39
44
|
# The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
|
|
40
45
|
# Do not change this value for productive environments.
|
|
41
|
-
mock_program_calls: bool = False
|
|
46
|
+
mock_program_calls: bool = False#TODO remove this variable. When someone want to mock program-calls then the ProgramRunnerMock can be used instead
|
|
42
47
|
# The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
|
|
43
48
|
execute_program_really_if_no_mock_call_is_defined: bool = False
|
|
44
49
|
__mocked_program_calls: list = None
|
|
45
50
|
program_runner: ProgramRunnerBase = None
|
|
51
|
+
call_program_runner_directly: bool = None
|
|
52
|
+
log: SCLog = None
|
|
46
53
|
|
|
47
54
|
def __init__(self):
|
|
48
55
|
self.program_runner = ProgramRunnerPopen()
|
|
56
|
+
self.call_program_runner_directly = None
|
|
49
57
|
self.__mocked_program_calls = list[ScriptCollectionCore.__MockProgramCall]()
|
|
58
|
+
self.log = SCLog(None, LogLevel.Warning, False)
|
|
50
59
|
|
|
51
60
|
@staticmethod
|
|
52
61
|
@GeneralUtilities.check_arguments
|
|
@@ -58,14 +67,14 @@ class ScriptCollectionCore:
|
|
|
58
67
|
errors = list()
|
|
59
68
|
filename = os.path.relpath(file, working_directory)
|
|
60
69
|
if treat_warnings_as_errors:
|
|
61
|
-
errorsonly_argument =
|
|
70
|
+
errorsonly_argument = GeneralUtilities.empty_string
|
|
62
71
|
else:
|
|
63
72
|
errorsonly_argument = " --errors-only"
|
|
64
73
|
(exit_code, stdout, stderr, _) = self.run_program("pylint", filename + errorsonly_argument, working_directory, throw_exception_if_exitcode_is_not_zero=False)
|
|
65
74
|
if (exit_code != 0):
|
|
66
75
|
errors.append(f"Linting-issues of {file}:")
|
|
67
76
|
errors.append(f"Pylint-exitcode: {exit_code}")
|
|
68
|
-
for line in GeneralUtilities.string_to_lines(stdout):
|
|
77
|
+
for line in GeneralUtilities.string_to_lines(stdout):
|
|
69
78
|
errors.append(line)
|
|
70
79
|
for line in GeneralUtilities.string_to_lines(stderr):
|
|
71
80
|
errors.append(line)
|
|
@@ -108,31 +117,48 @@ class ScriptCollectionCore:
|
|
|
108
117
|
raise ValueError(f"Version '{current_version}' does not match version-regex '{versiononlyregex}'")
|
|
109
118
|
|
|
110
119
|
@GeneralUtilities.check_arguments
|
|
111
|
-
def push_nuget_build_artifact(self, nupkg_file: str, registry_address: str, api_key: str
|
|
120
|
+
def push_nuget_build_artifact(self, nupkg_file: str, registry_address: str, api_key: str = None):
|
|
112
121
|
nupkg_file_name = os.path.basename(nupkg_file)
|
|
113
122
|
nupkg_file_folder = os.path.dirname(nupkg_file)
|
|
114
|
-
|
|
123
|
+
argument = f"nuget push {nupkg_file_name} --force-english-output --source {registry_address}"
|
|
124
|
+
if api_key is not None:
|
|
125
|
+
argument = f"{argument} --api-key {api_key}"
|
|
126
|
+
self.run_program("dotnet", argument, nupkg_file_folder)
|
|
115
127
|
|
|
116
128
|
@GeneralUtilities.check_arguments
|
|
117
|
-
def dotnet_build(self,
|
|
118
|
-
self.run_program("dotnet", f"clean -c {configuration}",
|
|
119
|
-
self.run_program("dotnet", f"build {projectname}/{projectname}.csproj -c {configuration}",
|
|
129
|
+
def dotnet_build(self, folder: str, projectname: str, configuration: str):
|
|
130
|
+
self.run_program("dotnet", f"clean -c {configuration}", folder)
|
|
131
|
+
self.run_program("dotnet", f"build {projectname}/{projectname}.csproj -c {configuration}", folder)
|
|
120
132
|
|
|
121
133
|
@GeneralUtilities.check_arguments
|
|
122
|
-
def find_file_by_extension(self, folder: str,
|
|
123
|
-
|
|
134
|
+
def find_file_by_extension(self, folder: str, extension_without_dot: str):
|
|
135
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
136
|
+
result = [file for file in self.list_content(folder, True, False, False) if file.endswith(f".{extension_without_dot}")]
|
|
124
137
|
result_length = len(result)
|
|
125
138
|
if result_length == 0:
|
|
126
|
-
raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{
|
|
139
|
+
raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension_without_dot}'.")
|
|
127
140
|
if result_length == 1:
|
|
128
141
|
return result[0]
|
|
129
142
|
else:
|
|
130
|
-
raise ValueError(f"Multiple values available in folder '{folder}' with extension '{
|
|
143
|
+
raise ValueError(f"Multiple values available in folder '{folder}' with extension '{extension_without_dot}'.")
|
|
144
|
+
|
|
145
|
+
@GeneralUtilities.check_arguments
|
|
146
|
+
def find_last_file_by_extension(self, folder: str, extension_without_dot: str) -> str:
|
|
147
|
+
files: list[str] = GeneralUtilities.get_direct_files_of_folder(folder)
|
|
148
|
+
possible_results: list[str] = []
|
|
149
|
+
for file in files:
|
|
150
|
+
if file.endswith(f".{extension_without_dot}"):
|
|
151
|
+
possible_results.append(file)
|
|
152
|
+
result_length = len(possible_results)
|
|
153
|
+
if result_length == 0:
|
|
154
|
+
raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension_without_dot}'.")
|
|
155
|
+
else:
|
|
156
|
+
return possible_results[-1]
|
|
131
157
|
|
|
132
158
|
@GeneralUtilities.check_arguments
|
|
133
159
|
def commit_is_signed_by_key(self, repository_folder: str, revision_identifier: str, key: str) -> bool:
|
|
134
|
-
|
|
135
|
-
|
|
160
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
161
|
+
result = self.run_program("git", f"verify-commit {revision_identifier}", repository_folder, throw_exception_if_exitcode_is_not_zero=False)
|
|
136
162
|
if (result[0] != 0):
|
|
137
163
|
return False
|
|
138
164
|
if (not GeneralUtilities.contains_line(result[1].splitlines(), f"gpg\\:\\ using\\ [A-Za-z0-9]+\\ key\\ [A-Za-z0-9]+{key}")):
|
|
@@ -145,35 +171,18 @@ class ScriptCollectionCore:
|
|
|
145
171
|
|
|
146
172
|
@GeneralUtilities.check_arguments
|
|
147
173
|
def get_parent_commit_ids_of_commit(self, repository_folder: str, commit_id: str) -> str:
|
|
148
|
-
|
|
174
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
175
|
+
return self.run_program("git", f'log --pretty=%P -n 1 "{commit_id}"', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string).split(" ")
|
|
149
176
|
|
|
150
|
-
@GeneralUtilities.check_arguments
|
|
151
|
-
def get_all_authors_and_committers_of_repository(self, repository_folder: str, subfolder: str = None, verbosity: int = 1) -> list[tuple[str, str]]:
|
|
152
|
-
space_character = "_"
|
|
153
|
-
if subfolder is None:
|
|
154
|
-
subfolder_argument = ""
|
|
155
|
-
else:
|
|
156
|
-
subfolder_argument = f" -- {subfolder}"
|
|
157
|
-
log_result = self.run_program("git", f'log --pretty=%aN{space_character}%aE%n%cN{space_character}%cE HEAD{subfolder_argument}', repository_folder, verbosity=0)
|
|
158
|
-
plain_content: list[str] = list(
|
|
159
|
-
set([line for line in log_result[1].split("\n") if len(line) > 0]))
|
|
160
|
-
result: list[tuple[str, str]] = []
|
|
161
|
-
for item in plain_content:
|
|
162
|
-
if len(re.findall(space_character, item)) == 1:
|
|
163
|
-
splitted = item.split(space_character)
|
|
164
|
-
result.append((splitted[0], splitted[1]))
|
|
165
|
-
else:
|
|
166
|
-
raise ValueError(f'Unexpected author: "{item}"')
|
|
167
|
-
return result
|
|
168
177
|
|
|
169
178
|
@GeneralUtilities.check_arguments
|
|
170
179
|
def get_commit_ids_between_dates(self, repository_folder: str, since: datetime, until: datetime, ignore_commits_which_are_not_in_history_of_head: bool = True) -> None:
|
|
180
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
171
181
|
since_as_string = self.__datetime_to_string_for_git(since)
|
|
172
182
|
until_as_string = self.__datetime_to_string_for_git(until)
|
|
173
|
-
result = filter(lambda line: not GeneralUtilities.string_is_none_or_whitespace(line), self.run_program("git", f'log --since "{since_as_string}" --until "{until_as_string}" --pretty=format:"%H" --no-patch', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].split("\n").replace("\r",
|
|
183
|
+
result = filter(lambda line: not GeneralUtilities.string_is_none_or_whitespace(line), self.run_program("git", f'log --since "{since_as_string}" --until "{until_as_string}" --pretty=format:"%H" --no-patch', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].split("\n").replace("\r", GeneralUtilities.empty_string))
|
|
174
184
|
if ignore_commits_which_are_not_in_history_of_head:
|
|
175
|
-
result = [commit_id for commit_id in result if self.git_commit_is_ancestor(
|
|
176
|
-
repository_folder, commit_id)]
|
|
185
|
+
result = [commit_id for commit_id in result if self.git_commit_is_ancestor(repository_folder, commit_id)]
|
|
177
186
|
return result
|
|
178
187
|
|
|
179
188
|
@GeneralUtilities.check_arguments
|
|
@@ -182,44 +191,52 @@ class ScriptCollectionCore:
|
|
|
182
191
|
|
|
183
192
|
@GeneralUtilities.check_arguments
|
|
184
193
|
def git_commit_is_ancestor(self, repository_folder: str, ancestor: str, descendant: str = "HEAD") -> bool:
|
|
185
|
-
|
|
194
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
195
|
+
result = self.run_program_argsasarray("git", ["merge-base", "--is-ancestor", ancestor, descendant], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
|
|
196
|
+
exit_code = result[0]
|
|
186
197
|
if exit_code == 0:
|
|
187
198
|
return True
|
|
188
199
|
elif exit_code == 1:
|
|
189
200
|
return False
|
|
190
201
|
else:
|
|
191
|
-
raise ValueError(f
|
|
202
|
+
raise ValueError(f'Can not calculate if {ancestor} is an ancestor of {descendant} in repository {repository_folder}. Outout of "{repository_folder}> git merge-base --is-ancestor {ancestor} {descendant}": Exitcode: {exit_code}; StdOut: {result[1]}; StdErr: {result[2]}.')
|
|
192
203
|
|
|
193
204
|
@GeneralUtilities.check_arguments
|
|
194
205
|
def __git_changes_helper(self, repository_folder: str, arguments_as_array: list[str]) -> bool:
|
|
195
|
-
|
|
206
|
+
self.assert_is_git_repository(repository_folder)
|
|
207
|
+
lines = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", arguments_as_array, repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1], False)
|
|
196
208
|
for line in lines:
|
|
197
209
|
if GeneralUtilities.string_has_content(line):
|
|
198
210
|
return True
|
|
199
211
|
return False
|
|
200
212
|
|
|
201
213
|
@GeneralUtilities.check_arguments
|
|
202
|
-
def git_repository_has_new_untracked_files(self,
|
|
203
|
-
|
|
214
|
+
def git_repository_has_new_untracked_files(self, repository_folder: str):
|
|
215
|
+
self.assert_is_git_repository(repository_folder)
|
|
216
|
+
return self.__git_changes_helper(repository_folder, ["ls-files", "--exclude-standard", "--others"])
|
|
204
217
|
|
|
205
218
|
@GeneralUtilities.check_arguments
|
|
206
|
-
def git_repository_has_unstaged_changes_of_tracked_files(self,
|
|
207
|
-
|
|
219
|
+
def git_repository_has_unstaged_changes_of_tracked_files(self, repository_folder: str):
|
|
220
|
+
self.assert_is_git_repository(repository_folder)
|
|
221
|
+
return self.__git_changes_helper(repository_folder, ["--no-pager", "diff"])
|
|
208
222
|
|
|
209
223
|
@GeneralUtilities.check_arguments
|
|
210
|
-
def git_repository_has_staged_changes(self,
|
|
211
|
-
|
|
224
|
+
def git_repository_has_staged_changes(self, repository_folder: str):
|
|
225
|
+
self.assert_is_git_repository(repository_folder)
|
|
226
|
+
return self.__git_changes_helper(repository_folder, ["--no-pager", "diff", "--cached"])
|
|
212
227
|
|
|
213
228
|
@GeneralUtilities.check_arguments
|
|
214
|
-
def git_repository_has_uncommitted_changes(self,
|
|
215
|
-
|
|
229
|
+
def git_repository_has_uncommitted_changes(self, repository_folder: str) -> bool:
|
|
230
|
+
self.assert_is_git_repository(repository_folder)
|
|
231
|
+
if (self.git_repository_has_unstaged_changes(repository_folder)):
|
|
216
232
|
return True
|
|
217
|
-
if (self.git_repository_has_staged_changes(
|
|
233
|
+
if (self.git_repository_has_staged_changes(repository_folder)):
|
|
218
234
|
return True
|
|
219
235
|
return False
|
|
220
236
|
|
|
221
237
|
@GeneralUtilities.check_arguments
|
|
222
238
|
def git_repository_has_unstaged_changes(self, repository_folder: str) -> bool:
|
|
239
|
+
self.assert_is_git_repository(repository_folder)
|
|
223
240
|
if (self.git_repository_has_unstaged_changes_of_tracked_files(repository_folder)):
|
|
224
241
|
return True
|
|
225
242
|
if (self.git_repository_has_new_untracked_files(repository_folder)):
|
|
@@ -228,47 +245,73 @@ class ScriptCollectionCore:
|
|
|
228
245
|
|
|
229
246
|
@GeneralUtilities.check_arguments
|
|
230
247
|
def git_get_commit_id(self, repository_folder: str, commit: str = "HEAD") -> str:
|
|
231
|
-
|
|
248
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
249
|
+
result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["rev-parse", "--verify", commit], repository_folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
232
250
|
return result[1].replace('\n', '')
|
|
233
251
|
|
|
234
252
|
@GeneralUtilities.check_arguments
|
|
235
253
|
def git_get_commit_date(self, repository_folder: str, commit: str = "HEAD") -> datetime:
|
|
236
|
-
|
|
254
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
255
|
+
result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["show", "-s", "--format=%ci", commit], repository_folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
237
256
|
date_as_string = result[1].replace('\n', '')
|
|
238
257
|
result = datetime.strptime(date_as_string, '%Y-%m-%d %H:%M:%S %z')
|
|
239
258
|
return result
|
|
240
259
|
|
|
260
|
+
@GeneralUtilities.check_arguments
|
|
261
|
+
def git_fetch_with_retry(self, folder: str, remotename: str = "--all", amount_of_attempts: int = 5) -> None:
|
|
262
|
+
GeneralUtilities.retry_action(lambda: self.git_fetch(folder, remotename), amount_of_attempts)
|
|
263
|
+
|
|
241
264
|
@GeneralUtilities.check_arguments
|
|
242
265
|
def git_fetch(self, folder: str, remotename: str = "--all") -> None:
|
|
243
|
-
self.
|
|
266
|
+
self.is_git_or_bare_git_repository(folder)
|
|
267
|
+
self.run_program_argsasarray("git", ["fetch", remotename, "--tags", "--prune"], folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
244
268
|
|
|
245
269
|
@GeneralUtilities.check_arguments
|
|
246
270
|
def git_fetch_in_bare_repository(self, folder: str, remotename, localbranch: str, remotebranch: str) -> None:
|
|
247
|
-
self.
|
|
271
|
+
self.is_git_or_bare_git_repository(folder)
|
|
272
|
+
self.run_program_argsasarray("git", ["fetch", remotename, f"{remotebranch}:{localbranch}"], folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
248
273
|
|
|
249
274
|
@GeneralUtilities.check_arguments
|
|
250
275
|
def git_remove_branch(self, folder: str, branchname: str) -> None:
|
|
251
|
-
self.
|
|
276
|
+
self.is_git_or_bare_git_repository(folder)
|
|
277
|
+
self.run_program("git", f"branch -D {branchname}", folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
252
278
|
|
|
253
279
|
@GeneralUtilities.check_arguments
|
|
254
|
-
def
|
|
255
|
-
|
|
280
|
+
def git_push_with_retry(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: LogLevel = LogLevel.Quiet, amount_of_attempts: int = 5) -> None:
|
|
281
|
+
GeneralUtilities.retry_action(lambda: self.git_push(folder, remotename, localbranchname, remotebranchname, forcepush, pushalltags, verbosity), amount_of_attempts)
|
|
282
|
+
|
|
283
|
+
@GeneralUtilities.check_arguments
|
|
284
|
+
def git_push(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: LogLevel = LogLevel.Quiet,resurse_submodules:bool=False) -> None:
|
|
285
|
+
self.is_git_or_bare_git_repository(folder)
|
|
286
|
+
argument = ["push"]
|
|
287
|
+
if resurse_submodules:
|
|
288
|
+
argument = argument + ["--recurse-submodules=on-demand"]
|
|
289
|
+
argument = argument + [remotename, f"{localbranchname}:{remotebranchname}"]
|
|
256
290
|
if (forcepush):
|
|
257
291
|
argument.append("--force")
|
|
258
292
|
if (pushalltags):
|
|
259
293
|
argument.append("--tags")
|
|
260
|
-
result: tuple[int, str, str, int] = self.run_program_argsasarray("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True,
|
|
294
|
+
result: tuple[int, str, str, int] = self.run_program_argsasarray("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True, print_errors_as_information=True)
|
|
261
295
|
return result[1].replace('\r', '').replace('\n', '')
|
|
262
296
|
|
|
263
297
|
@GeneralUtilities.check_arguments
|
|
264
|
-
def
|
|
265
|
-
self.
|
|
298
|
+
def git_pull_with_retry(self, folder: str, remote: str, localbranchname: str, remotebranchname: str, force: bool = False, amount_of_attempts: int = 5) -> None:
|
|
299
|
+
GeneralUtilities.retry_action(lambda: self.git_pull(folder, remote, localbranchname, remotebranchname), amount_of_attempts)
|
|
300
|
+
|
|
301
|
+
@GeneralUtilities.check_arguments
|
|
302
|
+
def git_pull(self, folder: str, remote: str, localbranchname: str, remotebranchname: str, force: bool = False) -> None:
|
|
303
|
+
self.is_git_or_bare_git_repository(folder)
|
|
304
|
+
argument = f"pull {remote} {remotebranchname}:{localbranchname}"
|
|
305
|
+
if force:
|
|
306
|
+
argument = f"{argument} --force"
|
|
307
|
+
self.run_program("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
266
308
|
|
|
267
309
|
@GeneralUtilities.check_arguments
|
|
268
310
|
def git_list_remote_branches(self, folder: str, remote: str, fetch: bool) -> list[str]:
|
|
311
|
+
self.is_git_or_bare_git_repository(folder)
|
|
269
312
|
if fetch:
|
|
270
313
|
self.git_fetch(folder, remote)
|
|
271
|
-
run_program_result = self.run_program("git", f"branch -rl {remote}/*", folder, throw_exception_if_exitcode_is_not_zero=True
|
|
314
|
+
run_program_result = self.run_program("git", f"branch -rl {remote}/*", folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
272
315
|
output = GeneralUtilities.string_to_lines(run_program_result[1])
|
|
273
316
|
result = list[str]()
|
|
274
317
|
for item in output:
|
|
@@ -295,61 +338,72 @@ class ScriptCollectionCore:
|
|
|
295
338
|
args.append("--remote-submodules")
|
|
296
339
|
if mirror:
|
|
297
340
|
args.append("--mirror")
|
|
298
|
-
self.run_program_argsasarray("git", args, os.getcwd(), throw_exception_if_exitcode_is_not_zero=True
|
|
341
|
+
self.run_program_argsasarray("git", args, os.getcwd(), throw_exception_if_exitcode_is_not_zero=True)
|
|
299
342
|
|
|
300
343
|
@GeneralUtilities.check_arguments
|
|
301
344
|
def git_get_all_remote_names(self, directory: str) -> list[str]:
|
|
302
|
-
|
|
345
|
+
self.is_git_or_bare_git_repository(directory)
|
|
346
|
+
result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote"], directory, throw_exception_if_exitcode_is_not_zero=True)[1], False)
|
|
303
347
|
return result
|
|
304
348
|
|
|
305
349
|
@GeneralUtilities.check_arguments
|
|
306
350
|
def git_get_remote_url(self, directory: str, remote_name: str) -> str:
|
|
307
|
-
|
|
351
|
+
self.is_git_or_bare_git_repository(directory)
|
|
352
|
+
result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote", "get-url", remote_name], directory, throw_exception_if_exitcode_is_not_zero=True)[1], False)
|
|
308
353
|
return result[0].replace('\n', '')
|
|
309
354
|
|
|
310
355
|
@GeneralUtilities.check_arguments
|
|
311
356
|
def repository_has_remote_with_specific_name(self, directory: str, remote_name: str) -> bool:
|
|
357
|
+
self.is_git_or_bare_git_repository(directory)
|
|
312
358
|
return remote_name in self.git_get_all_remote_names(directory)
|
|
313
359
|
|
|
314
360
|
@GeneralUtilities.check_arguments
|
|
315
361
|
def git_add_or_set_remote_address(self, directory: str, remote_name: str, remote_address: str) -> None:
|
|
362
|
+
self.assert_is_git_repository(directory)
|
|
316
363
|
if (self.repository_has_remote_with_specific_name(directory, remote_name)):
|
|
317
|
-
self.run_program_argsasarray("git", ['remote', 'set-url', 'remote_name', remote_address], directory, throw_exception_if_exitcode_is_not_zero=True
|
|
364
|
+
self.run_program_argsasarray("git", ['remote', 'set-url', 'remote_name', remote_address], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
318
365
|
else:
|
|
319
|
-
self.run_program_argsasarray("git", ['remote', 'add', remote_name, remote_address], directory, throw_exception_if_exitcode_is_not_zero=True
|
|
366
|
+
self.run_program_argsasarray("git", ['remote', 'add', remote_name, remote_address], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
320
367
|
|
|
321
368
|
@GeneralUtilities.check_arguments
|
|
322
369
|
def git_stage_all_changes(self, directory: str) -> None:
|
|
323
|
-
self.
|
|
370
|
+
self.assert_is_git_repository(directory)
|
|
371
|
+
self.run_program_argsasarray("git", ["add", "-A"], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
324
372
|
|
|
325
373
|
@GeneralUtilities.check_arguments
|
|
326
374
|
def git_unstage_all_changes(self, directory: str) -> None:
|
|
327
|
-
self.
|
|
375
|
+
self.assert_is_git_repository(directory)
|
|
376
|
+
self.run_program_argsasarray("git", ["reset"], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
377
|
+
# TODO check if this will also be done for submodules
|
|
328
378
|
|
|
329
379
|
@GeneralUtilities.check_arguments
|
|
330
380
|
def git_stage_file(self, directory: str, file: str) -> None:
|
|
331
|
-
self.
|
|
381
|
+
self.assert_is_git_repository(directory)
|
|
382
|
+
self.run_program_argsasarray("git", ['stage', file], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
332
383
|
|
|
333
384
|
@GeneralUtilities.check_arguments
|
|
334
385
|
def git_unstage_file(self, directory: str, file: str) -> None:
|
|
335
|
-
self.
|
|
386
|
+
self.assert_is_git_repository(directory)
|
|
387
|
+
self.run_program_argsasarray("git", ['reset', file], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
336
388
|
|
|
337
389
|
@GeneralUtilities.check_arguments
|
|
338
390
|
def git_discard_unstaged_changes_of_file(self, directory: str, file: str) -> None:
|
|
339
391
|
"""Caution: This method works really only for 'changed' files yet. So this method does not work properly for new or renamed files."""
|
|
340
|
-
self.
|
|
392
|
+
self.assert_is_git_repository(directory)
|
|
393
|
+
self.run_program_argsasarray("git", ['checkout', file], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
341
394
|
|
|
342
395
|
@GeneralUtilities.check_arguments
|
|
343
396
|
def git_discard_all_unstaged_changes(self, directory: str) -> None:
|
|
344
397
|
"""Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
|
|
345
|
-
self.
|
|
346
|
-
self.run_program_argsasarray("git", ['
|
|
398
|
+
self.assert_is_git_repository(directory)
|
|
399
|
+
self.run_program_argsasarray("git", ['clean', '-df'], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
400
|
+
self.run_program_argsasarray("git", ['checkout', '.'], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
401
|
+
# TODO check if this will also be done for submodules
|
|
347
402
|
|
|
348
403
|
@GeneralUtilities.check_arguments
|
|
349
|
-
def git_commit(self, directory: str, message: str, author_name: str = None, author_email: str = None, stage_all_changes: bool = True, no_changes_behavior: int = 0) -> str:
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
# no_changes_behavior=2 => Exception
|
|
404
|
+
def git_commit(self, directory: str, message: str = "Saved changes.", author_name: str = None, author_email: str = None, stage_all_changes: bool = True, no_changes_behavior: int = 0) -> str:
|
|
405
|
+
"""no_changes_behavior=0 => No commit; no_changes_behavior=1 => Commit anyway; no_changes_behavior=2 => Exception"""
|
|
406
|
+
self.assert_is_git_repository(directory)
|
|
353
407
|
author_name = GeneralUtilities.str_none_safe(author_name).strip()
|
|
354
408
|
author_email = GeneralUtilities.str_none_safe(author_email).strip()
|
|
355
409
|
argument = ['commit', '--quiet', '--allow-empty', '--message', message]
|
|
@@ -363,10 +417,10 @@ class ScriptCollectionCore:
|
|
|
363
417
|
self.git_stage_all_changes(directory)
|
|
364
418
|
else:
|
|
365
419
|
if no_changes_behavior == 0:
|
|
366
|
-
|
|
420
|
+
self.log.log(f"Commit '{message}' will not be done because there are no changes to commit in repository '{directory}'", LogLevel.Debug)
|
|
367
421
|
do_commit = False
|
|
368
422
|
elif no_changes_behavior == 1:
|
|
369
|
-
|
|
423
|
+
self.log.log(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will be done anyway.", LogLevel.Debug)
|
|
370
424
|
do_commit = True
|
|
371
425
|
elif no_changes_behavior == 2:
|
|
372
426
|
raise RuntimeError(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will not be done.")
|
|
@@ -374,37 +428,75 @@ class ScriptCollectionCore:
|
|
|
374
428
|
raise ValueError(f"Unknown value for no_changes_behavior: {GeneralUtilities.str_none_safe(no_changes_behavior)}")
|
|
375
429
|
|
|
376
430
|
if do_commit:
|
|
377
|
-
|
|
378
|
-
self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True
|
|
431
|
+
self.log.log(f"Commit changes in '{directory}'", LogLevel.Information)
|
|
432
|
+
self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
379
433
|
|
|
380
434
|
return self.git_get_commit_id(directory)
|
|
435
|
+
|
|
436
|
+
def search_repository_folder(self,some_file_in_repository:str)->str:
|
|
437
|
+
current_path:str=os.path.dirname(some_file_in_repository)
|
|
438
|
+
enabled:bool=True
|
|
439
|
+
while enabled:
|
|
440
|
+
try:
|
|
441
|
+
current_path=GeneralUtilities.resolve_relative_path("..",current_path)
|
|
442
|
+
if self.is_git_repository(current_path):
|
|
443
|
+
return current_path
|
|
444
|
+
except:
|
|
445
|
+
enabled=False
|
|
446
|
+
raise ValueError(f"Can not find git-repository for folder \"{some_file_in_repository}\".")
|
|
447
|
+
|
|
381
448
|
|
|
382
449
|
@GeneralUtilities.check_arguments
|
|
383
450
|
def git_create_tag(self, directory: str, target_for_tag: str, tag: str, sign: bool = False, message: str = None) -> None:
|
|
451
|
+
self.is_git_or_bare_git_repository(directory)
|
|
384
452
|
argument = ["tag", tag, target_for_tag]
|
|
385
453
|
if sign:
|
|
386
454
|
if message is None:
|
|
387
455
|
message = f"Created {target_for_tag}"
|
|
388
456
|
argument.extend(["-s", '-m', message])
|
|
389
|
-
self.run_program_argsasarray(
|
|
390
|
-
"git", argument, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
|
|
457
|
+
self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
391
458
|
|
|
392
459
|
@GeneralUtilities.check_arguments
|
|
393
460
|
def git_delete_tag(self, directory: str, tag: str) -> None:
|
|
394
|
-
self.
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
self.
|
|
461
|
+
self.is_git_or_bare_git_repository(directory)
|
|
462
|
+
self.run_program_argsasarray("git", ["tag", "--delete", tag], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
463
|
+
|
|
464
|
+
@GeneralUtilities.check_arguments
|
|
465
|
+
def git_checkout(self, directory: str, branch: str, undo_all_changes_after_checkout: bool = True, assert_no_uncommitted_changes: bool = True) -> None:
|
|
466
|
+
self.assert_is_git_repository(directory)
|
|
467
|
+
if assert_no_uncommitted_changes:
|
|
468
|
+
GeneralUtilities.assert_condition(not self.git_repository_has_uncommitted_changes(directory), f"Repository '{directory}' has uncommitted changes.")
|
|
469
|
+
self.run_program_argsasarray("git", ["checkout", branch], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
470
|
+
self.run_program_argsasarray("git", ["submodule", "update", "--recursive"], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
471
|
+
if undo_all_changes_after_checkout:
|
|
472
|
+
self.git_undo_all_changes(directory)
|
|
473
|
+
|
|
474
|
+
@GeneralUtilities.check_arguments
|
|
475
|
+
def merge_repository(self, repository_folder: str, remote: str, branch: str, pull_first_if_there_are_no_uncommitted_changes: bool = True):
|
|
476
|
+
if pull_first_if_there_are_no_uncommitted_changes:
|
|
477
|
+
uncommitted_changes = self.git_repository_has_uncommitted_changes(repository_folder)
|
|
478
|
+
if not uncommitted_changes:
|
|
479
|
+
is_pullable: bool = self.git_commit_is_ancestor(repository_folder, branch, f"{remote}/{branch}")
|
|
480
|
+
if is_pullable:
|
|
481
|
+
self.git_pull(repository_folder, remote, branch, branch)
|
|
482
|
+
uncommitted_changes = self.git_repository_has_uncommitted_changes(repository_folder)
|
|
483
|
+
GeneralUtilities.assert_condition(not uncommitted_changes, f"Pulling remote \"{remote}\" in \"{repository_folder}\" caused new uncommitted files.")
|
|
484
|
+
self.git_checkout(repository_folder, branch)
|
|
485
|
+
self.git_commit(repository_folder, "Automatic commit due to merge")
|
|
486
|
+
self.git_fetch(repository_folder, remote)
|
|
487
|
+
self.git_merge(repository_folder, f"{remote}/{branch}", branch)
|
|
488
|
+
self.git_push_with_retry(repository_folder, remote, branch, branch)
|
|
489
|
+
self.git_checkout(repository_folder, branch)
|
|
400
490
|
|
|
401
491
|
@GeneralUtilities.check_arguments
|
|
402
492
|
def git_merge_abort(self, directory: str) -> None:
|
|
403
|
-
self.
|
|
493
|
+
self.assert_is_git_repository(directory)
|
|
494
|
+
self.run_program_argsasarray("git", ["merge", "--abort"], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
404
495
|
|
|
405
496
|
@GeneralUtilities.check_arguments
|
|
406
|
-
def git_merge(self, directory: str, sourcebranch: str, targetbranch: str, fastforward: bool = True, commit: bool = True, commit_message: str = None) -> str:
|
|
407
|
-
self.
|
|
497
|
+
def git_merge(self, directory: str, sourcebranch: str, targetbranch: str, fastforward: bool = True, commit: bool = True, commit_message: str = None, undo_all_changes_after_checkout: bool = True, assert_no_uncommitted_changes: bool = True) -> str:
|
|
498
|
+
self.assert_is_git_repository(directory)
|
|
499
|
+
self.git_checkout(directory, targetbranch, undo_all_changes_after_checkout, assert_no_uncommitted_changes)
|
|
408
500
|
args = ["merge"]
|
|
409
501
|
if not commit:
|
|
410
502
|
args.append("--no-commit")
|
|
@@ -414,13 +506,14 @@ class ScriptCollectionCore:
|
|
|
414
506
|
args.append("-m")
|
|
415
507
|
args.append(commit_message)
|
|
416
508
|
args.append(sourcebranch)
|
|
417
|
-
self.run_program_argsasarray(
|
|
418
|
-
|
|
509
|
+
self.run_program_argsasarray("git", args, directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
510
|
+
self.run_program_argsasarray("git", ["submodule", "update"], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
419
511
|
return self.git_get_commit_id(directory)
|
|
420
512
|
|
|
421
513
|
@GeneralUtilities.check_arguments
|
|
422
514
|
def git_undo_all_changes(self, directory: str) -> None:
|
|
423
515
|
"""Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
|
|
516
|
+
self.assert_is_git_repository(directory)
|
|
424
517
|
self.git_unstage_all_changes(directory)
|
|
425
518
|
self.git_discard_all_unstaged_changes(directory)
|
|
426
519
|
|
|
@@ -438,22 +531,18 @@ class ScriptCollectionCore:
|
|
|
438
531
|
# clone
|
|
439
532
|
self.git_clone(target_repository, source_repository, include_submodules=True, mirror=True)
|
|
440
533
|
|
|
441
|
-
def get_git_submodules(self,
|
|
442
|
-
|
|
534
|
+
def get_git_submodules(self, directory: str) -> list[str]:
|
|
535
|
+
self.is_git_or_bare_git_repository(directory)
|
|
536
|
+
e = self.run_program("git", "submodule status", directory)
|
|
443
537
|
result = []
|
|
444
538
|
for submodule_line in GeneralUtilities.string_to_lines(e[1], False, True):
|
|
445
539
|
result.append(submodule_line.split(' ')[1])
|
|
446
540
|
return result
|
|
447
541
|
|
|
448
|
-
@GeneralUtilities.check_arguments
|
|
449
|
-
def is_git_repository(self, folder: str) -> bool:
|
|
450
|
-
combined = os.path.join(folder, ".git")
|
|
451
|
-
# TODO consider check for bare-repositories
|
|
452
|
-
return os.path.isdir(combined) or os.path.isfile(combined)
|
|
453
|
-
|
|
454
542
|
@GeneralUtilities.check_arguments
|
|
455
543
|
def file_is_git_ignored(self, file_in_repository: str, repositorybasefolder: str) -> None:
|
|
456
|
-
|
|
544
|
+
self.is_git_or_bare_git_repository(repositorybasefolder)
|
|
545
|
+
exit_code = self.run_program_argsasarray("git", ['check-ignore', file_in_repository], repositorybasefolder, throw_exception_if_exitcode_is_not_zero=False)[0]
|
|
457
546
|
if (exit_code == 0):
|
|
458
547
|
return True
|
|
459
548
|
if (exit_code == 1):
|
|
@@ -462,34 +551,39 @@ class ScriptCollectionCore:
|
|
|
462
551
|
|
|
463
552
|
@GeneralUtilities.check_arguments
|
|
464
553
|
def git_discard_all_changes(self, repository: str) -> None:
|
|
465
|
-
self.
|
|
466
|
-
self.run_program_argsasarray("git", ["
|
|
554
|
+
self.assert_is_git_repository(repository)
|
|
555
|
+
self.run_program_argsasarray("git", ["reset", "HEAD", "."], repository, throw_exception_if_exitcode_is_not_zero=True)
|
|
556
|
+
self.run_program_argsasarray("git", ["checkout", "."], repository, throw_exception_if_exitcode_is_not_zero=True)
|
|
467
557
|
|
|
468
558
|
@GeneralUtilities.check_arguments
|
|
469
559
|
def git_get_current_branch_name(self, repository: str) -> str:
|
|
470
|
-
|
|
471
|
-
|
|
560
|
+
self.assert_is_git_repository(repository)
|
|
561
|
+
result = self.run_program_argsasarray("git", ["rev-parse", "--abbrev-ref", "HEAD"], repository, throw_exception_if_exitcode_is_not_zero=True)
|
|
562
|
+
return result[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
|
|
472
563
|
|
|
473
564
|
@GeneralUtilities.check_arguments
|
|
474
565
|
def git_get_commitid_of_tag(self, repository: str, tag: str) -> str:
|
|
475
|
-
|
|
476
|
-
|
|
566
|
+
self.is_git_or_bare_git_repository(repository)
|
|
567
|
+
stdout = self.run_program_argsasarray("git", ["rev-list", "-n", "1", tag], repository)
|
|
568
|
+
result = stdout[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
|
|
477
569
|
return result
|
|
478
570
|
|
|
479
571
|
@GeneralUtilities.check_arguments
|
|
480
572
|
def git_get_tags(self, repository: str) -> list[str]:
|
|
481
|
-
|
|
573
|
+
self.is_git_or_bare_git_repository(repository)
|
|
574
|
+
tags = [line.replace("\r", GeneralUtilities.empty_string) for line in self.run_program_argsasarray(
|
|
482
575
|
"git", ["tag"], repository)[1].split("\n") if len(line) > 0]
|
|
483
576
|
return tags
|
|
484
577
|
|
|
485
578
|
@GeneralUtilities.check_arguments
|
|
486
579
|
def git_move_tags_to_another_branch(self, repository: str, tag_source_branch: str, tag_target_branch: str, sign: bool = False, message: str = None) -> None:
|
|
580
|
+
self.is_git_or_bare_git_repository(repository)
|
|
487
581
|
tags = self.git_get_tags(repository)
|
|
488
582
|
tags_count = len(tags)
|
|
489
583
|
counter = 0
|
|
490
584
|
for tag in tags:
|
|
491
585
|
counter = counter+1
|
|
492
|
-
|
|
586
|
+
self.log.log(f"Process tag {counter}/{tags_count}.", LogLevel.Information)
|
|
493
587
|
# tag is on source-branch
|
|
494
588
|
if self.git_commit_is_ancestor(repository, tag, tag_source_branch):
|
|
495
589
|
commit_id_old = self.git_get_commitid_of_tag(repository, tag)
|
|
@@ -504,27 +598,36 @@ class ScriptCollectionCore:
|
|
|
504
598
|
|
|
505
599
|
@GeneralUtilities.check_arguments
|
|
506
600
|
def get_current_git_branch_has_tag(self, repository_folder: str) -> bool:
|
|
507
|
-
|
|
601
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
602
|
+
result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
|
|
508
603
|
return result[0] == 0
|
|
509
604
|
|
|
510
605
|
@GeneralUtilities.check_arguments
|
|
511
606
|
def get_latest_git_tag(self, repository_folder: str) -> str:
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
result = result[1].replace("\r",
|
|
607
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
608
|
+
result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder)
|
|
609
|
+
result = result[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
|
|
515
610
|
return result
|
|
516
611
|
|
|
517
612
|
@GeneralUtilities.check_arguments
|
|
518
613
|
def get_staged_or_committed_git_ignored_files(self, repository_folder: str) -> list[str]:
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
614
|
+
self.assert_is_git_repository(repository_folder)
|
|
615
|
+
temp_result = self.run_program_argsasarray("git", ["ls-files", "-i", "-c", "--exclude-standard"], repository_folder)
|
|
616
|
+
temp_result = temp_result[1].replace("\r", GeneralUtilities.empty_string)
|
|
617
|
+
result = [line for line in temp_result.split("\n") if len(line) > 0]
|
|
522
618
|
return result
|
|
523
619
|
|
|
524
620
|
@GeneralUtilities.check_arguments
|
|
525
621
|
def git_repository_has_commits(self, repository_folder: str) -> bool:
|
|
622
|
+
self.assert_is_git_repository(repository_folder)
|
|
526
623
|
return self.run_program_argsasarray("git", ["rev-parse", "--verify", "HEAD"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)[0] == 0
|
|
527
624
|
|
|
625
|
+
@GeneralUtilities.check_arguments
|
|
626
|
+
def run_git_command_in_repository_and_submodules(self, repository_folder: str, arguments: list[str]) -> None:
|
|
627
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
628
|
+
self.run_program_argsasarray("git", arguments, repository_folder)
|
|
629
|
+
self.run_program_argsasarray("git", ["submodule", "foreach", "--recursive", "git"]+arguments, repository_folder)
|
|
630
|
+
|
|
528
631
|
@GeneralUtilities.check_arguments
|
|
529
632
|
def export_filemetadata(self, folder: str, target_file: str, encoding: str = "utf-8", filter_function=None) -> None:
|
|
530
633
|
folder = GeneralUtilities.resolve_relative_path_from_current_working_directory(folder)
|
|
@@ -576,6 +679,245 @@ class ScriptCollectionCore:
|
|
|
576
679
|
for renamed_item, original_name in renamed_items.items():
|
|
577
680
|
os.rename(renamed_item, original_name)
|
|
578
681
|
|
|
682
|
+
@GeneralUtilities.check_arguments
|
|
683
|
+
def is_git_repository(self, folder: str) -> bool:
|
|
684
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
685
|
+
folder=folder.replace("\\","/")
|
|
686
|
+
if folder.endswith("/"):
|
|
687
|
+
folder = folder[:-1]
|
|
688
|
+
if not self.is_folder(folder):
|
|
689
|
+
raise ValueError(f"Folder '{folder}' does not exist.")
|
|
690
|
+
git_folder_path = f"{folder}/.git"
|
|
691
|
+
return self.is_folder(git_folder_path) or self.is_file(git_folder_path)
|
|
692
|
+
|
|
693
|
+
@GeneralUtilities.check_arguments
|
|
694
|
+
def is_bare_git_repository(self, folder: str) -> bool:
|
|
695
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
696
|
+
if folder.endswith("/") or folder.endswith("\\"):
|
|
697
|
+
folder = folder[:-1]
|
|
698
|
+
if not self.is_folder(folder):
|
|
699
|
+
raise ValueError(f"Folder '{folder}' does not exist.")
|
|
700
|
+
return folder.endswith(".git")
|
|
701
|
+
|
|
702
|
+
@GeneralUtilities.check_arguments
|
|
703
|
+
def is_git_or_bare_git_repository(self, folder: str) -> bool:
|
|
704
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
705
|
+
return self.is_git_repository(folder) or self.is_bare_git_repository(folder)
|
|
706
|
+
|
|
707
|
+
@GeneralUtilities.check_arguments
|
|
708
|
+
def assert_is_git_repository(self, folder: str) -> str:
|
|
709
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
710
|
+
GeneralUtilities.assert_condition(self.is_git_repository(folder), f"'{folder}' is not a git-repository.")
|
|
711
|
+
|
|
712
|
+
@GeneralUtilities.check_arguments
|
|
713
|
+
def convert_git_repository_to_bare_repository(self, repository_folder: str):
|
|
714
|
+
repository_folder = repository_folder.replace("\\", "/")
|
|
715
|
+
self.assert_is_git_repository(repository_folder)
|
|
716
|
+
git_folder = repository_folder + "/.git"
|
|
717
|
+
if not self.is_folder(git_folder):
|
|
718
|
+
raise ValueError(f"Converting '{repository_folder}' to a bare repository not possible. The folder '{git_folder}' does not exist. Converting is currently only supported when the git-folder is a direct folder in a repository and not a reference to another location.")
|
|
719
|
+
target_folder: str = repository_folder + ".git"
|
|
720
|
+
GeneralUtilities.ensure_directory_exists(target_folder)
|
|
721
|
+
GeneralUtilities.move_content_of_folder(git_folder, target_folder)
|
|
722
|
+
GeneralUtilities.ensure_directory_does_not_exist(repository_folder)
|
|
723
|
+
self.run_program_argsasarray("git", ["config", "--bool", "core.bare", "true"], target_folder)
|
|
724
|
+
|
|
725
|
+
@GeneralUtilities.check_arguments
|
|
726
|
+
def assert_no_uncommitted_changes(self, repository_folder: str):
|
|
727
|
+
if self.git_repository_has_uncommitted_changes(repository_folder):
|
|
728
|
+
raise ValueError(f"Repository '{repository_folder}' has uncommitted changes.")
|
|
729
|
+
|
|
730
|
+
@GeneralUtilities.check_arguments
|
|
731
|
+
def list_content(self, path: str, include_files: bool, include_folder: bool, printonlynamewithoutpath: bool) -> list[str]:
|
|
732
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
733
|
+
result: list[str] = []
|
|
734
|
+
if self.program_runner.will_be_executed_locally():
|
|
735
|
+
if include_files:
|
|
736
|
+
result = result + GeneralUtilities.get_direct_files_of_folder(path)
|
|
737
|
+
if include_folder:
|
|
738
|
+
result = result + GeneralUtilities.get_direct_folders_of_folder(path)
|
|
739
|
+
else:
|
|
740
|
+
arguments = ["--path", path]
|
|
741
|
+
if not include_files:
|
|
742
|
+
arguments = arguments+["--excludefiles"]
|
|
743
|
+
if not include_folder:
|
|
744
|
+
arguments = arguments+["--excludedirectories"]
|
|
745
|
+
if printonlynamewithoutpath:
|
|
746
|
+
arguments = arguments+["--printonlynamewithoutpath"]
|
|
747
|
+
exit_code, stdout, stderr, _ = self.run_program_argsasarray("sclistfoldercontent", arguments)
|
|
748
|
+
if exit_code == 0:
|
|
749
|
+
for line in stdout.split("\n"):
|
|
750
|
+
normalized_line = line.replace("\r", "")
|
|
751
|
+
result.append(normalized_line)
|
|
752
|
+
else:
|
|
753
|
+
raise ValueError(f"Fatal error occurrs while checking whether file '{path}' exists. StdErr: '{stderr}'")
|
|
754
|
+
result = [item for item in result if GeneralUtilities.string_has_nonwhitespace_content(item)]
|
|
755
|
+
return result
|
|
756
|
+
|
|
757
|
+
@GeneralUtilities.check_arguments
|
|
758
|
+
def is_file(self, path: str) -> bool:
|
|
759
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
760
|
+
if self.program_runner.will_be_executed_locally():
|
|
761
|
+
return os.path.isfile(path) # works only locally, but much more performant than always running an external program
|
|
762
|
+
else:
|
|
763
|
+
exit_code, _, stderr, _ = self.run_program_argsasarray("scfileexists", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
764
|
+
if exit_code == 0:
|
|
765
|
+
return True
|
|
766
|
+
elif exit_code == 1:
|
|
767
|
+
raise ValueError(f"Not calculatable whether file '{path}' exists. StdErr: '{stderr}'")
|
|
768
|
+
elif exit_code == 2:
|
|
769
|
+
return False
|
|
770
|
+
raise ValueError(f"Fatal error occurrs while checking whether file '{path}' exists. StdErr: '{stderr}'")
|
|
771
|
+
|
|
772
|
+
@GeneralUtilities.check_arguments
|
|
773
|
+
def is_folder(self, path: str) -> bool:
|
|
774
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
775
|
+
if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
|
|
776
|
+
return os.path.isdir(path)
|
|
777
|
+
else:
|
|
778
|
+
exit_code, _, stderr, _ = self.run_program_argsasarray("scfolderexists", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
779
|
+
if exit_code == 0:
|
|
780
|
+
return True
|
|
781
|
+
elif exit_code == 1:
|
|
782
|
+
raise ValueError(f"Not calculatable whether folder '{path}' exists. StdErr: '{stderr}'")
|
|
783
|
+
elif exit_code == 2:
|
|
784
|
+
return False
|
|
785
|
+
raise ValueError(f"Fatal error occurrs while checking whether folder '{path}' exists. StdErr: '{stderr}'")
|
|
786
|
+
|
|
787
|
+
@GeneralUtilities.check_arguments
|
|
788
|
+
def get_file_content(self, path: str, encoding: str = "utf-8") -> str:
|
|
789
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
790
|
+
if self.program_runner.will_be_executed_locally():
|
|
791
|
+
return GeneralUtilities.read_text_from_file(path, encoding)
|
|
792
|
+
else:
|
|
793
|
+
result = self.run_program_argsasarray("scprintfilecontent", ["--path", path, "--encofing", encoding]) # works platform-indepent
|
|
794
|
+
return result[1].replace("\\n", "\n")
|
|
795
|
+
|
|
796
|
+
@GeneralUtilities.check_arguments
|
|
797
|
+
def set_file_content(self, path: str, content: str, encoding: str = "utf-8") -> None:
|
|
798
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
799
|
+
if self.program_runner.will_be_executed_locally():
|
|
800
|
+
GeneralUtilities.write_text_to_file(path, content, encoding)
|
|
801
|
+
else:
|
|
802
|
+
content_bytes = content.encode('utf-8')
|
|
803
|
+
base64_bytes = base64.b64encode(content_bytes)
|
|
804
|
+
base64_string = base64_bytes.decode('utf-8')
|
|
805
|
+
self.run_program_argsasarray("scsetfilecontent", ["--path", path, "--argumentisinbase64", "--content", base64_string]) # works platform-indepent
|
|
806
|
+
|
|
807
|
+
@GeneralUtilities.check_arguments
|
|
808
|
+
def remove(self, path: str) -> None:
|
|
809
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
810
|
+
if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
|
|
811
|
+
if os.path.isdir(path):
|
|
812
|
+
GeneralUtilities.ensure_directory_does_not_exist(path)
|
|
813
|
+
if os.path.isfile(path):
|
|
814
|
+
GeneralUtilities.ensure_file_does_not_exist(path)
|
|
815
|
+
else:
|
|
816
|
+
if self.is_file(path):
|
|
817
|
+
exit_code, _, stderr, _ = self.run_program_argsasarray("scremovefile", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
818
|
+
if exit_code != 0:
|
|
819
|
+
raise ValueError(f"Fatal error occurrs while removing file '{path}'. StdErr: '{stderr}'")
|
|
820
|
+
if self.is_folder(path):
|
|
821
|
+
exit_code, _, stderr, _ = self.run_program_argsasarray("scremovefolder", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
822
|
+
if exit_code != 0:
|
|
823
|
+
raise ValueError(f"Fatal error occurrs while removing folder '{path}'. StdErr: '{stderr}'")
|
|
824
|
+
|
|
825
|
+
@GeneralUtilities.check_arguments
|
|
826
|
+
def rename(self, source: str, target: str) -> None:
|
|
827
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
828
|
+
if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
|
|
829
|
+
os.rename(source, target)
|
|
830
|
+
else:
|
|
831
|
+
exit_code, _, stderr, _ = self.run_program_argsasarray("screname", ["--source", source, "--target", target], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
832
|
+
if exit_code != 0:
|
|
833
|
+
raise ValueError(f"Fatal error occurrs while renaming '{source}' to '{target}'. StdErr: '{stderr}'")
|
|
834
|
+
|
|
835
|
+
@GeneralUtilities.check_arguments
|
|
836
|
+
def copy(self, source: str, target: str) -> None:
|
|
837
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
838
|
+
if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
|
|
839
|
+
if os.path.isfile(target) or os.path.isdir(target):
|
|
840
|
+
raise ValueError(f"Can not copy to '{target}' because the target already exists.")
|
|
841
|
+
if os.path.isfile(source):
|
|
842
|
+
shutil.copyfile(source, target)
|
|
843
|
+
elif os.path.isdir(source):
|
|
844
|
+
GeneralUtilities.ensure_directory_exists(target)
|
|
845
|
+
GeneralUtilities.copy_content_of_folder(source, target)
|
|
846
|
+
else:
|
|
847
|
+
raise ValueError(f"'{source}' can not be copied because the path does not exist.")
|
|
848
|
+
else:
|
|
849
|
+
exit_code, _, stderr, _ = self.run_program_argsasarray("sccopy", ["--source", source, "--target", target], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
850
|
+
if exit_code != 0:
|
|
851
|
+
raise ValueError(f"Fatal error occurrs while copying '{source}' to '{target}'. StdErr: '{stderr}'")
|
|
852
|
+
|
|
853
|
+
@GeneralUtilities.check_arguments
|
|
854
|
+
def create_file(self, path: str, error_if_already_exists: bool, create_necessary_folder: bool) -> None:
|
|
855
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
856
|
+
if self.program_runner.will_be_executed_locally():
|
|
857
|
+
if not os.path.isabs(path):
|
|
858
|
+
path = os.path.join(os.getcwd(), path)
|
|
859
|
+
|
|
860
|
+
if os.path.isfile(path) and error_if_already_exists:
|
|
861
|
+
raise ValueError(f"File '{path}' already exists.")
|
|
862
|
+
|
|
863
|
+
# TODO maybe it should be checked if there is a folder with the same path which already exists.
|
|
864
|
+
|
|
865
|
+
folder = os.path.dirname(path)
|
|
866
|
+
|
|
867
|
+
if not os.path.isdir(folder):
|
|
868
|
+
if create_necessary_folder:
|
|
869
|
+
GeneralUtilities.ensure_directory_exists(folder) # TODO check if this also create nested folders if required
|
|
870
|
+
else:
|
|
871
|
+
raise ValueError(f"Folder '{folder}' does not exist.")
|
|
872
|
+
|
|
873
|
+
GeneralUtilities.ensure_file_exists(path)
|
|
874
|
+
else:
|
|
875
|
+
arguments = ["--path", path]
|
|
876
|
+
|
|
877
|
+
if error_if_already_exists:
|
|
878
|
+
arguments = arguments+["--errorwhenexists"]
|
|
879
|
+
|
|
880
|
+
if create_necessary_folder:
|
|
881
|
+
arguments = arguments+["--createnecessaryfolder"]
|
|
882
|
+
|
|
883
|
+
exit_code, _, stderr, _ = self.run_program_argsasarray("sccreatefile", arguments, throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
884
|
+
if exit_code != 0:
|
|
885
|
+
raise ValueError(f"Fatal error occurrs while create file '{path}'. StdErr: '{stderr}'")
|
|
886
|
+
|
|
887
|
+
@GeneralUtilities.check_arguments
|
|
888
|
+
def create_folder(self, path: str, error_if_already_exists: bool, create_necessary_folder: bool) -> None:
|
|
889
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
890
|
+
if self.program_runner.will_be_executed_locally():
|
|
891
|
+
if not os.path.isabs(path):
|
|
892
|
+
path = os.path.join(os.getcwd(), path)
|
|
893
|
+
|
|
894
|
+
if os.path.isdir(path) and error_if_already_exists:
|
|
895
|
+
raise ValueError(f"Folder '{path}' already exists.")
|
|
896
|
+
|
|
897
|
+
# TODO maybe it should be checked if there is a file with the same path which already exists.
|
|
898
|
+
|
|
899
|
+
folder = os.path.dirname(path)
|
|
900
|
+
|
|
901
|
+
if not os.path.isdir(folder):
|
|
902
|
+
if create_necessary_folder:
|
|
903
|
+
GeneralUtilities.ensure_directory_exists(folder) # TODO check if this also create nested folders if required
|
|
904
|
+
else:
|
|
905
|
+
raise ValueError(f"Folder '{folder}' does not exist.")
|
|
906
|
+
|
|
907
|
+
GeneralUtilities.ensure_directory_exists(path)
|
|
908
|
+
else:
|
|
909
|
+
arguments = ["--path", path]
|
|
910
|
+
|
|
911
|
+
if error_if_already_exists:
|
|
912
|
+
arguments = arguments+["--errorwhenexists"]
|
|
913
|
+
|
|
914
|
+
if create_necessary_folder:
|
|
915
|
+
arguments = arguments+["--createnecessaryfolder"]
|
|
916
|
+
|
|
917
|
+
exit_code, _, stderr, _ = self.run_program_argsasarray("sccreatefolder", arguments, throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
918
|
+
if exit_code != 0:
|
|
919
|
+
raise ValueError(f"Fatal error occurrs while create folder '{path}'. StdErr: '{stderr}'")
|
|
920
|
+
|
|
579
921
|
@GeneralUtilities.check_arguments
|
|
580
922
|
def __sort_fmd(self, line: str):
|
|
581
923
|
splitted: list = line.split(";")
|
|
@@ -619,7 +961,7 @@ class ScriptCollectionCore:
|
|
|
619
961
|
|
|
620
962
|
@GeneralUtilities.check_arguments
|
|
621
963
|
def __create_thumbnails(self, filename: str, fps: str, folder: str, tempname_for_thumbnails: str) -> list[str]:
|
|
622
|
-
argument = ['-i', filename, '-r',
|
|
964
|
+
argument = ['-i', filename, '-r', fps, '-vf', 'scale=-1:120', '-vcodec', 'png', f'{tempname_for_thumbnails}-%002d.png']
|
|
623
965
|
self.run_program_argsasarray("ffmpeg", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
624
966
|
files = GeneralUtilities.get_direct_files_of_folder(folder)
|
|
625
967
|
result: list[str] = []
|
|
@@ -636,8 +978,17 @@ class ScriptCollectionCore:
|
|
|
636
978
|
def __create_thumbnail(self, outputfilename: str, folder: str, length_in_seconds: float, tempname_for_thumbnails: str, amount_of_images: int) -> None:
|
|
637
979
|
duration = timedelta(seconds=length_in_seconds)
|
|
638
980
|
info = GeneralUtilities.timedelta_to_simple_string(duration)
|
|
639
|
-
|
|
640
|
-
|
|
981
|
+
next_square_number = GeneralUtilities.get_next_square_number(amount_of_images)
|
|
982
|
+
root = math.sqrt(next_square_number)
|
|
983
|
+
rows: int = root # 5
|
|
984
|
+
columns: int = root # math.ceil(amount_of_images/rows)
|
|
985
|
+
argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{rows}x{columns}', f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
|
|
986
|
+
self.run_program_argsasarray("montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
987
|
+
|
|
988
|
+
@GeneralUtilities.check_arguments
|
|
989
|
+
def __create_thumbnail2(self, outputfilename: str, folder: str, length_in_seconds: float, rows: int, columns: int, tempname_for_thumbnails: str, amount_of_images: int) -> None:
|
|
990
|
+
duration = timedelta(seconds=length_in_seconds)
|
|
991
|
+
info = GeneralUtilities.timedelta_to_simple_string(duration)
|
|
641
992
|
argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{rows}x{columns}', f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
|
|
642
993
|
self.run_program_argsasarray("montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
643
994
|
|
|
@@ -650,7 +1001,7 @@ class ScriptCollectionCore:
|
|
|
650
1001
|
return math.ceil(x * d) / d
|
|
651
1002
|
|
|
652
1003
|
@GeneralUtilities.check_arguments
|
|
653
|
-
def generate_thumbnail(self, file: str, frames_per_second:
|
|
1004
|
+
def generate_thumbnail(self, file: str, frames_per_second: float, tempname_for_thumbnails: str = None, hook=None) -> None:
|
|
654
1005
|
if tempname_for_thumbnails is None:
|
|
655
1006
|
tempname_for_thumbnails = "t_"+str(uuid.uuid4())
|
|
656
1007
|
|
|
@@ -661,16 +1012,9 @@ class ScriptCollectionCore:
|
|
|
661
1012
|
preview_files: list[str] = []
|
|
662
1013
|
try:
|
|
663
1014
|
length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
frames_per_second_as_string = str(frames_per_second)
|
|
668
|
-
amounf_of_previewframes = int(math.floor(length_in_seconds*frames_per_second))
|
|
669
|
-
else:
|
|
670
|
-
# concrete amount of frame, examples: frames_per_second="16" => 16 frames for entire video
|
|
671
|
-
amounf_of_previewframes = int(float(frames_per_second))
|
|
672
|
-
# self.roundup((amounf_of_previewframes-2)/length_in_seconds, 2)
|
|
673
|
-
frames_per_second_as_string = f"{amounf_of_previewframes-2}/{length_in_seconds}"
|
|
1015
|
+
# frames per second, example: frames_per_second="20fps" => 20 frames per second
|
|
1016
|
+
frames_per_second = self.__roundup(float(frames_per_second[:-3]), 2)
|
|
1017
|
+
frames_per_second_as_string = str(frames_per_second)
|
|
674
1018
|
preview_files = self.__create_thumbnails(filename, frames_per_second_as_string, folder, tempname_for_thumbnails)
|
|
675
1019
|
if hook is not None:
|
|
676
1020
|
hook(file, preview_files)
|
|
@@ -680,10 +1024,33 @@ class ScriptCollectionCore:
|
|
|
680
1024
|
for thumbnail_to_delete in preview_files:
|
|
681
1025
|
os.remove(thumbnail_to_delete)
|
|
682
1026
|
|
|
1027
|
+
@GeneralUtilities.check_arguments
|
|
1028
|
+
def generate_thumbnail_by_amount_of_pictures(self, file: str, amount_of_columns: int, amount_of_rows: int, tempname_for_thumbnails: str = None, hook=None) -> None:
|
|
1029
|
+
if tempname_for_thumbnails is None:
|
|
1030
|
+
tempname_for_thumbnails = "t_"+str(uuid.uuid4())
|
|
1031
|
+
|
|
1032
|
+
file = GeneralUtilities.resolve_relative_path_from_current_working_directory(file)
|
|
1033
|
+
filename = os.path.basename(file)
|
|
1034
|
+
folder = os.path.dirname(file)
|
|
1035
|
+
filename_without_extension = Path(file).stem
|
|
1036
|
+
preview_files: list[str] = []
|
|
1037
|
+
try:
|
|
1038
|
+
length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
|
|
1039
|
+
amounf_of_previewframes = int(amount_of_columns*amount_of_rows)
|
|
1040
|
+
frames_per_second_as_string = f"{amounf_of_previewframes-2}/{length_in_seconds}"
|
|
1041
|
+
preview_files = self.__create_thumbnails(filename, frames_per_second_as_string, folder, tempname_for_thumbnails)
|
|
1042
|
+
if hook is not None:
|
|
1043
|
+
hook(file, preview_files)
|
|
1044
|
+
actual_amounf_of_previewframes = len(preview_files)
|
|
1045
|
+
self.__create_thumbnail2(filename_without_extension, folder, length_in_seconds, amount_of_rows, amount_of_columns, tempname_for_thumbnails, actual_amounf_of_previewframes)
|
|
1046
|
+
finally:
|
|
1047
|
+
for thumbnail_to_delete in preview_files:
|
|
1048
|
+
os.remove(thumbnail_to_delete)
|
|
1049
|
+
|
|
683
1050
|
@GeneralUtilities.check_arguments
|
|
684
1051
|
def extract_pdf_pages(self, file: str, from_page: int, to_page: int, outputfile: str) -> None:
|
|
685
|
-
pdf_reader =
|
|
686
|
-
pdf_writer =
|
|
1052
|
+
pdf_reader: PdfReader = PdfReader(file)
|
|
1053
|
+
pdf_writer: PdfWriter = PdfWriter()
|
|
687
1054
|
start = from_page
|
|
688
1055
|
end = to_page
|
|
689
1056
|
while start <= end:
|
|
@@ -695,11 +1062,13 @@ class ScriptCollectionCore:
|
|
|
695
1062
|
@GeneralUtilities.check_arguments
|
|
696
1063
|
def merge_pdf_files(self, files: list[str], outputfile: str) -> None:
|
|
697
1064
|
# TODO add wildcard-option
|
|
698
|
-
pdfFileMerger =
|
|
1065
|
+
pdfFileMerger: PdfWriter = PdfWriter()
|
|
699
1066
|
for file in files:
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
1067
|
+
with open(file, "rb") as f:
|
|
1068
|
+
pdfFileMerger.append(f)
|
|
1069
|
+
with open(outputfile, "wb") as output:
|
|
1070
|
+
pdfFileMerger.write(output)
|
|
1071
|
+
pdfFileMerger.close()
|
|
703
1072
|
|
|
704
1073
|
@GeneralUtilities.check_arguments
|
|
705
1074
|
def pdf_to_image(self, file: str, outputfilename_without_extension: str) -> None:
|
|
@@ -737,10 +1106,10 @@ class ScriptCollectionCore:
|
|
|
737
1106
|
elif (size_string.endswith("gib")):
|
|
738
1107
|
size = int(size_string[:-3]) * pow(2, 30)
|
|
739
1108
|
else:
|
|
740
|
-
|
|
1109
|
+
self.log.log("Wrong format", LogLevel.Error)
|
|
741
1110
|
return 1
|
|
742
1111
|
else:
|
|
743
|
-
|
|
1112
|
+
self.log.log("Wrong format", LogLevel.Error)
|
|
744
1113
|
return 1
|
|
745
1114
|
with open(name, "wb") as f:
|
|
746
1115
|
f.seek(size-1)
|
|
@@ -804,7 +1173,7 @@ class ScriptCollectionCore:
|
|
|
804
1173
|
|
|
805
1174
|
return 0
|
|
806
1175
|
else:
|
|
807
|
-
|
|
1176
|
+
self.log.log(f"File '{file}' does not exist.", LogLevel.Error)
|
|
808
1177
|
return 1
|
|
809
1178
|
|
|
810
1179
|
@GeneralUtilities.check_arguments
|
|
@@ -880,11 +1249,19 @@ class ScriptCollectionCore:
|
|
|
880
1249
|
for file in GeneralUtilities.absolute_file_paths(folder):
|
|
881
1250
|
self.__check_file(file, searchstring)
|
|
882
1251
|
|
|
1252
|
+
@GeneralUtilities.check_arguments
|
|
1253
|
+
def get_string_as_qr_code(self,string: str) -> None:
|
|
1254
|
+
qr = qrcode.QRCode()
|
|
1255
|
+
qr.add_data(string)
|
|
1256
|
+
f = io.StringIO()
|
|
1257
|
+
qr.print_ascii(out=f)
|
|
1258
|
+
f.seek(0)
|
|
1259
|
+
return f.read()
|
|
1260
|
+
|
|
883
1261
|
@GeneralUtilities.check_arguments
|
|
884
1262
|
def __print_qr_code_by_csv_line(self, displayname: str, website: str, emailaddress: str, key: str, period: str) -> None:
|
|
885
1263
|
qrcode_content = f"otpauth://totp/{website}:{emailaddress}?secret={key}&issuer={displayname}&period={period}"
|
|
886
|
-
GeneralUtilities.write_message_to_stdout(
|
|
887
|
-
f"{displayname} ({emailaddress}):")
|
|
1264
|
+
GeneralUtilities.write_message_to_stdout(f"{displayname} ({emailaddress}):")
|
|
888
1265
|
GeneralUtilities.write_message_to_stdout(qrcode_content)
|
|
889
1266
|
qr = qrcode.QRCode()
|
|
890
1267
|
qr.add_data(qrcode_content)
|
|
@@ -895,14 +1272,11 @@ class ScriptCollectionCore:
|
|
|
895
1272
|
|
|
896
1273
|
@GeneralUtilities.check_arguments
|
|
897
1274
|
def SCShow2FAAsQRCode(self, csvfile: str) -> None:
|
|
898
|
-
separator_line = "--------------------------------------------------------"
|
|
899
1275
|
lines = GeneralUtilities.read_csv_file(csvfile, True)
|
|
900
1276
|
lines.sort(key=lambda items: ''.join(items).lower())
|
|
901
1277
|
for line in lines:
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
line[0], line[1], line[2], line[3], line[4])
|
|
905
|
-
GeneralUtilities.write_message_to_stdout(separator_line)
|
|
1278
|
+
self.__print_qr_code_by_csv_line(line[0], line[1], line[2], line[3], line[4])
|
|
1279
|
+
GeneralUtilities.write_message_to_stdout(GeneralUtilities.get_longline())
|
|
906
1280
|
|
|
907
1281
|
@GeneralUtilities.check_arguments
|
|
908
1282
|
def SCCalculateBitcoinBlockHash(self, block_version_number: str, previousblockhash: str, transactionsmerkleroot: str, timestamp: str, target: str, nonce: str) -> str:
|
|
@@ -935,7 +1309,7 @@ class ScriptCollectionCore:
|
|
|
935
1309
|
def __adjust_folder_name(self, folder: str) -> str:
|
|
936
1310
|
result = os.path.dirname(folder).replace("\\", "/")
|
|
937
1311
|
if result == "/":
|
|
938
|
-
return
|
|
1312
|
+
return GeneralUtilities.empty_string
|
|
939
1313
|
else:
|
|
940
1314
|
return result
|
|
941
1315
|
|
|
@@ -1068,15 +1442,15 @@ class ScriptCollectionCore:
|
|
|
1068
1442
|
|
|
1069
1443
|
@GeneralUtilities.check_arguments
|
|
1070
1444
|
def get_docker_debian_version(self, image_tag: str) -> str:
|
|
1071
|
-
result = ScriptCollectionCore().run_program_argsasarray(
|
|
1072
|
-
|
|
1073
|
-
result_line = GeneralUtilities.string_to_lines(result[1])[-2]
|
|
1445
|
+
result = ScriptCollectionCore().run_program_argsasarray("docker", ['run', f'debian:{image_tag}', 'bash', '-c', 'apt-get -y update && apt-get -y install lsb-release && lsb_release -cs'])
|
|
1446
|
+
result_line = GeneralUtilities.string_to_lines(result[1])[-1]
|
|
1074
1447
|
return result_line
|
|
1075
1448
|
|
|
1076
1449
|
@GeneralUtilities.check_arguments
|
|
1077
1450
|
def get_latest_tor_version_of_debian_repository(self, debian_version: str) -> str:
|
|
1078
1451
|
package_url: str = f"https://deb.torproject.org/torproject.org/dists/{debian_version}/main/binary-amd64/Packages"
|
|
1079
|
-
|
|
1452
|
+
headers = {'Cache-Control': 'no-cache'}
|
|
1453
|
+
r = requests.get(package_url, timeout=5, headers=headers)
|
|
1080
1454
|
if r.status_code != 200:
|
|
1081
1455
|
raise ValueError(f"Checking for latest tor package resulted in HTTP-response-code {r.status_code}.")
|
|
1082
1456
|
lines = GeneralUtilities.string_to_lines(GeneralUtilities.bytes_to_string(r.content))
|
|
@@ -1087,12 +1461,13 @@ class ScriptCollectionCore:
|
|
|
1087
1461
|
return tor_version
|
|
1088
1462
|
|
|
1089
1463
|
def run_testcases_for_python_project(self, repository_folder: str):
|
|
1464
|
+
self.assert_is_git_repository(repository_folder)
|
|
1090
1465
|
self.run_program("coverage", "run -m pytest", repository_folder)
|
|
1091
1466
|
self.run_program("coverage", "xml", repository_folder)
|
|
1092
1467
|
GeneralUtilities.ensure_directory_exists(os.path.join(repository_folder, "Other/TestCoverage"))
|
|
1093
1468
|
coveragefile = os.path.join(repository_folder, "Other/TestCoverage/TestCoverage.xml")
|
|
1094
1469
|
GeneralUtilities.ensure_file_does_not_exist(coveragefile)
|
|
1095
|
-
os.rename(os.path.join(repository_folder,
|
|
1470
|
+
os.rename(os.path.join(repository_folder, "coverage.xml"), coveragefile)
|
|
1096
1471
|
|
|
1097
1472
|
@GeneralUtilities.check_arguments
|
|
1098
1473
|
def get_file_permission(self, file: str) -> str:
|
|
@@ -1102,11 +1477,11 @@ class ScriptCollectionCore:
|
|
|
1102
1477
|
|
|
1103
1478
|
@GeneralUtilities.check_arguments
|
|
1104
1479
|
def __get_file_permission_helper(self, permissions: str) -> str:
|
|
1105
|
-
return str(self.__to_octet(permissions[0:3]))+str(self.__to_octet(permissions[3:6]))+str(self.__to_octet(permissions[6:9]))
|
|
1480
|
+
return str(self.__to_octet(permissions[0:3])) + str(self.__to_octet(permissions[3:6]))+str(self.__to_octet(permissions[6:9]))
|
|
1106
1481
|
|
|
1107
1482
|
@GeneralUtilities.check_arguments
|
|
1108
1483
|
def __to_octet(self, string: str) -> int:
|
|
1109
|
-
return int(self.__to_octet_helper(string[0])+self.__to_octet_helper(string[1])+self.__to_octet_helper(string[2]), 2)
|
|
1484
|
+
return int(self.__to_octet_helper(string[0]) + self.__to_octet_helper(string[1])+self.__to_octet_helper(string[2]), 2)
|
|
1110
1485
|
|
|
1111
1486
|
@GeneralUtilities.check_arguments
|
|
1112
1487
|
def __to_octet_helper(self, string: str) -> str:
|
|
@@ -1138,9 +1513,8 @@ class ScriptCollectionCore:
|
|
|
1138
1513
|
ls_result = self.run_program_argsasarray("ls", ["-ld", file_or_folder])
|
|
1139
1514
|
GeneralUtilities.assert_condition(ls_result[0] == 0, f"'ls -ld {file_or_folder}' resulted in exitcode {str(ls_result[0])}. StdErr: {ls_result[2]}")
|
|
1140
1515
|
GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(ls_result[1]), f"'ls -ld' of '{file_or_folder}' had an empty output. StdErr: '{ls_result[2]}'")
|
|
1141
|
-
GeneralUtilities.write_message_to_stdout(ls_result[1])
|
|
1142
1516
|
output = ls_result[1]
|
|
1143
|
-
result = output.replace("\n",
|
|
1517
|
+
result = output.replace("\n", GeneralUtilities.empty_string)
|
|
1144
1518
|
result = ' '.join(result.split()) # reduce multiple whitespaces to one
|
|
1145
1519
|
return result
|
|
1146
1520
|
|
|
@@ -1151,7 +1525,6 @@ class ScriptCollectionCore:
|
|
|
1151
1525
|
ls_result = self.run_program_argsasarray("ls", ["-la", file_or_folder])
|
|
1152
1526
|
GeneralUtilities.assert_condition(ls_result[0] == 0, f"'ls -la {file_or_folder}' resulted in exitcode {str(ls_result[0])}. StdErr: {ls_result[2]}")
|
|
1153
1527
|
GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(ls_result[1]), f"'ls -la' of '{file_or_folder}' had an empty output. StdErr: '{ls_result[2]}'")
|
|
1154
|
-
GeneralUtilities.write_message_to_stdout(ls_result[1])
|
|
1155
1528
|
output = ls_result[1]
|
|
1156
1529
|
result = output.split("\n")[3:] # skip the lines with "Total", "." and ".."
|
|
1157
1530
|
result = [' '.join(line.split()) for line in result] # reduce multiple whitespaces to one
|
|
@@ -1182,54 +1555,93 @@ class ScriptCollectionCore:
|
|
|
1182
1555
|
# <run programs>
|
|
1183
1556
|
|
|
1184
1557
|
@GeneralUtilities.check_arguments
|
|
1185
|
-
def __run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None,
|
|
1186
|
-
# Verbosity:
|
|
1187
|
-
# 0=Quiet (No output will be printed.)
|
|
1188
|
-
# 1=Normal (If the exitcode of the executed program is not 0 then the StdErr will be printed.)
|
|
1189
|
-
# 2=Full (Prints StdOut and StdErr of the executed program.)
|
|
1190
|
-
# 3=Verbose (Same as "Full" but with some more information.)
|
|
1191
|
-
|
|
1192
|
-
if isinstance(self.program_runner, ProgramRunnerEpew):
|
|
1193
|
-
custom_argument = CustomEpewArgument(print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, verbosity, arguments_for_log)
|
|
1558
|
+
def __run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> Popen:
|
|
1194
1559
|
popen: Popen = self.program_runner.run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, custom_argument, interactive)
|
|
1195
1560
|
return popen
|
|
1196
1561
|
|
|
1197
1562
|
@staticmethod
|
|
1198
|
-
def __enqueue_output(file, queue):
|
|
1563
|
+
def __enqueue_output(file: IO, queue: Queue):
|
|
1199
1564
|
for line in iter(file.readline, ''):
|
|
1200
1565
|
queue.put(line)
|
|
1201
1566
|
file.close()
|
|
1202
1567
|
|
|
1203
1568
|
@staticmethod
|
|
1204
|
-
def
|
|
1569
|
+
def __continue_process_reading(pid: int, p: Popen, q_stdout: Queue, q_stderr: Queue, reading_stdout_last_time_resulted_in_exception: bool, reading_stderr_last_time_resulted_in_exception: bool):
|
|
1570
|
+
if p.poll() is None:
|
|
1571
|
+
return True
|
|
1572
|
+
|
|
1573
|
+
# if reading_stdout_last_time_resulted_in_exception and reading_stderr_last_time_resulted_in_exception:
|
|
1574
|
+
# return False
|
|
1575
|
+
|
|
1576
|
+
if not q_stdout.empty():
|
|
1577
|
+
return True
|
|
1578
|
+
|
|
1579
|
+
if not q_stderr.empty():
|
|
1580
|
+
return True
|
|
1581
|
+
|
|
1582
|
+
return False
|
|
1583
|
+
|
|
1584
|
+
@staticmethod
|
|
1585
|
+
def __read_popen_pipes(p: Popen, print_live_output: bool, print_errors_as_information: bool, log: SCLog) -> tuple[list[str], list[str]]:
|
|
1586
|
+
p_id = p.pid
|
|
1205
1587
|
with ThreadPoolExecutor(2) as pool:
|
|
1206
1588
|
q_stdout = Queue()
|
|
1207
1589
|
q_stderr = Queue()
|
|
1208
1590
|
|
|
1209
1591
|
pool.submit(ScriptCollectionCore.__enqueue_output, p.stdout, q_stdout)
|
|
1210
1592
|
pool.submit(ScriptCollectionCore.__enqueue_output, p.stderr, q_stderr)
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1593
|
+
reading_stdout_last_time_resulted_in_exception: bool = False
|
|
1594
|
+
reading_stderr_last_time_resulted_in_exception: bool = False
|
|
1595
|
+
|
|
1596
|
+
stdout_result: list[str] = []
|
|
1597
|
+
stderr_result: list[str] = []
|
|
1598
|
+
|
|
1599
|
+
while (ScriptCollectionCore.__continue_process_reading(p_id, p, q_stdout, q_stderr, reading_stdout_last_time_resulted_in_exception, reading_stderr_last_time_resulted_in_exception)):
|
|
1215
1600
|
try:
|
|
1216
|
-
|
|
1601
|
+
while not q_stdout.empty():
|
|
1602
|
+
out_line: str = q_stdout.get_nowait()
|
|
1603
|
+
out_line = out_line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
|
|
1604
|
+
if GeneralUtilities.string_has_content(out_line):
|
|
1605
|
+
stdout_result.append(out_line)
|
|
1606
|
+
reading_stdout_last_time_resulted_in_exception = False
|
|
1607
|
+
if print_live_output:
|
|
1608
|
+
loglevel = LogLevel.Information
|
|
1609
|
+
if out_line.startswith("Debug: "):
|
|
1610
|
+
loglevel = LogLevel.Debug
|
|
1611
|
+
out_line = out_line[len("Debug: "):]
|
|
1612
|
+
if out_line.startswith("Diagnostic: "):
|
|
1613
|
+
loglevel = LogLevel.Diagnostic
|
|
1614
|
+
out_line = out_line[len("Diagnostic: "):]
|
|
1615
|
+
log.log(out_line, loglevel)
|
|
1217
1616
|
except Empty:
|
|
1218
|
-
|
|
1617
|
+
reading_stdout_last_time_resulted_in_exception = True
|
|
1618
|
+
|
|
1219
1619
|
try:
|
|
1220
|
-
|
|
1620
|
+
while not q_stderr.empty():
|
|
1621
|
+
err_line: str = q_stderr.get_nowait()
|
|
1622
|
+
err_line = err_line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
|
|
1623
|
+
if GeneralUtilities.string_has_content(err_line):
|
|
1624
|
+
stderr_result.append(err_line)
|
|
1625
|
+
reading_stderr_last_time_resulted_in_exception = False
|
|
1626
|
+
if print_live_output:
|
|
1627
|
+
loglevel = LogLevel.Error
|
|
1628
|
+
if err_line.startswith("Warning: "):
|
|
1629
|
+
loglevel = LogLevel.Warning
|
|
1630
|
+
err_line = err_line[len("Warning: "):]
|
|
1631
|
+
if print_errors_as_information: # "errors" in "print_errors_as_information" means: all what is written to std-err
|
|
1632
|
+
loglevel = LogLevel.Information
|
|
1633
|
+
log.log(err_line, loglevel)
|
|
1221
1634
|
except Empty:
|
|
1222
|
-
|
|
1635
|
+
reading_stderr_last_time_resulted_in_exception = True
|
|
1223
1636
|
|
|
1224
|
-
|
|
1637
|
+
time.sleep(0.01) # this is required to not finish too early
|
|
1638
|
+
|
|
1639
|
+
return (stdout_result, stderr_result)
|
|
1225
1640
|
|
|
1226
|
-
# Return-values program_runner: Exitcode, StdOut, StdErr, Pid
|
|
1227
1641
|
@GeneralUtilities.check_arguments
|
|
1228
|
-
def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None,
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
# verbosity 3: Logs and prints StdOut and StdErr of the executed program in realtime.
|
|
1232
|
-
# verbosity 4: Same as loglevel 3 but with some more overhead-information.
|
|
1642
|
+
def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False) -> tuple[int, str, str, int]:
|
|
1643
|
+
if self.call_program_runner_directly:
|
|
1644
|
+
return self.program_runner.run_program_argsasarray(program, arguments_as_array, working_directory, custom_argument, interactive)
|
|
1233
1645
|
try:
|
|
1234
1646
|
arguments_as_str = ' '.join(arguments_as_array)
|
|
1235
1647
|
mock_loader_result = self.__try_load_mock(program, arguments_as_str, working_directory)
|
|
@@ -1241,31 +1653,32 @@ class ScriptCollectionCore:
|
|
|
1241
1653
|
if arguments_for_log is None:
|
|
1242
1654
|
arguments_for_log = arguments_as_array
|
|
1243
1655
|
|
|
1244
|
-
|
|
1245
|
-
|
|
1656
|
+
cmd = f'{working_directory}>{program}'
|
|
1657
|
+
if 0 < len(arguments_for_log):
|
|
1658
|
+
arguments_for_log_as_string: str = ' '.join([f'"{argument_for_log}"' for argument_for_log in arguments_for_log])
|
|
1659
|
+
cmd = f'{cmd} {arguments_for_log_as_string}'
|
|
1246
1660
|
|
|
1247
1661
|
if GeneralUtilities.string_is_none_or_whitespace(title):
|
|
1248
1662
|
info_for_log = cmd
|
|
1249
1663
|
else:
|
|
1250
1664
|
info_for_log = title
|
|
1251
1665
|
|
|
1252
|
-
|
|
1253
|
-
GeneralUtilities.write_message_to_stdout(f"Run '{info_for_log}'.")
|
|
1254
|
-
|
|
1255
|
-
print_live_output = 1 < verbosity
|
|
1666
|
+
self.log.log(f"Run '{info_for_log}'.", LogLevel.Debug)
|
|
1256
1667
|
|
|
1257
1668
|
exit_code: int = None
|
|
1258
|
-
stdout: str =
|
|
1259
|
-
stderr: str =
|
|
1669
|
+
stdout: str = GeneralUtilities.empty_string
|
|
1670
|
+
stderr: str = GeneralUtilities.empty_string
|
|
1260
1671
|
pid: int = None
|
|
1261
1672
|
|
|
1262
|
-
with self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory,
|
|
1673
|
+
with self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive) as process:
|
|
1263
1674
|
|
|
1264
1675
|
if log_file is not None:
|
|
1265
1676
|
GeneralUtilities.ensure_file_exists(log_file)
|
|
1266
1677
|
pid = process.pid
|
|
1267
|
-
for out_line_plain, err_line_plain in ScriptCollectionCore.__read_popen_pipes(process): # see https://stackoverflow.com/a/57084403/3905529
|
|
1268
1678
|
|
|
1679
|
+
outputs: tuple[list[str], list[str]] = ScriptCollectionCore.__read_popen_pipes(process, print_live_output, print_errors_as_information, self.log)
|
|
1680
|
+
|
|
1681
|
+
for out_line_plain in outputs[0]:
|
|
1269
1682
|
if out_line_plain is not None:
|
|
1270
1683
|
out_line: str = None
|
|
1271
1684
|
if isinstance(out_line_plain, str):
|
|
@@ -1278,14 +1691,13 @@ class ScriptCollectionCore:
|
|
|
1278
1691
|
if out_line is not None and GeneralUtilities.string_has_content(out_line):
|
|
1279
1692
|
if out_line.endswith("\n"):
|
|
1280
1693
|
out_line = out_line[:-1]
|
|
1281
|
-
if print_live_output:
|
|
1282
|
-
print(out_line, end='\n', file=sys.stdout, flush=True)
|
|
1283
1694
|
if 0 < len(stdout):
|
|
1284
1695
|
stdout = stdout+"\n"
|
|
1285
1696
|
stdout = stdout+out_line
|
|
1286
1697
|
if log_file is not None:
|
|
1287
1698
|
GeneralUtilities.append_line_to_file(log_file, out_line)
|
|
1288
1699
|
|
|
1700
|
+
for err_line_plain in outputs[1]:
|
|
1289
1701
|
if err_line_plain is not None:
|
|
1290
1702
|
err_line: str = None
|
|
1291
1703
|
if isinstance(err_line_plain, str):
|
|
@@ -1297,8 +1709,6 @@ class ScriptCollectionCore:
|
|
|
1297
1709
|
if err_line is not None and GeneralUtilities.string_has_content(err_line):
|
|
1298
1710
|
if err_line.endswith("\n"):
|
|
1299
1711
|
err_line = err_line[:-1]
|
|
1300
|
-
if print_live_output:
|
|
1301
|
-
print(err_line, end='\n', file=sys.stderr, flush=True)
|
|
1302
1712
|
if 0 < len(stderr):
|
|
1303
1713
|
stderr = stderr+"\n"
|
|
1304
1714
|
stderr = stderr+err_line
|
|
@@ -1306,34 +1716,49 @@ class ScriptCollectionCore:
|
|
|
1306
1716
|
GeneralUtilities.append_line_to_file(log_file, err_line)
|
|
1307
1717
|
|
|
1308
1718
|
exit_code = process.returncode
|
|
1719
|
+
GeneralUtilities.assert_condition(exit_code is not None, f"Exitcode of program-run of '{info_for_log}' is None.")
|
|
1720
|
+
|
|
1721
|
+
result_message = f"Program '{info_for_log}' resulted in exitcode {exit_code}."
|
|
1722
|
+
|
|
1723
|
+
self.log.log(result_message, LogLevel.Debug)
|
|
1309
1724
|
|
|
1310
1725
|
if throw_exception_if_exitcode_is_not_zero and exit_code != 0:
|
|
1311
|
-
raise ValueError(f"
|
|
1726
|
+
raise ValueError(f"{result_message} (StdOut: '{stdout}', StdErr: '{stderr}')")
|
|
1312
1727
|
|
|
1313
|
-
GeneralUtilities.assert_condition(exit_code is not None, f"Exitcode of program-run of '{info_for_log}' is None.")
|
|
1314
1728
|
result = (exit_code, stdout, stderr, pid)
|
|
1315
1729
|
return result
|
|
1316
|
-
except Exception as e:
|
|
1317
|
-
raise
|
|
1730
|
+
except Exception as e:#pylint:disable=unused-variable, try-except-raise
|
|
1731
|
+
raise
|
|
1318
1732
|
|
|
1319
1733
|
# Return-values program_runner: Exitcode, StdOut, StdErr, Pid
|
|
1320
1734
|
@GeneralUtilities.check_arguments
|
|
1321
|
-
def
|
|
1322
|
-
return self.
|
|
1735
|
+
def run_program_with_retry(self, program: str, arguments: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False, amount_of_attempts: int = 5) -> tuple[int, str, str, int]:
|
|
1736
|
+
return GeneralUtilities.retry_action(lambda: self.run_program(program, arguments, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output), amount_of_attempts)
|
|
1737
|
+
|
|
1738
|
+
# Return-values program_runner: Exitcode, StdOut, StdErr, Pid
|
|
1739
|
+
@GeneralUtilities.check_arguments
|
|
1740
|
+
def run_program(self, program: str, arguments: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False) -> tuple[int, str, str, int]:
|
|
1741
|
+
if self.call_program_runner_directly:
|
|
1742
|
+
return self.program_runner.run_program(program, arguments, working_directory, custom_argument, interactive)
|
|
1743
|
+
return self.run_program_argsasarray(program, GeneralUtilities.arguments_to_array(arguments), working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output)
|
|
1323
1744
|
|
|
1324
1745
|
# Return-values program_runner: Pid
|
|
1325
1746
|
@GeneralUtilities.check_arguments
|
|
1326
|
-
def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None,
|
|
1747
|
+
def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
|
|
1748
|
+
if self.call_program_runner_directly:
|
|
1749
|
+
return self.program_runner.run_program_argsasarray_async(program, arguments_as_array, working_directory, custom_argument, interactive)
|
|
1327
1750
|
mock_loader_result = self.__try_load_mock(program, ' '.join(arguments_as_array), working_directory)
|
|
1328
1751
|
if mock_loader_result[0]:
|
|
1329
1752
|
return mock_loader_result[1]
|
|
1330
|
-
process: Popen = self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory,
|
|
1753
|
+
process: Popen = self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
|
|
1331
1754
|
return process.pid
|
|
1332
1755
|
|
|
1333
1756
|
# Return-values program_runner: Pid
|
|
1334
1757
|
@GeneralUtilities.check_arguments
|
|
1335
|
-
def run_program_async(self, program: str, arguments: str = "", working_directory: str = None,
|
|
1336
|
-
|
|
1758
|
+
def run_program_async(self, program: str, arguments: str = "", working_directory: str = None,print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
|
|
1759
|
+
if self.call_program_runner_directly:
|
|
1760
|
+
return self.program_runner.run_program_argsasarray_async(program, arguments, working_directory, custom_argument, interactive)
|
|
1761
|
+
return self.run_program_argsasarray_async(program, GeneralUtilities.arguments_to_array(arguments), working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
|
|
1337
1762
|
|
|
1338
1763
|
@GeneralUtilities.check_arguments
|
|
1339
1764
|
def __try_load_mock(self, program: str, arguments: str, working_directory: str) -> tuple[bool, tuple[int, str, str, int]]:
|
|
@@ -1347,10 +1772,17 @@ class ScriptCollectionCore:
|
|
|
1347
1772
|
|
|
1348
1773
|
@GeneralUtilities.check_arguments
|
|
1349
1774
|
def __adapt_workingdirectory(self, workingdirectory: str) -> str:
|
|
1775
|
+
result: str = None
|
|
1350
1776
|
if workingdirectory is None:
|
|
1351
|
-
|
|
1777
|
+
result = os.getcwd()
|
|
1352
1778
|
else:
|
|
1353
|
-
|
|
1779
|
+
if os.path.isabs(workingdirectory):
|
|
1780
|
+
result = workingdirectory
|
|
1781
|
+
else:
|
|
1782
|
+
result = GeneralUtilities.resolve_relative_path_from_current_working_directory(workingdirectory)
|
|
1783
|
+
if not os.path.isdir(result):
|
|
1784
|
+
raise ValueError(f"Working-directory '{workingdirectory}' does not exist.")
|
|
1785
|
+
return result
|
|
1354
1786
|
|
|
1355
1787
|
@GeneralUtilities.check_arguments
|
|
1356
1788
|
def verify_no_pending_mock_program_calls(self):
|
|
@@ -1405,13 +1837,28 @@ class ScriptCollectionCore:
|
|
|
1405
1837
|
stderr: str
|
|
1406
1838
|
pid: int
|
|
1407
1839
|
|
|
1840
|
+
@GeneralUtilities.check_arguments
|
|
1841
|
+
def run_with_epew(self, program: str, argument: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False,print_live_output:bool=False,encode_argument_in_base64:bool=False) -> tuple[int, str, str, int]:
|
|
1842
|
+
epew_argument=["-p",program ,"-w", working_directory]
|
|
1843
|
+
if encode_argument_in_base64:
|
|
1844
|
+
if arguments_for_log is None:
|
|
1845
|
+
arguments_for_log=epew_argument+["-a",f"\"{argument}\""]
|
|
1846
|
+
base64_string = base64.b64encode(argument.encode("utf-8")).decode("utf-8")
|
|
1847
|
+
epew_argument=epew_argument+["-a",base64_string,"-b"]
|
|
1848
|
+
else:
|
|
1849
|
+
epew_argument=epew_argument+["-a",argument]
|
|
1850
|
+
if arguments_for_log is None:
|
|
1851
|
+
arguments_for_log=epew_argument
|
|
1852
|
+
return self.run_program_argsasarray("epew", epew_argument, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive,print_live_output=print_live_output)
|
|
1853
|
+
|
|
1854
|
+
|
|
1408
1855
|
# </run programs>
|
|
1409
1856
|
|
|
1410
1857
|
@GeneralUtilities.check_arguments
|
|
1411
|
-
def extract_archive_with_7z(self, unzip_program_file: str,
|
|
1858
|
+
def extract_archive_with_7z(self, unzip_program_file: str, zip_file: str, password: str, output_directory: str) -> None:
|
|
1412
1859
|
password_set = not password is None
|
|
1413
|
-
file_name = Path(
|
|
1414
|
-
file_folder = os.path.dirname(
|
|
1860
|
+
file_name = Path(zip_file).name
|
|
1861
|
+
file_folder = os.path.dirname(zip_file)
|
|
1415
1862
|
argument = "x"
|
|
1416
1863
|
if password_set:
|
|
1417
1864
|
argument = f"{argument} -p\"{password}\""
|
|
@@ -1426,7 +1873,7 @@ class ScriptCollectionCore:
|
|
|
1426
1873
|
|
|
1427
1874
|
@GeneralUtilities.check_arguments
|
|
1428
1875
|
def system_time_equals_internet_time(self, maximal_tolerance_difference: timedelta) -> bool:
|
|
1429
|
-
return abs(
|
|
1876
|
+
return abs(GeneralUtilities.get_now() - self.get_internet_time()) < maximal_tolerance_difference
|
|
1430
1877
|
|
|
1431
1878
|
@GeneralUtilities.check_arguments
|
|
1432
1879
|
def system_time_equals_internet_time_with_default_tolerance(self) -> bool:
|
|
@@ -1462,6 +1909,7 @@ class ScriptCollectionCore:
|
|
|
1462
1909
|
|
|
1463
1910
|
@GeneralUtilities.check_arguments
|
|
1464
1911
|
def get_semver_version_from_gitversion(self, repository_folder: str) -> str:
|
|
1912
|
+
self.assert_is_git_repository(repository_folder)
|
|
1465
1913
|
if (self.git_repository_has_commits(repository_folder)):
|
|
1466
1914
|
result = self.get_version_from_gitversion(repository_folder, "MajorMinorPatch")
|
|
1467
1915
|
if self.git_repository_has_uncommitted_changes(repository_folder):
|
|
@@ -1483,8 +1931,8 @@ class ScriptCollectionCore:
|
|
|
1483
1931
|
@GeneralUtilities.check_arguments
|
|
1484
1932
|
def get_version_from_gitversion(self, folder: str, variable: str) -> str:
|
|
1485
1933
|
# called twice as workaround for issue 1877 in gitversion ( https://github.com/GitTools/GitVersion/issues/1877 )
|
|
1486
|
-
result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder
|
|
1487
|
-
result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder
|
|
1934
|
+
result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder)
|
|
1935
|
+
result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder)
|
|
1488
1936
|
result = GeneralUtilities.strip_new_line_character(result[1])
|
|
1489
1937
|
|
|
1490
1938
|
return result
|
|
@@ -1496,7 +1944,7 @@ class ScriptCollectionCore:
|
|
|
1496
1944
|
if password is None:
|
|
1497
1945
|
password = GeneralUtilities.generate_password()
|
|
1498
1946
|
GeneralUtilities.ensure_directory_exists(folder)
|
|
1499
|
-
self.
|
|
1947
|
+
self.run_program_argsasarray("openssl", ['req', '-new', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:prime256v1', '-days', str(days_until_expire), '-nodes', '-x509', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={name}/OU={subj_ou}', '-passout', f'pass:{password}', '-keyout', f'{name}.key', '-out', f'{name}.crt'], folder)
|
|
1500
1948
|
|
|
1501
1949
|
@GeneralUtilities.check_arguments
|
|
1502
1950
|
def generate_certificate(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
|
|
@@ -1505,9 +1953,9 @@ class ScriptCollectionCore:
|
|
|
1505
1953
|
if password is None:
|
|
1506
1954
|
password = GeneralUtilities.generate_password()
|
|
1507
1955
|
rsa_key_length = 4096
|
|
1508
|
-
self.
|
|
1509
|
-
self.
|
|
1510
|
-
self.
|
|
1956
|
+
self.run_program_argsasarray("openssl", ['genrsa', '-out', f'{filename}.key', f'{rsa_key_length}'], folder)
|
|
1957
|
+
self.run_program_argsasarray("openssl", ['req', '-new', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou}', '-x509', '-key', f'{filename}.key', '-out', f'{filename}.unsigned.crt', '-days', f'{days_until_expire}'], folder)
|
|
1958
|
+
self.run_program_argsasarray("openssl", ['pkcs12', '-export', '-out', f'{filename}.selfsigned.pfx', '-password', f'pass:{password}', '-inkey', f'{filename}.key', '-in', f'{filename}.unsigned.crt'], folder)
|
|
1511
1959
|
GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.password"), password)
|
|
1512
1960
|
GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.san.conf"), f"""[ req ]
|
|
1513
1961
|
default_bits = {rsa_key_length}
|
|
@@ -1534,7 +1982,7 @@ DNS = {domain}
|
|
|
1534
1982
|
|
|
1535
1983
|
@GeneralUtilities.check_arguments
|
|
1536
1984
|
def generate_certificate_sign_request(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str) -> None:
|
|
1537
|
-
self.
|
|
1985
|
+
self.run_program_argsasarray("openssl", ['req', '-new', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou}', '-key', f'{filename}.key', f'-out', f'{filename}.csr', f'-config', f'{filename}.san.conf'], folder)
|
|
1538
1986
|
|
|
1539
1987
|
@GeneralUtilities.check_arguments
|
|
1540
1988
|
def sign_certificate(self, folder: str, ca_folder: str, ca_name: str, domain: str, filename: str, days_until_expire: int = None) -> None:
|
|
@@ -1543,11 +1991,12 @@ DNS = {domain}
|
|
|
1543
1991
|
ca = os.path.join(ca_folder, ca_name)
|
|
1544
1992
|
password_file = os.path.join(folder, f"{filename}.password")
|
|
1545
1993
|
password = GeneralUtilities.read_text_from_file(password_file)
|
|
1546
|
-
self.
|
|
1547
|
-
self.
|
|
1994
|
+
self.run_program_argsasarray("openssl", ['x509', '-req', '-in', f'{filename}.csr', '-CA', f'{ca}.crt', '-CAkey', f'{ca}.key', '-CAcreateserial', '-CAserial', f'{ca}.srl', '-out', f'{filename}.crt', '-days', str(days_until_expire), '-sha256', '-extensions', 'v3_req', '-extfile', f'{filename}.san.conf'], folder)
|
|
1995
|
+
self.run_program_argsasarray("openssl", ['pkcs12', '-export', '-out', f'{filename}.pfx', f'-inkey', f'{filename}.key', '-in', f'{filename}.crt', '-password', f'pass:{password}'], folder)
|
|
1548
1996
|
|
|
1549
1997
|
@GeneralUtilities.check_arguments
|
|
1550
|
-
def update_dependencies_of_python_in_requirementstxt_file(self, file: str,
|
|
1998
|
+
def update_dependencies_of_python_in_requirementstxt_file(self, file: str, ignored_dependencies: list[str]):
|
|
1999
|
+
# TODO consider ignored_dependencies
|
|
1551
2000
|
lines = GeneralUtilities.read_lines_from_file(file)
|
|
1552
2001
|
new_lines = []
|
|
1553
2002
|
for line in lines:
|
|
@@ -1565,7 +2014,8 @@ DNS = {domain}
|
|
|
1565
2014
|
# (something like "cyclonedx-bom>=2.11.0" for example)
|
|
1566
2015
|
package = line.split(">")[0]
|
|
1567
2016
|
operator = ">=" if ">=" in line else ">"
|
|
1568
|
-
|
|
2017
|
+
headers = {'Cache-Control': 'no-cache'}
|
|
2018
|
+
response = requests.get(f'https://pypi.org/pypi/{package}/json', timeout=5, headers=headers)
|
|
1569
2019
|
latest_version = response.json()['info']['version']
|
|
1570
2020
|
# TODO update only minor- and patch-version
|
|
1571
2021
|
# TODO print info if there is a new major-version
|
|
@@ -1576,7 +2026,8 @@ DNS = {domain}
|
|
|
1576
2026
|
raise ValueError(f'Unexpected line in requirements-file: "{line}"')
|
|
1577
2027
|
|
|
1578
2028
|
@GeneralUtilities.check_arguments
|
|
1579
|
-
def update_dependencies_of_python_in_setupcfg_file(self, setup_cfg_file: str,
|
|
2029
|
+
def update_dependencies_of_python_in_setupcfg_file(self, setup_cfg_file: str, ignored_dependencies: list[str]):
|
|
2030
|
+
# TODO consider ignored_dependencies
|
|
1580
2031
|
lines = GeneralUtilities.read_lines_from_file(setup_cfg_file)
|
|
1581
2032
|
new_lines = []
|
|
1582
2033
|
requirement_parsing_mode = False
|
|
@@ -1595,21 +2046,22 @@ DNS = {domain}
|
|
|
1595
2046
|
GeneralUtilities.write_lines_to_file(setup_cfg_file, new_lines)
|
|
1596
2047
|
|
|
1597
2048
|
@GeneralUtilities.check_arguments
|
|
1598
|
-
def update_dependencies_of_dotnet_project(self, csproj_file: str,
|
|
2049
|
+
def update_dependencies_of_dotnet_project(self, csproj_file: str, ignored_dependencies: list[str]):
|
|
1599
2050
|
folder = os.path.dirname(csproj_file)
|
|
1600
2051
|
csproj_filename = os.path.basename(csproj_file)
|
|
1601
|
-
|
|
1602
|
-
result = self.
|
|
1603
|
-
for line in result[1].replace("\r",
|
|
2052
|
+
self.log.log(f"Check for updates in {csproj_filename}", LogLevel.Information)
|
|
2053
|
+
result = self.run_program_with_retry("dotnet", f"list {csproj_filename} package --outdated", folder, print_errors_as_information=True)
|
|
2054
|
+
for line in result[1].replace("\r", GeneralUtilities.empty_string).split("\n"):
|
|
1604
2055
|
# Relevant output-lines are something like " > NJsonSchema 10.7.0 10.7.0 10.9.0"
|
|
1605
2056
|
if ">" in line:
|
|
1606
|
-
package_name = line.replace(">",
|
|
1607
|
-
if not (
|
|
1608
|
-
|
|
1609
|
-
|
|
2057
|
+
package_name = line.replace(">", GeneralUtilities.empty_string).strip().split(" ")[0]
|
|
2058
|
+
if not (package_name in ignored_dependencies):
|
|
2059
|
+
self.log.log(f"Update package {package_name}...", LogLevel.Debug)
|
|
2060
|
+
time.sleep(1.1) # attempt to prevent rate-limit
|
|
2061
|
+
self.run_program_with_retry("dotnet", f"add {csproj_filename} package {package_name}", folder, print_errors_as_information=True)
|
|
1610
2062
|
|
|
1611
2063
|
@GeneralUtilities.check_arguments
|
|
1612
|
-
def create_deb_package(self, toolname: str, binary_folder: str, control_file_content: str, deb_output_folder: str,
|
|
2064
|
+
def create_deb_package(self, toolname: str, binary_folder: str, control_file_content: str, deb_output_folder: str, permission_of_executable_file_as_octet_triple: int) -> None:
|
|
1613
2065
|
|
|
1614
2066
|
# prepare
|
|
1615
2067
|
GeneralUtilities.ensure_directory_exists(deb_output_folder)
|
|
@@ -1672,9 +2124,9 @@ chmod {permission} {link_file}
|
|
|
1672
2124
|
|
|
1673
2125
|
# create debfile
|
|
1674
2126
|
deb_filename = f"{toolname}.deb"
|
|
1675
|
-
self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/control.tar.gz", "*"], packagecontent_control_folder
|
|
1676
|
-
self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/data.tar.gz", "*"], packagecontent_data_folder
|
|
1677
|
-
self.run_program_argsasarray("ar", ["r", deb_filename, "debian-binary", "control.tar.gz", "data.tar.gz"], packagecontent_entireresult_folder
|
|
2127
|
+
self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/control.tar.gz", "*"], packagecontent_control_folder)
|
|
2128
|
+
self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/data.tar.gz", "*"], packagecontent_data_folder)
|
|
2129
|
+
self.run_program_argsasarray("ar", ["r", deb_filename, "debian-binary", "control.tar.gz", "data.tar.gz"], packagecontent_entireresult_folder)
|
|
1678
2130
|
result_file = os.path.join(packagecontent_entireresult_folder, deb_filename)
|
|
1679
2131
|
shutil.copy(result_file, os.path.join(deb_output_folder, deb_filename))
|
|
1680
2132
|
|
|
@@ -1683,7 +2135,7 @@ chmod {permission} {link_file}
|
|
|
1683
2135
|
|
|
1684
2136
|
@GeneralUtilities.check_arguments
|
|
1685
2137
|
def update_year_in_copyright_tags(self, file: str) -> None:
|
|
1686
|
-
current_year = str(
|
|
2138
|
+
current_year = str(GeneralUtilities.get_now().year)
|
|
1687
2139
|
lines = GeneralUtilities.read_lines_from_file(file)
|
|
1688
2140
|
lines_result = []
|
|
1689
2141
|
for line in lines:
|
|
@@ -1698,31 +2150,28 @@ chmod {permission} {link_file}
|
|
|
1698
2150
|
|
|
1699
2151
|
@GeneralUtilities.check_arguments
|
|
1700
2152
|
def update_year_in_first_line_of_file(self, file: str) -> None:
|
|
1701
|
-
current_year = str(
|
|
2153
|
+
current_year = str(GeneralUtilities.get_now().year)
|
|
1702
2154
|
lines = GeneralUtilities.read_lines_from_file(file)
|
|
1703
2155
|
lines[0] = re.sub("\\d\\d\\d\\d", current_year, lines[0])
|
|
1704
2156
|
GeneralUtilities.write_lines_to_file(file, lines)
|
|
1705
2157
|
|
|
1706
2158
|
@GeneralUtilities.check_arguments
|
|
1707
|
-
def
|
|
1708
|
-
information = self.get_externalnetworkinformation_as_json_string(
|
|
2159
|
+
def get_external_ip_address(self) -> str:
|
|
2160
|
+
information = self.get_externalnetworkinformation_as_json_string()
|
|
1709
2161
|
parsed = json.loads(information)
|
|
1710
|
-
return parsed
|
|
2162
|
+
return parsed["IPAddress"]
|
|
1711
2163
|
|
|
1712
2164
|
@GeneralUtilities.check_arguments
|
|
1713
|
-
def
|
|
1714
|
-
information = self.get_externalnetworkinformation_as_json_string(
|
|
2165
|
+
def get_country_of_external_ip_address(self) -> str:
|
|
2166
|
+
information = self.get_externalnetworkinformation_as_json_string()
|
|
1715
2167
|
parsed = json.loads(information)
|
|
1716
|
-
return parsed
|
|
2168
|
+
return parsed["Country"]
|
|
1717
2169
|
|
|
1718
2170
|
@GeneralUtilities.check_arguments
|
|
1719
|
-
def get_externalnetworkinformation_as_json_string(self,
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
response = requests.get('https://ipinfo.io', proxies=proxies, timeout=5)
|
|
1724
|
-
network_information_as_json_string = GeneralUtilities.bytes_to_string(
|
|
1725
|
-
response.content)
|
|
2171
|
+
def get_externalnetworkinformation_as_json_string(self,clientinformation_link:str='https://clientinformation.anion327.de') -> str:
|
|
2172
|
+
headers = {'Cache-Control': 'no-cache'}
|
|
2173
|
+
response = requests.get(clientinformation_link, timeout=5, headers=headers)
|
|
2174
|
+
network_information_as_json_string = GeneralUtilities.bytes_to_string(response.content)
|
|
1726
2175
|
return network_information_as_json_string
|
|
1727
2176
|
|
|
1728
2177
|
@GeneralUtilities.check_arguments
|
|
@@ -1757,11 +2206,13 @@ chmod {permission} {link_file}
|
|
|
1757
2206
|
""".replace("XDX", "ODO"))
|
|
1758
2207
|
|
|
1759
2208
|
@GeneralUtilities.check_arguments
|
|
1760
|
-
def generate_arc42_reference_template(self, repository: str, productname: str = None):
|
|
2209
|
+
def generate_arc42_reference_template(self, repository: str, productname: str = None, subfolder: str = None):
|
|
1761
2210
|
productname: str
|
|
1762
2211
|
if productname is None:
|
|
1763
2212
|
productname = os.path.basename(repository)
|
|
1764
|
-
|
|
2213
|
+
if subfolder is None:
|
|
2214
|
+
subfolder = "Other/Reference"
|
|
2215
|
+
reference_root_folder = f"{repository}/{subfolder}"
|
|
1765
2216
|
reference_content_folder = reference_root_folder + "/Technical"
|
|
1766
2217
|
if os.path.isdir(reference_root_folder):
|
|
1767
2218
|
raise ValueError(f"The folder '{reference_root_folder}' does already exist.")
|
|
@@ -1770,7 +2221,7 @@ chmod {permission} {link_file}
|
|
|
1770
2221
|
main_reference_file = f"{reference_root_folder}/Reference.md"
|
|
1771
2222
|
GeneralUtilities.ensure_file_exists(main_reference_file)
|
|
1772
2223
|
GeneralUtilities.write_text_to_file(main_reference_file, f"""# {productname}
|
|
1773
|
-
|
|
2224
|
+
|
|
1774
2225
|
TXDX add minimal service-description here.
|
|
1775
2226
|
|
|
1776
2227
|
## Technical documentation
|
|
@@ -1780,11 +2231,11 @@ TXDX add minimal service-description here.
|
|
|
1780
2231
|
|
|
1781
2232
|
TXDX
|
|
1782
2233
|
|
|
1783
|
-
|
|
2234
|
+
## Quality goals
|
|
1784
2235
|
|
|
1785
|
-
|
|
2236
|
+
TXDX
|
|
1786
2237
|
|
|
1787
|
-
|
|
2238
|
+
## Stakeholder
|
|
1788
2239
|
|
|
1789
2240
|
| Name | How to contact | Reason |
|
|
1790
2241
|
| ---- | -------------- | ------ |""")
|
|
@@ -1803,7 +2254,7 @@ TXDX
|
|
|
1803
2254
|
|
|
1804
2255
|
## Scope
|
|
1805
2256
|
|
|
1806
|
-
|
|
2257
|
+
TXDX""")
|
|
1807
2258
|
self.__add_chapter(main_reference_file, reference_content_folder, 4, 'Solution Strategy', """TXDX""")
|
|
1808
2259
|
self.__add_chapter(main_reference_file, reference_content_folder, 5, 'Building Block View', """TXDX""")
|
|
1809
2260
|
self.__add_chapter(main_reference_file, reference_content_folder, 6, 'Runtime View', """TXDX""")
|
|
@@ -1817,8 +2268,7 @@ TXDX
|
|
|
1817
2268
|
|
|
1818
2269
|
## Deployment-proecsses
|
|
1819
2270
|
|
|
1820
|
-
TXDX
|
|
1821
|
-
""")
|
|
2271
|
+
TXDX""")
|
|
1822
2272
|
self.__add_chapter(main_reference_file, reference_content_folder, 8, 'Crosscutting Concepts', """TXDX""")
|
|
1823
2273
|
self.__add_chapter(main_reference_file, reference_content_folder, 9, 'Architectural Decisions', """## Decision-board
|
|
1824
2274
|
|
|
@@ -1861,5 +2311,231 @@ TXDX
|
|
|
1861
2311
|
- [Repository](TXDX)
|
|
1862
2312
|
- [Productive-System](TXDX)
|
|
1863
2313
|
- [QualityCheck-system](TXDX)
|
|
2314
|
+
""".replace("XDX", "ODO"))
|
|
1864
2315
|
|
|
1865
|
-
|
|
2316
|
+
@GeneralUtilities.check_arguments
|
|
2317
|
+
def run_with_timeout(self, method, timeout_in_seconds: float) -> bool:
|
|
2318
|
+
# Returns true if the method was terminated due to a timeout
|
|
2319
|
+
# Returns false if the method terminates in the given time
|
|
2320
|
+
p = multiprocessing.Process(target=method)
|
|
2321
|
+
p.start()
|
|
2322
|
+
p.join(timeout_in_seconds)
|
|
2323
|
+
if p.is_alive():
|
|
2324
|
+
p.kill()
|
|
2325
|
+
p.join()
|
|
2326
|
+
return True
|
|
2327
|
+
else:
|
|
2328
|
+
return False
|
|
2329
|
+
|
|
2330
|
+
@GeneralUtilities.check_arguments
|
|
2331
|
+
def ensure_local_docker_network_exists(self, network_name: str) -> None:
|
|
2332
|
+
if not self.local_docker_network_exists(network_name):
|
|
2333
|
+
self.create_local_docker_network(network_name)
|
|
2334
|
+
|
|
2335
|
+
@GeneralUtilities.check_arguments
|
|
2336
|
+
def ensure_local_docker_network_does_not_exist(self, network_name: str) -> None:
|
|
2337
|
+
if self.local_docker_network_exists(network_name):
|
|
2338
|
+
self.remove_local_docker_network(network_name)
|
|
2339
|
+
|
|
2340
|
+
@GeneralUtilities.check_arguments
|
|
2341
|
+
def local_docker_network_exists(self, network_name: str) -> bool:
|
|
2342
|
+
return network_name in self.get_all_local_existing_docker_networks()
|
|
2343
|
+
|
|
2344
|
+
@GeneralUtilities.check_arguments
|
|
2345
|
+
def get_all_local_existing_docker_networks(self) -> list[str]:
|
|
2346
|
+
program_call_result = self.run_program("docker", "network list")
|
|
2347
|
+
std_out = program_call_result[1]
|
|
2348
|
+
std_out_lines = std_out.split("\n")[1:]
|
|
2349
|
+
result: list[str] = []
|
|
2350
|
+
for std_out_line in std_out_lines:
|
|
2351
|
+
normalized_line = ';'.join(std_out_line.split())
|
|
2352
|
+
splitted = normalized_line.split(";")
|
|
2353
|
+
result.append(splitted[1])
|
|
2354
|
+
return result
|
|
2355
|
+
|
|
2356
|
+
@GeneralUtilities.check_arguments
|
|
2357
|
+
def remove_local_docker_network(self, network_name: str) -> None:
|
|
2358
|
+
self.run_program("docker", f"network remove {network_name}")
|
|
2359
|
+
|
|
2360
|
+
@GeneralUtilities.check_arguments
|
|
2361
|
+
def create_local_docker_network(self, network_name: str) -> None:
|
|
2362
|
+
self.run_program("docker", f"network create {network_name}")
|
|
2363
|
+
|
|
2364
|
+
@GeneralUtilities.check_arguments
|
|
2365
|
+
def format_xml_file(self, file: str) -> None:
|
|
2366
|
+
encoding = "utf-8"
|
|
2367
|
+
element = ET.XML(GeneralUtilities.read_text_from_file(file, encoding))
|
|
2368
|
+
ET.indent(element)
|
|
2369
|
+
GeneralUtilities.write_text_to_file(file, ET.tostring(element, encoding="unicode"), encoding)
|
|
2370
|
+
|
|
2371
|
+
@GeneralUtilities.check_arguments
|
|
2372
|
+
def install_requirementstxt_file(self, requirements_txt_file: str):
|
|
2373
|
+
folder: str = os.path.dirname(requirements_txt_file)
|
|
2374
|
+
filename: str = os.path.basename(requirements_txt_file)
|
|
2375
|
+
self.run_program_argsasarray("pip", ["install", "-r", filename], folder)
|
|
2376
|
+
|
|
2377
|
+
@GeneralUtilities.check_arguments
|
|
2378
|
+
def ocr_analysis_of_folder(self, folder: str, serviceaddress: str, extensions: list[str], languages: list[str]) -> list[str]: # Returns a list of changed files due to ocr-analysis.
|
|
2379
|
+
GeneralUtilities.write_message_to_stdout("Starting OCR analysis of folder " + folder)
|
|
2380
|
+
supported_extensions = ['png', 'jpg', 'jpeg', 'tiff', 'bmp', 'gif', 'pdf', 'docx', 'doc', 'xlsx', 'xls', 'pptx', 'ppt']
|
|
2381
|
+
changes_files: list[str] = []
|
|
2382
|
+
if extensions is None:
|
|
2383
|
+
extensions = supported_extensions
|
|
2384
|
+
for file in GeneralUtilities.get_direct_files_of_folder(folder):
|
|
2385
|
+
file_lower = file.lower()
|
|
2386
|
+
for extension in extensions:
|
|
2387
|
+
if file_lower.endswith("."+extension):
|
|
2388
|
+
if self.ocr_analysis_of_file(file, serviceaddress, languages):
|
|
2389
|
+
changes_files.append(file)
|
|
2390
|
+
break
|
|
2391
|
+
for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
|
|
2392
|
+
for file in self.ocr_analysis_of_folder(subfolder, serviceaddress, extensions, languages):
|
|
2393
|
+
changes_files.append(file)
|
|
2394
|
+
return changes_files
|
|
2395
|
+
|
|
2396
|
+
@GeneralUtilities.check_arguments
|
|
2397
|
+
def ocr_analysis_of_file(self, file: str, serviceaddress: str, languages: list[str]) -> bool: # Returns true if the ocr-file was generated or updated. Returns false if the existing ocr-file was not changed.
|
|
2398
|
+
GeneralUtilities.write_message_to_stdout("Do OCR analysis of file " + file)
|
|
2399
|
+
supported_extensions = ['png', 'jpg', 'jpeg', 'tiff', 'bmp', 'webp', 'gif', 'pdf', 'rtf', 'docx', 'doc', 'odt', 'xlsx', 'xls', 'ods', 'pptx', 'ppt', 'odp']
|
|
2400
|
+
for extension in supported_extensions:
|
|
2401
|
+
if file.lower().endswith("."+extension):
|
|
2402
|
+
raise ValueError(f"Extension '{extension}' is not supported. Supported extensions are: {', '.join(supported_extensions)}")
|
|
2403
|
+
target_file = file+".ocr.txt"
|
|
2404
|
+
hash_of_current_file: str = GeneralUtilities. get_sha256_of_file(file)
|
|
2405
|
+
if os.path.isfile(target_file):
|
|
2406
|
+
lines = GeneralUtilities.read_lines_from_file(target_file)
|
|
2407
|
+
previous_hash_of_current_file: str = lines[1].split(":")[1].strip()
|
|
2408
|
+
if hash_of_current_file == previous_hash_of_current_file:
|
|
2409
|
+
return False
|
|
2410
|
+
ocr_content = self.get_ocr_content_of_file(file, serviceaddress, languages)
|
|
2411
|
+
GeneralUtilities.ensure_file_exists(target_file)
|
|
2412
|
+
GeneralUtilities.write_text_to_file(file, f"""Name of file: \"{os.path.basename(file)}\""
|
|
2413
|
+
Hash of file: {hash_of_current_file}
|
|
2414
|
+
OCR-content:
|
|
2415
|
+
\"{ocr_content}\"""")
|
|
2416
|
+
return True
|
|
2417
|
+
|
|
2418
|
+
@GeneralUtilities.check_arguments
|
|
2419
|
+
def get_ocr_content_of_file(self, file: str, serviceaddress: str, languages: list[str]) -> str: # serviceaddress = None means local executable
|
|
2420
|
+
result: str = None
|
|
2421
|
+
extension = Path(file).suffix
|
|
2422
|
+
if serviceaddress is None:
|
|
2423
|
+
program_result = self.run_program_argsasarray("simpleocr", ["--File", file, "--Languages", "+".join(languages)] + languages)
|
|
2424
|
+
result = program_result[1]
|
|
2425
|
+
else:
|
|
2426
|
+
languages_for_url = '%2B'.join(languages)
|
|
2427
|
+
package_url: str = f"https://{serviceaddress}/GetOCRContent?languages={languages_for_url}&fileType={extension}"
|
|
2428
|
+
headers = {'Cache-Control': 'no-cache'}
|
|
2429
|
+
r = requests.put(package_url, timeout=5, headers=headers, data=GeneralUtilities.read_binary_from_file(file))
|
|
2430
|
+
if r.status_code != 200:
|
|
2431
|
+
raise ValueError(f"Checking for latest tor package resulted in HTTP-response-code {r.status_code}.")
|
|
2432
|
+
result = GeneralUtilities.bytes_to_string(r.content)
|
|
2433
|
+
return result
|
|
2434
|
+
|
|
2435
|
+
@GeneralUtilities.check_arguments
|
|
2436
|
+
def ocr_analysis_of_repository(self, folder: str, serviceaddress: str, extensions: list[str], languages: list[str]) -> None:
|
|
2437
|
+
self.assert_is_git_repository(folder)
|
|
2438
|
+
changed_files = self.ocr_analysis_of_folder(folder, serviceaddress, extensions, languages)
|
|
2439
|
+
for changed_ocr_file in changed_files:
|
|
2440
|
+
GeneralUtilities.assert_condition(changed_ocr_file.endswith(".ocr.txt"), f"File '{changed_ocr_file}' is not an OCR-file. It should end with '.ocr.txt'.")
|
|
2441
|
+
base_file = changed_ocr_file[:-len(".ocr.txt")]
|
|
2442
|
+
GeneralUtilities.assert_condition(os.path.isfile(base_file), f"Base file '{base_file}' does not exist. The OCR-file '{changed_ocr_file}' is not valid.")
|
|
2443
|
+
base_file_relative_path = os.path.relpath(base_file, folder)
|
|
2444
|
+
base_file_diff_program_result = self.run_program("git", f"diff --quiet -- \"{base_file_relative_path}\"", folder, throw_exception_if_exitcode_is_not_zero=False)
|
|
2445
|
+
has_staged_changes: bool = None
|
|
2446
|
+
if base_file_diff_program_result[0] == 0:
|
|
2447
|
+
has_staged_changes = False
|
|
2448
|
+
elif base_file_diff_program_result[0] == 1:
|
|
2449
|
+
has_staged_changes = True
|
|
2450
|
+
else:
|
|
2451
|
+
raise RuntimeError(f"Unexpected exit code {base_file_diff_program_result[0]} when checking for staged changes of file '{base_file_relative_path}'.")
|
|
2452
|
+
if has_staged_changes:
|
|
2453
|
+
changed_ocr_file_relative_path = os.path.relpath(changed_ocr_file, folder)
|
|
2454
|
+
self.run_program_argsasarray("git", ["add", changed_ocr_file_relative_path], folder)
|
|
2455
|
+
|
|
2456
|
+
@GeneralUtilities.check_arguments
|
|
2457
|
+
def update_timestamp_in_file(self, target_file: str) -> None:
|
|
2458
|
+
lines = GeneralUtilities.read_lines_from_file(target_file)
|
|
2459
|
+
new_lines = []
|
|
2460
|
+
prefix: str = "# last update: "
|
|
2461
|
+
for line in lines:
|
|
2462
|
+
if line.startswith(prefix):
|
|
2463
|
+
new_lines.append(prefix+GeneralUtilities.datetime_to_string_with_timezone(GeneralUtilities.get_now()))
|
|
2464
|
+
else:
|
|
2465
|
+
new_lines.append(line)
|
|
2466
|
+
GeneralUtilities.write_lines_to_file(target_file, new_lines)
|
|
2467
|
+
|
|
2468
|
+
def do_and_log_task(self, name_of_task: str, task):
|
|
2469
|
+
try:
|
|
2470
|
+
self.log.log(f"Start action \"{name_of_task}\".", LogLevel.Information)
|
|
2471
|
+
result = task()
|
|
2472
|
+
if result is None:
|
|
2473
|
+
result = 0
|
|
2474
|
+
return result
|
|
2475
|
+
except Exception as e:
|
|
2476
|
+
self.log.log_exception(f"Error while running action \"{name_of_task}\".", e, LogLevel.Error)
|
|
2477
|
+
return 1
|
|
2478
|
+
finally:
|
|
2479
|
+
self.log.log(f"Finished action \"{name_of_task}\".", LogLevel.Information)
|
|
2480
|
+
|
|
2481
|
+
def get_lines_of_code_with_default_excluded_patterns(self, repository: str) -> int:
|
|
2482
|
+
return self.get_lines_of_code(repository, self.default_excluded_patterns_for_loc)
|
|
2483
|
+
|
|
2484
|
+
default_excluded_patterns_for_loc: list[str] = [".txt", ".md", ".vscode", "Resources", "Reference", ".gitignore", ".gitattributes", "Other/Metrics"]
|
|
2485
|
+
|
|
2486
|
+
def get_lines_of_code(self, repository: str, excluded_pattern: list[str]) -> int:
|
|
2487
|
+
self.assert_is_git_repository(repository)
|
|
2488
|
+
result: int = 0
|
|
2489
|
+
self.log.log(f"Calculate lines of code in repository '{repository}' with excluded patterns: {', '.join(excluded_pattern)}",LogLevel.Debug)
|
|
2490
|
+
git_response = self.run_program("git", "ls-files", repository)
|
|
2491
|
+
files: list[str] = GeneralUtilities.string_to_lines(git_response[1])
|
|
2492
|
+
for file in files:
|
|
2493
|
+
if os.path.isfile(os.path.join(repository, file)):
|
|
2494
|
+
if self.__is_excluded_by_glob_pattern(file, excluded_pattern):
|
|
2495
|
+
self.log.log(f"File '{file}' is ignored because it matches an excluded pattern.",LogLevel.Diagnostic)
|
|
2496
|
+
else:
|
|
2497
|
+
full_file: str = os.path.join(repository, file)
|
|
2498
|
+
if GeneralUtilities.is_binary_file(full_file):
|
|
2499
|
+
self.log.log(f"File '{file}' is ignored because it is a binary-file.",LogLevel.Diagnostic)
|
|
2500
|
+
else:
|
|
2501
|
+
self.log.log(f"Count lines of file '{file}'.",LogLevel.Diagnostic)
|
|
2502
|
+
length = len(GeneralUtilities.read_nonempty_lines_from_file(full_file))
|
|
2503
|
+
result = result+length
|
|
2504
|
+
else:
|
|
2505
|
+
self.log.log(f"File '{file}' is ignored because it does not exist.",LogLevel.Diagnostic)
|
|
2506
|
+
return result
|
|
2507
|
+
|
|
2508
|
+
def __is_excluded_by_glob_pattern(self, file: str, excluded_patterns: list[str]) -> bool:
|
|
2509
|
+
for pattern in excluded_patterns:
|
|
2510
|
+
if fnmatch.fnmatch(file, f"*{pattern}*"):
|
|
2511
|
+
return True
|
|
2512
|
+
return False
|
|
2513
|
+
|
|
2514
|
+
@GeneralUtilities.check_arguments
|
|
2515
|
+
def create_zip_archive(self, folder:str,zip_file:str) -> None:
|
|
2516
|
+
GeneralUtilities.assert_folder_exists(folder)
|
|
2517
|
+
GeneralUtilities.assert_file_does_not_exist(zip_file)
|
|
2518
|
+
folder = os.path.abspath(folder)
|
|
2519
|
+
with zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED) as zipf:
|
|
2520
|
+
for root, _, files in os.walk(folder):
|
|
2521
|
+
for file in files:
|
|
2522
|
+
file_path = os.path.join(root, file)
|
|
2523
|
+
arcname = os.path.relpath(file_path, start=folder)
|
|
2524
|
+
zipf.write(file_path, arcname)
|
|
2525
|
+
|
|
2526
|
+
@GeneralUtilities.check_arguments
|
|
2527
|
+
def start_local_test_service(self, file: str):
|
|
2528
|
+
example_folder = os.path.dirname(file)
|
|
2529
|
+
docker_compose_file = os.path.join(example_folder, "docker-compose.yml")
|
|
2530
|
+
for service in self.get_services_from_yaml_file(docker_compose_file):
|
|
2531
|
+
self.kill_docker_container(service)
|
|
2532
|
+
example_name = os.path.basename(example_folder)
|
|
2533
|
+
title = f"Test{example_name}"
|
|
2534
|
+
self.run_program("docker", f"compose -p {title.lower()} up --detach", example_folder, title=title)
|
|
2535
|
+
|
|
2536
|
+
@GeneralUtilities.check_arguments
|
|
2537
|
+
def stop_local_test_service(self, file: str):
|
|
2538
|
+
example_folder = os.path.dirname(file)
|
|
2539
|
+
example_name = os.path.basename(example_folder)
|
|
2540
|
+
title = f"Test{example_name}"
|
|
2541
|
+
self.run_program("docker", f"compose -p {title.lower()} down", example_folder, title=title)
|