ScriptCollection 3.3.23__py3-none-any.whl → 4.0.78__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. ScriptCollection/AnionBuildPlatform.py +206 -0
  2. ScriptCollection/{UpdateCertificates.py → CertificateUpdater.py} +149 -128
  3. ScriptCollection/Executables.py +868 -292
  4. ScriptCollection/GeneralUtilities.py +609 -107
  5. ScriptCollection/ImageUpdater.py +648 -0
  6. ScriptCollection/ProcessesRunner.py +41 -0
  7. ScriptCollection/ProgramRunnerBase.py +47 -42
  8. ScriptCollection/ProgramRunnerMock.py +2 -0
  9. ScriptCollection/ProgramRunnerPopen.py +57 -50
  10. ScriptCollection/ProgramRunnerSudo.py +108 -0
  11. ScriptCollection/SCLog.py +115 -0
  12. ScriptCollection/ScriptCollectionCore.py +2541 -1383
  13. ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +95 -0
  14. ScriptCollection/TFCPS/Docker/__init__.py +0 -0
  15. ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py +8 -0
  16. ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py +6 -0
  17. ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py +7 -0
  18. ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +485 -0
  19. ScriptCollection/TFCPS/DotNet/__init__.py +0 -0
  20. ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +130 -0
  21. ScriptCollection/TFCPS/Flutter/__init__.py +0 -0
  22. ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py +74 -0
  23. ScriptCollection/TFCPS/Go/__init__.py +0 -0
  24. ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +131 -0
  25. ScriptCollection/TFCPS/NodeJS/__init__.py +0 -0
  26. ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +227 -0
  27. ScriptCollection/TFCPS/Python/__init__.py +0 -0
  28. ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +418 -0
  29. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +128 -0
  30. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +136 -0
  31. ScriptCollection/TFCPS/TFCPS_CreateRelease.py +95 -0
  32. ScriptCollection/TFCPS/TFCPS_Generic.py +43 -0
  33. ScriptCollection/TFCPS/TFCPS_MergeToMain.py +122 -0
  34. ScriptCollection/TFCPS/TFCPS_MergeToStable.py +350 -0
  35. ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +47 -0
  36. ScriptCollection/TFCPS/TFCPS_Tools_General.py +1356 -0
  37. ScriptCollection/TFCPS/__init__.py +0 -0
  38. {ScriptCollection-3.3.23.dist-info → scriptcollection-4.0.78.dist-info}/METADATA +26 -21
  39. scriptcollection-4.0.78.dist-info/RECORD +43 -0
  40. {ScriptCollection-3.3.23.dist-info → scriptcollection-4.0.78.dist-info}/WHEEL +1 -1
  41. scriptcollection-4.0.78.dist-info/entry_points.txt +64 -0
  42. ScriptCollection/Hardening.py +0 -59
  43. ScriptCollection/ProgramRunnerEpew.py +0 -122
  44. ScriptCollection/TasksForCommonProjectStructure.py +0 -1170
  45. ScriptCollection-3.3.23.dist-info/RECORD +0 -15
  46. ScriptCollection-3.3.23.dist-info/entry_points.txt +0 -24
  47. {ScriptCollection-3.3.23.dist-info → scriptcollection-4.0.78.dist-info}/top_level.txt +0 -0
@@ -1,1383 +1,2541 @@
1
- from datetime import timedelta, datetime
2
- import binascii
3
- import filecmp
4
- import sys
5
- import hashlib
6
- from io import BytesIO
7
- import itertools
8
- import math
9
- import os
10
- from pathlib import Path
11
- from random import randrange
12
- from subprocess import Popen
13
- import re
14
- import shutil
15
- import traceback
16
- import uuid
17
- import ntplib
18
- from lxml import etree
19
- import pycdlib
20
- import send2trash
21
- from PyPDF2 import PdfFileMerger
22
- from .GeneralUtilities import GeneralUtilities
23
- from .ProgramRunnerBase import ProgramRunnerBase
24
- from .ProgramRunnerPopen import ProgramRunnerPopen
25
- from .ProgramRunnerEpew import ProgramRunnerEpew, CustomEpewArgument
26
-
27
-
28
- version = "3.3.23"
29
- __version__ = version
30
-
31
-
32
- class ScriptCollectionCore:
33
-
34
- # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
35
- # Do not change this value for productive environments.
36
- mock_program_calls: bool = False
37
- # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
38
- execute_program_really_if_no_mock_call_is_defined: bool = False
39
- __mocked_program_calls: list = list()
40
- program_runner: ProgramRunnerBase = None
41
-
42
- def __init__(self):
43
- self.program_runner = ProgramRunnerPopen()
44
-
45
- @staticmethod
46
- @GeneralUtilities.check_arguments
47
- def get_scriptcollection_version() -> str:
48
- return __version__
49
-
50
- @GeneralUtilities.check_arguments
51
- def python_file_has_errors(self, file: str, working_directory: str, treat_warnings_as_errors: bool = True) -> tuple[bool, list[str]]:
52
- errors = list()
53
- filename = os.path.relpath(file, working_directory)
54
- if treat_warnings_as_errors:
55
- errorsonly_argument = ""
56
- else:
57
- errorsonly_argument = " --errors-only"
58
- (exit_code, stdout, stderr, _) = self.run_program("pylint", filename+errorsonly_argument, working_directory, throw_exception_if_exitcode_is_not_zero=False)
59
- if (exit_code != 0):
60
- errors.append(f"Linting-issues of {file}:")
61
- errors.append(f"Pylint-exitcode: {exit_code}")
62
- for line in GeneralUtilities.string_to_lines(stdout):
63
- errors.append(line)
64
- for line in GeneralUtilities.string_to_lines(stderr):
65
- errors.append(line)
66
- return (True, errors)
67
-
68
- return (False, errors)
69
-
70
- @GeneralUtilities.check_arguments
71
- def replace_version_in_dockerfile_file(self, dockerfile: str, new_version_value: str) -> None:
72
- GeneralUtilities.write_text_to_file(dockerfile, re.sub("ARG Version=\"\\d+\\.\\d+\\.\\d+\"", f"ARG Version=\"{new_version_value}\"",
73
- GeneralUtilities.read_text_from_file(dockerfile)))
74
-
75
- @GeneralUtilities.check_arguments
76
- def check_testcoverage(self, testcoverage_file_in_cobertura_format: str, threshold_in_percent: float):
77
- root: etree._ElementTree = etree.parse(testcoverage_file_in_cobertura_format)
78
- coverage_in_percent = round(float(str(root.xpath('//coverage/@line-rate')[0]))*100, 2)
79
- minimalrequiredtestcoverageinpercent = threshold_in_percent
80
- if (coverage_in_percent < minimalrequiredtestcoverageinpercent):
81
- raise ValueError(f"The testcoverage must be {minimalrequiredtestcoverageinpercent}% or more but is {coverage_in_percent}%.")
82
-
83
- @GeneralUtilities.check_arguments
84
- def replace_version_in_python_file(self, file: str, new_version_value: str):
85
- GeneralUtilities.write_text_to_file(file, re.sub("version = \"\\d+\\.\\d+\\.\\d+\"", f"version = \"{new_version_value}\"",
86
- GeneralUtilities.read_text_from_file(file)))
87
-
88
- @GeneralUtilities.check_arguments
89
- def replace_version_in_ini_file(self, file: str, new_version_value: str):
90
- GeneralUtilities.write_text_to_file(file, re.sub("version = \\d+\\.\\d+\\.\\d+", f"version = {new_version_value}",
91
- GeneralUtilities.read_text_from_file(file)))
92
-
93
- @GeneralUtilities.check_arguments
94
- def replace_version_in_nuspec_file(self, nuspec_file: str, new_version: str) -> None:
95
- # TODO use XSLT instead
96
- versionregex = "\\d+\\.\\d+\\.\\d+"
97
- versiononlyregex = f"^{versionregex}$"
98
- pattern = re.compile(versiononlyregex)
99
- if pattern.match(new_version):
100
- GeneralUtilities.write_text_to_file(nuspec_file, re.sub(f"<version>{versionregex}<\\/version>",
101
- f"<version>{new_version}</version>", GeneralUtilities.read_text_from_file(nuspec_file)))
102
- else:
103
- raise ValueError(f"Version '{new_version}' does not match version-regex '{versiononlyregex}'")
104
-
105
- @GeneralUtilities.check_arguments
106
- def replace_version_in_csproj_file(self, csproj_file: str, current_version: str):
107
- versionregex = "\\d+\\.\\d+\\.\\d+"
108
- versiononlyregex = f"^{versionregex}$"
109
- pattern = re.compile(versiononlyregex)
110
- if pattern.match(current_version):
111
- for tag in ["Version", "AssemblyVersion", "FileVersion"]:
112
- GeneralUtilities.write_text_to_file(csproj_file, re.sub(f"<{tag}>{versionregex}(.\\d+)?<\\/{tag}>",
113
- f"<{tag}>{current_version}</{tag}>", GeneralUtilities.read_text_from_file(csproj_file)))
114
- else:
115
- raise ValueError(f"Version '{current_version}' does not match version-regex '{versiononlyregex}'")
116
-
117
- @GeneralUtilities.check_arguments
118
- def push_nuget_build_artifact_of_repository_in_common_file_structure(self, nupkg_file: str, registry_address: str, api_key: str, verbosity: int = 1):
119
- nupkg_file_name = os.path.basename(nupkg_file)
120
- nupkg_file_folder = os.path.dirname(nupkg_file)
121
- self.run_program("dotnet", f"nuget push {nupkg_file_name} --force-english-output --source {registry_address} --api-key {api_key}",
122
- nupkg_file_folder, verbosity)
123
-
124
- @GeneralUtilities.check_arguments
125
- def dotnet_build(self, repository_folder: str, projectname: str, configuration: str):
126
- self.run_program("dotnet", f"clean -c {configuration}", repository_folder)
127
- self.run_program("dotnet", f"build {projectname}/{projectname}.csproj -c {configuration}", repository_folder)
128
-
129
- @GeneralUtilities.check_arguments
130
- def dotnet_sign(self, dllOrExefile: str, snkfile: str, verbosity: int, current_release_information: dict[str, str]) -> None:
131
- dllOrExeFile = GeneralUtilities.resolve_relative_path_from_current_working_directory(dllOrExefile)
132
- snkfile = GeneralUtilities.resolve_relative_path_from_current_working_directory(snkfile)
133
- directory = os.path.dirname(dllOrExeFile)
134
- filename = os.path.basename(dllOrExeFile)
135
- if filename.lower().endswith(".dll"):
136
- filename = filename[:-4]
137
- extension = "dll"
138
- elif filename.lower().endswith(".exe"):
139
- filename = filename[:-4]
140
- extension = "exe"
141
- else:
142
- raise Exception("Only .dll-files and .exe-files can be signed")
143
- self.run_program("ildasm",
144
- f'/all /typelist /text /out="{filename}.il" "{filename}.{extension}"',
145
- directory, verbosity, False, "Sign: ildasm")
146
- self.run_program("ilasm",
147
- f'/{extension} /res:"{filename}.res" /optimize /key="{snkfile}" "{filename}.il"',
148
- directory, verbosity, False, "Sign: ilasm")
149
- os.remove(directory+os.path.sep+filename+".il")
150
- os.remove(directory+os.path.sep+filename+".res")
151
-
152
- @GeneralUtilities.check_arguments
153
- def find_file_by_extension(self, folder: str, extension: str):
154
- result = [file for file in GeneralUtilities.get_direct_files_of_folder(folder) if file.endswith(f".{extension}")]
155
- result_length = len(result)
156
- if result_length == 0:
157
- raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension}'.")
158
- if result_length == 1:
159
- return result[0]
160
- else:
161
- raise ValueError(f"Multiple values available in folder '{folder}' with extension '{extension}'.")
162
-
163
- @GeneralUtilities.check_arguments
164
- def dotnet_sign_file(self, file: str, keyfile: str, verbosity: int):
165
- directory = os.path.dirname(file)
166
- filename = os.path.basename(file)
167
- if filename.lower().endswith(".dll"):
168
- filename = filename[:-4]
169
- extension = "dll"
170
- elif filename.lower().endswith(".exe"):
171
- filename = filename[:-4]
172
- extension = "exe"
173
- else:
174
- raise Exception("Only .dll-files and .exe-files can be signed")
175
- self.run_program("ildasm", f'/all /typelist /text /out={filename}.il {filename}.{extension}', directory, verbosity=verbosity)
176
- self.run_program("ilasm", f'/{extension} /res:{filename}.res /optimize /key={keyfile} {filename}.il', directory, verbosity=verbosity)
177
- os.remove(directory+os.path.sep+filename+".il")
178
- os.remove(directory+os.path.sep+filename+".res")
179
-
180
- @GeneralUtilities.check_arguments
181
- def commit_is_signed_by_key(self, repository_folder: str, revision_identifier: str, key: str) -> bool:
182
- result = self.run_program("git", f"verify-commit {revision_identifier}", repository_folder, throw_exception_if_exitcode_is_not_zero=False)
183
- if (result[0] != 0):
184
- return False
185
- if (not GeneralUtilities.contains_line(result[1].splitlines(), f"gpg\\:\\ using\\ [A-Za-z0-9]+\\ key\\ [A-Za-z0-9]+{key}")):
186
- # TODO check whether this works on machines where gpg is installed in another langauge than english
187
- return False
188
- if (not GeneralUtilities.contains_line(result[1].splitlines(), "gpg\\:\\ Good\\ signature\\ from")):
189
- # TODO check whether this works on machines where gpg is installed in another langauge than english
190
- return False
191
- return True
192
-
193
- @GeneralUtilities.check_arguments
194
- def get_parent_commit_ids_of_commit(self, repository_folder: str, commit_id: str) -> str:
195
- return self.run_program("git", f'log --pretty=%P -n 1 "{commit_id}"',
196
- repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].replace("\r", "").replace("\n", "").split(" ")
197
-
198
- @GeneralUtilities.check_arguments
199
- def get_commit_ids_between_dates(self, repository_folder: str, since: datetime, until: datetime, ignore_commits_which_are_not_in_history_of_head: bool = True) -> None:
200
- since_as_string = self.__datetime_to_string_for_git(since)
201
- until_as_string = self.__datetime_to_string_for_git(until)
202
- result = filter(lambda line: not GeneralUtilities.string_is_none_or_whitespace(line),
203
- self.run_program("git", f'log --since "{since_as_string}" --until "{until_as_string}" --pretty=format:"%H" --no-patch',
204
- repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].split("\n").replace("\r", ""))
205
- if ignore_commits_which_are_not_in_history_of_head:
206
- result = [commit_id for commit_id in result if self.git_commit_is_ancestor(repository_folder, commit_id)]
207
- return result
208
-
209
- @GeneralUtilities.check_arguments
210
- def __datetime_to_string_for_git(self, datetime_object: datetime) -> str:
211
- return datetime_object.strftime('%Y-%m-%d %H:%M:%S')
212
-
213
- @GeneralUtilities.check_arguments
214
- def git_commit_is_ancestor(self, repository_folder: str, ancestor: str, descendant: str = "HEAD") -> bool:
215
- return self.run_program_argsasarray("git", ["merge-base", "--is-ancestor", ancestor, descendant], repository_folder, throw_exception_if_exitcode_is_not_zero=False)[0] == 0
216
-
217
- @GeneralUtilities.check_arguments
218
- def __git_changes_helper(self, repository_folder: str, arguments_as_array: list[str]) -> bool:
219
- lines = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", arguments_as_array, repository_folder,
220
- throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
221
- for line in lines:
222
- if GeneralUtilities.string_has_content(line):
223
- return True
224
- return False
225
-
226
- @GeneralUtilities.check_arguments
227
- def git_repository_has_new_untracked_files(self, repositoryFolder: str):
228
- return self.__git_changes_helper(repositoryFolder, ["ls-files", "--exclude-standard", "--others"])
229
-
230
- @GeneralUtilities.check_arguments
231
- def git_repository_has_unstaged_changes_of_tracked_files(self, repositoryFolder: str):
232
- return self.__git_changes_helper(repositoryFolder, ["diff"])
233
-
234
- @GeneralUtilities.check_arguments
235
- def git_repository_has_staged_changes(self, repositoryFolder: str):
236
- return self.__git_changes_helper(repositoryFolder, ["diff", "--cached"])
237
-
238
- @GeneralUtilities.check_arguments
239
- def git_repository_has_uncommitted_changes(self, repositoryFolder: str) -> bool:
240
- if (self.git_repository_has_unstaged_changes(repositoryFolder)):
241
- return True
242
- if (self.git_repository_has_staged_changes(repositoryFolder)):
243
- return True
244
- return False
245
-
246
- @GeneralUtilities.check_arguments
247
- def git_repository_has_unstaged_changes(self, repository_folder: str) -> bool:
248
- if (self.git_repository_has_unstaged_changes_of_tracked_files(repository_folder)):
249
- return True
250
- if (self.git_repository_has_new_untracked_files(repository_folder)):
251
- return True
252
- return False
253
-
254
- @GeneralUtilities.check_arguments
255
- def git_get_commit_id(self, repository_folder: str, commit: str = "HEAD") -> str:
256
- result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["rev-parse", "--verify", commit],
257
- repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
258
- return result[1].replace('\n', '')
259
-
260
- @GeneralUtilities.check_arguments
261
- def git_get_commit_date(self, repository_folder: str, commit: str = "HEAD") -> datetime:
262
- result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["show", "-s", "--format=%ci", commit],
263
- repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
264
- date_as_string = result[1].replace('\n', '')
265
- result = datetime.strptime(date_as_string, '%Y-%m-%d %H:%M:%S %z')
266
- return result
267
-
268
- @GeneralUtilities.check_arguments
269
- def git_fetch(self, folder: str, remotename: str = "--all") -> None:
270
- self.run_program_argsasarray("git", ["fetch", remotename, "--tags", "--prune"], folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
271
-
272
- @GeneralUtilities.check_arguments
273
- def git_fetch_in_bare_repository(self, folder: str, remotename, localbranch: str, remotebranch: str) -> None:
274
- self.run_program_argsasarray("git", ["fetch", remotename, f"{remotebranch}:{localbranch}"], folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
275
-
276
- @GeneralUtilities.check_arguments
277
- def git_remove_branch(self, folder: str, branchname: str) -> None:
278
- self.run_program("git", f"branch -D {branchname}", folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
279
-
280
- @GeneralUtilities.check_arguments
281
- def git_push(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: int = 0) -> None:
282
- argument = ["push", "--recurse-submodules=on-demand", remotename, f"{localbranchname}:{remotebranchname}"]
283
- if (forcepush):
284
- argument.append("--force")
285
- if (pushalltags):
286
- argument.append("--tags")
287
- result: tuple[int, str, str, int] = self.run_program_argsasarray("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True,
288
- verbosity=verbosity, print_errors_as_information=True)
289
- return result[1].replace('\r', '').replace('\n', '')
290
-
291
- @GeneralUtilities.check_arguments
292
- def git_clone(self, clone_target_folder: str, remote_repository_path: str, include_submodules: bool = True, mirror: bool = False) -> None:
293
- if (os.path.isdir(clone_target_folder)):
294
- pass # TODO throw error
295
- else:
296
- args = ["clone", remote_repository_path, clone_target_folder]
297
- if include_submodules:
298
- args.append("--recurse-submodules")
299
- args.append("--remote-submodules")
300
- if mirror:
301
- args.append("--mirror")
302
- self.run_program_argsasarray("git", args, os.getcwd(), throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
303
-
304
- @GeneralUtilities.check_arguments
305
- def git_get_all_remote_names(self, directory) -> list[str]:
306
- result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
307
- return result
308
-
309
- @GeneralUtilities.check_arguments
310
- def repository_has_remote_with_specific_name(self, directory: str, remote_name: str) -> bool:
311
- return remote_name in self.git_get_all_remote_names(directory)
312
-
313
- @GeneralUtilities.check_arguments
314
- def git_add_or_set_remote_address(self, directory: str, remote_name: str, remote_address: str) -> None:
315
- if (self.repository_has_remote_with_specific_name(directory, remote_name)):
316
- self.run_program_argsasarray("git", ['remote', 'set-url', 'remote_name', remote_address], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
317
- else:
318
- self.run_program_argsasarray("git", ['remote', 'add', remote_name, remote_address], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
319
-
320
- @GeneralUtilities.check_arguments
321
- def git_stage_all_changes(self, directory: str) -> None:
322
- self.run_program_argsasarray("git", ["add", "-A"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
323
-
324
- @GeneralUtilities.check_arguments
325
- def git_unstage_all_changes(self, directory: str) -> None:
326
- self.run_program_argsasarray("git", ["reset"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
327
-
328
- @GeneralUtilities.check_arguments
329
- def git_stage_file(self, directory: str, file: str) -> None:
330
- self.run_program_argsasarray("git", ['stage', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
331
-
332
- @GeneralUtilities.check_arguments
333
- def git_unstage_file(self, directory: str, file: str) -> None:
334
- self.run_program_argsasarray("git", ['reset', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
335
-
336
- @GeneralUtilities.check_arguments
337
- def git_discard_unstaged_changes_of_file(self, directory: str, file: str) -> None:
338
- """Caution: This method works really only for 'changed' files yet. So this method does not work properly for new or renamed files."""
339
- self.run_program_argsasarray("git", ['checkout', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
340
-
341
- @GeneralUtilities.check_arguments
342
- def git_discard_all_unstaged_changes(self, directory: str) -> None:
343
- """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
344
- self.run_program_argsasarray("git", ['clean', '-df'], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
345
- self.run_program_argsasarray("git", ['checkout', '.'], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
346
-
347
- @GeneralUtilities.check_arguments
348
- def git_commit(self, directory: str, message: str, author_name: str = None, author_email: str = None, stage_all_changes: bool = True,
349
- no_changes_behavior: int = 0) -> str:
350
- # no_changes_behavior=0 => No commit
351
- # no_changes_behavior=1 => Commit anyway
352
- # no_changes_behavior=2 => Exception
353
- author_name = GeneralUtilities.str_none_safe(author_name).strip()
354
- author_email = GeneralUtilities.str_none_safe(author_email).strip()
355
- argument = ['commit', '--quiet', '--message', message]
356
- if (GeneralUtilities.string_has_content(author_name)):
357
- argument.append(f'--author="{author_name} <{author_email}>"')
358
- git_repository_has_uncommitted_changes = self.git_repository_has_uncommitted_changes(directory)
359
-
360
- if git_repository_has_uncommitted_changes:
361
- do_commit = True
362
- if stage_all_changes:
363
- self.git_stage_all_changes(directory)
364
- else:
365
- if no_changes_behavior == 0:
366
- GeneralUtilities.write_message_to_stdout(f"Commit '{message}' will not be done because there are no changes to commit in repository '{directory}'")
367
- do_commit = False
368
- if no_changes_behavior == 1:
369
- GeneralUtilities.write_message_to_stdout(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will be done anyway.")
370
- do_commit = True
371
- argument.append('--allow-empty')
372
- if no_changes_behavior == 2:
373
- raise RuntimeError(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will not be done.")
374
-
375
- if do_commit:
376
- GeneralUtilities.write_message_to_stdout(f"Commit changes in '{directory}'")
377
- self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
378
-
379
- return self.git_get_commit_id(directory)
380
-
381
- @GeneralUtilities.check_arguments
382
- def git_create_tag(self, directory: str, target_for_tag: str, tag: str, sign: bool = False, message: str = None) -> None:
383
- argument = ["tag", tag, target_for_tag]
384
- if sign:
385
- if message is None:
386
- message = f"Created {target_for_tag}"
387
- argument.extend(["-s", '-m', message])
388
- self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
389
-
390
- @GeneralUtilities.check_arguments
391
- def git_checkout(self, directory: str, branch: str) -> None:
392
- self.run_program_argsasarray("git", ["checkout", branch], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
393
-
394
- @GeneralUtilities.check_arguments
395
- def git_merge_abort(self, directory: str) -> None:
396
- self.run_program_argsasarray("git", ["merge", "--abort"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
397
-
398
- @GeneralUtilities.check_arguments
399
- def git_merge(self, directory: str, sourcebranch: str, targetbranch: str, fastforward: bool = True, commit: bool = True, commit_message: str = None) -> str:
400
- self.git_checkout(directory, targetbranch)
401
- args = ["merge"]
402
- if not commit:
403
- args.append("--no-commit")
404
- if not fastforward:
405
- args.append("--no-ff")
406
- if commit_message is not None:
407
- args.append("-m")
408
- args.append(commit_message)
409
- args.append(sourcebranch)
410
- self.run_program_argsasarray("git", args, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
411
- return self.git_get_commit_id(directory)
412
-
413
- @GeneralUtilities.check_arguments
414
- def git_undo_all_changes(self, directory: str) -> None:
415
- """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
416
- self.git_unstage_all_changes(directory)
417
- self.git_discard_all_unstaged_changes(directory)
418
-
419
- @GeneralUtilities.check_arguments
420
- def git_fetch_or_clone_all_in_directory(self, source_directory: str, target_directory: str) -> None:
421
- for subfolder in GeneralUtilities.get_direct_folders_of_folder(source_directory):
422
- foldername = os.path.basename(subfolder)
423
- if self.is_git_repository(subfolder):
424
- source_repository = subfolder
425
- target_repository = os.path.join(target_directory, foldername)
426
- if os.path.isdir(target_directory):
427
- # fetch
428
- self.git_fetch(target_directory)
429
- else:
430
- # clone
431
- self.git_clone(target_repository, source_repository, include_submodules=True, mirror=True)
432
-
433
- @GeneralUtilities.check_arguments
434
- def is_git_repository(self, folder: str) -> bool:
435
- combined = os.path.join(folder, ".git")
436
- # TODO consider check for bare-repositories
437
- return os.path.isdir(combined) or os.path.isfile(combined)
438
-
439
- @GeneralUtilities.check_arguments
440
- def file_is_git_ignored(self, file_in_repository: str, repositorybasefolder: str) -> None:
441
- exit_code = self.run_program_argsasarray("git", ['check-ignore', file_in_repository], repositorybasefolder, throw_exception_if_exitcode_is_not_zero=False, verbosity=0)[0]
442
- if (exit_code == 0):
443
- return True
444
- if (exit_code == 1):
445
- return False
446
- raise Exception(f"Unable to calculate whether '{file_in_repository}' in repository '{repositorybasefolder}' is ignored due to git-exitcode {exit_code}.")
447
-
448
- @GeneralUtilities.check_arguments
449
- def discard_all_changes(self, repository: str) -> None:
450
- self.run_program_argsasarray("git", ["reset", "HEAD", "."], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
451
- self.run_program_argsasarray("git", ["checkout", "."], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
452
-
453
- @GeneralUtilities.check_arguments
454
- def git_get_current_branch_name(self, repository: str) -> str:
455
- result = self.run_program_argsasarray("git", ["rev-parse", "--abbrev-ref", "HEAD"], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
456
- return result[1].replace("\r", "").replace("\n", "")
457
-
458
- @GeneralUtilities.check_arguments
459
- def export_filemetadata(self, folder: str, target_file: str, encoding: str = "utf-8", filter_function=None) -> None:
460
- folder = GeneralUtilities.resolve_relative_path_from_current_working_directory(folder)
461
- lines = list()
462
- path_prefix = len(folder)+1
463
- items = dict()
464
- for item in GeneralUtilities.get_all_folders_of_folder(folder):
465
- items[item] = "d"
466
- for item in GeneralUtilities.get_all_files_of_folder(folder):
467
- items[item] = "f"
468
- for file_or_folder, item_type in items.items():
469
- truncated_file = file_or_folder[path_prefix:]
470
- if (filter_function is None or filter_function(folder, truncated_file)):
471
- owner_and_permisssion = self.get_file_owner_and_file_permission(file_or_folder)
472
- user = owner_and_permisssion[0]
473
- permissions = owner_and_permisssion[1]
474
- lines.append(f"{truncated_file};{item_type};{user};{permissions}")
475
- lines = sorted(lines, key=str.casefold)
476
- with open(target_file, "w", encoding=encoding) as file_object:
477
- file_object.write("\n".join(lines))
478
-
479
- def escape_git_repositories_in_folder(self, folder: str) -> dict[str, str]:
480
- return self.__escape_git_repositories_in_folder_internal(folder, dict[str, str]())
481
-
482
- def __escape_git_repositories_in_folder_internal(self, folder: str, renamed_items: dict[str, str]) -> dict[str, str]:
483
- for file in GeneralUtilities.get_direct_files_of_folder(folder):
484
- filename = os.path.basename(file)
485
- if ".git" in filename:
486
- new_name = filename.replace(".git", ".gitx")
487
- target = os.path.join(folder, new_name)
488
- os.rename(file, target)
489
- renamed_items[target] = file
490
- for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
491
- foldername = os.path.basename(subfolder)
492
- if ".git" in foldername:
493
- new_name = foldername.replace(".git", ".gitx")
494
- subfolder2 = os.path.join(str(Path(subfolder).parent), new_name)
495
- os.rename(subfolder, subfolder2)
496
- renamed_items[subfolder2] = subfolder
497
- else:
498
- subfolder2 = subfolder
499
- self.__escape_git_repositories_in_folder_internal(subfolder2, renamed_items)
500
- return renamed_items
501
-
502
- def deescape_git_repositories_in_folder(self, renamed_items: dict[str, str]):
503
- for renamed_item, original_name in renamed_items.items():
504
- os.rename(renamed_item, original_name)
505
-
506
- def __sort_fmd(self, line: str):
507
- splitted: list = line.split(";")
508
- filetype: str = splitted[1]
509
- if filetype == "d":
510
- return -1
511
- if filetype == "f":
512
- return 1
513
- return 0
514
-
515
- @GeneralUtilities.check_arguments
516
- def restore_filemetadata(self, folder: str, source_file: str, strict=False, encoding: str = "utf-8", create_folder_is_not_exist: bool = True) -> None:
517
- lines = GeneralUtilities.read_lines_from_file(source_file, encoding)
518
- lines.sort(key=self.__sort_fmd)
519
- for line in lines:
520
- splitted: list = line.split(";")
521
- full_path_of_file_or_folder: str = os.path.join(folder, splitted[0])
522
- filetype: str = splitted[1]
523
- user: str = splitted[2]
524
- permissions: str = splitted[3]
525
- if filetype == "d" and create_folder_is_not_exist and not os.path.isdir(full_path_of_file_or_folder):
526
- GeneralUtilities.ensure_directory_exists(full_path_of_file_or_folder)
527
- if (filetype == "f" and os.path.isfile(full_path_of_file_or_folder)) or (filetype == "d" and os.path.isdir(full_path_of_file_or_folder)):
528
- self.set_owner(full_path_of_file_or_folder, user, os.name != 'nt')
529
- self.set_permission(full_path_of_file_or_folder, permissions)
530
- else:
531
- if strict:
532
- if filetype == "f":
533
- filetype_full = "File"
534
- if filetype == "d":
535
- filetype_full = "Directory"
536
- raise Exception(f"{filetype_full} '{full_path_of_file_or_folder}' does not exist")
537
-
538
- @GeneralUtilities.check_arguments
539
- def __calculate_lengh_in_seconds(self, filename: str, folder: str) -> float:
540
- argument = ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', filename]
541
- result = self.run_program_argsasarray("ffprobe", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
542
- return float(result[1].replace('\n', ''))
543
-
544
- @GeneralUtilities.check_arguments
545
- def __create_thumbnails(self, filename: str, fps: str, folder: str, tempname_for_thumbnails: str) -> None:
546
- argument = ['-i', filename, '-r', str(fps), '-vf', 'scale=-1:120', '-vcodec', 'png', f'{tempname_for_thumbnails}-%002d.png']
547
- self.run_program_argsasarray("ffmpeg", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
548
-
549
- @GeneralUtilities.check_arguments
550
- def __create_thumbnail(self, outputfilename: str, folder: str, length_in_seconds: float, tempname_for_thumbnails: str, amount_of_images: int) -> None:
551
- duration = timedelta(seconds=length_in_seconds)
552
- info = GeneralUtilities.timedelta_to_simple_string(duration)
553
- next_square_number = str(int(math.sqrt(GeneralUtilities.get_next_square_number(amount_of_images))))
554
- argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{next_square_number}x{next_square_number}',
555
- f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
556
- self.run_program_argsasarray("montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
557
-
558
- @GeneralUtilities.check_arguments
559
- def roundup(self, x: float, places: int) -> int:
560
- d = 10 ** places
561
- if x < 0:
562
- return math.floor(x * d) / d
563
- else:
564
- return math.ceil(x * d) / d
565
-
566
- @GeneralUtilities.check_arguments
567
- def generate_thumbnail(self, file: str, frames_per_second: str, tempname_for_thumbnails: str = None) -> None:
568
- if tempname_for_thumbnails is None:
569
- tempname_for_thumbnails = "t"+str(uuid.uuid4())
570
-
571
- file = GeneralUtilities.resolve_relative_path_from_current_working_directory(file)
572
- filename = os.path.basename(file)
573
- folder = os.path.dirname(file)
574
- filename_without_extension = Path(file).stem
575
-
576
- try:
577
- length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
578
- if (frames_per_second.endswith("fps")):
579
- # frames per second, example: frames_per_second="20fps" => 20 frames per second
580
- x = self.roundup(float(frames_per_second[:-3]), 2)
581
- frames_per_secondx = str(x)
582
- amounf_of_previewframes = int(math.floor(length_in_seconds*x))
583
- else:
584
- # concrete amount of frame, examples: frames_per_second="16" => 16 frames for entire video
585
- amounf_of_previewframes = int(float(frames_per_second))
586
- frames_per_secondx = f"{amounf_of_previewframes-2}/{length_in_seconds}" # self.roundup((amounf_of_previewframes-2)/length_in_seconds, 2)
587
- self.__create_thumbnails(filename, frames_per_secondx, folder, tempname_for_thumbnails)
588
- self.__create_thumbnail(filename_without_extension, folder, length_in_seconds, tempname_for_thumbnails, amounf_of_previewframes)
589
- finally:
590
- for thumbnail_to_delete in Path(folder).rglob(tempname_for_thumbnails+"-*"):
591
- file = str(thumbnail_to_delete)
592
- os.remove(file)
593
-
594
- @GeneralUtilities.check_arguments
595
- def merge_pdf_files(self, files, outputfile: str) -> None:
596
- # TODO add wildcard-option
597
- pdfFileMerger = PdfFileMerger()
598
- for file in files:
599
- pdfFileMerger.append(file.strip())
600
- pdfFileMerger.write(outputfile)
601
- pdfFileMerger.close()
602
- return 0
603
-
604
- @GeneralUtilities.check_arguments
605
- def SCShowMissingFiles(self, folderA: str, folderB: str):
606
- for file in GeneralUtilities.get_missing_files(folderA, folderB):
607
- GeneralUtilities.write_message_to_stdout(file)
608
-
609
- @GeneralUtilities.check_arguments
610
- def SCCreateEmptyFileWithSpecificSize(self, name: str, size_string: str) -> int:
611
- if size_string.isdigit():
612
- size = int(size_string)
613
- else:
614
- if len(size_string) >= 3:
615
- if (size_string.endswith("kb")):
616
- size = int(size_string[:-2]) * pow(10, 3)
617
- elif (size_string.endswith("mb")):
618
- size = int(size_string[:-2]) * pow(10, 6)
619
- elif (size_string.endswith("gb")):
620
- size = int(size_string[:-2]) * pow(10, 9)
621
- elif (size_string.endswith("kib")):
622
- size = int(size_string[:-3]) * pow(2, 10)
623
- elif (size_string.endswith("mib")):
624
- size = int(size_string[:-3]) * pow(2, 20)
625
- elif (size_string.endswith("gib")):
626
- size = int(size_string[:-3]) * pow(2, 30)
627
- else:
628
- GeneralUtilities.write_message_to_stderr("Wrong format")
629
- else:
630
- GeneralUtilities.write_message_to_stderr("Wrong format")
631
- return 1
632
- with open(name, "wb") as f:
633
- f.seek(size-1)
634
- f.write(b"\0")
635
- return 0
636
-
637
- @GeneralUtilities.check_arguments
638
- def SCCreateHashOfAllFiles(self, folder: str) -> None:
639
- for file in GeneralUtilities.absolute_file_paths(folder):
640
- with open(file+".sha256", "w+", encoding="utf-8") as f:
641
- f.write(GeneralUtilities.get_sha256_of_file(file))
642
-
643
- @GeneralUtilities.check_arguments
644
- def SCCreateSimpleMergeWithoutRelease(self, repository: str, sourcebranch: str, targetbranch: str, remotename: str, remove_source_branch: bool) -> None:
645
- commitid = self.git_merge(repository, sourcebranch, targetbranch, False, True)
646
- self.git_merge(repository, targetbranch, sourcebranch, True, True)
647
- created_version = self.get_semver_version_from_gitversion(repository)
648
- self.git_create_tag(repository, commitid, f"v{created_version}", True)
649
- self.git_push(repository, remotename, targetbranch, targetbranch, False, True)
650
- if (GeneralUtilities.string_has_nonwhitespace_content(remotename)):
651
- self.git_push(repository, remotename, sourcebranch, sourcebranch, False, True)
652
- if (remove_source_branch):
653
- self.git_remove_branch(repository, sourcebranch)
654
-
655
- @GeneralUtilities.check_arguments
656
- def sc_organize_lines_in_file(self, file: str, encoding: str, sort: bool = False, remove_duplicated_lines: bool = False, ignore_first_line: bool = False,
657
- remove_empty_lines: bool = True, ignored_start_character: list = list()) -> int:
658
- if os.path.isfile(file):
659
-
660
- # read file
661
- lines = GeneralUtilities.read_lines_from_file(file, encoding)
662
- if (len(lines) == 0):
663
- return 0
664
-
665
- # store first line if desiredpopd
666
-
667
- if (ignore_first_line):
668
- first_line = lines.pop(0)
669
-
670
- # remove empty lines if desired
671
- if remove_empty_lines:
672
- temp = lines
673
- lines = []
674
- for line in temp:
675
- if (not (GeneralUtilities.string_is_none_or_whitespace(line))):
676
- lines.append(line)
677
-
678
- # remove duplicated lines if desired
679
- if remove_duplicated_lines:
680
- lines = GeneralUtilities.remove_duplicates(lines)
681
-
682
- # sort lines if desired
683
- if sort:
684
- lines = sorted(lines, key=lambda singleline: self.__adapt_line_for_sorting(singleline, ignored_start_character))
685
-
686
- # reinsert first line
687
- if ignore_first_line:
688
- lines.insert(0, first_line)
689
-
690
- # write result to file
691
- GeneralUtilities.write_lines_to_file(file, lines, encoding)
692
-
693
- return 0
694
- else:
695
- GeneralUtilities.write_message_to_stdout(f"File '{file}' does not exist")
696
- return 1
697
-
698
- @GeneralUtilities.check_arguments
699
- def __adapt_line_for_sorting(self, line: str, ignored_start_characters: list):
700
- result = line.lower()
701
- while len(result) > 0 and result[0] in ignored_start_characters:
702
- result = result[1:]
703
- return result
704
-
705
- @GeneralUtilities.check_arguments
706
- def SCGenerateSnkFiles(self, outputfolder, keysize=4096, amountofkeys=10) -> int:
707
- GeneralUtilities.ensure_directory_exists(outputfolder)
708
- for _ in range(amountofkeys):
709
- file = os.path.join(outputfolder, str(uuid.uuid4())+".snk")
710
- argument = f"-k {keysize} {file}"
711
- self.run_program("sn", argument, outputfolder)
712
-
713
- @GeneralUtilities.check_arguments
714
- def __merge_files(self, sourcefile: str, targetfile: str) -> None:
715
- with open(sourcefile, "rb") as f:
716
- source_data = f.read()
717
- with open(targetfile, "ab") as fout:
718
- merge_separator = [0x0A]
719
- fout.write(bytes(merge_separator))
720
- fout.write(source_data)
721
-
722
- @GeneralUtilities.check_arguments
723
- def __process_file(self, file: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
724
- new_filename = os.path.join(os.path.dirname(file), os.path.basename(file).replace(substringInFilename, newSubstringInFilename))
725
- if file != new_filename:
726
- if os.path.isfile(new_filename):
727
- if filecmp.cmp(file, new_filename):
728
- send2trash.send2trash(file)
729
- else:
730
- if conflictResolveMode == "ignore":
731
- pass
732
- elif conflictResolveMode == "preservenewest":
733
- if (os.path.getmtime(file) - os.path.getmtime(new_filename) > 0):
734
- send2trash.send2trash(file)
735
- else:
736
- send2trash.send2trash(new_filename)
737
- os.rename(file, new_filename)
738
- elif (conflictResolveMode == "merge"):
739
- self.__merge_files(file, new_filename)
740
- send2trash.send2trash(file)
741
- else:
742
- raise Exception('Unknown conflict resolve mode')
743
- else:
744
- os.rename(file, new_filename)
745
-
746
- @GeneralUtilities.check_arguments
747
- def SCReplaceSubstringsInFilenames(self, folder: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
748
- for file in GeneralUtilities.absolute_file_paths(folder):
749
- self.__process_file(file, substringInFilename, newSubstringInFilename, conflictResolveMode)
750
-
751
- @GeneralUtilities.check_arguments
752
- def __check_file(self, file: str, searchstring: str) -> None:
753
- bytes_ascii = bytes(searchstring, "ascii")
754
- bytes_utf16 = bytes(searchstring, "utf-16") # often called "unicode-encoding"
755
- bytes_utf8 = bytes(searchstring, "utf-8")
756
- with open(file, mode='rb') as file_object:
757
- content = file_object.read()
758
- if bytes_ascii in content:
759
- GeneralUtilities.write_message_to_stdout(file)
760
- elif bytes_utf16 in content:
761
- GeneralUtilities.write_message_to_stdout(file)
762
- elif bytes_utf8 in content:
763
- GeneralUtilities.write_message_to_stdout(file)
764
-
765
- @GeneralUtilities.check_arguments
766
- def SCSearchInFiles(self, folder: str, searchstring: str) -> None:
767
- for file in GeneralUtilities.absolute_file_paths(folder):
768
- self.__check_file(file, searchstring)
769
-
770
- @GeneralUtilities.check_arguments
771
- def __print_qr_code_by_csv_line(self, displayname: str, website: str, emailaddress: str, key: str, period: str) -> None:
772
- qrcode_content = f"otpauth://totp/{website}:{emailaddress}?secret={key}&issuer={displayname}&period={period}"
773
- GeneralUtilities.write_message_to_stdout(f"{displayname} ({emailaddress}):")
774
- GeneralUtilities.write_message_to_stdout(qrcode_content)
775
- self.run_program("qr", qrcode_content)
776
-
777
- @GeneralUtilities.check_arguments
778
- def SCShow2FAAsQRCode(self, csvfile: str) -> None:
779
- separator_line = "--------------------------------------------------------"
780
- for line in GeneralUtilities.read_csv_file(csvfile, True):
781
- GeneralUtilities.write_message_to_stdout(separator_line)
782
- self.__print_qr_code_by_csv_line(line[0], line[1], line[2], line[3], line[4])
783
- GeneralUtilities.write_message_to_stdout(separator_line)
784
-
785
- @GeneralUtilities.check_arguments
786
- def SCUpdateNugetpackagesInCsharpProject(self, csprojfile: str) -> int:
787
- outdated_packages = self.get_nuget_packages_of_csproj_file(csprojfile, True)
788
- GeneralUtilities.write_message_to_stdout("The following packages will be updated:")
789
- for outdated_package in outdated_packages:
790
- GeneralUtilities.write_message_to_stdout(outdated_package)
791
- self.update_nuget_package(csprojfile, outdated_package)
792
- GeneralUtilities.write_message_to_stdout(f"{len(outdated_packages)} package(s) were updated")
793
- return len(outdated_packages) > 0
794
-
795
- @GeneralUtilities.check_arguments
796
- def SCUploadFileToFileHost(self, file: str, host: str) -> int:
797
- try:
798
- GeneralUtilities.write_message_to_stdout(self.upload_file_to_file_host(file, host))
799
- return 0
800
- except Exception as exception:
801
- GeneralUtilities.write_exception_to_stderr_with_traceback(exception, traceback)
802
- return 1
803
-
804
- @GeneralUtilities.check_arguments
805
- def SCFileIsAvailableOnFileHost(self, file: str) -> int:
806
- try:
807
- if self.file_is_available_on_file_host(file):
808
- GeneralUtilities.write_message_to_stdout(f"'{file}' is available")
809
- return 0
810
- else:
811
- GeneralUtilities.write_message_to_stdout(f"'{file}' is not available")
812
- return 1
813
- except Exception as exception:
814
- GeneralUtilities.write_exception_to_stderr_with_traceback(exception, traceback)
815
- return 2
816
-
817
- @GeneralUtilities.check_arguments
818
- def SCCalculateBitcoinBlockHash(self, block_version_number: str, previousblockhash: str, transactionsmerkleroot: str, timestamp: str, target: str, nonce: str) -> str:
819
- # Example-values:
820
- # block_version_number: "00000020"
821
- # previousblockhash: "66720b99e07d284bd4fe67ff8c49a5db1dd8514fcdab61000000000000000000"
822
- # transactionsmerkleroot: "7829844f4c3a41a537b3131ca992643eaa9d093b2383e4cdc060ad7dc5481187"
823
- # timestamp: "51eb505a"
824
- # target: "c1910018"
825
- # nonce: "de19b302"
826
- header = str(block_version_number + previousblockhash + transactionsmerkleroot + timestamp + target + nonce)
827
- return binascii.hexlify(hashlib.sha256(hashlib.sha256(binascii.unhexlify(header)).digest()).digest()[::-1]).decode('utf-8')
828
-
829
- @GeneralUtilities.check_arguments
830
- def SCChangeHashOfProgram(self, inputfile: str) -> None:
831
- valuetoappend = str(uuid.uuid4())
832
-
833
- outputfile = inputfile + '.modified'
834
-
835
- shutil.copy2(inputfile, outputfile)
836
- with open(outputfile, 'a', encoding="utf-8") as file:
837
- # TODO use rcedit for .exe-files instead of appending valuetoappend ( https://github.com/electron/rcedit/ )
838
- # background: you can retrieve the "original-filename" from the .exe-file like discussed here:
839
- # https://security.stackexchange.com/questions/210843/ is-it-possible-to-change-original-filename-of-an-exe
840
- # so removing the original filename with rcedit is probably a better way to make it more difficult to detect the programname.
841
- # this would obviously also change the hashvalue of the program so appending a whitespace is not required anymore.
842
- file.write(valuetoappend)
843
-
844
- @GeneralUtilities.check_arguments
845
- def __adjust_folder_name(self, folder: str) -> str:
846
- result = os.path.dirname(folder).replace("\\", "/")
847
- if result == "/":
848
- return ""
849
- else:
850
- return result
851
-
852
- @GeneralUtilities.check_arguments
853
- def __create_iso(self, folder, iso_file) -> None:
854
- created_directories = []
855
- files_directory = "FILES"
856
- iso = pycdlib.PyCdlib()
857
- iso.new()
858
- files_directory = files_directory.upper()
859
- iso.add_directory("/" + files_directory)
860
- created_directories.append("/" + files_directory)
861
- for root, _, files in os.walk(folder):
862
- for file in files:
863
- full_path = os.path.join(root, file)
864
- with (open(full_path, "rb").read()) as text_io_wrapper:
865
- content = text_io_wrapper
866
- path_in_iso = '/' + files_directory + self.__adjust_folder_name(full_path[len(folder)::1]).upper()
867
- if path_in_iso not in created_directories:
868
- iso.add_directory(path_in_iso)
869
- created_directories.append(path_in_iso)
870
- iso.add_fp(BytesIO(content), len(content), path_in_iso + '/' + file.upper() + ';1')
871
- iso.write(iso_file)
872
- iso.close()
873
-
874
- @GeneralUtilities.check_arguments
875
- def SCCreateISOFileWithObfuscatedFiles(self, inputfolder: str, outputfile: str, printtableheadline, createisofile, extensions) -> None:
876
- if (os.path.isdir(inputfolder)):
877
- namemappingfile = "name_map.csv"
878
- files_directory = inputfolder
879
- files_directory_obf = files_directory + "_Obfuscated"
880
- self.SCObfuscateFilesFolder(inputfolder, printtableheadline, namemappingfile, extensions)
881
- os.rename(namemappingfile, os.path.join(files_directory_obf, namemappingfile))
882
- if createisofile:
883
- self.__create_iso(files_directory_obf, outputfile)
884
- shutil.rmtree(files_directory_obf)
885
- else:
886
- raise Exception(f"Directory not found: '{inputfolder}'")
887
-
888
- @GeneralUtilities.check_arguments
889
- def SCFilenameObfuscator(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
890
- obfuscate_all_files = extensions == "*"
891
- if (not obfuscate_all_files):
892
- obfuscate_file_extensions = extensions.split(",")
893
-
894
- if (os.path.isdir(inputfolder)):
895
- printtableheadline = GeneralUtilities.string_to_boolean(printtableheadline)
896
- files = []
897
- if not os.path.isfile(namemappingfile):
898
- with open(namemappingfile, "a", encoding="utf-8"):
899
- pass
900
- if printtableheadline:
901
- GeneralUtilities.append_line_to_file(namemappingfile, "Original filename;new filename;SHA2-hash of file")
902
- for file in GeneralUtilities.absolute_file_paths(inputfolder):
903
- if os.path.isfile(os.path.join(inputfolder, file)):
904
- if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
905
- files.append(file)
906
- for file in files:
907
- hash_value = GeneralUtilities.get_sha256_of_file(file)
908
- extension = Path(file).suffix
909
- new_file_name_without_path = str(uuid.uuid4())[0:8] + extension
910
- new_file_name = os.path.join(os.path.dirname(file), new_file_name_without_path)
911
- os.rename(file, new_file_name)
912
- GeneralUtilities.append_line_to_file(namemappingfile, os.path.basename(file) + ";" + new_file_name_without_path + ";" + hash_value)
913
- else:
914
- raise Exception(f"Directory not found: '{inputfolder}'")
915
-
916
- @GeneralUtilities.check_arguments
917
- def __extension_matchs(self, file: str, obfuscate_file_extensions) -> bool:
918
- for extension in obfuscate_file_extensions:
919
- if file.lower().endswith("."+extension.lower()):
920
- return True
921
- return False
922
-
923
- @GeneralUtilities.check_arguments
924
- def SCHealthcheck(self, file: str) -> int:
925
- lines = GeneralUtilities.read_lines_from_file(file)
926
- for line in reversed(lines):
927
- if not GeneralUtilities.string_is_none_or_whitespace(line):
928
- if "RunningHealthy (" in line: # TODO use regex
929
- GeneralUtilities.write_message_to_stderr(f"Healthy running due to line '{line}' in file '{file}'.")
930
- return 0
931
- else:
932
- GeneralUtilities.write_message_to_stderr(f"Not healthy running due to line '{line}' in file '{file}'.")
933
- return 1
934
- GeneralUtilities.write_message_to_stderr(f"No valid line found for healthycheck in file '{file}'.")
935
- return 2
936
-
937
- @GeneralUtilities.check_arguments
938
- def SCObfuscateFilesFolder(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
939
- obfuscate_all_files = extensions == "*"
940
- if (not obfuscate_all_files):
941
- if "," in extensions:
942
- obfuscate_file_extensions = extensions.split(",")
943
- else:
944
- obfuscate_file_extensions = [extensions]
945
- newd = inputfolder+"_Obfuscated"
946
- shutil.copytree(inputfolder, newd)
947
- inputfolder = newd
948
- if (os.path.isdir(inputfolder)):
949
- for file in GeneralUtilities.absolute_file_paths(inputfolder):
950
- if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
951
- self.SCChangeHashOfProgram(file)
952
- os.remove(file)
953
- os.rename(file + ".modified", file)
954
- self.SCFilenameObfuscator(inputfolder, printtableheadline, namemappingfile, extensions)
955
- else:
956
- raise Exception(f"Directory not found: '{inputfolder}'")
957
-
958
- @GeneralUtilities.check_arguments
959
- def upload_file_to_file_host(self, file: str, host: str) -> int:
960
- if (host is None):
961
- return self.upload_file_to_random_filesharing_service(file)
962
- elif host == "anonfiles.com":
963
- return self.upload_file_to_anonfiles(file)
964
- elif host == "bayfiles.com":
965
- return self.upload_file_to_bayfiles(file)
966
- GeneralUtilities.write_message_to_stderr("Unknown host: "+host)
967
- return 1
968
-
969
- @GeneralUtilities.check_arguments
970
- def upload_file_to_random_filesharing_service(self, file: str) -> int:
971
- host = randrange(2)
972
- if host == 0:
973
- return self.upload_file_to_anonfiles(file)
974
- if host == 1:
975
- return self.upload_file_to_bayfiles(file)
976
- return 1
977
-
978
- @GeneralUtilities.check_arguments
979
- def upload_file_to_anonfiles(self, file) -> int:
980
- return self.upload_file_by_using_simple_curl_request("https://api.anonfiles.com/upload", file)
981
-
982
- @GeneralUtilities.check_arguments
983
- def upload_file_to_bayfiles(self, file) -> int:
984
- return self.upload_file_by_using_simple_curl_request("https://api.bayfiles.com/upload", file)
985
-
986
- @GeneralUtilities.check_arguments
987
- def upload_file_by_using_simple_curl_request(self, api_url: str, file: str) -> int:
988
- # TODO implement
989
- return 1
990
-
991
- @GeneralUtilities.check_arguments
992
- def file_is_available_on_file_host(self, file) -> int:
993
- # TODO implement
994
- return 1
995
-
996
- def run_testcases_for_python_project(self, repository_folder: str):
997
- self.run_program("coverage", "run -m pytest", repository_folder)
998
- self.run_program("coverage", "xml", repository_folder)
999
- GeneralUtilities.ensure_directory_exists(os.path.join(repository_folder, "Other/TestCoverage"))
1000
- coveragefile = os.path.join(repository_folder, "Other/TestCoverage/TestCoverage.xml")
1001
- GeneralUtilities.ensure_file_does_not_exist(coveragefile)
1002
- os.rename(os.path.join(repository_folder, "coverage.xml"), coveragefile)
1003
-
1004
- @GeneralUtilities.check_arguments
1005
- def get_nuget_packages_of_csproj_file(self, csproj_file: str, only_outdated_packages: bool) -> bool:
1006
- self.run_program("dotnet", f'restore --disable-parallel --force --force-evaluate "{csproj_file}"')
1007
- if only_outdated_packages:
1008
- only_outdated_packages_argument = " --outdated"
1009
- else:
1010
- only_outdated_packages_argument = ""
1011
- stdout = self.run_program("dotnet", f'list "{csproj_file}" package{only_outdated_packages_argument}')[1]
1012
- result = []
1013
- for line in stdout.splitlines():
1014
- trimmed_line = line.replace("\t", "").strip()
1015
- if trimmed_line.startswith(">"):
1016
- result.append(trimmed_line[2:].split(" ")[0])
1017
- return result
1018
-
1019
- @GeneralUtilities.check_arguments
1020
- def update_nuget_package(self, csproj_file: str, name: str) -> None:
1021
- self.run_program("dotnet", f'add "{csproj_file}" package {name}')
1022
-
1023
- @GeneralUtilities.check_arguments
1024
- def get_file_permission(self, file: str) -> str:
1025
- """This function returns an usual octet-triple, for example "0700"."""
1026
- ls_output = self.__ls(file)
1027
- return self.__get_file_permission_helper(ls_output)
1028
-
1029
- @GeneralUtilities.check_arguments
1030
- def __get_file_permission_helper(self, ls_output: str) -> str:
1031
- permissions = ' '.join(ls_output.split()).split(' ')[0][1:]
1032
- return str(self.__to_octet(permissions[0:3]))+str(self.__to_octet(permissions[3:6]))+str(self.__to_octet(permissions[6:9]))
1033
-
1034
- @GeneralUtilities.check_arguments
1035
- def __to_octet(self, string: str) -> int:
1036
- return int(self.__to_octet_helper(string[0])+self.__to_octet_helper(string[1])+self.__to_octet_helper(string[2]), 2)
1037
-
1038
- @GeneralUtilities.check_arguments
1039
- def __to_octet_helper(self, string: str) -> str:
1040
- if (string == "-"):
1041
- return "0"
1042
- else:
1043
- return "1"
1044
-
1045
- @GeneralUtilities.check_arguments
1046
- def get_file_owner(self, file: str) -> str:
1047
- """This function returns the user and the group in the format "user:group"."""
1048
- ls_output = self.__ls(file)
1049
- return self.__get_file_owner_helper(ls_output)
1050
-
1051
- @GeneralUtilities.check_arguments
1052
- def __get_file_owner_helper(self, ls_output: str) -> str:
1053
- try:
1054
- splitted = ' '.join(ls_output.split()).split(' ')
1055
- return f"{splitted[2]}:{splitted[3]}"
1056
- except Exception as exception:
1057
- raise ValueError(f"ls-output '{ls_output}' not parsable") from exception
1058
-
1059
- @GeneralUtilities.check_arguments
1060
- def get_file_owner_and_file_permission(self, file: str) -> str:
1061
- ls_output = self.__ls(file)
1062
- return [self.__get_file_owner_helper(ls_output), self.__get_file_permission_helper(ls_output)]
1063
-
1064
- @GeneralUtilities.check_arguments
1065
- def __ls(self, file: str) -> str:
1066
- file = file.replace("\\", "/")
1067
- GeneralUtilities.assert_condition(os.path.isfile(file) or os.path.isdir(file), f"Can not execute 'ls' because '{file}' does not exist")
1068
- result = self.run_program_argsasarray("ls", ["-ld", file])
1069
- GeneralUtilities.assert_condition(result[0] == 0, f"'ls -ld {file}' resulted in exitcode {str(result[0])}. StdErr: {result[2]}")
1070
- GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(result[1]), f"'ls' of '{file}' had an empty output. StdErr: '{result[2]}'")
1071
- return result[1]
1072
-
1073
- @GeneralUtilities.check_arguments
1074
- def set_permission(self, file_or_folder: str, permissions: str, recursive: bool = False) -> None:
1075
- """This function expects an usual octet-triple, for example "700"."""
1076
- args = []
1077
- if recursive:
1078
- args.append("--recursive")
1079
- args.append(permissions)
1080
- args.append(file_or_folder)
1081
- self.run_program_argsasarray("chmod", args)
1082
-
1083
- @GeneralUtilities.check_arguments
1084
- def set_owner(self, file_or_folder: str, owner: str, recursive: bool = False, follow_symlinks: bool = False) -> None:
1085
- """This function expects the user and the group in the format "user:group"."""
1086
- args = []
1087
- if recursive:
1088
- args.append("--recursive")
1089
- if follow_symlinks:
1090
- args.append("--no-dereference")
1091
- args.append(owner)
1092
- args.append(file_or_folder)
1093
- self.run_program_argsasarray("chown", args)
1094
-
1095
- # <run programs>
1096
-
1097
- @GeneralUtilities.check_arguments
1098
- def __run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1,
1099
- print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False,
1100
- title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None) -> Popen:
1101
- # Verbosity:
1102
- # 0=Quiet (No output will be printed.)
1103
- # 1=Normal (If the exitcode of the executed program is not 0 then the StdErr will be printed.)
1104
- # 2=Full (Prints StdOut and StdErr of the executed program.)
1105
- # 3=Verbose (Same as "Full" but with some more information.)
1106
-
1107
- if arguments_for_log is None:
1108
- arguments_for_log = ' '.join(arguments_as_array)
1109
- else:
1110
- arguments_for_log = ' '.join(arguments_for_log)
1111
- working_directory = self.__adapt_workingdirectory(working_directory)
1112
- cmd = f'{working_directory}>{program} {arguments_for_log}'
1113
-
1114
- if GeneralUtilities.string_is_none_or_whitespace(title):
1115
- info_for_log = cmd
1116
- else:
1117
- info_for_log = title
1118
-
1119
- if verbosity == 3:
1120
- GeneralUtilities.write_message_to_stdout(f"Run '{info_for_log}'.")
1121
-
1122
- if isinstance(self.program_runner, ProgramRunnerEpew):
1123
- GeneralUtilities.write_message_to_stdout("Using epew.")
1124
- custom_argument = CustomEpewArgument(print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, verbosity, arguments_for_log)
1125
- popen: Popen = self.program_runner.run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, custom_argument)
1126
- return popen
1127
-
1128
- # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
1129
-
1130
- @GeneralUtilities.check_arguments
1131
- def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1,
1132
- print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False,
1133
- title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None,
1134
- throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None) -> tuple[int, str, str, int]:
1135
- mock_loader_result = self.__try_load_mock(program, ' '.join(arguments_as_array), working_directory)
1136
- if mock_loader_result[0]:
1137
- return mock_loader_result[1]
1138
-
1139
- start_datetime = datetime.utcnow()
1140
-
1141
- if arguments_for_log is None:
1142
- arguments_for_log = arguments_as_array
1143
-
1144
- arguments_for_log_as_string = ' '.join(arguments_for_log)
1145
- cmd = f'{working_directory}>{program} {arguments_for_log_as_string}'
1146
- if GeneralUtilities.string_is_none_or_whitespace(title):
1147
- info_for_log = cmd
1148
- else:
1149
- info_for_log = title
1150
- if verbosity == 3:
1151
- GeneralUtilities.write_message_to_stdout(f"Run '{info_for_log}'.")
1152
-
1153
- epew_will_be_used = isinstance(self.program_runner, ProgramRunnerEpew)
1154
- program_manages_logging_itself = epew_will_be_used
1155
- program_manages_output_itself = epew_will_be_used
1156
-
1157
- process = self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, verbosity, print_errors_as_information, log_file,
1158
- timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument)
1159
- pid = process.pid
1160
-
1161
- if program_manages_output_itself:
1162
- for c in iter(lambda: process.stdout.read(1), b""):
1163
- sys.stdout.buffer.write(c)
1164
- for c in iter(lambda: process.stderr.read(1), b""):
1165
- sys.stderr.buffer.write(c)
1166
-
1167
- stdout, stderr = process.communicate()
1168
- exit_code = process.wait()
1169
- stdout = GeneralUtilities.bytes_to_string(stdout).replace('\r', '')
1170
- stderr = GeneralUtilities.bytes_to_string(stderr).replace('\r', '')
1171
- end_datetime = datetime.utcnow()
1172
-
1173
- if arguments_for_log is None:
1174
- arguments_for_log = ' '.join(arguments_as_array)
1175
- else:
1176
- arguments_for_log = ' '.join(arguments_for_log)
1177
-
1178
- duration: timedelta = end_datetime-start_datetime
1179
-
1180
- if GeneralUtilities.string_is_none_or_whitespace(title):
1181
- info_for_log = cmd
1182
- else:
1183
- info_for_log = title
1184
-
1185
- if not program_manages_logging_itself and log_file is not None:
1186
- GeneralUtilities.ensure_file_exists(log_file)
1187
- GeneralUtilities.append_line_to_file(log_file, stdout)
1188
- GeneralUtilities.append_line_to_file(log_file, stderr)
1189
-
1190
- if not program_manages_output_itself:
1191
- if verbosity == 1 and exit_code != 0:
1192
- self.__write_error_output(print_errors_as_information, stderr)
1193
- if verbosity == 2:
1194
- GeneralUtilities.write_message_to_stdout(stdout)
1195
- self.__write_error_output(print_errors_as_information, stderr)
1196
- if verbosity == 3:
1197
- GeneralUtilities.write_message_to_stdout(stdout)
1198
- self.__write_error_output(print_errors_as_information, stderr)
1199
- formatted = self.__format_program_execution_information(title=info_for_log, program=program, argument=arguments_for_log, workingdirectory=working_directory)
1200
- GeneralUtilities.write_message_to_stdout(f"Finished '{info_for_log}'. Details: '{formatted}")
1201
-
1202
- if throw_exception_if_exitcode_is_not_zero and exit_code != 0:
1203
- formatted = self.__format_program_execution_information(exit_code, stdout, stderr, program, arguments_for_log, working_directory, info_for_log, pid, duration)
1204
- raise ValueError(f"Finished '{info_for_log}'. Details: '{formatted}")
1205
-
1206
- result = (exit_code, stdout, stderr, pid)
1207
- return result
1208
-
1209
- # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
1210
- @GeneralUtilities.check_arguments
1211
- def run_program(self, program: str, arguments: str = "", working_directory: str = None, verbosity: int = 1,
1212
- print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False,
1213
- title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True,
1214
- custom_argument: object = None) -> tuple[int, str, str, int]:
1215
- return self.run_program_argsasarray(program, GeneralUtilities.arguments_to_array(arguments), working_directory, verbosity, print_errors_as_information,
1216
- log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument)
1217
-
1218
- # Return-values program_runner: Pid
1219
- @GeneralUtilities.check_arguments
1220
- def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1,
1221
- print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False,
1222
- title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None) -> int:
1223
- mock_loader_result = self.__try_load_mock(program, ' '.join(arguments_as_array), working_directory)
1224
- if mock_loader_result[0]:
1225
- return mock_loader_result[1]
1226
-
1227
- process: Popen = self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, verbosity,
1228
- print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument)
1229
- return process.pid
1230
-
1231
- # Return-values program_runner: Pid
1232
- @GeneralUtilities.check_arguments
1233
- def run_program_async(self, program: str, arguments: str = "", working_directory: str = None, verbosity: int = 1,
1234
- print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False,
1235
- title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None) -> int:
1236
- return self.run_program_argsasarray_async(program, GeneralUtilities.arguments_to_array(arguments), working_directory, verbosity,
1237
- print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument)
1238
-
1239
- @GeneralUtilities.check_arguments
1240
- def __try_load_mock(self, program: str, arguments: str, working_directory: str) -> tuple[bool, tuple[int, str, str, int]]:
1241
- if self.mock_program_calls:
1242
- try:
1243
- return [True, self.__get_mock_program_call(program, arguments, working_directory)]
1244
- except LookupError:
1245
- if not self.execute_program_really_if_no_mock_call_is_defined:
1246
- raise
1247
- return [False, None]
1248
-
1249
- @GeneralUtilities.check_arguments
1250
- def __adapt_workingdirectory(self, workingdirectory: str) -> str:
1251
- if workingdirectory is None:
1252
- return os.getcwd()
1253
- else:
1254
- return GeneralUtilities.resolve_relative_path_from_current_working_directory(workingdirectory)
1255
-
1256
- @GeneralUtilities.check_arguments
1257
- def __write_error_output(self, print_errors_as_information, stderr):
1258
- if print_errors_as_information:
1259
- GeneralUtilities.write_message_to_stdout(stderr)
1260
- else:
1261
- GeneralUtilities.write_message_to_stderr(stderr)
1262
-
1263
- @GeneralUtilities.check_arguments
1264
- def __format_program_execution_information(self, exitcode: int = None, stdout: str = None, stderr: str = None, program: str = None, argument: str = None,
1265
- workingdirectory: str = None, title: str = None, pid: int = None, execution_duration: timedelta = None):
1266
- result = ""
1267
- if (exitcode is not None and stdout is not None and stderr is not None):
1268
- result = f"{result} Exitcode: {exitcode}; StdOut: '{stdout}'; StdErr: '{stderr}'"
1269
- if (pid is not None):
1270
- result = f"Pid: '{pid}'; {result}"
1271
- if (program is not None and argument is not None and workingdirectory is not None):
1272
- result = f"Command: '{workingdirectory}> {program} {argument}'; {result}"
1273
- if (execution_duration is not None):
1274
- result = f"{result}; Duration: '{str(execution_duration)}'"
1275
- if (title is not None):
1276
- result = f"Title: '{title}'; {result}"
1277
- return result.strip()
1278
-
1279
- @GeneralUtilities.check_arguments
1280
- def verify_no_pending_mock_program_calls(self):
1281
- if (len(self.__mocked_program_calls) > 0):
1282
- raise AssertionError(
1283
- "The following mock-calls were not called:\n"+",\n ".join([self.__format_mock_program_call(r) for r in self.__mocked_program_calls]))
1284
-
1285
- @GeneralUtilities.check_arguments
1286
- def __format_mock_program_call(self, r) -> str:
1287
- r: ScriptCollectionCore.__MockProgramCall = r
1288
- return f"'{r.workingdirectory}>{r.program} {r.argument}' (" \
1289
- f"exitcode: {GeneralUtilities.str_none_safe(str(r.exit_code))}, " \
1290
- f"pid: {GeneralUtilities.str_none_safe(str(r.pid))}, "\
1291
- f"stdout: {GeneralUtilities.str_none_safe(str(r.stdout))}, " \
1292
- f"stderr: {GeneralUtilities.str_none_safe(str(r.stderr))})"
1293
-
1294
- @GeneralUtilities.check_arguments
1295
- def register_mock_program_call(self, program: str, argument: str, workingdirectory: str, result_exit_code: int, result_stdout: str, result_stderr: str,
1296
- result_pid: int, amount_of_expected_calls=1):
1297
- "This function is for test-purposes only"
1298
- for _ in itertools.repeat(None, amount_of_expected_calls):
1299
- mock_call = ScriptCollectionCore.__MockProgramCall()
1300
- mock_call.program = program
1301
- mock_call.argument = argument
1302
- mock_call.workingdirectory = workingdirectory
1303
- mock_call.exit_code = result_exit_code
1304
- mock_call.stdout = result_stdout
1305
- mock_call.stderr = result_stderr
1306
- mock_call.pid = result_pid
1307
- self.__mocked_program_calls.append(mock_call)
1308
-
1309
- @GeneralUtilities.check_arguments
1310
- def __get_mock_program_call(self, program: str, argument: str, workingdirectory: str):
1311
- result: ScriptCollectionCore.__MockProgramCall = None
1312
- for mock_call in self.__mocked_program_calls:
1313
- if ((re.match(mock_call.program, program) is not None)
1314
- and (re.match(mock_call.argument, argument) is not None)
1315
- and (re.match(mock_call.workingdirectory, workingdirectory) is not None)):
1316
- result = mock_call
1317
- break
1318
- if result is None:
1319
- raise LookupError(f"Tried to execute mock-call '{workingdirectory}>{program} {argument}' but no mock-call was defined for that execution")
1320
- else:
1321
- self.__mocked_program_calls.remove(result)
1322
- return (result.exit_code, result.stdout, result.stderr, result.pid)
1323
-
1324
- @GeneralUtilities.check_arguments
1325
- class __MockProgramCall:
1326
- program: str
1327
- argument: str
1328
- workingdirectory: str
1329
- exit_code: int
1330
- stdout: str
1331
- stderr: str
1332
- pid: int
1333
-
1334
- # </run programs>
1335
-
1336
- @GeneralUtilities.check_arguments
1337
- def extract_archive_with_7z(self, unzip_program_file: str, zipfile: str, password: str, output_directory: str) -> None:
1338
- password_set = not password is None
1339
- file_name = Path(zipfile).name
1340
- file_folder = os.path.dirname(zipfile)
1341
- argument = "x"
1342
- if password_set:
1343
- argument = f"{argument} -p\"{password}\""
1344
- argument = f"{argument} -o {output_directory}"
1345
- argument = f"{argument} {file_name}"
1346
- return self.run_program(unzip_program_file, argument, file_folder)
1347
-
1348
- @GeneralUtilities.check_arguments
1349
- def get_internet_time(self) -> datetime:
1350
- response = ntplib.NTPClient().request('pool.ntp.org')
1351
- return datetime.fromtimestamp(response.tx_time)
1352
-
1353
- @GeneralUtilities.check_arguments
1354
- def system_time_equals_internet_time(self, maximal_tolerance_difference: timedelta) -> bool:
1355
- return abs(datetime.now() - self.get_internet_time()) < maximal_tolerance_difference
1356
-
1357
- @GeneralUtilities.check_arguments
1358
- def system_time_equals_internet_time_with_default_tolerance(self) -> bool:
1359
- return self.system_time_equals_internet_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
1360
-
1361
- @GeneralUtilities.check_arguments
1362
- def check_system_time(self, maximal_tolerance_difference: timedelta):
1363
- if not self.system_time_equals_internet_time(maximal_tolerance_difference):
1364
- raise ValueError("System time may be wrong")
1365
-
1366
- @GeneralUtilities.check_arguments
1367
- def check_system_time_with_default_tolerance(self) -> None:
1368
- self.check_system_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
1369
-
1370
- @GeneralUtilities.check_arguments
1371
- def __get_default_tolerance_for_system_time_equals_internet_time(self) -> timedelta:
1372
- return timedelta(hours=0, minutes=0, seconds=3)
1373
-
1374
- @GeneralUtilities.check_arguments
1375
- def get_semver_version_from_gitversion(self, folder: str) -> str:
1376
- return self.get_version_from_gitversion(folder, "MajorMinorPatch")
1377
-
1378
- @GeneralUtilities.check_arguments
1379
- def get_version_from_gitversion(self, folder: str, variable: str) -> str:
1380
- # called twice as workaround for issue 1877 in gitversion ( https://github.com/GitTools/GitVersion/issues/1877 )
1381
- result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder)
1382
- result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder)
1383
- return GeneralUtilities.strip_new_line_character(result[1])
1
+ from datetime import timedelta, datetime
2
+ import json
3
+ import binascii
4
+ import filecmp
5
+ import hashlib
6
+ import multiprocessing
7
+ import time
8
+ from io import BytesIO
9
+ import itertools
10
+ import zipfile
11
+ import math
12
+ import base64
13
+ import os
14
+ from queue import Queue, Empty
15
+ from concurrent.futures import ThreadPoolExecutor
16
+ import xml.etree.ElementTree as ET
17
+ from pathlib import Path
18
+ from subprocess import Popen
19
+ import re
20
+ import shutil
21
+ from typing import IO
22
+ import fnmatch
23
+ import uuid
24
+ import tempfile
25
+ import io
26
+ import requests
27
+ import ntplib
28
+ import yaml
29
+ import qrcode
30
+ import pycdlib
31
+ import send2trash
32
+ from pypdf import PdfReader, PdfWriter
33
+ from .GeneralUtilities import GeneralUtilities
34
+ from .ProgramRunnerBase import ProgramRunnerBase
35
+ from .ProgramRunnerPopen import ProgramRunnerPopen
36
+ from .SCLog import SCLog, LogLevel
37
+
38
+ version = "4.0.78"
39
+ __version__ = version
40
+
41
+
42
+ class ScriptCollectionCore:
43
+
44
+ # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
45
+ # Do not change this value for productive environments.
46
+ mock_program_calls: bool = False#TODO remove this variable. When someone want to mock program-calls then the ProgramRunnerMock can be used instead
47
+ # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
48
+ execute_program_really_if_no_mock_call_is_defined: bool = False
49
+ __mocked_program_calls: list = None
50
+ program_runner: ProgramRunnerBase = None
51
+ call_program_runner_directly: bool = None
52
+ log: SCLog = None
53
+
54
+ def __init__(self):
55
+ self.program_runner = ProgramRunnerPopen()
56
+ self.call_program_runner_directly = None
57
+ self.__mocked_program_calls = list[ScriptCollectionCore.__MockProgramCall]()
58
+ self.log = SCLog(None, LogLevel.Warning, False)
59
+
60
+ @staticmethod
61
+ @GeneralUtilities.check_arguments
62
+ def get_scriptcollection_version() -> str:
63
+ return __version__
64
+
65
+ @GeneralUtilities.check_arguments
66
+ def python_file_has_errors(self, file: str, working_directory: str, treat_warnings_as_errors: bool = True) -> tuple[bool, list[str]]:
67
+ errors = list()
68
+ filename = os.path.relpath(file, working_directory)
69
+ if treat_warnings_as_errors:
70
+ errorsonly_argument = GeneralUtilities.empty_string
71
+ else:
72
+ errorsonly_argument = " --errors-only"
73
+ (exit_code, stdout, stderr, _) = self.run_program("pylint", filename + errorsonly_argument, working_directory, throw_exception_if_exitcode_is_not_zero=False)
74
+ if (exit_code != 0):
75
+ errors.append(f"Linting-issues of {file}:")
76
+ errors.append(f"Pylint-exitcode: {exit_code}")
77
+ for line in GeneralUtilities.string_to_lines(stdout):
78
+ errors.append(line)
79
+ for line in GeneralUtilities.string_to_lines(stderr):
80
+ errors.append(line)
81
+ return (True, errors)
82
+
83
+ return (False, errors)
84
+
85
+ @GeneralUtilities.check_arguments
86
+ def replace_version_in_dockerfile_file(self, dockerfile: str, new_version_value: str) -> None:
87
+ GeneralUtilities.write_text_to_file(dockerfile, re.sub("ARG Version=\"\\d+\\.\\d+\\.\\d+\"", f"ARG Version=\"{new_version_value}\"", GeneralUtilities.read_text_from_file(dockerfile)))
88
+
89
+ @GeneralUtilities.check_arguments
90
+ def replace_version_in_python_file(self, file: str, new_version_value: str):
91
+ GeneralUtilities.write_text_to_file(file, re.sub("version = \"\\d+\\.\\d+\\.\\d+\"", f"version = \"{new_version_value}\"", GeneralUtilities.read_text_from_file(file)))
92
+
93
+ @GeneralUtilities.check_arguments
94
+ def replace_version_in_ini_file(self, file: str, new_version_value: str):
95
+ GeneralUtilities.write_text_to_file(file, re.sub("version = \\d+\\.\\d+\\.\\d+", f"version = {new_version_value}", GeneralUtilities.read_text_from_file(file)))
96
+
97
+ @GeneralUtilities.check_arguments
98
+ def replace_version_in_nuspec_file(self, nuspec_file: str, new_version: str) -> None:
99
+ # TODO use XSLT instead
100
+ versionregex = "\\d+\\.\\d+\\.\\d+"
101
+ versiononlyregex = f"^{versionregex}$"
102
+ pattern = re.compile(versiononlyregex)
103
+ if pattern.match(new_version):
104
+ GeneralUtilities.write_text_to_file(nuspec_file, re.sub(f"<version>{versionregex}<\\/version>", f"<version>{new_version}</version>", GeneralUtilities.read_text_from_file(nuspec_file)))
105
+ else:
106
+ raise ValueError(f"Version '{new_version}' does not match version-regex '{versiononlyregex}'")
107
+
108
+ @GeneralUtilities.check_arguments
109
+ def replace_version_in_csproj_file(self, csproj_file: str, current_version: str):
110
+ versionregex = "\\d+\\.\\d+\\.\\d+"
111
+ versiononlyregex = f"^{versionregex}$"
112
+ pattern = re.compile(versiononlyregex)
113
+ if pattern.match(current_version):
114
+ for tag in ["Version", "AssemblyVersion", "FileVersion"]:
115
+ GeneralUtilities.write_text_to_file(csproj_file, re.sub(f"<{tag}>{versionregex}(.\\d+)?<\\/{tag}>", f"<{tag}>{current_version}</{tag}>", GeneralUtilities.read_text_from_file(csproj_file)))
116
+ else:
117
+ raise ValueError(f"Version '{current_version}' does not match version-regex '{versiononlyregex}'")
118
+
119
+ @GeneralUtilities.check_arguments
120
+ def push_nuget_build_artifact(self, nupkg_file: str, registry_address: str, api_key: str = None):
121
+ nupkg_file_name = os.path.basename(nupkg_file)
122
+ nupkg_file_folder = os.path.dirname(nupkg_file)
123
+ argument = f"nuget push {nupkg_file_name} --force-english-output --source {registry_address}"
124
+ if api_key is not None:
125
+ argument = f"{argument} --api-key {api_key}"
126
+ self.run_program("dotnet", argument, nupkg_file_folder)
127
+
128
+ @GeneralUtilities.check_arguments
129
+ def dotnet_build(self, folder: str, projectname: str, configuration: str):
130
+ self.run_program("dotnet", f"clean -c {configuration}", folder)
131
+ self.run_program("dotnet", f"build {projectname}/{projectname}.csproj -c {configuration}", folder)
132
+
133
+ @GeneralUtilities.check_arguments
134
+ def find_file_by_extension(self, folder: str, extension_without_dot: str):
135
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
136
+ result = [file for file in self.list_content(folder, True, False, False) if file.endswith(f".{extension_without_dot}")]
137
+ result_length = len(result)
138
+ if result_length == 0:
139
+ raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension_without_dot}'.")
140
+ if result_length == 1:
141
+ return result[0]
142
+ else:
143
+ raise ValueError(f"Multiple values available in folder '{folder}' with extension '{extension_without_dot}'.")
144
+
145
+ @GeneralUtilities.check_arguments
146
+ def find_last_file_by_extension(self, folder: str, extension_without_dot: str) -> str:
147
+ files: list[str] = GeneralUtilities.get_direct_files_of_folder(folder)
148
+ possible_results: list[str] = []
149
+ for file in files:
150
+ if file.endswith(f".{extension_without_dot}"):
151
+ possible_results.append(file)
152
+ result_length = len(possible_results)
153
+ if result_length == 0:
154
+ raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension_without_dot}'.")
155
+ else:
156
+ return possible_results[-1]
157
+
158
+ @GeneralUtilities.check_arguments
159
+ def commit_is_signed_by_key(self, repository_folder: str, revision_identifier: str, key: str) -> bool:
160
+ self.is_git_or_bare_git_repository(repository_folder)
161
+ result = self.run_program("git", f"verify-commit {revision_identifier}", repository_folder, throw_exception_if_exitcode_is_not_zero=False)
162
+ if (result[0] != 0):
163
+ return False
164
+ if (not GeneralUtilities.contains_line(result[1].splitlines(), f"gpg\\:\\ using\\ [A-Za-z0-9]+\\ key\\ [A-Za-z0-9]+{key}")):
165
+ # TODO check whether this works on machines where gpg is installed in another langauge than english
166
+ return False
167
+ if (not GeneralUtilities.contains_line(result[1].splitlines(), "gpg\\:\\ Good\\ signature\\ from")):
168
+ # TODO check whether this works on machines where gpg is installed in another langauge than english
169
+ return False
170
+ return True
171
+
172
+ @GeneralUtilities.check_arguments
173
+ def get_parent_commit_ids_of_commit(self, repository_folder: str, commit_id: str) -> str:
174
+ self.is_git_or_bare_git_repository(repository_folder)
175
+ return self.run_program("git", f'log --pretty=%P -n 1 "{commit_id}"', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string).split(" ")
176
+
177
+
178
+ @GeneralUtilities.check_arguments
179
+ def get_commit_ids_between_dates(self, repository_folder: str, since: datetime, until: datetime, ignore_commits_which_are_not_in_history_of_head: bool = True) -> None:
180
+ self.is_git_or_bare_git_repository(repository_folder)
181
+ since_as_string = self.__datetime_to_string_for_git(since)
182
+ until_as_string = self.__datetime_to_string_for_git(until)
183
+ result = filter(lambda line: not GeneralUtilities.string_is_none_or_whitespace(line), self.run_program("git", f'log --since "{since_as_string}" --until "{until_as_string}" --pretty=format:"%H" --no-patch', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].split("\n").replace("\r", GeneralUtilities.empty_string))
184
+ if ignore_commits_which_are_not_in_history_of_head:
185
+ result = [commit_id for commit_id in result if self.git_commit_is_ancestor(repository_folder, commit_id)]
186
+ return result
187
+
188
+ @GeneralUtilities.check_arguments
189
+ def __datetime_to_string_for_git(self, datetime_object: datetime) -> str:
190
+ return datetime_object.strftime('%Y-%m-%d %H:%M:%S')
191
+
192
+ @GeneralUtilities.check_arguments
193
+ def git_commit_is_ancestor(self, repository_folder: str, ancestor: str, descendant: str = "HEAD") -> bool:
194
+ self.is_git_or_bare_git_repository(repository_folder)
195
+ result = self.run_program_argsasarray("git", ["merge-base", "--is-ancestor", ancestor, descendant], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
196
+ exit_code = result[0]
197
+ if exit_code == 0:
198
+ return True
199
+ elif exit_code == 1:
200
+ return False
201
+ else:
202
+ raise ValueError(f'Can not calculate if {ancestor} is an ancestor of {descendant} in repository {repository_folder}. Outout of "{repository_folder}> git merge-base --is-ancestor {ancestor} {descendant}": Exitcode: {exit_code}; StdOut: {result[1]}; StdErr: {result[2]}.')
203
+
204
+ @GeneralUtilities.check_arguments
205
+ def __git_changes_helper(self, repository_folder: str, arguments_as_array: list[str]) -> bool:
206
+ self.assert_is_git_repository(repository_folder)
207
+ lines = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", arguments_as_array, repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1], False)
208
+ for line in lines:
209
+ if GeneralUtilities.string_has_content(line):
210
+ return True
211
+ return False
212
+
213
+ @GeneralUtilities.check_arguments
214
+ def git_repository_has_new_untracked_files(self, repository_folder: str):
215
+ self.assert_is_git_repository(repository_folder)
216
+ return self.__git_changes_helper(repository_folder, ["ls-files", "--exclude-standard", "--others"])
217
+
218
+ @GeneralUtilities.check_arguments
219
+ def git_repository_has_unstaged_changes_of_tracked_files(self, repository_folder: str):
220
+ self.assert_is_git_repository(repository_folder)
221
+ return self.__git_changes_helper(repository_folder, ["--no-pager", "diff"])
222
+
223
+ @GeneralUtilities.check_arguments
224
+ def git_repository_has_staged_changes(self, repository_folder: str):
225
+ self.assert_is_git_repository(repository_folder)
226
+ return self.__git_changes_helper(repository_folder, ["--no-pager", "diff", "--cached"])
227
+
228
+ @GeneralUtilities.check_arguments
229
+ def git_repository_has_uncommitted_changes(self, repository_folder: str) -> bool:
230
+ self.assert_is_git_repository(repository_folder)
231
+ if (self.git_repository_has_unstaged_changes(repository_folder)):
232
+ return True
233
+ if (self.git_repository_has_staged_changes(repository_folder)):
234
+ return True
235
+ return False
236
+
237
+ @GeneralUtilities.check_arguments
238
+ def git_repository_has_unstaged_changes(self, repository_folder: str) -> bool:
239
+ self.assert_is_git_repository(repository_folder)
240
+ if (self.git_repository_has_unstaged_changes_of_tracked_files(repository_folder)):
241
+ return True
242
+ if (self.git_repository_has_new_untracked_files(repository_folder)):
243
+ return True
244
+ return False
245
+
246
+ @GeneralUtilities.check_arguments
247
+ def git_get_commit_id(self, repository_folder: str, commit: str = "HEAD") -> str:
248
+ self.is_git_or_bare_git_repository(repository_folder)
249
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["rev-parse", "--verify", commit], repository_folder, throw_exception_if_exitcode_is_not_zero=True)
250
+ return result[1].replace('\n', '')
251
+
252
+ @GeneralUtilities.check_arguments
253
+ def git_get_commit_date(self, repository_folder: str, commit: str = "HEAD") -> datetime:
254
+ self.is_git_or_bare_git_repository(repository_folder)
255
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["show", "-s", "--format=%ci", commit], repository_folder, throw_exception_if_exitcode_is_not_zero=True)
256
+ date_as_string = result[1].replace('\n', '')
257
+ result = datetime.strptime(date_as_string, '%Y-%m-%d %H:%M:%S %z')
258
+ return result
259
+
260
+ @GeneralUtilities.check_arguments
261
+ def git_fetch_with_retry(self, folder: str, remotename: str = "--all", amount_of_attempts: int = 5) -> None:
262
+ GeneralUtilities.retry_action(lambda: self.git_fetch(folder, remotename), amount_of_attempts)
263
+
264
+ @GeneralUtilities.check_arguments
265
+ def git_fetch(self, folder: str, remotename: str = "--all") -> None:
266
+ self.is_git_or_bare_git_repository(folder)
267
+ self.run_program_argsasarray("git", ["fetch", remotename, "--tags", "--prune"], folder, throw_exception_if_exitcode_is_not_zero=True)
268
+
269
+ @GeneralUtilities.check_arguments
270
+ def git_fetch_in_bare_repository(self, folder: str, remotename, localbranch: str, remotebranch: str) -> None:
271
+ self.is_git_or_bare_git_repository(folder)
272
+ self.run_program_argsasarray("git", ["fetch", remotename, f"{remotebranch}:{localbranch}"], folder, throw_exception_if_exitcode_is_not_zero=True)
273
+
274
+ @GeneralUtilities.check_arguments
275
+ def git_remove_branch(self, folder: str, branchname: str) -> None:
276
+ self.is_git_or_bare_git_repository(folder)
277
+ self.run_program("git", f"branch -D {branchname}", folder, throw_exception_if_exitcode_is_not_zero=True)
278
+
279
+ @GeneralUtilities.check_arguments
280
+ def git_push_with_retry(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: LogLevel = LogLevel.Quiet, amount_of_attempts: int = 5) -> None:
281
+ GeneralUtilities.retry_action(lambda: self.git_push(folder, remotename, localbranchname, remotebranchname, forcepush, pushalltags, verbosity), amount_of_attempts)
282
+
283
+ @GeneralUtilities.check_arguments
284
+ def git_push(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: LogLevel = LogLevel.Quiet,resurse_submodules:bool=False) -> None:
285
+ self.is_git_or_bare_git_repository(folder)
286
+ argument = ["push"]
287
+ if resurse_submodules:
288
+ argument = argument + ["--recurse-submodules=on-demand"]
289
+ argument = argument + [remotename, f"{localbranchname}:{remotebranchname}"]
290
+ if (forcepush):
291
+ argument.append("--force")
292
+ if (pushalltags):
293
+ argument.append("--tags")
294
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True, print_errors_as_information=True)
295
+ return result[1].replace('\r', '').replace('\n', '')
296
+
297
+ @GeneralUtilities.check_arguments
298
+ def git_pull_with_retry(self, folder: str, remote: str, localbranchname: str, remotebranchname: str, force: bool = False, amount_of_attempts: int = 5) -> None:
299
+ GeneralUtilities.retry_action(lambda: self.git_pull(folder, remote, localbranchname, remotebranchname), amount_of_attempts)
300
+
301
+ @GeneralUtilities.check_arguments
302
+ def git_pull(self, folder: str, remote: str, localbranchname: str, remotebranchname: str, force: bool = False) -> None:
303
+ self.is_git_or_bare_git_repository(folder)
304
+ argument = f"pull {remote} {remotebranchname}:{localbranchname}"
305
+ if force:
306
+ argument = f"{argument} --force"
307
+ self.run_program("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
308
+
309
+ @GeneralUtilities.check_arguments
310
+ def git_list_remote_branches(self, folder: str, remote: str, fetch: bool) -> list[str]:
311
+ self.is_git_or_bare_git_repository(folder)
312
+ if fetch:
313
+ self.git_fetch(folder, remote)
314
+ run_program_result = self.run_program("git", f"branch -rl {remote}/*", folder, throw_exception_if_exitcode_is_not_zero=True)
315
+ output = GeneralUtilities.string_to_lines(run_program_result[1])
316
+ result = list[str]()
317
+ for item in output:
318
+ striped_item = item.strip()
319
+ if GeneralUtilities.string_has_content(striped_item):
320
+ branch: str = None
321
+ if " " in striped_item:
322
+ branch = striped_item.split(" ")[0]
323
+ else:
324
+ branch = striped_item
325
+ branchname = branch[len(remote)+1:]
326
+ if branchname != "HEAD":
327
+ result.append(branchname)
328
+ return result
329
+
330
+ @GeneralUtilities.check_arguments
331
+ def git_clone(self, clone_target_folder: str, remote_repository_path: str, include_submodules: bool = True, mirror: bool = False) -> None:
332
+ if (os.path.isdir(clone_target_folder)):
333
+ pass # TODO throw error
334
+ else:
335
+ args = ["clone", remote_repository_path, clone_target_folder]
336
+ if include_submodules:
337
+ args.append("--recurse-submodules")
338
+ args.append("--remote-submodules")
339
+ if mirror:
340
+ args.append("--mirror")
341
+ self.run_program_argsasarray("git", args, os.getcwd(), throw_exception_if_exitcode_is_not_zero=True)
342
+
343
+ @GeneralUtilities.check_arguments
344
+ def git_get_all_remote_names(self, directory: str) -> list[str]:
345
+ self.is_git_or_bare_git_repository(directory)
346
+ result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote"], directory, throw_exception_if_exitcode_is_not_zero=True)[1], False)
347
+ return result
348
+
349
+ @GeneralUtilities.check_arguments
350
+ def git_get_remote_url(self, directory: str, remote_name: str) -> str:
351
+ self.is_git_or_bare_git_repository(directory)
352
+ result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote", "get-url", remote_name], directory, throw_exception_if_exitcode_is_not_zero=True)[1], False)
353
+ return result[0].replace('\n', '')
354
+
355
+ @GeneralUtilities.check_arguments
356
+ def repository_has_remote_with_specific_name(self, directory: str, remote_name: str) -> bool:
357
+ self.is_git_or_bare_git_repository(directory)
358
+ return remote_name in self.git_get_all_remote_names(directory)
359
+
360
+ @GeneralUtilities.check_arguments
361
+ def git_add_or_set_remote_address(self, directory: str, remote_name: str, remote_address: str) -> None:
362
+ self.assert_is_git_repository(directory)
363
+ if (self.repository_has_remote_with_specific_name(directory, remote_name)):
364
+ self.run_program_argsasarray("git", ['remote', 'set-url', 'remote_name', remote_address], directory, throw_exception_if_exitcode_is_not_zero=True)
365
+ else:
366
+ self.run_program_argsasarray("git", ['remote', 'add', remote_name, remote_address], directory, throw_exception_if_exitcode_is_not_zero=True)
367
+
368
+ @GeneralUtilities.check_arguments
369
+ def git_stage_all_changes(self, directory: str) -> None:
370
+ self.assert_is_git_repository(directory)
371
+ self.run_program_argsasarray("git", ["add", "-A"], directory, throw_exception_if_exitcode_is_not_zero=True)
372
+
373
+ @GeneralUtilities.check_arguments
374
+ def git_unstage_all_changes(self, directory: str) -> None:
375
+ self.assert_is_git_repository(directory)
376
+ self.run_program_argsasarray("git", ["reset"], directory, throw_exception_if_exitcode_is_not_zero=True)
377
+ # TODO check if this will also be done for submodules
378
+
379
+ @GeneralUtilities.check_arguments
380
+ def git_stage_file(self, directory: str, file: str) -> None:
381
+ self.assert_is_git_repository(directory)
382
+ self.run_program_argsasarray("git", ['stage', file], directory, throw_exception_if_exitcode_is_not_zero=True)
383
+
384
+ @GeneralUtilities.check_arguments
385
+ def git_unstage_file(self, directory: str, file: str) -> None:
386
+ self.assert_is_git_repository(directory)
387
+ self.run_program_argsasarray("git", ['reset', file], directory, throw_exception_if_exitcode_is_not_zero=True)
388
+
389
+ @GeneralUtilities.check_arguments
390
+ def git_discard_unstaged_changes_of_file(self, directory: str, file: str) -> None:
391
+ """Caution: This method works really only for 'changed' files yet. So this method does not work properly for new or renamed files."""
392
+ self.assert_is_git_repository(directory)
393
+ self.run_program_argsasarray("git", ['checkout', file], directory, throw_exception_if_exitcode_is_not_zero=True)
394
+
395
+ @GeneralUtilities.check_arguments
396
+ def git_discard_all_unstaged_changes(self, directory: str) -> None:
397
+ """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
398
+ self.assert_is_git_repository(directory)
399
+ self.run_program_argsasarray("git", ['clean', '-df'], directory, throw_exception_if_exitcode_is_not_zero=True)
400
+ self.run_program_argsasarray("git", ['checkout', '.'], directory, throw_exception_if_exitcode_is_not_zero=True)
401
+ # TODO check if this will also be done for submodules
402
+
403
+ @GeneralUtilities.check_arguments
404
+ def git_commit(self, directory: str, message: str = "Saved changes.", author_name: str = None, author_email: str = None, stage_all_changes: bool = True, no_changes_behavior: int = 0) -> str:
405
+ """no_changes_behavior=0 => No commit; no_changes_behavior=1 => Commit anyway; no_changes_behavior=2 => Exception"""
406
+ self.assert_is_git_repository(directory)
407
+ author_name = GeneralUtilities.str_none_safe(author_name).strip()
408
+ author_email = GeneralUtilities.str_none_safe(author_email).strip()
409
+ argument = ['commit', '--quiet', '--allow-empty', '--message', message]
410
+ if (GeneralUtilities.string_has_content(author_name)):
411
+ argument.append(f'--author="{author_name} <{author_email}>"')
412
+ git_repository_has_uncommitted_changes = self.git_repository_has_uncommitted_changes(directory)
413
+
414
+ if git_repository_has_uncommitted_changes:
415
+ do_commit = True
416
+ if stage_all_changes:
417
+ self.git_stage_all_changes(directory)
418
+ else:
419
+ if no_changes_behavior == 0:
420
+ self.log.log(f"Commit '{message}' will not be done because there are no changes to commit in repository '{directory}'", LogLevel.Debug)
421
+ do_commit = False
422
+ elif no_changes_behavior == 1:
423
+ self.log.log(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will be done anyway.", LogLevel.Debug)
424
+ do_commit = True
425
+ elif no_changes_behavior == 2:
426
+ raise RuntimeError(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will not be done.")
427
+ else:
428
+ raise ValueError(f"Unknown value for no_changes_behavior: {GeneralUtilities.str_none_safe(no_changes_behavior)}")
429
+
430
+ if do_commit:
431
+ self.log.log(f"Commit changes in '{directory}'", LogLevel.Information)
432
+ self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True)
433
+
434
+ return self.git_get_commit_id(directory)
435
+
436
+ def search_repository_folder(self,some_file_in_repository:str)->str:
437
+ current_path:str=os.path.dirname(some_file_in_repository)
438
+ enabled:bool=True
439
+ while enabled:
440
+ try:
441
+ current_path=GeneralUtilities.resolve_relative_path("..",current_path)
442
+ if self.is_git_repository(current_path):
443
+ return current_path
444
+ except:
445
+ enabled=False
446
+ raise ValueError(f"Can not find git-repository for folder \"{some_file_in_repository}\".")
447
+
448
+
449
+ @GeneralUtilities.check_arguments
450
+ def git_create_tag(self, directory: str, target_for_tag: str, tag: str, sign: bool = False, message: str = None) -> None:
451
+ self.is_git_or_bare_git_repository(directory)
452
+ argument = ["tag", tag, target_for_tag]
453
+ if sign:
454
+ if message is None:
455
+ message = f"Created {target_for_tag}"
456
+ argument.extend(["-s", '-m', message])
457
+ self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True)
458
+
459
+ @GeneralUtilities.check_arguments
460
+ def git_delete_tag(self, directory: str, tag: str) -> None:
461
+ self.is_git_or_bare_git_repository(directory)
462
+ self.run_program_argsasarray("git", ["tag", "--delete", tag], directory, throw_exception_if_exitcode_is_not_zero=True)
463
+
464
+ @GeneralUtilities.check_arguments
465
+ def git_checkout(self, directory: str, branch: str, undo_all_changes_after_checkout: bool = True, assert_no_uncommitted_changes: bool = True) -> None:
466
+ self.assert_is_git_repository(directory)
467
+ if assert_no_uncommitted_changes:
468
+ GeneralUtilities.assert_condition(not self.git_repository_has_uncommitted_changes(directory), f"Repository '{directory}' has uncommitted changes.")
469
+ self.run_program_argsasarray("git", ["checkout", branch], directory, throw_exception_if_exitcode_is_not_zero=True)
470
+ self.run_program_argsasarray("git", ["submodule", "update", "--recursive"], directory, throw_exception_if_exitcode_is_not_zero=True)
471
+ if undo_all_changes_after_checkout:
472
+ self.git_undo_all_changes(directory)
473
+
474
+ @GeneralUtilities.check_arguments
475
+ def merge_repository(self, repository_folder: str, remote: str, branch: str, pull_first_if_there_are_no_uncommitted_changes: bool = True):
476
+ if pull_first_if_there_are_no_uncommitted_changes:
477
+ uncommitted_changes = self.git_repository_has_uncommitted_changes(repository_folder)
478
+ if not uncommitted_changes:
479
+ is_pullable: bool = self.git_commit_is_ancestor(repository_folder, branch, f"{remote}/{branch}")
480
+ if is_pullable:
481
+ self.git_pull(repository_folder, remote, branch, branch)
482
+ uncommitted_changes = self.git_repository_has_uncommitted_changes(repository_folder)
483
+ GeneralUtilities.assert_condition(not uncommitted_changes, f"Pulling remote \"{remote}\" in \"{repository_folder}\" caused new uncommitted files.")
484
+ self.git_checkout(repository_folder, branch)
485
+ self.git_commit(repository_folder, "Automatic commit due to merge")
486
+ self.git_fetch(repository_folder, remote)
487
+ self.git_merge(repository_folder, f"{remote}/{branch}", branch)
488
+ self.git_push_with_retry(repository_folder, remote, branch, branch)
489
+ self.git_checkout(repository_folder, branch)
490
+
491
+ @GeneralUtilities.check_arguments
492
+ def git_merge_abort(self, directory: str) -> None:
493
+ self.assert_is_git_repository(directory)
494
+ self.run_program_argsasarray("git", ["merge", "--abort"], directory, throw_exception_if_exitcode_is_not_zero=True)
495
+
496
+ @GeneralUtilities.check_arguments
497
+ def git_merge(self, directory: str, sourcebranch: str, targetbranch: str, fastforward: bool = True, commit: bool = True, commit_message: str = None, undo_all_changes_after_checkout: bool = True, assert_no_uncommitted_changes: bool = True) -> str:
498
+ self.assert_is_git_repository(directory)
499
+ self.git_checkout(directory, targetbranch, undo_all_changes_after_checkout, assert_no_uncommitted_changes)
500
+ args = ["merge"]
501
+ if not commit:
502
+ args.append("--no-commit")
503
+ if not fastforward:
504
+ args.append("--no-ff")
505
+ if commit_message is not None:
506
+ args.append("-m")
507
+ args.append(commit_message)
508
+ args.append(sourcebranch)
509
+ self.run_program_argsasarray("git", args, directory, throw_exception_if_exitcode_is_not_zero=True)
510
+ self.run_program_argsasarray("git", ["submodule", "update"], directory, throw_exception_if_exitcode_is_not_zero=True)
511
+ return self.git_get_commit_id(directory)
512
+
513
+ @GeneralUtilities.check_arguments
514
+ def git_undo_all_changes(self, directory: str) -> None:
515
+ """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
516
+ self.assert_is_git_repository(directory)
517
+ self.git_unstage_all_changes(directory)
518
+ self.git_discard_all_unstaged_changes(directory)
519
+
520
+ @GeneralUtilities.check_arguments
521
+ def git_fetch_or_clone_all_in_directory(self, source_directory: str, target_directory: str) -> None:
522
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(source_directory):
523
+ foldername = os.path.basename(subfolder)
524
+ if self.is_git_repository(subfolder):
525
+ source_repository = subfolder
526
+ target_repository = os.path.join(target_directory, foldername)
527
+ if os.path.isdir(target_directory):
528
+ # fetch
529
+ self.git_fetch(target_directory)
530
+ else:
531
+ # clone
532
+ self.git_clone(target_repository, source_repository, include_submodules=True, mirror=True)
533
+
534
+ def get_git_submodules(self, directory: str) -> list[str]:
535
+ self.is_git_or_bare_git_repository(directory)
536
+ e = self.run_program("git", "submodule status", directory)
537
+ result = []
538
+ for submodule_line in GeneralUtilities.string_to_lines(e[1], False, True):
539
+ result.append(submodule_line.split(' ')[1])
540
+ return result
541
+
542
+ @GeneralUtilities.check_arguments
543
+ def file_is_git_ignored(self, file_in_repository: str, repositorybasefolder: str) -> None:
544
+ self.is_git_or_bare_git_repository(repositorybasefolder)
545
+ exit_code = self.run_program_argsasarray("git", ['check-ignore', file_in_repository], repositorybasefolder, throw_exception_if_exitcode_is_not_zero=False)[0]
546
+ if (exit_code == 0):
547
+ return True
548
+ if (exit_code == 1):
549
+ return False
550
+ raise ValueError(f"Unable to calculate whether '{file_in_repository}' in repository '{repositorybasefolder}' is ignored due to git-exitcode {exit_code}.")
551
+
552
+ @GeneralUtilities.check_arguments
553
+ def git_discard_all_changes(self, repository: str) -> None:
554
+ self.assert_is_git_repository(repository)
555
+ self.run_program_argsasarray("git", ["reset", "HEAD", "."], repository, throw_exception_if_exitcode_is_not_zero=True)
556
+ self.run_program_argsasarray("git", ["checkout", "."], repository, throw_exception_if_exitcode_is_not_zero=True)
557
+
558
+ @GeneralUtilities.check_arguments
559
+ def git_get_current_branch_name(self, repository: str) -> str:
560
+ self.assert_is_git_repository(repository)
561
+ result = self.run_program_argsasarray("git", ["rev-parse", "--abbrev-ref", "HEAD"], repository, throw_exception_if_exitcode_is_not_zero=True)
562
+ return result[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
563
+
564
+ @GeneralUtilities.check_arguments
565
+ def git_get_commitid_of_tag(self, repository: str, tag: str) -> str:
566
+ self.is_git_or_bare_git_repository(repository)
567
+ stdout = self.run_program_argsasarray("git", ["rev-list", "-n", "1", tag], repository)
568
+ result = stdout[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
569
+ return result
570
+
571
+ @GeneralUtilities.check_arguments
572
+ def git_get_tags(self, repository: str) -> list[str]:
573
+ self.is_git_or_bare_git_repository(repository)
574
+ tags = [line.replace("\r", GeneralUtilities.empty_string) for line in self.run_program_argsasarray(
575
+ "git", ["tag"], repository)[1].split("\n") if len(line) > 0]
576
+ return tags
577
+
578
+ @GeneralUtilities.check_arguments
579
+ def git_move_tags_to_another_branch(self, repository: str, tag_source_branch: str, tag_target_branch: str, sign: bool = False, message: str = None) -> None:
580
+ self.is_git_or_bare_git_repository(repository)
581
+ tags = self.git_get_tags(repository)
582
+ tags_count = len(tags)
583
+ counter = 0
584
+ for tag in tags:
585
+ counter = counter+1
586
+ self.log.log(f"Process tag {counter}/{tags_count}.", LogLevel.Information)
587
+ # tag is on source-branch
588
+ if self.git_commit_is_ancestor(repository, tag, tag_source_branch):
589
+ commit_id_old = self.git_get_commitid_of_tag(repository, tag)
590
+ commit_date: datetime = self.git_get_commit_date(repository, commit_id_old)
591
+ date_as_string = self.__datetime_to_string_for_git(commit_date)
592
+ search_commit_result = self.run_program_argsasarray("git", ["log", f'--after="{date_as_string}"', f'--before="{date_as_string}"', "--pretty=format:%H", tag_target_branch], repository, throw_exception_if_exitcode_is_not_zero=False)
593
+ if search_commit_result[0] != 0 or not GeneralUtilities.string_has_nonwhitespace_content(search_commit_result[1]):
594
+ raise ValueError(f"Can not calculate corresponding commit for tag '{tag}'.")
595
+ commit_id_new = search_commit_result[1]
596
+ self.git_delete_tag(repository, tag)
597
+ self.git_create_tag(repository, commit_id_new, tag, sign, message)
598
+
599
+ @GeneralUtilities.check_arguments
600
+ def get_current_git_branch_has_tag(self, repository_folder: str) -> bool:
601
+ self.is_git_or_bare_git_repository(repository_folder)
602
+ result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
603
+ return result[0] == 0
604
+
605
+ @GeneralUtilities.check_arguments
606
+ def get_latest_git_tag(self, repository_folder: str) -> str:
607
+ self.is_git_or_bare_git_repository(repository_folder)
608
+ result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder)
609
+ result = result[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
610
+ return result
611
+
612
+ @GeneralUtilities.check_arguments
613
+ def get_staged_or_committed_git_ignored_files(self, repository_folder: str) -> list[str]:
614
+ self.assert_is_git_repository(repository_folder)
615
+ temp_result = self.run_program_argsasarray("git", ["ls-files", "-i", "-c", "--exclude-standard"], repository_folder)
616
+ temp_result = temp_result[1].replace("\r", GeneralUtilities.empty_string)
617
+ result = [line for line in temp_result.split("\n") if len(line) > 0]
618
+ return result
619
+
620
+ @GeneralUtilities.check_arguments
621
+ def git_repository_has_commits(self, repository_folder: str) -> bool:
622
+ self.assert_is_git_repository(repository_folder)
623
+ return self.run_program_argsasarray("git", ["rev-parse", "--verify", "HEAD"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)[0] == 0
624
+
625
+ @GeneralUtilities.check_arguments
626
+ def run_git_command_in_repository_and_submodules(self, repository_folder: str, arguments: list[str]) -> None:
627
+ self.is_git_or_bare_git_repository(repository_folder)
628
+ self.run_program_argsasarray("git", arguments, repository_folder)
629
+ self.run_program_argsasarray("git", ["submodule", "foreach", "--recursive", "git"]+arguments, repository_folder)
630
+
631
+ @GeneralUtilities.check_arguments
632
+ def export_filemetadata(self, folder: str, target_file: str, encoding: str = "utf-8", filter_function=None) -> None:
633
+ folder = GeneralUtilities.resolve_relative_path_from_current_working_directory(folder)
634
+ lines = list()
635
+ path_prefix = len(folder)+1
636
+ items = dict()
637
+ for item in GeneralUtilities.get_all_folders_of_folder(folder):
638
+ items[item] = "d"
639
+ for item in GeneralUtilities.get_all_files_of_folder(folder):
640
+ items[item] = "f"
641
+ for file_or_folder, item_type in items.items():
642
+ truncated_file = file_or_folder[path_prefix:]
643
+ if (filter_function is None or filter_function(folder, truncated_file)):
644
+ owner_and_permisssion = self.get_file_owner_and_file_permission(file_or_folder)
645
+ user = owner_and_permisssion[0]
646
+ permissions = owner_and_permisssion[1]
647
+ lines.append(f"{truncated_file};{item_type};{user};{permissions}")
648
+ lines = sorted(lines, key=str.casefold)
649
+ with open(target_file, "w", encoding=encoding) as file_object:
650
+ file_object.write("\n".join(lines))
651
+
652
+ @GeneralUtilities.check_arguments
653
+ def escape_git_repositories_in_folder(self, folder: str) -> dict[str, str]:
654
+ return self.__escape_git_repositories_in_folder_internal(folder, dict[str, str]())
655
+
656
+ @GeneralUtilities.check_arguments
657
+ def __escape_git_repositories_in_folder_internal(self, folder: str, renamed_items: dict[str, str]) -> dict[str, str]:
658
+ for file in GeneralUtilities.get_direct_files_of_folder(folder):
659
+ filename = os.path.basename(file)
660
+ if ".git" in filename:
661
+ new_name = filename.replace(".git", ".gitx")
662
+ target = os.path.join(folder, new_name)
663
+ os.rename(file, target)
664
+ renamed_items[target] = file
665
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
666
+ foldername = os.path.basename(subfolder)
667
+ if ".git" in foldername:
668
+ new_name = foldername.replace(".git", ".gitx")
669
+ subfolder2 = os.path.join(str(Path(subfolder).parent), new_name)
670
+ os.rename(subfolder, subfolder2)
671
+ renamed_items[subfolder2] = subfolder
672
+ else:
673
+ subfolder2 = subfolder
674
+ self.__escape_git_repositories_in_folder_internal(subfolder2, renamed_items)
675
+ return renamed_items
676
+
677
+ @GeneralUtilities.check_arguments
678
+ def deescape_git_repositories_in_folder(self, renamed_items: dict[str, str]):
679
+ for renamed_item, original_name in renamed_items.items():
680
+ os.rename(renamed_item, original_name)
681
+
682
+ @GeneralUtilities.check_arguments
683
+ def is_git_repository(self, folder: str) -> bool:
684
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
685
+ folder=folder.replace("\\","/")
686
+ if folder.endswith("/"):
687
+ folder = folder[:-1]
688
+ if not self.is_folder(folder):
689
+ raise ValueError(f"Folder '{folder}' does not exist.")
690
+ git_folder_path = f"{folder}/.git"
691
+ return self.is_folder(git_folder_path) or self.is_file(git_folder_path)
692
+
693
+ @GeneralUtilities.check_arguments
694
+ def is_bare_git_repository(self, folder: str) -> bool:
695
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
696
+ if folder.endswith("/") or folder.endswith("\\"):
697
+ folder = folder[:-1]
698
+ if not self.is_folder(folder):
699
+ raise ValueError(f"Folder '{folder}' does not exist.")
700
+ return folder.endswith(".git")
701
+
702
+ @GeneralUtilities.check_arguments
703
+ def is_git_or_bare_git_repository(self, folder: str) -> bool:
704
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
705
+ return self.is_git_repository(folder) or self.is_bare_git_repository(folder)
706
+
707
+ @GeneralUtilities.check_arguments
708
+ def assert_is_git_repository(self, folder: str) -> str:
709
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
710
+ GeneralUtilities.assert_condition(self.is_git_repository(folder), f"'{folder}' is not a git-repository.")
711
+
712
+ @GeneralUtilities.check_arguments
713
+ def convert_git_repository_to_bare_repository(self, repository_folder: str):
714
+ repository_folder = repository_folder.replace("\\", "/")
715
+ self.assert_is_git_repository(repository_folder)
716
+ git_folder = repository_folder + "/.git"
717
+ if not self.is_folder(git_folder):
718
+ raise ValueError(f"Converting '{repository_folder}' to a bare repository not possible. The folder '{git_folder}' does not exist. Converting is currently only supported when the git-folder is a direct folder in a repository and not a reference to another location.")
719
+ target_folder: str = repository_folder + ".git"
720
+ GeneralUtilities.ensure_directory_exists(target_folder)
721
+ GeneralUtilities.move_content_of_folder(git_folder, target_folder)
722
+ GeneralUtilities.ensure_directory_does_not_exist(repository_folder)
723
+ self.run_program_argsasarray("git", ["config", "--bool", "core.bare", "true"], target_folder)
724
+
725
+ @GeneralUtilities.check_arguments
726
+ def assert_no_uncommitted_changes(self, repository_folder: str):
727
+ if self.git_repository_has_uncommitted_changes(repository_folder):
728
+ raise ValueError(f"Repository '{repository_folder}' has uncommitted changes.")
729
+
730
+ @GeneralUtilities.check_arguments
731
+ def list_content(self, path: str, include_files: bool, include_folder: bool, printonlynamewithoutpath: bool) -> list[str]:
732
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
733
+ result: list[str] = []
734
+ if self.program_runner.will_be_executed_locally():
735
+ if include_files:
736
+ result = result + GeneralUtilities.get_direct_files_of_folder(path)
737
+ if include_folder:
738
+ result = result + GeneralUtilities.get_direct_folders_of_folder(path)
739
+ else:
740
+ arguments = ["--path", path]
741
+ if not include_files:
742
+ arguments = arguments+["--excludefiles"]
743
+ if not include_folder:
744
+ arguments = arguments+["--excludedirectories"]
745
+ if printonlynamewithoutpath:
746
+ arguments = arguments+["--printonlynamewithoutpath"]
747
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("sclistfoldercontent", arguments)
748
+ if exit_code == 0:
749
+ for line in stdout.split("\n"):
750
+ normalized_line = line.replace("\r", "")
751
+ result.append(normalized_line)
752
+ else:
753
+ raise ValueError(f"Fatal error occurrs while checking whether file '{path}' exists. StdErr: '{stderr}'")
754
+ result = [item for item in result if GeneralUtilities.string_has_nonwhitespace_content(item)]
755
+ return result
756
+
757
+ @GeneralUtilities.check_arguments
758
+ def is_file(self, path: str) -> bool:
759
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
760
+ if self.program_runner.will_be_executed_locally():
761
+ return os.path.isfile(path) # works only locally, but much more performant than always running an external program
762
+ else:
763
+ exit_code, _, stderr, _ = self.run_program_argsasarray("scfileexists", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
764
+ if exit_code == 0:
765
+ return True
766
+ elif exit_code == 1:
767
+ raise ValueError(f"Not calculatable whether file '{path}' exists. StdErr: '{stderr}'")
768
+ elif exit_code == 2:
769
+ return False
770
+ raise ValueError(f"Fatal error occurrs while checking whether file '{path}' exists. StdErr: '{stderr}'")
771
+
772
+ @GeneralUtilities.check_arguments
773
+ def is_folder(self, path: str) -> bool:
774
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
775
+ if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
776
+ return os.path.isdir(path)
777
+ else:
778
+ exit_code, _, stderr, _ = self.run_program_argsasarray("scfolderexists", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
779
+ if exit_code == 0:
780
+ return True
781
+ elif exit_code == 1:
782
+ raise ValueError(f"Not calculatable whether folder '{path}' exists. StdErr: '{stderr}'")
783
+ elif exit_code == 2:
784
+ return False
785
+ raise ValueError(f"Fatal error occurrs while checking whether folder '{path}' exists. StdErr: '{stderr}'")
786
+
787
+ @GeneralUtilities.check_arguments
788
+ def get_file_content(self, path: str, encoding: str = "utf-8") -> str:
789
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
790
+ if self.program_runner.will_be_executed_locally():
791
+ return GeneralUtilities.read_text_from_file(path, encoding)
792
+ else:
793
+ result = self.run_program_argsasarray("scprintfilecontent", ["--path", path, "--encofing", encoding]) # works platform-indepent
794
+ return result[1].replace("\\n", "\n")
795
+
796
+ @GeneralUtilities.check_arguments
797
+ def set_file_content(self, path: str, content: str, encoding: str = "utf-8") -> None:
798
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
799
+ if self.program_runner.will_be_executed_locally():
800
+ GeneralUtilities.write_text_to_file(path, content, encoding)
801
+ else:
802
+ content_bytes = content.encode('utf-8')
803
+ base64_bytes = base64.b64encode(content_bytes)
804
+ base64_string = base64_bytes.decode('utf-8')
805
+ self.run_program_argsasarray("scsetfilecontent", ["--path", path, "--argumentisinbase64", "--content", base64_string]) # works platform-indepent
806
+
807
+ @GeneralUtilities.check_arguments
808
+ def remove(self, path: str) -> None:
809
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
810
+ if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
811
+ if os.path.isdir(path):
812
+ GeneralUtilities.ensure_directory_does_not_exist(path)
813
+ if os.path.isfile(path):
814
+ GeneralUtilities.ensure_file_does_not_exist(path)
815
+ else:
816
+ if self.is_file(path):
817
+ exit_code, _, stderr, _ = self.run_program_argsasarray("scremovefile", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
818
+ if exit_code != 0:
819
+ raise ValueError(f"Fatal error occurrs while removing file '{path}'. StdErr: '{stderr}'")
820
+ if self.is_folder(path):
821
+ exit_code, _, stderr, _ = self.run_program_argsasarray("scremovefolder", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
822
+ if exit_code != 0:
823
+ raise ValueError(f"Fatal error occurrs while removing folder '{path}'. StdErr: '{stderr}'")
824
+
825
+ @GeneralUtilities.check_arguments
826
+ def rename(self, source: str, target: str) -> None:
827
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
828
+ if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
829
+ os.rename(source, target)
830
+ else:
831
+ exit_code, _, stderr, _ = self.run_program_argsasarray("screname", ["--source", source, "--target", target], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
832
+ if exit_code != 0:
833
+ raise ValueError(f"Fatal error occurrs while renaming '{source}' to '{target}'. StdErr: '{stderr}'")
834
+
835
+ @GeneralUtilities.check_arguments
836
+ def copy(self, source: str, target: str) -> None:
837
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
838
+ if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
839
+ if os.path.isfile(target) or os.path.isdir(target):
840
+ raise ValueError(f"Can not copy to '{target}' because the target already exists.")
841
+ if os.path.isfile(source):
842
+ shutil.copyfile(source, target)
843
+ elif os.path.isdir(source):
844
+ GeneralUtilities.ensure_directory_exists(target)
845
+ GeneralUtilities.copy_content_of_folder(source, target)
846
+ else:
847
+ raise ValueError(f"'{source}' can not be copied because the path does not exist.")
848
+ else:
849
+ exit_code, _, stderr, _ = self.run_program_argsasarray("sccopy", ["--source", source, "--target", target], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
850
+ if exit_code != 0:
851
+ raise ValueError(f"Fatal error occurrs while copying '{source}' to '{target}'. StdErr: '{stderr}'")
852
+
853
+ @GeneralUtilities.check_arguments
854
+ def create_file(self, path: str, error_if_already_exists: bool, create_necessary_folder: bool) -> None:
855
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
856
+ if self.program_runner.will_be_executed_locally():
857
+ if not os.path.isabs(path):
858
+ path = os.path.join(os.getcwd(), path)
859
+
860
+ if os.path.isfile(path) and error_if_already_exists:
861
+ raise ValueError(f"File '{path}' already exists.")
862
+
863
+ # TODO maybe it should be checked if there is a folder with the same path which already exists.
864
+
865
+ folder = os.path.dirname(path)
866
+
867
+ if not os.path.isdir(folder):
868
+ if create_necessary_folder:
869
+ GeneralUtilities.ensure_directory_exists(folder) # TODO check if this also create nested folders if required
870
+ else:
871
+ raise ValueError(f"Folder '{folder}' does not exist.")
872
+
873
+ GeneralUtilities.ensure_file_exists(path)
874
+ else:
875
+ arguments = ["--path", path]
876
+
877
+ if error_if_already_exists:
878
+ arguments = arguments+["--errorwhenexists"]
879
+
880
+ if create_necessary_folder:
881
+ arguments = arguments+["--createnecessaryfolder"]
882
+
883
+ exit_code, _, stderr, _ = self.run_program_argsasarray("sccreatefile", arguments, throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
884
+ if exit_code != 0:
885
+ raise ValueError(f"Fatal error occurrs while create file '{path}'. StdErr: '{stderr}'")
886
+
887
+ @GeneralUtilities.check_arguments
888
+ def create_folder(self, path: str, error_if_already_exists: bool, create_necessary_folder: bool) -> None:
889
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
890
+ if self.program_runner.will_be_executed_locally():
891
+ if not os.path.isabs(path):
892
+ path = os.path.join(os.getcwd(), path)
893
+
894
+ if os.path.isdir(path) and error_if_already_exists:
895
+ raise ValueError(f"Folder '{path}' already exists.")
896
+
897
+ # TODO maybe it should be checked if there is a file with the same path which already exists.
898
+
899
+ folder = os.path.dirname(path)
900
+
901
+ if not os.path.isdir(folder):
902
+ if create_necessary_folder:
903
+ GeneralUtilities.ensure_directory_exists(folder) # TODO check if this also create nested folders if required
904
+ else:
905
+ raise ValueError(f"Folder '{folder}' does not exist.")
906
+
907
+ GeneralUtilities.ensure_directory_exists(path)
908
+ else:
909
+ arguments = ["--path", path]
910
+
911
+ if error_if_already_exists:
912
+ arguments = arguments+["--errorwhenexists"]
913
+
914
+ if create_necessary_folder:
915
+ arguments = arguments+["--createnecessaryfolder"]
916
+
917
+ exit_code, _, stderr, _ = self.run_program_argsasarray("sccreatefolder", arguments, throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
918
+ if exit_code != 0:
919
+ raise ValueError(f"Fatal error occurrs while create folder '{path}'. StdErr: '{stderr}'")
920
+
921
+ @GeneralUtilities.check_arguments
922
+ def __sort_fmd(self, line: str):
923
+ splitted: list = line.split(";")
924
+ filetype: str = splitted[1]
925
+ if filetype == "d":
926
+ return -1
927
+ if filetype == "f":
928
+ return 1
929
+ return 0
930
+
931
+ @GeneralUtilities.check_arguments
932
+ def restore_filemetadata(self, folder: str, source_file: str, strict=False, encoding: str = "utf-8", create_folder_is_not_exist: bool = True) -> None:
933
+ lines = GeneralUtilities.read_lines_from_file(source_file, encoding)
934
+ lines.sort(key=self.__sort_fmd)
935
+ for line in lines:
936
+ splitted: list = line.split(";")
937
+ full_path_of_file_or_folder: str = os.path.join(folder, splitted[0])
938
+ filetype: str = splitted[1]
939
+ user: str = splitted[2]
940
+ permissions: str = splitted[3]
941
+ if filetype == "d" and create_folder_is_not_exist and not os.path.isdir(full_path_of_file_or_folder):
942
+ GeneralUtilities.ensure_directory_exists(full_path_of_file_or_folder)
943
+ if (filetype == "f" and os.path.isfile(full_path_of_file_or_folder)) or (filetype == "d" and os.path.isdir(full_path_of_file_or_folder)):
944
+ self.set_owner(full_path_of_file_or_folder, user, os.name != 'nt')
945
+ self.set_permission(full_path_of_file_or_folder, permissions)
946
+ else:
947
+ if strict:
948
+ if filetype == "f":
949
+ filetype_full = "File"
950
+ elif filetype == "d":
951
+ filetype_full = "Directory"
952
+ else:
953
+ raise ValueError(f"Unknown filetype: {GeneralUtilities.str_none_safe(filetype)}")
954
+ raise ValueError(f"{filetype_full} '{full_path_of_file_or_folder}' does not exist")
955
+
956
+ @GeneralUtilities.check_arguments
957
+ def __calculate_lengh_in_seconds(self, filename: str, folder: str) -> float:
958
+ argument = ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', filename]
959
+ result = self.run_program_argsasarray("ffprobe", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
960
+ return float(result[1].replace('\n', ''))
961
+
962
+ @GeneralUtilities.check_arguments
963
+ def __create_thumbnails(self, filename: str, fps: str, folder: str, tempname_for_thumbnails: str) -> list[str]:
964
+ argument = ['-i', filename, '-r', fps, '-vf', 'scale=-1:120', '-vcodec', 'png', f'{tempname_for_thumbnails}-%002d.png']
965
+ self.run_program_argsasarray("ffmpeg", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
966
+ files = GeneralUtilities.get_direct_files_of_folder(folder)
967
+ result: list[str] = []
968
+ regex = "^"+re.escape(tempname_for_thumbnails)+"\\-\\d+\\.png$"
969
+ regex_for_files = re.compile(regex)
970
+ for file in files:
971
+ filename = os.path.basename(file)
972
+ if regex_for_files.match(filename):
973
+ result.append(file)
974
+ GeneralUtilities.assert_condition(0 < len(result), "No thumbnail-files found.")
975
+ return result
976
+
977
+ @GeneralUtilities.check_arguments
978
+ def __create_thumbnail(self, outputfilename: str, folder: str, length_in_seconds: float, tempname_for_thumbnails: str, amount_of_images: int) -> None:
979
+ duration = timedelta(seconds=length_in_seconds)
980
+ info = GeneralUtilities.timedelta_to_simple_string(duration)
981
+ next_square_number = GeneralUtilities.get_next_square_number(amount_of_images)
982
+ root = math.sqrt(next_square_number)
983
+ rows: int = root # 5
984
+ columns: int = root # math.ceil(amount_of_images/rows)
985
+ argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{rows}x{columns}', f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
986
+ self.run_program_argsasarray("montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
987
+
988
+ @GeneralUtilities.check_arguments
989
+ def __create_thumbnail2(self, outputfilename: str, folder: str, length_in_seconds: float, rows: int, columns: int, tempname_for_thumbnails: str, amount_of_images: int) -> None:
990
+ duration = timedelta(seconds=length_in_seconds)
991
+ info = GeneralUtilities.timedelta_to_simple_string(duration)
992
+ argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{rows}x{columns}', f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
993
+ self.run_program_argsasarray("montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
994
+
995
+ @GeneralUtilities.check_arguments
996
+ def __roundup(self, x: float, places: int) -> int:
997
+ d = 10 ** places
998
+ if x < 0:
999
+ return math.floor(x * d) / d
1000
+ else:
1001
+ return math.ceil(x * d) / d
1002
+
1003
+ @GeneralUtilities.check_arguments
1004
+ def generate_thumbnail(self, file: str, frames_per_second: float, tempname_for_thumbnails: str = None, hook=None) -> None:
1005
+ if tempname_for_thumbnails is None:
1006
+ tempname_for_thumbnails = "t_"+str(uuid.uuid4())
1007
+
1008
+ file = GeneralUtilities.resolve_relative_path_from_current_working_directory(file)
1009
+ filename = os.path.basename(file)
1010
+ folder = os.path.dirname(file)
1011
+ filename_without_extension = Path(file).stem
1012
+ preview_files: list[str] = []
1013
+ try:
1014
+ length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
1015
+ # frames per second, example: frames_per_second="20fps" => 20 frames per second
1016
+ frames_per_second = self.__roundup(float(frames_per_second[:-3]), 2)
1017
+ frames_per_second_as_string = str(frames_per_second)
1018
+ preview_files = self.__create_thumbnails(filename, frames_per_second_as_string, folder, tempname_for_thumbnails)
1019
+ if hook is not None:
1020
+ hook(file, preview_files)
1021
+ actual_amounf_of_previewframes = len(preview_files)
1022
+ self.__create_thumbnail(filename_without_extension, folder, length_in_seconds, tempname_for_thumbnails, actual_amounf_of_previewframes)
1023
+ finally:
1024
+ for thumbnail_to_delete in preview_files:
1025
+ os.remove(thumbnail_to_delete)
1026
+
1027
+ @GeneralUtilities.check_arguments
1028
+ def generate_thumbnail_by_amount_of_pictures(self, file: str, amount_of_columns: int, amount_of_rows: int, tempname_for_thumbnails: str = None, hook=None) -> None:
1029
+ if tempname_for_thumbnails is None:
1030
+ tempname_for_thumbnails = "t_"+str(uuid.uuid4())
1031
+
1032
+ file = GeneralUtilities.resolve_relative_path_from_current_working_directory(file)
1033
+ filename = os.path.basename(file)
1034
+ folder = os.path.dirname(file)
1035
+ filename_without_extension = Path(file).stem
1036
+ preview_files: list[str] = []
1037
+ try:
1038
+ length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
1039
+ amounf_of_previewframes = int(amount_of_columns*amount_of_rows)
1040
+ frames_per_second_as_string = f"{amounf_of_previewframes-2}/{length_in_seconds}"
1041
+ preview_files = self.__create_thumbnails(filename, frames_per_second_as_string, folder, tempname_for_thumbnails)
1042
+ if hook is not None:
1043
+ hook(file, preview_files)
1044
+ actual_amounf_of_previewframes = len(preview_files)
1045
+ self.__create_thumbnail2(filename_without_extension, folder, length_in_seconds, amount_of_rows, amount_of_columns, tempname_for_thumbnails, actual_amounf_of_previewframes)
1046
+ finally:
1047
+ for thumbnail_to_delete in preview_files:
1048
+ os.remove(thumbnail_to_delete)
1049
+
1050
+ @GeneralUtilities.check_arguments
1051
+ def extract_pdf_pages(self, file: str, from_page: int, to_page: int, outputfile: str) -> None:
1052
+ pdf_reader: PdfReader = PdfReader(file)
1053
+ pdf_writer: PdfWriter = PdfWriter()
1054
+ start = from_page
1055
+ end = to_page
1056
+ while start <= end:
1057
+ pdf_writer.add_page(pdf_reader.pages[start-1])
1058
+ start += 1
1059
+ with open(outputfile, 'wb') as out:
1060
+ pdf_writer.write(out)
1061
+
1062
+ @GeneralUtilities.check_arguments
1063
+ def merge_pdf_files(self, files: list[str], outputfile: str) -> None:
1064
+ # TODO add wildcard-option
1065
+ pdfFileMerger: PdfWriter = PdfWriter()
1066
+ for file in files:
1067
+ with open(file, "rb") as f:
1068
+ pdfFileMerger.append(f)
1069
+ with open(outputfile, "wb") as output:
1070
+ pdfFileMerger.write(output)
1071
+ pdfFileMerger.close()
1072
+
1073
+ @GeneralUtilities.check_arguments
1074
+ def pdf_to_image(self, file: str, outputfilename_without_extension: str) -> None:
1075
+ raise ValueError("Function currently not available")
1076
+ # PyMuPDF can be used for that but sometimes it throws
1077
+ # "ImportError: DLL load failed while importing _fitz: Das angegebene Modul wurde nicht gefunden."
1078
+
1079
+ # doc = None # fitz.open(file)
1080
+ # for i, page in enumerate(doc):
1081
+ # pix = page.get_pixmap()
1082
+ # img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
1083
+ # img.save(f"{outputfilename_without_extension}_{i}.png", "PNG")
1084
+
1085
+ @GeneralUtilities.check_arguments
1086
+ def show_missing_files(self, folderA: str, folderB: str):
1087
+ for file in GeneralUtilities.get_missing_files(folderA, folderB):
1088
+ GeneralUtilities.write_message_to_stdout(file)
1089
+
1090
+ @GeneralUtilities.check_arguments
1091
+ def SCCreateEmptyFileWithSpecificSize(self, name: str, size_string: str) -> int:
1092
+ if size_string.isdigit():
1093
+ size = int(size_string)
1094
+ else:
1095
+ if len(size_string) >= 3:
1096
+ if (size_string.endswith("kb")):
1097
+ size = int(size_string[:-2]) * pow(10, 3)
1098
+ elif (size_string.endswith("mb")):
1099
+ size = int(size_string[:-2]) * pow(10, 6)
1100
+ elif (size_string.endswith("gb")):
1101
+ size = int(size_string[:-2]) * pow(10, 9)
1102
+ elif (size_string.endswith("kib")):
1103
+ size = int(size_string[:-3]) * pow(2, 10)
1104
+ elif (size_string.endswith("mib")):
1105
+ size = int(size_string[:-3]) * pow(2, 20)
1106
+ elif (size_string.endswith("gib")):
1107
+ size = int(size_string[:-3]) * pow(2, 30)
1108
+ else:
1109
+ self.log.log("Wrong format", LogLevel.Error)
1110
+ return 1
1111
+ else:
1112
+ self.log.log("Wrong format", LogLevel.Error)
1113
+ return 1
1114
+ with open(name, "wb") as f:
1115
+ f.seek(size-1)
1116
+ f.write(b"\0")
1117
+ return 0
1118
+
1119
+ @GeneralUtilities.check_arguments
1120
+ def SCCreateHashOfAllFiles(self, folder: str) -> None:
1121
+ for file in GeneralUtilities.absolute_file_paths(folder):
1122
+ with open(file+".sha256", "w+", encoding="utf-8") as f:
1123
+ f.write(GeneralUtilities.get_sha256_of_file(file))
1124
+
1125
+ @GeneralUtilities.check_arguments
1126
+ def SCCreateSimpleMergeWithoutRelease(self, repository: str, sourcebranch: str, targetbranch: str, remotename: str, remove_source_branch: bool) -> None:
1127
+ commitid = self.git_merge(repository, sourcebranch, targetbranch, False, True)
1128
+ self.git_merge(repository, targetbranch, sourcebranch, True, True)
1129
+ created_version = self.get_semver_version_from_gitversion(repository)
1130
+ self.git_create_tag(repository, commitid, f"v{created_version}", True)
1131
+ self.git_push(repository, remotename, targetbranch, targetbranch, False, True)
1132
+ if (GeneralUtilities.string_has_nonwhitespace_content(remotename)):
1133
+ self.git_push(repository, remotename, sourcebranch, sourcebranch, False, True)
1134
+ if (remove_source_branch):
1135
+ self.git_remove_branch(repository, sourcebranch)
1136
+
1137
+ @GeneralUtilities.check_arguments
1138
+ def sc_organize_lines_in_file(self, file: str, encoding: str, sort: bool = False, remove_duplicated_lines: bool = False, ignore_first_line: bool = False, remove_empty_lines: bool = True, ignored_start_character: list = list()) -> int:
1139
+ if os.path.isfile(file):
1140
+
1141
+ # read file
1142
+ lines = GeneralUtilities.read_lines_from_file(file, encoding)
1143
+ if (len(lines) == 0):
1144
+ return 0
1145
+
1146
+ # store first line if desiredpopd
1147
+
1148
+ if (ignore_first_line):
1149
+ first_line = lines.pop(0)
1150
+
1151
+ # remove empty lines if desired
1152
+ if remove_empty_lines:
1153
+ temp = lines
1154
+ lines = []
1155
+ for line in temp:
1156
+ if (not (GeneralUtilities.string_is_none_or_whitespace(line))):
1157
+ lines.append(line)
1158
+
1159
+ # remove duplicated lines if desired
1160
+ if remove_duplicated_lines:
1161
+ lines = GeneralUtilities.remove_duplicates(lines)
1162
+
1163
+ # sort lines if desired
1164
+ if sort:
1165
+ lines = sorted(lines, key=lambda singleline: self.__adapt_line_for_sorting(singleline, ignored_start_character))
1166
+
1167
+ # reinsert first line
1168
+ if ignore_first_line:
1169
+ lines.insert(0, first_line)
1170
+
1171
+ # write result to file
1172
+ GeneralUtilities.write_lines_to_file(file, lines, encoding)
1173
+
1174
+ return 0
1175
+ else:
1176
+ self.log.log(f"File '{file}' does not exist.", LogLevel.Error)
1177
+ return 1
1178
+
1179
+ @GeneralUtilities.check_arguments
1180
+ def __adapt_line_for_sorting(self, line: str, ignored_start_characters: list):
1181
+ result = line.lower()
1182
+ while len(result) > 0 and result[0] in ignored_start_characters:
1183
+ result = result[1:]
1184
+ return result
1185
+
1186
+ @GeneralUtilities.check_arguments
1187
+ def SCGenerateSnkFiles(self, outputfolder, keysize=4096, amountofkeys=10) -> int:
1188
+ GeneralUtilities.ensure_directory_exists(outputfolder)
1189
+ for _ in range(amountofkeys):
1190
+ file = os.path.join(outputfolder, str(uuid.uuid4())+".snk")
1191
+ argument = f"-k {keysize} {file}"
1192
+ self.run_program("sn", argument, outputfolder)
1193
+
1194
+ @GeneralUtilities.check_arguments
1195
+ def __merge_files(self, sourcefile: str, targetfile: str) -> None:
1196
+ with open(sourcefile, "rb") as f:
1197
+ source_data = f.read()
1198
+ with open(targetfile, "ab") as fout:
1199
+ merge_separator = [0x0A]
1200
+ fout.write(bytes(merge_separator))
1201
+ fout.write(source_data)
1202
+
1203
+ @GeneralUtilities.check_arguments
1204
+ def __process_file(self, file: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
1205
+ new_filename = os.path.join(os.path.dirname(file), os.path.basename(file).replace(substringInFilename, newSubstringInFilename))
1206
+ if file != new_filename:
1207
+ if os.path.isfile(new_filename):
1208
+ if filecmp.cmp(file, new_filename):
1209
+ send2trash.send2trash(file)
1210
+ else:
1211
+ if conflictResolveMode == "ignore":
1212
+ pass
1213
+ elif conflictResolveMode == "preservenewest":
1214
+ if (os.path.getmtime(file) - os.path.getmtime(new_filename) > 0):
1215
+ send2trash.send2trash(file)
1216
+ else:
1217
+ send2trash.send2trash(new_filename)
1218
+ os.rename(file, new_filename)
1219
+ elif (conflictResolveMode == "merge"):
1220
+ self.__merge_files(file, new_filename)
1221
+ send2trash.send2trash(file)
1222
+ else:
1223
+ raise ValueError('Unknown conflict resolve mode')
1224
+ else:
1225
+ os.rename(file, new_filename)
1226
+
1227
+ @GeneralUtilities.check_arguments
1228
+ def SCReplaceSubstringsInFilenames(self, folder: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
1229
+ for file in GeneralUtilities.absolute_file_paths(folder):
1230
+ self.__process_file(file, substringInFilename, newSubstringInFilename, conflictResolveMode)
1231
+
1232
+ @GeneralUtilities.check_arguments
1233
+ def __check_file(self, file: str, searchstring: str) -> None:
1234
+ bytes_ascii = bytes(searchstring, "ascii")
1235
+ # often called "unicode-encoding"
1236
+ bytes_utf16 = bytes(searchstring, "utf-16")
1237
+ bytes_utf8 = bytes(searchstring, "utf-8")
1238
+ with open(file, mode='rb') as file_object:
1239
+ content = file_object.read()
1240
+ if bytes_ascii in content:
1241
+ GeneralUtilities.write_message_to_stdout(file)
1242
+ elif bytes_utf16 in content:
1243
+ GeneralUtilities.write_message_to_stdout(file)
1244
+ elif bytes_utf8 in content:
1245
+ GeneralUtilities.write_message_to_stdout(file)
1246
+
1247
+ @GeneralUtilities.check_arguments
1248
+ def SCSearchInFiles(self, folder: str, searchstring: str) -> None:
1249
+ for file in GeneralUtilities.absolute_file_paths(folder):
1250
+ self.__check_file(file, searchstring)
1251
+
1252
+ @GeneralUtilities.check_arguments
1253
+ def get_string_as_qr_code(self,string: str) -> None:
1254
+ qr = qrcode.QRCode()
1255
+ qr.add_data(string)
1256
+ f = io.StringIO()
1257
+ qr.print_ascii(out=f)
1258
+ f.seek(0)
1259
+ return f.read()
1260
+
1261
+ @GeneralUtilities.check_arguments
1262
+ def __print_qr_code_by_csv_line(self, displayname: str, website: str, emailaddress: str, key: str, period: str) -> None:
1263
+ qrcode_content = f"otpauth://totp/{website}:{emailaddress}?secret={key}&issuer={displayname}&period={period}"
1264
+ GeneralUtilities.write_message_to_stdout(f"{displayname} ({emailaddress}):")
1265
+ GeneralUtilities.write_message_to_stdout(qrcode_content)
1266
+ qr = qrcode.QRCode()
1267
+ qr.add_data(qrcode_content)
1268
+ f = io.StringIO()
1269
+ qr.print_ascii(out=f)
1270
+ f.seek(0)
1271
+ GeneralUtilities.write_message_to_stdout(f.read())
1272
+
1273
+ @GeneralUtilities.check_arguments
1274
+ def SCShow2FAAsQRCode(self, csvfile: str) -> None:
1275
+ lines = GeneralUtilities.read_csv_file(csvfile, True)
1276
+ lines.sort(key=lambda items: ''.join(items).lower())
1277
+ for line in lines:
1278
+ self.__print_qr_code_by_csv_line(line[0], line[1], line[2], line[3], line[4])
1279
+ GeneralUtilities.write_message_to_stdout(GeneralUtilities.get_longline())
1280
+
1281
+ @GeneralUtilities.check_arguments
1282
+ def SCCalculateBitcoinBlockHash(self, block_version_number: str, previousblockhash: str, transactionsmerkleroot: str, timestamp: str, target: str, nonce: str) -> str:
1283
+ # Example-values:
1284
+ # block_version_number: "00000020"
1285
+ # previousblockhash: "66720b99e07d284bd4fe67ff8c49a5db1dd8514fcdab61000000000000000000"
1286
+ # transactionsmerkleroot: "7829844f4c3a41a537b3131ca992643eaa9d093b2383e4cdc060ad7dc5481187"
1287
+ # timestamp: "51eb505a"
1288
+ # target: "c1910018"
1289
+ # nonce: "de19b302"
1290
+ header = str(block_version_number + previousblockhash + transactionsmerkleroot + timestamp + target + nonce)
1291
+ return binascii.hexlify(hashlib.sha256(hashlib.sha256(binascii.unhexlify(header)).digest()).digest()[::-1]).decode('utf-8')
1292
+
1293
+ @GeneralUtilities.check_arguments
1294
+ def SCChangeHashOfProgram(self, inputfile: str) -> None:
1295
+ valuetoappend = str(uuid.uuid4())
1296
+
1297
+ outputfile = inputfile + '.modified'
1298
+
1299
+ shutil.copy2(inputfile, outputfile)
1300
+ with open(outputfile, 'a', encoding="utf-8") as file:
1301
+ # TODO use rcedit for .exe-files instead of appending valuetoappend ( https://github.com/electron/rcedit/ )
1302
+ # background: you can retrieve the "original-filename" from the .exe-file like discussed here:
1303
+ # https://security.stackexchange.com/questions/210843/ is-it-possible-to-change-original-filename-of-an-exe
1304
+ # so removing the original filename with rcedit is probably a better way to make it more difficult to detect the programname.
1305
+ # this would obviously also change the hashvalue of the program so appending a whitespace is not required anymore.
1306
+ file.write(valuetoappend)
1307
+
1308
+ @GeneralUtilities.check_arguments
1309
+ def __adjust_folder_name(self, folder: str) -> str:
1310
+ result = os.path.dirname(folder).replace("\\", "/")
1311
+ if result == "/":
1312
+ return GeneralUtilities.empty_string
1313
+ else:
1314
+ return result
1315
+
1316
+ @GeneralUtilities.check_arguments
1317
+ def __create_iso(self, folder, iso_file) -> None:
1318
+ created_directories = []
1319
+ files_directory = "FILES"
1320
+ iso = pycdlib.PyCdlib()
1321
+ iso.new()
1322
+ files_directory = files_directory.upper()
1323
+ iso.add_directory("/" + files_directory)
1324
+ created_directories.append("/" + files_directory)
1325
+ for root, _, files in os.walk(folder):
1326
+ for file in files:
1327
+ full_path = os.path.join(root, file)
1328
+ with (open(full_path, "rb").read()) as text_io_wrapper:
1329
+ content = text_io_wrapper
1330
+ path_in_iso = '/' + files_directory + \
1331
+ self.__adjust_folder_name(full_path[len(folder)::1]).upper()
1332
+ if path_in_iso not in created_directories:
1333
+ iso.add_directory(path_in_iso)
1334
+ created_directories.append(path_in_iso)
1335
+ iso.add_fp(BytesIO(content), len(content), path_in_iso + '/' + file.upper() + ';1')
1336
+ iso.write(iso_file)
1337
+ iso.close()
1338
+
1339
+ @GeneralUtilities.check_arguments
1340
+ def SCCreateISOFileWithObfuscatedFiles(self, inputfolder: str, outputfile: str, printtableheadline, createisofile, extensions) -> None:
1341
+ if (os.path.isdir(inputfolder)):
1342
+ namemappingfile = "name_map.csv"
1343
+ files_directory = inputfolder
1344
+ files_directory_obf = f"{files_directory}_Obfuscated"
1345
+ self.SCObfuscateFilesFolder(
1346
+ inputfolder, printtableheadline, namemappingfile, extensions)
1347
+ os.rename(namemappingfile, os.path.join(
1348
+ files_directory_obf, namemappingfile))
1349
+ if createisofile:
1350
+ self.__create_iso(files_directory_obf, outputfile)
1351
+ shutil.rmtree(files_directory_obf)
1352
+ else:
1353
+ raise ValueError(f"Directory not found: '{inputfolder}'")
1354
+
1355
+ @GeneralUtilities.check_arguments
1356
+ def SCFilenameObfuscator(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
1357
+ obfuscate_all_files = extensions == "*"
1358
+ if (obfuscate_all_files):
1359
+ obfuscate_file_extensions = None
1360
+ else:
1361
+ obfuscate_file_extensions = extensions.split(",")
1362
+ if (os.path.isdir(inputfolder)):
1363
+ printtableheadline = GeneralUtilities.string_to_boolean(
1364
+ printtableheadline)
1365
+ files = []
1366
+ if not os.path.isfile(namemappingfile):
1367
+ with open(namemappingfile, "a", encoding="utf-8"):
1368
+ pass
1369
+ if printtableheadline:
1370
+ GeneralUtilities.append_line_to_file(
1371
+ namemappingfile, "Original filename;new filename;SHA2-hash of file")
1372
+ for file in GeneralUtilities.absolute_file_paths(inputfolder):
1373
+ if os.path.isfile(os.path.join(inputfolder, file)):
1374
+ if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
1375
+ files.append(file)
1376
+ for file in files:
1377
+ hash_value = GeneralUtilities.get_sha256_of_file(file)
1378
+ extension = Path(file).suffix
1379
+ new_file_name_without_path = str(uuid.uuid4())[0:8] + extension
1380
+ new_file_name = os.path.join(
1381
+ os.path.dirname(file), new_file_name_without_path)
1382
+ os.rename(file, new_file_name)
1383
+ GeneralUtilities.append_line_to_file(namemappingfile, os.path.basename(file) + ";" + new_file_name_without_path + ";" + hash_value)
1384
+ else:
1385
+ raise ValueError(f"Directory not found: '{inputfolder}'")
1386
+
1387
+ @GeneralUtilities.check_arguments
1388
+ def __extension_matchs(self, file: str, obfuscate_file_extensions) -> bool:
1389
+ for extension in obfuscate_file_extensions:
1390
+ if file.lower().endswith("."+extension.lower()):
1391
+ return True
1392
+ return False
1393
+
1394
+ @GeneralUtilities.check_arguments
1395
+ def SCHealthcheck(self, file: str) -> int:
1396
+ lines = GeneralUtilities.read_lines_from_file(file)
1397
+ for line in reversed(lines):
1398
+ if not GeneralUtilities.string_is_none_or_whitespace(line):
1399
+ if "RunningHealthy (" in line: # TODO use regex
1400
+ GeneralUtilities.write_message_to_stderr(f"Healthy running due to line '{line}' in file '{file}'.")
1401
+ return 0
1402
+ else:
1403
+ GeneralUtilities.write_message_to_stderr(f"Not healthy running due to line '{line}' in file '{file}'.")
1404
+ return 1
1405
+ GeneralUtilities.write_message_to_stderr(f"No valid line found for healthycheck in file '{file}'.")
1406
+ return 2
1407
+
1408
+ @GeneralUtilities.check_arguments
1409
+ def SCObfuscateFilesFolder(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
1410
+ obfuscate_all_files = extensions == "*"
1411
+ if (obfuscate_all_files):
1412
+ obfuscate_file_extensions = None
1413
+ else:
1414
+ if "," in extensions:
1415
+ obfuscate_file_extensions = extensions.split(",")
1416
+ else:
1417
+ obfuscate_file_extensions = [extensions]
1418
+ newd = inputfolder+"_Obfuscated"
1419
+ shutil.copytree(inputfolder, newd)
1420
+ inputfolder = newd
1421
+ if (os.path.isdir(inputfolder)):
1422
+ for file in GeneralUtilities.absolute_file_paths(inputfolder):
1423
+ if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
1424
+ self.SCChangeHashOfProgram(file)
1425
+ os.remove(file)
1426
+ os.rename(file + ".modified", file)
1427
+ self.SCFilenameObfuscator(inputfolder, printtableheadline, namemappingfile, extensions)
1428
+ else:
1429
+ raise ValueError(f"Directory not found: '{inputfolder}'")
1430
+
1431
+ @GeneralUtilities.check_arguments
1432
+ def get_services_from_yaml_file(self, yaml_file: str) -> list[str]:
1433
+ with open(yaml_file, encoding="utf-8") as stream:
1434
+ loaded = yaml.safe_load(stream)
1435
+ services = loaded["services"]
1436
+ result = list(services.keys())
1437
+ return result
1438
+
1439
+ @GeneralUtilities.check_arguments
1440
+ def kill_docker_container(self, container_name: str) -> None:
1441
+ self.run_program("docker", f"container rm -f {container_name}")
1442
+
1443
+ @GeneralUtilities.check_arguments
1444
+ def get_docker_debian_version(self, image_tag: str) -> str:
1445
+ result = ScriptCollectionCore().run_program_argsasarray("docker", ['run', f'debian:{image_tag}', 'bash', '-c', 'apt-get -y update && apt-get -y install lsb-release && lsb_release -cs'])
1446
+ result_line = GeneralUtilities.string_to_lines(result[1])[-1]
1447
+ return result_line
1448
+
1449
+ @GeneralUtilities.check_arguments
1450
+ def get_latest_tor_version_of_debian_repository(self, debian_version: str) -> str:
1451
+ package_url: str = f"https://deb.torproject.org/torproject.org/dists/{debian_version}/main/binary-amd64/Packages"
1452
+ headers = {'Cache-Control': 'no-cache'}
1453
+ r = requests.get(package_url, timeout=5, headers=headers)
1454
+ if r.status_code != 200:
1455
+ raise ValueError(f"Checking for latest tor package resulted in HTTP-response-code {r.status_code}.")
1456
+ lines = GeneralUtilities.string_to_lines(GeneralUtilities.bytes_to_string(r.content))
1457
+ version_line_prefix = "Version: "
1458
+ version_content_line = [line for line in lines if line.startswith(version_line_prefix)][1]
1459
+ version_with_overhead = version_content_line[len(version_line_prefix):]
1460
+ tor_version = version_with_overhead.split("~")[0]
1461
+ return tor_version
1462
+
1463
+ def run_testcases_for_python_project(self, repository_folder: str):
1464
+ self.assert_is_git_repository(repository_folder)
1465
+ self.run_program("coverage", "run -m pytest", repository_folder)
1466
+ self.run_program("coverage", "xml", repository_folder)
1467
+ GeneralUtilities.ensure_directory_exists(os.path.join(repository_folder, "Other/TestCoverage"))
1468
+ coveragefile = os.path.join(repository_folder, "Other/TestCoverage/TestCoverage.xml")
1469
+ GeneralUtilities.ensure_file_does_not_exist(coveragefile)
1470
+ os.rename(os.path.join(repository_folder, "coverage.xml"), coveragefile)
1471
+
1472
+ @GeneralUtilities.check_arguments
1473
+ def get_file_permission(self, file: str) -> str:
1474
+ """This function returns an usual octet-triple, for example "700"."""
1475
+ ls_output: str = self.run_ls_for_folder(file)
1476
+ return self.__get_file_permission_helper(ls_output)
1477
+
1478
+ @GeneralUtilities.check_arguments
1479
+ def __get_file_permission_helper(self, permissions: str) -> str:
1480
+ return str(self.__to_octet(permissions[0:3])) + str(self.__to_octet(permissions[3:6]))+str(self.__to_octet(permissions[6:9]))
1481
+
1482
+ @GeneralUtilities.check_arguments
1483
+ def __to_octet(self, string: str) -> int:
1484
+ return int(self.__to_octet_helper(string[0]) + self.__to_octet_helper(string[1])+self.__to_octet_helper(string[2]), 2)
1485
+
1486
+ @GeneralUtilities.check_arguments
1487
+ def __to_octet_helper(self, string: str) -> str:
1488
+ if (string == "-"):
1489
+ return "0"
1490
+ else:
1491
+ return "1"
1492
+
1493
+ @GeneralUtilities.check_arguments
1494
+ def get_file_owner(self, file: str) -> str:
1495
+ """This function returns the user and the group in the format "user:group"."""
1496
+ ls_output: str = self.run_ls_for_folder(file)
1497
+ return self.__get_file_owner_helper(ls_output)
1498
+
1499
+ @GeneralUtilities.check_arguments
1500
+ def __get_file_owner_helper(self, ls_output: str) -> str:
1501
+ splitted = ls_output.split()
1502
+ return f"{splitted[2]}:{splitted[3]}"
1503
+
1504
+ @GeneralUtilities.check_arguments
1505
+ def get_file_owner_and_file_permission(self, file: str) -> str:
1506
+ ls_output: str = self.run_ls_for_folder(file)
1507
+ return [self.__get_file_owner_helper(ls_output), self.__get_file_permission_helper(ls_output)]
1508
+
1509
+ @GeneralUtilities.check_arguments
1510
+ def run_ls_for_folder(self, file_or_folder: str) -> str:
1511
+ file_or_folder = file_or_folder.replace("\\", "/")
1512
+ GeneralUtilities.assert_condition(os.path.isfile(file_or_folder) or os.path.isdir(file_or_folder), f"Can not execute 'ls -ld' because '{file_or_folder}' does not exist.")
1513
+ ls_result = self.run_program_argsasarray("ls", ["-ld", file_or_folder])
1514
+ GeneralUtilities.assert_condition(ls_result[0] == 0, f"'ls -ld {file_or_folder}' resulted in exitcode {str(ls_result[0])}. StdErr: {ls_result[2]}")
1515
+ GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(ls_result[1]), f"'ls -ld' of '{file_or_folder}' had an empty output. StdErr: '{ls_result[2]}'")
1516
+ output = ls_result[1]
1517
+ result = output.replace("\n", GeneralUtilities.empty_string)
1518
+ result = ' '.join(result.split()) # reduce multiple whitespaces to one
1519
+ return result
1520
+
1521
+ @GeneralUtilities.check_arguments
1522
+ def run_ls_for_folder_content(self, file_or_folder: str) -> list[str]:
1523
+ file_or_folder = file_or_folder.replace("\\", "/")
1524
+ GeneralUtilities.assert_condition(os.path.isfile(file_or_folder) or os.path.isdir(file_or_folder), f"Can not execute 'ls -la' because '{file_or_folder}' does not exist.")
1525
+ ls_result = self.run_program_argsasarray("ls", ["-la", file_or_folder])
1526
+ GeneralUtilities.assert_condition(ls_result[0] == 0, f"'ls -la {file_or_folder}' resulted in exitcode {str(ls_result[0])}. StdErr: {ls_result[2]}")
1527
+ GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(ls_result[1]), f"'ls -la' of '{file_or_folder}' had an empty output. StdErr: '{ls_result[2]}'")
1528
+ output = ls_result[1]
1529
+ result = output.split("\n")[3:] # skip the lines with "Total", "." and ".."
1530
+ result = [' '.join(line.split()) for line in result] # reduce multiple whitespaces to one
1531
+ return result
1532
+
1533
+ @GeneralUtilities.check_arguments
1534
+ def set_permission(self, file_or_folder: str, permissions: str, recursive: bool = False) -> None:
1535
+ """This function expects an usual octet-triple, for example "700"."""
1536
+ args = []
1537
+ if recursive:
1538
+ args.append("--recursive")
1539
+ args.append(permissions)
1540
+ args.append(file_or_folder)
1541
+ self.run_program_argsasarray("chmod", args)
1542
+
1543
+ @GeneralUtilities.check_arguments
1544
+ def set_owner(self, file_or_folder: str, owner: str, recursive: bool = False, follow_symlinks: bool = False) -> None:
1545
+ """This function expects the user and the group in the format "user:group"."""
1546
+ args = []
1547
+ if recursive:
1548
+ args.append("--recursive")
1549
+ if follow_symlinks:
1550
+ args.append("--no-dereference")
1551
+ args.append(owner)
1552
+ args.append(file_or_folder)
1553
+ self.run_program_argsasarray("chown", args)
1554
+
1555
+ # <run programs>
1556
+
1557
+ @GeneralUtilities.check_arguments
1558
+ def __run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> Popen:
1559
+ popen: Popen = self.program_runner.run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, custom_argument, interactive)
1560
+ return popen
1561
+
1562
+ @staticmethod
1563
+ def __enqueue_output(file: IO, queue: Queue):
1564
+ for line in iter(file.readline, ''):
1565
+ queue.put(line)
1566
+ file.close()
1567
+
1568
+ @staticmethod
1569
+ def __continue_process_reading(pid: int, p: Popen, q_stdout: Queue, q_stderr: Queue, reading_stdout_last_time_resulted_in_exception: bool, reading_stderr_last_time_resulted_in_exception: bool):
1570
+ if p.poll() is None:
1571
+ return True
1572
+
1573
+ # if reading_stdout_last_time_resulted_in_exception and reading_stderr_last_time_resulted_in_exception:
1574
+ # return False
1575
+
1576
+ if not q_stdout.empty():
1577
+ return True
1578
+
1579
+ if not q_stderr.empty():
1580
+ return True
1581
+
1582
+ return False
1583
+
1584
+ @staticmethod
1585
+ def __read_popen_pipes(p: Popen, print_live_output: bool, print_errors_as_information: bool, log: SCLog) -> tuple[list[str], list[str]]:
1586
+ p_id = p.pid
1587
+ with ThreadPoolExecutor(2) as pool:
1588
+ q_stdout = Queue()
1589
+ q_stderr = Queue()
1590
+
1591
+ pool.submit(ScriptCollectionCore.__enqueue_output, p.stdout, q_stdout)
1592
+ pool.submit(ScriptCollectionCore.__enqueue_output, p.stderr, q_stderr)
1593
+ reading_stdout_last_time_resulted_in_exception: bool = False
1594
+ reading_stderr_last_time_resulted_in_exception: bool = False
1595
+
1596
+ stdout_result: list[str] = []
1597
+ stderr_result: list[str] = []
1598
+
1599
+ while (ScriptCollectionCore.__continue_process_reading(p_id, p, q_stdout, q_stderr, reading_stdout_last_time_resulted_in_exception, reading_stderr_last_time_resulted_in_exception)):
1600
+ try:
1601
+ while not q_stdout.empty():
1602
+ out_line: str = q_stdout.get_nowait()
1603
+ out_line = out_line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
1604
+ if GeneralUtilities.string_has_content(out_line):
1605
+ stdout_result.append(out_line)
1606
+ reading_stdout_last_time_resulted_in_exception = False
1607
+ if print_live_output:
1608
+ loglevel = LogLevel.Information
1609
+ if out_line.startswith("Debug: "):
1610
+ loglevel = LogLevel.Debug
1611
+ out_line = out_line[len("Debug: "):]
1612
+ if out_line.startswith("Diagnostic: "):
1613
+ loglevel = LogLevel.Diagnostic
1614
+ out_line = out_line[len("Diagnostic: "):]
1615
+ log.log(out_line, loglevel)
1616
+ except Empty:
1617
+ reading_stdout_last_time_resulted_in_exception = True
1618
+
1619
+ try:
1620
+ while not q_stderr.empty():
1621
+ err_line: str = q_stderr.get_nowait()
1622
+ err_line = err_line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
1623
+ if GeneralUtilities.string_has_content(err_line):
1624
+ stderr_result.append(err_line)
1625
+ reading_stderr_last_time_resulted_in_exception = False
1626
+ if print_live_output:
1627
+ loglevel = LogLevel.Error
1628
+ if err_line.startswith("Warning: "):
1629
+ loglevel = LogLevel.Warning
1630
+ err_line = err_line[len("Warning: "):]
1631
+ if print_errors_as_information: # "errors" in "print_errors_as_information" means: all what is written to std-err
1632
+ loglevel = LogLevel.Information
1633
+ log.log(err_line, loglevel)
1634
+ except Empty:
1635
+ reading_stderr_last_time_resulted_in_exception = True
1636
+
1637
+ time.sleep(0.01) # this is required to not finish too early
1638
+
1639
+ return (stdout_result, stderr_result)
1640
+
1641
+ @GeneralUtilities.check_arguments
1642
+ def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False) -> tuple[int, str, str, int]:
1643
+ if self.call_program_runner_directly:
1644
+ return self.program_runner.run_program_argsasarray(program, arguments_as_array, working_directory, custom_argument, interactive)
1645
+ try:
1646
+ arguments_as_str = ' '.join(arguments_as_array)
1647
+ mock_loader_result = self.__try_load_mock(program, arguments_as_str, working_directory)
1648
+ if mock_loader_result[0]:
1649
+ return mock_loader_result[1]
1650
+
1651
+ working_directory = self.__adapt_workingdirectory(working_directory)
1652
+
1653
+ if arguments_for_log is None:
1654
+ arguments_for_log = arguments_as_array
1655
+
1656
+ cmd = f'{working_directory}>{program}'
1657
+ if 0 < len(arguments_for_log):
1658
+ arguments_for_log_as_string: str = ' '.join([f'"{argument_for_log}"' for argument_for_log in arguments_for_log])
1659
+ cmd = f'{cmd} {arguments_for_log_as_string}'
1660
+
1661
+ if GeneralUtilities.string_is_none_or_whitespace(title):
1662
+ info_for_log = cmd
1663
+ else:
1664
+ info_for_log = title
1665
+
1666
+ self.log.log(f"Run '{info_for_log}'.", LogLevel.Debug)
1667
+
1668
+ exit_code: int = None
1669
+ stdout: str = GeneralUtilities.empty_string
1670
+ stderr: str = GeneralUtilities.empty_string
1671
+ pid: int = None
1672
+
1673
+ with self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive) as process:
1674
+
1675
+ if log_file is not None:
1676
+ GeneralUtilities.ensure_file_exists(log_file)
1677
+ pid = process.pid
1678
+
1679
+ outputs: tuple[list[str], list[str]] = ScriptCollectionCore.__read_popen_pipes(process, print_live_output, print_errors_as_information, self.log)
1680
+
1681
+ for out_line_plain in outputs[0]:
1682
+ if out_line_plain is not None:
1683
+ out_line: str = None
1684
+ if isinstance(out_line_plain, str):
1685
+ out_line = out_line_plain
1686
+ elif isinstance(out_line_plain, bytes):
1687
+ out_line = GeneralUtilities.bytes_to_string(out_line_plain)
1688
+ else:
1689
+ raise ValueError(f"Unknown type of output: {str(type(out_line_plain))}")
1690
+
1691
+ if out_line is not None and GeneralUtilities.string_has_content(out_line):
1692
+ if out_line.endswith("\n"):
1693
+ out_line = out_line[:-1]
1694
+ if 0 < len(stdout):
1695
+ stdout = stdout+"\n"
1696
+ stdout = stdout+out_line
1697
+ if log_file is not None:
1698
+ GeneralUtilities.append_line_to_file(log_file, out_line)
1699
+
1700
+ for err_line_plain in outputs[1]:
1701
+ if err_line_plain is not None:
1702
+ err_line: str = None
1703
+ if isinstance(err_line_plain, str):
1704
+ err_line = err_line_plain
1705
+ elif isinstance(err_line_plain, bytes):
1706
+ err_line = GeneralUtilities.bytes_to_string(err_line_plain)
1707
+ else:
1708
+ raise ValueError(f"Unknown type of output: {str(type(err_line_plain))}")
1709
+ if err_line is not None and GeneralUtilities.string_has_content(err_line):
1710
+ if err_line.endswith("\n"):
1711
+ err_line = err_line[:-1]
1712
+ if 0 < len(stderr):
1713
+ stderr = stderr+"\n"
1714
+ stderr = stderr+err_line
1715
+ if log_file is not None:
1716
+ GeneralUtilities.append_line_to_file(log_file, err_line)
1717
+
1718
+ exit_code = process.returncode
1719
+ GeneralUtilities.assert_condition(exit_code is not None, f"Exitcode of program-run of '{info_for_log}' is None.")
1720
+
1721
+ result_message = f"Program '{info_for_log}' resulted in exitcode {exit_code}."
1722
+
1723
+ self.log.log(result_message, LogLevel.Debug)
1724
+
1725
+ if throw_exception_if_exitcode_is_not_zero and exit_code != 0:
1726
+ raise ValueError(f"{result_message} (StdOut: '{stdout}', StdErr: '{stderr}')")
1727
+
1728
+ result = (exit_code, stdout, stderr, pid)
1729
+ return result
1730
+ except Exception as e:#pylint:disable=unused-variable, try-except-raise
1731
+ raise
1732
+
1733
+ # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
1734
+ @GeneralUtilities.check_arguments
1735
+ def run_program_with_retry(self, program: str, arguments: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False, amount_of_attempts: int = 5) -> tuple[int, str, str, int]:
1736
+ return GeneralUtilities.retry_action(lambda: self.run_program(program, arguments, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output), amount_of_attempts)
1737
+
1738
+ # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
1739
+ @GeneralUtilities.check_arguments
1740
+ def run_program(self, program: str, arguments: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False) -> tuple[int, str, str, int]:
1741
+ if self.call_program_runner_directly:
1742
+ return self.program_runner.run_program(program, arguments, working_directory, custom_argument, interactive)
1743
+ return self.run_program_argsasarray(program, GeneralUtilities.arguments_to_array(arguments), working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output)
1744
+
1745
+ # Return-values program_runner: Pid
1746
+ @GeneralUtilities.check_arguments
1747
+ def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
1748
+ if self.call_program_runner_directly:
1749
+ return self.program_runner.run_program_argsasarray_async(program, arguments_as_array, working_directory, custom_argument, interactive)
1750
+ mock_loader_result = self.__try_load_mock(program, ' '.join(arguments_as_array), working_directory)
1751
+ if mock_loader_result[0]:
1752
+ return mock_loader_result[1]
1753
+ process: Popen = self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
1754
+ return process.pid
1755
+
1756
+ # Return-values program_runner: Pid
1757
+ @GeneralUtilities.check_arguments
1758
+ def run_program_async(self, program: str, arguments: str = "", working_directory: str = None,print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
1759
+ if self.call_program_runner_directly:
1760
+ return self.program_runner.run_program_argsasarray_async(program, arguments, working_directory, custom_argument, interactive)
1761
+ return self.run_program_argsasarray_async(program, GeneralUtilities.arguments_to_array(arguments), working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
1762
+
1763
+ @GeneralUtilities.check_arguments
1764
+ def __try_load_mock(self, program: str, arguments: str, working_directory: str) -> tuple[bool, tuple[int, str, str, int]]:
1765
+ if self.mock_program_calls:
1766
+ try:
1767
+ return [True, self.__get_mock_program_call(program, arguments, working_directory)]
1768
+ except LookupError:
1769
+ if not self.execute_program_really_if_no_mock_call_is_defined:
1770
+ raise
1771
+ return [False, None]
1772
+
1773
+ @GeneralUtilities.check_arguments
1774
+ def __adapt_workingdirectory(self, workingdirectory: str) -> str:
1775
+ result: str = None
1776
+ if workingdirectory is None:
1777
+ result = os.getcwd()
1778
+ else:
1779
+ if os.path.isabs(workingdirectory):
1780
+ result = workingdirectory
1781
+ else:
1782
+ result = GeneralUtilities.resolve_relative_path_from_current_working_directory(workingdirectory)
1783
+ if not os.path.isdir(result):
1784
+ raise ValueError(f"Working-directory '{workingdirectory}' does not exist.")
1785
+ return result
1786
+
1787
+ @GeneralUtilities.check_arguments
1788
+ def verify_no_pending_mock_program_calls(self):
1789
+ if (len(self.__mocked_program_calls) > 0):
1790
+ raise AssertionError("The following mock-calls were not called:\n"+",\n ".join([self.__format_mock_program_call(r) for r in self.__mocked_program_calls]))
1791
+
1792
+ @GeneralUtilities.check_arguments
1793
+ def __format_mock_program_call(self, r) -> str:
1794
+ r: ScriptCollectionCore.__MockProgramCall = r
1795
+ return f"'{r.workingdirectory}>{r.program} {r.argument}' (" \
1796
+ f"exitcode: {GeneralUtilities.str_none_safe(str(r.exit_code))}, " \
1797
+ f"pid: {GeneralUtilities.str_none_safe(str(r.pid))}, "\
1798
+ f"stdout: {GeneralUtilities.str_none_safe(str(r.stdout))}, " \
1799
+ f"stderr: {GeneralUtilities.str_none_safe(str(r.stderr))})"
1800
+
1801
+ @GeneralUtilities.check_arguments
1802
+ def register_mock_program_call(self, program: str, argument: str, workingdirectory: str, result_exit_code: int, result_stdout: str, result_stderr: str, result_pid: int, amount_of_expected_calls=1):
1803
+ "This function is for test-purposes only"
1804
+ for _ in itertools.repeat(None, amount_of_expected_calls):
1805
+ mock_call = ScriptCollectionCore.__MockProgramCall()
1806
+ mock_call.program = program
1807
+ mock_call.argument = argument
1808
+ mock_call.workingdirectory = workingdirectory
1809
+ mock_call.exit_code = result_exit_code
1810
+ mock_call.stdout = result_stdout
1811
+ mock_call.stderr = result_stderr
1812
+ mock_call.pid = result_pid
1813
+ self.__mocked_program_calls.append(mock_call)
1814
+
1815
+ @GeneralUtilities.check_arguments
1816
+ def __get_mock_program_call(self, program: str, argument: str, workingdirectory: str):
1817
+ result: ScriptCollectionCore.__MockProgramCall = None
1818
+ for mock_call in self.__mocked_program_calls:
1819
+ if ((re.match(mock_call.program, program) is not None)
1820
+ and (re.match(mock_call.argument, argument) is not None)
1821
+ and (re.match(mock_call.workingdirectory, workingdirectory) is not None)):
1822
+ result = mock_call
1823
+ break
1824
+ if result is None:
1825
+ raise LookupError(f"Tried to execute mock-call '{workingdirectory}>{program} {argument}' but no mock-call was defined for that execution")
1826
+ else:
1827
+ self.__mocked_program_calls.remove(result)
1828
+ return (result.exit_code, result.stdout, result.stderr, result.pid)
1829
+
1830
+ @GeneralUtilities.check_arguments
1831
+ class __MockProgramCall:
1832
+ program: str
1833
+ argument: str
1834
+ workingdirectory: str
1835
+ exit_code: int
1836
+ stdout: str
1837
+ stderr: str
1838
+ pid: int
1839
+
1840
+ @GeneralUtilities.check_arguments
1841
+ def run_with_epew(self, program: str, argument: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False,print_live_output:bool=False,encode_argument_in_base64:bool=False) -> tuple[int, str, str, int]:
1842
+ epew_argument=["-p",program ,"-w", working_directory]
1843
+ if encode_argument_in_base64:
1844
+ if arguments_for_log is None:
1845
+ arguments_for_log=epew_argument+["-a",f"\"{argument}\""]
1846
+ base64_string = base64.b64encode(argument.encode("utf-8")).decode("utf-8")
1847
+ epew_argument=epew_argument+["-a",base64_string,"-b"]
1848
+ else:
1849
+ epew_argument=epew_argument+["-a",argument]
1850
+ if arguments_for_log is None:
1851
+ arguments_for_log=epew_argument
1852
+ return self.run_program_argsasarray("epew", epew_argument, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive,print_live_output=print_live_output)
1853
+
1854
+
1855
+ # </run programs>
1856
+
1857
+ @GeneralUtilities.check_arguments
1858
+ def extract_archive_with_7z(self, unzip_program_file: str, zip_file: str, password: str, output_directory: str) -> None:
1859
+ password_set = not password is None
1860
+ file_name = Path(zip_file).name
1861
+ file_folder = os.path.dirname(zip_file)
1862
+ argument = "x"
1863
+ if password_set:
1864
+ argument = f"{argument} -p\"{password}\""
1865
+ argument = f"{argument} -o {output_directory}"
1866
+ argument = f"{argument} {file_name}"
1867
+ return self.run_program(unzip_program_file, argument, file_folder)
1868
+
1869
+ @GeneralUtilities.check_arguments
1870
+ def get_internet_time(self) -> datetime:
1871
+ response = ntplib.NTPClient().request('pool.ntp.org')
1872
+ return datetime.fromtimestamp(response.tx_time)
1873
+
1874
+ @GeneralUtilities.check_arguments
1875
+ def system_time_equals_internet_time(self, maximal_tolerance_difference: timedelta) -> bool:
1876
+ return abs(GeneralUtilities.get_now() - self.get_internet_time()) < maximal_tolerance_difference
1877
+
1878
+ @GeneralUtilities.check_arguments
1879
+ def system_time_equals_internet_time_with_default_tolerance(self) -> bool:
1880
+ return self.system_time_equals_internet_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
1881
+
1882
+ @GeneralUtilities.check_arguments
1883
+ def check_system_time(self, maximal_tolerance_difference: timedelta):
1884
+ if not self.system_time_equals_internet_time(maximal_tolerance_difference):
1885
+ raise ValueError("System time may be wrong")
1886
+
1887
+ @GeneralUtilities.check_arguments
1888
+ def check_system_time_with_default_tolerance(self) -> None:
1889
+ self.check_system_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
1890
+
1891
+ @GeneralUtilities.check_arguments
1892
+ def __get_default_tolerance_for_system_time_equals_internet_time(self) -> timedelta:
1893
+ return timedelta(hours=0, minutes=0, seconds=3)
1894
+
1895
+ @GeneralUtilities.check_arguments
1896
+ def increment_version(self, input_version: str, increment_major: bool, increment_minor: bool, increment_patch: bool) -> str:
1897
+ splitted = input_version.split(".")
1898
+ GeneralUtilities.assert_condition(len(splitted) == 3, f"Version '{input_version}' does not have the 'major.minor.patch'-pattern.")
1899
+ major = int(splitted[0])
1900
+ minor = int(splitted[1])
1901
+ patch = int(splitted[2])
1902
+ if increment_major:
1903
+ major = major+1
1904
+ if increment_minor:
1905
+ minor = minor+1
1906
+ if increment_patch:
1907
+ patch = patch+1
1908
+ return f"{major}.{minor}.{patch}"
1909
+
1910
+ @GeneralUtilities.check_arguments
1911
+ def get_semver_version_from_gitversion(self, repository_folder: str) -> str:
1912
+ self.assert_is_git_repository(repository_folder)
1913
+ if (self.git_repository_has_commits(repository_folder)):
1914
+ result = self.get_version_from_gitversion(repository_folder, "MajorMinorPatch")
1915
+ if self.git_repository_has_uncommitted_changes(repository_folder):
1916
+ if self.get_current_git_branch_has_tag(repository_folder):
1917
+ id_of_latest_tag = self.git_get_commitid_of_tag(repository_folder, self.get_latest_git_tag(repository_folder))
1918
+ current_commit = self.git_get_commit_id(repository_folder)
1919
+ current_commit_is_on_latest_tag = id_of_latest_tag == current_commit
1920
+ if current_commit_is_on_latest_tag:
1921
+ result = self.increment_version(result, False, False, True)
1922
+ else:
1923
+ result = "0.1.0"
1924
+ return result
1925
+
1926
+ @staticmethod
1927
+ @GeneralUtilities.check_arguments
1928
+ def is_patch_version(version_string: str) -> bool:
1929
+ return not version_string.endswith(".0")
1930
+
1931
+ @GeneralUtilities.check_arguments
1932
+ def get_version_from_gitversion(self, folder: str, variable: str) -> str:
1933
+ # called twice as workaround for issue 1877 in gitversion ( https://github.com/GitTools/GitVersion/issues/1877 )
1934
+ result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder)
1935
+ result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder)
1936
+ result = GeneralUtilities.strip_new_line_character(result[1])
1937
+
1938
+ return result
1939
+
1940
+ @GeneralUtilities.check_arguments
1941
+ def generate_certificate_authority(self, folder: str, name: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
1942
+ if days_until_expire is None:
1943
+ days_until_expire = 1825
1944
+ if password is None:
1945
+ password = GeneralUtilities.generate_password()
1946
+ GeneralUtilities.ensure_directory_exists(folder)
1947
+ self.run_program_argsasarray("openssl", ['req', '-new', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:prime256v1', '-days', str(days_until_expire), '-nodes', '-x509', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={name}/OU={subj_ou}', '-passout', f'pass:{password}', '-keyout', f'{name}.key', '-out', f'{name}.crt'], folder)
1948
+
1949
+ @GeneralUtilities.check_arguments
1950
+ def generate_certificate(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
1951
+ if days_until_expire is None:
1952
+ days_until_expire = 397
1953
+ if password is None:
1954
+ password = GeneralUtilities.generate_password()
1955
+ rsa_key_length = 4096
1956
+ self.run_program_argsasarray("openssl", ['genrsa', '-out', f'{filename}.key', f'{rsa_key_length}'], folder)
1957
+ self.run_program_argsasarray("openssl", ['req', '-new', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou}', '-x509', '-key', f'{filename}.key', '-out', f'{filename}.unsigned.crt', '-days', f'{days_until_expire}'], folder)
1958
+ self.run_program_argsasarray("openssl", ['pkcs12', '-export', '-out', f'{filename}.selfsigned.pfx', '-password', f'pass:{password}', '-inkey', f'{filename}.key', '-in', f'{filename}.unsigned.crt'], folder)
1959
+ GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.password"), password)
1960
+ GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.san.conf"), f"""[ req ]
1961
+ default_bits = {rsa_key_length}
1962
+ distinguished_name = req_distinguished_name
1963
+ req_extensions = v3_req
1964
+ default_md = sha256
1965
+ dirstring_type = nombstr
1966
+ prompt = no
1967
+
1968
+ [ req_distinguished_name ]
1969
+ countryName = {subj_c}
1970
+ stateOrProvinceName = {subj_st}
1971
+ localityName = {subj_l}
1972
+ organizationName = {subj_o}
1973
+ organizationUnit = {subj_ou}
1974
+ commonName = {domain}
1975
+
1976
+ [v3_req]
1977
+ subjectAltName = @subject_alt_name
1978
+
1979
+ [ subject_alt_name ]
1980
+ DNS = {domain}
1981
+ """)
1982
+
1983
+ @GeneralUtilities.check_arguments
1984
+ def generate_certificate_sign_request(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str) -> None:
1985
+ self.run_program_argsasarray("openssl", ['req', '-new', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou}', '-key', f'{filename}.key', f'-out', f'{filename}.csr', f'-config', f'{filename}.san.conf'], folder)
1986
+
1987
+ @GeneralUtilities.check_arguments
1988
+ def sign_certificate(self, folder: str, ca_folder: str, ca_name: str, domain: str, filename: str, days_until_expire: int = None) -> None:
1989
+ if days_until_expire is None:
1990
+ days_until_expire = 397
1991
+ ca = os.path.join(ca_folder, ca_name)
1992
+ password_file = os.path.join(folder, f"{filename}.password")
1993
+ password = GeneralUtilities.read_text_from_file(password_file)
1994
+ self.run_program_argsasarray("openssl", ['x509', '-req', '-in', f'{filename}.csr', '-CA', f'{ca}.crt', '-CAkey', f'{ca}.key', '-CAcreateserial', '-CAserial', f'{ca}.srl', '-out', f'{filename}.crt', '-days', str(days_until_expire), '-sha256', '-extensions', 'v3_req', '-extfile', f'{filename}.san.conf'], folder)
1995
+ self.run_program_argsasarray("openssl", ['pkcs12', '-export', '-out', f'{filename}.pfx', f'-inkey', f'{filename}.key', '-in', f'{filename}.crt', '-password', f'pass:{password}'], folder)
1996
+
1997
+ @GeneralUtilities.check_arguments
1998
+ def update_dependencies_of_python_in_requirementstxt_file(self, file: str, ignored_dependencies: list[str]):
1999
+ # TODO consider ignored_dependencies
2000
+ lines = GeneralUtilities.read_lines_from_file(file)
2001
+ new_lines = []
2002
+ for line in lines:
2003
+ if GeneralUtilities.string_has_content(line):
2004
+ new_lines.append(self.__get_updated_line_for_python_requirements(line.strip()))
2005
+ GeneralUtilities.write_lines_to_file(file, new_lines)
2006
+
2007
+ @GeneralUtilities.check_arguments
2008
+ def __get_updated_line_for_python_requirements(self, line: str) -> str:
2009
+ if "==" in line or "<" in line:
2010
+ return line
2011
+ elif ">" in line:
2012
+ try:
2013
+ # line is something like "cyclonedx-bom>=2.0.2" and the function must return with the updated version
2014
+ # (something like "cyclonedx-bom>=2.11.0" for example)
2015
+ package = line.split(">")[0]
2016
+ operator = ">=" if ">=" in line else ">"
2017
+ headers = {'Cache-Control': 'no-cache'}
2018
+ response = requests.get(f'https://pypi.org/pypi/{package}/json', timeout=5, headers=headers)
2019
+ latest_version = response.json()['info']['version']
2020
+ # TODO update only minor- and patch-version
2021
+ # TODO print info if there is a new major-version
2022
+ return package+operator+latest_version
2023
+ except:
2024
+ return line
2025
+ else:
2026
+ raise ValueError(f'Unexpected line in requirements-file: "{line}"')
2027
+
2028
+ @GeneralUtilities.check_arguments
2029
+ def update_dependencies_of_python_in_setupcfg_file(self, setup_cfg_file: str, ignored_dependencies: list[str]):
2030
+ # TODO consider ignored_dependencies
2031
+ lines = GeneralUtilities.read_lines_from_file(setup_cfg_file)
2032
+ new_lines = []
2033
+ requirement_parsing_mode = False
2034
+ for line in lines:
2035
+ new_line = line
2036
+ if (requirement_parsing_mode):
2037
+ if ("<" in line or "=" in line or ">" in line):
2038
+ updated_line = f" {self.__get_updated_line_for_python_requirements(line.strip())}"
2039
+ new_line = updated_line
2040
+ else:
2041
+ requirement_parsing_mode = False
2042
+ else:
2043
+ if line.startswith("install_requires ="):
2044
+ requirement_parsing_mode = True
2045
+ new_lines.append(new_line)
2046
+ GeneralUtilities.write_lines_to_file(setup_cfg_file, new_lines)
2047
+
2048
+ @GeneralUtilities.check_arguments
2049
+ def update_dependencies_of_dotnet_project(self, csproj_file: str, ignored_dependencies: list[str]):
2050
+ folder = os.path.dirname(csproj_file)
2051
+ csproj_filename = os.path.basename(csproj_file)
2052
+ self.log.log(f"Check for updates in {csproj_filename}", LogLevel.Information)
2053
+ result = self.run_program_with_retry("dotnet", f"list {csproj_filename} package --outdated", folder, print_errors_as_information=True)
2054
+ for line in result[1].replace("\r", GeneralUtilities.empty_string).split("\n"):
2055
+ # Relevant output-lines are something like " > NJsonSchema 10.7.0 10.7.0 10.9.0"
2056
+ if ">" in line:
2057
+ package_name = line.replace(">", GeneralUtilities.empty_string).strip().split(" ")[0]
2058
+ if not (package_name in ignored_dependencies):
2059
+ self.log.log(f"Update package {package_name}...", LogLevel.Debug)
2060
+ time.sleep(1.1) # attempt to prevent rate-limit
2061
+ self.run_program_with_retry("dotnet", f"add {csproj_filename} package {package_name}", folder, print_errors_as_information=True)
2062
+
2063
+ @GeneralUtilities.check_arguments
2064
+ def create_deb_package(self, toolname: str, binary_folder: str, control_file_content: str, deb_output_folder: str, permission_of_executable_file_as_octet_triple: int) -> None:
2065
+
2066
+ # prepare
2067
+ GeneralUtilities.ensure_directory_exists(deb_output_folder)
2068
+ temp_folder = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
2069
+ GeneralUtilities.ensure_directory_exists(temp_folder)
2070
+ bin_folder = binary_folder
2071
+ tool_content_folder_name = toolname+"Content"
2072
+
2073
+ # create folder
2074
+ GeneralUtilities.ensure_directory_exists(temp_folder)
2075
+ control_content_folder_name = "controlcontent"
2076
+ packagecontent_control_folder = os.path.join(temp_folder, control_content_folder_name)
2077
+ GeneralUtilities.ensure_directory_exists(packagecontent_control_folder)
2078
+ data_content_folder_name = "datacontent"
2079
+ packagecontent_data_folder = os.path.join(temp_folder, data_content_folder_name)
2080
+ GeneralUtilities.ensure_directory_exists(packagecontent_data_folder)
2081
+ entireresult_content_folder_name = "entireresultcontent"
2082
+ packagecontent_entireresult_folder = os.path.join(temp_folder, entireresult_content_folder_name)
2083
+ GeneralUtilities.ensure_directory_exists(packagecontent_entireresult_folder)
2084
+
2085
+ # create "debian-binary"-file
2086
+ debianbinary_file = os.path.join(packagecontent_entireresult_folder, "debian-binary")
2087
+ GeneralUtilities.ensure_file_exists(debianbinary_file)
2088
+ GeneralUtilities.write_text_to_file(debianbinary_file, "2.0\n")
2089
+
2090
+ # create control-content
2091
+
2092
+ # conffiles
2093
+ conffiles_file = os.path.join(packagecontent_control_folder, "conffiles")
2094
+ GeneralUtilities.ensure_file_exists(conffiles_file)
2095
+
2096
+ # postinst-script
2097
+ postinst_file = os.path.join(packagecontent_control_folder, "postinst")
2098
+ GeneralUtilities.ensure_file_exists(postinst_file)
2099
+ exe_file = f"/usr/bin/{tool_content_folder_name}/{toolname}"
2100
+ link_file = f"/usr/bin/{toolname.lower()}"
2101
+ permission = str(permission_of_executable_file_as_octet_triple)
2102
+ GeneralUtilities.write_text_to_file(postinst_file, f"""#!/bin/sh
2103
+ ln -s {exe_file} {link_file}
2104
+ chmod {permission} {exe_file}
2105
+ chmod {permission} {link_file}
2106
+ """)
2107
+
2108
+ # control
2109
+ control_file = os.path.join(packagecontent_control_folder, "control")
2110
+ GeneralUtilities.ensure_file_exists(control_file)
2111
+ GeneralUtilities.write_text_to_file(control_file, control_file_content)
2112
+
2113
+ # md5sums
2114
+ md5sums_file = os.path.join(packagecontent_control_folder, "md5sums")
2115
+ GeneralUtilities.ensure_file_exists(md5sums_file)
2116
+
2117
+ # create data-content
2118
+
2119
+ # copy binaries
2120
+ usr_bin_folder = os.path.join(packagecontent_data_folder, "usr/bin")
2121
+ GeneralUtilities.ensure_directory_exists(usr_bin_folder)
2122
+ usr_bin_content_folder = os.path.join(usr_bin_folder, tool_content_folder_name)
2123
+ GeneralUtilities.copy_content_of_folder(bin_folder, usr_bin_content_folder)
2124
+
2125
+ # create debfile
2126
+ deb_filename = f"{toolname}.deb"
2127
+ self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/control.tar.gz", "*"], packagecontent_control_folder)
2128
+ self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/data.tar.gz", "*"], packagecontent_data_folder)
2129
+ self.run_program_argsasarray("ar", ["r", deb_filename, "debian-binary", "control.tar.gz", "data.tar.gz"], packagecontent_entireresult_folder)
2130
+ result_file = os.path.join(packagecontent_entireresult_folder, deb_filename)
2131
+ shutil.copy(result_file, os.path.join(deb_output_folder, deb_filename))
2132
+
2133
+ # cleanup
2134
+ GeneralUtilities.ensure_directory_does_not_exist(temp_folder)
2135
+
2136
+ @GeneralUtilities.check_arguments
2137
+ def update_year_in_copyright_tags(self, file: str) -> None:
2138
+ current_year = str(GeneralUtilities.get_now().year)
2139
+ lines = GeneralUtilities.read_lines_from_file(file)
2140
+ lines_result = []
2141
+ for line in lines:
2142
+ if match := re.search("(.*<[Cc]opyright>.*)\\d\\d\\d\\d(.*<\\/[Cc]opyright>.*)", line):
2143
+ part1 = match.group(1)
2144
+ part2 = match.group(2)
2145
+ adapted = part1+current_year+part2
2146
+ else:
2147
+ adapted = line
2148
+ lines_result.append(adapted)
2149
+ GeneralUtilities.write_lines_to_file(file, lines_result)
2150
+
2151
+ @GeneralUtilities.check_arguments
2152
+ def update_year_in_first_line_of_file(self, file: str) -> None:
2153
+ current_year = str(GeneralUtilities.get_now().year)
2154
+ lines = GeneralUtilities.read_lines_from_file(file)
2155
+ lines[0] = re.sub("\\d\\d\\d\\d", current_year, lines[0])
2156
+ GeneralUtilities.write_lines_to_file(file, lines)
2157
+
2158
+ @GeneralUtilities.check_arguments
2159
+ def get_external_ip_address(self) -> str:
2160
+ information = self.get_externalnetworkinformation_as_json_string()
2161
+ parsed = json.loads(information)
2162
+ return parsed["IPAddress"]
2163
+
2164
+ @GeneralUtilities.check_arguments
2165
+ def get_country_of_external_ip_address(self) -> str:
2166
+ information = self.get_externalnetworkinformation_as_json_string()
2167
+ parsed = json.loads(information)
2168
+ return parsed["Country"]
2169
+
2170
+ @GeneralUtilities.check_arguments
2171
+ def get_externalnetworkinformation_as_json_string(self,clientinformation_link:str='https://clientinformation.anion327.de') -> str:
2172
+ headers = {'Cache-Control': 'no-cache'}
2173
+ response = requests.get(clientinformation_link, timeout=5, headers=headers)
2174
+ network_information_as_json_string = GeneralUtilities.bytes_to_string(response.content)
2175
+ return network_information_as_json_string
2176
+
2177
+ @GeneralUtilities.check_arguments
2178
+ def change_file_extensions(self, folder: str, from_extension: str, to_extension: str, recursive: bool, ignore_case: bool) -> None:
2179
+ extension_to_compare: str = None
2180
+ if ignore_case:
2181
+ extension_to_compare = from_extension.lower()
2182
+ else:
2183
+ extension_to_compare = from_extension
2184
+ for file in GeneralUtilities.get_direct_files_of_folder(folder):
2185
+ if (ignore_case and file.lower().endswith(f".{extension_to_compare}") or not ignore_case and file.endswith(f".{extension_to_compare}")):
2186
+ p = Path(file)
2187
+ p.rename(p.with_suffix('.'+to_extension))
2188
+ if recursive:
2189
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
2190
+ self.change_file_extensions(subfolder, from_extension, to_extension, recursive, ignore_case)
2191
+
2192
+ @GeneralUtilities.check_arguments
2193
+ def __add_chapter(self, main_reference_file, reference_content_folder, number: int, chaptertitle: str, content: str = None):
2194
+ if content is None:
2195
+ content = "TXDX add content here"
2196
+ filename = str(number).zfill(2)+"_"+chaptertitle.replace(' ', '-')
2197
+ file = f"{reference_content_folder}/{filename}.md"
2198
+ full_title = f"{number}. {chaptertitle}"
2199
+
2200
+ GeneralUtilities.append_line_to_file(main_reference_file, f"- [{full_title}](./{filename}.md)")
2201
+
2202
+ GeneralUtilities.ensure_file_exists(file)
2203
+ GeneralUtilities.write_text_to_file(file, f"""# {full_title}
2204
+
2205
+ {content}
2206
+ """.replace("XDX", "ODO"))
2207
+
2208
+ @GeneralUtilities.check_arguments
2209
+ def generate_arc42_reference_template(self, repository: str, productname: str = None, subfolder: str = None):
2210
+ productname: str
2211
+ if productname is None:
2212
+ productname = os.path.basename(repository)
2213
+ if subfolder is None:
2214
+ subfolder = "Other/Reference"
2215
+ reference_root_folder = f"{repository}/{subfolder}"
2216
+ reference_content_folder = reference_root_folder + "/Technical"
2217
+ if os.path.isdir(reference_root_folder):
2218
+ raise ValueError(f"The folder '{reference_root_folder}' does already exist.")
2219
+ GeneralUtilities.ensure_directory_exists(reference_root_folder)
2220
+ GeneralUtilities.ensure_directory_exists(reference_content_folder)
2221
+ main_reference_file = f"{reference_root_folder}/Reference.md"
2222
+ GeneralUtilities.ensure_file_exists(main_reference_file)
2223
+ GeneralUtilities.write_text_to_file(main_reference_file, f"""# {productname}
2224
+
2225
+ TXDX add minimal service-description here.
2226
+
2227
+ ## Technical documentation
2228
+
2229
+ """.replace("XDX", "ODO"))
2230
+ self.__add_chapter(main_reference_file, reference_content_folder, 1, 'Introduction and Goals', """## Overview
2231
+
2232
+ TXDX
2233
+
2234
+ ## Quality goals
2235
+
2236
+ TXDX
2237
+
2238
+ ## Stakeholder
2239
+
2240
+ | Name | How to contact | Reason |
2241
+ | ---- | -------------- | ------ |""")
2242
+ self.__add_chapter(main_reference_file, reference_content_folder, 2, 'Constraints', """## Technical constraints
2243
+
2244
+ | Constraint-identifier | Constraint | Reason |
2245
+ | --------------------- | ---------- | ------ |
2246
+
2247
+ ## Organizational constraints
2248
+
2249
+ | Constraint-identifier | Constraint | Reason |
2250
+ | --------------------- | ---------- | ------ |""")
2251
+ self.__add_chapter(main_reference_file, reference_content_folder, 3, 'Context and Scope', """## Context
2252
+
2253
+ TXDX
2254
+
2255
+ ## Scope
2256
+
2257
+ TXDX""")
2258
+ self.__add_chapter(main_reference_file, reference_content_folder, 4, 'Solution Strategy', """TXDX""")
2259
+ self.__add_chapter(main_reference_file, reference_content_folder, 5, 'Building Block View', """TXDX""")
2260
+ self.__add_chapter(main_reference_file, reference_content_folder, 6, 'Runtime View', """TXDX""")
2261
+ self.__add_chapter(main_reference_file, reference_content_folder, 7, 'Deployment View', """## Infrastructure-overview
2262
+
2263
+ TXDX
2264
+
2265
+ ## Infrastructure-requirements
2266
+
2267
+ TXDX
2268
+
2269
+ ## Deployment-proecsses
2270
+
2271
+ TXDX""")
2272
+ self.__add_chapter(main_reference_file, reference_content_folder, 8, 'Crosscutting Concepts', """TXDX""")
2273
+ self.__add_chapter(main_reference_file, reference_content_folder, 9, 'Architectural Decisions', """## Decision-board
2274
+
2275
+ | Decision-identifier | Date | Decision | Reason and notes |
2276
+ | ------------------- | ---- | -------- | ---------------- |""") # empty because there are no decsions yet
2277
+ self.__add_chapter(main_reference_file, reference_content_folder, 10, 'Quality Requirements', """TXDX""")
2278
+ self.__add_chapter(main_reference_file, reference_content_folder, 11, 'Risks and Technical Debt', """## Risks
2279
+
2280
+ Currently there are no known risks.
2281
+
2282
+ ## Technical debts
2283
+
2284
+ Currently there are no technical depts.""")
2285
+ self.__add_chapter(main_reference_file, reference_content_folder, 12, 'Glossary', """## Terms
2286
+
2287
+ | Term | Meaning |
2288
+ | ---- | ------- |
2289
+
2290
+ ## Abbreviations
2291
+
2292
+ | Abbreviation | Meaning |
2293
+ | ------------ | ------- |""")
2294
+
2295
+ GeneralUtilities.append_to_file(main_reference_file, """
2296
+
2297
+ ## Responsibilities
2298
+
2299
+ | Responsibility | Name and contact-information |
2300
+ | --------------- | ---------------------------- |
2301
+ | Pdocut-owner | TXDX |
2302
+ | Product-manager | TXDX |
2303
+ | Support | TXDX |
2304
+
2305
+ ## License & Pricing
2306
+
2307
+ TXDX
2308
+
2309
+ ## External resources
2310
+
2311
+ - [Repository](TXDX)
2312
+ - [Productive-System](TXDX)
2313
+ - [QualityCheck-system](TXDX)
2314
+ """.replace("XDX", "ODO"))
2315
+
2316
+ @GeneralUtilities.check_arguments
2317
+ def run_with_timeout(self, method, timeout_in_seconds: float) -> bool:
2318
+ # Returns true if the method was terminated due to a timeout
2319
+ # Returns false if the method terminates in the given time
2320
+ p = multiprocessing.Process(target=method)
2321
+ p.start()
2322
+ p.join(timeout_in_seconds)
2323
+ if p.is_alive():
2324
+ p.kill()
2325
+ p.join()
2326
+ return True
2327
+ else:
2328
+ return False
2329
+
2330
+ @GeneralUtilities.check_arguments
2331
+ def ensure_local_docker_network_exists(self, network_name: str) -> None:
2332
+ if not self.local_docker_network_exists(network_name):
2333
+ self.create_local_docker_network(network_name)
2334
+
2335
+ @GeneralUtilities.check_arguments
2336
+ def ensure_local_docker_network_does_not_exist(self, network_name: str) -> None:
2337
+ if self.local_docker_network_exists(network_name):
2338
+ self.remove_local_docker_network(network_name)
2339
+
2340
+ @GeneralUtilities.check_arguments
2341
+ def local_docker_network_exists(self, network_name: str) -> bool:
2342
+ return network_name in self.get_all_local_existing_docker_networks()
2343
+
2344
+ @GeneralUtilities.check_arguments
2345
+ def get_all_local_existing_docker_networks(self) -> list[str]:
2346
+ program_call_result = self.run_program("docker", "network list")
2347
+ std_out = program_call_result[1]
2348
+ std_out_lines = std_out.split("\n")[1:]
2349
+ result: list[str] = []
2350
+ for std_out_line in std_out_lines:
2351
+ normalized_line = ';'.join(std_out_line.split())
2352
+ splitted = normalized_line.split(";")
2353
+ result.append(splitted[1])
2354
+ return result
2355
+
2356
+ @GeneralUtilities.check_arguments
2357
+ def remove_local_docker_network(self, network_name: str) -> None:
2358
+ self.run_program("docker", f"network remove {network_name}")
2359
+
2360
+ @GeneralUtilities.check_arguments
2361
+ def create_local_docker_network(self, network_name: str) -> None:
2362
+ self.run_program("docker", f"network create {network_name}")
2363
+
2364
+ @GeneralUtilities.check_arguments
2365
+ def format_xml_file(self, file: str) -> None:
2366
+ encoding = "utf-8"
2367
+ element = ET.XML(GeneralUtilities.read_text_from_file(file, encoding))
2368
+ ET.indent(element)
2369
+ GeneralUtilities.write_text_to_file(file, ET.tostring(element, encoding="unicode"), encoding)
2370
+
2371
+ @GeneralUtilities.check_arguments
2372
+ def install_requirementstxt_file(self, requirements_txt_file: str):
2373
+ folder: str = os.path.dirname(requirements_txt_file)
2374
+ filename: str = os.path.basename(requirements_txt_file)
2375
+ self.run_program_argsasarray("pip", ["install", "-r", filename], folder)
2376
+
2377
+ @GeneralUtilities.check_arguments
2378
+ def ocr_analysis_of_folder(self, folder: str, serviceaddress: str, extensions: list[str], languages: list[str]) -> list[str]: # Returns a list of changed files due to ocr-analysis.
2379
+ GeneralUtilities.write_message_to_stdout("Starting OCR analysis of folder " + folder)
2380
+ supported_extensions = ['png', 'jpg', 'jpeg', 'tiff', 'bmp', 'gif', 'pdf', 'docx', 'doc', 'xlsx', 'xls', 'pptx', 'ppt']
2381
+ changes_files: list[str] = []
2382
+ if extensions is None:
2383
+ extensions = supported_extensions
2384
+ for file in GeneralUtilities.get_direct_files_of_folder(folder):
2385
+ file_lower = file.lower()
2386
+ for extension in extensions:
2387
+ if file_lower.endswith("."+extension):
2388
+ if self.ocr_analysis_of_file(file, serviceaddress, languages):
2389
+ changes_files.append(file)
2390
+ break
2391
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
2392
+ for file in self.ocr_analysis_of_folder(subfolder, serviceaddress, extensions, languages):
2393
+ changes_files.append(file)
2394
+ return changes_files
2395
+
2396
+ @GeneralUtilities.check_arguments
2397
+ def ocr_analysis_of_file(self, file: str, serviceaddress: str, languages: list[str]) -> bool: # Returns true if the ocr-file was generated or updated. Returns false if the existing ocr-file was not changed.
2398
+ GeneralUtilities.write_message_to_stdout("Do OCR analysis of file " + file)
2399
+ supported_extensions = ['png', 'jpg', 'jpeg', 'tiff', 'bmp', 'webp', 'gif', 'pdf', 'rtf', 'docx', 'doc', 'odt', 'xlsx', 'xls', 'ods', 'pptx', 'ppt', 'odp']
2400
+ for extension in supported_extensions:
2401
+ if file.lower().endswith("."+extension):
2402
+ raise ValueError(f"Extension '{extension}' is not supported. Supported extensions are: {', '.join(supported_extensions)}")
2403
+ target_file = file+".ocr.txt"
2404
+ hash_of_current_file: str = GeneralUtilities. get_sha256_of_file(file)
2405
+ if os.path.isfile(target_file):
2406
+ lines = GeneralUtilities.read_lines_from_file(target_file)
2407
+ previous_hash_of_current_file: str = lines[1].split(":")[1].strip()
2408
+ if hash_of_current_file == previous_hash_of_current_file:
2409
+ return False
2410
+ ocr_content = self.get_ocr_content_of_file(file, serviceaddress, languages)
2411
+ GeneralUtilities.ensure_file_exists(target_file)
2412
+ GeneralUtilities.write_text_to_file(file, f"""Name of file: \"{os.path.basename(file)}\""
2413
+ Hash of file: {hash_of_current_file}
2414
+ OCR-content:
2415
+ \"{ocr_content}\"""")
2416
+ return True
2417
+
2418
+ @GeneralUtilities.check_arguments
2419
+ def get_ocr_content_of_file(self, file: str, serviceaddress: str, languages: list[str]) -> str: # serviceaddress = None means local executable
2420
+ result: str = None
2421
+ extension = Path(file).suffix
2422
+ if serviceaddress is None:
2423
+ program_result = self.run_program_argsasarray("simpleocr", ["--File", file, "--Languages", "+".join(languages)] + languages)
2424
+ result = program_result[1]
2425
+ else:
2426
+ languages_for_url = '%2B'.join(languages)
2427
+ package_url: str = f"https://{serviceaddress}/GetOCRContent?languages={languages_for_url}&fileType={extension}"
2428
+ headers = {'Cache-Control': 'no-cache'}
2429
+ r = requests.put(package_url, timeout=5, headers=headers, data=GeneralUtilities.read_binary_from_file(file))
2430
+ if r.status_code != 200:
2431
+ raise ValueError(f"Checking for latest tor package resulted in HTTP-response-code {r.status_code}.")
2432
+ result = GeneralUtilities.bytes_to_string(r.content)
2433
+ return result
2434
+
2435
+ @GeneralUtilities.check_arguments
2436
+ def ocr_analysis_of_repository(self, folder: str, serviceaddress: str, extensions: list[str], languages: list[str]) -> None:
2437
+ self.assert_is_git_repository(folder)
2438
+ changed_files = self.ocr_analysis_of_folder(folder, serviceaddress, extensions, languages)
2439
+ for changed_ocr_file in changed_files:
2440
+ GeneralUtilities.assert_condition(changed_ocr_file.endswith(".ocr.txt"), f"File '{changed_ocr_file}' is not an OCR-file. It should end with '.ocr.txt'.")
2441
+ base_file = changed_ocr_file[:-len(".ocr.txt")]
2442
+ GeneralUtilities.assert_condition(os.path.isfile(base_file), f"Base file '{base_file}' does not exist. The OCR-file '{changed_ocr_file}' is not valid.")
2443
+ base_file_relative_path = os.path.relpath(base_file, folder)
2444
+ base_file_diff_program_result = self.run_program("git", f"diff --quiet -- \"{base_file_relative_path}\"", folder, throw_exception_if_exitcode_is_not_zero=False)
2445
+ has_staged_changes: bool = None
2446
+ if base_file_diff_program_result[0] == 0:
2447
+ has_staged_changes = False
2448
+ elif base_file_diff_program_result[0] == 1:
2449
+ has_staged_changes = True
2450
+ else:
2451
+ raise RuntimeError(f"Unexpected exit code {base_file_diff_program_result[0]} when checking for staged changes of file '{base_file_relative_path}'.")
2452
+ if has_staged_changes:
2453
+ changed_ocr_file_relative_path = os.path.relpath(changed_ocr_file, folder)
2454
+ self.run_program_argsasarray("git", ["add", changed_ocr_file_relative_path], folder)
2455
+
2456
+ @GeneralUtilities.check_arguments
2457
+ def update_timestamp_in_file(self, target_file: str) -> None:
2458
+ lines = GeneralUtilities.read_lines_from_file(target_file)
2459
+ new_lines = []
2460
+ prefix: str = "# last update: "
2461
+ for line in lines:
2462
+ if line.startswith(prefix):
2463
+ new_lines.append(prefix+GeneralUtilities.datetime_to_string_with_timezone(GeneralUtilities.get_now()))
2464
+ else:
2465
+ new_lines.append(line)
2466
+ GeneralUtilities.write_lines_to_file(target_file, new_lines)
2467
+
2468
+ def do_and_log_task(self, name_of_task: str, task):
2469
+ try:
2470
+ self.log.log(f"Start action \"{name_of_task}\".", LogLevel.Information)
2471
+ result = task()
2472
+ if result is None:
2473
+ result = 0
2474
+ return result
2475
+ except Exception as e:
2476
+ self.log.log_exception(f"Error while running action \"{name_of_task}\".", e, LogLevel.Error)
2477
+ return 1
2478
+ finally:
2479
+ self.log.log(f"Finished action \"{name_of_task}\".", LogLevel.Information)
2480
+
2481
+ def get_lines_of_code_with_default_excluded_patterns(self, repository: str) -> int:
2482
+ return self.get_lines_of_code(repository, self.default_excluded_patterns_for_loc)
2483
+
2484
+ default_excluded_patterns_for_loc: list[str] = [".txt", ".md", ".vscode", "Resources", "Reference", ".gitignore", ".gitattributes", "Other/Metrics"]
2485
+
2486
+ def get_lines_of_code(self, repository: str, excluded_pattern: list[str]) -> int:
2487
+ self.assert_is_git_repository(repository)
2488
+ result: int = 0
2489
+ self.log.log(f"Calculate lines of code in repository '{repository}' with excluded patterns: {', '.join(excluded_pattern)}",LogLevel.Debug)
2490
+ git_response = self.run_program("git", "ls-files", repository)
2491
+ files: list[str] = GeneralUtilities.string_to_lines(git_response[1])
2492
+ for file in files:
2493
+ if os.path.isfile(os.path.join(repository, file)):
2494
+ if self.__is_excluded_by_glob_pattern(file, excluded_pattern):
2495
+ self.log.log(f"File '{file}' is ignored because it matches an excluded pattern.",LogLevel.Diagnostic)
2496
+ else:
2497
+ full_file: str = os.path.join(repository, file)
2498
+ if GeneralUtilities.is_binary_file(full_file):
2499
+ self.log.log(f"File '{file}' is ignored because it is a binary-file.",LogLevel.Diagnostic)
2500
+ else:
2501
+ self.log.log(f"Count lines of file '{file}'.",LogLevel.Diagnostic)
2502
+ length = len(GeneralUtilities.read_nonempty_lines_from_file(full_file))
2503
+ result = result+length
2504
+ else:
2505
+ self.log.log(f"File '{file}' is ignored because it does not exist.",LogLevel.Diagnostic)
2506
+ return result
2507
+
2508
+ def __is_excluded_by_glob_pattern(self, file: str, excluded_patterns: list[str]) -> bool:
2509
+ for pattern in excluded_patterns:
2510
+ if fnmatch.fnmatch(file, f"*{pattern}*"):
2511
+ return True
2512
+ return False
2513
+
2514
+ @GeneralUtilities.check_arguments
2515
+ def create_zip_archive(self, folder:str,zip_file:str) -> None:
2516
+ GeneralUtilities.assert_folder_exists(folder)
2517
+ GeneralUtilities.assert_file_does_not_exist(zip_file)
2518
+ folder = os.path.abspath(folder)
2519
+ with zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED) as zipf:
2520
+ for root, _, files in os.walk(folder):
2521
+ for file in files:
2522
+ file_path = os.path.join(root, file)
2523
+ arcname = os.path.relpath(file_path, start=folder)
2524
+ zipf.write(file_path, arcname)
2525
+
2526
+ @GeneralUtilities.check_arguments
2527
+ def start_local_test_service(self, file: str):
2528
+ example_folder = os.path.dirname(file)
2529
+ docker_compose_file = os.path.join(example_folder, "docker-compose.yml")
2530
+ for service in self.get_services_from_yaml_file(docker_compose_file):
2531
+ self.kill_docker_container(service)
2532
+ example_name = os.path.basename(example_folder)
2533
+ title = f"Test{example_name}"
2534
+ self.run_program("docker", f"compose -p {title.lower()} up --detach", example_folder, title=title)
2535
+
2536
+ @GeneralUtilities.check_arguments
2537
+ def stop_local_test_service(self, file: str):
2538
+ example_folder = os.path.dirname(file)
2539
+ example_name = os.path.basename(example_folder)
2540
+ title = f"Test{example_name}"
2541
+ self.run_program("docker", f"compose -p {title.lower()} down", example_folder, title=title)