ScriptCollection 3.4.55__py3-none-any.whl → 3.5.44__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1703 +1,1930 @@
1
- from datetime import timedelta, datetime
2
- import binascii
3
- import filecmp
4
- import hashlib
5
- from io import BytesIO
6
- import itertools
7
- import math
8
- import os
9
- from pathlib import Path
10
- from random import randrange
11
- from subprocess import Popen
12
- import re
13
- import shutil
14
- import traceback
15
- import uuid
16
- import tempfile
17
- import io
18
- import requests
19
- import ntplib
20
- import qrcode
21
- import pycdlib
22
- import send2trash
23
- import PyPDF2
24
- from .GeneralUtilities import GeneralUtilities
25
- from .ProgramRunnerBase import ProgramRunnerBase
26
- from .ProgramRunnerPopen import ProgramRunnerPopen
27
- from .ProgramRunnerEpew import ProgramRunnerEpew, CustomEpewArgument
28
-
29
-
30
- version = "3.4.55"
31
- __version__ = version
32
-
33
-
34
- class ScriptCollectionCore:
35
-
36
- # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
37
- # Do not change this value for productive environments.
38
- mock_program_calls: bool = False
39
- # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
40
- execute_program_really_if_no_mock_call_is_defined: bool = False
41
- __mocked_program_calls: list = list()
42
- program_runner: ProgramRunnerBase = None
43
-
44
- def __init__(self):
45
- self.program_runner = ProgramRunnerPopen()
46
-
47
- @staticmethod
48
- @GeneralUtilities.check_arguments
49
- def get_scriptcollection_version() -> str:
50
- return __version__
51
-
52
- @GeneralUtilities.check_arguments
53
- def python_file_has_errors(self, file: str, working_directory: str, treat_warnings_as_errors: bool = True) -> tuple[bool, list[str]]:
54
- errors = list()
55
- filename = os.path.relpath(file, working_directory)
56
- if treat_warnings_as_errors:
57
- errorsonly_argument = ""
58
- else:
59
- errorsonly_argument = " --errors-only"
60
- (exit_code, stdout, stderr, _) = self.run_program("pylint", filename+errorsonly_argument, working_directory, throw_exception_if_exitcode_is_not_zero=False)
61
- if (exit_code != 0):
62
- errors.append(f"Linting-issues of {file}:")
63
- errors.append(f"Pylint-exitcode: {exit_code}")
64
- for line in GeneralUtilities.string_to_lines(stdout):
65
- errors.append(line)
66
- for line in GeneralUtilities.string_to_lines(stderr):
67
- errors.append(line)
68
- return (True, errors)
69
-
70
- return (False, errors)
71
-
72
- @GeneralUtilities.check_arguments
73
- def replace_version_in_dockerfile_file(self, dockerfile: str, new_version_value: str) -> None:
74
- GeneralUtilities.write_text_to_file(dockerfile, re.sub("ARG Version=\"\\d+\\.\\d+\\.\\d+\"", f"ARG Version=\"{new_version_value}\"",
75
- GeneralUtilities.read_text_from_file(dockerfile)))
76
-
77
- @GeneralUtilities.check_arguments
78
- def replace_version_in_python_file(self, file: str, new_version_value: str):
79
- GeneralUtilities.write_text_to_file(file, re.sub("version = \"\\d+\\.\\d+\\.\\d+\"", f"version = \"{new_version_value}\"",
80
- GeneralUtilities.read_text_from_file(file)))
81
-
82
- @GeneralUtilities.check_arguments
83
- def replace_version_in_ini_file(self, file: str, new_version_value: str):
84
- GeneralUtilities.write_text_to_file(file, re.sub("version = \\d+\\.\\d+\\.\\d+", f"version = {new_version_value}",
85
- GeneralUtilities.read_text_from_file(file)))
86
-
87
- @GeneralUtilities.check_arguments
88
- def replace_version_in_nuspec_file(self, nuspec_file: str, new_version: str) -> None:
89
- # TODO use XSLT instead
90
- versionregex = "\\d+\\.\\d+\\.\\d+"
91
- versiononlyregex = f"^{versionregex}$"
92
- pattern = re.compile(versiononlyregex)
93
- if pattern.match(new_version):
94
- GeneralUtilities.write_text_to_file(nuspec_file, re.sub(f"<version>{versionregex}<\\/version>",
95
- f"<version>{new_version}</version>", GeneralUtilities.read_text_from_file(nuspec_file)))
96
- else:
97
- raise ValueError(f"Version '{new_version}' does not match version-regex '{versiononlyregex}'")
98
-
99
- @GeneralUtilities.check_arguments
100
- def replace_version_in_csproj_file(self, csproj_file: str, current_version: str):
101
- versionregex = "\\d+\\.\\d+\\.\\d+"
102
- versiononlyregex = f"^{versionregex}$"
103
- pattern = re.compile(versiononlyregex)
104
- if pattern.match(current_version):
105
- for tag in ["Version", "AssemblyVersion", "FileVersion"]:
106
- GeneralUtilities.write_text_to_file(csproj_file, re.sub(f"<{tag}>{versionregex}(.\\d+)?<\\/{tag}>",
107
- f"<{tag}>{current_version}</{tag}>", GeneralUtilities.read_text_from_file(csproj_file)))
108
- else:
109
- raise ValueError(f"Version '{current_version}' does not match version-regex '{versiononlyregex}'")
110
-
111
- @GeneralUtilities.check_arguments
112
- def push_nuget_build_artifact(self, nupkg_file: str, registry_address: str, api_key: str, verbosity: int = 1):
113
- nupkg_file_name = os.path.basename(nupkg_file)
114
- nupkg_file_folder = os.path.dirname(nupkg_file)
115
- self.run_program("dotnet", f"nuget push {nupkg_file_name} --force-english-output --source {registry_address} --api-key {api_key}",
116
- nupkg_file_folder, verbosity)
117
-
118
- @GeneralUtilities.check_arguments
119
- def dotnet_build(self, repository_folder: str, projectname: str, configuration: str):
120
- self.run_program("dotnet", f"clean -c {configuration}", repository_folder)
121
- self.run_program("dotnet", f"build {projectname}/{projectname}.csproj -c {configuration}", repository_folder)
122
-
123
- @GeneralUtilities.check_arguments
124
- def find_file_by_extension(self, folder: str, extension: str):
125
- result = [file for file in GeneralUtilities.get_direct_files_of_folder(folder) if file.endswith(f".{extension}")]
126
- result_length = len(result)
127
- if result_length == 0:
128
- raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension}'.")
129
- if result_length == 1:
130
- return result[0]
131
- else:
132
- raise ValueError(f"Multiple values available in folder '{folder}' with extension '{extension}'.")
133
-
134
- @GeneralUtilities.check_arguments
135
- def commit_is_signed_by_key(self, repository_folder: str, revision_identifier: str, key: str) -> bool:
136
- result = self.run_program("git", f"verify-commit {revision_identifier}", repository_folder, throw_exception_if_exitcode_is_not_zero=False)
137
- if (result[0] != 0):
138
- return False
139
- if (not GeneralUtilities.contains_line(result[1].splitlines(), f"gpg\\:\\ using\\ [A-Za-z0-9]+\\ key\\ [A-Za-z0-9]+{key}")):
140
- # TODO check whether this works on machines where gpg is installed in another langauge than english
141
- return False
142
- if (not GeneralUtilities.contains_line(result[1].splitlines(), "gpg\\:\\ Good\\ signature\\ from")):
143
- # TODO check whether this works on machines where gpg is installed in another langauge than english
144
- return False
145
- return True
146
-
147
- @GeneralUtilities.check_arguments
148
- def get_parent_commit_ids_of_commit(self, repository_folder: str, commit_id: str) -> str:
149
- return self.run_program("git", f'log --pretty=%P -n 1 "{commit_id}"',
150
- repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].replace("\r", "").replace("\n", "").split(" ")
151
-
152
- @GeneralUtilities.check_arguments
153
- def get_all_authors_and_committers_of_repository(self, repository_folder: str, subfolder: str = None, verbosity: int = 1) -> list[tuple[str, str]]:
154
- space_character = "_"
155
- if subfolder is None:
156
- subfolder_argument = ""
157
- else:
158
- subfolder_argument = f" -- {subfolder}"
159
- log_result = self.run_program("git", f'log --pretty=%aN{space_character}%aE%n%cN{space_character}%cE HEAD{subfolder_argument}',
160
- repository_folder, verbosity=0)
161
- plain_content: list[str] = list(set([line for line in log_result[1].split("\n") if len(line) > 0]))
162
- result: list[tuple[str, str]] = []
163
- for item in plain_content:
164
- if len(re.findall(space_character, item)) == 1:
165
- splitted = item.split(space_character)
166
- result.append((splitted[0], splitted[1]))
167
- else:
168
- raise ValueError(f'Unexpected author: "{item}"')
169
- return result
170
-
171
- @GeneralUtilities.check_arguments
172
- def get_commit_ids_between_dates(self, repository_folder: str, since: datetime, until: datetime, ignore_commits_which_are_not_in_history_of_head: bool = True) -> None:
173
- since_as_string = self.__datetime_to_string_for_git(since)
174
- until_as_string = self.__datetime_to_string_for_git(until)
175
- result = filter(lambda line: not GeneralUtilities.string_is_none_or_whitespace(line),
176
- self.run_program("git", f'log --since "{since_as_string}" --until "{until_as_string}" --pretty=format:"%H" --no-patch',
177
- repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].split("\n").replace("\r", ""))
178
- if ignore_commits_which_are_not_in_history_of_head:
179
- result = [commit_id for commit_id in result if self.git_commit_is_ancestor(repository_folder, commit_id)]
180
- return result
181
-
182
- @GeneralUtilities.check_arguments
183
- def __datetime_to_string_for_git(self, datetime_object: datetime) -> str:
184
- return datetime_object.strftime('%Y-%m-%d %H:%M:%S')
185
-
186
- @GeneralUtilities.check_arguments
187
- def git_commit_is_ancestor(self, repository_folder: str, ancestor: str, descendant: str = "HEAD") -> bool:
188
- exit_code = self.run_program_argsasarray("git", ["merge-base", "--is-ancestor", ancestor, descendant],
189
- repository_folder, throw_exception_if_exitcode_is_not_zero=False)[0]
190
- if exit_code == 0:
191
- return True
192
- elif exit_code == 1:
193
- return False
194
- else:
195
- raise ValueError(f"Can not calculate if {ancestor} is an ancestor of {descendant} in repository {repository_folder}.")
196
-
197
- @GeneralUtilities.check_arguments
198
- def __git_changes_helper(self, repository_folder: str, arguments_as_array: list[str]) -> bool:
199
- lines = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", arguments_as_array, repository_folder,
200
- throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
201
- for line in lines:
202
- if GeneralUtilities.string_has_content(line):
203
- return True
204
- return False
205
-
206
- @GeneralUtilities.check_arguments
207
- def git_repository_has_new_untracked_files(self, repositoryFolder: str):
208
- return self.__git_changes_helper(repositoryFolder, ["ls-files", "--exclude-standard", "--others"])
209
-
210
- @GeneralUtilities.check_arguments
211
- def git_repository_has_unstaged_changes_of_tracked_files(self, repositoryFolder: str):
212
- return self.__git_changes_helper(repositoryFolder, ["diff"])
213
-
214
- @GeneralUtilities.check_arguments
215
- def git_repository_has_staged_changes(self, repositoryFolder: str):
216
- return self.__git_changes_helper(repositoryFolder, ["diff", "--cached"])
217
-
218
- @GeneralUtilities.check_arguments
219
- def git_repository_has_uncommitted_changes(self, repositoryFolder: str) -> bool:
220
- if (self.git_repository_has_unstaged_changes(repositoryFolder)):
221
- return True
222
- if (self.git_repository_has_staged_changes(repositoryFolder)):
223
- return True
224
- return False
225
-
226
- @GeneralUtilities.check_arguments
227
- def git_repository_has_unstaged_changes(self, repository_folder: str) -> bool:
228
- if (self.git_repository_has_unstaged_changes_of_tracked_files(repository_folder)):
229
- return True
230
- if (self.git_repository_has_new_untracked_files(repository_folder)):
231
- return True
232
- return False
233
-
234
- @GeneralUtilities.check_arguments
235
- def git_get_commit_id(self, repository_folder: str, commit: str = "HEAD") -> str:
236
- result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["rev-parse", "--verify", commit],
237
- repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
238
- return result[1].replace('\n', '')
239
-
240
- @GeneralUtilities.check_arguments
241
- def git_get_commit_date(self, repository_folder: str, commit: str = "HEAD") -> datetime:
242
- result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["show", "-s", "--format=%ci", commit],
243
- repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
244
- date_as_string = result[1].replace('\n', '')
245
- result = datetime.strptime(date_as_string, '%Y-%m-%d %H:%M:%S %z')
246
- return result
247
-
248
- @GeneralUtilities.check_arguments
249
- def git_fetch(self, folder: str, remotename: str = "--all") -> None:
250
- self.run_program_argsasarray("git", ["fetch", remotename, "--tags", "--prune"], folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
251
-
252
- @GeneralUtilities.check_arguments
253
- def git_fetch_in_bare_repository(self, folder: str, remotename, localbranch: str, remotebranch: str) -> None:
254
- self.run_program_argsasarray("git", ["fetch", remotename, f"{remotebranch}:{localbranch}"], folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
255
-
256
- @GeneralUtilities.check_arguments
257
- def git_remove_branch(self, folder: str, branchname: str) -> None:
258
- self.run_program("git", f"branch -D {branchname}", folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
259
-
260
- @GeneralUtilities.check_arguments
261
- def git_push(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: int = 0) -> None:
262
- argument = ["push", "--recurse-submodules=on-demand", remotename, f"{localbranchname}:{remotebranchname}"]
263
- if (forcepush):
264
- argument.append("--force")
265
- if (pushalltags):
266
- argument.append("--tags")
267
- result: tuple[int, str, str, int] = self.run_program_argsasarray("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True,
268
- verbosity=verbosity, print_errors_as_information=True)
269
- return result[1].replace('\r', '').replace('\n', '')
270
-
271
- @GeneralUtilities.check_arguments
272
- def git_clone(self, clone_target_folder: str, remote_repository_path: str, include_submodules: bool = True, mirror: bool = False) -> None:
273
- if (os.path.isdir(clone_target_folder)):
274
- pass # TODO throw error
275
- else:
276
- args = ["clone", remote_repository_path, clone_target_folder]
277
- if include_submodules:
278
- args.append("--recurse-submodules")
279
- args.append("--remote-submodules")
280
- if mirror:
281
- args.append("--mirror")
282
- self.run_program_argsasarray("git", args, os.getcwd(), throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
283
-
284
- @GeneralUtilities.check_arguments
285
- def git_get_all_remote_names(self, directory: str) -> list[str]:
286
- result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
287
- return result
288
-
289
- @GeneralUtilities.check_arguments
290
- def git_get_remote_url(self, directory: str, remote_name: str) -> str:
291
- result = GeneralUtilities.string_to_lines(self.run_program_argsasarray(
292
- "git", ["remote", "get-url", remote_name], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
293
- return result[0].replace('\n', '')
294
-
295
- @GeneralUtilities.check_arguments
296
- def repository_has_remote_with_specific_name(self, directory: str, remote_name: str) -> bool:
297
- return remote_name in self.git_get_all_remote_names(directory)
298
-
299
- @GeneralUtilities.check_arguments
300
- def git_add_or_set_remote_address(self, directory: str, remote_name: str, remote_address: str) -> None:
301
- if (self.repository_has_remote_with_specific_name(directory, remote_name)):
302
- self.run_program_argsasarray("git", ['remote', 'set-url', 'remote_name', remote_address], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
303
- else:
304
- self.run_program_argsasarray("git", ['remote', 'add', remote_name, remote_address], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
305
-
306
- @GeneralUtilities.check_arguments
307
- def git_stage_all_changes(self, directory: str) -> None:
308
- self.run_program_argsasarray("git", ["add", "-A"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
309
-
310
- @GeneralUtilities.check_arguments
311
- def git_unstage_all_changes(self, directory: str) -> None:
312
- self.run_program_argsasarray("git", ["reset"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
313
-
314
- @GeneralUtilities.check_arguments
315
- def git_stage_file(self, directory: str, file: str) -> None:
316
- self.run_program_argsasarray("git", ['stage', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
317
-
318
- @GeneralUtilities.check_arguments
319
- def git_unstage_file(self, directory: str, file: str) -> None:
320
- self.run_program_argsasarray("git", ['reset', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
321
-
322
- @GeneralUtilities.check_arguments
323
- def git_discard_unstaged_changes_of_file(self, directory: str, file: str) -> None:
324
- """Caution: This method works really only for 'changed' files yet. So this method does not work properly for new or renamed files."""
325
- self.run_program_argsasarray("git", ['checkout', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
326
-
327
- @GeneralUtilities.check_arguments
328
- def git_discard_all_unstaged_changes(self, directory: str) -> None:
329
- """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
330
- self.run_program_argsasarray("git", ['clean', '-df'], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
331
- self.run_program_argsasarray("git", ['checkout', '.'], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
332
-
333
- @GeneralUtilities.check_arguments
334
- def git_commit(self, directory: str, message: str, author_name: str = None, author_email: str = None, stage_all_changes: bool = True,
335
- no_changes_behavior: int = 0) -> str:
336
- # no_changes_behavior=0 => No commit
337
- # no_changes_behavior=1 => Commit anyway
338
- # no_changes_behavior=2 => Exception
339
- author_name = GeneralUtilities.str_none_safe(author_name).strip()
340
- author_email = GeneralUtilities.str_none_safe(author_email).strip()
341
- argument = ['commit', '--quiet', '--allow-empty', '--message', message]
342
- if (GeneralUtilities.string_has_content(author_name)):
343
- argument.append(f'--author="{author_name} <{author_email}>"')
344
- git_repository_has_uncommitted_changes = self.git_repository_has_uncommitted_changes(directory)
345
-
346
- if git_repository_has_uncommitted_changes:
347
- do_commit = True
348
- if stage_all_changes:
349
- self.git_stage_all_changes(directory)
350
- else:
351
- if no_changes_behavior == 0:
352
- GeneralUtilities.write_message_to_stdout(f"Commit '{message}' will not be done because there are no changes to commit in repository '{directory}'")
353
- do_commit = False
354
- if no_changes_behavior == 1:
355
- GeneralUtilities.write_message_to_stdout(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will be done anyway.")
356
- do_commit = True
357
- if no_changes_behavior == 2:
358
- raise RuntimeError(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will not be done.")
359
-
360
- if do_commit:
361
- GeneralUtilities.write_message_to_stdout(f"Commit changes in '{directory}'")
362
- self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
363
-
364
- return self.git_get_commit_id(directory)
365
-
366
- @GeneralUtilities.check_arguments
367
- def git_create_tag(self, directory: str, target_for_tag: str, tag: str, sign: bool = False, message: str = None) -> None:
368
- argument = ["tag", tag, target_for_tag]
369
- if sign:
370
- if message is None:
371
- message = f"Created {target_for_tag}"
372
- argument.extend(["-s", '-m', message])
373
- self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
374
-
375
- @GeneralUtilities.check_arguments
376
- def git_delete_tag(self, directory: str, tag: str) -> None:
377
- self.run_program_argsasarray("git", ["tag", "--delete", tag], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
378
-
379
- @GeneralUtilities.check_arguments
380
- def git_checkout(self, directory: str, branch: str) -> None:
381
- self.run_program_argsasarray("git", ["checkout", branch], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
382
- self.run_program_argsasarray("git", ["submodule", "update", "--recursive"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
383
-
384
- @GeneralUtilities.check_arguments
385
- def git_merge_abort(self, directory: str) -> None:
386
- self.run_program_argsasarray("git", ["merge", "--abort"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
387
-
388
- @GeneralUtilities.check_arguments
389
- def git_merge(self, directory: str, sourcebranch: str, targetbranch: str, fastforward: bool = True, commit: bool = True, commit_message: str = None) -> str:
390
- self.git_checkout(directory, targetbranch)
391
- args = ["merge"]
392
- if not commit:
393
- args.append("--no-commit")
394
- if not fastforward:
395
- args.append("--no-ff")
396
- if commit_message is not None:
397
- args.append("-m")
398
- args.append(commit_message)
399
- args.append(sourcebranch)
400
- self.run_program_argsasarray("git", args, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
401
- return self.git_get_commit_id(directory)
402
-
403
- @GeneralUtilities.check_arguments
404
- def git_undo_all_changes(self, directory: str) -> None:
405
- """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
406
- self.git_unstage_all_changes(directory)
407
- self.git_discard_all_unstaged_changes(directory)
408
-
409
- @GeneralUtilities.check_arguments
410
- def git_fetch_or_clone_all_in_directory(self, source_directory: str, target_directory: str) -> None:
411
- for subfolder in GeneralUtilities.get_direct_folders_of_folder(source_directory):
412
- foldername = os.path.basename(subfolder)
413
- if self.is_git_repository(subfolder):
414
- source_repository = subfolder
415
- target_repository = os.path.join(target_directory, foldername)
416
- if os.path.isdir(target_directory):
417
- # fetch
418
- self.git_fetch(target_directory)
419
- else:
420
- # clone
421
- self.git_clone(target_repository, source_repository, include_submodules=True, mirror=True)
422
-
423
- def get_git_submodules(self, folder: str) -> list[str]:
424
- e = self.run_program("git", "submodule status", folder)
425
- result = []
426
- for submodule_line in GeneralUtilities.string_to_lines(e[1], False, True):
427
- result.append(submodule_line.split(' ')[1])
428
- return result
429
-
430
- @GeneralUtilities.check_arguments
431
- def is_git_repository(self, folder: str) -> bool:
432
- combined = os.path.join(folder, ".git")
433
- # TODO consider check for bare-repositories
434
- return os.path.isdir(combined) or os.path.isfile(combined)
435
-
436
- @GeneralUtilities.check_arguments
437
- def file_is_git_ignored(self, file_in_repository: str, repositorybasefolder: str) -> None:
438
- exit_code = self.run_program_argsasarray("git", ['check-ignore', file_in_repository], repositorybasefolder, throw_exception_if_exitcode_is_not_zero=False, verbosity=0)[0]
439
- if (exit_code == 0):
440
- return True
441
- if (exit_code == 1):
442
- return False
443
- raise ValueError(f"Unable to calculate whether '{file_in_repository}' in repository '{repositorybasefolder}' is ignored due to git-exitcode {exit_code}.")
444
-
445
- @GeneralUtilities.check_arguments
446
- def discard_all_changes(self, repository: str) -> None:
447
- self.run_program_argsasarray("git", ["reset", "HEAD", "."], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
448
- self.run_program_argsasarray("git", ["checkout", "."], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
449
-
450
- @GeneralUtilities.check_arguments
451
- def git_get_current_branch_name(self, repository: str) -> str:
452
- result = self.run_program_argsasarray("git", ["rev-parse", "--abbrev-ref", "HEAD"], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
453
- return result[1].replace("\r", "").replace("\n", "")
454
-
455
- @GeneralUtilities.check_arguments
456
- def git_get_commitid_of_tag(self, repository: str, tag: str) -> str:
457
- stdout = self.run_program_argsasarray("git", ["rev-list", "-n", "1", tag], repository, verbosity=0)
458
- result = stdout[1].replace("\r", "").replace("\n", "")
459
- return result
460
-
461
- @GeneralUtilities.check_arguments
462
- def git_get_tags(self, repository: str) -> list[str]:
463
- tags = [line for line in self.run_program_argsasarray("git", ["tag"], repository)[1].split("\n") if len(line) > 0]
464
- return tags
465
-
466
- @GeneralUtilities.check_arguments
467
- def git_move_tags_to_another_branch(self, repository: str, tag_source_branch: str, tag_target_branch: str,
468
- sign: bool = False, message: str = None) -> None:
469
- tags = self.git_get_tags(repository)
470
- tags_count = len(tags)
471
- counter = 0
472
- for tag in tags:
473
- counter = counter+1
474
- GeneralUtilities.write_message_to_stdout(f"Process tag {counter}/{tags_count}.")
475
- if self.git_commit_is_ancestor(repository, tag, tag_source_branch): # tag is on source-branch
476
- commit_id_old = self.git_get_commitid_of_tag(repository, tag)
477
- commit_date: datetime = self.git_get_commit_date(repository, commit_id_old)
478
- date_as_string = self.__datetime_to_string_for_git(commit_date)
479
- search_commit_result = self.run_program_argsasarray("git", ["log", f'--after="{date_as_string}"', f'--before="{date_as_string}"',
480
- "--pretty=format:%H", tag_target_branch], repository,
481
- throw_exception_if_exitcode_is_not_zero=False)
482
- if search_commit_result[0] != 0 or not GeneralUtilities.string_has_nonwhitespace_content(search_commit_result[1]):
483
- raise ValueError(f"Can not calculate corresponding commit for tag '{tag}'.")
484
- commit_id_new = search_commit_result[1]
485
- self.git_delete_tag(repository, tag)
486
- self.git_create_tag(repository, commit_id_new, tag, sign, message)
487
-
488
- @GeneralUtilities.check_arguments
489
- def get_current_branch_has_tag(self, repository_folder: str) -> str:
490
- result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder, verbosity=0, throw_exception_if_exitcode_is_not_zero=False)
491
- return result[0] == 0
492
-
493
- @GeneralUtilities.check_arguments
494
- def get_latest_tag(self, repository_folder: str) -> str:
495
- result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder, verbosity=0)
496
- result = result[1].replace("\r", "").replace("\n", "")
497
- return result
498
-
499
- @GeneralUtilities.check_arguments
500
- def export_filemetadata(self, folder: str, target_file: str, encoding: str = "utf-8", filter_function=None) -> None:
501
- folder = GeneralUtilities.resolve_relative_path_from_current_working_directory(folder)
502
- lines = list()
503
- path_prefix = len(folder)+1
504
- items = dict()
505
- for item in GeneralUtilities.get_all_folders_of_folder(folder):
506
- items[item] = "d"
507
- for item in GeneralUtilities.get_all_files_of_folder(folder):
508
- items[item] = "f"
509
- for file_or_folder, item_type in items.items():
510
- truncated_file = file_or_folder[path_prefix:]
511
- if (filter_function is None or filter_function(folder, truncated_file)):
512
- owner_and_permisssion = self.get_file_owner_and_file_permission(file_or_folder)
513
- user = owner_and_permisssion[0]
514
- permissions = owner_and_permisssion[1]
515
- lines.append(f"{truncated_file};{item_type};{user};{permissions}")
516
- lines = sorted(lines, key=str.casefold)
517
- with open(target_file, "w", encoding=encoding) as file_object:
518
- file_object.write("\n".join(lines))
519
-
520
- def escape_git_repositories_in_folder(self, folder: str) -> dict[str, str]:
521
- return self.__escape_git_repositories_in_folder_internal(folder, dict[str, str]())
522
-
523
- def __escape_git_repositories_in_folder_internal(self, folder: str, renamed_items: dict[str, str]) -> dict[str, str]:
524
- for file in GeneralUtilities.get_direct_files_of_folder(folder):
525
- filename = os.path.basename(file)
526
- if ".git" in filename:
527
- new_name = filename.replace(".git", ".gitx")
528
- target = os.path.join(folder, new_name)
529
- os.rename(file, target)
530
- renamed_items[target] = file
531
- for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
532
- foldername = os.path.basename(subfolder)
533
- if ".git" in foldername:
534
- new_name = foldername.replace(".git", ".gitx")
535
- subfolder2 = os.path.join(str(Path(subfolder).parent), new_name)
536
- os.rename(subfolder, subfolder2)
537
- renamed_items[subfolder2] = subfolder
538
- else:
539
- subfolder2 = subfolder
540
- self.__escape_git_repositories_in_folder_internal(subfolder2, renamed_items)
541
- return renamed_items
542
-
543
- def deescape_git_repositories_in_folder(self, renamed_items: dict[str, str]):
544
- for renamed_item, original_name in renamed_items.items():
545
- os.rename(renamed_item, original_name)
546
-
547
- def __sort_fmd(self, line: str):
548
- splitted: list = line.split(";")
549
- filetype: str = splitted[1]
550
- if filetype == "d":
551
- return -1
552
- if filetype == "f":
553
- return 1
554
- return 0
555
-
556
- @GeneralUtilities.check_arguments
557
- def restore_filemetadata(self, folder: str, source_file: str, strict=False, encoding: str = "utf-8", create_folder_is_not_exist: bool = True) -> None:
558
- lines = GeneralUtilities.read_lines_from_file(source_file, encoding)
559
- lines.sort(key=self.__sort_fmd)
560
- for line in lines:
561
- splitted: list = line.split(";")
562
- full_path_of_file_or_folder: str = os.path.join(folder, splitted[0])
563
- filetype: str = splitted[1]
564
- user: str = splitted[2]
565
- permissions: str = splitted[3]
566
- if filetype == "d" and create_folder_is_not_exist and not os.path.isdir(full_path_of_file_or_folder):
567
- GeneralUtilities.ensure_directory_exists(full_path_of_file_or_folder)
568
- if (filetype == "f" and os.path.isfile(full_path_of_file_or_folder)) or (filetype == "d" and os.path.isdir(full_path_of_file_or_folder)):
569
- self.set_owner(full_path_of_file_or_folder, user, os.name != 'nt')
570
- self.set_permission(full_path_of_file_or_folder, permissions)
571
- else:
572
- if strict:
573
- if filetype == "f":
574
- filetype_full = "File"
575
- if filetype == "d":
576
- filetype_full = "Directory"
577
- raise ValueError(f"{filetype_full} '{full_path_of_file_or_folder}' does not exist")
578
-
579
- @GeneralUtilities.check_arguments
580
- def __calculate_lengh_in_seconds(self, filename: str, folder: str) -> float:
581
- argument = ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', filename]
582
- result = self.run_program_argsasarray("ffprobe", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
583
- return float(result[1].replace('\n', ''))
584
-
585
- @GeneralUtilities.check_arguments
586
- def __create_thumbnails(self, filename: str, fps: str, folder: str, tempname_for_thumbnails: str) -> None:
587
- argument = ['-i', filename, '-r', str(fps), '-vf', 'scale=-1:120', '-vcodec', 'png', f'{tempname_for_thumbnails}-%002d.png']
588
- self.run_program_argsasarray("ffmpeg", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
589
-
590
- @GeneralUtilities.check_arguments
591
- def __create_thumbnail(self, outputfilename: str, folder: str, length_in_seconds: float, tempname_for_thumbnails: str, amount_of_images: int) -> None:
592
- duration = timedelta(seconds=length_in_seconds)
593
- info = GeneralUtilities.timedelta_to_simple_string(duration)
594
- next_square_number = str(int(math.sqrt(GeneralUtilities.get_next_square_number(amount_of_images))))
595
- argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{next_square_number}x{next_square_number}',
596
- f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
597
- self.run_program_argsasarray("montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
598
-
599
- @GeneralUtilities.check_arguments
600
- def roundup(self, x: float, places: int) -> int:
601
- d = 10 ** places
602
- if x < 0:
603
- return math.floor(x * d) / d
604
- else:
605
- return math.ceil(x * d) / d
606
-
607
- @GeneralUtilities.check_arguments
608
- def generate_thumbnail(self, file: str, frames_per_second: str, tempname_for_thumbnails: str = None) -> None:
609
- if tempname_for_thumbnails is None:
610
- tempname_for_thumbnails = "t"+str(uuid.uuid4())
611
-
612
- file = GeneralUtilities.resolve_relative_path_from_current_working_directory(file)
613
- filename = os.path.basename(file)
614
- folder = os.path.dirname(file)
615
- filename_without_extension = Path(file).stem
616
-
617
- try:
618
- length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
619
- if (frames_per_second.endswith("fps")):
620
- # frames per second, example: frames_per_second="20fps" => 20 frames per second
621
- x = self.roundup(float(frames_per_second[:-3]), 2)
622
- frames_per_secondx = str(x)
623
- amounf_of_previewframes = int(math.floor(length_in_seconds*x))
624
- else:
625
- # concrete amount of frame, examples: frames_per_second="16" => 16 frames for entire video
626
- amounf_of_previewframes = int(float(frames_per_second))
627
- frames_per_secondx = f"{amounf_of_previewframes-2}/{length_in_seconds}" # self.roundup((amounf_of_previewframes-2)/length_in_seconds, 2)
628
- self.__create_thumbnails(filename, frames_per_secondx, folder, tempname_for_thumbnails)
629
- self.__create_thumbnail(filename_without_extension, folder, length_in_seconds, tempname_for_thumbnails, amounf_of_previewframes)
630
- finally:
631
- for thumbnail_to_delete in Path(folder).rglob(tempname_for_thumbnails+"-*"):
632
- file = str(thumbnail_to_delete)
633
- os.remove(file)
634
-
635
- @GeneralUtilities.check_arguments
636
- def extract_pdf_pages(self, file: str, from_page: int, to_page: int, outputfile: str) -> None:
637
- pdf_reader = PyPDF2.PdfReader(file)
638
- pdf_writer = PyPDF2.PdfWriter()
639
- start = from_page
640
- end = to_page
641
- while start <= end:
642
- pdf_writer.add_page(pdf_reader.pages[start-1])
643
- start += 1
644
- with open(outputfile, 'wb') as out:
645
- pdf_writer.write(out)
646
-
647
- @GeneralUtilities.check_arguments
648
- def merge_pdf_files(self, files: list[str], outputfile: str) -> None:
649
- # TODO add wildcard-option
650
- pdfFileMerger = PyPDF2.PdfFileMerger()
651
- for file in files:
652
- pdfFileMerger.append(file.strip())
653
- pdfFileMerger.write(outputfile)
654
- pdfFileMerger.close()
655
-
656
- @GeneralUtilities.check_arguments
657
- def pdf_to_image(self, file: str, outputfilename_without_extension: str) -> None:
658
- raise ValueError("Function currently not available")
659
- # PyMuPDF can be used for that but sometimes it throws
660
- # "ImportError: DLL load failed while importing _fitz: Das angegebene Modul wurde nicht gefunden."
661
-
662
- # doc = None # fitz.open(file)
663
- # for i, page in enumerate(doc):
664
- # pix = page.get_pixmap()
665
- # img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
666
- # img.save(f"{outputfilename_without_extension}_{i}.png", "PNG")
667
-
668
- @GeneralUtilities.check_arguments
669
- def show_missing_files(self, folderA: str, folderB: str):
670
- for file in GeneralUtilities.get_missing_files(folderA, folderB):
671
- GeneralUtilities.write_message_to_stdout(file)
672
-
673
- @GeneralUtilities.check_arguments
674
- def SCCreateEmptyFileWithSpecificSize(self, name: str, size_string: str) -> int:
675
- if size_string.isdigit():
676
- size = int(size_string)
677
- else:
678
- if len(size_string) >= 3:
679
- if (size_string.endswith("kb")):
680
- size = int(size_string[:-2]) * pow(10, 3)
681
- elif (size_string.endswith("mb")):
682
- size = int(size_string[:-2]) * pow(10, 6)
683
- elif (size_string.endswith("gb")):
684
- size = int(size_string[:-2]) * pow(10, 9)
685
- elif (size_string.endswith("kib")):
686
- size = int(size_string[:-3]) * pow(2, 10)
687
- elif (size_string.endswith("mib")):
688
- size = int(size_string[:-3]) * pow(2, 20)
689
- elif (size_string.endswith("gib")):
690
- size = int(size_string[:-3]) * pow(2, 30)
691
- else:
692
- GeneralUtilities.write_message_to_stderr("Wrong format")
693
- else:
694
- GeneralUtilities.write_message_to_stderr("Wrong format")
695
- return 1
696
- with open(name, "wb") as f:
697
- f.seek(size-1)
698
- f.write(b"\0")
699
- return 0
700
-
701
- @GeneralUtilities.check_arguments
702
- def SCCreateHashOfAllFiles(self, folder: str) -> None:
703
- for file in GeneralUtilities.absolute_file_paths(folder):
704
- with open(file+".sha256", "w+", encoding="utf-8") as f:
705
- f.write(GeneralUtilities.get_sha256_of_file(file))
706
-
707
- @GeneralUtilities.check_arguments
708
- def SCCreateSimpleMergeWithoutRelease(self, repository: str, sourcebranch: str, targetbranch: str, remotename: str, remove_source_branch: bool) -> None:
709
- commitid = self.git_merge(repository, sourcebranch, targetbranch, False, True)
710
- self.git_merge(repository, targetbranch, sourcebranch, True, True)
711
- created_version = self.get_semver_version_from_gitversion(repository)
712
- self.git_create_tag(repository, commitid, f"v{created_version}", True)
713
- self.git_push(repository, remotename, targetbranch, targetbranch, False, True)
714
- if (GeneralUtilities.string_has_nonwhitespace_content(remotename)):
715
- self.git_push(repository, remotename, sourcebranch, sourcebranch, False, True)
716
- if (remove_source_branch):
717
- self.git_remove_branch(repository, sourcebranch)
718
-
719
- @GeneralUtilities.check_arguments
720
- def sc_organize_lines_in_file(self, file: str, encoding: str, sort: bool = False, remove_duplicated_lines: bool = False, ignore_first_line: bool = False,
721
- remove_empty_lines: bool = True, ignored_start_character: list = list()) -> int:
722
- if os.path.isfile(file):
723
-
724
- # read file
725
- lines = GeneralUtilities.read_lines_from_file(file, encoding)
726
- if (len(lines) == 0):
727
- return 0
728
-
729
- # store first line if desiredpopd
730
-
731
- if (ignore_first_line):
732
- first_line = lines.pop(0)
733
-
734
- # remove empty lines if desired
735
- if remove_empty_lines:
736
- temp = lines
737
- lines = []
738
- for line in temp:
739
- if (not (GeneralUtilities.string_is_none_or_whitespace(line))):
740
- lines.append(line)
741
-
742
- # remove duplicated lines if desired
743
- if remove_duplicated_lines:
744
- lines = GeneralUtilities.remove_duplicates(lines)
745
-
746
- # sort lines if desired
747
- if sort:
748
- lines = sorted(lines, key=lambda singleline: self.__adapt_line_for_sorting(singleline, ignored_start_character))
749
-
750
- # reinsert first line
751
- if ignore_first_line:
752
- lines.insert(0, first_line)
753
-
754
- # write result to file
755
- GeneralUtilities.write_lines_to_file(file, lines, encoding)
756
-
757
- return 0
758
- else:
759
- GeneralUtilities.write_message_to_stdout(f"File '{file}' does not exist")
760
- return 1
761
-
762
- @GeneralUtilities.check_arguments
763
- def __adapt_line_for_sorting(self, line: str, ignored_start_characters: list):
764
- result = line.lower()
765
- while len(result) > 0 and result[0] in ignored_start_characters:
766
- result = result[1:]
767
- return result
768
-
769
- @GeneralUtilities.check_arguments
770
- def SCGenerateSnkFiles(self, outputfolder, keysize=4096, amountofkeys=10) -> int:
771
- GeneralUtilities.ensure_directory_exists(outputfolder)
772
- for _ in range(amountofkeys):
773
- file = os.path.join(outputfolder, str(uuid.uuid4())+".snk")
774
- argument = f"-k {keysize} {file}"
775
- self.run_program("sn", argument, outputfolder)
776
-
777
- @GeneralUtilities.check_arguments
778
- def __merge_files(self, sourcefile: str, targetfile: str) -> None:
779
- with open(sourcefile, "rb") as f:
780
- source_data = f.read()
781
- with open(targetfile, "ab") as fout:
782
- merge_separator = [0x0A]
783
- fout.write(bytes(merge_separator))
784
- fout.write(source_data)
785
-
786
- @GeneralUtilities.check_arguments
787
- def __process_file(self, file: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
788
- new_filename = os.path.join(os.path.dirname(file), os.path.basename(file).replace(substringInFilename, newSubstringInFilename))
789
- if file != new_filename:
790
- if os.path.isfile(new_filename):
791
- if filecmp.cmp(file, new_filename):
792
- send2trash.send2trash(file)
793
- else:
794
- if conflictResolveMode == "ignore":
795
- pass
796
- elif conflictResolveMode == "preservenewest":
797
- if (os.path.getmtime(file) - os.path.getmtime(new_filename) > 0):
798
- send2trash.send2trash(file)
799
- else:
800
- send2trash.send2trash(new_filename)
801
- os.rename(file, new_filename)
802
- elif (conflictResolveMode == "merge"):
803
- self.__merge_files(file, new_filename)
804
- send2trash.send2trash(file)
805
- else:
806
- raise ValueError('Unknown conflict resolve mode')
807
- else:
808
- os.rename(file, new_filename)
809
-
810
- @GeneralUtilities.check_arguments
811
- def SCReplaceSubstringsInFilenames(self, folder: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
812
- for file in GeneralUtilities.absolute_file_paths(folder):
813
- self.__process_file(file, substringInFilename, newSubstringInFilename, conflictResolveMode)
814
-
815
- @GeneralUtilities.check_arguments
816
- def __check_file(self, file: str, searchstring: str) -> None:
817
- bytes_ascii = bytes(searchstring, "ascii")
818
- bytes_utf16 = bytes(searchstring, "utf-16") # often called "unicode-encoding"
819
- bytes_utf8 = bytes(searchstring, "utf-8")
820
- with open(file, mode='rb') as file_object:
821
- content = file_object.read()
822
- if bytes_ascii in content:
823
- GeneralUtilities.write_message_to_stdout(file)
824
- elif bytes_utf16 in content:
825
- GeneralUtilities.write_message_to_stdout(file)
826
- elif bytes_utf8 in content:
827
- GeneralUtilities.write_message_to_stdout(file)
828
-
829
- @GeneralUtilities.check_arguments
830
- def SCSearchInFiles(self, folder: str, searchstring: str) -> None:
831
- for file in GeneralUtilities.absolute_file_paths(folder):
832
- self.__check_file(file, searchstring)
833
-
834
- @GeneralUtilities.check_arguments
835
- def __print_qr_code_by_csv_line(self, displayname: str, website: str, emailaddress: str, key: str, period: str) -> None:
836
- qrcode_content = f"otpauth://totp/{website}:{emailaddress}?secret={key}&issuer={displayname}&period={period}"
837
- GeneralUtilities.write_message_to_stdout(f"{displayname} ({emailaddress}):")
838
- GeneralUtilities.write_message_to_stdout(qrcode_content)
839
- qr = qrcode.QRCode()
840
- qr.add_data(qrcode_content)
841
- f = io.StringIO()
842
- qr.print_ascii(out=f)
843
- f.seek(0)
844
- GeneralUtilities.write_message_to_stdout(f.read())
845
-
846
- @GeneralUtilities.check_arguments
847
- def SCShow2FAAsQRCode(self, csvfile: str) -> None:
848
- separator_line = "--------------------------------------------------------"
849
- lines = GeneralUtilities.read_csv_file(csvfile, True)
850
- lines.sort(key=lambda items: ''.join(items).lower())
851
- for line in lines:
852
- GeneralUtilities.write_message_to_stdout(separator_line)
853
- self.__print_qr_code_by_csv_line(line[0], line[1], line[2], line[3], line[4])
854
- GeneralUtilities.write_message_to_stdout(separator_line)
855
-
856
- @GeneralUtilities.check_arguments
857
- def SCUploadFileToFileHost(self, file: str, host: str) -> int:
858
- try:
859
- GeneralUtilities.write_message_to_stdout(self.upload_file_to_file_host(file, host))
860
- return 0
861
- except Exception as exception:
862
- GeneralUtilities.write_exception_to_stderr_with_traceback(exception, traceback)
863
- return 1
864
-
865
- @GeneralUtilities.check_arguments
866
- def SCFileIsAvailableOnFileHost(self, file: str) -> int:
867
- try:
868
- if self.file_is_available_on_file_host(file):
869
- GeneralUtilities.write_message_to_stdout(f"'{file}' is available")
870
- return 0
871
- else:
872
- GeneralUtilities.write_message_to_stdout(f"'{file}' is not available")
873
- return 1
874
- except Exception as exception:
875
- GeneralUtilities.write_exception_to_stderr_with_traceback(exception, traceback)
876
- return 2
877
-
878
- @GeneralUtilities.check_arguments
879
- def SCCalculateBitcoinBlockHash(self, block_version_number: str, previousblockhash: str, transactionsmerkleroot: str, timestamp: str, target: str, nonce: str) -> str:
880
- # Example-values:
881
- # block_version_number: "00000020"
882
- # previousblockhash: "66720b99e07d284bd4fe67ff8c49a5db1dd8514fcdab61000000000000000000"
883
- # transactionsmerkleroot: "7829844f4c3a41a537b3131ca992643eaa9d093b2383e4cdc060ad7dc5481187"
884
- # timestamp: "51eb505a"
885
- # target: "c1910018"
886
- # nonce: "de19b302"
887
- header = str(block_version_number + previousblockhash + transactionsmerkleroot + timestamp + target + nonce)
888
- return binascii.hexlify(hashlib.sha256(hashlib.sha256(binascii.unhexlify(header)).digest()).digest()[::-1]).decode('utf-8')
889
-
890
- @GeneralUtilities.check_arguments
891
- def SCChangeHashOfProgram(self, inputfile: str) -> None:
892
- valuetoappend = str(uuid.uuid4())
893
-
894
- outputfile = inputfile + '.modified'
895
-
896
- shutil.copy2(inputfile, outputfile)
897
- with open(outputfile, 'a', encoding="utf-8") as file:
898
- # TODO use rcedit for .exe-files instead of appending valuetoappend ( https://github.com/electron/rcedit/ )
899
- # background: you can retrieve the "original-filename" from the .exe-file like discussed here:
900
- # https://security.stackexchange.com/questions/210843/ is-it-possible-to-change-original-filename-of-an-exe
901
- # so removing the original filename with rcedit is probably a better way to make it more difficult to detect the programname.
902
- # this would obviously also change the hashvalue of the program so appending a whitespace is not required anymore.
903
- file.write(valuetoappend)
904
-
905
- @GeneralUtilities.check_arguments
906
- def __adjust_folder_name(self, folder: str) -> str:
907
- result = os.path.dirname(folder).replace("\\", "/")
908
- if result == "/":
909
- return ""
910
- else:
911
- return result
912
-
913
- @GeneralUtilities.check_arguments
914
- def __create_iso(self, folder, iso_file) -> None:
915
- created_directories = []
916
- files_directory = "FILES"
917
- iso = pycdlib.PyCdlib()
918
- iso.new()
919
- files_directory = files_directory.upper()
920
- iso.add_directory("/" + files_directory)
921
- created_directories.append("/" + files_directory)
922
- for root, _, files in os.walk(folder):
923
- for file in files:
924
- full_path = os.path.join(root, file)
925
- with (open(full_path, "rb").read()) as text_io_wrapper:
926
- content = text_io_wrapper
927
- path_in_iso = '/' + files_directory + self.__adjust_folder_name(full_path[len(folder)::1]).upper()
928
- if path_in_iso not in created_directories:
929
- iso.add_directory(path_in_iso)
930
- created_directories.append(path_in_iso)
931
- iso.add_fp(BytesIO(content), len(content), path_in_iso + '/' + file.upper() + ';1')
932
- iso.write(iso_file)
933
- iso.close()
934
-
935
- @GeneralUtilities.check_arguments
936
- def SCCreateISOFileWithObfuscatedFiles(self, inputfolder: str, outputfile: str, printtableheadline, createisofile, extensions) -> None:
937
- if (os.path.isdir(inputfolder)):
938
- namemappingfile = "name_map.csv"
939
- files_directory = inputfolder
940
- files_directory_obf = files_directory + "_Obfuscated"
941
- self.SCObfuscateFilesFolder(inputfolder, printtableheadline, namemappingfile, extensions)
942
- os.rename(namemappingfile, os.path.join(files_directory_obf, namemappingfile))
943
- if createisofile:
944
- self.__create_iso(files_directory_obf, outputfile)
945
- shutil.rmtree(files_directory_obf)
946
- else:
947
- raise ValueError(f"Directory not found: '{inputfolder}'")
948
-
949
- @GeneralUtilities.check_arguments
950
- def SCFilenameObfuscator(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
951
- obfuscate_all_files = extensions == "*"
952
- if (not obfuscate_all_files):
953
- obfuscate_file_extensions = extensions.split(",")
954
-
955
- if (os.path.isdir(inputfolder)):
956
- printtableheadline = GeneralUtilities.string_to_boolean(printtableheadline)
957
- files = []
958
- if not os.path.isfile(namemappingfile):
959
- with open(namemappingfile, "a", encoding="utf-8"):
960
- pass
961
- if printtableheadline:
962
- GeneralUtilities.append_line_to_file(namemappingfile, "Original filename;new filename;SHA2-hash of file")
963
- for file in GeneralUtilities.absolute_file_paths(inputfolder):
964
- if os.path.isfile(os.path.join(inputfolder, file)):
965
- if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
966
- files.append(file)
967
- for file in files:
968
- hash_value = GeneralUtilities.get_sha256_of_file(file)
969
- extension = Path(file).suffix
970
- new_file_name_without_path = str(uuid.uuid4())[0:8] + extension
971
- new_file_name = os.path.join(os.path.dirname(file), new_file_name_without_path)
972
- os.rename(file, new_file_name)
973
- GeneralUtilities.append_line_to_file(namemappingfile, os.path.basename(file) + ";" + new_file_name_without_path + ";" + hash_value)
974
- else:
975
- raise ValueError(f"Directory not found: '{inputfolder}'")
976
-
977
- @GeneralUtilities.check_arguments
978
- def __extension_matchs(self, file: str, obfuscate_file_extensions) -> bool:
979
- for extension in obfuscate_file_extensions:
980
- if file.lower().endswith("."+extension.lower()):
981
- return True
982
- return False
983
-
984
- @GeneralUtilities.check_arguments
985
- def SCHealthcheck(self, file: str) -> int:
986
- lines = GeneralUtilities.read_lines_from_file(file)
987
- for line in reversed(lines):
988
- if not GeneralUtilities.string_is_none_or_whitespace(line):
989
- if "RunningHealthy (" in line: # TODO use regex
990
- GeneralUtilities.write_message_to_stderr(f"Healthy running due to line '{line}' in file '{file}'.")
991
- return 0
992
- else:
993
- GeneralUtilities.write_message_to_stderr(f"Not healthy running due to line '{line}' in file '{file}'.")
994
- return 1
995
- GeneralUtilities.write_message_to_stderr(f"No valid line found for healthycheck in file '{file}'.")
996
- return 2
997
-
998
- @GeneralUtilities.check_arguments
999
- def SCObfuscateFilesFolder(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
1000
- obfuscate_all_files = extensions == "*"
1001
- if (not obfuscate_all_files):
1002
- if "," in extensions:
1003
- obfuscate_file_extensions = extensions.split(",")
1004
- else:
1005
- obfuscate_file_extensions = [extensions]
1006
- newd = inputfolder+"_Obfuscated"
1007
- shutil.copytree(inputfolder, newd)
1008
- inputfolder = newd
1009
- if (os.path.isdir(inputfolder)):
1010
- for file in GeneralUtilities.absolute_file_paths(inputfolder):
1011
- if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
1012
- self.SCChangeHashOfProgram(file)
1013
- os.remove(file)
1014
- os.rename(file + ".modified", file)
1015
- self.SCFilenameObfuscator(inputfolder, printtableheadline, namemappingfile, extensions)
1016
- else:
1017
- raise ValueError(f"Directory not found: '{inputfolder}'")
1018
-
1019
- @GeneralUtilities.check_arguments
1020
- def get_docker_debian_version(self, image_tag: str) -> str:
1021
- result = ScriptCollectionCore().run_program_argsasarray(
1022
- "docker", ['run', f'debian:{image_tag}', 'bash', '-c', 'apt-get -y update && apt-get -y install lsb-release && lsb_release -cs'])
1023
- result_line = GeneralUtilities.string_to_lines(result[1])[-2]
1024
- return result_line
1025
-
1026
- @GeneralUtilities.check_arguments
1027
- def get_latest_tor_version_of_debian_repository(self, debian_version: str) -> str:
1028
- package_url: str = f"https://deb.torproject.org/torproject.org/dists/{debian_version}/main/binary-amd64/Packages"
1029
- r = requests.get(package_url, timeout=5)
1030
- if r.status_code != 200:
1031
- raise ValueError(f"Checking for latest tor package resulted in HTTP-response-code {r.status_code}.")
1032
- lines = GeneralUtilities.string_to_lines(GeneralUtilities.bytes_to_string(r.content))
1033
- version_line_prefix = "Version: "
1034
- version_content_line = [line for line in lines if line.startswith(version_line_prefix)][1]
1035
- version_with_overhead = version_content_line[len(version_line_prefix):]
1036
- tor_version = version_with_overhead.split("~")[0]
1037
- return tor_version
1038
-
1039
- @GeneralUtilities.check_arguments
1040
- def upload_file_to_file_host(self, file: str, host: str) -> int:
1041
- if (host is None):
1042
- return self.upload_file_to_random_filesharing_service(file)
1043
- elif host == "anonfiles.com":
1044
- return self.upload_file_to_anonfiles(file)
1045
- elif host == "bayfiles.com":
1046
- return self.upload_file_to_bayfiles(file)
1047
- GeneralUtilities.write_message_to_stderr("Unknown host: "+host)
1048
- return 1
1049
-
1050
- @GeneralUtilities.check_arguments
1051
- def upload_file_to_random_filesharing_service(self, file: str) -> int:
1052
- host = randrange(2)
1053
- if host == 0:
1054
- return self.upload_file_to_anonfiles(file)
1055
- if host == 1:
1056
- return self.upload_file_to_bayfiles(file)
1057
- return 1
1058
-
1059
- @GeneralUtilities.check_arguments
1060
- def upload_file_to_anonfiles(self, file) -> int:
1061
- return self.upload_file_by_using_simple_curl_request("https://api.anonfiles.com/upload", file)
1062
-
1063
- @GeneralUtilities.check_arguments
1064
- def upload_file_to_bayfiles(self, file) -> int:
1065
- return self.upload_file_by_using_simple_curl_request("https://api.bayfiles.com/upload", file)
1066
-
1067
- @GeneralUtilities.check_arguments
1068
- def upload_file_by_using_simple_curl_request(self, api_url: str, file: str) -> int:
1069
- # TODO implement
1070
- return 1
1071
-
1072
- @GeneralUtilities.check_arguments
1073
- def file_is_available_on_file_host(self, file) -> int:
1074
- # TODO implement
1075
- return 1
1076
-
1077
- def run_testcases_for_python_project(self, repository_folder: str):
1078
- self.run_program("coverage", "run -m pytest", repository_folder)
1079
- self.run_program("coverage", "xml", repository_folder)
1080
- GeneralUtilities.ensure_directory_exists(os.path.join(repository_folder, "Other/TestCoverage"))
1081
- coveragefile = os.path.join(repository_folder, "Other/TestCoverage/TestCoverage.xml")
1082
- GeneralUtilities.ensure_file_does_not_exist(coveragefile)
1083
- os.rename(os.path.join(repository_folder, "coverage.xml"), coveragefile)
1084
-
1085
- @GeneralUtilities.check_arguments
1086
- def get_file_permission(self, file: str) -> str:
1087
- """This function returns an usual octet-triple, for example "0700"."""
1088
- ls_output = self.__ls(file)
1089
- return self.__get_file_permission_helper(ls_output)
1090
-
1091
- @GeneralUtilities.check_arguments
1092
- def __get_file_permission_helper(self, ls_output: str) -> str:
1093
- permissions = ' '.join(ls_output.split()).split(' ')[0][1:]
1094
- return str(self.__to_octet(permissions[0:3]))+str(self.__to_octet(permissions[3:6]))+str(self.__to_octet(permissions[6:9]))
1095
-
1096
- @GeneralUtilities.check_arguments
1097
- def __to_octet(self, string: str) -> int:
1098
- return int(self.__to_octet_helper(string[0])+self.__to_octet_helper(string[1])+self.__to_octet_helper(string[2]), 2)
1099
-
1100
- @GeneralUtilities.check_arguments
1101
- def __to_octet_helper(self, string: str) -> str:
1102
- if (string == "-"):
1103
- return "0"
1104
- else:
1105
- return "1"
1106
-
1107
- @GeneralUtilities.check_arguments
1108
- def get_file_owner(self, file: str) -> str:
1109
- """This function returns the user and the group in the format "user:group"."""
1110
- ls_output = self.__ls(file)
1111
- return self.__get_file_owner_helper(ls_output)
1112
-
1113
- @GeneralUtilities.check_arguments
1114
- def __get_file_owner_helper(self, ls_output: str) -> str:
1115
- try:
1116
- splitted = ' '.join(ls_output.split()).split(' ')
1117
- return f"{splitted[2]}:{splitted[3]}"
1118
- except Exception as exception:
1119
- raise ValueError(f"ls-output '{ls_output}' not parsable") from exception
1120
-
1121
- @GeneralUtilities.check_arguments
1122
- def get_file_owner_and_file_permission(self, file: str) -> str:
1123
- ls_output = self.__ls(file)
1124
- return [self.__get_file_owner_helper(ls_output), self.__get_file_permission_helper(ls_output)]
1125
-
1126
- @GeneralUtilities.check_arguments
1127
- def __ls(self, file: str) -> str:
1128
- file = file.replace("\\", "/")
1129
- GeneralUtilities.assert_condition(os.path.isfile(file) or os.path.isdir(file), f"Can not execute 'ls' because '{file}' does not exist")
1130
- result = self.run_program_argsasarray("ls", ["-ld", file])
1131
- GeneralUtilities.assert_condition(result[0] == 0, f"'ls -ld {file}' resulted in exitcode {str(result[0])}. StdErr: {result[2]}")
1132
- GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(result[1]), f"'ls' of '{file}' had an empty output. StdErr: '{result[2]}'")
1133
- return result[1]
1134
-
1135
- @GeneralUtilities.check_arguments
1136
- def set_permission(self, file_or_folder: str, permissions: str, recursive: bool = False) -> None:
1137
- """This function expects an usual octet-triple, for example "700"."""
1138
- args = []
1139
- if recursive:
1140
- args.append("--recursive")
1141
- args.append(permissions)
1142
- args.append(file_or_folder)
1143
- self.run_program_argsasarray("chmod", args)
1144
-
1145
- @GeneralUtilities.check_arguments
1146
- def set_owner(self, file_or_folder: str, owner: str, recursive: bool = False, follow_symlinks: bool = False) -> None:
1147
- """This function expects the user and the group in the format "user:group"."""
1148
- args = []
1149
- if recursive:
1150
- args.append("--recursive")
1151
- if follow_symlinks:
1152
- args.append("--no-dereference")
1153
- args.append(owner)
1154
- args.append(file_or_folder)
1155
- self.run_program_argsasarray("chown", args)
1156
-
1157
- # <run programs>
1158
-
1159
- @GeneralUtilities.check_arguments
1160
- def __run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1,
1161
- print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False,
1162
- title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None) -> Popen:
1163
- # Verbosity:
1164
- # 0=Quiet (No output will be printed.)
1165
- # 1=Normal (If the exitcode of the executed program is not 0 then the StdErr will be printed.)
1166
- # 2=Full (Prints StdOut and StdErr of the executed program.)
1167
- # 3=Verbose (Same as "Full" but with some more information.)
1168
-
1169
- if arguments_for_log is None:
1170
- arguments_for_log = ' '.join(arguments_as_array)
1171
- else:
1172
- arguments_for_log = ' '.join(arguments_for_log)
1173
- working_directory = self.__adapt_workingdirectory(working_directory)
1174
- cmd = f'{working_directory}>{program} {arguments_for_log}'
1175
-
1176
- if GeneralUtilities.string_is_none_or_whitespace(title):
1177
- info_for_log = cmd
1178
- else:
1179
- info_for_log = title
1180
-
1181
- if verbosity >= 3:
1182
- GeneralUtilities.write_message_to_stdout(f"Run '{info_for_log}'.")
1183
-
1184
- if isinstance(self.program_runner, ProgramRunnerEpew):
1185
- custom_argument = CustomEpewArgument(print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, verbosity, arguments_for_log)
1186
- popen: Popen = self.program_runner.run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, custom_argument)
1187
- return popen
1188
-
1189
- # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
1190
-
1191
- @GeneralUtilities.check_arguments
1192
- def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1,
1193
- print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False,
1194
- title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None,
1195
- throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None) -> tuple[int, str, str, int]:
1196
- mock_loader_result = self.__try_load_mock(program, ' '.join(arguments_as_array), working_directory)
1197
- if mock_loader_result[0]:
1198
- return mock_loader_result[1]
1199
-
1200
- if arguments_for_log is None:
1201
- arguments_for_log = arguments_as_array
1202
-
1203
- arguments_for_log_as_string = ' '.join(arguments_for_log)
1204
- cmd = f'{working_directory}>{program} {arguments_for_log_as_string}'
1205
- if GeneralUtilities.string_is_none_or_whitespace(title):
1206
- info_for_log = cmd
1207
- else:
1208
- info_for_log = title
1209
-
1210
- epew_will_be_used = isinstance(self.program_runner, ProgramRunnerEpew)
1211
- program_manages_output_itself = epew_will_be_used and False # TODO fix stdout-/stderr-reading-block below
1212
- program_manages_logging_itself = epew_will_be_used
1213
-
1214
- process = self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, verbosity, print_errors_as_information, log_file,
1215
- timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument)
1216
- pid = process.pid
1217
-
1218
- if program_manages_output_itself:
1219
- stdout_readable = process.stdout.readable()
1220
- stderr_readable = process.stderr.readable()
1221
- while stdout_readable or stderr_readable:
1222
-
1223
- if stdout_readable:
1224
- stdout_line = GeneralUtilities.bytes_to_string(process.stdout.readline()).strip()
1225
- if (len(stdout_line)) > 0:
1226
- GeneralUtilities.write_message_to_stdout(stdout_line)
1227
-
1228
- if stderr_readable:
1229
- stderr_line = GeneralUtilities.bytes_to_string(process.stderr.readline()).strip()
1230
- if (len(stderr_line)) > 0:
1231
- GeneralUtilities.write_message_to_stderr(stderr_line)
1232
-
1233
- stdout_readable = process.stdout.readable()
1234
- stderr_readable = process.stderr.readable()
1235
-
1236
- stdout, stderr = process.communicate()
1237
- exit_code = process.wait()
1238
- stdout = GeneralUtilities.bytes_to_string(stdout).replace('\r', '')
1239
- stderr = GeneralUtilities.bytes_to_string(stderr).replace('\r', '')
1240
-
1241
- if arguments_for_log_as_string is None:
1242
- arguments_for_log_as_string = ' '.join(arguments_as_array)
1243
- else:
1244
- arguments_for_log_as_string = ' '.join(arguments_for_log)
1245
-
1246
- if GeneralUtilities.string_is_none_or_whitespace(title):
1247
- info_for_log = cmd
1248
- else:
1249
- info_for_log = title
1250
-
1251
- if not program_manages_logging_itself and log_file is not None:
1252
- GeneralUtilities.ensure_file_exists(log_file)
1253
- GeneralUtilities.append_line_to_file(log_file, stdout)
1254
- GeneralUtilities.append_line_to_file(log_file, stderr)
1255
-
1256
- if not program_manages_output_itself:
1257
- if verbosity == 1 and exit_code != 0:
1258
- self.__write_error_output(print_errors_as_information, stderr)
1259
- if verbosity == 2:
1260
- GeneralUtilities.write_message_to_stdout(stdout)
1261
- self.__write_error_output(print_errors_as_information, stderr)
1262
- if verbosity == 3:
1263
- GeneralUtilities.write_message_to_stdout(stdout)
1264
- self.__write_error_output(print_errors_as_information, stderr)
1265
- formatted = self.__format_program_execution_information(title=info_for_log, program=program,
1266
- argument=arguments_for_log_as_string, workingdirectory=working_directory)
1267
- GeneralUtilities.write_message_to_stdout(f"Finished '{info_for_log}'. Details: '{formatted}")
1268
-
1269
- if throw_exception_if_exitcode_is_not_zero and exit_code != 0:
1270
- arguments_for_log_as_string = ' '.join(arguments_for_log)
1271
- raise ValueError(f"Program '{working_directory}>{program} {arguments_for_log_as_string}' resulted in exitcode {exit_code}. (StdOut: '{stdout}', StdErr: '{stderr}')")
1272
-
1273
- result = (exit_code, stdout, stderr, pid)
1274
- return result
1275
-
1276
- # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
1277
- @GeneralUtilities.check_arguments
1278
- def run_program(self, program: str, arguments: str = "", working_directory: str = None, verbosity: int = 1,
1279
- print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False,
1280
- title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True,
1281
- custom_argument: object = None) -> tuple[int, str, str, int]:
1282
- return self.run_program_argsasarray(program, GeneralUtilities.arguments_to_array(arguments), working_directory, verbosity, print_errors_as_information,
1283
- log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument)
1284
-
1285
- # Return-values program_runner: Pid
1286
- @GeneralUtilities.check_arguments
1287
- def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1,
1288
- print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False,
1289
- title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None) -> int:
1290
- mock_loader_result = self.__try_load_mock(program, ' '.join(arguments_as_array), working_directory)
1291
- if mock_loader_result[0]:
1292
- return mock_loader_result[1]
1293
-
1294
- process: Popen = self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, verbosity,
1295
- print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument)
1296
- return process.pid
1297
-
1298
- # Return-values program_runner: Pid
1299
- @GeneralUtilities.check_arguments
1300
- def run_program_async(self, program: str, arguments: str = "", working_directory: str = None, verbosity: int = 1,
1301
- print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False,
1302
- title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None) -> int:
1303
- return self.run_program_argsasarray_async(program, GeneralUtilities.arguments_to_array(arguments), working_directory, verbosity,
1304
- print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument)
1305
-
1306
- @GeneralUtilities.check_arguments
1307
- def __try_load_mock(self, program: str, arguments: str, working_directory: str) -> tuple[bool, tuple[int, str, str, int]]:
1308
- if self.mock_program_calls:
1309
- try:
1310
- return [True, self.__get_mock_program_call(program, arguments, working_directory)]
1311
- except LookupError:
1312
- if not self.execute_program_really_if_no_mock_call_is_defined:
1313
- raise
1314
- return [False, None]
1315
-
1316
- @GeneralUtilities.check_arguments
1317
- def __adapt_workingdirectory(self, workingdirectory: str) -> str:
1318
- if workingdirectory is None:
1319
- return os.getcwd()
1320
- else:
1321
- return GeneralUtilities.resolve_relative_path_from_current_working_directory(workingdirectory)
1322
-
1323
- @GeneralUtilities.check_arguments
1324
- def __write_error_output(self, print_errors_as_information: bool, stderr: str):
1325
- if print_errors_as_information:
1326
- GeneralUtilities.write_message_to_stdout(stderr)
1327
- else:
1328
- GeneralUtilities.write_message_to_stderr(stderr)
1329
-
1330
- @GeneralUtilities.check_arguments
1331
- def __format_program_execution_information(self, exitcode: int = None, stdout: str = None, stderr: str = None, program: str = None, argument: str = None,
1332
- workingdirectory: str = None, title: str = None, pid: int = None, execution_duration: timedelta = None):
1333
- result = ""
1334
- if (exitcode is not None and stdout is not None and stderr is not None):
1335
- result = f"{result} Exitcode: {exitcode}; StdOut: '{stdout}'; StdErr: '{stderr}'"
1336
- if (pid is not None):
1337
- result = f"Pid: '{pid}'; {result}"
1338
- if (program is not None and argument is not None and workingdirectory is not None):
1339
- result = f"Command: '{workingdirectory}> {program} {argument}'; {result}"
1340
- if (execution_duration is not None):
1341
- result = f"{result}; Duration: '{str(execution_duration)}'"
1342
- if (title is not None):
1343
- result = f"Title: '{title}'; {result}"
1344
- return result.strip()
1345
-
1346
- @GeneralUtilities.check_arguments
1347
- def verify_no_pending_mock_program_calls(self):
1348
- if (len(self.__mocked_program_calls) > 0):
1349
- raise AssertionError(
1350
- "The following mock-calls were not called:\n"+",\n ".join([self.__format_mock_program_call(r) for r in self.__mocked_program_calls]))
1351
-
1352
- @GeneralUtilities.check_arguments
1353
- def __format_mock_program_call(self, r) -> str:
1354
- r: ScriptCollectionCore.__MockProgramCall = r
1355
- return f"'{r.workingdirectory}>{r.program} {r.argument}' (" \
1356
- f"exitcode: {GeneralUtilities.str_none_safe(str(r.exit_code))}, " \
1357
- f"pid: {GeneralUtilities.str_none_safe(str(r.pid))}, "\
1358
- f"stdout: {GeneralUtilities.str_none_safe(str(r.stdout))}, " \
1359
- f"stderr: {GeneralUtilities.str_none_safe(str(r.stderr))})"
1360
-
1361
- @GeneralUtilities.check_arguments
1362
- def register_mock_program_call(self, program: str, argument: str, workingdirectory: str, result_exit_code: int, result_stdout: str, result_stderr: str,
1363
- result_pid: int, amount_of_expected_calls=1):
1364
- "This function is for test-purposes only"
1365
- for _ in itertools.repeat(None, amount_of_expected_calls):
1366
- mock_call = ScriptCollectionCore.__MockProgramCall()
1367
- mock_call.program = program
1368
- mock_call.argument = argument
1369
- mock_call.workingdirectory = workingdirectory
1370
- mock_call.exit_code = result_exit_code
1371
- mock_call.stdout = result_stdout
1372
- mock_call.stderr = result_stderr
1373
- mock_call.pid = result_pid
1374
- self.__mocked_program_calls.append(mock_call)
1375
-
1376
- @GeneralUtilities.check_arguments
1377
- def __get_mock_program_call(self, program: str, argument: str, workingdirectory: str):
1378
- result: ScriptCollectionCore.__MockProgramCall = None
1379
- for mock_call in self.__mocked_program_calls:
1380
- if ((re.match(mock_call.program, program) is not None)
1381
- and (re.match(mock_call.argument, argument) is not None)
1382
- and (re.match(mock_call.workingdirectory, workingdirectory) is not None)):
1383
- result = mock_call
1384
- break
1385
- if result is None:
1386
- raise LookupError(f"Tried to execute mock-call '{workingdirectory}>{program} {argument}' but no mock-call was defined for that execution")
1387
- else:
1388
- self.__mocked_program_calls.remove(result)
1389
- return (result.exit_code, result.stdout, result.stderr, result.pid)
1390
-
1391
- @GeneralUtilities.check_arguments
1392
- class __MockProgramCall:
1393
- program: str
1394
- argument: str
1395
- workingdirectory: str
1396
- exit_code: int
1397
- stdout: str
1398
- stderr: str
1399
- pid: int
1400
-
1401
- # </run programs>
1402
-
1403
- @GeneralUtilities.check_arguments
1404
- def extract_archive_with_7z(self, unzip_program_file: str, zipfile: str, password: str, output_directory: str) -> None:
1405
- password_set = not password is None
1406
- file_name = Path(zipfile).name
1407
- file_folder = os.path.dirname(zipfile)
1408
- argument = "x"
1409
- if password_set:
1410
- argument = f"{argument} -p\"{password}\""
1411
- argument = f"{argument} -o {output_directory}"
1412
- argument = f"{argument} {file_name}"
1413
- return self.run_program(unzip_program_file, argument, file_folder)
1414
-
1415
- @GeneralUtilities.check_arguments
1416
- def get_internet_time(self) -> datetime:
1417
- response = ntplib.NTPClient().request('pool.ntp.org')
1418
- return datetime.fromtimestamp(response.tx_time)
1419
-
1420
- @GeneralUtilities.check_arguments
1421
- def system_time_equals_internet_time(self, maximal_tolerance_difference: timedelta) -> bool:
1422
- return abs(datetime.now() - self.get_internet_time()) < maximal_tolerance_difference
1423
-
1424
- @GeneralUtilities.check_arguments
1425
- def system_time_equals_internet_time_with_default_tolerance(self) -> bool:
1426
- return self.system_time_equals_internet_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
1427
-
1428
- @GeneralUtilities.check_arguments
1429
- def check_system_time(self, maximal_tolerance_difference: timedelta):
1430
- if not self.system_time_equals_internet_time(maximal_tolerance_difference):
1431
- raise ValueError("System time may be wrong")
1432
-
1433
- @GeneralUtilities.check_arguments
1434
- def check_system_time_with_default_tolerance(self) -> None:
1435
- self.check_system_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
1436
-
1437
- @GeneralUtilities.check_arguments
1438
- def __get_default_tolerance_for_system_time_equals_internet_time(self) -> timedelta:
1439
- return timedelta(hours=0, minutes=0, seconds=3)
1440
-
1441
- @GeneralUtilities.check_arguments
1442
- def increment_version(self, input_version: str, increment_major: bool, increment_minor: bool, increment_patch: bool) -> str:
1443
- splitted = input_version.split(".")
1444
- GeneralUtilities.assert_condition(len(splitted) == 3, f"Version '{input_version}' does not have the 'major.minor.patch'-pattern.")
1445
- major = int(splitted[0])
1446
- minor = int(splitted[1])
1447
- patch = int(splitted[2])
1448
- if increment_major:
1449
- major = major+1
1450
- if increment_minor:
1451
- minor = minor+1
1452
- if increment_patch:
1453
- patch = patch+1
1454
- return f"{major}.{minor}.{patch}"
1455
-
1456
- @GeneralUtilities.check_arguments
1457
- def get_semver_version_from_gitversion(self, repository_folder: str) -> str:
1458
- result = self.get_version_from_gitversion(repository_folder, "MajorMinorPatch")
1459
-
1460
- if self.git_repository_has_uncommitted_changes(repository_folder):
1461
- if self.get_current_branch_has_tag(repository_folder):
1462
- id_of_latest_tag = self.git_get_commitid_of_tag(repository_folder, self.get_latest_tag(repository_folder))
1463
- current_commit = self.git_get_commit_id(repository_folder)
1464
- current_commit_is_on_latest_tag = id_of_latest_tag == current_commit
1465
- if current_commit_is_on_latest_tag:
1466
- result = self.increment_version(result, False, False, True)
1467
-
1468
- return result
1469
-
1470
- @staticmethod
1471
- @GeneralUtilities.check_arguments
1472
- def is_patch_version(version_string: str) -> bool:
1473
- return not version_string.endswith(".0")
1474
-
1475
- @GeneralUtilities.check_arguments
1476
- def get_version_from_gitversion(self, folder: str, variable: str) -> str:
1477
- # called twice as workaround for issue 1877 in gitversion ( https://github.com/GitTools/GitVersion/issues/1877 )
1478
- result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder)
1479
- result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder)
1480
- result = GeneralUtilities.strip_new_line_character(result[1])
1481
-
1482
- return result
1483
-
1484
- @GeneralUtilities.check_arguments
1485
- def generate_certificate_authority(self, folder: str, name: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str,
1486
- days_until_expire: int = None, password: str = None) -> None:
1487
- if days_until_expire is None:
1488
- days_until_expire = 1825
1489
- if password is None:
1490
- password = GeneralUtilities.generate_password()
1491
- self.run_program("openssl", f'req -new -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -days {days_until_expire} -nodes -x509 -subj ' +
1492
- f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={name}/OU={subj_ou} -passout pass:{password} ' +
1493
- f'-keyout {name}.key -out {name}.crt', folder)
1494
-
1495
- @GeneralUtilities.check_arguments
1496
- def generate_certificate(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str,
1497
- days_until_expire: int = None, password: str = None) -> None:
1498
- if days_until_expire is None:
1499
- days_until_expire = 397
1500
- if password is None:
1501
- password = GeneralUtilities.generate_password()
1502
- rsa_key_length = 4096
1503
- self.run_program("openssl", f'genrsa -out {filename}.key {rsa_key_length}', folder)
1504
- self.run_program("openssl", f'req -new -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou} -x509 ' +
1505
- f'-key {filename}.key -out {filename}.unsigned.crt -days {days_until_expire}', folder)
1506
- self.run_program("openssl", f'pkcs12 -export -out {filename}.selfsigned.pfx -password pass:{password} -inkey {filename}.key -in {filename}.unsigned.crt', folder)
1507
- GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.password"), password)
1508
- GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.san.conf"), f"""[ req ]
1509
- default_bits = {rsa_key_length}
1510
- distinguished_name = req_distinguished_name
1511
- req_extensions = v3_req
1512
- default_md = sha256
1513
- dirstring_type = nombstr
1514
- prompt = no
1515
-
1516
- [ req_distinguished_name ]
1517
- countryName = {subj_c}
1518
- stateOrProvinceName = {subj_st}
1519
- localityName = {subj_l}
1520
- organizationName = {subj_o}
1521
- organizationUnit = {subj_ou}
1522
- commonName = {domain}
1523
-
1524
- [v3_req]
1525
- subjectAltName = @subject_alt_name
1526
-
1527
- [ subject_alt_name ]
1528
- DNS = {domain}
1529
- """)
1530
-
1531
- @GeneralUtilities.check_arguments
1532
- def generate_certificate_sign_request(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str) -> None:
1533
- self.run_program("openssl", f'req -new -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou} ' +
1534
- f'-key {filename}.key -out {filename}.csr -config {filename}.san.conf', folder)
1535
-
1536
- @GeneralUtilities.check_arguments
1537
- def sign_certificate(self, folder: str, ca_folder: str, ca_name: str, domain: str, filename: str, days_until_expire: int = None) -> None:
1538
- if days_until_expire is None:
1539
- days_until_expire = 397
1540
- ca = os.path.join(ca_folder, ca_name)
1541
- password_file = os.path.join(folder, f"{filename}.password")
1542
- password = GeneralUtilities.read_text_from_file(password_file)
1543
- self.run_program("openssl", f'x509 -req -in {filename}.csr -CA {ca}.crt -CAkey {ca}.key -CAcreateserial -CAserial {ca}.srl ' +
1544
- f'-out {filename}.crt -days {days_until_expire} -sha256 -extensions v3_req -extfile {filename}.san.conf', folder)
1545
- self.run_program("openssl", f'pkcs12 -export -out {filename}.pfx -inkey {filename}.key -in {filename}.crt -password pass:{password}', folder)
1546
-
1547
- @GeneralUtilities.check_arguments
1548
- def update_dependencies_of_python_in_requirementstxt_file(self, file: str, verbosity: int):
1549
- lines = GeneralUtilities.read_lines_from_file(file)
1550
- new_lines = []
1551
- for line in lines:
1552
- new_lines.append(self.__get_updated_line_for_python_requirements(line.strip()))
1553
- GeneralUtilities.write_lines_to_file(file, new_lines)
1554
-
1555
- @GeneralUtilities.check_arguments
1556
- def __get_updated_line_for_python_requirements(self, line: str) -> str:
1557
- if "==" in line or "<" in line:
1558
- return line
1559
- elif ">" in line:
1560
- try:
1561
- # line is something like "cyclonedx-bom>=2.0.2" and the function must return with the updated version
1562
- # (something like "cyclonedx-bom>=3.11.0" for example)
1563
- package = line.split(">")[0]
1564
- operator = ">=" if ">=" in line else ">"
1565
- response = requests.get(f'https://pypi.org/pypi/{package}/json', timeout=5)
1566
- latest_version = response.json()['info']['version']
1567
- return package+operator+latest_version
1568
- except:
1569
- return line
1570
- else:
1571
- raise ValueError(f'Unexpected line in requirements-file: "{line}"')
1572
-
1573
- @GeneralUtilities.check_arguments
1574
- def update_dependencies_of_python_in_setupcfg_file(self, setup_cfg_file: str, verbosity: int):
1575
- lines = GeneralUtilities.read_lines_from_file(setup_cfg_file)
1576
- new_lines = []
1577
- requirement_parsing_mode = False
1578
- for line in lines:
1579
- new_line = line
1580
- if (requirement_parsing_mode):
1581
- if ("<" in line or "=" in line or ">" in line):
1582
- updated_line = f" {self.__get_updated_line_for_python_requirements(line.strip())}"
1583
- new_line = updated_line
1584
- else:
1585
- requirement_parsing_mode = False
1586
- else:
1587
- if line.startswith("install_requires ="):
1588
- requirement_parsing_mode = True
1589
- new_lines.append(new_line)
1590
- GeneralUtilities.write_lines_to_file(setup_cfg_file, new_lines)
1591
-
1592
- @GeneralUtilities.check_arguments
1593
- def update_dependencies_of_dotnet_project(self, csproj_file: str, verbosity: int):
1594
- folder = os.path.dirname(csproj_file)
1595
- csproj_filename = os.path.basename(csproj_file)
1596
- GeneralUtilities.write_message_to_stderr(f"Check for updates in {csproj_filename}")
1597
- result = self.run_program("dotnet", f"list {csproj_filename} package --outdated", folder)
1598
- for line in result[1].replace("\r", "").split("\n"):
1599
- # Relevant output-lines are something like " > NJsonSchema 10.7.0 10.7.0 10.9.0"
1600
- if ">" in line:
1601
- package_name = line.replace(">", "").strip().split(" ")[0]
1602
- GeneralUtilities.write_message_to_stderr(f"Update package {package_name}")
1603
- self.run_program("dotnet", f"add {csproj_filename} package {package_name}", folder)
1604
-
1605
- @GeneralUtilities.check_arguments
1606
- def create_deb_package(self, toolname: str, binary_folder: str, control_file_content: str,
1607
- deb_output_folder: str, verbosity: int, permission_of_executable_file_as_octet_triple: int) -> None:
1608
-
1609
- # prepare
1610
- GeneralUtilities.ensure_directory_exists(deb_output_folder)
1611
- temp_folder = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
1612
- GeneralUtilities.ensure_directory_exists(temp_folder)
1613
- bin_folder = binary_folder
1614
- tool_content_folder_name = toolname+"Content"
1615
-
1616
- # create folder
1617
- GeneralUtilities.ensure_directory_exists(temp_folder)
1618
- control_content_folder_name = "controlcontent"
1619
- packagecontent_control_folder = os.path.join(temp_folder, control_content_folder_name)
1620
- GeneralUtilities.ensure_directory_exists(packagecontent_control_folder)
1621
- data_content_folder_name = "datacontent"
1622
- packagecontent_data_folder = os.path.join(temp_folder, data_content_folder_name)
1623
- GeneralUtilities.ensure_directory_exists(packagecontent_data_folder)
1624
- entireresult_content_folder_name = "entireresultcontent"
1625
- packagecontent_entireresult_folder = os.path.join(temp_folder, entireresult_content_folder_name)
1626
- GeneralUtilities.ensure_directory_exists(packagecontent_entireresult_folder)
1627
-
1628
- # create "debian-binary"-file
1629
- debianbinary_file = os.path.join(packagecontent_entireresult_folder, "debian-binary")
1630
- GeneralUtilities.ensure_file_exists(debianbinary_file)
1631
- GeneralUtilities.write_text_to_file(debianbinary_file, "2.0\n")
1632
-
1633
- # create control-content
1634
-
1635
- # conffiles
1636
- conffiles_file = os.path.join(packagecontent_control_folder, "conffiles")
1637
- GeneralUtilities.ensure_file_exists(conffiles_file)
1638
-
1639
- # postinst-script
1640
- postinst_file = os.path.join(packagecontent_control_folder, "postinst")
1641
- GeneralUtilities.ensure_file_exists(postinst_file)
1642
- exe_file = f"/usr/bin/{tool_content_folder_name}/{toolname}"
1643
- link_file = f"/usr/bin/{toolname.lower()}"
1644
- permission = str(permission_of_executable_file_as_octet_triple)
1645
- GeneralUtilities.write_text_to_file(postinst_file, f"""#!/bin/sh
1646
- ln -s {exe_file} {link_file}
1647
- chmod {permission} {exe_file}
1648
- chmod {permission} {link_file}
1649
- """)
1650
-
1651
- # control
1652
- control_file = os.path.join(packagecontent_control_folder, "control")
1653
- GeneralUtilities.ensure_file_exists(control_file)
1654
- GeneralUtilities.write_text_to_file(control_file, control_file_content)
1655
-
1656
- # md5sums
1657
- md5sums_file = os.path.join(packagecontent_control_folder, "md5sums")
1658
- GeneralUtilities.ensure_file_exists(md5sums_file)
1659
-
1660
- # create data-content
1661
-
1662
- # copy binaries
1663
- usr_bin_folder = os.path.join(packagecontent_data_folder, "usr/bin")
1664
- GeneralUtilities.ensure_directory_exists(usr_bin_folder)
1665
- usr_bin_content_folder = os.path.join(usr_bin_folder, tool_content_folder_name)
1666
- GeneralUtilities.copy_content_of_folder(bin_folder, usr_bin_content_folder)
1667
-
1668
- # create debfile
1669
- deb_filename = f"{toolname}.deb"
1670
- self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/control.tar.gz", "*"],
1671
- packagecontent_control_folder, verbosity=verbosity)
1672
- self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/data.tar.gz", "*"],
1673
- packagecontent_data_folder, verbosity=verbosity)
1674
- self.run_program_argsasarray("ar", ["r", deb_filename, "debian-binary", "control.tar.gz", "data.tar.gz"],
1675
- packagecontent_entireresult_folder, verbosity=verbosity)
1676
- result_file = os.path.join(packagecontent_entireresult_folder, deb_filename)
1677
- shutil.copy(result_file, os.path.join(deb_output_folder, deb_filename))
1678
-
1679
- # cleanup
1680
- GeneralUtilities.ensure_directory_does_not_exist(temp_folder)
1681
-
1682
-
1683
- @GeneralUtilities.check_arguments
1684
- def update_year_in_copyright_tags(self, file: str) -> None:
1685
- current_year=str(datetime.now().year)
1686
- lines=GeneralUtilities.read_lines_from_file(file)
1687
- lines_result=[]
1688
- for line in lines:
1689
- if match := re.search("(.*<[Cc]opyright>.*)\\d\\d\\d\\d(.*<\\/[Cc]opyright>.*)", line):
1690
- part1 = match.group(1)
1691
- part2 = match.group(2)
1692
- adapted=part1+current_year+part2
1693
- else:
1694
- adapted=line
1695
- lines_result.append(adapted)
1696
- GeneralUtilities.write_lines_to_file(file,lines_result)
1697
-
1698
- @GeneralUtilities.check_arguments
1699
- def update_year_in_first_line_of_file(self, file: str) -> None:
1700
- current_year=str(datetime.now().year)
1701
- lines=GeneralUtilities.read_lines_from_file(file)
1702
- lines[0]=re.sub("\\d\\d\\d\\d",current_year,lines[0])
1703
- GeneralUtilities.write_lines_to_file(file,lines)
1
+ import sys
2
+ from datetime import timedelta, datetime
3
+ import json
4
+ import binascii
5
+ import filecmp
6
+ import hashlib
7
+ import multiprocessing
8
+ import time
9
+ from io import BytesIO
10
+ import itertools
11
+ import math
12
+ import os
13
+ from queue import Queue, Empty
14
+ from concurrent.futures import ThreadPoolExecutor
15
+ from pathlib import Path
16
+ from subprocess import Popen
17
+ import re
18
+ import shutil
19
+ import uuid
20
+ import tempfile
21
+ import io
22
+ import requests
23
+ import ntplib
24
+ import yaml
25
+ import qrcode
26
+ import pycdlib
27
+ import send2trash
28
+ import PyPDF2
29
+ from .GeneralUtilities import GeneralUtilities
30
+ from .ProgramRunnerBase import ProgramRunnerBase
31
+ from .ProgramRunnerPopen import ProgramRunnerPopen
32
+ from .ProgramRunnerEpew import ProgramRunnerEpew, CustomEpewArgument
33
+
34
+ version = "3.5.44"
35
+ __version__ = version
36
+
37
+
38
+ class ScriptCollectionCore:
39
+
40
+ # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
41
+ # Do not change this value for productive environments.
42
+ mock_program_calls: bool = False
43
+ # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
44
+ execute_program_really_if_no_mock_call_is_defined: bool = False
45
+ __mocked_program_calls: list = None
46
+ program_runner: ProgramRunnerBase = None
47
+ call_program_runner_directly: bool = None
48
+
49
+ def __init__(self):
50
+ self.program_runner = ProgramRunnerPopen()
51
+ self.call_program_runner_directly = None
52
+ self.__mocked_program_calls = list[ScriptCollectionCore.__MockProgramCall]()
53
+
54
+ @staticmethod
55
+ @GeneralUtilities.check_arguments
56
+ def get_scriptcollection_version() -> str:
57
+ return __version__
58
+
59
+ @GeneralUtilities.check_arguments
60
+ def python_file_has_errors(self, file: str, working_directory: str, treat_warnings_as_errors: bool = True) -> tuple[bool, list[str]]:
61
+ errors = list()
62
+ filename = os.path.relpath(file, working_directory)
63
+ if treat_warnings_as_errors:
64
+ errorsonly_argument = ""
65
+ else:
66
+ errorsonly_argument = " --errors-only"
67
+ (exit_code, stdout, stderr, _) = self.run_program("pylint", filename + errorsonly_argument, working_directory, throw_exception_if_exitcode_is_not_zero=False)
68
+ if (exit_code != 0):
69
+ errors.append(f"Linting-issues of {file}:")
70
+ errors.append(f"Pylint-exitcode: {exit_code}")
71
+ for line in GeneralUtilities.string_to_lines(stdout):
72
+ errors.append(line)
73
+ for line in GeneralUtilities.string_to_lines(stderr):
74
+ errors.append(line)
75
+ return (True, errors)
76
+
77
+ return (False, errors)
78
+
79
+ @GeneralUtilities.check_arguments
80
+ def replace_version_in_dockerfile_file(self, dockerfile: str, new_version_value: str) -> None:
81
+ GeneralUtilities.write_text_to_file(dockerfile, re.sub("ARG Version=\"\\d+\\.\\d+\\.\\d+\"", f"ARG Version=\"{new_version_value}\"", GeneralUtilities.read_text_from_file(dockerfile)))
82
+
83
+ @GeneralUtilities.check_arguments
84
+ def replace_version_in_python_file(self, file: str, new_version_value: str):
85
+ GeneralUtilities.write_text_to_file(file, re.sub("version = \"\\d+\\.\\d+\\.\\d+\"", f"version = \"{new_version_value}\"", GeneralUtilities.read_text_from_file(file)))
86
+
87
+ @GeneralUtilities.check_arguments
88
+ def replace_version_in_ini_file(self, file: str, new_version_value: str):
89
+ GeneralUtilities.write_text_to_file(file, re.sub("version = \\d+\\.\\d+\\.\\d+", f"version = {new_version_value}", GeneralUtilities.read_text_from_file(file)))
90
+
91
+ @GeneralUtilities.check_arguments
92
+ def replace_version_in_nuspec_file(self, nuspec_file: str, new_version: str) -> None:
93
+ # TODO use XSLT instead
94
+ versionregex = "\\d+\\.\\d+\\.\\d+"
95
+ versiononlyregex = f"^{versionregex}$"
96
+ pattern = re.compile(versiononlyregex)
97
+ if pattern.match(new_version):
98
+ GeneralUtilities.write_text_to_file(nuspec_file, re.sub(f"<version>{versionregex}<\\/version>", f"<version>{new_version}</version>", GeneralUtilities.read_text_from_file(nuspec_file)))
99
+ else:
100
+ raise ValueError(f"Version '{new_version}' does not match version-regex '{versiononlyregex}'")
101
+
102
+ @GeneralUtilities.check_arguments
103
+ def replace_version_in_csproj_file(self, csproj_file: str, current_version: str):
104
+ versionregex = "\\d+\\.\\d+\\.\\d+"
105
+ versiononlyregex = f"^{versionregex}$"
106
+ pattern = re.compile(versiononlyregex)
107
+ if pattern.match(current_version):
108
+ for tag in ["Version", "AssemblyVersion", "FileVersion"]:
109
+ GeneralUtilities.write_text_to_file(csproj_file, re.sub(f"<{tag}>{versionregex}(.\\d+)?<\\/{tag}>", f"<{tag}>{current_version}</{tag}>", GeneralUtilities.read_text_from_file(csproj_file)))
110
+ else:
111
+ raise ValueError(f"Version '{current_version}' does not match version-regex '{versiononlyregex}'")
112
+
113
+ @GeneralUtilities.check_arguments
114
+ def push_nuget_build_artifact(self, nupkg_file: str, registry_address: str, api_key: str, verbosity: int = 1):
115
+ nupkg_file_name = os.path.basename(nupkg_file)
116
+ nupkg_file_folder = os.path.dirname(nupkg_file)
117
+ self.run_program("dotnet", f"nuget push {nupkg_file_name} --force-english-output --source {registry_address} --api-key {api_key}", nupkg_file_folder, verbosity)
118
+
119
+ @GeneralUtilities.check_arguments
120
+ def dotnet_build(self, repository_folder: str, projectname: str, configuration: str):
121
+ self.run_program("dotnet", f"clean -c {configuration}", repository_folder)
122
+ self.run_program("dotnet", f"build {projectname}/{projectname}.csproj -c {configuration}", repository_folder)
123
+
124
+ @GeneralUtilities.check_arguments
125
+ def find_file_by_extension(self, folder: str, extension: str):
126
+ result = [file for file in GeneralUtilities.get_direct_files_of_folder(folder) if file.endswith(f".{extension}")]
127
+ result_length = len(result)
128
+ if result_length == 0:
129
+ raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension}'.")
130
+ if result_length == 1:
131
+ return result[0]
132
+ else:
133
+ raise ValueError(f"Multiple values available in folder '{folder}' with extension '{extension}'.")
134
+
135
+ @GeneralUtilities.check_arguments
136
+ def find_latest_file_by_extension(self, folder: str, extension: str) -> str:
137
+ files: list[str] = GeneralUtilities.get_direct_files_of_folder(folder)
138
+ possible_results: list[str] = []
139
+ for file in files:
140
+ if file.endswith(f".{extension}"):
141
+ possible_results.append(file)
142
+ result_length = len(possible_results)
143
+ if result_length == 0:
144
+ raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension}'.")
145
+ else:
146
+ return possible_results[-1]
147
+
148
+ @GeneralUtilities.check_arguments
149
+ def commit_is_signed_by_key(self, repository_folder: str, revision_identifier: str, key: str) -> bool:
150
+ result = self.run_program(
151
+ "git", f"verify-commit {revision_identifier}", repository_folder, throw_exception_if_exitcode_is_not_zero=False)
152
+ if (result[0] != 0):
153
+ return False
154
+ if (not GeneralUtilities.contains_line(result[1].splitlines(), f"gpg\\:\\ using\\ [A-Za-z0-9]+\\ key\\ [A-Za-z0-9]+{key}")):
155
+ # TODO check whether this works on machines where gpg is installed in another langauge than english
156
+ return False
157
+ if (not GeneralUtilities.contains_line(result[1].splitlines(), "gpg\\:\\ Good\\ signature\\ from")):
158
+ # TODO check whether this works on machines where gpg is installed in another langauge than english
159
+ return False
160
+ return True
161
+
162
+ @GeneralUtilities.check_arguments
163
+ def get_parent_commit_ids_of_commit(self, repository_folder: str, commit_id: str) -> str:
164
+ return self.run_program("git", f'log --pretty=%P -n 1 "{commit_id}"', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].replace("\r", "").replace("\n", "").split(" ")
165
+
166
+ @GeneralUtilities.check_arguments
167
+ def get_all_authors_and_committers_of_repository(self, repository_folder: str, subfolder: str = None, verbosity: int = 1) -> list[tuple[str, str]]:
168
+ space_character = "_"
169
+ if subfolder is None:
170
+ subfolder_argument = ""
171
+ else:
172
+ subfolder_argument = f" -- {subfolder}"
173
+ log_result = self.run_program("git", f'log --pretty=%aN{space_character}%aE%n%cN{space_character}%cE HEAD{subfolder_argument}', repository_folder, verbosity=0)
174
+ plain_content: list[str] = list(
175
+ set([line for line in log_result[1].split("\n") if len(line) > 0]))
176
+ result: list[tuple[str, str]] = []
177
+ for item in plain_content:
178
+ if len(re.findall(space_character, item)) == 1:
179
+ splitted = item.split(space_character)
180
+ result.append((splitted[0], splitted[1]))
181
+ else:
182
+ raise ValueError(f'Unexpected author: "{item}"')
183
+ return result
184
+
185
+ @GeneralUtilities.check_arguments
186
+ def get_commit_ids_between_dates(self, repository_folder: str, since: datetime, until: datetime, ignore_commits_which_are_not_in_history_of_head: bool = True) -> None:
187
+ since_as_string = self.__datetime_to_string_for_git(since)
188
+ until_as_string = self.__datetime_to_string_for_git(until)
189
+ result = filter(lambda line: not GeneralUtilities.string_is_none_or_whitespace(line), self.run_program("git", f'log --since "{since_as_string}" --until "{until_as_string}" --pretty=format:"%H" --no-patch', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].split("\n").replace("\r", ""))
190
+ if ignore_commits_which_are_not_in_history_of_head:
191
+ result = [commit_id for commit_id in result if self.git_commit_is_ancestor(
192
+ repository_folder, commit_id)]
193
+ return result
194
+
195
+ @GeneralUtilities.check_arguments
196
+ def __datetime_to_string_for_git(self, datetime_object: datetime) -> str:
197
+ return datetime_object.strftime('%Y-%m-%d %H:%M:%S')
198
+
199
+ @GeneralUtilities.check_arguments
200
+ def git_commit_is_ancestor(self, repository_folder: str, ancestor: str, descendant: str = "HEAD") -> bool:
201
+ result = self.run_program_argsasarray("git", ["merge-base", "--is-ancestor", ancestor, descendant], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
202
+ exit_code = result[0]
203
+ if exit_code == 0:
204
+ return True
205
+ elif exit_code == 1:
206
+ return False
207
+ else:
208
+ raise ValueError(f'Can not calculate if {ancestor} is an ancestor of {descendant} in repository {repository_folder}. Outout of "{repository_folder}> git merge-base --is-ancestor {ancestor} {descendant}": Exitcode: {exit_code}; StdOut: {result[1]}; StdErr: {result[2]}.')
209
+
210
+ @GeneralUtilities.check_arguments
211
+ def __git_changes_helper(self, repository_folder: str, arguments_as_array: list[str]) -> bool:
212
+ lines = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", arguments_as_array, repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
213
+ for line in lines:
214
+ if GeneralUtilities.string_has_content(line):
215
+ return True
216
+ return False
217
+
218
+ @GeneralUtilities.check_arguments
219
+ def git_repository_has_new_untracked_files(self, repositoryFolder: str):
220
+ return self.__git_changes_helper(repositoryFolder, ["ls-files", "--exclude-standard", "--others"])
221
+
222
+ @GeneralUtilities.check_arguments
223
+ def git_repository_has_unstaged_changes_of_tracked_files(self, repositoryFolder: str):
224
+ return self.__git_changes_helper(repositoryFolder, ["--no-pager", "diff"])
225
+
226
+ @GeneralUtilities.check_arguments
227
+ def git_repository_has_staged_changes(self, repositoryFolder: str):
228
+ return self.__git_changes_helper(repositoryFolder, ["--no-pager", "diff", "--cached"])
229
+
230
+ @GeneralUtilities.check_arguments
231
+ def git_repository_has_uncommitted_changes(self, repositoryFolder: str) -> bool:
232
+ if (self.git_repository_has_unstaged_changes(repositoryFolder)):
233
+ return True
234
+ if (self.git_repository_has_staged_changes(repositoryFolder)):
235
+ return True
236
+ return False
237
+
238
+ @GeneralUtilities.check_arguments
239
+ def git_repository_has_unstaged_changes(self, repository_folder: str) -> bool:
240
+ if (self.git_repository_has_unstaged_changes_of_tracked_files(repository_folder)):
241
+ return True
242
+ if (self.git_repository_has_new_untracked_files(repository_folder)):
243
+ return True
244
+ return False
245
+
246
+ @GeneralUtilities.check_arguments
247
+ def git_get_commit_id(self, repository_folder: str, commit: str = "HEAD") -> str:
248
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["rev-parse", "--verify", commit], repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
249
+ return result[1].replace('\n', '')
250
+
251
+ @GeneralUtilities.check_arguments
252
+ def git_get_commit_date(self, repository_folder: str, commit: str = "HEAD") -> datetime:
253
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["show", "-s", "--format=%ci", commit], repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
254
+ date_as_string = result[1].replace('\n', '')
255
+ result = datetime.strptime(date_as_string, '%Y-%m-%d %H:%M:%S %z')
256
+ return result
257
+
258
+ @GeneralUtilities.check_arguments
259
+ def git_fetch(self, folder: str, remotename: str = "--all") -> None:
260
+ self.run_program_argsasarray("git", ["fetch", remotename, "--tags", "--prune"], folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
261
+
262
+ @GeneralUtilities.check_arguments
263
+ def git_fetch_in_bare_repository(self, folder: str, remotename, localbranch: str, remotebranch: str) -> None:
264
+ self.run_program_argsasarray("git", ["fetch", remotename, f"{remotebranch}:{localbranch}"], folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
265
+
266
+ @GeneralUtilities.check_arguments
267
+ def git_remove_branch(self, folder: str, branchname: str) -> None:
268
+ self.run_program("git", f"branch -D {branchname}", folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
269
+
270
+ @GeneralUtilities.check_arguments
271
+ def git_push(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: int = 0) -> None:
272
+ argument = ["push", "--recurse-submodules=on-demand", remotename, f"{localbranchname}:{remotebranchname}"]
273
+ if (forcepush):
274
+ argument.append("--force")
275
+ if (pushalltags):
276
+ argument.append("--tags")
277
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=verbosity, print_errors_as_information=True)
278
+ return result[1].replace('\r', '').replace('\n', '')
279
+
280
+ @GeneralUtilities.check_arguments
281
+ def git_pull(self, folder: str, remote: str, localbranchname: str, remotebranchname: str, force: bool = False) -> None:
282
+ argument = f"pull {remote} {remotebranchname}:{localbranchname}"
283
+ if force:
284
+ argument = f"{argument} --force"
285
+ self.run_program("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
286
+
287
+ @GeneralUtilities.check_arguments
288
+ def git_list_remote_branches(self, folder: str, remote: str, fetch: bool) -> list[str]:
289
+ if fetch:
290
+ self.git_fetch(folder, remote)
291
+ run_program_result = self.run_program("git", f"branch -rl {remote}/*", folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
292
+ output = GeneralUtilities.string_to_lines(run_program_result[1])
293
+ result = list[str]()
294
+ for item in output:
295
+ striped_item = item.strip()
296
+ if GeneralUtilities.string_has_content(striped_item):
297
+ branch: str = None
298
+ if " " in striped_item:
299
+ branch = striped_item.split(" ")[0]
300
+ else:
301
+ branch = striped_item
302
+ branchname = branch[len(remote)+1:]
303
+ if branchname != "HEAD":
304
+ result.append(branchname)
305
+ return result
306
+
307
+ @GeneralUtilities.check_arguments
308
+ def git_clone(self, clone_target_folder: str, remote_repository_path: str, include_submodules: bool = True, mirror: bool = False) -> None:
309
+ if (os.path.isdir(clone_target_folder)):
310
+ pass # TODO throw error
311
+ else:
312
+ args = ["clone", remote_repository_path, clone_target_folder]
313
+ if include_submodules:
314
+ args.append("--recurse-submodules")
315
+ args.append("--remote-submodules")
316
+ if mirror:
317
+ args.append("--mirror")
318
+ self.run_program_argsasarray("git", args, os.getcwd(), throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
319
+
320
+ @GeneralUtilities.check_arguments
321
+ def git_get_all_remote_names(self, directory: str) -> list[str]:
322
+ result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
323
+ return result
324
+
325
+ @GeneralUtilities.check_arguments
326
+ def git_get_remote_url(self, directory: str, remote_name: str) -> str:
327
+ result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote", "get-url", remote_name], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
328
+ return result[0].replace('\n', '')
329
+
330
+ @GeneralUtilities.check_arguments
331
+ def repository_has_remote_with_specific_name(self, directory: str, remote_name: str) -> bool:
332
+ return remote_name in self.git_get_all_remote_names(directory)
333
+
334
+ @GeneralUtilities.check_arguments
335
+ def git_add_or_set_remote_address(self, directory: str, remote_name: str, remote_address: str) -> None:
336
+ if (self.repository_has_remote_with_specific_name(directory, remote_name)):
337
+ self.run_program_argsasarray("git", ['remote', 'set-url', 'remote_name', remote_address], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
338
+ else:
339
+ self.run_program_argsasarray("git", ['remote', 'add', remote_name, remote_address], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
340
+
341
+ @GeneralUtilities.check_arguments
342
+ def git_stage_all_changes(self, directory: str) -> None:
343
+ self.run_program_argsasarray("git", ["add", "-A"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
344
+
345
+ @GeneralUtilities.check_arguments
346
+ def git_unstage_all_changes(self, directory: str) -> None:
347
+ self.run_program_argsasarray("git", ["reset"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
348
+
349
+ @GeneralUtilities.check_arguments
350
+ def git_stage_file(self, directory: str, file: str) -> None:
351
+ self.run_program_argsasarray("git", ['stage', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
352
+
353
+ @GeneralUtilities.check_arguments
354
+ def git_unstage_file(self, directory: str, file: str) -> None:
355
+ self.run_program_argsasarray("git", ['reset', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
356
+
357
+ @GeneralUtilities.check_arguments
358
+ def git_discard_unstaged_changes_of_file(self, directory: str, file: str) -> None:
359
+ """Caution: This method works really only for 'changed' files yet. So this method does not work properly for new or renamed files."""
360
+ self.run_program_argsasarray("git", ['checkout', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
361
+
362
+ @GeneralUtilities.check_arguments
363
+ def git_discard_all_unstaged_changes(self, directory: str) -> None:
364
+ """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
365
+ self.run_program_argsasarray("git", ['clean', '-df'], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
366
+ self.run_program_argsasarray("git", ['checkout', '.'], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
367
+
368
+ @GeneralUtilities.check_arguments
369
+ def git_commit(self, directory: str, message: str, author_name: str = None, author_email: str = None, stage_all_changes: bool = True, no_changes_behavior: int = 0) -> str:
370
+ # no_changes_behavior=0 => No commit
371
+ # no_changes_behavior=1 => Commit anyway
372
+ # no_changes_behavior=2 => Exception
373
+ author_name = GeneralUtilities.str_none_safe(author_name).strip()
374
+ author_email = GeneralUtilities.str_none_safe(author_email).strip()
375
+ argument = ['commit', '--quiet', '--allow-empty', '--message', message]
376
+ if (GeneralUtilities.string_has_content(author_name)):
377
+ argument.append(f'--author="{author_name} <{author_email}>"')
378
+ git_repository_has_uncommitted_changes = self.git_repository_has_uncommitted_changes(directory)
379
+
380
+ if git_repository_has_uncommitted_changes:
381
+ do_commit = True
382
+ if stage_all_changes:
383
+ self.git_stage_all_changes(directory)
384
+ else:
385
+ if no_changes_behavior == 0:
386
+ GeneralUtilities.write_message_to_stdout(f"Commit '{message}' will not be done because there are no changes to commit in repository '{directory}'")
387
+ do_commit = False
388
+ elif no_changes_behavior == 1:
389
+ GeneralUtilities.write_message_to_stdout(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will be done anyway.")
390
+ do_commit = True
391
+ elif no_changes_behavior == 2:
392
+ raise RuntimeError(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will not be done.")
393
+ else:
394
+ raise ValueError(f"Unknown value for no_changes_behavior: {GeneralUtilities.str_none_safe(no_changes_behavior)}")
395
+
396
+ if do_commit:
397
+ GeneralUtilities.write_message_to_stdout(f"Commit changes in '{directory}'")
398
+ self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
399
+
400
+ return self.git_get_commit_id(directory)
401
+
402
+ @GeneralUtilities.check_arguments
403
+ def git_create_tag(self, directory: str, target_for_tag: str, tag: str, sign: bool = False, message: str = None) -> None:
404
+ argument = ["tag", tag, target_for_tag]
405
+ if sign:
406
+ if message is None:
407
+ message = f"Created {target_for_tag}"
408
+ argument.extend(["-s", '-m', message])
409
+ self.run_program_argsasarray(
410
+ "git", argument, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
411
+
412
+ @GeneralUtilities.check_arguments
413
+ def git_delete_tag(self, directory: str, tag: str) -> None:
414
+ self.run_program_argsasarray("git", ["tag", "--delete", tag], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
415
+
416
+ @GeneralUtilities.check_arguments
417
+ def git_checkout(self, directory: str, branch: str) -> None:
418
+ self.run_program_argsasarray("git", ["checkout", branch], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
419
+ self.run_program_argsasarray("git", ["submodule", "update", "--recursive"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
420
+
421
+ @GeneralUtilities.check_arguments
422
+ def git_merge_abort(self, directory: str) -> None:
423
+ self.run_program_argsasarray("git", ["merge", "--abort"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
424
+
425
+ @GeneralUtilities.check_arguments
426
+ def git_merge(self, directory: str, sourcebranch: str, targetbranch: str, fastforward: bool = True, commit: bool = True, commit_message: str = None) -> str:
427
+ self.git_checkout(directory, targetbranch)
428
+ args = ["merge"]
429
+ if not commit:
430
+ args.append("--no-commit")
431
+ if not fastforward:
432
+ args.append("--no-ff")
433
+ if commit_message is not None:
434
+ args.append("-m")
435
+ args.append(commit_message)
436
+ args.append(sourcebranch)
437
+ self.run_program_argsasarray("git", args, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
438
+ self.run_program_argsasarray("git", ["submodule", "update"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
439
+ return self.git_get_commit_id(directory)
440
+
441
+ @GeneralUtilities.check_arguments
442
+ def git_undo_all_changes(self, directory: str) -> None:
443
+ """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
444
+ self.git_unstage_all_changes(directory)
445
+ self.git_discard_all_unstaged_changes(directory)
446
+
447
+ @GeneralUtilities.check_arguments
448
+ def git_fetch_or_clone_all_in_directory(self, source_directory: str, target_directory: str) -> None:
449
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(source_directory):
450
+ foldername = os.path.basename(subfolder)
451
+ if self.is_git_repository(subfolder):
452
+ source_repository = subfolder
453
+ target_repository = os.path.join(target_directory, foldername)
454
+ if os.path.isdir(target_directory):
455
+ # fetch
456
+ self.git_fetch(target_directory)
457
+ else:
458
+ # clone
459
+ self.git_clone(target_repository, source_repository, include_submodules=True, mirror=True)
460
+
461
+ def get_git_submodules(self, folder: str) -> list[str]:
462
+ e = self.run_program("git", "submodule status", folder)
463
+ result = []
464
+ for submodule_line in GeneralUtilities.string_to_lines(e[1], False, True):
465
+ result.append(submodule_line.split(' ')[1])
466
+ return result
467
+
468
+ @GeneralUtilities.check_arguments
469
+ def is_git_repository(self, folder: str) -> bool:
470
+ combined = os.path.join(folder, ".git")
471
+ # TODO consider check for bare-repositories
472
+ return os.path.isdir(combined) or os.path.isfile(combined)
473
+
474
+ @GeneralUtilities.check_arguments
475
+ def file_is_git_ignored(self, file_in_repository: str, repositorybasefolder: str) -> None:
476
+ exit_code = self.run_program_argsasarray("git", ['check-ignore', file_in_repository], repositorybasefolder, throw_exception_if_exitcode_is_not_zero=False, verbosity=0)[0]
477
+ if (exit_code == 0):
478
+ return True
479
+ if (exit_code == 1):
480
+ return False
481
+ raise ValueError(f"Unable to calculate whether '{file_in_repository}' in repository '{repositorybasefolder}' is ignored due to git-exitcode {exit_code}.")
482
+
483
+ @GeneralUtilities.check_arguments
484
+ def git_discard_all_changes(self, repository: str) -> None:
485
+ self.run_program_argsasarray("git", ["reset", "HEAD", "."], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
486
+ self.run_program_argsasarray("git", ["checkout", "."], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
487
+
488
+ @GeneralUtilities.check_arguments
489
+ def git_get_current_branch_name(self, repository: str) -> str:
490
+ result = self.run_program_argsasarray("git", ["rev-parse", "--abbrev-ref", "HEAD"], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
491
+ return result[1].replace("\r", "").replace("\n", "")
492
+
493
+ @GeneralUtilities.check_arguments
494
+ def git_get_commitid_of_tag(self, repository: str, tag: str) -> str:
495
+ stdout = self.run_program_argsasarray("git", ["rev-list", "-n", "1", tag], repository, verbosity=0)
496
+ result = stdout[1].replace("\r", "").replace("\n", "")
497
+ return result
498
+
499
+ @GeneralUtilities.check_arguments
500
+ def git_get_tags(self, repository: str) -> list[str]:
501
+ tags = [line.replace("\r", "") for line in self.run_program_argsasarray(
502
+ "git", ["tag"], repository)[1].split("\n") if len(line) > 0]
503
+ return tags
504
+
505
+ @GeneralUtilities.check_arguments
506
+ def git_move_tags_to_another_branch(self, repository: str, tag_source_branch: str, tag_target_branch: str, sign: bool = False, message: str = None) -> None:
507
+ tags = self.git_get_tags(repository)
508
+ tags_count = len(tags)
509
+ counter = 0
510
+ for tag in tags:
511
+ counter = counter+1
512
+ GeneralUtilities.write_message_to_stdout(f"Process tag {counter}/{tags_count}.")
513
+ # tag is on source-branch
514
+ if self.git_commit_is_ancestor(repository, tag, tag_source_branch):
515
+ commit_id_old = self.git_get_commitid_of_tag(repository, tag)
516
+ commit_date: datetime = self.git_get_commit_date(repository, commit_id_old)
517
+ date_as_string = self.__datetime_to_string_for_git(commit_date)
518
+ search_commit_result = self.run_program_argsasarray("git", ["log", f'--after="{date_as_string}"', f'--before="{date_as_string}"', "--pretty=format:%H", tag_target_branch], repository, throw_exception_if_exitcode_is_not_zero=False)
519
+ if search_commit_result[0] != 0 or not GeneralUtilities.string_has_nonwhitespace_content(search_commit_result[1]):
520
+ raise ValueError(f"Can not calculate corresponding commit for tag '{tag}'.")
521
+ commit_id_new = search_commit_result[1]
522
+ self.git_delete_tag(repository, tag)
523
+ self.git_create_tag(repository, commit_id_new, tag, sign, message)
524
+
525
+ @GeneralUtilities.check_arguments
526
+ def get_current_git_branch_has_tag(self, repository_folder: str) -> bool:
527
+ result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder, verbosity=0, throw_exception_if_exitcode_is_not_zero=False)
528
+ return result[0] == 0
529
+
530
+ @GeneralUtilities.check_arguments
531
+ def get_latest_git_tag(self, repository_folder: str) -> str:
532
+ result = self.run_program_argsasarray(
533
+ "git", ["describe", "--tags", "--abbrev=0"], repository_folder, verbosity=0)
534
+ result = result[1].replace("\r", "").replace("\n", "")
535
+ return result
536
+
537
+ @GeneralUtilities.check_arguments
538
+ def get_staged_or_committed_git_ignored_files(self, repository_folder: str) -> list[str]:
539
+ tresult = self.run_program_argsasarray("git", ["ls-files", "-i", "-c", "--exclude-standard"], repository_folder, verbosity=0)
540
+ tresult = tresult[1].replace("\r", "")
541
+ result = [line for line in tresult.split("\n") if len(line) > 0]
542
+ return result
543
+
544
+ @GeneralUtilities.check_arguments
545
+ def git_repository_has_commits(self, repository_folder: str) -> bool:
546
+ return self.run_program_argsasarray("git", ["rev-parse", "--verify", "HEAD"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)[0] == 0
547
+
548
+ @GeneralUtilities.check_arguments
549
+ def export_filemetadata(self, folder: str, target_file: str, encoding: str = "utf-8", filter_function=None) -> None:
550
+ folder = GeneralUtilities.resolve_relative_path_from_current_working_directory(folder)
551
+ lines = list()
552
+ path_prefix = len(folder)+1
553
+ items = dict()
554
+ for item in GeneralUtilities.get_all_folders_of_folder(folder):
555
+ items[item] = "d"
556
+ for item in GeneralUtilities.get_all_files_of_folder(folder):
557
+ items[item] = "f"
558
+ for file_or_folder, item_type in items.items():
559
+ truncated_file = file_or_folder[path_prefix:]
560
+ if (filter_function is None or filter_function(folder, truncated_file)):
561
+ owner_and_permisssion = self.get_file_owner_and_file_permission(file_or_folder)
562
+ user = owner_and_permisssion[0]
563
+ permissions = owner_and_permisssion[1]
564
+ lines.append(f"{truncated_file};{item_type};{user};{permissions}")
565
+ lines = sorted(lines, key=str.casefold)
566
+ with open(target_file, "w", encoding=encoding) as file_object:
567
+ file_object.write("\n".join(lines))
568
+
569
+ @GeneralUtilities.check_arguments
570
+ def escape_git_repositories_in_folder(self, folder: str) -> dict[str, str]:
571
+ return self.__escape_git_repositories_in_folder_internal(folder, dict[str, str]())
572
+
573
+ @GeneralUtilities.check_arguments
574
+ def __escape_git_repositories_in_folder_internal(self, folder: str, renamed_items: dict[str, str]) -> dict[str, str]:
575
+ for file in GeneralUtilities.get_direct_files_of_folder(folder):
576
+ filename = os.path.basename(file)
577
+ if ".git" in filename:
578
+ new_name = filename.replace(".git", ".gitx")
579
+ target = os.path.join(folder, new_name)
580
+ os.rename(file, target)
581
+ renamed_items[target] = file
582
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
583
+ foldername = os.path.basename(subfolder)
584
+ if ".git" in foldername:
585
+ new_name = foldername.replace(".git", ".gitx")
586
+ subfolder2 = os.path.join(str(Path(subfolder).parent), new_name)
587
+ os.rename(subfolder, subfolder2)
588
+ renamed_items[subfolder2] = subfolder
589
+ else:
590
+ subfolder2 = subfolder
591
+ self.__escape_git_repositories_in_folder_internal(subfolder2, renamed_items)
592
+ return renamed_items
593
+
594
+ @GeneralUtilities.check_arguments
595
+ def deescape_git_repositories_in_folder(self, renamed_items: dict[str, str]):
596
+ for renamed_item, original_name in renamed_items.items():
597
+ os.rename(renamed_item, original_name)
598
+
599
+ @GeneralUtilities.check_arguments
600
+ def __sort_fmd(self, line: str):
601
+ splitted: list = line.split(";")
602
+ filetype: str = splitted[1]
603
+ if filetype == "d":
604
+ return -1
605
+ if filetype == "f":
606
+ return 1
607
+ return 0
608
+
609
+ @GeneralUtilities.check_arguments
610
+ def restore_filemetadata(self, folder: str, source_file: str, strict=False, encoding: str = "utf-8", create_folder_is_not_exist: bool = True) -> None:
611
+ lines = GeneralUtilities.read_lines_from_file(source_file, encoding)
612
+ lines.sort(key=self.__sort_fmd)
613
+ for line in lines:
614
+ splitted: list = line.split(";")
615
+ full_path_of_file_or_folder: str = os.path.join(folder, splitted[0])
616
+ filetype: str = splitted[1]
617
+ user: str = splitted[2]
618
+ permissions: str = splitted[3]
619
+ if filetype == "d" and create_folder_is_not_exist and not os.path.isdir(full_path_of_file_or_folder):
620
+ GeneralUtilities.ensure_directory_exists(full_path_of_file_or_folder)
621
+ if (filetype == "f" and os.path.isfile(full_path_of_file_or_folder)) or (filetype == "d" and os.path.isdir(full_path_of_file_or_folder)):
622
+ self.set_owner(full_path_of_file_or_folder, user, os.name != 'nt')
623
+ self.set_permission(full_path_of_file_or_folder, permissions)
624
+ else:
625
+ if strict:
626
+ if filetype == "f":
627
+ filetype_full = "File"
628
+ elif filetype == "d":
629
+ filetype_full = "Directory"
630
+ else:
631
+ raise ValueError(f"Unknown filetype: {GeneralUtilities.str_none_safe(filetype)}")
632
+ raise ValueError(f"{filetype_full} '{full_path_of_file_or_folder}' does not exist")
633
+
634
+ @GeneralUtilities.check_arguments
635
+ def __calculate_lengh_in_seconds(self, filename: str, folder: str) -> float:
636
+ argument = ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', filename]
637
+ result = self.run_program_argsasarray("ffprobe", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
638
+ return float(result[1].replace('\n', ''))
639
+
640
+ @GeneralUtilities.check_arguments
641
+ def __create_thumbnails(self, filename: str, fps: str, folder: str, tempname_for_thumbnails: str) -> list[str]:
642
+ argument = ['-i', filename, '-r', str(fps), '-vf', 'scale=-1:120', '-vcodec', 'png', f'{tempname_for_thumbnails}-%002d.png']
643
+ self.run_program_argsasarray("ffmpeg", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
644
+ files = GeneralUtilities.get_direct_files_of_folder(folder)
645
+ result: list[str] = []
646
+ regex = "^"+re.escape(tempname_for_thumbnails)+"\\-\\d+\\.png$"
647
+ regex_for_files = re.compile(regex)
648
+ for file in files:
649
+ filename = os.path.basename(file)
650
+ if regex_for_files.match(filename):
651
+ result.append(file)
652
+ GeneralUtilities.assert_condition(0 < len(result), "No thumbnail-files found.")
653
+ return result
654
+
655
+ @GeneralUtilities.check_arguments
656
+ def __create_thumbnail(self, outputfilename: str, folder: str, length_in_seconds: float, tempname_for_thumbnails: str, amount_of_images: int) -> None:
657
+ duration = timedelta(seconds=length_in_seconds)
658
+ info = GeneralUtilities.timedelta_to_simple_string(duration)
659
+ rows: int = 5
660
+ columns: int = math.ceil(amount_of_images/rows)
661
+ argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{rows}x{columns}', f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
662
+ self.run_program_argsasarray("montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
663
+
664
+ @GeneralUtilities.check_arguments
665
+ def __roundup(self, x: float, places: int) -> int:
666
+ d = 10 ** places
667
+ if x < 0:
668
+ return math.floor(x * d) / d
669
+ else:
670
+ return math.ceil(x * d) / d
671
+
672
+ @GeneralUtilities.check_arguments
673
+ def generate_thumbnail(self, file: str, frames_per_second: str, tempname_for_thumbnails: str = None, hook=None) -> None:
674
+ if tempname_for_thumbnails is None:
675
+ tempname_for_thumbnails = "t_"+str(uuid.uuid4())
676
+
677
+ file = GeneralUtilities.resolve_relative_path_from_current_working_directory(file)
678
+ filename = os.path.basename(file)
679
+ folder = os.path.dirname(file)
680
+ filename_without_extension = Path(file).stem
681
+ preview_files: list[str] = []
682
+ try:
683
+ length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
684
+ if (frames_per_second.endswith("fps")):
685
+ # frames per second, example: frames_per_second="20fps" => 20 frames per second
686
+ frames_per_second = self.__roundup(float(frames_per_second[:-3]), 2)
687
+ frames_per_second_as_string = str(frames_per_second)
688
+ amounf_of_previewframes = int(math.floor(length_in_seconds*frames_per_second))
689
+ else:
690
+ # concrete amount of frame, examples: frames_per_second="16" => 16 frames for entire video
691
+ amounf_of_previewframes = int(float(frames_per_second))
692
+ # self.roundup((amounf_of_previewframes-2)/length_in_seconds, 2)
693
+ frames_per_second_as_string = f"{amounf_of_previewframes-2}/{length_in_seconds}"
694
+ preview_files = self.__create_thumbnails(filename, frames_per_second_as_string, folder, tempname_for_thumbnails)
695
+ if hook is not None:
696
+ hook(file, preview_files)
697
+ actual_amounf_of_previewframes = len(preview_files)
698
+ self.__create_thumbnail(filename_without_extension, folder, length_in_seconds, tempname_for_thumbnails, actual_amounf_of_previewframes)
699
+ finally:
700
+ for thumbnail_to_delete in preview_files:
701
+ os.remove(thumbnail_to_delete)
702
+
703
+ @GeneralUtilities.check_arguments
704
+ def extract_pdf_pages(self, file: str, from_page: int, to_page: int, outputfile: str) -> None:
705
+ pdf_reader = PyPDF2.PdfReader(file)
706
+ pdf_writer = PyPDF2.PdfWriter()
707
+ start = from_page
708
+ end = to_page
709
+ while start <= end:
710
+ pdf_writer.add_page(pdf_reader.pages[start-1])
711
+ start += 1
712
+ with open(outputfile, 'wb') as out:
713
+ pdf_writer.write(out)
714
+
715
+ @GeneralUtilities.check_arguments
716
+ def merge_pdf_files(self, files: list[str], outputfile: str) -> None:
717
+ # TODO add wildcard-option
718
+ pdfFileMerger = PyPDF2.PdfFileMerger()
719
+ for file in files:
720
+ pdfFileMerger.append(file.strip())
721
+ pdfFileMerger.write(outputfile)
722
+ pdfFileMerger.close()
723
+
724
+ @GeneralUtilities.check_arguments
725
+ def pdf_to_image(self, file: str, outputfilename_without_extension: str) -> None:
726
+ raise ValueError("Function currently not available")
727
+ # PyMuPDF can be used for that but sometimes it throws
728
+ # "ImportError: DLL load failed while importing _fitz: Das angegebene Modul wurde nicht gefunden."
729
+
730
+ # doc = None # fitz.open(file)
731
+ # for i, page in enumerate(doc):
732
+ # pix = page.get_pixmap()
733
+ # img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
734
+ # img.save(f"{outputfilename_without_extension}_{i}.png", "PNG")
735
+
736
+ @GeneralUtilities.check_arguments
737
+ def show_missing_files(self, folderA: str, folderB: str):
738
+ for file in GeneralUtilities.get_missing_files(folderA, folderB):
739
+ GeneralUtilities.write_message_to_stdout(file)
740
+
741
+ @GeneralUtilities.check_arguments
742
+ def SCCreateEmptyFileWithSpecificSize(self, name: str, size_string: str) -> int:
743
+ if size_string.isdigit():
744
+ size = int(size_string)
745
+ else:
746
+ if len(size_string) >= 3:
747
+ if (size_string.endswith("kb")):
748
+ size = int(size_string[:-2]) * pow(10, 3)
749
+ elif (size_string.endswith("mb")):
750
+ size = int(size_string[:-2]) * pow(10, 6)
751
+ elif (size_string.endswith("gb")):
752
+ size = int(size_string[:-2]) * pow(10, 9)
753
+ elif (size_string.endswith("kib")):
754
+ size = int(size_string[:-3]) * pow(2, 10)
755
+ elif (size_string.endswith("mib")):
756
+ size = int(size_string[:-3]) * pow(2, 20)
757
+ elif (size_string.endswith("gib")):
758
+ size = int(size_string[:-3]) * pow(2, 30)
759
+ else:
760
+ GeneralUtilities.write_message_to_stderr("Wrong format")
761
+ return 1
762
+ else:
763
+ GeneralUtilities.write_message_to_stderr("Wrong format")
764
+ return 1
765
+ with open(name, "wb") as f:
766
+ f.seek(size-1)
767
+ f.write(b"\0")
768
+ return 0
769
+
770
+ @GeneralUtilities.check_arguments
771
+ def SCCreateHashOfAllFiles(self, folder: str) -> None:
772
+ for file in GeneralUtilities.absolute_file_paths(folder):
773
+ with open(file+".sha256", "w+", encoding="utf-8") as f:
774
+ f.write(GeneralUtilities.get_sha256_of_file(file))
775
+
776
+ @GeneralUtilities.check_arguments
777
+ def SCCreateSimpleMergeWithoutRelease(self, repository: str, sourcebranch: str, targetbranch: str, remotename: str, remove_source_branch: bool) -> None:
778
+ commitid = self.git_merge(repository, sourcebranch, targetbranch, False, True)
779
+ self.git_merge(repository, targetbranch, sourcebranch, True, True)
780
+ created_version = self.get_semver_version_from_gitversion(repository)
781
+ self.git_create_tag(repository, commitid, f"v{created_version}", True)
782
+ self.git_push(repository, remotename, targetbranch, targetbranch, False, True)
783
+ if (GeneralUtilities.string_has_nonwhitespace_content(remotename)):
784
+ self.git_push(repository, remotename, sourcebranch, sourcebranch, False, True)
785
+ if (remove_source_branch):
786
+ self.git_remove_branch(repository, sourcebranch)
787
+
788
+ @GeneralUtilities.check_arguments
789
+ def sc_organize_lines_in_file(self, file: str, encoding: str, sort: bool = False, remove_duplicated_lines: bool = False, ignore_first_line: bool = False, remove_empty_lines: bool = True, ignored_start_character: list = list()) -> int:
790
+ if os.path.isfile(file):
791
+
792
+ # read file
793
+ lines = GeneralUtilities.read_lines_from_file(file, encoding)
794
+ if (len(lines) == 0):
795
+ return 0
796
+
797
+ # store first line if desiredpopd
798
+
799
+ if (ignore_first_line):
800
+ first_line = lines.pop(0)
801
+
802
+ # remove empty lines if desired
803
+ if remove_empty_lines:
804
+ temp = lines
805
+ lines = []
806
+ for line in temp:
807
+ if (not (GeneralUtilities.string_is_none_or_whitespace(line))):
808
+ lines.append(line)
809
+
810
+ # remove duplicated lines if desired
811
+ if remove_duplicated_lines:
812
+ lines = GeneralUtilities.remove_duplicates(lines)
813
+
814
+ # sort lines if desired
815
+ if sort:
816
+ lines = sorted(lines, key=lambda singleline: self.__adapt_line_for_sorting(singleline, ignored_start_character))
817
+
818
+ # reinsert first line
819
+ if ignore_first_line:
820
+ lines.insert(0, first_line)
821
+
822
+ # write result to file
823
+ GeneralUtilities.write_lines_to_file(file, lines, encoding)
824
+
825
+ return 0
826
+ else:
827
+ GeneralUtilities.write_message_to_stdout(f"File '{file}' does not exist")
828
+ return 1
829
+
830
+ @GeneralUtilities.check_arguments
831
+ def __adapt_line_for_sorting(self, line: str, ignored_start_characters: list):
832
+ result = line.lower()
833
+ while len(result) > 0 and result[0] in ignored_start_characters:
834
+ result = result[1:]
835
+ return result
836
+
837
+ @GeneralUtilities.check_arguments
838
+ def SCGenerateSnkFiles(self, outputfolder, keysize=4096, amountofkeys=10) -> int:
839
+ GeneralUtilities.ensure_directory_exists(outputfolder)
840
+ for _ in range(amountofkeys):
841
+ file = os.path.join(outputfolder, str(uuid.uuid4())+".snk")
842
+ argument = f"-k {keysize} {file}"
843
+ self.run_program("sn", argument, outputfolder)
844
+
845
+ @GeneralUtilities.check_arguments
846
+ def __merge_files(self, sourcefile: str, targetfile: str) -> None:
847
+ with open(sourcefile, "rb") as f:
848
+ source_data = f.read()
849
+ with open(targetfile, "ab") as fout:
850
+ merge_separator = [0x0A]
851
+ fout.write(bytes(merge_separator))
852
+ fout.write(source_data)
853
+
854
+ @GeneralUtilities.check_arguments
855
+ def __process_file(self, file: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
856
+ new_filename = os.path.join(os.path.dirname(file), os.path.basename(file).replace(substringInFilename, newSubstringInFilename))
857
+ if file != new_filename:
858
+ if os.path.isfile(new_filename):
859
+ if filecmp.cmp(file, new_filename):
860
+ send2trash.send2trash(file)
861
+ else:
862
+ if conflictResolveMode == "ignore":
863
+ pass
864
+ elif conflictResolveMode == "preservenewest":
865
+ if (os.path.getmtime(file) - os.path.getmtime(new_filename) > 0):
866
+ send2trash.send2trash(file)
867
+ else:
868
+ send2trash.send2trash(new_filename)
869
+ os.rename(file, new_filename)
870
+ elif (conflictResolveMode == "merge"):
871
+ self.__merge_files(file, new_filename)
872
+ send2trash.send2trash(file)
873
+ else:
874
+ raise ValueError('Unknown conflict resolve mode')
875
+ else:
876
+ os.rename(file, new_filename)
877
+
878
+ @GeneralUtilities.check_arguments
879
+ def SCReplaceSubstringsInFilenames(self, folder: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
880
+ for file in GeneralUtilities.absolute_file_paths(folder):
881
+ self.__process_file(file, substringInFilename, newSubstringInFilename, conflictResolveMode)
882
+
883
+ @GeneralUtilities.check_arguments
884
+ def __check_file(self, file: str, searchstring: str) -> None:
885
+ bytes_ascii = bytes(searchstring, "ascii")
886
+ # often called "unicode-encoding"
887
+ bytes_utf16 = bytes(searchstring, "utf-16")
888
+ bytes_utf8 = bytes(searchstring, "utf-8")
889
+ with open(file, mode='rb') as file_object:
890
+ content = file_object.read()
891
+ if bytes_ascii in content:
892
+ GeneralUtilities.write_message_to_stdout(file)
893
+ elif bytes_utf16 in content:
894
+ GeneralUtilities.write_message_to_stdout(file)
895
+ elif bytes_utf8 in content:
896
+ GeneralUtilities.write_message_to_stdout(file)
897
+
898
+ @GeneralUtilities.check_arguments
899
+ def SCSearchInFiles(self, folder: str, searchstring: str) -> None:
900
+ for file in GeneralUtilities.absolute_file_paths(folder):
901
+ self.__check_file(file, searchstring)
902
+
903
+ @GeneralUtilities.check_arguments
904
+ def __print_qr_code_by_csv_line(self, displayname: str, website: str, emailaddress: str, key: str, period: str) -> None:
905
+ qrcode_content = f"otpauth://totp/{website}:{emailaddress}?secret={key}&issuer={displayname}&period={period}"
906
+ GeneralUtilities.write_message_to_stdout(
907
+ f"{displayname} ({emailaddress}):")
908
+ GeneralUtilities.write_message_to_stdout(qrcode_content)
909
+ qr = qrcode.QRCode()
910
+ qr.add_data(qrcode_content)
911
+ f = io.StringIO()
912
+ qr.print_ascii(out=f)
913
+ f.seek(0)
914
+ GeneralUtilities.write_message_to_stdout(f.read())
915
+
916
+ @GeneralUtilities.check_arguments
917
+ def SCShow2FAAsQRCode(self, csvfile: str) -> None:
918
+ separator_line = "--------------------------------------------------------"
919
+ lines = GeneralUtilities.read_csv_file(csvfile, True)
920
+ lines.sort(key=lambda items: ''.join(items).lower())
921
+ for line in lines:
922
+ GeneralUtilities.write_message_to_stdout(separator_line)
923
+ self.__print_qr_code_by_csv_line(
924
+ line[0], line[1], line[2], line[3], line[4])
925
+ GeneralUtilities.write_message_to_stdout(separator_line)
926
+
927
+ @GeneralUtilities.check_arguments
928
+ def SCCalculateBitcoinBlockHash(self, block_version_number: str, previousblockhash: str, transactionsmerkleroot: str, timestamp: str, target: str, nonce: str) -> str:
929
+ # Example-values:
930
+ # block_version_number: "00000020"
931
+ # previousblockhash: "66720b99e07d284bd4fe67ff8c49a5db1dd8514fcdab61000000000000000000"
932
+ # transactionsmerkleroot: "7829844f4c3a41a537b3131ca992643eaa9d093b2383e4cdc060ad7dc5481187"
933
+ # timestamp: "51eb505a"
934
+ # target: "c1910018"
935
+ # nonce: "de19b302"
936
+ header = str(block_version_number + previousblockhash + transactionsmerkleroot + timestamp + target + nonce)
937
+ return binascii.hexlify(hashlib.sha256(hashlib.sha256(binascii.unhexlify(header)).digest()).digest()[::-1]).decode('utf-8')
938
+
939
+ @GeneralUtilities.check_arguments
940
+ def SCChangeHashOfProgram(self, inputfile: str) -> None:
941
+ valuetoappend = str(uuid.uuid4())
942
+
943
+ outputfile = inputfile + '.modified'
944
+
945
+ shutil.copy2(inputfile, outputfile)
946
+ with open(outputfile, 'a', encoding="utf-8") as file:
947
+ # TODO use rcedit for .exe-files instead of appending valuetoappend ( https://github.com/electron/rcedit/ )
948
+ # background: you can retrieve the "original-filename" from the .exe-file like discussed here:
949
+ # https://security.stackexchange.com/questions/210843/ is-it-possible-to-change-original-filename-of-an-exe
950
+ # so removing the original filename with rcedit is probably a better way to make it more difficult to detect the programname.
951
+ # this would obviously also change the hashvalue of the program so appending a whitespace is not required anymore.
952
+ file.write(valuetoappend)
953
+
954
+ @GeneralUtilities.check_arguments
955
+ def __adjust_folder_name(self, folder: str) -> str:
956
+ result = os.path.dirname(folder).replace("\\", "/")
957
+ if result == "/":
958
+ return ""
959
+ else:
960
+ return result
961
+
962
+ @GeneralUtilities.check_arguments
963
+ def __create_iso(self, folder, iso_file) -> None:
964
+ created_directories = []
965
+ files_directory = "FILES"
966
+ iso = pycdlib.PyCdlib()
967
+ iso.new()
968
+ files_directory = files_directory.upper()
969
+ iso.add_directory("/" + files_directory)
970
+ created_directories.append("/" + files_directory)
971
+ for root, _, files in os.walk(folder):
972
+ for file in files:
973
+ full_path = os.path.join(root, file)
974
+ with (open(full_path, "rb").read()) as text_io_wrapper:
975
+ content = text_io_wrapper
976
+ path_in_iso = '/' + files_directory + \
977
+ self.__adjust_folder_name(full_path[len(folder)::1]).upper()
978
+ if path_in_iso not in created_directories:
979
+ iso.add_directory(path_in_iso)
980
+ created_directories.append(path_in_iso)
981
+ iso.add_fp(BytesIO(content), len(content), path_in_iso + '/' + file.upper() + ';1')
982
+ iso.write(iso_file)
983
+ iso.close()
984
+
985
+ @GeneralUtilities.check_arguments
986
+ def SCCreateISOFileWithObfuscatedFiles(self, inputfolder: str, outputfile: str, printtableheadline, createisofile, extensions) -> None:
987
+ if (os.path.isdir(inputfolder)):
988
+ namemappingfile = "name_map.csv"
989
+ files_directory = inputfolder
990
+ files_directory_obf = f"{files_directory}_Obfuscated"
991
+ self.SCObfuscateFilesFolder(
992
+ inputfolder, printtableheadline, namemappingfile, extensions)
993
+ os.rename(namemappingfile, os.path.join(
994
+ files_directory_obf, namemappingfile))
995
+ if createisofile:
996
+ self.__create_iso(files_directory_obf, outputfile)
997
+ shutil.rmtree(files_directory_obf)
998
+ else:
999
+ raise ValueError(f"Directory not found: '{inputfolder}'")
1000
+
1001
+ @GeneralUtilities.check_arguments
1002
+ def SCFilenameObfuscator(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
1003
+ obfuscate_all_files = extensions == "*"
1004
+ if (obfuscate_all_files):
1005
+ obfuscate_file_extensions = None
1006
+ else:
1007
+ obfuscate_file_extensions = extensions.split(",")
1008
+ if (os.path.isdir(inputfolder)):
1009
+ printtableheadline = GeneralUtilities.string_to_boolean(
1010
+ printtableheadline)
1011
+ files = []
1012
+ if not os.path.isfile(namemappingfile):
1013
+ with open(namemappingfile, "a", encoding="utf-8"):
1014
+ pass
1015
+ if printtableheadline:
1016
+ GeneralUtilities.append_line_to_file(
1017
+ namemappingfile, "Original filename;new filename;SHA2-hash of file")
1018
+ for file in GeneralUtilities.absolute_file_paths(inputfolder):
1019
+ if os.path.isfile(os.path.join(inputfolder, file)):
1020
+ if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
1021
+ files.append(file)
1022
+ for file in files:
1023
+ hash_value = GeneralUtilities.get_sha256_of_file(file)
1024
+ extension = Path(file).suffix
1025
+ new_file_name_without_path = str(uuid.uuid4())[0:8] + extension
1026
+ new_file_name = os.path.join(
1027
+ os.path.dirname(file), new_file_name_without_path)
1028
+ os.rename(file, new_file_name)
1029
+ GeneralUtilities.append_line_to_file(namemappingfile, os.path.basename(file) + ";" + new_file_name_without_path + ";" + hash_value)
1030
+ else:
1031
+ raise ValueError(f"Directory not found: '{inputfolder}'")
1032
+
1033
+ @GeneralUtilities.check_arguments
1034
+ def __extension_matchs(self, file: str, obfuscate_file_extensions) -> bool:
1035
+ for extension in obfuscate_file_extensions:
1036
+ if file.lower().endswith("."+extension.lower()):
1037
+ return True
1038
+ return False
1039
+
1040
+ @GeneralUtilities.check_arguments
1041
+ def SCHealthcheck(self, file: str) -> int:
1042
+ lines = GeneralUtilities.read_lines_from_file(file)
1043
+ for line in reversed(lines):
1044
+ if not GeneralUtilities.string_is_none_or_whitespace(line):
1045
+ if "RunningHealthy (" in line: # TODO use regex
1046
+ GeneralUtilities.write_message_to_stderr(f"Healthy running due to line '{line}' in file '{file}'.")
1047
+ return 0
1048
+ else:
1049
+ GeneralUtilities.write_message_to_stderr(f"Not healthy running due to line '{line}' in file '{file}'.")
1050
+ return 1
1051
+ GeneralUtilities.write_message_to_stderr(f"No valid line found for healthycheck in file '{file}'.")
1052
+ return 2
1053
+
1054
+ @GeneralUtilities.check_arguments
1055
+ def SCObfuscateFilesFolder(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
1056
+ obfuscate_all_files = extensions == "*"
1057
+ if (obfuscate_all_files):
1058
+ obfuscate_file_extensions = None
1059
+ else:
1060
+ if "," in extensions:
1061
+ obfuscate_file_extensions = extensions.split(",")
1062
+ else:
1063
+ obfuscate_file_extensions = [extensions]
1064
+ newd = inputfolder+"_Obfuscated"
1065
+ shutil.copytree(inputfolder, newd)
1066
+ inputfolder = newd
1067
+ if (os.path.isdir(inputfolder)):
1068
+ for file in GeneralUtilities.absolute_file_paths(inputfolder):
1069
+ if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
1070
+ self.SCChangeHashOfProgram(file)
1071
+ os.remove(file)
1072
+ os.rename(file + ".modified", file)
1073
+ self.SCFilenameObfuscator(inputfolder, printtableheadline, namemappingfile, extensions)
1074
+ else:
1075
+ raise ValueError(f"Directory not found: '{inputfolder}'")
1076
+
1077
+ @GeneralUtilities.check_arguments
1078
+ def get_services_from_yaml_file(self, yaml_file: str) -> list[str]:
1079
+ with open(yaml_file, encoding="utf-8") as stream:
1080
+ loaded = yaml.safe_load(stream)
1081
+ services = loaded["services"]
1082
+ result = list(services.keys())
1083
+ return result
1084
+
1085
+ @GeneralUtilities.check_arguments
1086
+ def kill_docker_container(self, container_name: str) -> None:
1087
+ self.run_program("docker", f"container rm -f {container_name}")
1088
+
1089
+ @GeneralUtilities.check_arguments
1090
+ def get_docker_debian_version(self, image_tag: str) -> str:
1091
+ result = ScriptCollectionCore().run_program_argsasarray("docker", ['run', f'debian:{image_tag}', 'bash', '-c', 'apt-get -y update && apt-get -y install lsb-release && lsb_release -cs'])
1092
+ result_line = GeneralUtilities.string_to_lines(result[1])[-1]
1093
+ return result_line
1094
+
1095
+ @GeneralUtilities.check_arguments
1096
+ def get_latest_tor_version_of_debian_repository(self, debian_version: str) -> str:
1097
+ package_url: str = f"https://deb.torproject.org/torproject.org/dists/{debian_version}/main/binary-amd64/Packages"
1098
+ headers = {'Cache-Control': 'no-cache'}
1099
+ r = requests.get(package_url, timeout=5, headers=headers)
1100
+ if r.status_code != 200:
1101
+ raise ValueError(f"Checking for latest tor package resulted in HTTP-response-code {r.status_code}.")
1102
+ lines = GeneralUtilities.string_to_lines(GeneralUtilities.bytes_to_string(r.content))
1103
+ version_line_prefix = "Version: "
1104
+ version_content_line = [line for line in lines if line.startswith(version_line_prefix)][1]
1105
+ version_with_overhead = version_content_line[len(version_line_prefix):]
1106
+ tor_version = version_with_overhead.split("~")[0]
1107
+ return tor_version
1108
+
1109
+ def run_testcases_for_python_project(self, repository_folder: str):
1110
+ self.run_program("coverage", "run -m pytest", repository_folder)
1111
+ self.run_program("coverage", "xml", repository_folder)
1112
+ GeneralUtilities.ensure_directory_exists(os.path.join(repository_folder, "Other/TestCoverage"))
1113
+ coveragefile = os.path.join(repository_folder, "Other/TestCoverage/TestCoverage.xml")
1114
+ GeneralUtilities.ensure_file_does_not_exist(coveragefile)
1115
+ os.rename(os.path.join(repository_folder, "coverage.xml"), coveragefile)
1116
+
1117
+ @GeneralUtilities.check_arguments
1118
+ def get_file_permission(self, file: str) -> str:
1119
+ """This function returns an usual octet-triple, for example "700"."""
1120
+ ls_output: str = self.run_ls_for_folder(file)
1121
+ return self.__get_file_permission_helper(ls_output)
1122
+
1123
+ @GeneralUtilities.check_arguments
1124
+ def __get_file_permission_helper(self, permissions: str) -> str:
1125
+ return str(self.__to_octet(permissions[0:3])) + str(self.__to_octet(permissions[3:6]))+str(self.__to_octet(permissions[6:9]))
1126
+
1127
+ @GeneralUtilities.check_arguments
1128
+ def __to_octet(self, string: str) -> int:
1129
+ return int(self.__to_octet_helper(string[0]) + self.__to_octet_helper(string[1])+self.__to_octet_helper(string[2]), 2)
1130
+
1131
+ @GeneralUtilities.check_arguments
1132
+ def __to_octet_helper(self, string: str) -> str:
1133
+ if (string == "-"):
1134
+ return "0"
1135
+ else:
1136
+ return "1"
1137
+
1138
+ @GeneralUtilities.check_arguments
1139
+ def get_file_owner(self, file: str) -> str:
1140
+ """This function returns the user and the group in the format "user:group"."""
1141
+ ls_output: str = self.run_ls_for_folder(file)
1142
+ return self.__get_file_owner_helper(ls_output)
1143
+
1144
+ @GeneralUtilities.check_arguments
1145
+ def __get_file_owner_helper(self, ls_output: str) -> str:
1146
+ splitted = ls_output.split()
1147
+ return f"{splitted[2]}:{splitted[3]}"
1148
+
1149
+ @GeneralUtilities.check_arguments
1150
+ def get_file_owner_and_file_permission(self, file: str) -> str:
1151
+ ls_output: str = self.run_ls_for_folder(file)
1152
+ return [self.__get_file_owner_helper(ls_output), self.__get_file_permission_helper(ls_output)]
1153
+
1154
+ @GeneralUtilities.check_arguments
1155
+ def run_ls_for_folder(self, file_or_folder: str) -> str:
1156
+ file_or_folder = file_or_folder.replace("\\", "/")
1157
+ GeneralUtilities.assert_condition(os.path.isfile(file_or_folder) or os.path.isdir(file_or_folder), f"Can not execute 'ls -ld' because '{file_or_folder}' does not exist.")
1158
+ ls_result = self.run_program_argsasarray("ls", ["-ld", file_or_folder])
1159
+ GeneralUtilities.assert_condition(ls_result[0] == 0, f"'ls -ld {file_or_folder}' resulted in exitcode {str(ls_result[0])}. StdErr: {ls_result[2]}")
1160
+ GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(ls_result[1]), f"'ls -ld' of '{file_or_folder}' had an empty output. StdErr: '{ls_result[2]}'")
1161
+ GeneralUtilities.write_message_to_stdout(ls_result[1])
1162
+ output = ls_result[1]
1163
+ result = output.replace("\n", "")
1164
+ result = ' '.join(result.split()) # reduce multiple whitespaces to one
1165
+ return result
1166
+
1167
+ @GeneralUtilities.check_arguments
1168
+ def run_ls_for_folder_content(self, file_or_folder: str) -> list[str]:
1169
+ file_or_folder = file_or_folder.replace("\\", "/")
1170
+ GeneralUtilities.assert_condition(os.path.isfile(file_or_folder) or os.path.isdir(file_or_folder), f"Can not execute 'ls -la' because '{file_or_folder}' does not exist.")
1171
+ ls_result = self.run_program_argsasarray("ls", ["-la", file_or_folder])
1172
+ GeneralUtilities.assert_condition(ls_result[0] == 0, f"'ls -la {file_or_folder}' resulted in exitcode {str(ls_result[0])}. StdErr: {ls_result[2]}")
1173
+ GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(ls_result[1]), f"'ls -la' of '{file_or_folder}' had an empty output. StdErr: '{ls_result[2]}'")
1174
+ GeneralUtilities.write_message_to_stdout(ls_result[1])
1175
+ output = ls_result[1]
1176
+ result = output.split("\n")[3:] # skip the lines with "Total", "." and ".."
1177
+ result = [' '.join(line.split()) for line in result] # reduce multiple whitespaces to one
1178
+ return result
1179
+
1180
+ @GeneralUtilities.check_arguments
1181
+ def set_permission(self, file_or_folder: str, permissions: str, recursive: bool = False) -> None:
1182
+ """This function expects an usual octet-triple, for example "700"."""
1183
+ args = []
1184
+ if recursive:
1185
+ args.append("--recursive")
1186
+ args.append(permissions)
1187
+ args.append(file_or_folder)
1188
+ self.run_program_argsasarray("chmod", args)
1189
+
1190
+ @GeneralUtilities.check_arguments
1191
+ def set_owner(self, file_or_folder: str, owner: str, recursive: bool = False, follow_symlinks: bool = False) -> None:
1192
+ """This function expects the user and the group in the format "user:group"."""
1193
+ args = []
1194
+ if recursive:
1195
+ args.append("--recursive")
1196
+ if follow_symlinks:
1197
+ args.append("--no-dereference")
1198
+ args.append(owner)
1199
+ args.append(file_or_folder)
1200
+ self.run_program_argsasarray("chown", args)
1201
+
1202
+ # <run programs>
1203
+
1204
+ @GeneralUtilities.check_arguments
1205
+ def __run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> Popen:
1206
+ # Verbosity:
1207
+ # 0=Quiet (No output will be printed.)
1208
+ # 1=Normal (If the exitcode of the executed program is not 0 then the StdErr will be printed.)
1209
+ # 2=Full (Prints StdOut and StdErr of the executed program.)
1210
+ # 3=Verbose (Same as "Full" but with some more information.)
1211
+
1212
+ if isinstance(self.program_runner, ProgramRunnerEpew):
1213
+ custom_argument = CustomEpewArgument(print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, verbosity, arguments_for_log)
1214
+ popen: Popen = self.program_runner.run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, custom_argument, interactive)
1215
+ return popen
1216
+
1217
+ @staticmethod
1218
+ def __enqueue_output(file, queue):
1219
+ for line in iter(file.readline, ''):
1220
+ queue.put(line)
1221
+ file.close()
1222
+
1223
+ @staticmethod
1224
+ def __continue_process_reading(pid: int, p: Popen, q_stdout: Queue, q_stderr: Queue, reading_stdout_last_time_resulted_in_exception: bool, reading_stderr_last_time_resulted_in_exception: bool):
1225
+ if p.poll() is None:
1226
+ return True
1227
+
1228
+ # if reading_stdout_last_time_resulted_in_exception and reading_stderr_last_time_resulted_in_exception:
1229
+ # return False
1230
+
1231
+ if not q_stdout.empty():
1232
+ return True
1233
+
1234
+ if not q_stderr.empty():
1235
+ return True
1236
+
1237
+ return False
1238
+
1239
+ @staticmethod
1240
+ def __read_popen_pipes(p: Popen):
1241
+ p_id = p.pid
1242
+ with ThreadPoolExecutor(2) as pool:
1243
+ q_stdout = Queue()
1244
+ q_stderr = Queue()
1245
+
1246
+ pool.submit(ScriptCollectionCore.__enqueue_output, p.stdout, q_stdout)
1247
+ pool.submit(ScriptCollectionCore.__enqueue_output, p.stderr, q_stderr)
1248
+ reading_stdout_last_time_resulted_in_exception: bool = False
1249
+ reading_stderr_last_time_resulted_in_exception: bool = False
1250
+ while (ScriptCollectionCore.__continue_process_reading(p_id, p, q_stdout, q_stderr, reading_stdout_last_time_resulted_in_exception, reading_stderr_last_time_resulted_in_exception)):
1251
+ out_line = None
1252
+ err_line = None
1253
+ try:
1254
+ out_line = q_stdout.get_nowait()
1255
+ reading_stdout_last_time_resulted_in_exception = False
1256
+ except Empty:
1257
+ reading_stdout_last_time_resulted_in_exception = True
1258
+
1259
+ try:
1260
+ err_line = q_stderr.get_nowait()
1261
+ reading_stderr_last_time_resulted_in_exception = False
1262
+ except Empty:
1263
+ reading_stderr_last_time_resulted_in_exception = True
1264
+
1265
+ time.sleep(0.01)
1266
+
1267
+ yield (out_line, err_line)
1268
+
1269
+ # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
1270
+ @GeneralUtilities.check_arguments
1271
+ def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False) -> tuple[int, str, str, int]:
1272
+ # verbosity 1: No output will be logged.
1273
+ # verbosity 2: If the exitcode of the executed program is not 0 then the StdErr will be logged. This is supposed to be the default verbosity-level.
1274
+ # verbosity 3: Logs and prints StdOut and StdErr of the executed program in realtime.
1275
+ # verbosity 4: Same as loglevel 3 but with some more overhead-information.
1276
+ if self.call_program_runner_directly:
1277
+ return self.program_runner.run_program_argsasarray(program, arguments_as_array, working_directory, custom_argument, interactive)
1278
+ try:
1279
+ arguments_as_str = ' '.join(arguments_as_array)
1280
+ mock_loader_result = self.__try_load_mock(program, arguments_as_str, working_directory)
1281
+ if mock_loader_result[0]:
1282
+ return mock_loader_result[1]
1283
+
1284
+ working_directory = self.__adapt_workingdirectory(working_directory)
1285
+
1286
+ if arguments_for_log is None:
1287
+ arguments_for_log = arguments_as_array
1288
+
1289
+ arguments_for_log_as_string: str = ' '.join(arguments_for_log)
1290
+ cmd = f'{working_directory}>{program} {arguments_for_log_as_string}'
1291
+
1292
+ if GeneralUtilities.string_is_none_or_whitespace(title):
1293
+ info_for_log = cmd
1294
+ else:
1295
+ info_for_log = title
1296
+
1297
+ if verbosity >= 3:
1298
+ GeneralUtilities.write_message_to_stdout(f"Run '{info_for_log}'.")
1299
+
1300
+ print_live_output = 1 < verbosity
1301
+
1302
+ exit_code: int = None
1303
+ stdout: str = ""
1304
+ stderr: str = ""
1305
+ pid: int = None
1306
+
1307
+ with self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive) as process:
1308
+
1309
+ if log_file is not None:
1310
+ GeneralUtilities.ensure_file_exists(log_file)
1311
+ pid = process.pid
1312
+ for out_line_plain, err_line_plain in ScriptCollectionCore.__read_popen_pipes(process): # see https://stackoverflow.com/a/57084403/3905529
1313
+
1314
+ if out_line_plain is not None:
1315
+ out_line: str = None
1316
+ if isinstance(out_line_plain, str):
1317
+ out_line = out_line_plain
1318
+ elif isinstance(out_line_plain, bytes):
1319
+ out_line = GeneralUtilities.bytes_to_string(out_line_plain)
1320
+ else:
1321
+ raise ValueError(f"Unknown type of output: {str(type(out_line_plain))}")
1322
+
1323
+ if out_line is not None and GeneralUtilities.string_has_content(out_line):
1324
+ if out_line.endswith("\n"):
1325
+ out_line = out_line[:-1]
1326
+ if print_live_output:
1327
+ print(out_line, end='\n', file=sys.stdout, flush=True)
1328
+ if 0 < len(stdout):
1329
+ stdout = stdout+"\n"
1330
+ stdout = stdout+out_line
1331
+ if log_file is not None:
1332
+ GeneralUtilities.append_line_to_file(log_file, out_line)
1333
+
1334
+ if err_line_plain is not None:
1335
+ err_line: str = None
1336
+ if isinstance(err_line_plain, str):
1337
+ err_line = err_line_plain
1338
+ elif isinstance(err_line_plain, bytes):
1339
+ err_line = GeneralUtilities.bytes_to_string(err_line_plain)
1340
+ else:
1341
+ raise ValueError(f"Unknown type of output: {str(type(err_line_plain))}")
1342
+ if err_line is not None and GeneralUtilities.string_has_content(err_line):
1343
+ if err_line.endswith("\n"):
1344
+ err_line = err_line[:-1]
1345
+ if print_live_output:
1346
+ print(err_line, end='\n', file=sys.stderr, flush=True)
1347
+ if 0 < len(stderr):
1348
+ stderr = stderr+"\n"
1349
+ stderr = stderr+err_line
1350
+ if log_file is not None:
1351
+ GeneralUtilities.append_line_to_file(log_file, err_line)
1352
+
1353
+ exit_code = process.returncode
1354
+
1355
+ if throw_exception_if_exitcode_is_not_zero and exit_code != 0:
1356
+ raise ValueError(f"Program '{working_directory}>{program} {arguments_for_log_as_string}' resulted in exitcode {exit_code}. (StdOut: '{stdout}', StdErr: '{stderr}')")
1357
+
1358
+ GeneralUtilities.assert_condition(exit_code is not None, f"Exitcode of program-run of '{info_for_log}' is None.")
1359
+ result = (exit_code, stdout, stderr, pid)
1360
+ return result
1361
+ except Exception as e:
1362
+ raise e
1363
+
1364
+ # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
1365
+ @GeneralUtilities.check_arguments
1366
+ def run_program(self, program: str, arguments: str = "", working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False) -> tuple[int, str, str, int]:
1367
+ if self.call_program_runner_directly:
1368
+ return self.program_runner.run_program(program, arguments, working_directory, custom_argument, interactive)
1369
+ return self.run_program_argsasarray(program, GeneralUtilities.arguments_to_array(arguments), working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive)
1370
+
1371
+ # Return-values program_runner: Pid
1372
+ @GeneralUtilities.check_arguments
1373
+ def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
1374
+ if self.call_program_runner_directly:
1375
+ return self.program_runner.run_program_argsasarray_async(program, arguments_as_array, working_directory, custom_argument, interactive)
1376
+ mock_loader_result = self.__try_load_mock(program, ' '.join(arguments_as_array), working_directory)
1377
+ if mock_loader_result[0]:
1378
+ return mock_loader_result[1]
1379
+ process: Popen = self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
1380
+ return process.pid
1381
+
1382
+ # Return-values program_runner: Pid
1383
+ @GeneralUtilities.check_arguments
1384
+ def run_program_async(self, program: str, arguments: str = "", working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
1385
+ if self.call_program_runner_directly:
1386
+ return self.program_runner.run_program_argsasarray_async(program, arguments, working_directory, custom_argument, interactive)
1387
+ return self.run_program_argsasarray_async(program, GeneralUtilities.arguments_to_array(arguments), working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
1388
+
1389
+ @GeneralUtilities.check_arguments
1390
+ def __try_load_mock(self, program: str, arguments: str, working_directory: str) -> tuple[bool, tuple[int, str, str, int]]:
1391
+ if self.mock_program_calls:
1392
+ try:
1393
+ return [True, self.__get_mock_program_call(program, arguments, working_directory)]
1394
+ except LookupError:
1395
+ if not self.execute_program_really_if_no_mock_call_is_defined:
1396
+ raise
1397
+ return [False, None]
1398
+
1399
+ @GeneralUtilities.check_arguments
1400
+ def __adapt_workingdirectory(self, workingdirectory: str) -> str:
1401
+ if workingdirectory is None:
1402
+ return os.getcwd()
1403
+ else:
1404
+ return GeneralUtilities.resolve_relative_path_from_current_working_directory(workingdirectory)
1405
+
1406
+ @GeneralUtilities.check_arguments
1407
+ def verify_no_pending_mock_program_calls(self):
1408
+ if (len(self.__mocked_program_calls) > 0):
1409
+ raise AssertionError("The following mock-calls were not called:\n"+",\n ".join([self.__format_mock_program_call(r) for r in self.__mocked_program_calls]))
1410
+
1411
+ @GeneralUtilities.check_arguments
1412
+ def __format_mock_program_call(self, r) -> str:
1413
+ r: ScriptCollectionCore.__MockProgramCall = r
1414
+ return f"'{r.workingdirectory}>{r.program} {r.argument}' (" \
1415
+ f"exitcode: {GeneralUtilities.str_none_safe(str(r.exit_code))}, " \
1416
+ f"pid: {GeneralUtilities.str_none_safe(str(r.pid))}, "\
1417
+ f"stdout: {GeneralUtilities.str_none_safe(str(r.stdout))}, " \
1418
+ f"stderr: {GeneralUtilities.str_none_safe(str(r.stderr))})"
1419
+
1420
+ @GeneralUtilities.check_arguments
1421
+ def register_mock_program_call(self, program: str, argument: str, workingdirectory: str, result_exit_code: int, result_stdout: str, result_stderr: str, result_pid: int, amount_of_expected_calls=1):
1422
+ "This function is for test-purposes only"
1423
+ for _ in itertools.repeat(None, amount_of_expected_calls):
1424
+ mock_call = ScriptCollectionCore.__MockProgramCall()
1425
+ mock_call.program = program
1426
+ mock_call.argument = argument
1427
+ mock_call.workingdirectory = workingdirectory
1428
+ mock_call.exit_code = result_exit_code
1429
+ mock_call.stdout = result_stdout
1430
+ mock_call.stderr = result_stderr
1431
+ mock_call.pid = result_pid
1432
+ self.__mocked_program_calls.append(mock_call)
1433
+
1434
+ @GeneralUtilities.check_arguments
1435
+ def __get_mock_program_call(self, program: str, argument: str, workingdirectory: str):
1436
+ result: ScriptCollectionCore.__MockProgramCall = None
1437
+ for mock_call in self.__mocked_program_calls:
1438
+ if ((re.match(mock_call.program, program) is not None)
1439
+ and (re.match(mock_call.argument, argument) is not None)
1440
+ and (re.match(mock_call.workingdirectory, workingdirectory) is not None)):
1441
+ result = mock_call
1442
+ break
1443
+ if result is None:
1444
+ raise LookupError(f"Tried to execute mock-call '{workingdirectory}>{program} {argument}' but no mock-call was defined for that execution")
1445
+ else:
1446
+ self.__mocked_program_calls.remove(result)
1447
+ return (result.exit_code, result.stdout, result.stderr, result.pid)
1448
+
1449
+ @GeneralUtilities.check_arguments
1450
+ class __MockProgramCall:
1451
+ program: str
1452
+ argument: str
1453
+ workingdirectory: str
1454
+ exit_code: int
1455
+ stdout: str
1456
+ stderr: str
1457
+ pid: int
1458
+
1459
+ # </run programs>
1460
+
1461
+ @GeneralUtilities.check_arguments
1462
+ def extract_archive_with_7z(self, unzip_program_file: str, zipfile: str, password: str, output_directory: str) -> None:
1463
+ password_set = not password is None
1464
+ file_name = Path(zipfile).name
1465
+ file_folder = os.path.dirname(zipfile)
1466
+ argument = "x"
1467
+ if password_set:
1468
+ argument = f"{argument} -p\"{password}\""
1469
+ argument = f"{argument} -o {output_directory}"
1470
+ argument = f"{argument} {file_name}"
1471
+ return self.run_program(unzip_program_file, argument, file_folder)
1472
+
1473
+ @GeneralUtilities.check_arguments
1474
+ def get_internet_time(self) -> datetime:
1475
+ response = ntplib.NTPClient().request('pool.ntp.org')
1476
+ return datetime.fromtimestamp(response.tx_time)
1477
+
1478
+ @GeneralUtilities.check_arguments
1479
+ def system_time_equals_internet_time(self, maximal_tolerance_difference: timedelta) -> bool:
1480
+ return abs(datetime.now() - self.get_internet_time()) < maximal_tolerance_difference
1481
+
1482
+ @GeneralUtilities.check_arguments
1483
+ def system_time_equals_internet_time_with_default_tolerance(self) -> bool:
1484
+ return self.system_time_equals_internet_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
1485
+
1486
+ @GeneralUtilities.check_arguments
1487
+ def check_system_time(self, maximal_tolerance_difference: timedelta):
1488
+ if not self.system_time_equals_internet_time(maximal_tolerance_difference):
1489
+ raise ValueError("System time may be wrong")
1490
+
1491
+ @GeneralUtilities.check_arguments
1492
+ def check_system_time_with_default_tolerance(self) -> None:
1493
+ self.check_system_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
1494
+
1495
+ @GeneralUtilities.check_arguments
1496
+ def __get_default_tolerance_for_system_time_equals_internet_time(self) -> timedelta:
1497
+ return timedelta(hours=0, minutes=0, seconds=3)
1498
+
1499
+ @GeneralUtilities.check_arguments
1500
+ def increment_version(self, input_version: str, increment_major: bool, increment_minor: bool, increment_patch: bool) -> str:
1501
+ splitted = input_version.split(".")
1502
+ GeneralUtilities.assert_condition(len(splitted) == 3, f"Version '{input_version}' does not have the 'major.minor.patch'-pattern.")
1503
+ major = int(splitted[0])
1504
+ minor = int(splitted[1])
1505
+ patch = int(splitted[2])
1506
+ if increment_major:
1507
+ major = major+1
1508
+ if increment_minor:
1509
+ minor = minor+1
1510
+ if increment_patch:
1511
+ patch = patch+1
1512
+ return f"{major}.{minor}.{patch}"
1513
+
1514
+ @GeneralUtilities.check_arguments
1515
+ def get_semver_version_from_gitversion(self, repository_folder: str) -> str:
1516
+ if (self.git_repository_has_commits(repository_folder)):
1517
+ result = self.get_version_from_gitversion(repository_folder, "MajorMinorPatch")
1518
+ if self.git_repository_has_uncommitted_changes(repository_folder):
1519
+ if self.get_current_git_branch_has_tag(repository_folder):
1520
+ id_of_latest_tag = self.git_get_commitid_of_tag(repository_folder, self.get_latest_git_tag(repository_folder))
1521
+ current_commit = self.git_get_commit_id(repository_folder)
1522
+ current_commit_is_on_latest_tag = id_of_latest_tag == current_commit
1523
+ if current_commit_is_on_latest_tag:
1524
+ result = self.increment_version(result, False, False, True)
1525
+ else:
1526
+ result = "0.1.0"
1527
+ return result
1528
+
1529
+ @staticmethod
1530
+ @GeneralUtilities.check_arguments
1531
+ def is_patch_version(version_string: str) -> bool:
1532
+ return not version_string.endswith(".0")
1533
+
1534
+ @GeneralUtilities.check_arguments
1535
+ def get_version_from_gitversion(self, folder: str, variable: str) -> str:
1536
+ # called twice as workaround for issue 1877 in gitversion ( https://github.com/GitTools/GitVersion/issues/1877 )
1537
+ result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder, verbosity=0)
1538
+ result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder, verbosity=0)
1539
+ result = GeneralUtilities.strip_new_line_character(result[1])
1540
+
1541
+ return result
1542
+
1543
+ @GeneralUtilities.check_arguments
1544
+ def generate_certificate_authority(self, folder: str, name: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
1545
+ if days_until_expire is None:
1546
+ days_until_expire = 1825
1547
+ if password is None:
1548
+ password = GeneralUtilities.generate_password()
1549
+ GeneralUtilities.ensure_directory_exists(folder)
1550
+ self.run_program("openssl", f'req -new -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -days {days_until_expire} -nodes -x509 -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={name}/OU={subj_ou} -passout pass:{password} -keyout {name}.key -out {name}.crt', folder)
1551
+
1552
+ @GeneralUtilities.check_arguments
1553
+ def generate_certificate(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
1554
+ if days_until_expire is None:
1555
+ days_until_expire = 397
1556
+ if password is None:
1557
+ password = GeneralUtilities.generate_password()
1558
+ rsa_key_length = 4096
1559
+ self.run_program("openssl", f'genrsa -out {filename}.key {rsa_key_length}', folder)
1560
+ self.run_program("openssl", f'req -new -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou} -x509 -key {filename}.key -out {filename}.unsigned.crt -days {days_until_expire}', folder)
1561
+ self.run_program("openssl", f'pkcs12 -export -out {filename}.selfsigned.pfx -password pass:{password} -inkey {filename}.key -in {filename}.unsigned.crt', folder)
1562
+ GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.password"), password)
1563
+ GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.san.conf"), f"""[ req ]
1564
+ default_bits = {rsa_key_length}
1565
+ distinguished_name = req_distinguished_name
1566
+ req_extensions = v3_req
1567
+ default_md = sha256
1568
+ dirstring_type = nombstr
1569
+ prompt = no
1570
+
1571
+ [ req_distinguished_name ]
1572
+ countryName = {subj_c}
1573
+ stateOrProvinceName = {subj_st}
1574
+ localityName = {subj_l}
1575
+ organizationName = {subj_o}
1576
+ organizationUnit = {subj_ou}
1577
+ commonName = {domain}
1578
+
1579
+ [v3_req]
1580
+ subjectAltName = @subject_alt_name
1581
+
1582
+ [ subject_alt_name ]
1583
+ DNS = {domain}
1584
+ """)
1585
+
1586
+ @GeneralUtilities.check_arguments
1587
+ def generate_certificate_sign_request(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str) -> None:
1588
+ self.run_program("openssl", f'req -new -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou} -key {filename}.key -out {filename}.csr -config {filename}.san.conf', folder)
1589
+
1590
+ @GeneralUtilities.check_arguments
1591
+ def sign_certificate(self, folder: str, ca_folder: str, ca_name: str, domain: str, filename: str, days_until_expire: int = None) -> None:
1592
+ if days_until_expire is None:
1593
+ days_until_expire = 397
1594
+ ca = os.path.join(ca_folder, ca_name)
1595
+ password_file = os.path.join(folder, f"{filename}.password")
1596
+ password = GeneralUtilities.read_text_from_file(password_file)
1597
+ self.run_program("openssl", f'x509 -req -in {filename}.csr -CA {ca}.crt -CAkey {ca}.key -CAcreateserial -CAserial {ca}.srl -out {filename}.crt -days {days_until_expire} -sha256 -extensions v3_req -extfile {filename}.san.conf', folder)
1598
+ self.run_program("openssl", f'pkcs12 -export -out {filename}.pfx -inkey {filename}.key -in {filename}.crt -password pass:{password}', folder)
1599
+
1600
+ @GeneralUtilities.check_arguments
1601
+ def update_dependencies_of_python_in_requirementstxt_file(self, file: str, verbosity: int):
1602
+ lines = GeneralUtilities.read_lines_from_file(file)
1603
+ new_lines = []
1604
+ for line in lines:
1605
+ if GeneralUtilities.string_has_content(line):
1606
+ new_lines.append(self.__get_updated_line_for_python_requirements(line.strip()))
1607
+ GeneralUtilities.write_lines_to_file(file, new_lines)
1608
+
1609
+ @GeneralUtilities.check_arguments
1610
+ def __get_updated_line_for_python_requirements(self, line: str) -> str:
1611
+ if "==" in line or "<" in line:
1612
+ return line
1613
+ elif ">" in line:
1614
+ try:
1615
+ # line is something like "cyclonedx-bom>=2.0.2" and the function must return with the updated version
1616
+ # (something like "cyclonedx-bom>=2.11.0" for example)
1617
+ package = line.split(">")[0]
1618
+ operator = ">=" if ">=" in line else ">"
1619
+ headers = {'Cache-Control': 'no-cache'}
1620
+ response = requests.get(f'https://pypi.org/pypi/{package}/json', timeout=5, headers=headers)
1621
+ latest_version = response.json()['info']['version']
1622
+ # TODO update only minor- and patch-version
1623
+ # TODO print info if there is a new major-version
1624
+ return package+operator+latest_version
1625
+ except:
1626
+ return line
1627
+ else:
1628
+ raise ValueError(f'Unexpected line in requirements-file: "{line}"')
1629
+
1630
+ @GeneralUtilities.check_arguments
1631
+ def update_dependencies_of_python_in_setupcfg_file(self, setup_cfg_file: str, verbosity: int):
1632
+ lines = GeneralUtilities.read_lines_from_file(setup_cfg_file)
1633
+ new_lines = []
1634
+ requirement_parsing_mode = False
1635
+ for line in lines:
1636
+ new_line = line
1637
+ if (requirement_parsing_mode):
1638
+ if ("<" in line or "=" in line or ">" in line):
1639
+ updated_line = f" {self.__get_updated_line_for_python_requirements(line.strip())}"
1640
+ new_line = updated_line
1641
+ else:
1642
+ requirement_parsing_mode = False
1643
+ else:
1644
+ if line.startswith("install_requires ="):
1645
+ requirement_parsing_mode = True
1646
+ new_lines.append(new_line)
1647
+ GeneralUtilities.write_lines_to_file(setup_cfg_file, new_lines)
1648
+
1649
+ @GeneralUtilities.check_arguments
1650
+ def update_dependencies_of_dotnet_project(self, csproj_file: str, verbosity: int, ignored_dependencies: list[str]):
1651
+ folder = os.path.dirname(csproj_file)
1652
+ csproj_filename = os.path.basename(csproj_file)
1653
+ GeneralUtilities.write_message_to_stderr(f"Check for updates in {csproj_filename}")
1654
+ result = self.run_program("dotnet", f"list {csproj_filename} package --outdated", folder)
1655
+ for line in result[1].replace("\r", "").split("\n"):
1656
+ # Relevant output-lines are something like " > NJsonSchema 10.7.0 10.7.0 10.9.0"
1657
+ if ">" in line:
1658
+ package_name = line.replace(">", "").strip().split(" ")[0]
1659
+ if not (package_name in ignored_dependencies):
1660
+ GeneralUtilities.write_message_to_stderr(f"Update package {package_name}")
1661
+ self.run_program("dotnet", f"add {csproj_filename} package {package_name}", folder)
1662
+
1663
+ @GeneralUtilities.check_arguments
1664
+ def create_deb_package(self, toolname: str, binary_folder: str, control_file_content: str, deb_output_folder: str, verbosity: int, permission_of_executable_file_as_octet_triple: int) -> None:
1665
+
1666
+ # prepare
1667
+ GeneralUtilities.ensure_directory_exists(deb_output_folder)
1668
+ temp_folder = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
1669
+ GeneralUtilities.ensure_directory_exists(temp_folder)
1670
+ bin_folder = binary_folder
1671
+ tool_content_folder_name = toolname+"Content"
1672
+
1673
+ # create folder
1674
+ GeneralUtilities.ensure_directory_exists(temp_folder)
1675
+ control_content_folder_name = "controlcontent"
1676
+ packagecontent_control_folder = os.path.join(temp_folder, control_content_folder_name)
1677
+ GeneralUtilities.ensure_directory_exists(packagecontent_control_folder)
1678
+ data_content_folder_name = "datacontent"
1679
+ packagecontent_data_folder = os.path.join(temp_folder, data_content_folder_name)
1680
+ GeneralUtilities.ensure_directory_exists(packagecontent_data_folder)
1681
+ entireresult_content_folder_name = "entireresultcontent"
1682
+ packagecontent_entireresult_folder = os.path.join(temp_folder, entireresult_content_folder_name)
1683
+ GeneralUtilities.ensure_directory_exists(packagecontent_entireresult_folder)
1684
+
1685
+ # create "debian-binary"-file
1686
+ debianbinary_file = os.path.join(packagecontent_entireresult_folder, "debian-binary")
1687
+ GeneralUtilities.ensure_file_exists(debianbinary_file)
1688
+ GeneralUtilities.write_text_to_file(debianbinary_file, "2.0\n")
1689
+
1690
+ # create control-content
1691
+
1692
+ # conffiles
1693
+ conffiles_file = os.path.join(packagecontent_control_folder, "conffiles")
1694
+ GeneralUtilities.ensure_file_exists(conffiles_file)
1695
+
1696
+ # postinst-script
1697
+ postinst_file = os.path.join(packagecontent_control_folder, "postinst")
1698
+ GeneralUtilities.ensure_file_exists(postinst_file)
1699
+ exe_file = f"/usr/bin/{tool_content_folder_name}/{toolname}"
1700
+ link_file = f"/usr/bin/{toolname.lower()}"
1701
+ permission = str(permission_of_executable_file_as_octet_triple)
1702
+ GeneralUtilities.write_text_to_file(postinst_file, f"""#!/bin/sh
1703
+ ln -s {exe_file} {link_file}
1704
+ chmod {permission} {exe_file}
1705
+ chmod {permission} {link_file}
1706
+ """)
1707
+
1708
+ # control
1709
+ control_file = os.path.join(packagecontent_control_folder, "control")
1710
+ GeneralUtilities.ensure_file_exists(control_file)
1711
+ GeneralUtilities.write_text_to_file(control_file, control_file_content)
1712
+
1713
+ # md5sums
1714
+ md5sums_file = os.path.join(packagecontent_control_folder, "md5sums")
1715
+ GeneralUtilities.ensure_file_exists(md5sums_file)
1716
+
1717
+ # create data-content
1718
+
1719
+ # copy binaries
1720
+ usr_bin_folder = os.path.join(packagecontent_data_folder, "usr/bin")
1721
+ GeneralUtilities.ensure_directory_exists(usr_bin_folder)
1722
+ usr_bin_content_folder = os.path.join(usr_bin_folder, tool_content_folder_name)
1723
+ GeneralUtilities.copy_content_of_folder(bin_folder, usr_bin_content_folder)
1724
+
1725
+ # create debfile
1726
+ deb_filename = f"{toolname}.deb"
1727
+ self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/control.tar.gz", "*"], packagecontent_control_folder, verbosity=verbosity)
1728
+ self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/data.tar.gz", "*"], packagecontent_data_folder, verbosity=verbosity)
1729
+ self.run_program_argsasarray("ar", ["r", deb_filename, "debian-binary", "control.tar.gz", "data.tar.gz"], packagecontent_entireresult_folder, verbosity=verbosity)
1730
+ result_file = os.path.join(packagecontent_entireresult_folder, deb_filename)
1731
+ shutil.copy(result_file, os.path.join(deb_output_folder, deb_filename))
1732
+
1733
+ # cleanup
1734
+ GeneralUtilities.ensure_directory_does_not_exist(temp_folder)
1735
+
1736
+ @GeneralUtilities.check_arguments
1737
+ def update_year_in_copyright_tags(self, file: str) -> None:
1738
+ current_year = str(datetime.now().year)
1739
+ lines = GeneralUtilities.read_lines_from_file(file)
1740
+ lines_result = []
1741
+ for line in lines:
1742
+ if match := re.search("(.*<[Cc]opyright>.*)\\d\\d\\d\\d(.*<\\/[Cc]opyright>.*)", line):
1743
+ part1 = match.group(1)
1744
+ part2 = match.group(2)
1745
+ adapted = part1+current_year+part2
1746
+ else:
1747
+ adapted = line
1748
+ lines_result.append(adapted)
1749
+ GeneralUtilities.write_lines_to_file(file, lines_result)
1750
+
1751
+ @GeneralUtilities.check_arguments
1752
+ def update_year_in_first_line_of_file(self, file: str) -> None:
1753
+ current_year = str(datetime.now().year)
1754
+ lines = GeneralUtilities.read_lines_from_file(file)
1755
+ lines[0] = re.sub("\\d\\d\\d\\d", current_year, lines[0])
1756
+ GeneralUtilities.write_lines_to_file(file, lines)
1757
+
1758
+ @GeneralUtilities.check_arguments
1759
+ def get_external_ip(self) -> str:
1760
+ information = self.get_externalnetworkinformation_as_json_string()
1761
+ parsed = json.loads(information)
1762
+ return parsed.IPAddress
1763
+
1764
+ @GeneralUtilities.check_arguments
1765
+ def get_country_of_external_ip(self) -> str:
1766
+ information = self.get_externalnetworkinformation_as_json_string()
1767
+ parsed = json.loads(information)
1768
+ return parsed.Country
1769
+
1770
+ @GeneralUtilities.check_arguments
1771
+ def get_externalnetworkinformation_as_json_string(self) -> str:
1772
+ headers = {'Cache-Control': 'no-cache'}
1773
+ response = requests.get('https://clientinformation.anion327.de/API/v1/ClientInformationBackendController/Information', timeout=5, headers=headers)
1774
+ network_information_as_json_string = GeneralUtilities.bytes_to_string(response.content)
1775
+ return network_information_as_json_string
1776
+
1777
+ @GeneralUtilities.check_arguments
1778
+ def change_file_extensions(self, folder: str, from_extension: str, to_extension: str, recursive: bool, ignore_case: bool) -> None:
1779
+ extension_to_compare: str = None
1780
+ if ignore_case:
1781
+ extension_to_compare = from_extension.lower()
1782
+ else:
1783
+ extension_to_compare = from_extension
1784
+ for file in GeneralUtilities.get_direct_files_of_folder(folder):
1785
+ if (ignore_case and file.lower().endswith(f".{extension_to_compare}") or not ignore_case and file.endswith(f".{extension_to_compare}")):
1786
+ p = Path(file)
1787
+ p.rename(p.with_suffix('.'+to_extension))
1788
+ if recursive:
1789
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
1790
+ self.change_file_extensions(subfolder, from_extension, to_extension, recursive, ignore_case)
1791
+
1792
+ @GeneralUtilities.check_arguments
1793
+ def __add_chapter(self, main_reference_file, reference_content_folder, number: int, chaptertitle: str, content: str = None):
1794
+ if content is None:
1795
+ content = "TXDX add content here"
1796
+ filename = str(number).zfill(2)+"_"+chaptertitle.replace(' ', '-')
1797
+ file = f"{reference_content_folder}/{filename}.md"
1798
+ full_title = f"{number}. {chaptertitle}"
1799
+
1800
+ GeneralUtilities.append_line_to_file(main_reference_file, f"- [{full_title}](./{filename}.md)")
1801
+
1802
+ GeneralUtilities.ensure_file_exists(file)
1803
+ GeneralUtilities.write_text_to_file(file, f"""# {full_title}
1804
+
1805
+ {content}
1806
+ """.replace("XDX", "ODO"))
1807
+
1808
+ @GeneralUtilities.check_arguments
1809
+ def generate_arc42_reference_template(self, repository: str, productname: str = None, subfolder: str = None):
1810
+ productname: str
1811
+ if productname is None:
1812
+ productname = os.path.basename(repository)
1813
+ if subfolder is None:
1814
+ subfolder = "Other/Resources/Reference"
1815
+ reference_root_folder = f"{repository}/{subfolder}"
1816
+ reference_content_folder = reference_root_folder + "/Technical"
1817
+ if os.path.isdir(reference_root_folder):
1818
+ raise ValueError(f"The folder '{reference_root_folder}' does already exist.")
1819
+ GeneralUtilities.ensure_directory_exists(reference_root_folder)
1820
+ GeneralUtilities.ensure_directory_exists(reference_content_folder)
1821
+ main_reference_file = f"{reference_root_folder}/Reference.md"
1822
+ GeneralUtilities.ensure_file_exists(main_reference_file)
1823
+ GeneralUtilities.write_text_to_file(main_reference_file, f"""# {productname}
1824
+
1825
+ TXDX add minimal service-description here.
1826
+
1827
+ ## Technical documentation
1828
+
1829
+ """.replace("XDX", "ODO"))
1830
+ self.__add_chapter(main_reference_file, reference_content_folder, 1, 'Introduction and Goals', """## Overview
1831
+
1832
+ TXDX
1833
+
1834
+ ## Quality goals
1835
+
1836
+ TXDX
1837
+
1838
+ ## Stakeholder
1839
+
1840
+ | Name | How to contact | Reason |
1841
+ | ---- | -------------- | ------ |""")
1842
+ self.__add_chapter(main_reference_file, reference_content_folder, 2, 'Constraints', """## Technical constraints
1843
+
1844
+ | Constraint-identifier | Constraint | Reason |
1845
+ | --------------------- | ---------- | ------ |
1846
+
1847
+ ## Organizational constraints
1848
+
1849
+ | Constraint-identifier | Constraint | Reason |
1850
+ | --------------------- | ---------- | ------ |""")
1851
+ self.__add_chapter(main_reference_file, reference_content_folder, 3, 'Context and Scope', """## Context
1852
+
1853
+ TXDX
1854
+
1855
+ ## Scope
1856
+
1857
+ TXDX""")
1858
+ self.__add_chapter(main_reference_file, reference_content_folder, 4, 'Solution Strategy', """TXDX""")
1859
+ self.__add_chapter(main_reference_file, reference_content_folder, 5, 'Building Block View', """TXDX""")
1860
+ self.__add_chapter(main_reference_file, reference_content_folder, 6, 'Runtime View', """TXDX""")
1861
+ self.__add_chapter(main_reference_file, reference_content_folder, 7, 'Deployment View', """## Infrastructure-overview
1862
+
1863
+ TXDX
1864
+
1865
+ ## Infrastructure-requirements
1866
+
1867
+ TXDX
1868
+
1869
+ ## Deployment-proecsses
1870
+
1871
+ TXDX
1872
+ """)
1873
+ self.__add_chapter(main_reference_file, reference_content_folder, 8, 'Crosscutting Concepts', """TXDX""")
1874
+ self.__add_chapter(main_reference_file, reference_content_folder, 9, 'Architectural Decisions', """## Decision-board
1875
+
1876
+ | Decision-identifier | Date | Decision | Reason and notes |
1877
+ | ------------------- | ---- | -------- | ---------------- |""") # empty because there are no decsions yet
1878
+ self.__add_chapter(main_reference_file, reference_content_folder, 10, 'Quality Requirements', """TXDX""")
1879
+ self.__add_chapter(main_reference_file, reference_content_folder, 11, 'Risks and Technical Debt', """## Risks
1880
+
1881
+ Currently there are no known risks.
1882
+
1883
+ ## Technical debts
1884
+
1885
+ Currently there are no technical depts.""")
1886
+ self.__add_chapter(main_reference_file, reference_content_folder, 12, 'Glossary', """## Terms
1887
+
1888
+ | Term | Meaning |
1889
+ | ---- | ------- |
1890
+
1891
+ ## Abbreviations
1892
+
1893
+ | Abbreviation | Meaning |
1894
+ | ------------ | ------- |""")
1895
+
1896
+ GeneralUtilities.append_to_file(main_reference_file, """
1897
+
1898
+ ## Responsibilities
1899
+
1900
+ | Responsibility | Name and contact-information |
1901
+ | --------------- | ---------------------------- |
1902
+ | Pdocut-owner | TXDX |
1903
+ | Product-manager | TXDX |
1904
+ | Support | TXDX |
1905
+
1906
+ ## License & Pricing
1907
+
1908
+ TXDX
1909
+
1910
+ ## External resources
1911
+
1912
+ - [Repository](TXDX)
1913
+ - [Productive-System](TXDX)
1914
+ - [QualityCheck-system](TXDX)
1915
+
1916
+ """.replace("XDX", "ODO"))
1917
+
1918
+ @GeneralUtilities.check_arguments
1919
+ def run_with_timeout(self, method, timeout_in_seconds: float) -> bool:
1920
+ # Returns true if the method was terminated due to a timeout
1921
+ # Returns false if the method terminates in the given time
1922
+ p = multiprocessing.Process(target=method)
1923
+ p.start()
1924
+ p.join(timeout_in_seconds)
1925
+ if p.is_alive():
1926
+ p.kill()
1927
+ p.join()
1928
+ return True
1929
+ else:
1930
+ return False