ScriptCollection 3.5.10__py3-none-any.whl → 3.5.12__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1782 +1,1798 @@
1
- import time
2
- from datetime import timedelta, datetime
3
- import json
4
- import binascii
5
- import filecmp
6
- import hashlib
7
- from io import BufferedReader, BytesIO
8
- import itertools
9
- import math
10
- import os
11
- from pathlib import Path
12
- from random import randrange
13
- from subprocess import Popen
14
- import re
15
- import shutil
16
- import traceback
17
- import uuid
18
- import tempfile
19
- import io
20
- import requests
21
- import ntplib
22
- import yaml
23
- import qrcode
24
- import pycdlib
25
- import send2trash
26
- import PyPDF2
27
- from .GeneralUtilities import GeneralUtilities
28
- from .ProgramRunnerBase import ProgramRunnerBase
29
- from .ProgramRunnerPopen import ProgramRunnerPopen
30
- from .ProgramRunnerEpew import ProgramRunnerEpew, CustomEpewArgument
31
-
32
- version = "3.5.10"
33
- __version__ = version
34
-
35
-
36
- class ScriptCollectionCore:
37
-
38
- # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
39
- # Do not change this value for productive environments.
40
- mock_program_calls: bool = False
41
- # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
42
- execute_program_really_if_no_mock_call_is_defined: bool = False
43
- __mocked_program_calls: list = list()
44
- program_runner: ProgramRunnerBase = None
45
-
46
- def __init__(self):
47
- self.program_runner = ProgramRunnerPopen()
48
-
49
- @staticmethod
50
- @GeneralUtilities.check_arguments
51
- def get_scriptcollection_version() -> str:
52
- return __version__
53
-
54
- @GeneralUtilities.check_arguments
55
- def python_file_has_errors(self, file: str, working_directory: str, treat_warnings_as_errors: bool = True) -> tuple[bool, list[str]]:
56
- errors = list()
57
- filename = os.path.relpath(file, working_directory)
58
- if treat_warnings_as_errors:
59
- errorsonly_argument = ""
60
- else:
61
- errorsonly_argument = " --errors-only"
62
- (exit_code, stdout, stderr, _) = self.run_program("pylint", filename +
63
- errorsonly_argument, working_directory, throw_exception_if_exitcode_is_not_zero=False)
64
- if (exit_code != 0):
65
- errors.append(f"Linting-issues of {file}:")
66
- errors.append(f"Pylint-exitcode: {exit_code}")
67
- for line in GeneralUtilities.string_to_lines(stdout):
68
- errors.append(line)
69
- for line in GeneralUtilities.string_to_lines(stderr):
70
- errors.append(line)
71
- return (True, errors)
72
-
73
- return (False, errors)
74
-
75
- @GeneralUtilities.check_arguments
76
- def replace_version_in_dockerfile_file(self, dockerfile: str, new_version_value: str) -> None:
77
- GeneralUtilities.write_text_to_file(dockerfile, re.sub(
78
- "ARG Version=\"\\d+\\.\\d+\\.\\d+\"", f"ARG Version=\"{new_version_value}\"", GeneralUtilities.read_text_from_file(dockerfile)))
79
-
80
- @GeneralUtilities.check_arguments
81
- def replace_version_in_python_file(self, file: str, new_version_value: str):
82
- GeneralUtilities.write_text_to_file(file, re.sub(
83
- "version = \"\\d+\\.\\d+\\.\\d+\"", f"version = \"{new_version_value}\"", GeneralUtilities.read_text_from_file(file)))
84
-
85
- @GeneralUtilities.check_arguments
86
- def replace_version_in_ini_file(self, file: str, new_version_value: str):
87
- GeneralUtilities.write_text_to_file(file, re.sub(
88
- "version = \\d+\\.\\d+\\.\\d+", f"version = {new_version_value}", GeneralUtilities.read_text_from_file(file)))
89
-
90
- @GeneralUtilities.check_arguments
91
- def replace_version_in_nuspec_file(self, nuspec_file: str, new_version: str) -> None:
92
- # TODO use XSLT instead
93
- versionregex = "\\d+\\.\\d+\\.\\d+"
94
- versiononlyregex = f"^{versionregex}$"
95
- pattern = re.compile(versiononlyregex)
96
- if pattern.match(new_version):
97
- GeneralUtilities.write_text_to_file(nuspec_file, re.sub(f"<version>{versionregex}<\\/version>", f"<version>{new_version}</version>", GeneralUtilities.read_text_from_file(nuspec_file)))
98
- else:
99
- raise ValueError(
100
- f"Version '{new_version}' does not match version-regex '{versiononlyregex}'")
101
-
102
- @GeneralUtilities.check_arguments
103
- def replace_version_in_csproj_file(self, csproj_file: str, current_version: str):
104
- versionregex = "\\d+\\.\\d+\\.\\d+"
105
- versiononlyregex = f"^{versionregex}$"
106
- pattern = re.compile(versiononlyregex)
107
- if pattern.match(current_version):
108
- for tag in ["Version", "AssemblyVersion", "FileVersion"]:
109
- GeneralUtilities.write_text_to_file(csproj_file, re.sub(
110
- f"<{tag}>{versionregex}(.\\d+)?<\\/{tag}>", f"<{tag}>{current_version}</{tag}>", GeneralUtilities.read_text_from_file(csproj_file)))
111
- else:
112
- raise ValueError(
113
- f"Version '{current_version}' does not match version-regex '{versiononlyregex}'")
114
-
115
- @GeneralUtilities.check_arguments
116
- def push_nuget_build_artifact(self, nupkg_file: str, registry_address: str, api_key: str, verbosity: int = 1):
117
- nupkg_file_name = os.path.basename(nupkg_file)
118
- nupkg_file_folder = os.path.dirname(nupkg_file)
119
- self.run_program(
120
- "dotnet", f"nuget push {nupkg_file_name} --force-english-output --source {registry_address} --api-key {api_key}", nupkg_file_folder, verbosity)
121
-
122
- @GeneralUtilities.check_arguments
123
- def dotnet_build(self, repository_folder: str, projectname: str, configuration: str):
124
- self.run_program(
125
- "dotnet", f"clean -c {configuration}", repository_folder)
126
- self.run_program(
127
- "dotnet", f"build {projectname}/{projectname}.csproj -c {configuration}", repository_folder)
128
-
129
- @GeneralUtilities.check_arguments
130
- def find_file_by_extension(self, folder: str, extension: str):
131
- result = [file for file in GeneralUtilities.get_direct_files_of_folder(folder) if file.endswith(f".{extension}")]
132
- result_length = len(result)
133
- if result_length == 0:
134
- raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension}'.")
135
- if result_length == 1:
136
- return result[0]
137
- else:
138
- raise ValueError(f"Multiple values available in folder '{folder}' with extension '{extension}'.")
139
-
140
- @GeneralUtilities.check_arguments
141
- def commit_is_signed_by_key(self, repository_folder: str, revision_identifier: str, key: str) -> bool:
142
- result = self.run_program(
143
- "git", f"verify-commit {revision_identifier}", repository_folder, throw_exception_if_exitcode_is_not_zero=False)
144
- if (result[0] != 0):
145
- return False
146
- if (not GeneralUtilities.contains_line(result[1].splitlines(), f"gpg\\:\\ using\\ [A-Za-z0-9]+\\ key\\ [A-Za-z0-9]+{key}")):
147
- # TODO check whether this works on machines where gpg is installed in another langauge than english
148
- return False
149
- if (not GeneralUtilities.contains_line(result[1].splitlines(), "gpg\\:\\ Good\\ signature\\ from")):
150
- # TODO check whether this works on machines where gpg is installed in another langauge than english
151
- return False
152
- return True
153
-
154
- @GeneralUtilities.check_arguments
155
- def get_parent_commit_ids_of_commit(self, repository_folder: str, commit_id: str) -> str:
156
- return self.run_program("git", f'log --pretty=%P -n 1 "{commit_id}"', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].replace("\r", "").replace("\n", "").split(" ")
157
-
158
- @GeneralUtilities.check_arguments
159
- def get_all_authors_and_committers_of_repository(self, repository_folder: str, subfolder: str = None, verbosity: int = 1) -> list[tuple[str, str]]:
160
- space_character = "_"
161
- if subfolder is None:
162
- subfolder_argument = ""
163
- else:
164
- subfolder_argument = f" -- {subfolder}"
165
- log_result = self.run_program(
166
- "git", f'log --pretty=%aN{space_character}%aE%n%cN{space_character}%cE HEAD{subfolder_argument}', repository_folder, verbosity=0)
167
- plain_content: list[str] = list(
168
- set([line for line in log_result[1].split("\n") if len(line) > 0]))
169
- result: list[tuple[str, str]] = []
170
- for item in plain_content:
171
- if len(re.findall(space_character, item)) == 1:
172
- splitted = item.split(space_character)
173
- result.append((splitted[0], splitted[1]))
174
- else:
175
- raise ValueError(f'Unexpected author: "{item}"')
176
- return result
177
-
178
- @GeneralUtilities.check_arguments
179
- def get_commit_ids_between_dates(self, repository_folder: str, since: datetime, until: datetime, ignore_commits_which_are_not_in_history_of_head: bool = True) -> None:
180
- since_as_string = self.__datetime_to_string_for_git(since)
181
- until_as_string = self.__datetime_to_string_for_git(until)
182
- result = filter(lambda line: not GeneralUtilities.string_is_none_or_whitespace(line), self.run_program(
183
- "git", f'log --since "{since_as_string}" --until "{until_as_string}" --pretty=format:"%H" --no-patch', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].split("\n").replace("\r", ""))
184
- if ignore_commits_which_are_not_in_history_of_head:
185
- result = [commit_id for commit_id in result if self.git_commit_is_ancestor(
186
- repository_folder, commit_id)]
187
- return result
188
-
189
- @GeneralUtilities.check_arguments
190
- def __datetime_to_string_for_git(self, datetime_object: datetime) -> str:
191
- return datetime_object.strftime('%Y-%m-%d %H:%M:%S')
192
-
193
- @GeneralUtilities.check_arguments
194
- def git_commit_is_ancestor(self, repository_folder: str, ancestor: str, descendant: str = "HEAD") -> bool:
195
- exit_code = self.run_program_argsasarray(
196
- "git", ["merge-base", "--is-ancestor", ancestor, descendant], repository_folder, throw_exception_if_exitcode_is_not_zero=False)[0]
197
- if exit_code == 0:
198
- return True
199
- elif exit_code == 1:
200
- return False
201
- else:
202
- raise ValueError(f"Can not calculate if {ancestor} is an ancestor of {descendant} in repository {repository_folder}.")
203
-
204
- @GeneralUtilities.check_arguments
205
- def __git_changes_helper(self, repository_folder: str, arguments_as_array: list[str]) -> bool:
206
- lines = GeneralUtilities.string_to_lines(self.run_program_argsasarray(
207
- "git", arguments_as_array, repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
208
- for line in lines:
209
- if GeneralUtilities.string_has_content(line):
210
- return True
211
- return False
212
-
213
- @GeneralUtilities.check_arguments
214
- def git_repository_has_new_untracked_files(self, repositoryFolder: str):
215
- return self.__git_changes_helper(repositoryFolder, ["ls-files", "--exclude-standard", "--others"])
216
-
217
- @GeneralUtilities.check_arguments
218
- def git_repository_has_unstaged_changes_of_tracked_files(self, repositoryFolder: str):
219
- return self.__git_changes_helper(repositoryFolder, ["diff"])
220
-
221
- @GeneralUtilities.check_arguments
222
- def git_repository_has_staged_changes(self, repositoryFolder: str):
223
- return self.__git_changes_helper(repositoryFolder, ["diff", "--cached"])
224
-
225
- @GeneralUtilities.check_arguments
226
- def git_repository_has_uncommitted_changes(self, repositoryFolder: str) -> bool:
227
- if (self.git_repository_has_unstaged_changes(repositoryFolder)):
228
- return True
229
- if (self.git_repository_has_staged_changes(repositoryFolder)):
230
- return True
231
- return False
232
-
233
- @GeneralUtilities.check_arguments
234
- def git_repository_has_unstaged_changes(self, repository_folder: str) -> bool:
235
- if (self.git_repository_has_unstaged_changes_of_tracked_files(repository_folder)):
236
- return True
237
- if (self.git_repository_has_new_untracked_files(repository_folder)):
238
- return True
239
- return False
240
-
241
- @GeneralUtilities.check_arguments
242
- def git_get_commit_id(self, repository_folder: str, commit: str = "HEAD") -> str:
243
- result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["rev-parse", "--verify", commit], repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
244
- return result[1].replace('\n', '')
245
-
246
- @GeneralUtilities.check_arguments
247
- def git_get_commit_date(self, repository_folder: str, commit: str = "HEAD") -> datetime:
248
- result: tuple[int, str, str, int] = self.run_program_argsasarray(
249
- "git", ["show", "-s", "--format=%ci", commit], repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
250
- date_as_string = result[1].replace('\n', '')
251
- result = datetime.strptime(date_as_string, '%Y-%m-%d %H:%M:%S %z')
252
- return result
253
-
254
- @GeneralUtilities.check_arguments
255
- def git_fetch(self, folder: str, remotename: str = "--all") -> None:
256
- self.run_program_argsasarray("git", ["fetch", remotename, "--tags", "--prune"], folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
257
-
258
- @GeneralUtilities.check_arguments
259
- def git_fetch_in_bare_repository(self, folder: str, remotename, localbranch: str, remotebranch: str) -> None:
260
- self.run_program_argsasarray("git", ["fetch", remotename, f"{remotebranch}:{localbranch}"], folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
261
-
262
- @GeneralUtilities.check_arguments
263
- def git_remove_branch(self, folder: str, branchname: str) -> None:
264
- self.run_program("git", f"branch -D {branchname}", folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
265
-
266
- @GeneralUtilities.check_arguments
267
- def git_push(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: int = 0) -> None:
268
- argument = ["push", "--recurse-submodules=on-demand", remotename, f"{localbranchname}:{remotebranchname}"]
269
- if (forcepush):
270
- argument.append("--force")
271
- if (pushalltags):
272
- argument.append("--tags")
273
- result: tuple[int, str, str, int] = self.run_program_argsasarray("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=verbosity, print_errors_as_information=True)
274
- return result[1].replace('\r', '').replace('\n', '')
275
-
276
- @GeneralUtilities.check_arguments
277
- def git_pull(self, folder: str, remote: str, localbranchname: str, remotebranchname: str) -> None:
278
- self.run_program("git", f"pull {remote} {remotebranchname}:{localbranchname}", folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
279
-
280
- @GeneralUtilities.check_arguments
281
- def git_list_remote_branches(self, folder: str, remote: str, fetch: bool) -> list[str]:
282
- if fetch:
283
- self.git_fetch(folder, remote)
284
- run_program_result = self.run_program("git", f"branch -rl {remote}/*", folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
285
- output = GeneralUtilities.string_to_lines(run_program_result[1])
286
- result = list[str]()
287
- for item in output:
288
- striped_item = item.strip()
289
- if GeneralUtilities.string_has_content(striped_item):
290
- branch: str = None
291
- if " " in striped_item:
292
- branch = striped_item.split(" ")[0]
293
- else:
294
- branch = striped_item
295
- branchname = branch[len(remote)+1:]
296
- if branchname != "HEAD":
297
- result.append(branchname)
298
- return result
299
-
300
- @GeneralUtilities.check_arguments
301
- def git_clone(self, clone_target_folder: str, remote_repository_path: str, include_submodules: bool = True, mirror: bool = False) -> None:
302
- if (os.path.isdir(clone_target_folder)):
303
- pass # TODO throw error
304
- else:
305
- args = ["clone", remote_repository_path, clone_target_folder]
306
- if include_submodules:
307
- args.append("--recurse-submodules")
308
- args.append("--remote-submodules")
309
- if mirror:
310
- args.append("--mirror")
311
- self.run_program_argsasarray("git", args, os.getcwd(), throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
312
-
313
- @GeneralUtilities.check_arguments
314
- def git_get_all_remote_names(self, directory: str) -> list[str]:
315
- result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
316
- return result
317
-
318
- @GeneralUtilities.check_arguments
319
- def git_get_remote_url(self, directory: str, remote_name: str) -> str:
320
- result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote", "get-url", remote_name], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
321
- return result[0].replace('\n', '')
322
-
323
- @GeneralUtilities.check_arguments
324
- def repository_has_remote_with_specific_name(self, directory: str, remote_name: str) -> bool:
325
- return remote_name in self.git_get_all_remote_names(directory)
326
-
327
- @GeneralUtilities.check_arguments
328
- def git_add_or_set_remote_address(self, directory: str, remote_name: str, remote_address: str) -> None:
329
- if (self.repository_has_remote_with_specific_name(directory, remote_name)):
330
- self.run_program_argsasarray("git", ['remote', 'set-url', 'remote_name', remote_address], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
331
- else:
332
- self.run_program_argsasarray("git", ['remote', 'add', remote_name, remote_address], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
333
-
334
- @GeneralUtilities.check_arguments
335
- def git_stage_all_changes(self, directory: str) -> None:
336
- self.run_program_argsasarray("git", ["add", "-A"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
337
-
338
- @GeneralUtilities.check_arguments
339
- def git_unstage_all_changes(self, directory: str) -> None:
340
- self.run_program_argsasarray("git", ["reset"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
341
-
342
- @GeneralUtilities.check_arguments
343
- def git_stage_file(self, directory: str, file: str) -> None:
344
- self.run_program_argsasarray("git", ['stage', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
345
-
346
- @GeneralUtilities.check_arguments
347
- def git_unstage_file(self, directory: str, file: str) -> None:
348
- self.run_program_argsasarray("git", ['reset', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
349
-
350
- @GeneralUtilities.check_arguments
351
- def git_discard_unstaged_changes_of_file(self, directory: str, file: str) -> None:
352
- """Caution: This method works really only for 'changed' files yet. So this method does not work properly for new or renamed files."""
353
- self.run_program_argsasarray("git", ['checkout', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
354
-
355
- @GeneralUtilities.check_arguments
356
- def git_discard_all_unstaged_changes(self, directory: str) -> None:
357
- """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
358
- self.run_program_argsasarray("git", ['clean', '-df'], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
359
- self.run_program_argsasarray("git", ['checkout', '.'], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
360
-
361
- @GeneralUtilities.check_arguments
362
- def git_commit(self, directory: str, message: str, author_name: str = None, author_email: str = None, stage_all_changes: bool = True,
363
- no_changes_behavior: int = 0) -> str:
364
- # no_changes_behavior=0 => No commit
365
- # no_changes_behavior=1 => Commit anyway
366
- # no_changes_behavior=2 => Exception
367
- author_name = GeneralUtilities.str_none_safe(author_name).strip()
368
- author_email = GeneralUtilities.str_none_safe(author_email).strip()
369
- argument = ['commit', '--quiet', '--allow-empty', '--message', message]
370
- if (GeneralUtilities.string_has_content(author_name)):
371
- argument.append(f'--author="{author_name} <{author_email}>"')
372
- git_repository_has_uncommitted_changes = self.git_repository_has_uncommitted_changes(directory)
373
-
374
- if git_repository_has_uncommitted_changes:
375
- do_commit = True
376
- if stage_all_changes:
377
- self.git_stage_all_changes(directory)
378
- else:
379
- if no_changes_behavior == 0:
380
- GeneralUtilities.write_message_to_stdout(f"Commit '{message}' will not be done because there are no changes to commit in repository '{directory}'")
381
- do_commit = False
382
- elif no_changes_behavior == 1:
383
- GeneralUtilities.write_message_to_stdout(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will be done anyway.")
384
- do_commit = True
385
- elif no_changes_behavior == 2:
386
- raise RuntimeError(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will not be done.")
387
- else:
388
- raise ValueError(f"Unknown value for no_changes_behavior: {GeneralUtilities.str_none_safe(no_changes_behavior)}")
389
-
390
- if do_commit:
391
- GeneralUtilities.write_message_to_stdout(f"Commit changes in '{directory}'")
392
- self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
393
-
394
- return self.git_get_commit_id(directory)
395
-
396
- @GeneralUtilities.check_arguments
397
- def git_create_tag(self, directory: str, target_for_tag: str, tag: str, sign: bool = False, message: str = None) -> None:
398
- argument = ["tag", tag, target_for_tag]
399
- if sign:
400
- if message is None:
401
- message = f"Created {target_for_tag}"
402
- argument.extend(["-s", '-m', message])
403
- self.run_program_argsasarray(
404
- "git", argument, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
405
-
406
- @GeneralUtilities.check_arguments
407
- def git_delete_tag(self, directory: str, tag: str) -> None:
408
- self.run_program_argsasarray("git", ["tag", "--delete", tag], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
409
-
410
- @GeneralUtilities.check_arguments
411
- def git_checkout(self, directory: str, branch: str) -> None:
412
- self.run_program_argsasarray("git", ["checkout", branch], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
413
- self.run_program_argsasarray("git", ["submodule", "update", "--recursive"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
414
-
415
- @GeneralUtilities.check_arguments
416
- def git_merge_abort(self, directory: str) -> None:
417
- self.run_program_argsasarray("git", ["merge", "--abort"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
418
-
419
- @GeneralUtilities.check_arguments
420
- def git_merge(self, directory: str, sourcebranch: str, targetbranch: str, fastforward: bool = True, commit: bool = True, commit_message: str = None) -> str:
421
- self.git_checkout(directory, targetbranch)
422
- args = ["merge"]
423
- if not commit:
424
- args.append("--no-commit")
425
- if not fastforward:
426
- args.append("--no-ff")
427
- if commit_message is not None:
428
- args.append("-m")
429
- args.append(commit_message)
430
- args.append(sourcebranch)
431
- self.run_program_argsasarray(
432
- "git", args, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
433
- return self.git_get_commit_id(directory)
434
-
435
- @GeneralUtilities.check_arguments
436
- def git_undo_all_changes(self, directory: str) -> None:
437
- """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
438
- self.git_unstage_all_changes(directory)
439
- self.git_discard_all_unstaged_changes(directory)
440
-
441
- @GeneralUtilities.check_arguments
442
- def git_fetch_or_clone_all_in_directory(self, source_directory: str, target_directory: str) -> None:
443
- for subfolder in GeneralUtilities.get_direct_folders_of_folder(source_directory):
444
- foldername = os.path.basename(subfolder)
445
- if self.is_git_repository(subfolder):
446
- source_repository = subfolder
447
- target_repository = os.path.join(target_directory, foldername)
448
- if os.path.isdir(target_directory):
449
- # fetch
450
- self.git_fetch(target_directory)
451
- else:
452
- # clone
453
- self.git_clone(target_repository, source_repository, include_submodules=True, mirror=True)
454
-
455
- def get_git_submodules(self, folder: str) -> list[str]:
456
- e = self.run_program("git", "submodule status", folder)
457
- result = []
458
- for submodule_line in GeneralUtilities.string_to_lines(e[1], False, True):
459
- result.append(submodule_line.split(' ')[1])
460
- return result
461
-
462
- @GeneralUtilities.check_arguments
463
- def is_git_repository(self, folder: str) -> bool:
464
- combined = os.path.join(folder, ".git")
465
- # TODO consider check for bare-repositories
466
- return os.path.isdir(combined) or os.path.isfile(combined)
467
-
468
- @GeneralUtilities.check_arguments
469
- def file_is_git_ignored(self, file_in_repository: str, repositorybasefolder: str) -> None:
470
- exit_code = self.run_program_argsasarray("git", ['check-ignore', file_in_repository], repositorybasefolder, throw_exception_if_exitcode_is_not_zero=False, verbosity=0)[0]
471
- if (exit_code == 0):
472
- return True
473
- if (exit_code == 1):
474
- return False
475
- raise ValueError(
476
- f"Unable to calculate whether '{file_in_repository}' in repository '{repositorybasefolder}' is ignored due to git-exitcode {exit_code}.")
477
-
478
- @GeneralUtilities.check_arguments
479
- def git_discard_all_changes(self, repository: str) -> None:
480
- self.run_program_argsasarray("git", ["reset", "HEAD", "."], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
481
- self.run_program_argsasarray("git", ["checkout", "."], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
482
-
483
- @GeneralUtilities.check_arguments
484
- def git_get_current_branch_name(self, repository: str) -> str:
485
- result = self.run_program_argsasarray("git", ["rev-parse", "--abbrev-ref", "HEAD"], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
486
- return result[1].replace("\r", "").replace("\n", "")
487
-
488
- @GeneralUtilities.check_arguments
489
- def git_get_commitid_of_tag(self, repository: str, tag: str) -> str:
490
- stdout = self.run_program_argsasarray("git", ["rev-list", "-n", "1", tag], repository, verbosity=0)
491
- result = stdout[1].replace("\r", "").replace("\n", "")
492
- return result
493
-
494
- @GeneralUtilities.check_arguments
495
- def git_get_tags(self, repository: str) -> list[str]:
496
- tags = [line.replace("\r", "") for line in self.run_program_argsasarray(
497
- "git", ["tag"], repository)[1].split("\n") if len(line) > 0]
498
- return tags
499
-
500
- @GeneralUtilities.check_arguments
501
- def git_move_tags_to_another_branch(self, repository: str, tag_source_branch: str, tag_target_branch: str, sign: bool = False, message: str = None) -> None:
502
- tags = self.git_get_tags(repository)
503
- tags_count = len(tags)
504
- counter = 0
505
- for tag in tags:
506
- counter = counter+1
507
- GeneralUtilities.write_message_to_stdout(f"Process tag {counter}/{tags_count}.")
508
- # tag is on source-branch
509
- if self.git_commit_is_ancestor(repository, tag, tag_source_branch):
510
- commit_id_old = self.git_get_commitid_of_tag(repository, tag)
511
- commit_date: datetime = self.git_get_commit_date(repository, commit_id_old)
512
- date_as_string = self.__datetime_to_string_for_git(commit_date)
513
- search_commit_result = self.run_program_argsasarray("git", ["log", f'--after="{date_as_string}"', f'--before="{date_as_string}"', "--pretty=format:%H", tag_target_branch], repository, throw_exception_if_exitcode_is_not_zero=False)
514
- if search_commit_result[0] != 0 or not GeneralUtilities.string_has_nonwhitespace_content(search_commit_result[1]):
515
- raise ValueError(f"Can not calculate corresponding commit for tag '{tag}'.")
516
- commit_id_new = search_commit_result[1]
517
- self.git_delete_tag(repository, tag)
518
- self.git_create_tag(repository, commit_id_new, tag, sign, message)
519
-
520
- @GeneralUtilities.check_arguments
521
- def get_current_git_branch_has_tag(self, repository_folder: str) -> bool:
522
- result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder, verbosity=0, throw_exception_if_exitcode_is_not_zero=False)
523
- return result[0] == 0
524
-
525
- @GeneralUtilities.check_arguments
526
- def get_latest_git_tag(self, repository_folder: str) -> str:
527
- result = self.run_program_argsasarray(
528
- "git", ["describe", "--tags", "--abbrev=0"], repository_folder, verbosity=0)
529
- result = result[1].replace("\r", "").replace("\n", "")
530
- return result
531
-
532
- @GeneralUtilities.check_arguments
533
- def get_staged_or_committed_git_ignored_files(self, repository_folder: str) -> list[str]:
534
- tresult = self.run_program_argsasarray("git", ["ls-files", "-i", "-c", "--exclude-standard"], repository_folder, verbosity=0)
535
- tresult = tresult[1].replace("\r", "")
536
- result = [line for line in tresult.split("\n") if len(line) > 0]
537
- return result
538
-
539
- @GeneralUtilities.check_arguments
540
- def git_repository_has_commits(self, repository_folder: str) -> bool:
541
- return self.run_program_argsasarray("git", ["rev-parse", "--verify", "HEAD"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)[0] == 0
542
-
543
- @GeneralUtilities.check_arguments
544
- def export_filemetadata(self, folder: str, target_file: str, encoding: str = "utf-8", filter_function=None) -> None:
545
- folder = GeneralUtilities.resolve_relative_path_from_current_working_directory(folder)
546
- lines = list()
547
- path_prefix = len(folder)+1
548
- items = dict()
549
- for item in GeneralUtilities.get_all_folders_of_folder(folder):
550
- items[item] = "d"
551
- for item in GeneralUtilities.get_all_files_of_folder(folder):
552
- items[item] = "f"
553
- for file_or_folder, item_type in items.items():
554
- truncated_file = file_or_folder[path_prefix:]
555
- if (filter_function is None or filter_function(folder, truncated_file)):
556
- owner_and_permisssion = self.get_file_owner_and_file_permission(
557
- file_or_folder)
558
- user = owner_and_permisssion[0]
559
- permissions = owner_and_permisssion[1]
560
- lines.append(f"{truncated_file};{item_type};{user};{permissions}")
561
- lines = sorted(lines, key=str.casefold)
562
- with open(target_file, "w", encoding=encoding) as file_object:
563
- file_object.write("\n".join(lines))
564
-
565
- def escape_git_repositories_in_folder(self, folder: str) -> dict[str, str]:
566
- return self.__escape_git_repositories_in_folder_internal(folder, dict[str, str]())
567
-
568
- def __escape_git_repositories_in_folder_internal(self, folder: str, renamed_items: dict[str, str]) -> dict[str, str]:
569
- for file in GeneralUtilities.get_direct_files_of_folder(folder):
570
- filename = os.path.basename(file)
571
- if ".git" in filename:
572
- new_name = filename.replace(".git", ".gitx")
573
- target = os.path.join(folder, new_name)
574
- os.rename(file, target)
575
- renamed_items[target] = file
576
- for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
577
- foldername = os.path.basename(subfolder)
578
- if ".git" in foldername:
579
- new_name = foldername.replace(".git", ".gitx")
580
- subfolder2 = os.path.join(
581
- str(Path(subfolder).parent), new_name)
582
- os.rename(subfolder, subfolder2)
583
- renamed_items[subfolder2] = subfolder
584
- else:
585
- subfolder2 = subfolder
586
- self.__escape_git_repositories_in_folder_internal(subfolder2, renamed_items)
587
- return renamed_items
588
-
589
- def deescape_git_repositories_in_folder(self, renamed_items: dict[str, str]):
590
- for renamed_item, original_name in renamed_items.items():
591
- os.rename(renamed_item, original_name)
592
-
593
- def __sort_fmd(self, line: str):
594
- splitted: list = line.split(";")
595
- filetype: str = splitted[1]
596
- if filetype == "d":
597
- return -1
598
- if filetype == "f":
599
- return 1
600
- return 0
601
-
602
- @GeneralUtilities.check_arguments
603
- def restore_filemetadata(self, folder: str, source_file: str, strict=False, encoding: str = "utf-8", create_folder_is_not_exist: bool = True) -> None:
604
- lines = GeneralUtilities.read_lines_from_file(source_file, encoding)
605
- lines.sort(key=self.__sort_fmd)
606
- for line in lines:
607
- splitted: list = line.split(";")
608
- full_path_of_file_or_folder: str = os.path.join(folder, splitted[0])
609
- filetype: str = splitted[1]
610
- user: str = splitted[2]
611
- permissions: str = splitted[3]
612
- if filetype == "d" and create_folder_is_not_exist and not os.path.isdir(full_path_of_file_or_folder):
613
- GeneralUtilities.ensure_directory_exists(full_path_of_file_or_folder)
614
- if (filetype == "f" and os.path.isfile(full_path_of_file_or_folder)) or (filetype == "d" and os.path.isdir(full_path_of_file_or_folder)):
615
- self.set_owner(full_path_of_file_or_folder, user, os.name != 'nt')
616
- self.set_permission(full_path_of_file_or_folder, permissions)
617
- else:
618
- if strict:
619
- if filetype == "f":
620
- filetype_full = "File"
621
- elif filetype == "d":
622
- filetype_full = "Directory"
623
- else:
624
- raise ValueError(f"Unknown filetype: {GeneralUtilities.str_none_safe(filetype)}")
625
- raise ValueError(f"{filetype_full} '{full_path_of_file_or_folder}' does not exist")
626
-
627
- @GeneralUtilities.check_arguments
628
- def __calculate_lengh_in_seconds(self, filename: str, folder: str) -> float:
629
- argument = ['-v', 'error', '-show_entries', 'format=duration',
630
- '-of', 'default=noprint_wrappers=1:nokey=1', filename]
631
- result = self.run_program_argsasarray("ffprobe", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
632
- return float(result[1].replace('\n', ''))
633
-
634
- @GeneralUtilities.check_arguments
635
- def __create_thumbnails(self, filename: str, fps: str, folder: str, tempname_for_thumbnails: str) -> None:
636
- argument = ['-i', filename, '-r', str(fps), '-vf', 'scale=-1:120',
637
- '-vcodec', 'png', f'{tempname_for_thumbnails}-%002d.png']
638
- self.run_program_argsasarray("ffmpeg", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
639
-
640
- @GeneralUtilities.check_arguments
641
- def __create_thumbnail(self, outputfilename: str, folder: str, length_in_seconds: float, tempname_for_thumbnails: str, amount_of_images: int) -> None:
642
- duration = timedelta(seconds=length_in_seconds)
643
- info = GeneralUtilities.timedelta_to_simple_string(duration)
644
- next_square_number = str(int(math.sqrt(GeneralUtilities.get_next_square_number(amount_of_images))))
645
- argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{next_square_number}x{next_square_number}',
646
- f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
647
- self.run_program_argsasarray(
648
- "montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
649
-
650
- @GeneralUtilities.check_arguments
651
- def roundup(self, x: float, places: int) -> int:
652
- d = 10 ** places
653
- if x < 0:
654
- return math.floor(x * d) / d
655
- else:
656
- return math.ceil(x * d) / d
657
-
658
- @GeneralUtilities.check_arguments
659
- def generate_thumbnail(self, file: str, frames_per_second: str, tempname_for_thumbnails: str = None) -> None:
660
- if tempname_for_thumbnails is None:
661
- tempname_for_thumbnails = "t"+str(uuid.uuid4())
662
-
663
- file = GeneralUtilities.resolve_relative_path_from_current_working_directory(file)
664
- filename = os.path.basename(file)
665
- folder = os.path.dirname(file)
666
- filename_without_extension = Path(file).stem
667
-
668
- try:
669
- length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
670
- if (frames_per_second.endswith("fps")):
671
- # frames per second, example: frames_per_second="20fps" => 20 frames per second
672
- x = self.roundup(float(frames_per_second[:-3]), 2)
673
- frames_per_secondx = str(x)
674
- amounf_of_previewframes = int(math.floor(length_in_seconds*x))
675
- else:
676
- # concrete amount of frame, examples: frames_per_second="16" => 16 frames for entire video
677
- amounf_of_previewframes = int(float(frames_per_second))
678
- # self.roundup((amounf_of_previewframes-2)/length_in_seconds, 2)
679
- frames_per_secondx = f"{amounf_of_previewframes-2}/{length_in_seconds}"
680
- self.__create_thumbnails(filename, frames_per_secondx, folder, tempname_for_thumbnails)
681
- self.__create_thumbnail(filename_without_extension, folder, length_in_seconds, tempname_for_thumbnails, amounf_of_previewframes)
682
- finally:
683
- for thumbnail_to_delete in Path(folder).rglob(tempname_for_thumbnails+"-*"):
684
- file = str(thumbnail_to_delete)
685
- os.remove(file)
686
-
687
- @GeneralUtilities.check_arguments
688
- def extract_pdf_pages(self, file: str, from_page: int, to_page: int, outputfile: str) -> None:
689
- pdf_reader = PyPDF2.PdfReader(file)
690
- pdf_writer = PyPDF2.PdfWriter()
691
- start = from_page
692
- end = to_page
693
- while start <= end:
694
- pdf_writer.add_page(pdf_reader.pages[start-1])
695
- start += 1
696
- with open(outputfile, 'wb') as out:
697
- pdf_writer.write(out)
698
-
699
- @GeneralUtilities.check_arguments
700
- def merge_pdf_files(self, files: list[str], outputfile: str) -> None:
701
- # TODO add wildcard-option
702
- pdfFileMerger = PyPDF2.PdfFileMerger()
703
- for file in files:
704
- pdfFileMerger.append(file.strip())
705
- pdfFileMerger.write(outputfile)
706
- pdfFileMerger.close()
707
-
708
- @GeneralUtilities.check_arguments
709
- def pdf_to_image(self, file: str, outputfilename_without_extension: str) -> None:
710
- raise ValueError("Function currently not available")
711
- # PyMuPDF can be used for that but sometimes it throws
712
- # "ImportError: DLL load failed while importing _fitz: Das angegebene Modul wurde nicht gefunden."
713
-
714
- # doc = None # fitz.open(file)
715
- # for i, page in enumerate(doc):
716
- # pix = page.get_pixmap()
717
- # img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
718
- # img.save(f"{outputfilename_without_extension}_{i}.png", "PNG")
719
-
720
- @GeneralUtilities.check_arguments
721
- def show_missing_files(self, folderA: str, folderB: str):
722
- for file in GeneralUtilities.get_missing_files(folderA, folderB):
723
- GeneralUtilities.write_message_to_stdout(file)
724
-
725
- @GeneralUtilities.check_arguments
726
- def SCCreateEmptyFileWithSpecificSize(self, name: str, size_string: str) -> int:
727
- if size_string.isdigit():
728
- size = int(size_string)
729
- else:
730
- if len(size_string) >= 3:
731
- if (size_string.endswith("kb")):
732
- size = int(size_string[:-2]) * pow(10, 3)
733
- elif (size_string.endswith("mb")):
734
- size = int(size_string[:-2]) * pow(10, 6)
735
- elif (size_string.endswith("gb")):
736
- size = int(size_string[:-2]) * pow(10, 9)
737
- elif (size_string.endswith("kib")):
738
- size = int(size_string[:-3]) * pow(2, 10)
739
- elif (size_string.endswith("mib")):
740
- size = int(size_string[:-3]) * pow(2, 20)
741
- elif (size_string.endswith("gib")):
742
- size = int(size_string[:-3]) * pow(2, 30)
743
- else:
744
- GeneralUtilities.write_message_to_stderr("Wrong format")
745
- return 1
746
- else:
747
- GeneralUtilities.write_message_to_stderr("Wrong format")
748
- return 1
749
- with open(name, "wb") as f:
750
- f.seek(size-1)
751
- f.write(b"\0")
752
- return 0
753
-
754
- @GeneralUtilities.check_arguments
755
- def SCCreateHashOfAllFiles(self, folder: str) -> None:
756
- for file in GeneralUtilities.absolute_file_paths(folder):
757
- with open(file+".sha256", "w+", encoding="utf-8") as f:
758
- f.write(GeneralUtilities.get_sha256_of_file(file))
759
-
760
- @GeneralUtilities.check_arguments
761
- def SCCreateSimpleMergeWithoutRelease(self, repository: str, sourcebranch: str, targetbranch: str, remotename: str, remove_source_branch: bool) -> None:
762
- commitid = self.git_merge(repository, sourcebranch, targetbranch, False, True)
763
- self.git_merge(repository, targetbranch, sourcebranch, True, True)
764
- created_version = self.get_semver_version_from_gitversion(repository)
765
- self.git_create_tag(repository, commitid, f"v{created_version}", True)
766
- self.git_push(repository, remotename, targetbranch, targetbranch, False, True)
767
- if (GeneralUtilities.string_has_nonwhitespace_content(remotename)):
768
- self.git_push(repository, remotename, sourcebranch, sourcebranch, False, True)
769
- if (remove_source_branch):
770
- self.git_remove_branch(repository, sourcebranch)
771
-
772
- @GeneralUtilities.check_arguments
773
- def sc_organize_lines_in_file(self, file: str, encoding: str, sort: bool = False, remove_duplicated_lines: bool = False, ignore_first_line: bool = False, remove_empty_lines: bool = True, ignored_start_character: list = list()) -> int:
774
- if os.path.isfile(file):
775
-
776
- # read file
777
- lines = GeneralUtilities.read_lines_from_file(file, encoding)
778
- if (len(lines) == 0):
779
- return 0
780
-
781
- # store first line if desiredpopd
782
-
783
- if (ignore_first_line):
784
- first_line = lines.pop(0)
785
-
786
- # remove empty lines if desired
787
- if remove_empty_lines:
788
- temp = lines
789
- lines = []
790
- for line in temp:
791
- if (not (GeneralUtilities.string_is_none_or_whitespace(line))):
792
- lines.append(line)
793
-
794
- # remove duplicated lines if desired
795
- if remove_duplicated_lines:
796
- lines = GeneralUtilities.remove_duplicates(lines)
797
-
798
- # sort lines if desired
799
- if sort:
800
- lines = sorted(lines, key=lambda singleline: self.__adapt_line_for_sorting(singleline, ignored_start_character))
801
-
802
- # reinsert first line
803
- if ignore_first_line:
804
- lines.insert(0, first_line)
805
-
806
- # write result to file
807
- GeneralUtilities.write_lines_to_file(file, lines, encoding)
808
-
809
- return 0
810
- else:
811
- GeneralUtilities.write_message_to_stdout(f"File '{file}' does not exist")
812
- return 1
813
-
814
- @GeneralUtilities.check_arguments
815
- def __adapt_line_for_sorting(self, line: str, ignored_start_characters: list):
816
- result = line.lower()
817
- while len(result) > 0 and result[0] in ignored_start_characters:
818
- result = result[1:]
819
- return result
820
-
821
- @GeneralUtilities.check_arguments
822
- def SCGenerateSnkFiles(self, outputfolder, keysize=4096, amountofkeys=10) -> int:
823
- GeneralUtilities.ensure_directory_exists(outputfolder)
824
- for _ in range(amountofkeys):
825
- file = os.path.join(outputfolder, str(uuid.uuid4())+".snk")
826
- argument = f"-k {keysize} {file}"
827
- self.run_program("sn", argument, outputfolder)
828
-
829
- @GeneralUtilities.check_arguments
830
- def __merge_files(self, sourcefile: str, targetfile: str) -> None:
831
- with open(sourcefile, "rb") as f:
832
- source_data = f.read()
833
- with open(targetfile, "ab") as fout:
834
- merge_separator = [0x0A]
835
- fout.write(bytes(merge_separator))
836
- fout.write(source_data)
837
-
838
- @GeneralUtilities.check_arguments
839
- def __process_file(self, file: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
840
- new_filename = os.path.join(os.path.dirname(file), os.path.basename(file).replace(substringInFilename, newSubstringInFilename))
841
- if file != new_filename:
842
- if os.path.isfile(new_filename):
843
- if filecmp.cmp(file, new_filename):
844
- send2trash.send2trash(file)
845
- else:
846
- if conflictResolveMode == "ignore":
847
- pass
848
- elif conflictResolveMode == "preservenewest":
849
- if (os.path.getmtime(file) - os.path.getmtime(new_filename) > 0):
850
- send2trash.send2trash(file)
851
- else:
852
- send2trash.send2trash(new_filename)
853
- os.rename(file, new_filename)
854
- elif (conflictResolveMode == "merge"):
855
- self.__merge_files(file, new_filename)
856
- send2trash.send2trash(file)
857
- else:
858
- raise ValueError('Unknown conflict resolve mode')
859
- else:
860
- os.rename(file, new_filename)
861
-
862
- @GeneralUtilities.check_arguments
863
- def SCReplaceSubstringsInFilenames(self, folder: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
864
- for file in GeneralUtilities.absolute_file_paths(folder):
865
- self.__process_file(file, substringInFilename, newSubstringInFilename, conflictResolveMode)
866
-
867
- @GeneralUtilities.check_arguments
868
- def __check_file(self, file: str, searchstring: str) -> None:
869
- bytes_ascii = bytes(searchstring, "ascii")
870
- # often called "unicode-encoding"
871
- bytes_utf16 = bytes(searchstring, "utf-16")
872
- bytes_utf8 = bytes(searchstring, "utf-8")
873
- with open(file, mode='rb') as file_object:
874
- content = file_object.read()
875
- if bytes_ascii in content:
876
- GeneralUtilities.write_message_to_stdout(file)
877
- elif bytes_utf16 in content:
878
- GeneralUtilities.write_message_to_stdout(file)
879
- elif bytes_utf8 in content:
880
- GeneralUtilities.write_message_to_stdout(file)
881
-
882
- @GeneralUtilities.check_arguments
883
- def SCSearchInFiles(self, folder: str, searchstring: str) -> None:
884
- for file in GeneralUtilities.absolute_file_paths(folder):
885
- self.__check_file(file, searchstring)
886
-
887
- @GeneralUtilities.check_arguments
888
- def __print_qr_code_by_csv_line(self, displayname: str, website: str, emailaddress: str, key: str, period: str) -> None:
889
- qrcode_content = f"otpauth://totp/{website}:{emailaddress}?secret={key}&issuer={displayname}&period={period}"
890
- GeneralUtilities.write_message_to_stdout(
891
- f"{displayname} ({emailaddress}):")
892
- GeneralUtilities.write_message_to_stdout(qrcode_content)
893
- qr = qrcode.QRCode()
894
- qr.add_data(qrcode_content)
895
- f = io.StringIO()
896
- qr.print_ascii(out=f)
897
- f.seek(0)
898
- GeneralUtilities.write_message_to_stdout(f.read())
899
-
900
- @GeneralUtilities.check_arguments
901
- def SCShow2FAAsQRCode(self, csvfile: str) -> None:
902
- separator_line = "--------------------------------------------------------"
903
- lines = GeneralUtilities.read_csv_file(csvfile, True)
904
- lines.sort(key=lambda items: ''.join(items).lower())
905
- for line in lines:
906
- GeneralUtilities.write_message_to_stdout(separator_line)
907
- self.__print_qr_code_by_csv_line(
908
- line[0], line[1], line[2], line[3], line[4])
909
- GeneralUtilities.write_message_to_stdout(separator_line)
910
-
911
- @GeneralUtilities.check_arguments
912
- def SCUploadFileToFileHost(self, file: str, host: str) -> int:
913
- try:
914
- GeneralUtilities.write_message_to_stdout(
915
- self.upload_file_to_file_host(file, host))
916
- return 0
917
- except Exception as exception:
918
- GeneralUtilities.write_exception_to_stderr_with_traceback(exception, traceback)
919
- return 1
920
-
921
- @GeneralUtilities.check_arguments
922
- def SCFileIsAvailableOnFileHost(self, file: str) -> int:
923
- try:
924
- if self.file_is_available_on_file_host(file):
925
- GeneralUtilities.write_message_to_stdout(f"'{file}' is available")
926
- return 0
927
- else:
928
- GeneralUtilities.write_message_to_stdout(f"'{file}' is not available")
929
- return 1
930
- except Exception as exception:
931
- GeneralUtilities.write_exception_to_stderr_with_traceback(exception, traceback)
932
- return 2
933
-
934
- @GeneralUtilities.check_arguments
935
- def SCCalculateBitcoinBlockHash(self, block_version_number: str, previousblockhash: str, transactionsmerkleroot: str, timestamp: str, target: str, nonce: str) -> str:
936
- # Example-values:
937
- # block_version_number: "00000020"
938
- # previousblockhash: "66720b99e07d284bd4fe67ff8c49a5db1dd8514fcdab61000000000000000000"
939
- # transactionsmerkleroot: "7829844f4c3a41a537b3131ca992643eaa9d093b2383e4cdc060ad7dc5481187"
940
- # timestamp: "51eb505a"
941
- # target: "c1910018"
942
- # nonce: "de19b302"
943
- header = str(block_version_number + previousblockhash + transactionsmerkleroot + timestamp + target + nonce)
944
- return binascii.hexlify(hashlib.sha256(hashlib.sha256(binascii.unhexlify(header)).digest()).digest()[::-1]).decode('utf-8')
945
-
946
- @GeneralUtilities.check_arguments
947
- def SCChangeHashOfProgram(self, inputfile: str) -> None:
948
- valuetoappend = str(uuid.uuid4())
949
-
950
- outputfile = inputfile + '.modified'
951
-
952
- shutil.copy2(inputfile, outputfile)
953
- with open(outputfile, 'a', encoding="utf-8") as file:
954
- # TODO use rcedit for .exe-files instead of appending valuetoappend ( https://github.com/electron/rcedit/ )
955
- # background: you can retrieve the "original-filename" from the .exe-file like discussed here:
956
- # https://security.stackexchange.com/questions/210843/ is-it-possible-to-change-original-filename-of-an-exe
957
- # so removing the original filename with rcedit is probably a better way to make it more difficult to detect the programname.
958
- # this would obviously also change the hashvalue of the program so appending a whitespace is not required anymore.
959
- file.write(valuetoappend)
960
-
961
- @GeneralUtilities.check_arguments
962
- def __adjust_folder_name(self, folder: str) -> str:
963
- result = os.path.dirname(folder).replace("\\", "/")
964
- if result == "/":
965
- return ""
966
- else:
967
- return result
968
-
969
- @GeneralUtilities.check_arguments
970
- def __create_iso(self, folder, iso_file) -> None:
971
- created_directories = []
972
- files_directory = "FILES"
973
- iso = pycdlib.PyCdlib()
974
- iso.new()
975
- files_directory = files_directory.upper()
976
- iso.add_directory("/" + files_directory)
977
- created_directories.append("/" + files_directory)
978
- for root, _, files in os.walk(folder):
979
- for file in files:
980
- full_path = os.path.join(root, file)
981
- with (open(full_path, "rb").read()) as text_io_wrapper:
982
- content = text_io_wrapper
983
- path_in_iso = '/' + files_directory + \
984
- self.__adjust_folder_name(
985
- full_path[len(folder)::1]).upper()
986
- if path_in_iso not in created_directories:
987
- iso.add_directory(path_in_iso)
988
- created_directories.append(path_in_iso)
989
- iso.add_fp(BytesIO(content), len(content),
990
- path_in_iso + '/' + file.upper() + ';1')
991
- iso.write(iso_file)
992
- iso.close()
993
-
994
- @GeneralUtilities.check_arguments
995
- def SCCreateISOFileWithObfuscatedFiles(self, inputfolder: str, outputfile: str, printtableheadline, createisofile, extensions) -> None:
996
- if (os.path.isdir(inputfolder)):
997
- namemappingfile = "name_map.csv"
998
- files_directory = inputfolder
999
- files_directory_obf = f"{files_directory}_Obfuscated"
1000
- self.SCObfuscateFilesFolder(
1001
- inputfolder, printtableheadline, namemappingfile, extensions)
1002
- os.rename(namemappingfile, os.path.join(
1003
- files_directory_obf, namemappingfile))
1004
- if createisofile:
1005
- self.__create_iso(files_directory_obf, outputfile)
1006
- shutil.rmtree(files_directory_obf)
1007
- else:
1008
- raise ValueError(f"Directory not found: '{inputfolder}'")
1009
-
1010
- @GeneralUtilities.check_arguments
1011
- def SCFilenameObfuscator(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
1012
- obfuscate_all_files = extensions == "*"
1013
- if (obfuscate_all_files):
1014
- obfuscate_file_extensions = None
1015
- else:
1016
- obfuscate_file_extensions = extensions.split(",")
1017
- if (os.path.isdir(inputfolder)):
1018
- printtableheadline = GeneralUtilities.string_to_boolean(
1019
- printtableheadline)
1020
- files = []
1021
- if not os.path.isfile(namemappingfile):
1022
- with open(namemappingfile, "a", encoding="utf-8"):
1023
- pass
1024
- if printtableheadline:
1025
- GeneralUtilities.append_line_to_file(
1026
- namemappingfile, "Original filename;new filename;SHA2-hash of file")
1027
- for file in GeneralUtilities.absolute_file_paths(inputfolder):
1028
- if os.path.isfile(os.path.join(inputfolder, file)):
1029
- if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
1030
- files.append(file)
1031
- for file in files:
1032
- hash_value = GeneralUtilities.get_sha256_of_file(file)
1033
- extension = Path(file).suffix
1034
- new_file_name_without_path = str(uuid.uuid4())[0:8] + extension
1035
- new_file_name = os.path.join(
1036
- os.path.dirname(file), new_file_name_without_path)
1037
- os.rename(file, new_file_name)
1038
- GeneralUtilities.append_line_to_file(namemappingfile, os.path.basename(
1039
- file) + ";" + new_file_name_without_path + ";" + hash_value)
1040
- else:
1041
- raise ValueError(f"Directory not found: '{inputfolder}'")
1042
-
1043
- @GeneralUtilities.check_arguments
1044
- def __extension_matchs(self, file: str, obfuscate_file_extensions) -> bool:
1045
- for extension in obfuscate_file_extensions:
1046
- if file.lower().endswith("."+extension.lower()):
1047
- return True
1048
- return False
1049
-
1050
- @GeneralUtilities.check_arguments
1051
- def SCHealthcheck(self, file: str) -> int:
1052
- lines = GeneralUtilities.read_lines_from_file(file)
1053
- for line in reversed(lines):
1054
- if not GeneralUtilities.string_is_none_or_whitespace(line):
1055
- if "RunningHealthy (" in line: # TODO use regex
1056
- GeneralUtilities.write_message_to_stderr(
1057
- f"Healthy running due to line '{line}' in file '{file}'.")
1058
- return 0
1059
- else:
1060
- GeneralUtilities.write_message_to_stderr(
1061
- f"Not healthy running due to line '{line}' in file '{file}'.")
1062
- return 1
1063
- GeneralUtilities.write_message_to_stderr(
1064
- f"No valid line found for healthycheck in file '{file}'.")
1065
- return 2
1066
-
1067
- @GeneralUtilities.check_arguments
1068
- def SCObfuscateFilesFolder(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
1069
- obfuscate_all_files = extensions == "*"
1070
- if (obfuscate_all_files):
1071
- obfuscate_file_extensions = None
1072
- else:
1073
- if "," in extensions:
1074
- obfuscate_file_extensions = extensions.split(",")
1075
- else:
1076
- obfuscate_file_extensions = [extensions]
1077
- newd = inputfolder+"_Obfuscated"
1078
- shutil.copytree(inputfolder, newd)
1079
- inputfolder = newd
1080
- if (os.path.isdir(inputfolder)):
1081
- for file in GeneralUtilities.absolute_file_paths(inputfolder):
1082
- if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
1083
- self.SCChangeHashOfProgram(file)
1084
- os.remove(file)
1085
- os.rename(file + ".modified", file)
1086
- self.SCFilenameObfuscator(inputfolder, printtableheadline, namemappingfile, extensions)
1087
- else:
1088
- raise ValueError(f"Directory not found: '{inputfolder}'")
1089
-
1090
- @GeneralUtilities.check_arguments
1091
- def get_services_from_yaml_file(self, yaml_file: str) -> list[str]:
1092
- with open(yaml_file, encoding="utf-8") as stream:
1093
- loaded = yaml.safe_load(stream)
1094
- services = loaded["services"]
1095
- result = list(services.keys())
1096
- return result
1097
-
1098
- @GeneralUtilities.check_arguments
1099
- def kill_docker_container(self, container_name: str) -> None:
1100
- self.run_program("docker", f"container rm -f {container_name}")
1101
-
1102
- @GeneralUtilities.check_arguments
1103
- def get_docker_debian_version(self, image_tag: str) -> str:
1104
- result = ScriptCollectionCore().run_program_argsasarray(
1105
- "docker", ['run', f'debian:{image_tag}', 'bash', '-c', 'apt-get -y update && apt-get -y install lsb-release && lsb_release -cs'])
1106
- result_line = GeneralUtilities.string_to_lines(result[1])[-2]
1107
- return result_line
1108
-
1109
- @GeneralUtilities.check_arguments
1110
- def get_latest_tor_version_of_debian_repository(self, debian_version: str) -> str:
1111
- package_url: str = f"https://deb.torproject.org/torproject.org/dists/{debian_version}/main/binary-amd64/Packages"
1112
- r = requests.get(package_url, timeout=5)
1113
- if r.status_code != 200:
1114
- raise ValueError(f"Checking for latest tor package resulted in HTTP-response-code {r.status_code}.")
1115
- lines = GeneralUtilities.string_to_lines(GeneralUtilities.bytes_to_string(r.content))
1116
- version_line_prefix = "Version: "
1117
- version_content_line = [line for line in lines if line.startswith(version_line_prefix)][1]
1118
- version_with_overhead = version_content_line[len(version_line_prefix):]
1119
- tor_version = version_with_overhead.split("~")[0]
1120
- return tor_version
1121
-
1122
- @GeneralUtilities.check_arguments
1123
- def upload_file_to_file_host(self, file: str, host: str) -> int:
1124
- if (host is None):
1125
- return self.upload_file_to_random_filesharing_service(file)
1126
- elif host == "anonfiles.com":
1127
- return self.upload_file_to_anonfiles(file)
1128
- elif host == "bayfiles.com":
1129
- return self.upload_file_to_bayfiles(file)
1130
- GeneralUtilities.write_message_to_stderr("Unknown host: "+host)
1131
- return 1
1132
-
1133
- @GeneralUtilities.check_arguments
1134
- def upload_file_to_random_filesharing_service(self, file: str) -> int:
1135
- host = randrange(2)
1136
- if host == 0:
1137
- return self.upload_file_to_anonfiles(file)
1138
- if host == 1:
1139
- return self.upload_file_to_bayfiles(file)
1140
- return 1
1141
-
1142
- @GeneralUtilities.check_arguments
1143
- def upload_file_to_anonfiles(self, file) -> int:
1144
- return self.upload_file_by_using_simple_curl_request("https://api.anonfiles.com/upload", file)
1145
-
1146
- @GeneralUtilities.check_arguments
1147
- def upload_file_to_bayfiles(self, file) -> int:
1148
- return self.upload_file_by_using_simple_curl_request("https://api.bayfiles.com/upload", file)
1149
-
1150
- @GeneralUtilities.check_arguments
1151
- def upload_file_by_using_simple_curl_request(self, api_url: str, file: str) -> int:
1152
- # TODO implement
1153
- return 1
1154
-
1155
- @GeneralUtilities.check_arguments
1156
- def file_is_available_on_file_host(self, file) -> int:
1157
- # TODO implement
1158
- return 1
1159
-
1160
- def run_testcases_for_python_project(self, repository_folder: str):
1161
- self.run_program("coverage", "run -m pytest", repository_folder)
1162
- self.run_program("coverage", "xml", repository_folder)
1163
- GeneralUtilities.ensure_directory_exists(os.path.join(repository_folder, "Other/TestCoverage"))
1164
- coveragefile = os.path.join(repository_folder, "Other/TestCoverage/TestCoverage.xml")
1165
- GeneralUtilities.ensure_file_does_not_exist(coveragefile)
1166
- os.rename(os.path.join(repository_folder, "coverage.xml"), coveragefile)
1167
-
1168
- @GeneralUtilities.check_arguments
1169
- def get_file_permission(self, file: str) -> str:
1170
- """This function returns an usual octet-triple, for example "0700"."""
1171
- ls_output = self.__ls(file)
1172
- return self.__get_file_permission_helper(ls_output)
1173
-
1174
- @GeneralUtilities.check_arguments
1175
- def __get_file_permission_helper(self, ls_output: str) -> str:
1176
- permissions = ' '.join(ls_output.split()).split(' ')[0][1:]
1177
- return str(self.__to_octet(permissions[0:3]))+str(self.__to_octet(permissions[3:6]))+str(self.__to_octet(permissions[6:9]))
1178
-
1179
- @GeneralUtilities.check_arguments
1180
- def __to_octet(self, string: str) -> int:
1181
- return int(self.__to_octet_helper(string[0])+self.__to_octet_helper(string[1])+self.__to_octet_helper(string[2]), 2)
1182
-
1183
- @GeneralUtilities.check_arguments
1184
- def __to_octet_helper(self, string: str) -> str:
1185
- if (string == "-"):
1186
- return "0"
1187
- else:
1188
- return "1"
1189
-
1190
- @GeneralUtilities.check_arguments
1191
- def get_file_owner(self, file: str) -> str:
1192
- """This function returns the user and the group in the format "user:group"."""
1193
- ls_output = self.__ls(file)
1194
- return self.__get_file_owner_helper(ls_output)
1195
-
1196
- @GeneralUtilities.check_arguments
1197
- def __get_file_owner_helper(self, ls_output: str) -> str:
1198
- try:
1199
- splitted = ' '.join(ls_output.split()).split(' ')
1200
- return f"{splitted[2]}:{splitted[3]}"
1201
- except Exception as exception:
1202
- raise ValueError(
1203
- f"ls-output '{ls_output}' not parsable") from exception
1204
-
1205
- @GeneralUtilities.check_arguments
1206
- def get_file_owner_and_file_permission(self, file: str) -> str:
1207
- ls_output = self.__ls(file)
1208
- return [self.__get_file_owner_helper(ls_output), self.__get_file_permission_helper(ls_output)]
1209
-
1210
- @GeneralUtilities.check_arguments
1211
- def __ls(self, file: str) -> str:
1212
- file = file.replace("\\", "/")
1213
- GeneralUtilities.assert_condition(os.path.isfile(file) or os.path.isdir(
1214
- file), f"Can not execute 'ls' because '{file}' does not exist")
1215
- result = self.run_program_argsasarray("ls", ["-ld", file])
1216
- GeneralUtilities.assert_condition(result[0] == 0, f"'ls -ld {file}' resulted in exitcode {str(result[0])}. StdErr: {result[2]}")
1217
- GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(
1218
- result[1]), f"'ls' of '{file}' had an empty output. StdErr: '{result[2]}'")
1219
- return result[1]
1220
-
1221
- @GeneralUtilities.check_arguments
1222
- def set_permission(self, file_or_folder: str, permissions: str, recursive: bool = False) -> None:
1223
- """This function expects an usual octet-triple, for example "700"."""
1224
- args = []
1225
- if recursive:
1226
- args.append("--recursive")
1227
- args.append(permissions)
1228
- args.append(file_or_folder)
1229
- self.run_program_argsasarray("chmod", args)
1230
-
1231
- @GeneralUtilities.check_arguments
1232
- def set_owner(self, file_or_folder: str, owner: str, recursive: bool = False, follow_symlinks: bool = False) -> None:
1233
- """This function expects the user and the group in the format "user:group"."""
1234
- args = []
1235
- if recursive:
1236
- args.append("--recursive")
1237
- if follow_symlinks:
1238
- args.append("--no-dereference")
1239
- args.append(owner)
1240
- args.append(file_or_folder)
1241
- self.run_program_argsasarray("chown", args)
1242
-
1243
- # <run programs>
1244
-
1245
- @GeneralUtilities.check_arguments
1246
- def __run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> Popen:
1247
- # Verbosity:
1248
- # 0=Quiet (No output will be printed.)
1249
- # 1=Normal (If the exitcode of the executed program is not 0 then the StdErr will be printed.)
1250
- # 2=Full (Prints StdOut and StdErr of the executed program.)
1251
- # 3=Verbose (Same as "Full" but with some more information.)
1252
-
1253
- if isinstance(self.program_runner, ProgramRunnerEpew):
1254
- custom_argument = CustomEpewArgument(print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, verbosity, arguments_for_log)
1255
- popen: Popen = self.program_runner.run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, custom_argument, interactive)
1256
- return popen
1257
-
1258
- # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
1259
-
1260
- @GeneralUtilities.check_arguments
1261
- def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False) -> tuple[int, str, str, int]:
1262
- # verbosity 1: No output will be logged.
1263
- # verbosity 2: If the exitcode of the executed program is not 0 then the StdErr will be logged. This is supposed to be the default verbosity-level.
1264
- # verbosity 3: Logs and prints StdOut and StdErr of the executed program in realtime.
1265
- # verbosity 4: Same as loglevel 3 but with some more overhead-information.
1266
- try:
1267
- arguments_as_str = ' '.join(arguments_as_array)
1268
- mock_loader_result = self.__try_load_mock(program, arguments_as_str, working_directory)
1269
- if mock_loader_result[0]:
1270
- return mock_loader_result[1]
1271
-
1272
- working_directory = self.__adapt_workingdirectory(working_directory)
1273
-
1274
- if arguments_for_log is None:
1275
- arguments_for_log = arguments_as_array
1276
-
1277
- arguments_for_log_as_string: str = ' '.join(arguments_for_log)
1278
- cmd = f'{working_directory}>{program} {arguments_for_log_as_string}'
1279
-
1280
- if GeneralUtilities.string_is_none_or_whitespace(title):
1281
- info_for_log = cmd
1282
- else:
1283
- info_for_log = title
1284
-
1285
- if verbosity >= 3:
1286
- GeneralUtilities.write_message_to_stdout(f"Run '{info_for_log}'.")
1287
- with self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive) as process:
1288
- pid = process.pid
1289
-
1290
- stdout_lines = list[str]()
1291
- stderr_lines = list[str]()
1292
-
1293
- live_console_output_printing = 2 < verbosity and interactive
1294
-
1295
- log_to_file = log_file is not None
1296
- if log_to_file:
1297
- GeneralUtilities.ensure_file_exists(log_file)
1298
-
1299
- if interactive:
1300
- # there are 2 issues in this part:
1301
- # 1.: ctrl+c
1302
- # 2.: sometimes this function does not terminate even if the started process exited
1303
- def stream_process(process) -> bool:
1304
- try:
1305
- go: bool = process.poll() is None
1306
-
1307
- stdoutreader: BufferedReader = process.stdout
1308
- if stdoutreader.readable():
1309
- stdoutresultb: bytes = stdoutreader.read()
1310
- stdoutresult = GeneralUtilities.bytes_to_string(
1311
- stdoutresultb)
1312
- stdoutlines = GeneralUtilities.string_to_lines(stdoutresult)
1313
- for line in stdoutlines:
1314
- line_stripped = line.replace("\r", "").strip()
1315
- if len(line_stripped) > 0:
1316
- line_str = line_stripped
1317
- stdout_lines.append(line_str)
1318
- if live_console_output_printing:
1319
- GeneralUtilities.write_message_to_stdout(line_str)
1320
- if log_to_file:
1321
- GeneralUtilities.append_line_to_file(
1322
- log_file, line_str)
1323
-
1324
- stderrreader: BufferedReader = process.stderr
1325
- if stderrreader.readable():
1326
- stderrresultb: bytes = stderrreader.read()
1327
- stderrresult = GeneralUtilities.bytes_to_string(stderrresultb)
1328
- stderrlines = GeneralUtilities.string_to_lines(stderrresult)
1329
- for line in stderrlines:
1330
- line_stripped = line.replace("\r", "").strip()
1331
- if len(line_stripped) > 0:
1332
- line_str = line_stripped
1333
- stderr_lines.append(line_str)
1334
- if live_console_output_printing:
1335
- if print_errors_as_information:
1336
- GeneralUtilities.write_message_to_stdout(line_str)
1337
- else:
1338
- GeneralUtilities.write_message_to_stderr(line_str)
1339
- if log_to_file:
1340
- GeneralUtilities.append_line_to_file(log_file, line_str)
1341
-
1342
- return go
1343
- except Exception:
1344
- return False
1345
-
1346
- while stream_process(process):
1347
- time.sleep(0.1)
1348
-
1349
- exit_code = process.poll()
1350
- stdout = '\n'.join(stdout_lines)
1351
- stderr = '\n'.join(stderr_lines)
1352
- else:
1353
- stdout, stderr = process.communicate()
1354
- exit_code = process.wait()
1355
- stdout = GeneralUtilities.bytes_to_string(stdout).replace('\r', '')
1356
- stderr = GeneralUtilities.bytes_to_string(stderr).replace('\r', '')
1357
-
1358
- if throw_exception_if_exitcode_is_not_zero and exit_code != 0:
1359
- raise ValueError(f"Program '{working_directory}>{program} {arguments_for_log_as_string}' resulted in exitcode {exit_code}. (StdOut: '{stdout}', StdErr: '{stderr}')")
1360
-
1361
- result = (exit_code, stdout, stderr, pid)
1362
- return result
1363
- except Exception as e:
1364
- raise e
1365
-
1366
- # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
1367
- @GeneralUtilities.check_arguments
1368
- def run_program(self, program: str, arguments: str = "", working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False) -> tuple[int, str, str, int]:
1369
- return self.run_program_argsasarray(program, GeneralUtilities.arguments_to_array(arguments), working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive)
1370
-
1371
- # Return-values program_runner: Pid
1372
- @GeneralUtilities.check_arguments
1373
- def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
1374
- mock_loader_result = self.__try_load_mock(program, ' '.join(arguments_as_array), working_directory)
1375
- if mock_loader_result[0]:
1376
- return mock_loader_result[1]
1377
- process: Popen = self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
1378
- return process.pid
1379
-
1380
- # Return-values program_runner: Pid
1381
- @GeneralUtilities.check_arguments
1382
- def run_program_async(self, program: str, arguments: str = "", working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
1383
- return self.run_program_argsasarray_async(program, GeneralUtilities.arguments_to_array(arguments), working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
1384
-
1385
- @GeneralUtilities.check_arguments
1386
- def __try_load_mock(self, program: str, arguments: str, working_directory: str) -> tuple[bool, tuple[int, str, str, int]]:
1387
- if self.mock_program_calls:
1388
- try:
1389
- return [True, self.__get_mock_program_call(program, arguments, working_directory)]
1390
- except LookupError:
1391
- if not self.execute_program_really_if_no_mock_call_is_defined:
1392
- raise
1393
- return [False, None]
1394
-
1395
- @GeneralUtilities.check_arguments
1396
- def __adapt_workingdirectory(self, workingdirectory: str) -> str:
1397
- if workingdirectory is None:
1398
- return os.getcwd()
1399
- else:
1400
- return GeneralUtilities.resolve_relative_path_from_current_working_directory(workingdirectory)
1401
-
1402
- @GeneralUtilities.check_arguments
1403
- def verify_no_pending_mock_program_calls(self):
1404
- if (len(self.__mocked_program_calls) > 0):
1405
- raise AssertionError(
1406
- "The following mock-calls were not called:\n"+",\n ".join([self.__format_mock_program_call(r) for r in self.__mocked_program_calls]))
1407
-
1408
- @GeneralUtilities.check_arguments
1409
- def __format_mock_program_call(self, r) -> str:
1410
- r: ScriptCollectionCore.__MockProgramCall = r
1411
- return f"'{r.workingdirectory}>{r.program} {r.argument}' (" \
1412
- f"exitcode: {GeneralUtilities.str_none_safe(str(r.exit_code))}, " \
1413
- f"pid: {GeneralUtilities.str_none_safe(str(r.pid))}, "\
1414
- f"stdout: {GeneralUtilities.str_none_safe(str(r.stdout))}, " \
1415
- f"stderr: {GeneralUtilities.str_none_safe(str(r.stderr))})"
1416
-
1417
- @GeneralUtilities.check_arguments
1418
- def register_mock_program_call(self, program: str, argument: str, workingdirectory: str, result_exit_code: int, result_stdout: str, result_stderr: str, result_pid: int, amount_of_expected_calls=1):
1419
- "This function is for test-purposes only"
1420
- for _ in itertools.repeat(None, amount_of_expected_calls):
1421
- mock_call = ScriptCollectionCore.__MockProgramCall()
1422
- mock_call.program = program
1423
- mock_call.argument = argument
1424
- mock_call.workingdirectory = workingdirectory
1425
- mock_call.exit_code = result_exit_code
1426
- mock_call.stdout = result_stdout
1427
- mock_call.stderr = result_stderr
1428
- mock_call.pid = result_pid
1429
- self.__mocked_program_calls.append(mock_call)
1430
-
1431
- @GeneralUtilities.check_arguments
1432
- def __get_mock_program_call(self, program: str, argument: str, workingdirectory: str):
1433
- result: ScriptCollectionCore.__MockProgramCall = None
1434
- for mock_call in self.__mocked_program_calls:
1435
- if ((re.match(mock_call.program, program) is not None)
1436
- and (re.match(mock_call.argument, argument) is not None)
1437
- and (re.match(mock_call.workingdirectory, workingdirectory) is not None)):
1438
- result = mock_call
1439
- break
1440
- if result is None:
1441
- raise LookupError(
1442
- f"Tried to execute mock-call '{workingdirectory}>{program} {argument}' but no mock-call was defined for that execution")
1443
- else:
1444
- self.__mocked_program_calls.remove(result)
1445
- return (result.exit_code, result.stdout, result.stderr, result.pid)
1446
-
1447
- @GeneralUtilities.check_arguments
1448
- class __MockProgramCall:
1449
- program: str
1450
- argument: str
1451
- workingdirectory: str
1452
- exit_code: int
1453
- stdout: str
1454
- stderr: str
1455
- pid: int
1456
-
1457
- # </run programs>
1458
-
1459
- @GeneralUtilities.check_arguments
1460
- def extract_archive_with_7z(self, unzip_program_file: str, zipfile: str, password: str, output_directory: str) -> None:
1461
- password_set = not password is None
1462
- file_name = Path(zipfile).name
1463
- file_folder = os.path.dirname(zipfile)
1464
- argument = "x"
1465
- if password_set:
1466
- argument = f"{argument} -p\"{password}\""
1467
- argument = f"{argument} -o {output_directory}"
1468
- argument = f"{argument} {file_name}"
1469
- return self.run_program(unzip_program_file, argument, file_folder)
1470
-
1471
- @GeneralUtilities.check_arguments
1472
- def get_internet_time(self) -> datetime:
1473
- response = ntplib.NTPClient().request('pool.ntp.org')
1474
- return datetime.fromtimestamp(response.tx_time)
1475
-
1476
- @GeneralUtilities.check_arguments
1477
- def system_time_equals_internet_time(self, maximal_tolerance_difference: timedelta) -> bool:
1478
- return abs(datetime.now() - self.get_internet_time()) < maximal_tolerance_difference
1479
-
1480
- @GeneralUtilities.check_arguments
1481
- def system_time_equals_internet_time_with_default_tolerance(self) -> bool:
1482
- return self.system_time_equals_internet_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
1483
-
1484
- @GeneralUtilities.check_arguments
1485
- def check_system_time(self, maximal_tolerance_difference: timedelta):
1486
- if not self.system_time_equals_internet_time(maximal_tolerance_difference):
1487
- raise ValueError("System time may be wrong")
1488
-
1489
- @GeneralUtilities.check_arguments
1490
- def check_system_time_with_default_tolerance(self) -> None:
1491
- self.check_system_time(
1492
- self.__get_default_tolerance_for_system_time_equals_internet_time())
1493
-
1494
- @GeneralUtilities.check_arguments
1495
- def __get_default_tolerance_for_system_time_equals_internet_time(self) -> timedelta:
1496
- return timedelta(hours=0, minutes=0, seconds=3)
1497
-
1498
- @GeneralUtilities.check_arguments
1499
- def increment_version(self, input_version: str, increment_major: bool, increment_minor: bool, increment_patch: bool) -> str:
1500
- splitted = input_version.split(".")
1501
- GeneralUtilities.assert_condition(len(splitted) == 3, f"Version '{input_version}' does not have the 'major.minor.patch'-pattern.")
1502
- major = int(splitted[0])
1503
- minor = int(splitted[1])
1504
- patch = int(splitted[2])
1505
- if increment_major:
1506
- major = major+1
1507
- if increment_minor:
1508
- minor = minor+1
1509
- if increment_patch:
1510
- patch = patch+1
1511
- return f"{major}.{minor}.{patch}"
1512
-
1513
- @GeneralUtilities.check_arguments
1514
- def get_semver_version_from_gitversion(self, repository_folder: str) -> str:
1515
- if (self.git_repository_has_commits(repository_folder)):
1516
- result = self.get_version_from_gitversion(repository_folder, "MajorMinorPatch")
1517
- if self.git_repository_has_uncommitted_changes(repository_folder):
1518
- if self.get_current_git_branch_has_tag(repository_folder):
1519
- id_of_latest_tag = self.git_get_commitid_of_tag(
1520
- repository_folder, self.get_latest_git_tag(repository_folder))
1521
- current_commit = self.git_get_commit_id(repository_folder)
1522
- current_commit_is_on_latest_tag = id_of_latest_tag == current_commit
1523
- if current_commit_is_on_latest_tag:
1524
- result = self.increment_version(
1525
- result, False, False, True)
1526
- else:
1527
- result = "0.1.0"
1528
- return result
1529
-
1530
- @staticmethod
1531
- @GeneralUtilities.check_arguments
1532
- def is_patch_version(version_string: str) -> bool:
1533
- return not version_string.endswith(".0")
1534
-
1535
- @GeneralUtilities.check_arguments
1536
- def get_version_from_gitversion(self, folder: str, variable: str) -> str:
1537
- # called twice as workaround for issue 1877 in gitversion ( https://github.com/GitTools/GitVersion/issues/1877 )
1538
- result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder, verbosity=0)
1539
- result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder, verbosity=0)
1540
- result = GeneralUtilities.strip_new_line_character(result[1])
1541
-
1542
- return result
1543
-
1544
- @GeneralUtilities.check_arguments
1545
- def generate_certificate_authority(self, folder: str, name: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
1546
- if days_until_expire is None:
1547
- days_until_expire = 1825
1548
- if password is None:
1549
- password = GeneralUtilities.generate_password()
1550
- GeneralUtilities.ensure_directory_exists(folder)
1551
- self.run_program("openssl", f'req -new -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -days {days_until_expire} -nodes -x509 -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={name}/OU={subj_ou} -passout pass:{password} -keyout {name}.key -out {name}.crt', folder)
1552
-
1553
- @GeneralUtilities.check_arguments
1554
- def generate_certificate(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
1555
- if days_until_expire is None:
1556
- days_until_expire = 397
1557
- if password is None:
1558
- password = GeneralUtilities.generate_password()
1559
- rsa_key_length = 4096
1560
- self.run_program("openssl", f'genrsa -out {filename}.key {rsa_key_length}', folder)
1561
- self.run_program("openssl", f'req -new -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou} -x509 -key {filename}.key -out {filename}.unsigned.crt -days {days_until_expire}', folder)
1562
- self.run_program("openssl", f'pkcs12 -export -out {filename}.selfsigned.pfx -password pass:{password} -inkey {filename}.key -in {filename}.unsigned.crt', folder)
1563
- GeneralUtilities.write_text_to_file(
1564
- os.path.join(folder, f"{filename}.password"), password)
1565
- GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.san.conf"), f"""[ req ]
1566
- default_bits = {rsa_key_length}
1567
- distinguished_name = req_distinguished_name
1568
- req_extensions = v3_req
1569
- default_md = sha256
1570
- dirstring_type = nombstr
1571
- prompt = no
1572
-
1573
- [ req_distinguished_name ]
1574
- countryName = {subj_c}
1575
- stateOrProvinceName = {subj_st}
1576
- localityName = {subj_l}
1577
- organizationName = {subj_o}
1578
- organizationUnit = {subj_ou}
1579
- commonName = {domain}
1580
-
1581
- [v3_req]
1582
- subjectAltName = @subject_alt_name
1583
-
1584
- [ subject_alt_name ]
1585
- DNS = {domain}
1586
- """)
1587
-
1588
- @GeneralUtilities.check_arguments
1589
- def generate_certificate_sign_request(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str) -> None:
1590
- self.run_program(
1591
- "openssl", f'req -new -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou} -key {filename}.key -out {filename}.csr -config {filename}.san.conf', folder)
1592
-
1593
- @GeneralUtilities.check_arguments
1594
- def sign_certificate(self, folder: str, ca_folder: str, ca_name: str, domain: str, filename: str, days_until_expire: int = None) -> None:
1595
- if days_until_expire is None:
1596
- days_until_expire = 397
1597
- ca = os.path.join(ca_folder, ca_name)
1598
- password_file = os.path.join(folder, f"{filename}.password")
1599
- password = GeneralUtilities.read_text_from_file(password_file)
1600
- self.run_program("openssl", f'x509 -req -in {filename}.csr -CA {ca}.crt -CAkey {ca}.key -CAcreateserial -CAserial {ca}.srl -out {filename}.crt -days {days_until_expire} -sha256 -extensions v3_req -extfile {filename}.san.conf', folder)
1601
- self.run_program("openssl", f'pkcs12 -export -out {filename}.pfx -inkey {filename}.key -in {filename}.crt -password pass:{password}', folder)
1602
-
1603
- @GeneralUtilities.check_arguments
1604
- def update_dependencies_of_python_in_requirementstxt_file(self, file: str, verbosity: int):
1605
- lines = GeneralUtilities.read_lines_from_file(file)
1606
- new_lines = []
1607
- for line in lines:
1608
- if GeneralUtilities.string_has_content(line):
1609
- new_lines.append(self.__get_updated_line_for_python_requirements(line.strip()))
1610
- GeneralUtilities.write_lines_to_file(file, new_lines)
1611
-
1612
- @GeneralUtilities.check_arguments
1613
- def __get_updated_line_for_python_requirements(self, line: str) -> str:
1614
- if "==" in line or "<" in line:
1615
- return line
1616
- elif ">" in line:
1617
- try:
1618
- # line is something like "cyclonedx-bom>=2.0.2" and the function must return with the updated version
1619
- # (something like "cyclonedx-bom>=3.11.0" for example)
1620
- package = line.split(">")[0]
1621
- operator = ">=" if ">=" in line else ">"
1622
- response = requests.get(f'https://pypi.org/pypi/{package}/json', timeout=5)
1623
- latest_version = response.json()['info']['version']
1624
- return package+operator+latest_version
1625
- except:
1626
- return line
1627
- else:
1628
- raise ValueError(f'Unexpected line in requirements-file: "{line}"')
1629
-
1630
- @GeneralUtilities.check_arguments
1631
- def update_dependencies_of_python_in_setupcfg_file(self, setup_cfg_file: str, verbosity: int):
1632
- lines = GeneralUtilities.read_lines_from_file(setup_cfg_file)
1633
- new_lines = []
1634
- requirement_parsing_mode = False
1635
- for line in lines:
1636
- new_line = line
1637
- if (requirement_parsing_mode):
1638
- if ("<" in line or "=" in line or ">" in line):
1639
- updated_line = f" {self.__get_updated_line_for_python_requirements(line.strip())}"
1640
- new_line = updated_line
1641
- else:
1642
- requirement_parsing_mode = False
1643
- else:
1644
- if line.startswith("install_requires ="):
1645
- requirement_parsing_mode = True
1646
- new_lines.append(new_line)
1647
- GeneralUtilities.write_lines_to_file(setup_cfg_file, new_lines)
1648
-
1649
- @GeneralUtilities.check_arguments
1650
- def update_dependencies_of_dotnet_project(self, csproj_file: str, verbosity: int, ignored_dependencies: list[str]):
1651
- folder = os.path.dirname(csproj_file)
1652
- csproj_filename = os.path.basename(csproj_file)
1653
- GeneralUtilities.write_message_to_stderr(f"Check for updates in {csproj_filename}")
1654
- result = self.run_program("dotnet", f"list {csproj_filename} package --outdated", folder)
1655
- for line in result[1].replace("\r", "").split("\n"):
1656
- # Relevant output-lines are something like " > NJsonSchema 10.7.0 10.7.0 10.9.0"
1657
- if ">" in line:
1658
- package_name = line.replace(">", "").strip().split(" ")[0]
1659
- if not (ignored_dependencies in package_name):
1660
- GeneralUtilities.write_message_to_stderr(f"Update package {package_name}")
1661
- self.run_program("dotnet", f"add {csproj_filename} package {package_name}", folder)
1662
-
1663
- @GeneralUtilities.check_arguments
1664
- def create_deb_package(self, toolname: str, binary_folder: str, control_file_content: str, deb_output_folder: str, verbosity: int, permission_of_executable_file_as_octet_triple: int) -> None:
1665
-
1666
- # prepare
1667
- GeneralUtilities.ensure_directory_exists(deb_output_folder)
1668
- temp_folder = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
1669
- GeneralUtilities.ensure_directory_exists(temp_folder)
1670
- bin_folder = binary_folder
1671
- tool_content_folder_name = toolname+"Content"
1672
-
1673
- # create folder
1674
- GeneralUtilities.ensure_directory_exists(temp_folder)
1675
- control_content_folder_name = "controlcontent"
1676
- packagecontent_control_folder = os.path.join(temp_folder, control_content_folder_name)
1677
- GeneralUtilities.ensure_directory_exists(packagecontent_control_folder)
1678
- data_content_folder_name = "datacontent"
1679
- packagecontent_data_folder = os.path.join(temp_folder, data_content_folder_name)
1680
- GeneralUtilities.ensure_directory_exists(packagecontent_data_folder)
1681
- entireresult_content_folder_name = "entireresultcontent"
1682
- packagecontent_entireresult_folder = os.path.join(temp_folder, entireresult_content_folder_name)
1683
- GeneralUtilities.ensure_directory_exists(packagecontent_entireresult_folder)
1684
-
1685
- # create "debian-binary"-file
1686
- debianbinary_file = os.path.join(
1687
- packagecontent_entireresult_folder, "debian-binary")
1688
- GeneralUtilities.ensure_file_exists(debianbinary_file)
1689
- GeneralUtilities.write_text_to_file(debianbinary_file, "2.0\n")
1690
-
1691
- # create control-content
1692
-
1693
- # conffiles
1694
- conffiles_file = os.path.join(packagecontent_control_folder, "conffiles")
1695
- GeneralUtilities.ensure_file_exists(conffiles_file)
1696
-
1697
- # postinst-script
1698
- postinst_file = os.path.join(packagecontent_control_folder, "postinst")
1699
- GeneralUtilities.ensure_file_exists(postinst_file)
1700
- exe_file = f"/usr/bin/{tool_content_folder_name}/{toolname}"
1701
- link_file = f"/usr/bin/{toolname.lower()}"
1702
- permission = str(permission_of_executable_file_as_octet_triple)
1703
- GeneralUtilities.write_text_to_file(postinst_file, f"""#!/bin/sh
1704
- ln -s {exe_file} {link_file}
1705
- chmod {permission} {exe_file}
1706
- chmod {permission} {link_file}
1707
- """)
1708
-
1709
- # control
1710
- control_file = os.path.join(packagecontent_control_folder, "control")
1711
- GeneralUtilities.ensure_file_exists(control_file)
1712
- GeneralUtilities.write_text_to_file(control_file, control_file_content)
1713
-
1714
- # md5sums
1715
- md5sums_file = os.path.join(packagecontent_control_folder, "md5sums")
1716
- GeneralUtilities.ensure_file_exists(md5sums_file)
1717
-
1718
- # create data-content
1719
-
1720
- # copy binaries
1721
- usr_bin_folder = os.path.join(packagecontent_data_folder, "usr/bin")
1722
- GeneralUtilities.ensure_directory_exists(usr_bin_folder)
1723
- usr_bin_content_folder = os.path.join(
1724
- usr_bin_folder, tool_content_folder_name)
1725
- GeneralUtilities.copy_content_of_folder(bin_folder, usr_bin_content_folder)
1726
-
1727
- # create debfile
1728
- deb_filename = f"{toolname}.deb"
1729
- self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/control.tar.gz", "*"], packagecontent_control_folder, verbosity=verbosity)
1730
- self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/data.tar.gz", "*"], packagecontent_data_folder, verbosity=verbosity)
1731
- self.run_program_argsasarray("ar", ["r", deb_filename, "debian-binary", "control.tar.gz",
1732
- "data.tar.gz"], packagecontent_entireresult_folder, verbosity=verbosity)
1733
- result_file = os.path.join(packagecontent_entireresult_folder, deb_filename)
1734
- shutil.copy(result_file, os.path.join(deb_output_folder, deb_filename))
1735
-
1736
- # cleanup
1737
- GeneralUtilities.ensure_directory_does_not_exist(temp_folder)
1738
-
1739
- @GeneralUtilities.check_arguments
1740
- def update_year_in_copyright_tags(self, file: str) -> None:
1741
- current_year = str(datetime.now().year)
1742
- lines = GeneralUtilities.read_lines_from_file(file)
1743
- lines_result = []
1744
- for line in lines:
1745
- if match := re.search("(.*<[Cc]opyright>.*)\\d\\d\\d\\d(.*<\\/[Cc]opyright>.*)", line):
1746
- part1 = match.group(1)
1747
- part2 = match.group(2)
1748
- adapted = part1+current_year+part2
1749
- else:
1750
- adapted = line
1751
- lines_result.append(adapted)
1752
- GeneralUtilities.write_lines_to_file(file, lines_result)
1753
-
1754
- @GeneralUtilities.check_arguments
1755
- def update_year_in_first_line_of_file(self, file: str) -> None:
1756
- current_year = str(datetime.now().year)
1757
- lines = GeneralUtilities.read_lines_from_file(file)
1758
- lines[0] = re.sub("\\d\\d\\d\\d", current_year, lines[0])
1759
- GeneralUtilities.write_lines_to_file(file, lines)
1760
-
1761
- @GeneralUtilities.check_arguments
1762
- def get_external_ip(self, proxy: str) -> str:
1763
- information = self.get_externalnetworkinformation_as_json_string(proxy)
1764
- parsed = json.loads(information)
1765
- return parsed.ip
1766
-
1767
- @GeneralUtilities.check_arguments
1768
- def get_country_of_external_ip(self, proxy: str) -> str:
1769
- information = self.get_externalnetworkinformation_as_json_string(proxy)
1770
- parsed = json.loads(information)
1771
- return parsed.country
1772
-
1773
- @GeneralUtilities.check_arguments
1774
- def get_externalnetworkinformation_as_json_string(self, proxy: str) -> str:
1775
- proxies = None
1776
- if GeneralUtilities.string_has_content(proxy):
1777
- proxies = {"http": proxy}
1778
- response = requests.get('https://ipinfo.io',
1779
- proxies=proxies, timeout=5)
1780
- network_information_as_json_string = GeneralUtilities.bytes_to_string(
1781
- response.content)
1782
- return network_information_as_json_string
1
+ import time
2
+ from datetime import timedelta, datetime
3
+ import json
4
+ import binascii
5
+ import filecmp
6
+ import hashlib
7
+ from io import BufferedReader, BytesIO
8
+ import itertools
9
+ import math
10
+ import os
11
+ from pathlib import Path
12
+ from random import randrange
13
+ from subprocess import Popen
14
+ import re
15
+ import shutil
16
+ import traceback
17
+ import uuid
18
+ import tempfile
19
+ import io
20
+ import requests
21
+ import ntplib
22
+ import yaml
23
+ import qrcode
24
+ import pycdlib
25
+ import send2trash
26
+ import PyPDF2
27
+ from .GeneralUtilities import GeneralUtilities
28
+ from .ProgramRunnerBase import ProgramRunnerBase
29
+ from .ProgramRunnerPopen import ProgramRunnerPopen
30
+ from .ProgramRunnerEpew import ProgramRunnerEpew, CustomEpewArgument
31
+
32
+ version = "3.5.12"
33
+ __version__ = version
34
+
35
+
36
+ class ScriptCollectionCore:
37
+
38
+ # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
39
+ # Do not change this value for productive environments.
40
+ mock_program_calls: bool = False
41
+ # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
42
+ execute_program_really_if_no_mock_call_is_defined: bool = False
43
+ __mocked_program_calls: list = list()
44
+ program_runner: ProgramRunnerBase = None
45
+
46
+ def __init__(self):
47
+ self.program_runner = ProgramRunnerPopen()
48
+
49
+ @staticmethod
50
+ @GeneralUtilities.check_arguments
51
+ def get_scriptcollection_version() -> str:
52
+ return __version__
53
+
54
+ @GeneralUtilities.check_arguments
55
+ def python_file_has_errors(self, file: str, working_directory: str, treat_warnings_as_errors: bool = True) -> tuple[bool, list[str]]:
56
+ errors = list()
57
+ filename = os.path.relpath(file, working_directory)
58
+ if treat_warnings_as_errors:
59
+ errorsonly_argument = ""
60
+ else:
61
+ errorsonly_argument = " --errors-only"
62
+ (exit_code, stdout, stderr, _) = self.run_program("pylint", filename +
63
+ errorsonly_argument, working_directory, throw_exception_if_exitcode_is_not_zero=False)
64
+ if (exit_code != 0):
65
+ errors.append(f"Linting-issues of {file}:")
66
+ errors.append(f"Pylint-exitcode: {exit_code}")
67
+ for line in GeneralUtilities.string_to_lines(stdout):
68
+ errors.append(line)
69
+ for line in GeneralUtilities.string_to_lines(stderr):
70
+ errors.append(line)
71
+ return (True, errors)
72
+
73
+ return (False, errors)
74
+
75
+ @GeneralUtilities.check_arguments
76
+ def replace_version_in_dockerfile_file(self, dockerfile: str, new_version_value: str) -> None:
77
+ GeneralUtilities.write_text_to_file(dockerfile, re.sub(
78
+ "ARG Version=\"\\d+\\.\\d+\\.\\d+\"", f"ARG Version=\"{new_version_value}\"", GeneralUtilities.read_text_from_file(dockerfile)))
79
+
80
+ @GeneralUtilities.check_arguments
81
+ def replace_version_in_python_file(self, file: str, new_version_value: str):
82
+ GeneralUtilities.write_text_to_file(file, re.sub(
83
+ "version = \"\\d+\\.\\d+\\.\\d+\"", f"version = \"{new_version_value}\"", GeneralUtilities.read_text_from_file(file)))
84
+
85
+ @GeneralUtilities.check_arguments
86
+ def replace_version_in_ini_file(self, file: str, new_version_value: str):
87
+ GeneralUtilities.write_text_to_file(file, re.sub(
88
+ "version = \\d+\\.\\d+\\.\\d+", f"version = {new_version_value}", GeneralUtilities.read_text_from_file(file)))
89
+
90
+ @GeneralUtilities.check_arguments
91
+ def replace_version_in_nuspec_file(self, nuspec_file: str, new_version: str) -> None:
92
+ # TODO use XSLT instead
93
+ versionregex = "\\d+\\.\\d+\\.\\d+"
94
+ versiononlyregex = f"^{versionregex}$"
95
+ pattern = re.compile(versiononlyregex)
96
+ if pattern.match(new_version):
97
+ GeneralUtilities.write_text_to_file(nuspec_file, re.sub(f"<version>{versionregex}<\\/version>", f"<version>{new_version}</version>", GeneralUtilities.read_text_from_file(nuspec_file)))
98
+ else:
99
+ raise ValueError(
100
+ f"Version '{new_version}' does not match version-regex '{versiononlyregex}'")
101
+
102
+ @GeneralUtilities.check_arguments
103
+ def replace_version_in_csproj_file(self, csproj_file: str, current_version: str):
104
+ versionregex = "\\d+\\.\\d+\\.\\d+"
105
+ versiononlyregex = f"^{versionregex}$"
106
+ pattern = re.compile(versiononlyregex)
107
+ if pattern.match(current_version):
108
+ for tag in ["Version", "AssemblyVersion", "FileVersion"]:
109
+ GeneralUtilities.write_text_to_file(csproj_file, re.sub(
110
+ f"<{tag}>{versionregex}(.\\d+)?<\\/{tag}>", f"<{tag}>{current_version}</{tag}>", GeneralUtilities.read_text_from_file(csproj_file)))
111
+ else:
112
+ raise ValueError(
113
+ f"Version '{current_version}' does not match version-regex '{versiononlyregex}'")
114
+
115
+ @GeneralUtilities.check_arguments
116
+ def push_nuget_build_artifact(self, nupkg_file: str, registry_address: str, api_key: str, verbosity: int = 1):
117
+ nupkg_file_name = os.path.basename(nupkg_file)
118
+ nupkg_file_folder = os.path.dirname(nupkg_file)
119
+ self.run_program(
120
+ "dotnet", f"nuget push {nupkg_file_name} --force-english-output --source {registry_address} --api-key {api_key}", nupkg_file_folder, verbosity)
121
+
122
+ @GeneralUtilities.check_arguments
123
+ def dotnet_build(self, repository_folder: str, projectname: str, configuration: str):
124
+ self.run_program(
125
+ "dotnet", f"clean -c {configuration}", repository_folder)
126
+ self.run_program(
127
+ "dotnet", f"build {projectname}/{projectname}.csproj -c {configuration}", repository_folder)
128
+
129
+ @GeneralUtilities.check_arguments
130
+ def find_file_by_extension(self, folder: str, extension: str):
131
+ result = [file for file in GeneralUtilities.get_direct_files_of_folder(folder) if file.endswith(f".{extension}")]
132
+ result_length = len(result)
133
+ if result_length == 0:
134
+ raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension}'.")
135
+ if result_length == 1:
136
+ return result[0]
137
+ else:
138
+ raise ValueError(f"Multiple values available in folder '{folder}' with extension '{extension}'.")
139
+
140
+ @GeneralUtilities.check_arguments
141
+ def commit_is_signed_by_key(self, repository_folder: str, revision_identifier: str, key: str) -> bool:
142
+ result = self.run_program(
143
+ "git", f"verify-commit {revision_identifier}", repository_folder, throw_exception_if_exitcode_is_not_zero=False)
144
+ if (result[0] != 0):
145
+ return False
146
+ if (not GeneralUtilities.contains_line(result[1].splitlines(), f"gpg\\:\\ using\\ [A-Za-z0-9]+\\ key\\ [A-Za-z0-9]+{key}")):
147
+ # TODO check whether this works on machines where gpg is installed in another langauge than english
148
+ return False
149
+ if (not GeneralUtilities.contains_line(result[1].splitlines(), "gpg\\:\\ Good\\ signature\\ from")):
150
+ # TODO check whether this works on machines where gpg is installed in another langauge than english
151
+ return False
152
+ return True
153
+
154
+ @GeneralUtilities.check_arguments
155
+ def get_parent_commit_ids_of_commit(self, repository_folder: str, commit_id: str) -> str:
156
+ return self.run_program("git", f'log --pretty=%P -n 1 "{commit_id}"', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].replace("\r", "").replace("\n", "").split(" ")
157
+
158
+ @GeneralUtilities.check_arguments
159
+ def get_all_authors_and_committers_of_repository(self, repository_folder: str, subfolder: str = None, verbosity: int = 1) -> list[tuple[str, str]]:
160
+ space_character = "_"
161
+ if subfolder is None:
162
+ subfolder_argument = ""
163
+ else:
164
+ subfolder_argument = f" -- {subfolder}"
165
+ log_result = self.run_program(
166
+ "git", f'log --pretty=%aN{space_character}%aE%n%cN{space_character}%cE HEAD{subfolder_argument}', repository_folder, verbosity=0)
167
+ plain_content: list[str] = list(
168
+ set([line for line in log_result[1].split("\n") if len(line) > 0]))
169
+ result: list[tuple[str, str]] = []
170
+ for item in plain_content:
171
+ if len(re.findall(space_character, item)) == 1:
172
+ splitted = item.split(space_character)
173
+ result.append((splitted[0], splitted[1]))
174
+ else:
175
+ raise ValueError(f'Unexpected author: "{item}"')
176
+ return result
177
+
178
+ @GeneralUtilities.check_arguments
179
+ def get_commit_ids_between_dates(self, repository_folder: str, since: datetime, until: datetime, ignore_commits_which_are_not_in_history_of_head: bool = True) -> None:
180
+ since_as_string = self.__datetime_to_string_for_git(since)
181
+ until_as_string = self.__datetime_to_string_for_git(until)
182
+ result = filter(lambda line: not GeneralUtilities.string_is_none_or_whitespace(line), self.run_program(
183
+ "git", f'log --since "{since_as_string}" --until "{until_as_string}" --pretty=format:"%H" --no-patch', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].split("\n").replace("\r", ""))
184
+ if ignore_commits_which_are_not_in_history_of_head:
185
+ result = [commit_id for commit_id in result if self.git_commit_is_ancestor(
186
+ repository_folder, commit_id)]
187
+ return result
188
+
189
+ @GeneralUtilities.check_arguments
190
+ def __datetime_to_string_for_git(self, datetime_object: datetime) -> str:
191
+ return datetime_object.strftime('%Y-%m-%d %H:%M:%S')
192
+
193
+ @GeneralUtilities.check_arguments
194
+ def git_commit_is_ancestor(self, repository_folder: str, ancestor: str, descendant: str = "HEAD") -> bool:
195
+ exit_code = self.run_program_argsasarray(
196
+ "git", ["merge-base", "--is-ancestor", ancestor, descendant], repository_folder, throw_exception_if_exitcode_is_not_zero=False)[0]
197
+ if exit_code == 0:
198
+ return True
199
+ elif exit_code == 1:
200
+ return False
201
+ else:
202
+ raise ValueError(f"Can not calculate if {ancestor} is an ancestor of {descendant} in repository {repository_folder}.")
203
+
204
+ @GeneralUtilities.check_arguments
205
+ def __git_changes_helper(self, repository_folder: str, arguments_as_array: list[str]) -> bool:
206
+ lines = GeneralUtilities.string_to_lines(self.run_program_argsasarray(
207
+ "git", arguments_as_array, repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
208
+ for line in lines:
209
+ if GeneralUtilities.string_has_content(line):
210
+ return True
211
+ return False
212
+
213
+ @GeneralUtilities.check_arguments
214
+ def git_repository_has_new_untracked_files(self, repositoryFolder: str):
215
+ return self.__git_changes_helper(repositoryFolder, ["ls-files", "--exclude-standard", "--others"])
216
+
217
+ @GeneralUtilities.check_arguments
218
+ def git_repository_has_unstaged_changes_of_tracked_files(self, repositoryFolder: str):
219
+ return self.__git_changes_helper(repositoryFolder, ["diff"])
220
+
221
+ @GeneralUtilities.check_arguments
222
+ def git_repository_has_staged_changes(self, repositoryFolder: str):
223
+ return self.__git_changes_helper(repositoryFolder, ["diff", "--cached"])
224
+
225
+ @GeneralUtilities.check_arguments
226
+ def git_repository_has_uncommitted_changes(self, repositoryFolder: str) -> bool:
227
+ if (self.git_repository_has_unstaged_changes(repositoryFolder)):
228
+ return True
229
+ if (self.git_repository_has_staged_changes(repositoryFolder)):
230
+ return True
231
+ return False
232
+
233
+ @GeneralUtilities.check_arguments
234
+ def git_repository_has_unstaged_changes(self, repository_folder: str) -> bool:
235
+ if (self.git_repository_has_unstaged_changes_of_tracked_files(repository_folder)):
236
+ return True
237
+ if (self.git_repository_has_new_untracked_files(repository_folder)):
238
+ return True
239
+ return False
240
+
241
+ @GeneralUtilities.check_arguments
242
+ def git_get_commit_id(self, repository_folder: str, commit: str = "HEAD") -> str:
243
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["rev-parse", "--verify", commit], repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
244
+ return result[1].replace('\n', '')
245
+
246
+ @GeneralUtilities.check_arguments
247
+ def git_get_commit_date(self, repository_folder: str, commit: str = "HEAD") -> datetime:
248
+ result: tuple[int, str, str, int] = self.run_program_argsasarray(
249
+ "git", ["show", "-s", "--format=%ci", commit], repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
250
+ date_as_string = result[1].replace('\n', '')
251
+ result = datetime.strptime(date_as_string, '%Y-%m-%d %H:%M:%S %z')
252
+ return result
253
+
254
+ @GeneralUtilities.check_arguments
255
+ def git_fetch(self, folder: str, remotename: str = "--all") -> None:
256
+ self.run_program_argsasarray("git", ["fetch", remotename, "--tags", "--prune"], folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
257
+
258
+ @GeneralUtilities.check_arguments
259
+ def git_fetch_in_bare_repository(self, folder: str, remotename, localbranch: str, remotebranch: str) -> None:
260
+ self.run_program_argsasarray("git", ["fetch", remotename, f"{remotebranch}:{localbranch}"], folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
261
+
262
+ @GeneralUtilities.check_arguments
263
+ def git_remove_branch(self, folder: str, branchname: str) -> None:
264
+ self.run_program("git", f"branch -D {branchname}", folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
265
+
266
+ @GeneralUtilities.check_arguments
267
+ def git_push(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: int = 0) -> None:
268
+ argument = ["push", "--recurse-submodules=on-demand", remotename, f"{localbranchname}:{remotebranchname}"]
269
+ if (forcepush):
270
+ argument.append("--force")
271
+ if (pushalltags):
272
+ argument.append("--tags")
273
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=verbosity, print_errors_as_information=True)
274
+ return result[1].replace('\r', '').replace('\n', '')
275
+
276
+ @GeneralUtilities.check_arguments
277
+ def git_pull(self, folder: str, remote: str, localbranchname: str, remotebranchname: str) -> None:
278
+ self.run_program("git", f"pull {remote} {remotebranchname}:{localbranchname}", folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
279
+
280
+ @GeneralUtilities.check_arguments
281
+ def git_list_remote_branches(self, folder: str, remote: str, fetch: bool) -> list[str]:
282
+ if fetch:
283
+ self.git_fetch(folder, remote)
284
+ run_program_result = self.run_program("git", f"branch -rl {remote}/*", folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
285
+ output = GeneralUtilities.string_to_lines(run_program_result[1])
286
+ result = list[str]()
287
+ for item in output:
288
+ striped_item = item.strip()
289
+ if GeneralUtilities.string_has_content(striped_item):
290
+ branch: str = None
291
+ if " " in striped_item:
292
+ branch = striped_item.split(" ")[0]
293
+ else:
294
+ branch = striped_item
295
+ branchname = branch[len(remote)+1:]
296
+ if branchname != "HEAD":
297
+ result.append(branchname)
298
+ return result
299
+
300
+ @GeneralUtilities.check_arguments
301
+ def git_clone(self, clone_target_folder: str, remote_repository_path: str, include_submodules: bool = True, mirror: bool = False) -> None:
302
+ if (os.path.isdir(clone_target_folder)):
303
+ pass # TODO throw error
304
+ else:
305
+ args = ["clone", remote_repository_path, clone_target_folder]
306
+ if include_submodules:
307
+ args.append("--recurse-submodules")
308
+ args.append("--remote-submodules")
309
+ if mirror:
310
+ args.append("--mirror")
311
+ self.run_program_argsasarray("git", args, os.getcwd(), throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
312
+
313
+ @GeneralUtilities.check_arguments
314
+ def git_get_all_remote_names(self, directory: str) -> list[str]:
315
+ result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
316
+ return result
317
+
318
+ @GeneralUtilities.check_arguments
319
+ def git_get_remote_url(self, directory: str, remote_name: str) -> str:
320
+ result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote", "get-url", remote_name], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
321
+ return result[0].replace('\n', '')
322
+
323
+ @GeneralUtilities.check_arguments
324
+ def repository_has_remote_with_specific_name(self, directory: str, remote_name: str) -> bool:
325
+ return remote_name in self.git_get_all_remote_names(directory)
326
+
327
+ @GeneralUtilities.check_arguments
328
+ def git_add_or_set_remote_address(self, directory: str, remote_name: str, remote_address: str) -> None:
329
+ if (self.repository_has_remote_with_specific_name(directory, remote_name)):
330
+ self.run_program_argsasarray("git", ['remote', 'set-url', 'remote_name', remote_address], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
331
+ else:
332
+ self.run_program_argsasarray("git", ['remote', 'add', remote_name, remote_address], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
333
+
334
+ @GeneralUtilities.check_arguments
335
+ def git_stage_all_changes(self, directory: str) -> None:
336
+ self.run_program_argsasarray("git", ["add", "-A"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
337
+
338
+ @GeneralUtilities.check_arguments
339
+ def git_unstage_all_changes(self, directory: str) -> None:
340
+ self.run_program_argsasarray("git", ["reset"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
341
+
342
+ @GeneralUtilities.check_arguments
343
+ def git_stage_file(self, directory: str, file: str) -> None:
344
+ self.run_program_argsasarray("git", ['stage', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
345
+
346
+ @GeneralUtilities.check_arguments
347
+ def git_unstage_file(self, directory: str, file: str) -> None:
348
+ self.run_program_argsasarray("git", ['reset', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
349
+
350
+ @GeneralUtilities.check_arguments
351
+ def git_discard_unstaged_changes_of_file(self, directory: str, file: str) -> None:
352
+ """Caution: This method works really only for 'changed' files yet. So this method does not work properly for new or renamed files."""
353
+ self.run_program_argsasarray("git", ['checkout', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
354
+
355
+ @GeneralUtilities.check_arguments
356
+ def git_discard_all_unstaged_changes(self, directory: str) -> None:
357
+ """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
358
+ self.run_program_argsasarray("git", ['clean', '-df'], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
359
+ self.run_program_argsasarray("git", ['checkout', '.'], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
360
+
361
+ @GeneralUtilities.check_arguments
362
+ def git_commit(self, directory: str, message: str, author_name: str = None, author_email: str = None, stage_all_changes: bool = True,
363
+ no_changes_behavior: int = 0) -> str:
364
+ # no_changes_behavior=0 => No commit
365
+ # no_changes_behavior=1 => Commit anyway
366
+ # no_changes_behavior=2 => Exception
367
+ author_name = GeneralUtilities.str_none_safe(author_name).strip()
368
+ author_email = GeneralUtilities.str_none_safe(author_email).strip()
369
+ argument = ['commit', '--quiet', '--allow-empty', '--message', message]
370
+ if (GeneralUtilities.string_has_content(author_name)):
371
+ argument.append(f'--author="{author_name} <{author_email}>"')
372
+ git_repository_has_uncommitted_changes = self.git_repository_has_uncommitted_changes(directory)
373
+
374
+ if git_repository_has_uncommitted_changes:
375
+ do_commit = True
376
+ if stage_all_changes:
377
+ self.git_stage_all_changes(directory)
378
+ else:
379
+ if no_changes_behavior == 0:
380
+ GeneralUtilities.write_message_to_stdout(f"Commit '{message}' will not be done because there are no changes to commit in repository '{directory}'")
381
+ do_commit = False
382
+ elif no_changes_behavior == 1:
383
+ GeneralUtilities.write_message_to_stdout(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will be done anyway.")
384
+ do_commit = True
385
+ elif no_changes_behavior == 2:
386
+ raise RuntimeError(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will not be done.")
387
+ else:
388
+ raise ValueError(f"Unknown value for no_changes_behavior: {GeneralUtilities.str_none_safe(no_changes_behavior)}")
389
+
390
+ if do_commit:
391
+ GeneralUtilities.write_message_to_stdout(f"Commit changes in '{directory}'")
392
+ self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
393
+
394
+ return self.git_get_commit_id(directory)
395
+
396
+ @GeneralUtilities.check_arguments
397
+ def git_create_tag(self, directory: str, target_for_tag: str, tag: str, sign: bool = False, message: str = None) -> None:
398
+ argument = ["tag", tag, target_for_tag]
399
+ if sign:
400
+ if message is None:
401
+ message = f"Created {target_for_tag}"
402
+ argument.extend(["-s", '-m', message])
403
+ self.run_program_argsasarray(
404
+ "git", argument, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
405
+
406
+ @GeneralUtilities.check_arguments
407
+ def git_delete_tag(self, directory: str, tag: str) -> None:
408
+ self.run_program_argsasarray("git", ["tag", "--delete", tag], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
409
+
410
+ @GeneralUtilities.check_arguments
411
+ def git_checkout(self, directory: str, branch: str) -> None:
412
+ self.run_program_argsasarray("git", ["checkout", branch], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
413
+ self.run_program_argsasarray("git", ["submodule", "update", "--recursive"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
414
+
415
+ @GeneralUtilities.check_arguments
416
+ def git_merge_abort(self, directory: str) -> None:
417
+ self.run_program_argsasarray("git", ["merge", "--abort"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
418
+
419
+ @GeneralUtilities.check_arguments
420
+ def git_merge(self, directory: str, sourcebranch: str, targetbranch: str, fastforward: bool = True, commit: bool = True, commit_message: str = None) -> str:
421
+ self.git_checkout(directory, targetbranch)
422
+ args = ["merge"]
423
+ if not commit:
424
+ args.append("--no-commit")
425
+ if not fastforward:
426
+ args.append("--no-ff")
427
+ if commit_message is not None:
428
+ args.append("-m")
429
+ args.append(commit_message)
430
+ args.append(sourcebranch)
431
+ self.run_program_argsasarray(
432
+ "git", args, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
433
+ return self.git_get_commit_id(directory)
434
+
435
+ @GeneralUtilities.check_arguments
436
+ def git_undo_all_changes(self, directory: str) -> None:
437
+ """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
438
+ self.git_unstage_all_changes(directory)
439
+ self.git_discard_all_unstaged_changes(directory)
440
+
441
+ @GeneralUtilities.check_arguments
442
+ def git_fetch_or_clone_all_in_directory(self, source_directory: str, target_directory: str) -> None:
443
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(source_directory):
444
+ foldername = os.path.basename(subfolder)
445
+ if self.is_git_repository(subfolder):
446
+ source_repository = subfolder
447
+ target_repository = os.path.join(target_directory, foldername)
448
+ if os.path.isdir(target_directory):
449
+ # fetch
450
+ self.git_fetch(target_directory)
451
+ else:
452
+ # clone
453
+ self.git_clone(target_repository, source_repository, include_submodules=True, mirror=True)
454
+
455
+ def get_git_submodules(self, folder: str) -> list[str]:
456
+ e = self.run_program("git", "submodule status", folder)
457
+ result = []
458
+ for submodule_line in GeneralUtilities.string_to_lines(e[1], False, True):
459
+ result.append(submodule_line.split(' ')[1])
460
+ return result
461
+
462
+ @GeneralUtilities.check_arguments
463
+ def is_git_repository(self, folder: str) -> bool:
464
+ combined = os.path.join(folder, ".git")
465
+ # TODO consider check for bare-repositories
466
+ return os.path.isdir(combined) or os.path.isfile(combined)
467
+
468
+ @GeneralUtilities.check_arguments
469
+ def file_is_git_ignored(self, file_in_repository: str, repositorybasefolder: str) -> None:
470
+ exit_code = self.run_program_argsasarray("git", ['check-ignore', file_in_repository], repositorybasefolder, throw_exception_if_exitcode_is_not_zero=False, verbosity=0)[0]
471
+ if (exit_code == 0):
472
+ return True
473
+ if (exit_code == 1):
474
+ return False
475
+ raise ValueError(
476
+ f"Unable to calculate whether '{file_in_repository}' in repository '{repositorybasefolder}' is ignored due to git-exitcode {exit_code}.")
477
+
478
+ @GeneralUtilities.check_arguments
479
+ def git_discard_all_changes(self, repository: str) -> None:
480
+ self.run_program_argsasarray("git", ["reset", "HEAD", "."], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
481
+ self.run_program_argsasarray("git", ["checkout", "."], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
482
+
483
+ @GeneralUtilities.check_arguments
484
+ def git_get_current_branch_name(self, repository: str) -> str:
485
+ result = self.run_program_argsasarray("git", ["rev-parse", "--abbrev-ref", "HEAD"], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
486
+ return result[1].replace("\r", "").replace("\n", "")
487
+
488
+ @GeneralUtilities.check_arguments
489
+ def git_get_commitid_of_tag(self, repository: str, tag: str) -> str:
490
+ stdout = self.run_program_argsasarray("git", ["rev-list", "-n", "1", tag], repository, verbosity=0)
491
+ result = stdout[1].replace("\r", "").replace("\n", "")
492
+ return result
493
+
494
+ @GeneralUtilities.check_arguments
495
+ def git_get_tags(self, repository: str) -> list[str]:
496
+ tags = [line.replace("\r", "") for line in self.run_program_argsasarray(
497
+ "git", ["tag"], repository)[1].split("\n") if len(line) > 0]
498
+ return tags
499
+
500
+ @GeneralUtilities.check_arguments
501
+ def git_move_tags_to_another_branch(self, repository: str, tag_source_branch: str, tag_target_branch: str, sign: bool = False, message: str = None) -> None:
502
+ tags = self.git_get_tags(repository)
503
+ tags_count = len(tags)
504
+ counter = 0
505
+ for tag in tags:
506
+ counter = counter+1
507
+ GeneralUtilities.write_message_to_stdout(f"Process tag {counter}/{tags_count}.")
508
+ # tag is on source-branch
509
+ if self.git_commit_is_ancestor(repository, tag, tag_source_branch):
510
+ commit_id_old = self.git_get_commitid_of_tag(repository, tag)
511
+ commit_date: datetime = self.git_get_commit_date(repository, commit_id_old)
512
+ date_as_string = self.__datetime_to_string_for_git(commit_date)
513
+ search_commit_result = self.run_program_argsasarray("git", ["log", f'--after="{date_as_string}"', f'--before="{date_as_string}"', "--pretty=format:%H", tag_target_branch], repository, throw_exception_if_exitcode_is_not_zero=False)
514
+ if search_commit_result[0] != 0 or not GeneralUtilities.string_has_nonwhitespace_content(search_commit_result[1]):
515
+ raise ValueError(f"Can not calculate corresponding commit for tag '{tag}'.")
516
+ commit_id_new = search_commit_result[1]
517
+ self.git_delete_tag(repository, tag)
518
+ self.git_create_tag(repository, commit_id_new, tag, sign, message)
519
+
520
+ @GeneralUtilities.check_arguments
521
+ def get_current_git_branch_has_tag(self, repository_folder: str) -> bool:
522
+ result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder, verbosity=0, throw_exception_if_exitcode_is_not_zero=False)
523
+ return result[0] == 0
524
+
525
+ @GeneralUtilities.check_arguments
526
+ def get_latest_git_tag(self, repository_folder: str) -> str:
527
+ result = self.run_program_argsasarray(
528
+ "git", ["describe", "--tags", "--abbrev=0"], repository_folder, verbosity=0)
529
+ result = result[1].replace("\r", "").replace("\n", "")
530
+ return result
531
+
532
+ @GeneralUtilities.check_arguments
533
+ def get_staged_or_committed_git_ignored_files(self, repository_folder: str) -> list[str]:
534
+ tresult = self.run_program_argsasarray("git", ["ls-files", "-i", "-c", "--exclude-standard"], repository_folder, verbosity=0)
535
+ tresult = tresult[1].replace("\r", "")
536
+ result = [line for line in tresult.split("\n") if len(line) > 0]
537
+ return result
538
+
539
+ @GeneralUtilities.check_arguments
540
+ def git_repository_has_commits(self, repository_folder: str) -> bool:
541
+ return self.run_program_argsasarray("git", ["rev-parse", "--verify", "HEAD"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)[0] == 0
542
+
543
+ @GeneralUtilities.check_arguments
544
+ def export_filemetadata(self, folder: str, target_file: str, encoding: str = "utf-8", filter_function=None) -> None:
545
+ folder = GeneralUtilities.resolve_relative_path_from_current_working_directory(folder)
546
+ lines = list()
547
+ path_prefix = len(folder)+1
548
+ items = dict()
549
+ for item in GeneralUtilities.get_all_folders_of_folder(folder):
550
+ items[item] = "d"
551
+ for item in GeneralUtilities.get_all_files_of_folder(folder):
552
+ items[item] = "f"
553
+ for file_or_folder, item_type in items.items():
554
+ truncated_file = file_or_folder[path_prefix:]
555
+ if (filter_function is None or filter_function(folder, truncated_file)):
556
+ owner_and_permisssion = self.get_file_owner_and_file_permission(
557
+ file_or_folder)
558
+ user = owner_and_permisssion[0]
559
+ permissions = owner_and_permisssion[1]
560
+ lines.append(f"{truncated_file};{item_type};{user};{permissions}")
561
+ lines = sorted(lines, key=str.casefold)
562
+ with open(target_file, "w", encoding=encoding) as file_object:
563
+ file_object.write("\n".join(lines))
564
+
565
+ def escape_git_repositories_in_folder(self, folder: str) -> dict[str, str]:
566
+ return self.__escape_git_repositories_in_folder_internal(folder, dict[str, str]())
567
+
568
+ def __escape_git_repositories_in_folder_internal(self, folder: str, renamed_items: dict[str, str]) -> dict[str, str]:
569
+ for file in GeneralUtilities.get_direct_files_of_folder(folder):
570
+ filename = os.path.basename(file)
571
+ if ".git" in filename:
572
+ new_name = filename.replace(".git", ".gitx")
573
+ target = os.path.join(folder, new_name)
574
+ os.rename(file, target)
575
+ renamed_items[target] = file
576
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
577
+ foldername = os.path.basename(subfolder)
578
+ if ".git" in foldername:
579
+ new_name = foldername.replace(".git", ".gitx")
580
+ subfolder2 = os.path.join(
581
+ str(Path(subfolder).parent), new_name)
582
+ os.rename(subfolder, subfolder2)
583
+ renamed_items[subfolder2] = subfolder
584
+ else:
585
+ subfolder2 = subfolder
586
+ self.__escape_git_repositories_in_folder_internal(subfolder2, renamed_items)
587
+ return renamed_items
588
+
589
+ def deescape_git_repositories_in_folder(self, renamed_items: dict[str, str]):
590
+ for renamed_item, original_name in renamed_items.items():
591
+ os.rename(renamed_item, original_name)
592
+
593
+ def __sort_fmd(self, line: str):
594
+ splitted: list = line.split(";")
595
+ filetype: str = splitted[1]
596
+ if filetype == "d":
597
+ return -1
598
+ if filetype == "f":
599
+ return 1
600
+ return 0
601
+
602
+ @GeneralUtilities.check_arguments
603
+ def restore_filemetadata(self, folder: str, source_file: str, strict=False, encoding: str = "utf-8", create_folder_is_not_exist: bool = True) -> None:
604
+ lines = GeneralUtilities.read_lines_from_file(source_file, encoding)
605
+ lines.sort(key=self.__sort_fmd)
606
+ for line in lines:
607
+ splitted: list = line.split(";")
608
+ full_path_of_file_or_folder: str = os.path.join(folder, splitted[0])
609
+ filetype: str = splitted[1]
610
+ user: str = splitted[2]
611
+ permissions: str = splitted[3]
612
+ if filetype == "d" and create_folder_is_not_exist and not os.path.isdir(full_path_of_file_or_folder):
613
+ GeneralUtilities.ensure_directory_exists(full_path_of_file_or_folder)
614
+ if (filetype == "f" and os.path.isfile(full_path_of_file_or_folder)) or (filetype == "d" and os.path.isdir(full_path_of_file_or_folder)):
615
+ self.set_owner(full_path_of_file_or_folder, user, os.name != 'nt')
616
+ self.set_permission(full_path_of_file_or_folder, permissions)
617
+ else:
618
+ if strict:
619
+ if filetype == "f":
620
+ filetype_full = "File"
621
+ elif filetype == "d":
622
+ filetype_full = "Directory"
623
+ else:
624
+ raise ValueError(f"Unknown filetype: {GeneralUtilities.str_none_safe(filetype)}")
625
+ raise ValueError(f"{filetype_full} '{full_path_of_file_or_folder}' does not exist")
626
+
627
+ @GeneralUtilities.check_arguments
628
+ def __calculate_lengh_in_seconds(self, filename: str, folder: str) -> float:
629
+ argument = ['-v', 'error', '-show_entries', 'format=duration',
630
+ '-of', 'default=noprint_wrappers=1:nokey=1', filename]
631
+ result = self.run_program_argsasarray("ffprobe", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
632
+ return float(result[1].replace('\n', ''))
633
+
634
+ @GeneralUtilities.check_arguments
635
+ def __create_thumbnails(self, filename: str, fps: str, folder: str, tempname_for_thumbnails: str) -> None:
636
+ argument = ['-i', filename, '-r', str(fps), '-vf', 'scale=-1:120',
637
+ '-vcodec', 'png', f'{tempname_for_thumbnails}-%002d.png']
638
+ self.run_program_argsasarray("ffmpeg", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
639
+
640
+ @GeneralUtilities.check_arguments
641
+ def __create_thumbnail(self, outputfilename: str, folder: str, length_in_seconds: float, tempname_for_thumbnails: str, amount_of_images: int) -> None:
642
+ duration = timedelta(seconds=length_in_seconds)
643
+ info = GeneralUtilities.timedelta_to_simple_string(duration)
644
+ next_square_number = str(int(math.sqrt(GeneralUtilities.get_next_square_number(amount_of_images))))
645
+ argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{next_square_number}x{next_square_number}',
646
+ f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
647
+ self.run_program_argsasarray(
648
+ "montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
649
+
650
+ @GeneralUtilities.check_arguments
651
+ def roundup(self, x: float, places: int) -> int:
652
+ d = 10 ** places
653
+ if x < 0:
654
+ return math.floor(x * d) / d
655
+ else:
656
+ return math.ceil(x * d) / d
657
+
658
+ @GeneralUtilities.check_arguments
659
+ def generate_thumbnail(self, file: str, frames_per_second: str, tempname_for_thumbnails: str = None) -> None:
660
+ if tempname_for_thumbnails is None:
661
+ tempname_for_thumbnails = "t"+str(uuid.uuid4())
662
+
663
+ file = GeneralUtilities.resolve_relative_path_from_current_working_directory(file)
664
+ filename = os.path.basename(file)
665
+ folder = os.path.dirname(file)
666
+ filename_without_extension = Path(file).stem
667
+
668
+ try:
669
+ length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
670
+ if (frames_per_second.endswith("fps")):
671
+ # frames per second, example: frames_per_second="20fps" => 20 frames per second
672
+ x = self.roundup(float(frames_per_second[:-3]), 2)
673
+ frames_per_secondx = str(x)
674
+ amounf_of_previewframes = int(math.floor(length_in_seconds*x))
675
+ else:
676
+ # concrete amount of frame, examples: frames_per_second="16" => 16 frames for entire video
677
+ amounf_of_previewframes = int(float(frames_per_second))
678
+ # self.roundup((amounf_of_previewframes-2)/length_in_seconds, 2)
679
+ frames_per_secondx = f"{amounf_of_previewframes-2}/{length_in_seconds}"
680
+ self.__create_thumbnails(filename, frames_per_secondx, folder, tempname_for_thumbnails)
681
+ self.__create_thumbnail(filename_without_extension, folder, length_in_seconds, tempname_for_thumbnails, amounf_of_previewframes)
682
+ finally:
683
+ for thumbnail_to_delete in Path(folder).rglob(tempname_for_thumbnails+"-*"):
684
+ file = str(thumbnail_to_delete)
685
+ os.remove(file)
686
+
687
+ @GeneralUtilities.check_arguments
688
+ def extract_pdf_pages(self, file: str, from_page: int, to_page: int, outputfile: str) -> None:
689
+ pdf_reader = PyPDF2.PdfReader(file)
690
+ pdf_writer = PyPDF2.PdfWriter()
691
+ start = from_page
692
+ end = to_page
693
+ while start <= end:
694
+ pdf_writer.add_page(pdf_reader.pages[start-1])
695
+ start += 1
696
+ with open(outputfile, 'wb') as out:
697
+ pdf_writer.write(out)
698
+
699
+ @GeneralUtilities.check_arguments
700
+ def merge_pdf_files(self, files: list[str], outputfile: str) -> None:
701
+ # TODO add wildcard-option
702
+ pdfFileMerger = PyPDF2.PdfFileMerger()
703
+ for file in files:
704
+ pdfFileMerger.append(file.strip())
705
+ pdfFileMerger.write(outputfile)
706
+ pdfFileMerger.close()
707
+
708
+ @GeneralUtilities.check_arguments
709
+ def pdf_to_image(self, file: str, outputfilename_without_extension: str) -> None:
710
+ raise ValueError("Function currently not available")
711
+ # PyMuPDF can be used for that but sometimes it throws
712
+ # "ImportError: DLL load failed while importing _fitz: Das angegebene Modul wurde nicht gefunden."
713
+
714
+ # doc = None # fitz.open(file)
715
+ # for i, page in enumerate(doc):
716
+ # pix = page.get_pixmap()
717
+ # img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
718
+ # img.save(f"{outputfilename_without_extension}_{i}.png", "PNG")
719
+
720
+ @GeneralUtilities.check_arguments
721
+ def show_missing_files(self, folderA: str, folderB: str):
722
+ for file in GeneralUtilities.get_missing_files(folderA, folderB):
723
+ GeneralUtilities.write_message_to_stdout(file)
724
+
725
+ @GeneralUtilities.check_arguments
726
+ def SCCreateEmptyFileWithSpecificSize(self, name: str, size_string: str) -> int:
727
+ if size_string.isdigit():
728
+ size = int(size_string)
729
+ else:
730
+ if len(size_string) >= 3:
731
+ if (size_string.endswith("kb")):
732
+ size = int(size_string[:-2]) * pow(10, 3)
733
+ elif (size_string.endswith("mb")):
734
+ size = int(size_string[:-2]) * pow(10, 6)
735
+ elif (size_string.endswith("gb")):
736
+ size = int(size_string[:-2]) * pow(10, 9)
737
+ elif (size_string.endswith("kib")):
738
+ size = int(size_string[:-3]) * pow(2, 10)
739
+ elif (size_string.endswith("mib")):
740
+ size = int(size_string[:-3]) * pow(2, 20)
741
+ elif (size_string.endswith("gib")):
742
+ size = int(size_string[:-3]) * pow(2, 30)
743
+ else:
744
+ GeneralUtilities.write_message_to_stderr("Wrong format")
745
+ return 1
746
+ else:
747
+ GeneralUtilities.write_message_to_stderr("Wrong format")
748
+ return 1
749
+ with open(name, "wb") as f:
750
+ f.seek(size-1)
751
+ f.write(b"\0")
752
+ return 0
753
+
754
+ @GeneralUtilities.check_arguments
755
+ def SCCreateHashOfAllFiles(self, folder: str) -> None:
756
+ for file in GeneralUtilities.absolute_file_paths(folder):
757
+ with open(file+".sha256", "w+", encoding="utf-8") as f:
758
+ f.write(GeneralUtilities.get_sha256_of_file(file))
759
+
760
+ @GeneralUtilities.check_arguments
761
+ def SCCreateSimpleMergeWithoutRelease(self, repository: str, sourcebranch: str, targetbranch: str, remotename: str, remove_source_branch: bool) -> None:
762
+ commitid = self.git_merge(repository, sourcebranch, targetbranch, False, True)
763
+ self.git_merge(repository, targetbranch, sourcebranch, True, True)
764
+ created_version = self.get_semver_version_from_gitversion(repository)
765
+ self.git_create_tag(repository, commitid, f"v{created_version}", True)
766
+ self.git_push(repository, remotename, targetbranch, targetbranch, False, True)
767
+ if (GeneralUtilities.string_has_nonwhitespace_content(remotename)):
768
+ self.git_push(repository, remotename, sourcebranch, sourcebranch, False, True)
769
+ if (remove_source_branch):
770
+ self.git_remove_branch(repository, sourcebranch)
771
+
772
+ @GeneralUtilities.check_arguments
773
+ def sc_organize_lines_in_file(self, file: str, encoding: str, sort: bool = False, remove_duplicated_lines: bool = False, ignore_first_line: bool = False, remove_empty_lines: bool = True, ignored_start_character: list = list()) -> int:
774
+ if os.path.isfile(file):
775
+
776
+ # read file
777
+ lines = GeneralUtilities.read_lines_from_file(file, encoding)
778
+ if (len(lines) == 0):
779
+ return 0
780
+
781
+ # store first line if desiredpopd
782
+
783
+ if (ignore_first_line):
784
+ first_line = lines.pop(0)
785
+
786
+ # remove empty lines if desired
787
+ if remove_empty_lines:
788
+ temp = lines
789
+ lines = []
790
+ for line in temp:
791
+ if (not (GeneralUtilities.string_is_none_or_whitespace(line))):
792
+ lines.append(line)
793
+
794
+ # remove duplicated lines if desired
795
+ if remove_duplicated_lines:
796
+ lines = GeneralUtilities.remove_duplicates(lines)
797
+
798
+ # sort lines if desired
799
+ if sort:
800
+ lines = sorted(lines, key=lambda singleline: self.__adapt_line_for_sorting(singleline, ignored_start_character))
801
+
802
+ # reinsert first line
803
+ if ignore_first_line:
804
+ lines.insert(0, first_line)
805
+
806
+ # write result to file
807
+ GeneralUtilities.write_lines_to_file(file, lines, encoding)
808
+
809
+ return 0
810
+ else:
811
+ GeneralUtilities.write_message_to_stdout(f"File '{file}' does not exist")
812
+ return 1
813
+
814
+ @GeneralUtilities.check_arguments
815
+ def __adapt_line_for_sorting(self, line: str, ignored_start_characters: list):
816
+ result = line.lower()
817
+ while len(result) > 0 and result[0] in ignored_start_characters:
818
+ result = result[1:]
819
+ return result
820
+
821
+ @GeneralUtilities.check_arguments
822
+ def SCGenerateSnkFiles(self, outputfolder, keysize=4096, amountofkeys=10) -> int:
823
+ GeneralUtilities.ensure_directory_exists(outputfolder)
824
+ for _ in range(amountofkeys):
825
+ file = os.path.join(outputfolder, str(uuid.uuid4())+".snk")
826
+ argument = f"-k {keysize} {file}"
827
+ self.run_program("sn", argument, outputfolder)
828
+
829
+ @GeneralUtilities.check_arguments
830
+ def __merge_files(self, sourcefile: str, targetfile: str) -> None:
831
+ with open(sourcefile, "rb") as f:
832
+ source_data = f.read()
833
+ with open(targetfile, "ab") as fout:
834
+ merge_separator = [0x0A]
835
+ fout.write(bytes(merge_separator))
836
+ fout.write(source_data)
837
+
838
+ @GeneralUtilities.check_arguments
839
+ def __process_file(self, file: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
840
+ new_filename = os.path.join(os.path.dirname(file), os.path.basename(file).replace(substringInFilename, newSubstringInFilename))
841
+ if file != new_filename:
842
+ if os.path.isfile(new_filename):
843
+ if filecmp.cmp(file, new_filename):
844
+ send2trash.send2trash(file)
845
+ else:
846
+ if conflictResolveMode == "ignore":
847
+ pass
848
+ elif conflictResolveMode == "preservenewest":
849
+ if (os.path.getmtime(file) - os.path.getmtime(new_filename) > 0):
850
+ send2trash.send2trash(file)
851
+ else:
852
+ send2trash.send2trash(new_filename)
853
+ os.rename(file, new_filename)
854
+ elif (conflictResolveMode == "merge"):
855
+ self.__merge_files(file, new_filename)
856
+ send2trash.send2trash(file)
857
+ else:
858
+ raise ValueError('Unknown conflict resolve mode')
859
+ else:
860
+ os.rename(file, new_filename)
861
+
862
+ @GeneralUtilities.check_arguments
863
+ def SCReplaceSubstringsInFilenames(self, folder: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
864
+ for file in GeneralUtilities.absolute_file_paths(folder):
865
+ self.__process_file(file, substringInFilename, newSubstringInFilename, conflictResolveMode)
866
+
867
+ @GeneralUtilities.check_arguments
868
+ def __check_file(self, file: str, searchstring: str) -> None:
869
+ bytes_ascii = bytes(searchstring, "ascii")
870
+ # often called "unicode-encoding"
871
+ bytes_utf16 = bytes(searchstring, "utf-16")
872
+ bytes_utf8 = bytes(searchstring, "utf-8")
873
+ with open(file, mode='rb') as file_object:
874
+ content = file_object.read()
875
+ if bytes_ascii in content:
876
+ GeneralUtilities.write_message_to_stdout(file)
877
+ elif bytes_utf16 in content:
878
+ GeneralUtilities.write_message_to_stdout(file)
879
+ elif bytes_utf8 in content:
880
+ GeneralUtilities.write_message_to_stdout(file)
881
+
882
+ @GeneralUtilities.check_arguments
883
+ def SCSearchInFiles(self, folder: str, searchstring: str) -> None:
884
+ for file in GeneralUtilities.absolute_file_paths(folder):
885
+ self.__check_file(file, searchstring)
886
+
887
+ @GeneralUtilities.check_arguments
888
+ def __print_qr_code_by_csv_line(self, displayname: str, website: str, emailaddress: str, key: str, period: str) -> None:
889
+ qrcode_content = f"otpauth://totp/{website}:{emailaddress}?secret={key}&issuer={displayname}&period={period}"
890
+ GeneralUtilities.write_message_to_stdout(
891
+ f"{displayname} ({emailaddress}):")
892
+ GeneralUtilities.write_message_to_stdout(qrcode_content)
893
+ qr = qrcode.QRCode()
894
+ qr.add_data(qrcode_content)
895
+ f = io.StringIO()
896
+ qr.print_ascii(out=f)
897
+ f.seek(0)
898
+ GeneralUtilities.write_message_to_stdout(f.read())
899
+
900
+ @GeneralUtilities.check_arguments
901
+ def SCShow2FAAsQRCode(self, csvfile: str) -> None:
902
+ separator_line = "--------------------------------------------------------"
903
+ lines = GeneralUtilities.read_csv_file(csvfile, True)
904
+ lines.sort(key=lambda items: ''.join(items).lower())
905
+ for line in lines:
906
+ GeneralUtilities.write_message_to_stdout(separator_line)
907
+ self.__print_qr_code_by_csv_line(
908
+ line[0], line[1], line[2], line[3], line[4])
909
+ GeneralUtilities.write_message_to_stdout(separator_line)
910
+
911
+ @GeneralUtilities.check_arguments
912
+ def SCUploadFileToFileHost(self, file: str, host: str) -> int:
913
+ try:
914
+ GeneralUtilities.write_message_to_stdout(
915
+ self.upload_file_to_file_host(file, host))
916
+ return 0
917
+ except Exception as exception:
918
+ GeneralUtilities.write_exception_to_stderr_with_traceback(exception, traceback)
919
+ return 1
920
+
921
+ @GeneralUtilities.check_arguments
922
+ def SCFileIsAvailableOnFileHost(self, file: str) -> int:
923
+ try:
924
+ if self.file_is_available_on_file_host(file):
925
+ GeneralUtilities.write_message_to_stdout(f"'{file}' is available")
926
+ return 0
927
+ else:
928
+ GeneralUtilities.write_message_to_stdout(f"'{file}' is not available")
929
+ return 1
930
+ except Exception as exception:
931
+ GeneralUtilities.write_exception_to_stderr_with_traceback(exception, traceback)
932
+ return 2
933
+
934
+ @GeneralUtilities.check_arguments
935
+ def SCCalculateBitcoinBlockHash(self, block_version_number: str, previousblockhash: str, transactionsmerkleroot: str, timestamp: str, target: str, nonce: str) -> str:
936
+ # Example-values:
937
+ # block_version_number: "00000020"
938
+ # previousblockhash: "66720b99e07d284bd4fe67ff8c49a5db1dd8514fcdab61000000000000000000"
939
+ # transactionsmerkleroot: "7829844f4c3a41a537b3131ca992643eaa9d093b2383e4cdc060ad7dc5481187"
940
+ # timestamp: "51eb505a"
941
+ # target: "c1910018"
942
+ # nonce: "de19b302"
943
+ header = str(block_version_number + previousblockhash + transactionsmerkleroot + timestamp + target + nonce)
944
+ return binascii.hexlify(hashlib.sha256(hashlib.sha256(binascii.unhexlify(header)).digest()).digest()[::-1]).decode('utf-8')
945
+
946
+ @GeneralUtilities.check_arguments
947
+ def SCChangeHashOfProgram(self, inputfile: str) -> None:
948
+ valuetoappend = str(uuid.uuid4())
949
+
950
+ outputfile = inputfile + '.modified'
951
+
952
+ shutil.copy2(inputfile, outputfile)
953
+ with open(outputfile, 'a', encoding="utf-8") as file:
954
+ # TODO use rcedit for .exe-files instead of appending valuetoappend ( https://github.com/electron/rcedit/ )
955
+ # background: you can retrieve the "original-filename" from the .exe-file like discussed here:
956
+ # https://security.stackexchange.com/questions/210843/ is-it-possible-to-change-original-filename-of-an-exe
957
+ # so removing the original filename with rcedit is probably a better way to make it more difficult to detect the programname.
958
+ # this would obviously also change the hashvalue of the program so appending a whitespace is not required anymore.
959
+ file.write(valuetoappend)
960
+
961
+ @GeneralUtilities.check_arguments
962
+ def __adjust_folder_name(self, folder: str) -> str:
963
+ result = os.path.dirname(folder).replace("\\", "/")
964
+ if result == "/":
965
+ return ""
966
+ else:
967
+ return result
968
+
969
+ @GeneralUtilities.check_arguments
970
+ def __create_iso(self, folder, iso_file) -> None:
971
+ created_directories = []
972
+ files_directory = "FILES"
973
+ iso = pycdlib.PyCdlib()
974
+ iso.new()
975
+ files_directory = files_directory.upper()
976
+ iso.add_directory("/" + files_directory)
977
+ created_directories.append("/" + files_directory)
978
+ for root, _, files in os.walk(folder):
979
+ for file in files:
980
+ full_path = os.path.join(root, file)
981
+ with (open(full_path, "rb").read()) as text_io_wrapper:
982
+ content = text_io_wrapper
983
+ path_in_iso = '/' + files_directory + \
984
+ self.__adjust_folder_name(
985
+ full_path[len(folder)::1]).upper()
986
+ if path_in_iso not in created_directories:
987
+ iso.add_directory(path_in_iso)
988
+ created_directories.append(path_in_iso)
989
+ iso.add_fp(BytesIO(content), len(content),
990
+ path_in_iso + '/' + file.upper() + ';1')
991
+ iso.write(iso_file)
992
+ iso.close()
993
+
994
+ @GeneralUtilities.check_arguments
995
+ def SCCreateISOFileWithObfuscatedFiles(self, inputfolder: str, outputfile: str, printtableheadline, createisofile, extensions) -> None:
996
+ if (os.path.isdir(inputfolder)):
997
+ namemappingfile = "name_map.csv"
998
+ files_directory = inputfolder
999
+ files_directory_obf = f"{files_directory}_Obfuscated"
1000
+ self.SCObfuscateFilesFolder(
1001
+ inputfolder, printtableheadline, namemappingfile, extensions)
1002
+ os.rename(namemappingfile, os.path.join(
1003
+ files_directory_obf, namemappingfile))
1004
+ if createisofile:
1005
+ self.__create_iso(files_directory_obf, outputfile)
1006
+ shutil.rmtree(files_directory_obf)
1007
+ else:
1008
+ raise ValueError(f"Directory not found: '{inputfolder}'")
1009
+
1010
+ @GeneralUtilities.check_arguments
1011
+ def SCFilenameObfuscator(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
1012
+ obfuscate_all_files = extensions == "*"
1013
+ if (obfuscate_all_files):
1014
+ obfuscate_file_extensions = None
1015
+ else:
1016
+ obfuscate_file_extensions = extensions.split(",")
1017
+ if (os.path.isdir(inputfolder)):
1018
+ printtableheadline = GeneralUtilities.string_to_boolean(
1019
+ printtableheadline)
1020
+ files = []
1021
+ if not os.path.isfile(namemappingfile):
1022
+ with open(namemappingfile, "a", encoding="utf-8"):
1023
+ pass
1024
+ if printtableheadline:
1025
+ GeneralUtilities.append_line_to_file(
1026
+ namemappingfile, "Original filename;new filename;SHA2-hash of file")
1027
+ for file in GeneralUtilities.absolute_file_paths(inputfolder):
1028
+ if os.path.isfile(os.path.join(inputfolder, file)):
1029
+ if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
1030
+ files.append(file)
1031
+ for file in files:
1032
+ hash_value = GeneralUtilities.get_sha256_of_file(file)
1033
+ extension = Path(file).suffix
1034
+ new_file_name_without_path = str(uuid.uuid4())[0:8] + extension
1035
+ new_file_name = os.path.join(
1036
+ os.path.dirname(file), new_file_name_without_path)
1037
+ os.rename(file, new_file_name)
1038
+ GeneralUtilities.append_line_to_file(namemappingfile, os.path.basename(
1039
+ file) + ";" + new_file_name_without_path + ";" + hash_value)
1040
+ else:
1041
+ raise ValueError(f"Directory not found: '{inputfolder}'")
1042
+
1043
+ @GeneralUtilities.check_arguments
1044
+ def __extension_matchs(self, file: str, obfuscate_file_extensions) -> bool:
1045
+ for extension in obfuscate_file_extensions:
1046
+ if file.lower().endswith("."+extension.lower()):
1047
+ return True
1048
+ return False
1049
+
1050
+ @GeneralUtilities.check_arguments
1051
+ def SCHealthcheck(self, file: str) -> int:
1052
+ lines = GeneralUtilities.read_lines_from_file(file)
1053
+ for line in reversed(lines):
1054
+ if not GeneralUtilities.string_is_none_or_whitespace(line):
1055
+ if "RunningHealthy (" in line: # TODO use regex
1056
+ GeneralUtilities.write_message_to_stderr(
1057
+ f"Healthy running due to line '{line}' in file '{file}'.")
1058
+ return 0
1059
+ else:
1060
+ GeneralUtilities.write_message_to_stderr(
1061
+ f"Not healthy running due to line '{line}' in file '{file}'.")
1062
+ return 1
1063
+ GeneralUtilities.write_message_to_stderr(
1064
+ f"No valid line found for healthycheck in file '{file}'.")
1065
+ return 2
1066
+
1067
+ @GeneralUtilities.check_arguments
1068
+ def SCObfuscateFilesFolder(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
1069
+ obfuscate_all_files = extensions == "*"
1070
+ if (obfuscate_all_files):
1071
+ obfuscate_file_extensions = None
1072
+ else:
1073
+ if "," in extensions:
1074
+ obfuscate_file_extensions = extensions.split(",")
1075
+ else:
1076
+ obfuscate_file_extensions = [extensions]
1077
+ newd = inputfolder+"_Obfuscated"
1078
+ shutil.copytree(inputfolder, newd)
1079
+ inputfolder = newd
1080
+ if (os.path.isdir(inputfolder)):
1081
+ for file in GeneralUtilities.absolute_file_paths(inputfolder):
1082
+ if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
1083
+ self.SCChangeHashOfProgram(file)
1084
+ os.remove(file)
1085
+ os.rename(file + ".modified", file)
1086
+ self.SCFilenameObfuscator(inputfolder, printtableheadline, namemappingfile, extensions)
1087
+ else:
1088
+ raise ValueError(f"Directory not found: '{inputfolder}'")
1089
+
1090
+ @GeneralUtilities.check_arguments
1091
+ def get_services_from_yaml_file(self, yaml_file: str) -> list[str]:
1092
+ with open(yaml_file, encoding="utf-8") as stream:
1093
+ loaded = yaml.safe_load(stream)
1094
+ services = loaded["services"]
1095
+ result = list(services.keys())
1096
+ return result
1097
+
1098
+ @GeneralUtilities.check_arguments
1099
+ def kill_docker_container(self, container_name: str) -> None:
1100
+ self.run_program("docker", f"container rm -f {container_name}")
1101
+
1102
+ @GeneralUtilities.check_arguments
1103
+ def get_docker_debian_version(self, image_tag: str) -> str:
1104
+ result = ScriptCollectionCore().run_program_argsasarray(
1105
+ "docker", ['run', f'debian:{image_tag}', 'bash', '-c', 'apt-get -y update && apt-get -y install lsb-release && lsb_release -cs'])
1106
+ result_line = GeneralUtilities.string_to_lines(result[1])[-2]
1107
+ return result_line
1108
+
1109
+ @GeneralUtilities.check_arguments
1110
+ def get_latest_tor_version_of_debian_repository(self, debian_version: str) -> str:
1111
+ package_url: str = f"https://deb.torproject.org/torproject.org/dists/{debian_version}/main/binary-amd64/Packages"
1112
+ r = requests.get(package_url, timeout=5)
1113
+ if r.status_code != 200:
1114
+ raise ValueError(f"Checking for latest tor package resulted in HTTP-response-code {r.status_code}.")
1115
+ lines = GeneralUtilities.string_to_lines(GeneralUtilities.bytes_to_string(r.content))
1116
+ version_line_prefix = "Version: "
1117
+ version_content_line = [line for line in lines if line.startswith(version_line_prefix)][1]
1118
+ version_with_overhead = version_content_line[len(version_line_prefix):]
1119
+ tor_version = version_with_overhead.split("~")[0]
1120
+ return tor_version
1121
+
1122
+ @GeneralUtilities.check_arguments
1123
+ def upload_file_to_file_host(self, file: str, host: str) -> int:
1124
+ if (host is None):
1125
+ return self.upload_file_to_random_filesharing_service(file)
1126
+ elif host == "anonfiles.com":
1127
+ return self.upload_file_to_anonfiles(file)
1128
+ elif host == "bayfiles.com":
1129
+ return self.upload_file_to_bayfiles(file)
1130
+ GeneralUtilities.write_message_to_stderr("Unknown host: "+host)
1131
+ return 1
1132
+
1133
+ @GeneralUtilities.check_arguments
1134
+ def upload_file_to_random_filesharing_service(self, file: str) -> int:
1135
+ host = randrange(2)
1136
+ if host == 0:
1137
+ return self.upload_file_to_anonfiles(file)
1138
+ if host == 1:
1139
+ return self.upload_file_to_bayfiles(file)
1140
+ return 1
1141
+
1142
+ @GeneralUtilities.check_arguments
1143
+ def upload_file_to_anonfiles(self, file) -> int:
1144
+ return self.upload_file_by_using_simple_curl_request("https://api.anonfiles.com/upload", file)
1145
+
1146
+ @GeneralUtilities.check_arguments
1147
+ def upload_file_to_bayfiles(self, file) -> int:
1148
+ return self.upload_file_by_using_simple_curl_request("https://api.bayfiles.com/upload", file)
1149
+
1150
+ @GeneralUtilities.check_arguments
1151
+ def upload_file_by_using_simple_curl_request(self, api_url: str, file: str) -> int:
1152
+ # TODO implement
1153
+ return 1
1154
+
1155
+ @GeneralUtilities.check_arguments
1156
+ def file_is_available_on_file_host(self, file) -> int:
1157
+ # TODO implement
1158
+ return 1
1159
+
1160
+ def run_testcases_for_python_project(self, repository_folder: str):
1161
+ self.run_program("coverage", "run -m pytest", repository_folder)
1162
+ self.run_program("coverage", "xml", repository_folder)
1163
+ GeneralUtilities.ensure_directory_exists(os.path.join(repository_folder, "Other/TestCoverage"))
1164
+ coveragefile = os.path.join(repository_folder, "Other/TestCoverage/TestCoverage.xml")
1165
+ GeneralUtilities.ensure_file_does_not_exist(coveragefile)
1166
+ os.rename(os.path.join(repository_folder, "coverage.xml"), coveragefile)
1167
+
1168
+ @GeneralUtilities.check_arguments
1169
+ def get_file_permission(self, file: str) -> str:
1170
+ """This function returns an usual octet-triple, for example "0700"."""
1171
+ ls_output = self.__ls(file)
1172
+ return self.__get_file_permission_helper(ls_output)
1173
+
1174
+ @GeneralUtilities.check_arguments
1175
+ def __get_file_permission_helper(self, ls_output: str) -> str:
1176
+ permissions = ' '.join(ls_output.split()).split(' ')[0][1:]
1177
+ return str(self.__to_octet(permissions[0:3]))+str(self.__to_octet(permissions[3:6]))+str(self.__to_octet(permissions[6:9]))
1178
+
1179
+ @GeneralUtilities.check_arguments
1180
+ def __to_octet(self, string: str) -> int:
1181
+ return int(self.__to_octet_helper(string[0])+self.__to_octet_helper(string[1])+self.__to_octet_helper(string[2]), 2)
1182
+
1183
+ @GeneralUtilities.check_arguments
1184
+ def __to_octet_helper(self, string: str) -> str:
1185
+ if (string == "-"):
1186
+ return "0"
1187
+ else:
1188
+ return "1"
1189
+
1190
+ @GeneralUtilities.check_arguments
1191
+ def get_file_owner(self, file: str) -> str:
1192
+ """This function returns the user and the group in the format "user:group"."""
1193
+ ls_output = self.__ls(file)
1194
+ return self.__get_file_owner_helper(ls_output)
1195
+
1196
+ @GeneralUtilities.check_arguments
1197
+ def __get_file_owner_helper(self, ls_output: str) -> str:
1198
+ try:
1199
+ splitted = ' '.join(ls_output.split()).split(' ')
1200
+ return f"{splitted[2]}:{splitted[3]}"
1201
+ except Exception as exception:
1202
+ raise ValueError(
1203
+ f"ls-output '{ls_output}' not parsable") from exception
1204
+
1205
+ @GeneralUtilities.check_arguments
1206
+ def get_file_owner_and_file_permission(self, file: str) -> str:
1207
+ ls_output = self.__ls(file)
1208
+ return [self.__get_file_owner_helper(ls_output), self.__get_file_permission_helper(ls_output)]
1209
+
1210
+ @GeneralUtilities.check_arguments
1211
+ def __ls(self, file: str) -> str:
1212
+ file = file.replace("\\", "/")
1213
+ GeneralUtilities.assert_condition(os.path.isfile(file) or os.path.isdir(
1214
+ file), f"Can not execute 'ls' because '{file}' does not exist")
1215
+ result = self.run_program_argsasarray("ls", ["-ld", file])
1216
+ GeneralUtilities.assert_condition(result[0] == 0, f"'ls -ld {file}' resulted in exitcode {str(result[0])}. StdErr: {result[2]}")
1217
+ GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(
1218
+ result[1]), f"'ls' of '{file}' had an empty output. StdErr: '{result[2]}'")
1219
+ return result[1]
1220
+
1221
+ @GeneralUtilities.check_arguments
1222
+ def set_permission(self, file_or_folder: str, permissions: str, recursive: bool = False) -> None:
1223
+ """This function expects an usual octet-triple, for example "700"."""
1224
+ args = []
1225
+ if recursive:
1226
+ args.append("--recursive")
1227
+ args.append(permissions)
1228
+ args.append(file_or_folder)
1229
+ self.run_program_argsasarray("chmod", args)
1230
+
1231
+ @GeneralUtilities.check_arguments
1232
+ def set_owner(self, file_or_folder: str, owner: str, recursive: bool = False, follow_symlinks: bool = False) -> None:
1233
+ """This function expects the user and the group in the format "user:group"."""
1234
+ args = []
1235
+ if recursive:
1236
+ args.append("--recursive")
1237
+ if follow_symlinks:
1238
+ args.append("--no-dereference")
1239
+ args.append(owner)
1240
+ args.append(file_or_folder)
1241
+ self.run_program_argsasarray("chown", args)
1242
+
1243
+ # <run programs>
1244
+
1245
+ @GeneralUtilities.check_arguments
1246
+ def __run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> Popen:
1247
+ # Verbosity:
1248
+ # 0=Quiet (No output will be printed.)
1249
+ # 1=Normal (If the exitcode of the executed program is not 0 then the StdErr will be printed.)
1250
+ # 2=Full (Prints StdOut and StdErr of the executed program.)
1251
+ # 3=Verbose (Same as "Full" but with some more information.)
1252
+
1253
+ if isinstance(self.program_runner, ProgramRunnerEpew):
1254
+ custom_argument = CustomEpewArgument(print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, verbosity, arguments_for_log)
1255
+ popen: Popen = self.program_runner.run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, custom_argument, interactive)
1256
+ return popen
1257
+
1258
+ # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
1259
+
1260
+ @GeneralUtilities.check_arguments
1261
+ def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False) -> tuple[int, str, str, int]:
1262
+ # verbosity 1: No output will be logged.
1263
+ # verbosity 2: If the exitcode of the executed program is not 0 then the StdErr will be logged. This is supposed to be the default verbosity-level.
1264
+ # verbosity 3: Logs and prints StdOut and StdErr of the executed program in realtime.
1265
+ # verbosity 4: Same as loglevel 3 but with some more overhead-information.
1266
+ try:
1267
+ arguments_as_str = ' '.join(arguments_as_array)
1268
+ mock_loader_result = self.__try_load_mock(program, arguments_as_str, working_directory)
1269
+ if mock_loader_result[0]:
1270
+ return mock_loader_result[1]
1271
+
1272
+ working_directory = self.__adapt_workingdirectory(working_directory)
1273
+
1274
+ if arguments_for_log is None:
1275
+ arguments_for_log = arguments_as_array
1276
+
1277
+ arguments_for_log_as_string: str = ' '.join(arguments_for_log)
1278
+ cmd = f'{working_directory}>{program} {arguments_for_log_as_string}'
1279
+
1280
+ if GeneralUtilities.string_is_none_or_whitespace(title):
1281
+ info_for_log = cmd
1282
+ else:
1283
+ info_for_log = title
1284
+
1285
+ if verbosity >= 3:
1286
+ GeneralUtilities.write_message_to_stdout(f"Run '{info_for_log}'.")
1287
+ with self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive) as process:
1288
+ pid = process.pid
1289
+
1290
+ stdout_lines = list[str]()
1291
+ stderr_lines = list[str]()
1292
+
1293
+ live_console_output_printing = 2 < verbosity and interactive
1294
+
1295
+ log_to_file = log_file is not None
1296
+ if log_to_file:
1297
+ GeneralUtilities.ensure_file_exists(log_file)
1298
+
1299
+ if interactive:
1300
+ # there are 2 issues in this part:
1301
+ # 1.: ctrl+c
1302
+ # 2.: sometimes this function does not terminate even if the started process exited
1303
+ def stream_process(process) -> bool:
1304
+ try:
1305
+ go: bool = process.poll() is None
1306
+
1307
+ stdoutreader: BufferedReader = process.stdout
1308
+ if stdoutreader.readable():
1309
+ stdoutresultb: bytes = stdoutreader.read()
1310
+ stdoutresult = GeneralUtilities.bytes_to_string(
1311
+ stdoutresultb)
1312
+ stdoutlines = GeneralUtilities.string_to_lines(stdoutresult)
1313
+ for line in stdoutlines:
1314
+ line_stripped = line.replace("\r", "").strip()
1315
+ if len(line_stripped) > 0:
1316
+ line_str = line_stripped
1317
+ stdout_lines.append(line_str)
1318
+ if live_console_output_printing:
1319
+ GeneralUtilities.write_message_to_stdout(line_str)
1320
+ if log_to_file:
1321
+ GeneralUtilities.append_line_to_file(
1322
+ log_file, line_str)
1323
+
1324
+ stderrreader: BufferedReader = process.stderr
1325
+ if stderrreader.readable():
1326
+ stderrresultb: bytes = stderrreader.read()
1327
+ stderrresult = GeneralUtilities.bytes_to_string(stderrresultb)
1328
+ stderrlines = GeneralUtilities.string_to_lines(stderrresult)
1329
+ for line in stderrlines:
1330
+ line_stripped = line.replace("\r", "").strip()
1331
+ if len(line_stripped) > 0:
1332
+ line_str = line_stripped
1333
+ stderr_lines.append(line_str)
1334
+ if live_console_output_printing:
1335
+ if print_errors_as_information:
1336
+ GeneralUtilities.write_message_to_stdout(line_str)
1337
+ else:
1338
+ GeneralUtilities.write_message_to_stderr(line_str)
1339
+ if log_to_file:
1340
+ GeneralUtilities.append_line_to_file(log_file, line_str)
1341
+
1342
+ return go
1343
+ except Exception:
1344
+ return False
1345
+
1346
+ while stream_process(process):
1347
+ time.sleep(0.1)
1348
+
1349
+ exit_code = process.poll()
1350
+ stdout = '\n'.join(stdout_lines)
1351
+ stderr = '\n'.join(stderr_lines)
1352
+ else:
1353
+ stdout, stderr = process.communicate()
1354
+ exit_code = process.wait()
1355
+ stdout = GeneralUtilities.bytes_to_string(stdout).replace('\r', '')
1356
+ stderr = GeneralUtilities.bytes_to_string(stderr).replace('\r', '')
1357
+
1358
+ if throw_exception_if_exitcode_is_not_zero and exit_code != 0:
1359
+ raise ValueError(f"Program '{working_directory}>{program} {arguments_for_log_as_string}' resulted in exitcode {exit_code}. (StdOut: '{stdout}', StdErr: '{stderr}')")
1360
+
1361
+ result = (exit_code, stdout, stderr, pid)
1362
+ return result
1363
+ except Exception as e:
1364
+ raise e
1365
+
1366
+ # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
1367
+ @GeneralUtilities.check_arguments
1368
+ def run_program(self, program: str, arguments: str = "", working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False) -> tuple[int, str, str, int]:
1369
+ return self.run_program_argsasarray(program, GeneralUtilities.arguments_to_array(arguments), working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive)
1370
+
1371
+ # Return-values program_runner: Pid
1372
+ @GeneralUtilities.check_arguments
1373
+ def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
1374
+ mock_loader_result = self.__try_load_mock(program, ' '.join(arguments_as_array), working_directory)
1375
+ if mock_loader_result[0]:
1376
+ return mock_loader_result[1]
1377
+ process: Popen = self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
1378
+ return process.pid
1379
+
1380
+ # Return-values program_runner: Pid
1381
+ @GeneralUtilities.check_arguments
1382
+ def run_program_async(self, program: str, arguments: str = "", working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
1383
+ return self.run_program_argsasarray_async(program, GeneralUtilities.arguments_to_array(arguments), working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
1384
+
1385
+ @GeneralUtilities.check_arguments
1386
+ def __try_load_mock(self, program: str, arguments: str, working_directory: str) -> tuple[bool, tuple[int, str, str, int]]:
1387
+ if self.mock_program_calls:
1388
+ try:
1389
+ return [True, self.__get_mock_program_call(program, arguments, working_directory)]
1390
+ except LookupError:
1391
+ if not self.execute_program_really_if_no_mock_call_is_defined:
1392
+ raise
1393
+ return [False, None]
1394
+
1395
+ @GeneralUtilities.check_arguments
1396
+ def __adapt_workingdirectory(self, workingdirectory: str) -> str:
1397
+ if workingdirectory is None:
1398
+ return os.getcwd()
1399
+ else:
1400
+ return GeneralUtilities.resolve_relative_path_from_current_working_directory(workingdirectory)
1401
+
1402
+ @GeneralUtilities.check_arguments
1403
+ def verify_no_pending_mock_program_calls(self):
1404
+ if (len(self.__mocked_program_calls) > 0):
1405
+ raise AssertionError(
1406
+ "The following mock-calls were not called:\n"+",\n ".join([self.__format_mock_program_call(r) for r in self.__mocked_program_calls]))
1407
+
1408
+ @GeneralUtilities.check_arguments
1409
+ def __format_mock_program_call(self, r) -> str:
1410
+ r: ScriptCollectionCore.__MockProgramCall = r
1411
+ return f"'{r.workingdirectory}>{r.program} {r.argument}' (" \
1412
+ f"exitcode: {GeneralUtilities.str_none_safe(str(r.exit_code))}, " \
1413
+ f"pid: {GeneralUtilities.str_none_safe(str(r.pid))}, "\
1414
+ f"stdout: {GeneralUtilities.str_none_safe(str(r.stdout))}, " \
1415
+ f"stderr: {GeneralUtilities.str_none_safe(str(r.stderr))})"
1416
+
1417
+ @GeneralUtilities.check_arguments
1418
+ def register_mock_program_call(self, program: str, argument: str, workingdirectory: str, result_exit_code: int, result_stdout: str, result_stderr: str, result_pid: int, amount_of_expected_calls=1):
1419
+ "This function is for test-purposes only"
1420
+ for _ in itertools.repeat(None, amount_of_expected_calls):
1421
+ mock_call = ScriptCollectionCore.__MockProgramCall()
1422
+ mock_call.program = program
1423
+ mock_call.argument = argument
1424
+ mock_call.workingdirectory = workingdirectory
1425
+ mock_call.exit_code = result_exit_code
1426
+ mock_call.stdout = result_stdout
1427
+ mock_call.stderr = result_stderr
1428
+ mock_call.pid = result_pid
1429
+ self.__mocked_program_calls.append(mock_call)
1430
+
1431
+ @GeneralUtilities.check_arguments
1432
+ def __get_mock_program_call(self, program: str, argument: str, workingdirectory: str):
1433
+ result: ScriptCollectionCore.__MockProgramCall = None
1434
+ for mock_call in self.__mocked_program_calls:
1435
+ if ((re.match(mock_call.program, program) is not None)
1436
+ and (re.match(mock_call.argument, argument) is not None)
1437
+ and (re.match(mock_call.workingdirectory, workingdirectory) is not None)):
1438
+ result = mock_call
1439
+ break
1440
+ if result is None:
1441
+ raise LookupError(
1442
+ f"Tried to execute mock-call '{workingdirectory}>{program} {argument}' but no mock-call was defined for that execution")
1443
+ else:
1444
+ self.__mocked_program_calls.remove(result)
1445
+ return (result.exit_code, result.stdout, result.stderr, result.pid)
1446
+
1447
+ @GeneralUtilities.check_arguments
1448
+ class __MockProgramCall:
1449
+ program: str
1450
+ argument: str
1451
+ workingdirectory: str
1452
+ exit_code: int
1453
+ stdout: str
1454
+ stderr: str
1455
+ pid: int
1456
+
1457
+ # </run programs>
1458
+
1459
+ @GeneralUtilities.check_arguments
1460
+ def extract_archive_with_7z(self, unzip_program_file: str, zipfile: str, password: str, output_directory: str) -> None:
1461
+ password_set = not password is None
1462
+ file_name = Path(zipfile).name
1463
+ file_folder = os.path.dirname(zipfile)
1464
+ argument = "x"
1465
+ if password_set:
1466
+ argument = f"{argument} -p\"{password}\""
1467
+ argument = f"{argument} -o {output_directory}"
1468
+ argument = f"{argument} {file_name}"
1469
+ return self.run_program(unzip_program_file, argument, file_folder)
1470
+
1471
+ @GeneralUtilities.check_arguments
1472
+ def get_internet_time(self) -> datetime:
1473
+ response = ntplib.NTPClient().request('pool.ntp.org')
1474
+ return datetime.fromtimestamp(response.tx_time)
1475
+
1476
+ @GeneralUtilities.check_arguments
1477
+ def system_time_equals_internet_time(self, maximal_tolerance_difference: timedelta) -> bool:
1478
+ return abs(datetime.now() - self.get_internet_time()) < maximal_tolerance_difference
1479
+
1480
+ @GeneralUtilities.check_arguments
1481
+ def system_time_equals_internet_time_with_default_tolerance(self) -> bool:
1482
+ return self.system_time_equals_internet_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
1483
+
1484
+ @GeneralUtilities.check_arguments
1485
+ def check_system_time(self, maximal_tolerance_difference: timedelta):
1486
+ if not self.system_time_equals_internet_time(maximal_tolerance_difference):
1487
+ raise ValueError("System time may be wrong")
1488
+
1489
+ @GeneralUtilities.check_arguments
1490
+ def check_system_time_with_default_tolerance(self) -> None:
1491
+ self.check_system_time(
1492
+ self.__get_default_tolerance_for_system_time_equals_internet_time())
1493
+
1494
+ @GeneralUtilities.check_arguments
1495
+ def __get_default_tolerance_for_system_time_equals_internet_time(self) -> timedelta:
1496
+ return timedelta(hours=0, minutes=0, seconds=3)
1497
+
1498
+ @GeneralUtilities.check_arguments
1499
+ def increment_version(self, input_version: str, increment_major: bool, increment_minor: bool, increment_patch: bool) -> str:
1500
+ splitted = input_version.split(".")
1501
+ GeneralUtilities.assert_condition(len(splitted) == 3, f"Version '{input_version}' does not have the 'major.minor.patch'-pattern.")
1502
+ major = int(splitted[0])
1503
+ minor = int(splitted[1])
1504
+ patch = int(splitted[2])
1505
+ if increment_major:
1506
+ major = major+1
1507
+ if increment_minor:
1508
+ minor = minor+1
1509
+ if increment_patch:
1510
+ patch = patch+1
1511
+ return f"{major}.{minor}.{patch}"
1512
+
1513
+ @GeneralUtilities.check_arguments
1514
+ def get_semver_version_from_gitversion(self, repository_folder: str) -> str:
1515
+ if (self.git_repository_has_commits(repository_folder)):
1516
+ result = self.get_version_from_gitversion(repository_folder, "MajorMinorPatch")
1517
+ if self.git_repository_has_uncommitted_changes(repository_folder):
1518
+ if self.get_current_git_branch_has_tag(repository_folder):
1519
+ id_of_latest_tag = self.git_get_commitid_of_tag(
1520
+ repository_folder, self.get_latest_git_tag(repository_folder))
1521
+ current_commit = self.git_get_commit_id(repository_folder)
1522
+ current_commit_is_on_latest_tag = id_of_latest_tag == current_commit
1523
+ if current_commit_is_on_latest_tag:
1524
+ result = self.increment_version(
1525
+ result, False, False, True)
1526
+ else:
1527
+ result = "0.1.0"
1528
+ return result
1529
+
1530
+ @staticmethod
1531
+ @GeneralUtilities.check_arguments
1532
+ def is_patch_version(version_string: str) -> bool:
1533
+ return not version_string.endswith(".0")
1534
+
1535
+ @GeneralUtilities.check_arguments
1536
+ def get_version_from_gitversion(self, folder: str, variable: str) -> str:
1537
+ # called twice as workaround for issue 1877 in gitversion ( https://github.com/GitTools/GitVersion/issues/1877 )
1538
+ result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder, verbosity=0)
1539
+ result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder, verbosity=0)
1540
+ result = GeneralUtilities.strip_new_line_character(result[1])
1541
+
1542
+ return result
1543
+
1544
+ @GeneralUtilities.check_arguments
1545
+ def generate_certificate_authority(self, folder: str, name: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
1546
+ if days_until_expire is None:
1547
+ days_until_expire = 1825
1548
+ if password is None:
1549
+ password = GeneralUtilities.generate_password()
1550
+ GeneralUtilities.ensure_directory_exists(folder)
1551
+ self.run_program("openssl", f'req -new -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -days {days_until_expire} -nodes -x509 -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={name}/OU={subj_ou} -passout pass:{password} -keyout {name}.key -out {name}.crt', folder)
1552
+
1553
+ @GeneralUtilities.check_arguments
1554
+ def generate_certificate(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
1555
+ if days_until_expire is None:
1556
+ days_until_expire = 397
1557
+ if password is None:
1558
+ password = GeneralUtilities.generate_password()
1559
+ rsa_key_length = 4096
1560
+ self.run_program("openssl", f'genrsa -out {filename}.key {rsa_key_length}', folder)
1561
+ self.run_program("openssl", f'req -new -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou} -x509 -key {filename}.key -out {filename}.unsigned.crt -days {days_until_expire}', folder)
1562
+ self.run_program("openssl", f'pkcs12 -export -out {filename}.selfsigned.pfx -password pass:{password} -inkey {filename}.key -in {filename}.unsigned.crt', folder)
1563
+ GeneralUtilities.write_text_to_file(
1564
+ os.path.join(folder, f"{filename}.password"), password)
1565
+ GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.san.conf"), f"""[ req ]
1566
+ default_bits = {rsa_key_length}
1567
+ distinguished_name = req_distinguished_name
1568
+ req_extensions = v3_req
1569
+ default_md = sha256
1570
+ dirstring_type = nombstr
1571
+ prompt = no
1572
+
1573
+ [ req_distinguished_name ]
1574
+ countryName = {subj_c}
1575
+ stateOrProvinceName = {subj_st}
1576
+ localityName = {subj_l}
1577
+ organizationName = {subj_o}
1578
+ organizationUnit = {subj_ou}
1579
+ commonName = {domain}
1580
+
1581
+ [v3_req]
1582
+ subjectAltName = @subject_alt_name
1583
+
1584
+ [ subject_alt_name ]
1585
+ DNS = {domain}
1586
+ """)
1587
+
1588
+ @GeneralUtilities.check_arguments
1589
+ def generate_certificate_sign_request(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str) -> None:
1590
+ self.run_program(
1591
+ "openssl", f'req -new -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou} -key {filename}.key -out {filename}.csr -config {filename}.san.conf', folder)
1592
+
1593
+ @GeneralUtilities.check_arguments
1594
+ def sign_certificate(self, folder: str, ca_folder: str, ca_name: str, domain: str, filename: str, days_until_expire: int = None) -> None:
1595
+ if days_until_expire is None:
1596
+ days_until_expire = 397
1597
+ ca = os.path.join(ca_folder, ca_name)
1598
+ password_file = os.path.join(folder, f"{filename}.password")
1599
+ password = GeneralUtilities.read_text_from_file(password_file)
1600
+ self.run_program("openssl", f'x509 -req -in {filename}.csr -CA {ca}.crt -CAkey {ca}.key -CAcreateserial -CAserial {ca}.srl -out {filename}.crt -days {days_until_expire} -sha256 -extensions v3_req -extfile {filename}.san.conf', folder)
1601
+ self.run_program("openssl", f'pkcs12 -export -out {filename}.pfx -inkey {filename}.key -in {filename}.crt -password pass:{password}', folder)
1602
+
1603
+ @GeneralUtilities.check_arguments
1604
+ def update_dependencies_of_python_in_requirementstxt_file(self, file: str, verbosity: int):
1605
+ lines = GeneralUtilities.read_lines_from_file(file)
1606
+ new_lines = []
1607
+ for line in lines:
1608
+ if GeneralUtilities.string_has_content(line):
1609
+ new_lines.append(self.__get_updated_line_for_python_requirements(line.strip()))
1610
+ GeneralUtilities.write_lines_to_file(file, new_lines)
1611
+
1612
+ @GeneralUtilities.check_arguments
1613
+ def __get_updated_line_for_python_requirements(self, line: str) -> str:
1614
+ if "==" in line or "<" in line:
1615
+ return line
1616
+ elif ">" in line:
1617
+ try:
1618
+ # line is something like "cyclonedx-bom>=2.0.2" and the function must return with the updated version
1619
+ # (something like "cyclonedx-bom>=3.11.0" for example)
1620
+ package = line.split(">")[0]
1621
+ operator = ">=" if ">=" in line else ">"
1622
+ response = requests.get(f'https://pypi.org/pypi/{package}/json', timeout=5)
1623
+ latest_version = response.json()['info']['version']
1624
+ return package+operator+latest_version
1625
+ except:
1626
+ return line
1627
+ else:
1628
+ raise ValueError(f'Unexpected line in requirements-file: "{line}"')
1629
+
1630
+ @GeneralUtilities.check_arguments
1631
+ def update_dependencies_of_python_in_setupcfg_file(self, setup_cfg_file: str, verbosity: int):
1632
+ lines = GeneralUtilities.read_lines_from_file(setup_cfg_file)
1633
+ new_lines = []
1634
+ requirement_parsing_mode = False
1635
+ for line in lines:
1636
+ new_line = line
1637
+ if (requirement_parsing_mode):
1638
+ if ("<" in line or "=" in line or ">" in line):
1639
+ updated_line = f" {self.__get_updated_line_for_python_requirements(line.strip())}"
1640
+ new_line = updated_line
1641
+ else:
1642
+ requirement_parsing_mode = False
1643
+ else:
1644
+ if line.startswith("install_requires ="):
1645
+ requirement_parsing_mode = True
1646
+ new_lines.append(new_line)
1647
+ GeneralUtilities.write_lines_to_file(setup_cfg_file, new_lines)
1648
+
1649
+ @GeneralUtilities.check_arguments
1650
+ def update_dependencies_of_dotnet_project(self, csproj_file: str, verbosity: int, ignored_dependencies: list[str]):
1651
+ folder = os.path.dirname(csproj_file)
1652
+ csproj_filename = os.path.basename(csproj_file)
1653
+ GeneralUtilities.write_message_to_stderr(f"Check for updates in {csproj_filename}")
1654
+ result = self.run_program("dotnet", f"list {csproj_filename} package --outdated", folder)
1655
+ for line in result[1].replace("\r", "").split("\n"):
1656
+ # Relevant output-lines are something like " > NJsonSchema 10.7.0 10.7.0 10.9.0"
1657
+ if ">" in line:
1658
+ package_name = line.replace(">", "").strip().split(" ")[0]
1659
+ if not (ignored_dependencies in package_name):
1660
+ GeneralUtilities.write_message_to_stderr(f"Update package {package_name}")
1661
+ self.run_program("dotnet", f"add {csproj_filename} package {package_name}", folder)
1662
+
1663
+ @GeneralUtilities.check_arguments
1664
+ def create_deb_package(self, toolname: str, binary_folder: str, control_file_content: str, deb_output_folder: str, verbosity: int, permission_of_executable_file_as_octet_triple: int) -> None:
1665
+
1666
+ # prepare
1667
+ GeneralUtilities.ensure_directory_exists(deb_output_folder)
1668
+ temp_folder = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
1669
+ GeneralUtilities.ensure_directory_exists(temp_folder)
1670
+ bin_folder = binary_folder
1671
+ tool_content_folder_name = toolname+"Content"
1672
+
1673
+ # create folder
1674
+ GeneralUtilities.ensure_directory_exists(temp_folder)
1675
+ control_content_folder_name = "controlcontent"
1676
+ packagecontent_control_folder = os.path.join(temp_folder, control_content_folder_name)
1677
+ GeneralUtilities.ensure_directory_exists(packagecontent_control_folder)
1678
+ data_content_folder_name = "datacontent"
1679
+ packagecontent_data_folder = os.path.join(temp_folder, data_content_folder_name)
1680
+ GeneralUtilities.ensure_directory_exists(packagecontent_data_folder)
1681
+ entireresult_content_folder_name = "entireresultcontent"
1682
+ packagecontent_entireresult_folder = os.path.join(temp_folder, entireresult_content_folder_name)
1683
+ GeneralUtilities.ensure_directory_exists(packagecontent_entireresult_folder)
1684
+
1685
+ # create "debian-binary"-file
1686
+ debianbinary_file = os.path.join(
1687
+ packagecontent_entireresult_folder, "debian-binary")
1688
+ GeneralUtilities.ensure_file_exists(debianbinary_file)
1689
+ GeneralUtilities.write_text_to_file(debianbinary_file, "2.0\n")
1690
+
1691
+ # create control-content
1692
+
1693
+ # conffiles
1694
+ conffiles_file = os.path.join(packagecontent_control_folder, "conffiles")
1695
+ GeneralUtilities.ensure_file_exists(conffiles_file)
1696
+
1697
+ # postinst-script
1698
+ postinst_file = os.path.join(packagecontent_control_folder, "postinst")
1699
+ GeneralUtilities.ensure_file_exists(postinst_file)
1700
+ exe_file = f"/usr/bin/{tool_content_folder_name}/{toolname}"
1701
+ link_file = f"/usr/bin/{toolname.lower()}"
1702
+ permission = str(permission_of_executable_file_as_octet_triple)
1703
+ GeneralUtilities.write_text_to_file(postinst_file, f"""#!/bin/sh
1704
+ ln -s {exe_file} {link_file}
1705
+ chmod {permission} {exe_file}
1706
+ chmod {permission} {link_file}
1707
+ """)
1708
+
1709
+ # control
1710
+ control_file = os.path.join(packagecontent_control_folder, "control")
1711
+ GeneralUtilities.ensure_file_exists(control_file)
1712
+ GeneralUtilities.write_text_to_file(control_file, control_file_content)
1713
+
1714
+ # md5sums
1715
+ md5sums_file = os.path.join(packagecontent_control_folder, "md5sums")
1716
+ GeneralUtilities.ensure_file_exists(md5sums_file)
1717
+
1718
+ # create data-content
1719
+
1720
+ # copy binaries
1721
+ usr_bin_folder = os.path.join(packagecontent_data_folder, "usr/bin")
1722
+ GeneralUtilities.ensure_directory_exists(usr_bin_folder)
1723
+ usr_bin_content_folder = os.path.join(
1724
+ usr_bin_folder, tool_content_folder_name)
1725
+ GeneralUtilities.copy_content_of_folder(bin_folder, usr_bin_content_folder)
1726
+
1727
+ # create debfile
1728
+ deb_filename = f"{toolname}.deb"
1729
+ self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/control.tar.gz", "*"], packagecontent_control_folder, verbosity=verbosity)
1730
+ self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/data.tar.gz", "*"], packagecontent_data_folder, verbosity=verbosity)
1731
+ self.run_program_argsasarray("ar", ["r", deb_filename, "debian-binary", "control.tar.gz",
1732
+ "data.tar.gz"], packagecontent_entireresult_folder, verbosity=verbosity)
1733
+ result_file = os.path.join(packagecontent_entireresult_folder, deb_filename)
1734
+ shutil.copy(result_file, os.path.join(deb_output_folder, deb_filename))
1735
+
1736
+ # cleanup
1737
+ GeneralUtilities.ensure_directory_does_not_exist(temp_folder)
1738
+
1739
+ @GeneralUtilities.check_arguments
1740
+ def update_year_in_copyright_tags(self, file: str) -> None:
1741
+ current_year = str(datetime.now().year)
1742
+ lines = GeneralUtilities.read_lines_from_file(file)
1743
+ lines_result = []
1744
+ for line in lines:
1745
+ if match := re.search("(.*<[Cc]opyright>.*)\\d\\d\\d\\d(.*<\\/[Cc]opyright>.*)", line):
1746
+ part1 = match.group(1)
1747
+ part2 = match.group(2)
1748
+ adapted = part1+current_year+part2
1749
+ else:
1750
+ adapted = line
1751
+ lines_result.append(adapted)
1752
+ GeneralUtilities.write_lines_to_file(file, lines_result)
1753
+
1754
+ @GeneralUtilities.check_arguments
1755
+ def update_year_in_first_line_of_file(self, file: str) -> None:
1756
+ current_year = str(datetime.now().year)
1757
+ lines = GeneralUtilities.read_lines_from_file(file)
1758
+ lines[0] = re.sub("\\d\\d\\d\\d", current_year, lines[0])
1759
+ GeneralUtilities.write_lines_to_file(file, lines)
1760
+
1761
+ @GeneralUtilities.check_arguments
1762
+ def get_external_ip(self, proxy: str) -> str:
1763
+ information = self.get_externalnetworkinformation_as_json_string(proxy)
1764
+ parsed = json.loads(information)
1765
+ return parsed.ip
1766
+
1767
+ @GeneralUtilities.check_arguments
1768
+ def get_country_of_external_ip(self, proxy: str) -> str:
1769
+ information = self.get_externalnetworkinformation_as_json_string(proxy)
1770
+ parsed = json.loads(information)
1771
+ return parsed.country
1772
+
1773
+ @GeneralUtilities.check_arguments
1774
+ def get_externalnetworkinformation_as_json_string(self, proxy: str) -> str:
1775
+ proxies = None
1776
+ if GeneralUtilities.string_has_content(proxy):
1777
+ proxies = {"http": proxy}
1778
+ response = requests.get('https://ipinfo.io',
1779
+ proxies=proxies, timeout=5)
1780
+ network_information_as_json_string = GeneralUtilities.bytes_to_string(
1781
+ response.content)
1782
+ return network_information_as_json_string
1783
+
1784
+ @GeneralUtilities.check_arguments
1785
+ def change_file_extensions(self, folder: str, from_extension: str, to_extension: str, recursive: bool, ignore_case: bool) -> None:
1786
+ extension_to_compare:str=None
1787
+ if ignore_case:
1788
+ extension_to_compare=from_extension.lower()
1789
+ else:
1790
+ extension_to_compare=from_extension
1791
+ for file in GeneralUtilities.get_direct_files_of_folder(folder):
1792
+ if (ignore_case and file.lower().endswith(f".{extension_to_compare}")
1793
+ or not ignore_case and file.endswith(f".{extension_to_compare}")):
1794
+ p = Path(file)
1795
+ p.rename(p.with_suffix('.'+to_extension))
1796
+ if recursive:
1797
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
1798
+ self.change_file_extensions(subfolder, from_extension, to_extension, recursive,ignore_case)