ScriptCollection 3.5.16__py3-none-any.whl → 4.0.78__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. ScriptCollection/AnionBuildPlatform.py +206 -0
  2. ScriptCollection/{UpdateCertificates.py → CertificateUpdater.py} +69 -46
  3. ScriptCollection/Executables.py +515 -18
  4. ScriptCollection/GeneralUtilities.py +1272 -873
  5. ScriptCollection/ImageUpdater.py +648 -0
  6. ScriptCollection/ProgramRunnerBase.py +10 -10
  7. ScriptCollection/ProgramRunnerMock.py +2 -0
  8. ScriptCollection/ProgramRunnerPopen.py +7 -1
  9. ScriptCollection/ProgramRunnerSudo.py +108 -0
  10. ScriptCollection/SCLog.py +115 -0
  11. ScriptCollection/ScriptCollectionCore.py +942 -266
  12. ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +95 -0
  13. ScriptCollection/TFCPS/Docker/__init__.py +0 -0
  14. ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py +8 -0
  15. ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py +6 -0
  16. ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py +7 -0
  17. ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +485 -0
  18. ScriptCollection/TFCPS/DotNet/__init__.py +0 -0
  19. ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +130 -0
  20. ScriptCollection/TFCPS/Flutter/__init__.py +0 -0
  21. ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py +74 -0
  22. ScriptCollection/TFCPS/Go/__init__.py +0 -0
  23. ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +131 -0
  24. ScriptCollection/TFCPS/NodeJS/__init__.py +0 -0
  25. ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +227 -0
  26. ScriptCollection/TFCPS/Python/__init__.py +0 -0
  27. ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +418 -0
  28. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +128 -0
  29. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +136 -0
  30. ScriptCollection/TFCPS/TFCPS_CreateRelease.py +95 -0
  31. ScriptCollection/TFCPS/TFCPS_Generic.py +43 -0
  32. ScriptCollection/TFCPS/TFCPS_MergeToMain.py +122 -0
  33. ScriptCollection/TFCPS/TFCPS_MergeToStable.py +350 -0
  34. ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +47 -0
  35. ScriptCollection/TFCPS/TFCPS_Tools_General.py +1356 -0
  36. ScriptCollection/TFCPS/__init__.py +0 -0
  37. {ScriptCollection-3.5.16.dist-info → scriptcollection-4.0.78.dist-info}/METADATA +23 -22
  38. scriptcollection-4.0.78.dist-info/RECORD +43 -0
  39. {ScriptCollection-3.5.16.dist-info → scriptcollection-4.0.78.dist-info}/WHEEL +1 -1
  40. {ScriptCollection-3.5.16.dist-info → scriptcollection-4.0.78.dist-info}/entry_points.txt +32 -0
  41. ScriptCollection/ProgramRunnerEpew.py +0 -122
  42. ScriptCollection/RPStream.py +0 -42
  43. ScriptCollection/TasksForCommonProjectStructure.py +0 -2625
  44. ScriptCollection-3.5.16.dist-info/RECORD +0 -16
  45. {ScriptCollection-3.5.16.dist-info → scriptcollection-4.0.78.dist-info}/top_level.txt +0 -0
@@ -1,20 +1,25 @@
1
- import sys
2
1
  from datetime import timedelta, datetime
3
2
  import json
4
3
  import binascii
5
4
  import filecmp
6
5
  import hashlib
6
+ import multiprocessing
7
7
  import time
8
8
  from io import BytesIO
9
9
  import itertools
10
+ import zipfile
10
11
  import math
12
+ import base64
11
13
  import os
12
14
  from queue import Queue, Empty
13
15
  from concurrent.futures import ThreadPoolExecutor
16
+ import xml.etree.ElementTree as ET
14
17
  from pathlib import Path
15
18
  from subprocess import Popen
16
19
  import re
17
20
  import shutil
21
+ from typing import IO
22
+ import fnmatch
18
23
  import uuid
19
24
  import tempfile
20
25
  import io
@@ -24,13 +29,13 @@ import yaml
24
29
  import qrcode
25
30
  import pycdlib
26
31
  import send2trash
27
- import PyPDF2
32
+ from pypdf import PdfReader, PdfWriter
28
33
  from .GeneralUtilities import GeneralUtilities
29
34
  from .ProgramRunnerBase import ProgramRunnerBase
30
35
  from .ProgramRunnerPopen import ProgramRunnerPopen
31
- from .ProgramRunnerEpew import ProgramRunnerEpew, CustomEpewArgument
36
+ from .SCLog import SCLog, LogLevel
32
37
 
33
- version = "3.5.16"
38
+ version = "4.0.78"
34
39
  __version__ = version
35
40
 
36
41
 
@@ -38,15 +43,19 @@ class ScriptCollectionCore:
38
43
 
39
44
  # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
40
45
  # Do not change this value for productive environments.
41
- mock_program_calls: bool = False
46
+ mock_program_calls: bool = False#TODO remove this variable. When someone want to mock program-calls then the ProgramRunnerMock can be used instead
42
47
  # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
43
48
  execute_program_really_if_no_mock_call_is_defined: bool = False
44
49
  __mocked_program_calls: list = None
45
50
  program_runner: ProgramRunnerBase = None
51
+ call_program_runner_directly: bool = None
52
+ log: SCLog = None
46
53
 
47
54
  def __init__(self):
48
55
  self.program_runner = ProgramRunnerPopen()
56
+ self.call_program_runner_directly = None
49
57
  self.__mocked_program_calls = list[ScriptCollectionCore.__MockProgramCall]()
58
+ self.log = SCLog(None, LogLevel.Warning, False)
50
59
 
51
60
  @staticmethod
52
61
  @GeneralUtilities.check_arguments
@@ -58,14 +67,14 @@ class ScriptCollectionCore:
58
67
  errors = list()
59
68
  filename = os.path.relpath(file, working_directory)
60
69
  if treat_warnings_as_errors:
61
- errorsonly_argument = ""
70
+ errorsonly_argument = GeneralUtilities.empty_string
62
71
  else:
63
72
  errorsonly_argument = " --errors-only"
64
73
  (exit_code, stdout, stderr, _) = self.run_program("pylint", filename + errorsonly_argument, working_directory, throw_exception_if_exitcode_is_not_zero=False)
65
74
  if (exit_code != 0):
66
75
  errors.append(f"Linting-issues of {file}:")
67
76
  errors.append(f"Pylint-exitcode: {exit_code}")
68
- for line in GeneralUtilities.string_to_lines(stdout):
77
+ for line in GeneralUtilities.string_to_lines(stdout):
69
78
  errors.append(line)
70
79
  for line in GeneralUtilities.string_to_lines(stderr):
71
80
  errors.append(line)
@@ -108,31 +117,48 @@ class ScriptCollectionCore:
108
117
  raise ValueError(f"Version '{current_version}' does not match version-regex '{versiononlyregex}'")
109
118
 
110
119
  @GeneralUtilities.check_arguments
111
- def push_nuget_build_artifact(self, nupkg_file: str, registry_address: str, api_key: str, verbosity: int = 1):
120
+ def push_nuget_build_artifact(self, nupkg_file: str, registry_address: str, api_key: str = None):
112
121
  nupkg_file_name = os.path.basename(nupkg_file)
113
122
  nupkg_file_folder = os.path.dirname(nupkg_file)
114
- self.run_program("dotnet", f"nuget push {nupkg_file_name} --force-english-output --source {registry_address} --api-key {api_key}", nupkg_file_folder, verbosity)
123
+ argument = f"nuget push {nupkg_file_name} --force-english-output --source {registry_address}"
124
+ if api_key is not None:
125
+ argument = f"{argument} --api-key {api_key}"
126
+ self.run_program("dotnet", argument, nupkg_file_folder)
115
127
 
116
128
  @GeneralUtilities.check_arguments
117
- def dotnet_build(self, repository_folder: str, projectname: str, configuration: str):
118
- self.run_program("dotnet", f"clean -c {configuration}", repository_folder)
119
- self.run_program("dotnet", f"build {projectname}/{projectname}.csproj -c {configuration}", repository_folder)
129
+ def dotnet_build(self, folder: str, projectname: str, configuration: str):
130
+ self.run_program("dotnet", f"clean -c {configuration}", folder)
131
+ self.run_program("dotnet", f"build {projectname}/{projectname}.csproj -c {configuration}", folder)
120
132
 
121
133
  @GeneralUtilities.check_arguments
122
- def find_file_by_extension(self, folder: str, extension: str):
123
- result = [file for file in GeneralUtilities.get_direct_files_of_folder(folder) if file.endswith(f".{extension}")]
134
+ def find_file_by_extension(self, folder: str, extension_without_dot: str):
135
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
136
+ result = [file for file in self.list_content(folder, True, False, False) if file.endswith(f".{extension_without_dot}")]
124
137
  result_length = len(result)
125
138
  if result_length == 0:
126
- raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension}'.")
139
+ raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension_without_dot}'.")
127
140
  if result_length == 1:
128
141
  return result[0]
129
142
  else:
130
- raise ValueError(f"Multiple values available in folder '{folder}' with extension '{extension}'.")
143
+ raise ValueError(f"Multiple values available in folder '{folder}' with extension '{extension_without_dot}'.")
144
+
145
+ @GeneralUtilities.check_arguments
146
+ def find_last_file_by_extension(self, folder: str, extension_without_dot: str) -> str:
147
+ files: list[str] = GeneralUtilities.get_direct_files_of_folder(folder)
148
+ possible_results: list[str] = []
149
+ for file in files:
150
+ if file.endswith(f".{extension_without_dot}"):
151
+ possible_results.append(file)
152
+ result_length = len(possible_results)
153
+ if result_length == 0:
154
+ raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension_without_dot}'.")
155
+ else:
156
+ return possible_results[-1]
131
157
 
132
158
  @GeneralUtilities.check_arguments
133
159
  def commit_is_signed_by_key(self, repository_folder: str, revision_identifier: str, key: str) -> bool:
134
- result = self.run_program(
135
- "git", f"verify-commit {revision_identifier}", repository_folder, throw_exception_if_exitcode_is_not_zero=False)
160
+ self.is_git_or_bare_git_repository(repository_folder)
161
+ result = self.run_program("git", f"verify-commit {revision_identifier}", repository_folder, throw_exception_if_exitcode_is_not_zero=False)
136
162
  if (result[0] != 0):
137
163
  return False
138
164
  if (not GeneralUtilities.contains_line(result[1].splitlines(), f"gpg\\:\\ using\\ [A-Za-z0-9]+\\ key\\ [A-Za-z0-9]+{key}")):
@@ -145,35 +171,18 @@ class ScriptCollectionCore:
145
171
 
146
172
  @GeneralUtilities.check_arguments
147
173
  def get_parent_commit_ids_of_commit(self, repository_folder: str, commit_id: str) -> str:
148
- return self.run_program("git", f'log --pretty=%P -n 1 "{commit_id}"', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].replace("\r", "").replace("\n", "").split(" ")
174
+ self.is_git_or_bare_git_repository(repository_folder)
175
+ return self.run_program("git", f'log --pretty=%P -n 1 "{commit_id}"', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string).split(" ")
149
176
 
150
- @GeneralUtilities.check_arguments
151
- def get_all_authors_and_committers_of_repository(self, repository_folder: str, subfolder: str = None, verbosity: int = 1) -> list[tuple[str, str]]:
152
- space_character = "_"
153
- if subfolder is None:
154
- subfolder_argument = ""
155
- else:
156
- subfolder_argument = f" -- {subfolder}"
157
- log_result = self.run_program("git", f'log --pretty=%aN{space_character}%aE%n%cN{space_character}%cE HEAD{subfolder_argument}', repository_folder, verbosity=0)
158
- plain_content: list[str] = list(
159
- set([line for line in log_result[1].split("\n") if len(line) > 0]))
160
- result: list[tuple[str, str]] = []
161
- for item in plain_content:
162
- if len(re.findall(space_character, item)) == 1:
163
- splitted = item.split(space_character)
164
- result.append((splitted[0], splitted[1]))
165
- else:
166
- raise ValueError(f'Unexpected author: "{item}"')
167
- return result
168
177
 
169
178
  @GeneralUtilities.check_arguments
170
179
  def get_commit_ids_between_dates(self, repository_folder: str, since: datetime, until: datetime, ignore_commits_which_are_not_in_history_of_head: bool = True) -> None:
180
+ self.is_git_or_bare_git_repository(repository_folder)
171
181
  since_as_string = self.__datetime_to_string_for_git(since)
172
182
  until_as_string = self.__datetime_to_string_for_git(until)
173
- result = filter(lambda line: not GeneralUtilities.string_is_none_or_whitespace(line), self.run_program("git", f'log --since "{since_as_string}" --until "{until_as_string}" --pretty=format:"%H" --no-patch', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].split("\n").replace("\r", ""))
183
+ result = filter(lambda line: not GeneralUtilities.string_is_none_or_whitespace(line), self.run_program("git", f'log --since "{since_as_string}" --until "{until_as_string}" --pretty=format:"%H" --no-patch', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].split("\n").replace("\r", GeneralUtilities.empty_string))
174
184
  if ignore_commits_which_are_not_in_history_of_head:
175
- result = [commit_id for commit_id in result if self.git_commit_is_ancestor(
176
- repository_folder, commit_id)]
185
+ result = [commit_id for commit_id in result if self.git_commit_is_ancestor(repository_folder, commit_id)]
177
186
  return result
178
187
 
179
188
  @GeneralUtilities.check_arguments
@@ -182,44 +191,52 @@ class ScriptCollectionCore:
182
191
 
183
192
  @GeneralUtilities.check_arguments
184
193
  def git_commit_is_ancestor(self, repository_folder: str, ancestor: str, descendant: str = "HEAD") -> bool:
185
- exit_code = self.run_program_argsasarray("git", ["merge-base", "--is-ancestor", ancestor, descendant], repository_folder, throw_exception_if_exitcode_is_not_zero=False)[0]
194
+ self.is_git_or_bare_git_repository(repository_folder)
195
+ result = self.run_program_argsasarray("git", ["merge-base", "--is-ancestor", ancestor, descendant], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
196
+ exit_code = result[0]
186
197
  if exit_code == 0:
187
198
  return True
188
199
  elif exit_code == 1:
189
200
  return False
190
201
  else:
191
- raise ValueError(f"Can not calculate if {ancestor} is an ancestor of {descendant} in repository {repository_folder}.")
202
+ raise ValueError(f'Can not calculate if {ancestor} is an ancestor of {descendant} in repository {repository_folder}. Outout of "{repository_folder}> git merge-base --is-ancestor {ancestor} {descendant}": Exitcode: {exit_code}; StdOut: {result[1]}; StdErr: {result[2]}.')
192
203
 
193
204
  @GeneralUtilities.check_arguments
194
205
  def __git_changes_helper(self, repository_folder: str, arguments_as_array: list[str]) -> bool:
195
- lines = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", arguments_as_array, repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
206
+ self.assert_is_git_repository(repository_folder)
207
+ lines = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", arguments_as_array, repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1], False)
196
208
  for line in lines:
197
209
  if GeneralUtilities.string_has_content(line):
198
210
  return True
199
211
  return False
200
212
 
201
213
  @GeneralUtilities.check_arguments
202
- def git_repository_has_new_untracked_files(self, repositoryFolder: str):
203
- return self.__git_changes_helper(repositoryFolder, ["ls-files", "--exclude-standard", "--others"])
214
+ def git_repository_has_new_untracked_files(self, repository_folder: str):
215
+ self.assert_is_git_repository(repository_folder)
216
+ return self.__git_changes_helper(repository_folder, ["ls-files", "--exclude-standard", "--others"])
204
217
 
205
218
  @GeneralUtilities.check_arguments
206
- def git_repository_has_unstaged_changes_of_tracked_files(self, repositoryFolder: str):
207
- return self.__git_changes_helper(repositoryFolder, ["diff"])
219
+ def git_repository_has_unstaged_changes_of_tracked_files(self, repository_folder: str):
220
+ self.assert_is_git_repository(repository_folder)
221
+ return self.__git_changes_helper(repository_folder, ["--no-pager", "diff"])
208
222
 
209
223
  @GeneralUtilities.check_arguments
210
- def git_repository_has_staged_changes(self, repositoryFolder: str):
211
- return self.__git_changes_helper(repositoryFolder, ["diff", "--cached"])
224
+ def git_repository_has_staged_changes(self, repository_folder: str):
225
+ self.assert_is_git_repository(repository_folder)
226
+ return self.__git_changes_helper(repository_folder, ["--no-pager", "diff", "--cached"])
212
227
 
213
228
  @GeneralUtilities.check_arguments
214
- def git_repository_has_uncommitted_changes(self, repositoryFolder: str) -> bool:
215
- if (self.git_repository_has_unstaged_changes(repositoryFolder)):
229
+ def git_repository_has_uncommitted_changes(self, repository_folder: str) -> bool:
230
+ self.assert_is_git_repository(repository_folder)
231
+ if (self.git_repository_has_unstaged_changes(repository_folder)):
216
232
  return True
217
- if (self.git_repository_has_staged_changes(repositoryFolder)):
233
+ if (self.git_repository_has_staged_changes(repository_folder)):
218
234
  return True
219
235
  return False
220
236
 
221
237
  @GeneralUtilities.check_arguments
222
238
  def git_repository_has_unstaged_changes(self, repository_folder: str) -> bool:
239
+ self.assert_is_git_repository(repository_folder)
223
240
  if (self.git_repository_has_unstaged_changes_of_tracked_files(repository_folder)):
224
241
  return True
225
242
  if (self.git_repository_has_new_untracked_files(repository_folder)):
@@ -228,47 +245,73 @@ class ScriptCollectionCore:
228
245
 
229
246
  @GeneralUtilities.check_arguments
230
247
  def git_get_commit_id(self, repository_folder: str, commit: str = "HEAD") -> str:
231
- result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["rev-parse", "--verify", commit], repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
248
+ self.is_git_or_bare_git_repository(repository_folder)
249
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["rev-parse", "--verify", commit], repository_folder, throw_exception_if_exitcode_is_not_zero=True)
232
250
  return result[1].replace('\n', '')
233
251
 
234
252
  @GeneralUtilities.check_arguments
235
253
  def git_get_commit_date(self, repository_folder: str, commit: str = "HEAD") -> datetime:
236
- result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["show", "-s", "--format=%ci", commit], repository_folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
254
+ self.is_git_or_bare_git_repository(repository_folder)
255
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["show", "-s", "--format=%ci", commit], repository_folder, throw_exception_if_exitcode_is_not_zero=True)
237
256
  date_as_string = result[1].replace('\n', '')
238
257
  result = datetime.strptime(date_as_string, '%Y-%m-%d %H:%M:%S %z')
239
258
  return result
240
259
 
260
+ @GeneralUtilities.check_arguments
261
+ def git_fetch_with_retry(self, folder: str, remotename: str = "--all", amount_of_attempts: int = 5) -> None:
262
+ GeneralUtilities.retry_action(lambda: self.git_fetch(folder, remotename), amount_of_attempts)
263
+
241
264
  @GeneralUtilities.check_arguments
242
265
  def git_fetch(self, folder: str, remotename: str = "--all") -> None:
243
- self.run_program_argsasarray("git", ["fetch", remotename, "--tags", "--prune"], folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
266
+ self.is_git_or_bare_git_repository(folder)
267
+ self.run_program_argsasarray("git", ["fetch", remotename, "--tags", "--prune"], folder, throw_exception_if_exitcode_is_not_zero=True)
244
268
 
245
269
  @GeneralUtilities.check_arguments
246
270
  def git_fetch_in_bare_repository(self, folder: str, remotename, localbranch: str, remotebranch: str) -> None:
247
- self.run_program_argsasarray("git", ["fetch", remotename, f"{remotebranch}:{localbranch}"], folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
271
+ self.is_git_or_bare_git_repository(folder)
272
+ self.run_program_argsasarray("git", ["fetch", remotename, f"{remotebranch}:{localbranch}"], folder, throw_exception_if_exitcode_is_not_zero=True)
248
273
 
249
274
  @GeneralUtilities.check_arguments
250
275
  def git_remove_branch(self, folder: str, branchname: str) -> None:
251
- self.run_program("git", f"branch -D {branchname}", folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
276
+ self.is_git_or_bare_git_repository(folder)
277
+ self.run_program("git", f"branch -D {branchname}", folder, throw_exception_if_exitcode_is_not_zero=True)
252
278
 
253
279
  @GeneralUtilities.check_arguments
254
- def git_push(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: int = 0) -> None:
255
- argument = ["push", "--recurse-submodules=on-demand", remotename, f"{localbranchname}:{remotebranchname}"]
280
+ def git_push_with_retry(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: LogLevel = LogLevel.Quiet, amount_of_attempts: int = 5) -> None:
281
+ GeneralUtilities.retry_action(lambda: self.git_push(folder, remotename, localbranchname, remotebranchname, forcepush, pushalltags, verbosity), amount_of_attempts)
282
+
283
+ @GeneralUtilities.check_arguments
284
+ def git_push(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: LogLevel = LogLevel.Quiet,resurse_submodules:bool=False) -> None:
285
+ self.is_git_or_bare_git_repository(folder)
286
+ argument = ["push"]
287
+ if resurse_submodules:
288
+ argument = argument + ["--recurse-submodules=on-demand"]
289
+ argument = argument + [remotename, f"{localbranchname}:{remotebranchname}"]
256
290
  if (forcepush):
257
291
  argument.append("--force")
258
292
  if (pushalltags):
259
293
  argument.append("--tags")
260
- result: tuple[int, str, str, int] = self.run_program_argsasarray("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=verbosity, print_errors_as_information=True)
294
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True, print_errors_as_information=True)
261
295
  return result[1].replace('\r', '').replace('\n', '')
262
296
 
263
297
  @GeneralUtilities.check_arguments
264
- def git_pull(self, folder: str, remote: str, localbranchname: str, remotebranchname: str) -> None:
265
- self.run_program("git", f"pull {remote} {remotebranchname}:{localbranchname}", folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
298
+ def git_pull_with_retry(self, folder: str, remote: str, localbranchname: str, remotebranchname: str, force: bool = False, amount_of_attempts: int = 5) -> None:
299
+ GeneralUtilities.retry_action(lambda: self.git_pull(folder, remote, localbranchname, remotebranchname), amount_of_attempts)
300
+
301
+ @GeneralUtilities.check_arguments
302
+ def git_pull(self, folder: str, remote: str, localbranchname: str, remotebranchname: str, force: bool = False) -> None:
303
+ self.is_git_or_bare_git_repository(folder)
304
+ argument = f"pull {remote} {remotebranchname}:{localbranchname}"
305
+ if force:
306
+ argument = f"{argument} --force"
307
+ self.run_program("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
266
308
 
267
309
  @GeneralUtilities.check_arguments
268
310
  def git_list_remote_branches(self, folder: str, remote: str, fetch: bool) -> list[str]:
311
+ self.is_git_or_bare_git_repository(folder)
269
312
  if fetch:
270
313
  self.git_fetch(folder, remote)
271
- run_program_result = self.run_program("git", f"branch -rl {remote}/*", folder, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
314
+ run_program_result = self.run_program("git", f"branch -rl {remote}/*", folder, throw_exception_if_exitcode_is_not_zero=True)
272
315
  output = GeneralUtilities.string_to_lines(run_program_result[1])
273
316
  result = list[str]()
274
317
  for item in output:
@@ -295,61 +338,72 @@ class ScriptCollectionCore:
295
338
  args.append("--remote-submodules")
296
339
  if mirror:
297
340
  args.append("--mirror")
298
- self.run_program_argsasarray("git", args, os.getcwd(), throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
341
+ self.run_program_argsasarray("git", args, os.getcwd(), throw_exception_if_exitcode_is_not_zero=True)
299
342
 
300
343
  @GeneralUtilities.check_arguments
301
344
  def git_get_all_remote_names(self, directory: str) -> list[str]:
302
- result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
345
+ self.is_git_or_bare_git_repository(directory)
346
+ result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote"], directory, throw_exception_if_exitcode_is_not_zero=True)[1], False)
303
347
  return result
304
348
 
305
349
  @GeneralUtilities.check_arguments
306
350
  def git_get_remote_url(self, directory: str, remote_name: str) -> str:
307
- result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote", "get-url", remote_name], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)[1], False)
351
+ self.is_git_or_bare_git_repository(directory)
352
+ result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote", "get-url", remote_name], directory, throw_exception_if_exitcode_is_not_zero=True)[1], False)
308
353
  return result[0].replace('\n', '')
309
354
 
310
355
  @GeneralUtilities.check_arguments
311
356
  def repository_has_remote_with_specific_name(self, directory: str, remote_name: str) -> bool:
357
+ self.is_git_or_bare_git_repository(directory)
312
358
  return remote_name in self.git_get_all_remote_names(directory)
313
359
 
314
360
  @GeneralUtilities.check_arguments
315
361
  def git_add_or_set_remote_address(self, directory: str, remote_name: str, remote_address: str) -> None:
362
+ self.assert_is_git_repository(directory)
316
363
  if (self.repository_has_remote_with_specific_name(directory, remote_name)):
317
- self.run_program_argsasarray("git", ['remote', 'set-url', 'remote_name', remote_address], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
364
+ self.run_program_argsasarray("git", ['remote', 'set-url', 'remote_name', remote_address], directory, throw_exception_if_exitcode_is_not_zero=True)
318
365
  else:
319
- self.run_program_argsasarray("git", ['remote', 'add', remote_name, remote_address], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
366
+ self.run_program_argsasarray("git", ['remote', 'add', remote_name, remote_address], directory, throw_exception_if_exitcode_is_not_zero=True)
320
367
 
321
368
  @GeneralUtilities.check_arguments
322
369
  def git_stage_all_changes(self, directory: str) -> None:
323
- self.run_program_argsasarray("git", ["add", "-A"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
370
+ self.assert_is_git_repository(directory)
371
+ self.run_program_argsasarray("git", ["add", "-A"], directory, throw_exception_if_exitcode_is_not_zero=True)
324
372
 
325
373
  @GeneralUtilities.check_arguments
326
374
  def git_unstage_all_changes(self, directory: str) -> None:
327
- self.run_program_argsasarray("git", ["reset"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
375
+ self.assert_is_git_repository(directory)
376
+ self.run_program_argsasarray("git", ["reset"], directory, throw_exception_if_exitcode_is_not_zero=True)
377
+ # TODO check if this will also be done for submodules
328
378
 
329
379
  @GeneralUtilities.check_arguments
330
380
  def git_stage_file(self, directory: str, file: str) -> None:
331
- self.run_program_argsasarray("git", ['stage', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
381
+ self.assert_is_git_repository(directory)
382
+ self.run_program_argsasarray("git", ['stage', file], directory, throw_exception_if_exitcode_is_not_zero=True)
332
383
 
333
384
  @GeneralUtilities.check_arguments
334
385
  def git_unstage_file(self, directory: str, file: str) -> None:
335
- self.run_program_argsasarray("git", ['reset', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
386
+ self.assert_is_git_repository(directory)
387
+ self.run_program_argsasarray("git", ['reset', file], directory, throw_exception_if_exitcode_is_not_zero=True)
336
388
 
337
389
  @GeneralUtilities.check_arguments
338
390
  def git_discard_unstaged_changes_of_file(self, directory: str, file: str) -> None:
339
391
  """Caution: This method works really only for 'changed' files yet. So this method does not work properly for new or renamed files."""
340
- self.run_program_argsasarray("git", ['checkout', file], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
392
+ self.assert_is_git_repository(directory)
393
+ self.run_program_argsasarray("git", ['checkout', file], directory, throw_exception_if_exitcode_is_not_zero=True)
341
394
 
342
395
  @GeneralUtilities.check_arguments
343
396
  def git_discard_all_unstaged_changes(self, directory: str) -> None:
344
397
  """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
345
- self.run_program_argsasarray("git", ['clean', '-df'], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
346
- self.run_program_argsasarray("git", ['checkout', '.'], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
398
+ self.assert_is_git_repository(directory)
399
+ self.run_program_argsasarray("git", ['clean', '-df'], directory, throw_exception_if_exitcode_is_not_zero=True)
400
+ self.run_program_argsasarray("git", ['checkout', '.'], directory, throw_exception_if_exitcode_is_not_zero=True)
401
+ # TODO check if this will also be done for submodules
347
402
 
348
403
  @GeneralUtilities.check_arguments
349
- def git_commit(self, directory: str, message: str, author_name: str = None, author_email: str = None, stage_all_changes: bool = True, no_changes_behavior: int = 0) -> str:
350
- # no_changes_behavior=0 => No commit
351
- # no_changes_behavior=1 => Commit anyway
352
- # no_changes_behavior=2 => Exception
404
+ def git_commit(self, directory: str, message: str = "Saved changes.", author_name: str = None, author_email: str = None, stage_all_changes: bool = True, no_changes_behavior: int = 0) -> str:
405
+ """no_changes_behavior=0 => No commit; no_changes_behavior=1 => Commit anyway; no_changes_behavior=2 => Exception"""
406
+ self.assert_is_git_repository(directory)
353
407
  author_name = GeneralUtilities.str_none_safe(author_name).strip()
354
408
  author_email = GeneralUtilities.str_none_safe(author_email).strip()
355
409
  argument = ['commit', '--quiet', '--allow-empty', '--message', message]
@@ -363,10 +417,10 @@ class ScriptCollectionCore:
363
417
  self.git_stage_all_changes(directory)
364
418
  else:
365
419
  if no_changes_behavior == 0:
366
- GeneralUtilities.write_message_to_stdout(f"Commit '{message}' will not be done because there are no changes to commit in repository '{directory}'")
420
+ self.log.log(f"Commit '{message}' will not be done because there are no changes to commit in repository '{directory}'", LogLevel.Debug)
367
421
  do_commit = False
368
422
  elif no_changes_behavior == 1:
369
- GeneralUtilities.write_message_to_stdout(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will be done anyway.")
423
+ self.log.log(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will be done anyway.", LogLevel.Debug)
370
424
  do_commit = True
371
425
  elif no_changes_behavior == 2:
372
426
  raise RuntimeError(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will not be done.")
@@ -374,37 +428,75 @@ class ScriptCollectionCore:
374
428
  raise ValueError(f"Unknown value for no_changes_behavior: {GeneralUtilities.str_none_safe(no_changes_behavior)}")
375
429
 
376
430
  if do_commit:
377
- GeneralUtilities.write_message_to_stdout(f"Commit changes in '{directory}'")
378
- self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
431
+ self.log.log(f"Commit changes in '{directory}'", LogLevel.Information)
432
+ self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True)
379
433
 
380
434
  return self.git_get_commit_id(directory)
435
+
436
+ def search_repository_folder(self,some_file_in_repository:str)->str:
437
+ current_path:str=os.path.dirname(some_file_in_repository)
438
+ enabled:bool=True
439
+ while enabled:
440
+ try:
441
+ current_path=GeneralUtilities.resolve_relative_path("..",current_path)
442
+ if self.is_git_repository(current_path):
443
+ return current_path
444
+ except:
445
+ enabled=False
446
+ raise ValueError(f"Can not find git-repository for folder \"{some_file_in_repository}\".")
447
+
381
448
 
382
449
  @GeneralUtilities.check_arguments
383
450
  def git_create_tag(self, directory: str, target_for_tag: str, tag: str, sign: bool = False, message: str = None) -> None:
451
+ self.is_git_or_bare_git_repository(directory)
384
452
  argument = ["tag", tag, target_for_tag]
385
453
  if sign:
386
454
  if message is None:
387
455
  message = f"Created {target_for_tag}"
388
456
  argument.extend(["-s", '-m', message])
389
- self.run_program_argsasarray(
390
- "git", argument, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
457
+ self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True)
391
458
 
392
459
  @GeneralUtilities.check_arguments
393
460
  def git_delete_tag(self, directory: str, tag: str) -> None:
394
- self.run_program_argsasarray("git", ["tag", "--delete", tag], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
395
-
396
- @GeneralUtilities.check_arguments
397
- def git_checkout(self, directory: str, branch: str) -> None:
398
- self.run_program_argsasarray("git", ["checkout", branch], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
399
- self.run_program_argsasarray("git", ["submodule", "update", "--recursive"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
461
+ self.is_git_or_bare_git_repository(directory)
462
+ self.run_program_argsasarray("git", ["tag", "--delete", tag], directory, throw_exception_if_exitcode_is_not_zero=True)
463
+
464
+ @GeneralUtilities.check_arguments
465
+ def git_checkout(self, directory: str, branch: str, undo_all_changes_after_checkout: bool = True, assert_no_uncommitted_changes: bool = True) -> None:
466
+ self.assert_is_git_repository(directory)
467
+ if assert_no_uncommitted_changes:
468
+ GeneralUtilities.assert_condition(not self.git_repository_has_uncommitted_changes(directory), f"Repository '{directory}' has uncommitted changes.")
469
+ self.run_program_argsasarray("git", ["checkout", branch], directory, throw_exception_if_exitcode_is_not_zero=True)
470
+ self.run_program_argsasarray("git", ["submodule", "update", "--recursive"], directory, throw_exception_if_exitcode_is_not_zero=True)
471
+ if undo_all_changes_after_checkout:
472
+ self.git_undo_all_changes(directory)
473
+
474
+ @GeneralUtilities.check_arguments
475
+ def merge_repository(self, repository_folder: str, remote: str, branch: str, pull_first_if_there_are_no_uncommitted_changes: bool = True):
476
+ if pull_first_if_there_are_no_uncommitted_changes:
477
+ uncommitted_changes = self.git_repository_has_uncommitted_changes(repository_folder)
478
+ if not uncommitted_changes:
479
+ is_pullable: bool = self.git_commit_is_ancestor(repository_folder, branch, f"{remote}/{branch}")
480
+ if is_pullable:
481
+ self.git_pull(repository_folder, remote, branch, branch)
482
+ uncommitted_changes = self.git_repository_has_uncommitted_changes(repository_folder)
483
+ GeneralUtilities.assert_condition(not uncommitted_changes, f"Pulling remote \"{remote}\" in \"{repository_folder}\" caused new uncommitted files.")
484
+ self.git_checkout(repository_folder, branch)
485
+ self.git_commit(repository_folder, "Automatic commit due to merge")
486
+ self.git_fetch(repository_folder, remote)
487
+ self.git_merge(repository_folder, f"{remote}/{branch}", branch)
488
+ self.git_push_with_retry(repository_folder, remote, branch, branch)
489
+ self.git_checkout(repository_folder, branch)
400
490
 
401
491
  @GeneralUtilities.check_arguments
402
492
  def git_merge_abort(self, directory: str) -> None:
403
- self.run_program_argsasarray("git", ["merge", "--abort"], directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
493
+ self.assert_is_git_repository(directory)
494
+ self.run_program_argsasarray("git", ["merge", "--abort"], directory, throw_exception_if_exitcode_is_not_zero=True)
404
495
 
405
496
  @GeneralUtilities.check_arguments
406
- def git_merge(self, directory: str, sourcebranch: str, targetbranch: str, fastforward: bool = True, commit: bool = True, commit_message: str = None) -> str:
407
- self.git_checkout(directory, targetbranch)
497
+ def git_merge(self, directory: str, sourcebranch: str, targetbranch: str, fastforward: bool = True, commit: bool = True, commit_message: str = None, undo_all_changes_after_checkout: bool = True, assert_no_uncommitted_changes: bool = True) -> str:
498
+ self.assert_is_git_repository(directory)
499
+ self.git_checkout(directory, targetbranch, undo_all_changes_after_checkout, assert_no_uncommitted_changes)
408
500
  args = ["merge"]
409
501
  if not commit:
410
502
  args.append("--no-commit")
@@ -414,13 +506,14 @@ class ScriptCollectionCore:
414
506
  args.append("-m")
415
507
  args.append(commit_message)
416
508
  args.append(sourcebranch)
417
- self.run_program_argsasarray(
418
- "git", args, directory, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
509
+ self.run_program_argsasarray("git", args, directory, throw_exception_if_exitcode_is_not_zero=True)
510
+ self.run_program_argsasarray("git", ["submodule", "update"], directory, throw_exception_if_exitcode_is_not_zero=True)
419
511
  return self.git_get_commit_id(directory)
420
512
 
421
513
  @GeneralUtilities.check_arguments
422
514
  def git_undo_all_changes(self, directory: str) -> None:
423
515
  """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
516
+ self.assert_is_git_repository(directory)
424
517
  self.git_unstage_all_changes(directory)
425
518
  self.git_discard_all_unstaged_changes(directory)
426
519
 
@@ -438,22 +531,18 @@ class ScriptCollectionCore:
438
531
  # clone
439
532
  self.git_clone(target_repository, source_repository, include_submodules=True, mirror=True)
440
533
 
441
- def get_git_submodules(self, folder: str) -> list[str]:
442
- e = self.run_program("git", "submodule status", folder)
534
+ def get_git_submodules(self, directory: str) -> list[str]:
535
+ self.is_git_or_bare_git_repository(directory)
536
+ e = self.run_program("git", "submodule status", directory)
443
537
  result = []
444
538
  for submodule_line in GeneralUtilities.string_to_lines(e[1], False, True):
445
539
  result.append(submodule_line.split(' ')[1])
446
540
  return result
447
541
 
448
- @GeneralUtilities.check_arguments
449
- def is_git_repository(self, folder: str) -> bool:
450
- combined = os.path.join(folder, ".git")
451
- # TODO consider check for bare-repositories
452
- return os.path.isdir(combined) or os.path.isfile(combined)
453
-
454
542
  @GeneralUtilities.check_arguments
455
543
  def file_is_git_ignored(self, file_in_repository: str, repositorybasefolder: str) -> None:
456
- exit_code = self.run_program_argsasarray("git", ['check-ignore', file_in_repository], repositorybasefolder, throw_exception_if_exitcode_is_not_zero=False, verbosity=0)[0]
544
+ self.is_git_or_bare_git_repository(repositorybasefolder)
545
+ exit_code = self.run_program_argsasarray("git", ['check-ignore', file_in_repository], repositorybasefolder, throw_exception_if_exitcode_is_not_zero=False)[0]
457
546
  if (exit_code == 0):
458
547
  return True
459
548
  if (exit_code == 1):
@@ -462,34 +551,39 @@ class ScriptCollectionCore:
462
551
 
463
552
  @GeneralUtilities.check_arguments
464
553
  def git_discard_all_changes(self, repository: str) -> None:
465
- self.run_program_argsasarray("git", ["reset", "HEAD", "."], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
466
- self.run_program_argsasarray("git", ["checkout", "."], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
554
+ self.assert_is_git_repository(repository)
555
+ self.run_program_argsasarray("git", ["reset", "HEAD", "."], repository, throw_exception_if_exitcode_is_not_zero=True)
556
+ self.run_program_argsasarray("git", ["checkout", "."], repository, throw_exception_if_exitcode_is_not_zero=True)
467
557
 
468
558
  @GeneralUtilities.check_arguments
469
559
  def git_get_current_branch_name(self, repository: str) -> str:
470
- result = self.run_program_argsasarray("git", ["rev-parse", "--abbrev-ref", "HEAD"], repository, throw_exception_if_exitcode_is_not_zero=True, verbosity=0)
471
- return result[1].replace("\r", "").replace("\n", "")
560
+ self.assert_is_git_repository(repository)
561
+ result = self.run_program_argsasarray("git", ["rev-parse", "--abbrev-ref", "HEAD"], repository, throw_exception_if_exitcode_is_not_zero=True)
562
+ return result[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
472
563
 
473
564
  @GeneralUtilities.check_arguments
474
565
  def git_get_commitid_of_tag(self, repository: str, tag: str) -> str:
475
- stdout = self.run_program_argsasarray("git", ["rev-list", "-n", "1", tag], repository, verbosity=0)
476
- result = stdout[1].replace("\r", "").replace("\n", "")
566
+ self.is_git_or_bare_git_repository(repository)
567
+ stdout = self.run_program_argsasarray("git", ["rev-list", "-n", "1", tag], repository)
568
+ result = stdout[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
477
569
  return result
478
570
 
479
571
  @GeneralUtilities.check_arguments
480
572
  def git_get_tags(self, repository: str) -> list[str]:
481
- tags = [line.replace("\r", "") for line in self.run_program_argsasarray(
573
+ self.is_git_or_bare_git_repository(repository)
574
+ tags = [line.replace("\r", GeneralUtilities.empty_string) for line in self.run_program_argsasarray(
482
575
  "git", ["tag"], repository)[1].split("\n") if len(line) > 0]
483
576
  return tags
484
577
 
485
578
  @GeneralUtilities.check_arguments
486
579
  def git_move_tags_to_another_branch(self, repository: str, tag_source_branch: str, tag_target_branch: str, sign: bool = False, message: str = None) -> None:
580
+ self.is_git_or_bare_git_repository(repository)
487
581
  tags = self.git_get_tags(repository)
488
582
  tags_count = len(tags)
489
583
  counter = 0
490
584
  for tag in tags:
491
585
  counter = counter+1
492
- GeneralUtilities.write_message_to_stdout(f"Process tag {counter}/{tags_count}.")
586
+ self.log.log(f"Process tag {counter}/{tags_count}.", LogLevel.Information)
493
587
  # tag is on source-branch
494
588
  if self.git_commit_is_ancestor(repository, tag, tag_source_branch):
495
589
  commit_id_old = self.git_get_commitid_of_tag(repository, tag)
@@ -504,27 +598,36 @@ class ScriptCollectionCore:
504
598
 
505
599
  @GeneralUtilities.check_arguments
506
600
  def get_current_git_branch_has_tag(self, repository_folder: str) -> bool:
507
- result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder, verbosity=0, throw_exception_if_exitcode_is_not_zero=False)
601
+ self.is_git_or_bare_git_repository(repository_folder)
602
+ result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
508
603
  return result[0] == 0
509
604
 
510
605
  @GeneralUtilities.check_arguments
511
606
  def get_latest_git_tag(self, repository_folder: str) -> str:
512
- result = self.run_program_argsasarray(
513
- "git", ["describe", "--tags", "--abbrev=0"], repository_folder, verbosity=0)
514
- result = result[1].replace("\r", "").replace("\n", "")
607
+ self.is_git_or_bare_git_repository(repository_folder)
608
+ result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder)
609
+ result = result[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
515
610
  return result
516
611
 
517
612
  @GeneralUtilities.check_arguments
518
613
  def get_staged_or_committed_git_ignored_files(self, repository_folder: str) -> list[str]:
519
- tresult = self.run_program_argsasarray("git", ["ls-files", "-i", "-c", "--exclude-standard"], repository_folder, verbosity=0)
520
- tresult = tresult[1].replace("\r", "")
521
- result = [line for line in tresult.split("\n") if len(line) > 0]
614
+ self.assert_is_git_repository(repository_folder)
615
+ temp_result = self.run_program_argsasarray("git", ["ls-files", "-i", "-c", "--exclude-standard"], repository_folder)
616
+ temp_result = temp_result[1].replace("\r", GeneralUtilities.empty_string)
617
+ result = [line for line in temp_result.split("\n") if len(line) > 0]
522
618
  return result
523
619
 
524
620
  @GeneralUtilities.check_arguments
525
621
  def git_repository_has_commits(self, repository_folder: str) -> bool:
622
+ self.assert_is_git_repository(repository_folder)
526
623
  return self.run_program_argsasarray("git", ["rev-parse", "--verify", "HEAD"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)[0] == 0
527
624
 
625
+ @GeneralUtilities.check_arguments
626
+ def run_git_command_in_repository_and_submodules(self, repository_folder: str, arguments: list[str]) -> None:
627
+ self.is_git_or_bare_git_repository(repository_folder)
628
+ self.run_program_argsasarray("git", arguments, repository_folder)
629
+ self.run_program_argsasarray("git", ["submodule", "foreach", "--recursive", "git"]+arguments, repository_folder)
630
+
528
631
  @GeneralUtilities.check_arguments
529
632
  def export_filemetadata(self, folder: str, target_file: str, encoding: str = "utf-8", filter_function=None) -> None:
530
633
  folder = GeneralUtilities.resolve_relative_path_from_current_working_directory(folder)
@@ -576,6 +679,245 @@ class ScriptCollectionCore:
576
679
  for renamed_item, original_name in renamed_items.items():
577
680
  os.rename(renamed_item, original_name)
578
681
 
682
+ @GeneralUtilities.check_arguments
683
+ def is_git_repository(self, folder: str) -> bool:
684
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
685
+ folder=folder.replace("\\","/")
686
+ if folder.endswith("/"):
687
+ folder = folder[:-1]
688
+ if not self.is_folder(folder):
689
+ raise ValueError(f"Folder '{folder}' does not exist.")
690
+ git_folder_path = f"{folder}/.git"
691
+ return self.is_folder(git_folder_path) or self.is_file(git_folder_path)
692
+
693
+ @GeneralUtilities.check_arguments
694
+ def is_bare_git_repository(self, folder: str) -> bool:
695
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
696
+ if folder.endswith("/") or folder.endswith("\\"):
697
+ folder = folder[:-1]
698
+ if not self.is_folder(folder):
699
+ raise ValueError(f"Folder '{folder}' does not exist.")
700
+ return folder.endswith(".git")
701
+
702
+ @GeneralUtilities.check_arguments
703
+ def is_git_or_bare_git_repository(self, folder: str) -> bool:
704
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
705
+ return self.is_git_repository(folder) or self.is_bare_git_repository(folder)
706
+
707
+ @GeneralUtilities.check_arguments
708
+ def assert_is_git_repository(self, folder: str) -> str:
709
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
710
+ GeneralUtilities.assert_condition(self.is_git_repository(folder), f"'{folder}' is not a git-repository.")
711
+
712
+ @GeneralUtilities.check_arguments
713
+ def convert_git_repository_to_bare_repository(self, repository_folder: str):
714
+ repository_folder = repository_folder.replace("\\", "/")
715
+ self.assert_is_git_repository(repository_folder)
716
+ git_folder = repository_folder + "/.git"
717
+ if not self.is_folder(git_folder):
718
+ raise ValueError(f"Converting '{repository_folder}' to a bare repository not possible. The folder '{git_folder}' does not exist. Converting is currently only supported when the git-folder is a direct folder in a repository and not a reference to another location.")
719
+ target_folder: str = repository_folder + ".git"
720
+ GeneralUtilities.ensure_directory_exists(target_folder)
721
+ GeneralUtilities.move_content_of_folder(git_folder, target_folder)
722
+ GeneralUtilities.ensure_directory_does_not_exist(repository_folder)
723
+ self.run_program_argsasarray("git", ["config", "--bool", "core.bare", "true"], target_folder)
724
+
725
+ @GeneralUtilities.check_arguments
726
+ def assert_no_uncommitted_changes(self, repository_folder: str):
727
+ if self.git_repository_has_uncommitted_changes(repository_folder):
728
+ raise ValueError(f"Repository '{repository_folder}' has uncommitted changes.")
729
+
730
+ @GeneralUtilities.check_arguments
731
+ def list_content(self, path: str, include_files: bool, include_folder: bool, printonlynamewithoutpath: bool) -> list[str]:
732
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
733
+ result: list[str] = []
734
+ if self.program_runner.will_be_executed_locally():
735
+ if include_files:
736
+ result = result + GeneralUtilities.get_direct_files_of_folder(path)
737
+ if include_folder:
738
+ result = result + GeneralUtilities.get_direct_folders_of_folder(path)
739
+ else:
740
+ arguments = ["--path", path]
741
+ if not include_files:
742
+ arguments = arguments+["--excludefiles"]
743
+ if not include_folder:
744
+ arguments = arguments+["--excludedirectories"]
745
+ if printonlynamewithoutpath:
746
+ arguments = arguments+["--printonlynamewithoutpath"]
747
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("sclistfoldercontent", arguments)
748
+ if exit_code == 0:
749
+ for line in stdout.split("\n"):
750
+ normalized_line = line.replace("\r", "")
751
+ result.append(normalized_line)
752
+ else:
753
+ raise ValueError(f"Fatal error occurrs while checking whether file '{path}' exists. StdErr: '{stderr}'")
754
+ result = [item for item in result if GeneralUtilities.string_has_nonwhitespace_content(item)]
755
+ return result
756
+
757
+ @GeneralUtilities.check_arguments
758
+ def is_file(self, path: str) -> bool:
759
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
760
+ if self.program_runner.will_be_executed_locally():
761
+ return os.path.isfile(path) # works only locally, but much more performant than always running an external program
762
+ else:
763
+ exit_code, _, stderr, _ = self.run_program_argsasarray("scfileexists", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
764
+ if exit_code == 0:
765
+ return True
766
+ elif exit_code == 1:
767
+ raise ValueError(f"Not calculatable whether file '{path}' exists. StdErr: '{stderr}'")
768
+ elif exit_code == 2:
769
+ return False
770
+ raise ValueError(f"Fatal error occurrs while checking whether file '{path}' exists. StdErr: '{stderr}'")
771
+
772
+ @GeneralUtilities.check_arguments
773
+ def is_folder(self, path: str) -> bool:
774
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
775
+ if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
776
+ return os.path.isdir(path)
777
+ else:
778
+ exit_code, _, stderr, _ = self.run_program_argsasarray("scfolderexists", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
779
+ if exit_code == 0:
780
+ return True
781
+ elif exit_code == 1:
782
+ raise ValueError(f"Not calculatable whether folder '{path}' exists. StdErr: '{stderr}'")
783
+ elif exit_code == 2:
784
+ return False
785
+ raise ValueError(f"Fatal error occurrs while checking whether folder '{path}' exists. StdErr: '{stderr}'")
786
+
787
+ @GeneralUtilities.check_arguments
788
+ def get_file_content(self, path: str, encoding: str = "utf-8") -> str:
789
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
790
+ if self.program_runner.will_be_executed_locally():
791
+ return GeneralUtilities.read_text_from_file(path, encoding)
792
+ else:
793
+ result = self.run_program_argsasarray("scprintfilecontent", ["--path", path, "--encofing", encoding]) # works platform-indepent
794
+ return result[1].replace("\\n", "\n")
795
+
796
+ @GeneralUtilities.check_arguments
797
+ def set_file_content(self, path: str, content: str, encoding: str = "utf-8") -> None:
798
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
799
+ if self.program_runner.will_be_executed_locally():
800
+ GeneralUtilities.write_text_to_file(path, content, encoding)
801
+ else:
802
+ content_bytes = content.encode('utf-8')
803
+ base64_bytes = base64.b64encode(content_bytes)
804
+ base64_string = base64_bytes.decode('utf-8')
805
+ self.run_program_argsasarray("scsetfilecontent", ["--path", path, "--argumentisinbase64", "--content", base64_string]) # works platform-indepent
806
+
807
+ @GeneralUtilities.check_arguments
808
+ def remove(self, path: str) -> None:
809
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
810
+ if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
811
+ if os.path.isdir(path):
812
+ GeneralUtilities.ensure_directory_does_not_exist(path)
813
+ if os.path.isfile(path):
814
+ GeneralUtilities.ensure_file_does_not_exist(path)
815
+ else:
816
+ if self.is_file(path):
817
+ exit_code, _, stderr, _ = self.run_program_argsasarray("scremovefile", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
818
+ if exit_code != 0:
819
+ raise ValueError(f"Fatal error occurrs while removing file '{path}'. StdErr: '{stderr}'")
820
+ if self.is_folder(path):
821
+ exit_code, _, stderr, _ = self.run_program_argsasarray("scremovefolder", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
822
+ if exit_code != 0:
823
+ raise ValueError(f"Fatal error occurrs while removing folder '{path}'. StdErr: '{stderr}'")
824
+
825
+ @GeneralUtilities.check_arguments
826
+ def rename(self, source: str, target: str) -> None:
827
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
828
+ if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
829
+ os.rename(source, target)
830
+ else:
831
+ exit_code, _, stderr, _ = self.run_program_argsasarray("screname", ["--source", source, "--target", target], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
832
+ if exit_code != 0:
833
+ raise ValueError(f"Fatal error occurrs while renaming '{source}' to '{target}'. StdErr: '{stderr}'")
834
+
835
+ @GeneralUtilities.check_arguments
836
+ def copy(self, source: str, target: str) -> None:
837
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
838
+ if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
839
+ if os.path.isfile(target) or os.path.isdir(target):
840
+ raise ValueError(f"Can not copy to '{target}' because the target already exists.")
841
+ if os.path.isfile(source):
842
+ shutil.copyfile(source, target)
843
+ elif os.path.isdir(source):
844
+ GeneralUtilities.ensure_directory_exists(target)
845
+ GeneralUtilities.copy_content_of_folder(source, target)
846
+ else:
847
+ raise ValueError(f"'{source}' can not be copied because the path does not exist.")
848
+ else:
849
+ exit_code, _, stderr, _ = self.run_program_argsasarray("sccopy", ["--source", source, "--target", target], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
850
+ if exit_code != 0:
851
+ raise ValueError(f"Fatal error occurrs while copying '{source}' to '{target}'. StdErr: '{stderr}'")
852
+
853
+ @GeneralUtilities.check_arguments
854
+ def create_file(self, path: str, error_if_already_exists: bool, create_necessary_folder: bool) -> None:
855
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
856
+ if self.program_runner.will_be_executed_locally():
857
+ if not os.path.isabs(path):
858
+ path = os.path.join(os.getcwd(), path)
859
+
860
+ if os.path.isfile(path) and error_if_already_exists:
861
+ raise ValueError(f"File '{path}' already exists.")
862
+
863
+ # TODO maybe it should be checked if there is a folder with the same path which already exists.
864
+
865
+ folder = os.path.dirname(path)
866
+
867
+ if not os.path.isdir(folder):
868
+ if create_necessary_folder:
869
+ GeneralUtilities.ensure_directory_exists(folder) # TODO check if this also create nested folders if required
870
+ else:
871
+ raise ValueError(f"Folder '{folder}' does not exist.")
872
+
873
+ GeneralUtilities.ensure_file_exists(path)
874
+ else:
875
+ arguments = ["--path", path]
876
+
877
+ if error_if_already_exists:
878
+ arguments = arguments+["--errorwhenexists"]
879
+
880
+ if create_necessary_folder:
881
+ arguments = arguments+["--createnecessaryfolder"]
882
+
883
+ exit_code, _, stderr, _ = self.run_program_argsasarray("sccreatefile", arguments, throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
884
+ if exit_code != 0:
885
+ raise ValueError(f"Fatal error occurrs while create file '{path}'. StdErr: '{stderr}'")
886
+
887
+ @GeneralUtilities.check_arguments
888
+ def create_folder(self, path: str, error_if_already_exists: bool, create_necessary_folder: bool) -> None:
889
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
890
+ if self.program_runner.will_be_executed_locally():
891
+ if not os.path.isabs(path):
892
+ path = os.path.join(os.getcwd(), path)
893
+
894
+ if os.path.isdir(path) and error_if_already_exists:
895
+ raise ValueError(f"Folder '{path}' already exists.")
896
+
897
+ # TODO maybe it should be checked if there is a file with the same path which already exists.
898
+
899
+ folder = os.path.dirname(path)
900
+
901
+ if not os.path.isdir(folder):
902
+ if create_necessary_folder:
903
+ GeneralUtilities.ensure_directory_exists(folder) # TODO check if this also create nested folders if required
904
+ else:
905
+ raise ValueError(f"Folder '{folder}' does not exist.")
906
+
907
+ GeneralUtilities.ensure_directory_exists(path)
908
+ else:
909
+ arguments = ["--path", path]
910
+
911
+ if error_if_already_exists:
912
+ arguments = arguments+["--errorwhenexists"]
913
+
914
+ if create_necessary_folder:
915
+ arguments = arguments+["--createnecessaryfolder"]
916
+
917
+ exit_code, _, stderr, _ = self.run_program_argsasarray("sccreatefolder", arguments, throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
918
+ if exit_code != 0:
919
+ raise ValueError(f"Fatal error occurrs while create folder '{path}'. StdErr: '{stderr}'")
920
+
579
921
  @GeneralUtilities.check_arguments
580
922
  def __sort_fmd(self, line: str):
581
923
  splitted: list = line.split(";")
@@ -619,7 +961,7 @@ class ScriptCollectionCore:
619
961
 
620
962
  @GeneralUtilities.check_arguments
621
963
  def __create_thumbnails(self, filename: str, fps: str, folder: str, tempname_for_thumbnails: str) -> list[str]:
622
- argument = ['-i', filename, '-r', str(fps), '-vf', 'scale=-1:120', '-vcodec', 'png', f'{tempname_for_thumbnails}-%002d.png']
964
+ argument = ['-i', filename, '-r', fps, '-vf', 'scale=-1:120', '-vcodec', 'png', f'{tempname_for_thumbnails}-%002d.png']
623
965
  self.run_program_argsasarray("ffmpeg", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
624
966
  files = GeneralUtilities.get_direct_files_of_folder(folder)
625
967
  result: list[str] = []
@@ -636,8 +978,17 @@ class ScriptCollectionCore:
636
978
  def __create_thumbnail(self, outputfilename: str, folder: str, length_in_seconds: float, tempname_for_thumbnails: str, amount_of_images: int) -> None:
637
979
  duration = timedelta(seconds=length_in_seconds)
638
980
  info = GeneralUtilities.timedelta_to_simple_string(duration)
639
- rows: int = 5
640
- columns: int = math.ceil(amount_of_images/rows)
981
+ next_square_number = GeneralUtilities.get_next_square_number(amount_of_images)
982
+ root = math.sqrt(next_square_number)
983
+ rows: int = root # 5
984
+ columns: int = root # math.ceil(amount_of_images/rows)
985
+ argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{rows}x{columns}', f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
986
+ self.run_program_argsasarray("montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
987
+
988
+ @GeneralUtilities.check_arguments
989
+ def __create_thumbnail2(self, outputfilename: str, folder: str, length_in_seconds: float, rows: int, columns: int, tempname_for_thumbnails: str, amount_of_images: int) -> None:
990
+ duration = timedelta(seconds=length_in_seconds)
991
+ info = GeneralUtilities.timedelta_to_simple_string(duration)
641
992
  argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{rows}x{columns}', f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
642
993
  self.run_program_argsasarray("montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
643
994
 
@@ -650,7 +1001,7 @@ class ScriptCollectionCore:
650
1001
  return math.ceil(x * d) / d
651
1002
 
652
1003
  @GeneralUtilities.check_arguments
653
- def generate_thumbnail(self, file: str, frames_per_second: str, tempname_for_thumbnails: str = None, hook=None) -> None:
1004
+ def generate_thumbnail(self, file: str, frames_per_second: float, tempname_for_thumbnails: str = None, hook=None) -> None:
654
1005
  if tempname_for_thumbnails is None:
655
1006
  tempname_for_thumbnails = "t_"+str(uuid.uuid4())
656
1007
 
@@ -661,16 +1012,9 @@ class ScriptCollectionCore:
661
1012
  preview_files: list[str] = []
662
1013
  try:
663
1014
  length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
664
- if (frames_per_second.endswith("fps")):
665
- # frames per second, example: frames_per_second="20fps" => 20 frames per second
666
- frames_per_second = self.__roundup(float(frames_per_second[:-3]), 2)
667
- frames_per_second_as_string = str(frames_per_second)
668
- amounf_of_previewframes = int(math.floor(length_in_seconds*frames_per_second))
669
- else:
670
- # concrete amount of frame, examples: frames_per_second="16" => 16 frames for entire video
671
- amounf_of_previewframes = int(float(frames_per_second))
672
- # self.roundup((amounf_of_previewframes-2)/length_in_seconds, 2)
673
- frames_per_second_as_string = f"{amounf_of_previewframes-2}/{length_in_seconds}"
1015
+ # frames per second, example: frames_per_second="20fps" => 20 frames per second
1016
+ frames_per_second = self.__roundup(float(frames_per_second[:-3]), 2)
1017
+ frames_per_second_as_string = str(frames_per_second)
674
1018
  preview_files = self.__create_thumbnails(filename, frames_per_second_as_string, folder, tempname_for_thumbnails)
675
1019
  if hook is not None:
676
1020
  hook(file, preview_files)
@@ -680,10 +1024,33 @@ class ScriptCollectionCore:
680
1024
  for thumbnail_to_delete in preview_files:
681
1025
  os.remove(thumbnail_to_delete)
682
1026
 
1027
+ @GeneralUtilities.check_arguments
1028
+ def generate_thumbnail_by_amount_of_pictures(self, file: str, amount_of_columns: int, amount_of_rows: int, tempname_for_thumbnails: str = None, hook=None) -> None:
1029
+ if tempname_for_thumbnails is None:
1030
+ tempname_for_thumbnails = "t_"+str(uuid.uuid4())
1031
+
1032
+ file = GeneralUtilities.resolve_relative_path_from_current_working_directory(file)
1033
+ filename = os.path.basename(file)
1034
+ folder = os.path.dirname(file)
1035
+ filename_without_extension = Path(file).stem
1036
+ preview_files: list[str] = []
1037
+ try:
1038
+ length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
1039
+ amounf_of_previewframes = int(amount_of_columns*amount_of_rows)
1040
+ frames_per_second_as_string = f"{amounf_of_previewframes-2}/{length_in_seconds}"
1041
+ preview_files = self.__create_thumbnails(filename, frames_per_second_as_string, folder, tempname_for_thumbnails)
1042
+ if hook is not None:
1043
+ hook(file, preview_files)
1044
+ actual_amounf_of_previewframes = len(preview_files)
1045
+ self.__create_thumbnail2(filename_without_extension, folder, length_in_seconds, amount_of_rows, amount_of_columns, tempname_for_thumbnails, actual_amounf_of_previewframes)
1046
+ finally:
1047
+ for thumbnail_to_delete in preview_files:
1048
+ os.remove(thumbnail_to_delete)
1049
+
683
1050
  @GeneralUtilities.check_arguments
684
1051
  def extract_pdf_pages(self, file: str, from_page: int, to_page: int, outputfile: str) -> None:
685
- pdf_reader = PyPDF2.PdfReader(file)
686
- pdf_writer = PyPDF2.PdfWriter()
1052
+ pdf_reader: PdfReader = PdfReader(file)
1053
+ pdf_writer: PdfWriter = PdfWriter()
687
1054
  start = from_page
688
1055
  end = to_page
689
1056
  while start <= end:
@@ -695,11 +1062,13 @@ class ScriptCollectionCore:
695
1062
  @GeneralUtilities.check_arguments
696
1063
  def merge_pdf_files(self, files: list[str], outputfile: str) -> None:
697
1064
  # TODO add wildcard-option
698
- pdfFileMerger = PyPDF2.PdfFileMerger()
1065
+ pdfFileMerger: PdfWriter = PdfWriter()
699
1066
  for file in files:
700
- pdfFileMerger.append(file.strip())
701
- pdfFileMerger.write(outputfile)
702
- pdfFileMerger.close()
1067
+ with open(file, "rb") as f:
1068
+ pdfFileMerger.append(f)
1069
+ with open(outputfile, "wb") as output:
1070
+ pdfFileMerger.write(output)
1071
+ pdfFileMerger.close()
703
1072
 
704
1073
  @GeneralUtilities.check_arguments
705
1074
  def pdf_to_image(self, file: str, outputfilename_without_extension: str) -> None:
@@ -737,10 +1106,10 @@ class ScriptCollectionCore:
737
1106
  elif (size_string.endswith("gib")):
738
1107
  size = int(size_string[:-3]) * pow(2, 30)
739
1108
  else:
740
- GeneralUtilities.write_message_to_stderr("Wrong format")
1109
+ self.log.log("Wrong format", LogLevel.Error)
741
1110
  return 1
742
1111
  else:
743
- GeneralUtilities.write_message_to_stderr("Wrong format")
1112
+ self.log.log("Wrong format", LogLevel.Error)
744
1113
  return 1
745
1114
  with open(name, "wb") as f:
746
1115
  f.seek(size-1)
@@ -804,7 +1173,7 @@ class ScriptCollectionCore:
804
1173
 
805
1174
  return 0
806
1175
  else:
807
- GeneralUtilities.write_message_to_stdout(f"File '{file}' does not exist")
1176
+ self.log.log(f"File '{file}' does not exist.", LogLevel.Error)
808
1177
  return 1
809
1178
 
810
1179
  @GeneralUtilities.check_arguments
@@ -880,11 +1249,19 @@ class ScriptCollectionCore:
880
1249
  for file in GeneralUtilities.absolute_file_paths(folder):
881
1250
  self.__check_file(file, searchstring)
882
1251
 
1252
+ @GeneralUtilities.check_arguments
1253
+ def get_string_as_qr_code(self,string: str) -> None:
1254
+ qr = qrcode.QRCode()
1255
+ qr.add_data(string)
1256
+ f = io.StringIO()
1257
+ qr.print_ascii(out=f)
1258
+ f.seek(0)
1259
+ return f.read()
1260
+
883
1261
  @GeneralUtilities.check_arguments
884
1262
  def __print_qr_code_by_csv_line(self, displayname: str, website: str, emailaddress: str, key: str, period: str) -> None:
885
1263
  qrcode_content = f"otpauth://totp/{website}:{emailaddress}?secret={key}&issuer={displayname}&period={period}"
886
- GeneralUtilities.write_message_to_stdout(
887
- f"{displayname} ({emailaddress}):")
1264
+ GeneralUtilities.write_message_to_stdout(f"{displayname} ({emailaddress}):")
888
1265
  GeneralUtilities.write_message_to_stdout(qrcode_content)
889
1266
  qr = qrcode.QRCode()
890
1267
  qr.add_data(qrcode_content)
@@ -895,14 +1272,11 @@ class ScriptCollectionCore:
895
1272
 
896
1273
  @GeneralUtilities.check_arguments
897
1274
  def SCShow2FAAsQRCode(self, csvfile: str) -> None:
898
- separator_line = "--------------------------------------------------------"
899
1275
  lines = GeneralUtilities.read_csv_file(csvfile, True)
900
1276
  lines.sort(key=lambda items: ''.join(items).lower())
901
1277
  for line in lines:
902
- GeneralUtilities.write_message_to_stdout(separator_line)
903
- self.__print_qr_code_by_csv_line(
904
- line[0], line[1], line[2], line[3], line[4])
905
- GeneralUtilities.write_message_to_stdout(separator_line)
1278
+ self.__print_qr_code_by_csv_line(line[0], line[1], line[2], line[3], line[4])
1279
+ GeneralUtilities.write_message_to_stdout(GeneralUtilities.get_longline())
906
1280
 
907
1281
  @GeneralUtilities.check_arguments
908
1282
  def SCCalculateBitcoinBlockHash(self, block_version_number: str, previousblockhash: str, transactionsmerkleroot: str, timestamp: str, target: str, nonce: str) -> str:
@@ -935,7 +1309,7 @@ class ScriptCollectionCore:
935
1309
  def __adjust_folder_name(self, folder: str) -> str:
936
1310
  result = os.path.dirname(folder).replace("\\", "/")
937
1311
  if result == "/":
938
- return ""
1312
+ return GeneralUtilities.empty_string
939
1313
  else:
940
1314
  return result
941
1315
 
@@ -1068,15 +1442,15 @@ class ScriptCollectionCore:
1068
1442
 
1069
1443
  @GeneralUtilities.check_arguments
1070
1444
  def get_docker_debian_version(self, image_tag: str) -> str:
1071
- result = ScriptCollectionCore().run_program_argsasarray(
1072
- "docker", ['run', f'debian:{image_tag}', 'bash', '-c', 'apt-get -y update && apt-get -y install lsb-release && lsb_release -cs'])
1073
- result_line = GeneralUtilities.string_to_lines(result[1])[-2]
1445
+ result = ScriptCollectionCore().run_program_argsasarray("docker", ['run', f'debian:{image_tag}', 'bash', '-c', 'apt-get -y update && apt-get -y install lsb-release && lsb_release -cs'])
1446
+ result_line = GeneralUtilities.string_to_lines(result[1])[-1]
1074
1447
  return result_line
1075
1448
 
1076
1449
  @GeneralUtilities.check_arguments
1077
1450
  def get_latest_tor_version_of_debian_repository(self, debian_version: str) -> str:
1078
1451
  package_url: str = f"https://deb.torproject.org/torproject.org/dists/{debian_version}/main/binary-amd64/Packages"
1079
- r = requests.get(package_url, timeout=5)
1452
+ headers = {'Cache-Control': 'no-cache'}
1453
+ r = requests.get(package_url, timeout=5, headers=headers)
1080
1454
  if r.status_code != 200:
1081
1455
  raise ValueError(f"Checking for latest tor package resulted in HTTP-response-code {r.status_code}.")
1082
1456
  lines = GeneralUtilities.string_to_lines(GeneralUtilities.bytes_to_string(r.content))
@@ -1087,12 +1461,13 @@ class ScriptCollectionCore:
1087
1461
  return tor_version
1088
1462
 
1089
1463
  def run_testcases_for_python_project(self, repository_folder: str):
1464
+ self.assert_is_git_repository(repository_folder)
1090
1465
  self.run_program("coverage", "run -m pytest", repository_folder)
1091
1466
  self.run_program("coverage", "xml", repository_folder)
1092
1467
  GeneralUtilities.ensure_directory_exists(os.path.join(repository_folder, "Other/TestCoverage"))
1093
1468
  coveragefile = os.path.join(repository_folder, "Other/TestCoverage/TestCoverage.xml")
1094
1469
  GeneralUtilities.ensure_file_does_not_exist(coveragefile)
1095
- os.rename(os.path.join(repository_folder, "coverage.xml"), coveragefile)
1470
+ os.rename(os.path.join(repository_folder, "coverage.xml"), coveragefile)
1096
1471
 
1097
1472
  @GeneralUtilities.check_arguments
1098
1473
  def get_file_permission(self, file: str) -> str:
@@ -1102,11 +1477,11 @@ class ScriptCollectionCore:
1102
1477
 
1103
1478
  @GeneralUtilities.check_arguments
1104
1479
  def __get_file_permission_helper(self, permissions: str) -> str:
1105
- return str(self.__to_octet(permissions[0:3]))+str(self.__to_octet(permissions[3:6]))+str(self.__to_octet(permissions[6:9]))
1480
+ return str(self.__to_octet(permissions[0:3])) + str(self.__to_octet(permissions[3:6]))+str(self.__to_octet(permissions[6:9]))
1106
1481
 
1107
1482
  @GeneralUtilities.check_arguments
1108
1483
  def __to_octet(self, string: str) -> int:
1109
- return int(self.__to_octet_helper(string[0])+self.__to_octet_helper(string[1])+self.__to_octet_helper(string[2]), 2)
1484
+ return int(self.__to_octet_helper(string[0]) + self.__to_octet_helper(string[1])+self.__to_octet_helper(string[2]), 2)
1110
1485
 
1111
1486
  @GeneralUtilities.check_arguments
1112
1487
  def __to_octet_helper(self, string: str) -> str:
@@ -1138,9 +1513,8 @@ class ScriptCollectionCore:
1138
1513
  ls_result = self.run_program_argsasarray("ls", ["-ld", file_or_folder])
1139
1514
  GeneralUtilities.assert_condition(ls_result[0] == 0, f"'ls -ld {file_or_folder}' resulted in exitcode {str(ls_result[0])}. StdErr: {ls_result[2]}")
1140
1515
  GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(ls_result[1]), f"'ls -ld' of '{file_or_folder}' had an empty output. StdErr: '{ls_result[2]}'")
1141
- GeneralUtilities.write_message_to_stdout(ls_result[1])
1142
1516
  output = ls_result[1]
1143
- result = output.replace("\n", "")
1517
+ result = output.replace("\n", GeneralUtilities.empty_string)
1144
1518
  result = ' '.join(result.split()) # reduce multiple whitespaces to one
1145
1519
  return result
1146
1520
 
@@ -1151,7 +1525,6 @@ class ScriptCollectionCore:
1151
1525
  ls_result = self.run_program_argsasarray("ls", ["-la", file_or_folder])
1152
1526
  GeneralUtilities.assert_condition(ls_result[0] == 0, f"'ls -la {file_or_folder}' resulted in exitcode {str(ls_result[0])}. StdErr: {ls_result[2]}")
1153
1527
  GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(ls_result[1]), f"'ls -la' of '{file_or_folder}' had an empty output. StdErr: '{ls_result[2]}'")
1154
- GeneralUtilities.write_message_to_stdout(ls_result[1])
1155
1528
  output = ls_result[1]
1156
1529
  result = output.split("\n")[3:] # skip the lines with "Total", "." and ".."
1157
1530
  result = [' '.join(line.split()) for line in result] # reduce multiple whitespaces to one
@@ -1182,54 +1555,93 @@ class ScriptCollectionCore:
1182
1555
  # <run programs>
1183
1556
 
1184
1557
  @GeneralUtilities.check_arguments
1185
- def __run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> Popen:
1186
- # Verbosity:
1187
- # 0=Quiet (No output will be printed.)
1188
- # 1=Normal (If the exitcode of the executed program is not 0 then the StdErr will be printed.)
1189
- # 2=Full (Prints StdOut and StdErr of the executed program.)
1190
- # 3=Verbose (Same as "Full" but with some more information.)
1191
-
1192
- if isinstance(self.program_runner, ProgramRunnerEpew):
1193
- custom_argument = CustomEpewArgument(print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, verbosity, arguments_for_log)
1558
+ def __run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> Popen:
1194
1559
  popen: Popen = self.program_runner.run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, custom_argument, interactive)
1195
1560
  return popen
1196
1561
 
1197
1562
  @staticmethod
1198
- def __enqueue_output(file, queue):
1563
+ def __enqueue_output(file: IO, queue: Queue):
1199
1564
  for line in iter(file.readline, ''):
1200
1565
  queue.put(line)
1201
1566
  file.close()
1202
1567
 
1203
1568
  @staticmethod
1204
- def __read_popen_pipes(p: Popen):
1569
+ def __continue_process_reading(pid: int, p: Popen, q_stdout: Queue, q_stderr: Queue, reading_stdout_last_time_resulted_in_exception: bool, reading_stderr_last_time_resulted_in_exception: bool):
1570
+ if p.poll() is None:
1571
+ return True
1572
+
1573
+ # if reading_stdout_last_time_resulted_in_exception and reading_stderr_last_time_resulted_in_exception:
1574
+ # return False
1575
+
1576
+ if not q_stdout.empty():
1577
+ return True
1578
+
1579
+ if not q_stderr.empty():
1580
+ return True
1581
+
1582
+ return False
1583
+
1584
+ @staticmethod
1585
+ def __read_popen_pipes(p: Popen, print_live_output: bool, print_errors_as_information: bool, log: SCLog) -> tuple[list[str], list[str]]:
1586
+ p_id = p.pid
1205
1587
  with ThreadPoolExecutor(2) as pool:
1206
1588
  q_stdout = Queue()
1207
1589
  q_stderr = Queue()
1208
1590
 
1209
1591
  pool.submit(ScriptCollectionCore.__enqueue_output, p.stdout, q_stdout)
1210
1592
  pool.submit(ScriptCollectionCore.__enqueue_output, p.stderr, q_stderr)
1211
- while (p.poll() is None) or (not q_stdout.empty()) or (not q_stderr.empty()):
1212
- time.sleep(0.01)
1213
- out_line = None
1214
- err_line = None
1593
+ reading_stdout_last_time_resulted_in_exception: bool = False
1594
+ reading_stderr_last_time_resulted_in_exception: bool = False
1595
+
1596
+ stdout_result: list[str] = []
1597
+ stderr_result: list[str] = []
1598
+
1599
+ while (ScriptCollectionCore.__continue_process_reading(p_id, p, q_stdout, q_stderr, reading_stdout_last_time_resulted_in_exception, reading_stderr_last_time_resulted_in_exception)):
1215
1600
  try:
1216
- out_line = q_stdout.get_nowait()
1601
+ while not q_stdout.empty():
1602
+ out_line: str = q_stdout.get_nowait()
1603
+ out_line = out_line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
1604
+ if GeneralUtilities.string_has_content(out_line):
1605
+ stdout_result.append(out_line)
1606
+ reading_stdout_last_time_resulted_in_exception = False
1607
+ if print_live_output:
1608
+ loglevel = LogLevel.Information
1609
+ if out_line.startswith("Debug: "):
1610
+ loglevel = LogLevel.Debug
1611
+ out_line = out_line[len("Debug: "):]
1612
+ if out_line.startswith("Diagnostic: "):
1613
+ loglevel = LogLevel.Diagnostic
1614
+ out_line = out_line[len("Diagnostic: "):]
1615
+ log.log(out_line, loglevel)
1217
1616
  except Empty:
1218
- pass
1617
+ reading_stdout_last_time_resulted_in_exception = True
1618
+
1219
1619
  try:
1220
- err_line = q_stderr.get_nowait()
1620
+ while not q_stderr.empty():
1621
+ err_line: str = q_stderr.get_nowait()
1622
+ err_line = err_line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
1623
+ if GeneralUtilities.string_has_content(err_line):
1624
+ stderr_result.append(err_line)
1625
+ reading_stderr_last_time_resulted_in_exception = False
1626
+ if print_live_output:
1627
+ loglevel = LogLevel.Error
1628
+ if err_line.startswith("Warning: "):
1629
+ loglevel = LogLevel.Warning
1630
+ err_line = err_line[len("Warning: "):]
1631
+ if print_errors_as_information: # "errors" in "print_errors_as_information" means: all what is written to std-err
1632
+ loglevel = LogLevel.Information
1633
+ log.log(err_line, loglevel)
1221
1634
  except Empty:
1222
- pass
1635
+ reading_stderr_last_time_resulted_in_exception = True
1223
1636
 
1224
- yield (out_line, err_line)
1637
+ time.sleep(0.01) # this is required to not finish too early
1638
+
1639
+ return (stdout_result, stderr_result)
1225
1640
 
1226
- # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
1227
1641
  @GeneralUtilities.check_arguments
1228
- def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False) -> tuple[int, str, str, int]:
1229
- # verbosity 1: No output will be logged.
1230
- # verbosity 2: If the exitcode of the executed program is not 0 then the StdErr will be logged. This is supposed to be the default verbosity-level.
1231
- # verbosity 3: Logs and prints StdOut and StdErr of the executed program in realtime.
1232
- # verbosity 4: Same as loglevel 3 but with some more overhead-information.
1642
+ def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False) -> tuple[int, str, str, int]:
1643
+ if self.call_program_runner_directly:
1644
+ return self.program_runner.run_program_argsasarray(program, arguments_as_array, working_directory, custom_argument, interactive)
1233
1645
  try:
1234
1646
  arguments_as_str = ' '.join(arguments_as_array)
1235
1647
  mock_loader_result = self.__try_load_mock(program, arguments_as_str, working_directory)
@@ -1241,31 +1653,32 @@ class ScriptCollectionCore:
1241
1653
  if arguments_for_log is None:
1242
1654
  arguments_for_log = arguments_as_array
1243
1655
 
1244
- arguments_for_log_as_string: str = ' '.join(arguments_for_log)
1245
- cmd = f'{working_directory}>{program} {arguments_for_log_as_string}'
1656
+ cmd = f'{working_directory}>{program}'
1657
+ if 0 < len(arguments_for_log):
1658
+ arguments_for_log_as_string: str = ' '.join([f'"{argument_for_log}"' for argument_for_log in arguments_for_log])
1659
+ cmd = f'{cmd} {arguments_for_log_as_string}'
1246
1660
 
1247
1661
  if GeneralUtilities.string_is_none_or_whitespace(title):
1248
1662
  info_for_log = cmd
1249
1663
  else:
1250
1664
  info_for_log = title
1251
1665
 
1252
- if verbosity >= 3:
1253
- GeneralUtilities.write_message_to_stdout(f"Run '{info_for_log}'.")
1254
-
1255
- print_live_output = 1 < verbosity
1666
+ self.log.log(f"Run '{info_for_log}'.", LogLevel.Debug)
1256
1667
 
1257
1668
  exit_code: int = None
1258
- stdout: str = ""
1259
- stderr: str = ""
1669
+ stdout: str = GeneralUtilities.empty_string
1670
+ stderr: str = GeneralUtilities.empty_string
1260
1671
  pid: int = None
1261
1672
 
1262
- with self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive) as process:
1673
+ with self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive) as process:
1263
1674
 
1264
1675
  if log_file is not None:
1265
1676
  GeneralUtilities.ensure_file_exists(log_file)
1266
1677
  pid = process.pid
1267
- for out_line_plain, err_line_plain in ScriptCollectionCore.__read_popen_pipes(process): # see https://stackoverflow.com/a/57084403/3905529
1268
1678
 
1679
+ outputs: tuple[list[str], list[str]] = ScriptCollectionCore.__read_popen_pipes(process, print_live_output, print_errors_as_information, self.log)
1680
+
1681
+ for out_line_plain in outputs[0]:
1269
1682
  if out_line_plain is not None:
1270
1683
  out_line: str = None
1271
1684
  if isinstance(out_line_plain, str):
@@ -1278,14 +1691,13 @@ class ScriptCollectionCore:
1278
1691
  if out_line is not None and GeneralUtilities.string_has_content(out_line):
1279
1692
  if out_line.endswith("\n"):
1280
1693
  out_line = out_line[:-1]
1281
- if print_live_output:
1282
- print(out_line, end='\n', file=sys.stdout, flush=True)
1283
1694
  if 0 < len(stdout):
1284
1695
  stdout = stdout+"\n"
1285
1696
  stdout = stdout+out_line
1286
1697
  if log_file is not None:
1287
1698
  GeneralUtilities.append_line_to_file(log_file, out_line)
1288
1699
 
1700
+ for err_line_plain in outputs[1]:
1289
1701
  if err_line_plain is not None:
1290
1702
  err_line: str = None
1291
1703
  if isinstance(err_line_plain, str):
@@ -1297,8 +1709,6 @@ class ScriptCollectionCore:
1297
1709
  if err_line is not None and GeneralUtilities.string_has_content(err_line):
1298
1710
  if err_line.endswith("\n"):
1299
1711
  err_line = err_line[:-1]
1300
- if print_live_output:
1301
- print(err_line, end='\n', file=sys.stderr, flush=True)
1302
1712
  if 0 < len(stderr):
1303
1713
  stderr = stderr+"\n"
1304
1714
  stderr = stderr+err_line
@@ -1306,34 +1716,49 @@ class ScriptCollectionCore:
1306
1716
  GeneralUtilities.append_line_to_file(log_file, err_line)
1307
1717
 
1308
1718
  exit_code = process.returncode
1719
+ GeneralUtilities.assert_condition(exit_code is not None, f"Exitcode of program-run of '{info_for_log}' is None.")
1720
+
1721
+ result_message = f"Program '{info_for_log}' resulted in exitcode {exit_code}."
1722
+
1723
+ self.log.log(result_message, LogLevel.Debug)
1309
1724
 
1310
1725
  if throw_exception_if_exitcode_is_not_zero and exit_code != 0:
1311
- raise ValueError(f"Program '{working_directory}>{program} {arguments_for_log_as_string}' resulted in exitcode {exit_code}. (StdOut: '{stdout}', StdErr: '{stderr}')")
1726
+ raise ValueError(f"{result_message} (StdOut: '{stdout}', StdErr: '{stderr}')")
1312
1727
 
1313
- GeneralUtilities.assert_condition(exit_code is not None, f"Exitcode of program-run of '{info_for_log}' is None.")
1314
1728
  result = (exit_code, stdout, stderr, pid)
1315
1729
  return result
1316
- except Exception as e:
1317
- raise e
1730
+ except Exception as e:#pylint:disable=unused-variable, try-except-raise
1731
+ raise
1318
1732
 
1319
1733
  # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
1320
1734
  @GeneralUtilities.check_arguments
1321
- def run_program(self, program: str, arguments: str = "", working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False) -> tuple[int, str, str, int]:
1322
- return self.run_program_argsasarray(program, GeneralUtilities.arguments_to_array(arguments), working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive)
1735
+ def run_program_with_retry(self, program: str, arguments: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False, amount_of_attempts: int = 5) -> tuple[int, str, str, int]:
1736
+ return GeneralUtilities.retry_action(lambda: self.run_program(program, arguments, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output), amount_of_attempts)
1737
+
1738
+ # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
1739
+ @GeneralUtilities.check_arguments
1740
+ def run_program(self, program: str, arguments: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False) -> tuple[int, str, str, int]:
1741
+ if self.call_program_runner_directly:
1742
+ return self.program_runner.run_program(program, arguments, working_directory, custom_argument, interactive)
1743
+ return self.run_program_argsasarray(program, GeneralUtilities.arguments_to_array(arguments), working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output)
1323
1744
 
1324
1745
  # Return-values program_runner: Pid
1325
1746
  @GeneralUtilities.check_arguments
1326
- def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
1747
+ def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
1748
+ if self.call_program_runner_directly:
1749
+ return self.program_runner.run_program_argsasarray_async(program, arguments_as_array, working_directory, custom_argument, interactive)
1327
1750
  mock_loader_result = self.__try_load_mock(program, ' '.join(arguments_as_array), working_directory)
1328
1751
  if mock_loader_result[0]:
1329
1752
  return mock_loader_result[1]
1330
- process: Popen = self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
1753
+ process: Popen = self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
1331
1754
  return process.pid
1332
1755
 
1333
1756
  # Return-values program_runner: Pid
1334
1757
  @GeneralUtilities.check_arguments
1335
- def run_program_async(self, program: str, arguments: str = "", working_directory: str = None, verbosity: int = 1, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
1336
- return self.run_program_argsasarray_async(program, GeneralUtilities.arguments_to_array(arguments), working_directory, verbosity, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
1758
+ def run_program_async(self, program: str, arguments: str = "", working_directory: str = None,print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
1759
+ if self.call_program_runner_directly:
1760
+ return self.program_runner.run_program_argsasarray_async(program, arguments, working_directory, custom_argument, interactive)
1761
+ return self.run_program_argsasarray_async(program, GeneralUtilities.arguments_to_array(arguments), working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
1337
1762
 
1338
1763
  @GeneralUtilities.check_arguments
1339
1764
  def __try_load_mock(self, program: str, arguments: str, working_directory: str) -> tuple[bool, tuple[int, str, str, int]]:
@@ -1347,10 +1772,17 @@ class ScriptCollectionCore:
1347
1772
 
1348
1773
  @GeneralUtilities.check_arguments
1349
1774
  def __adapt_workingdirectory(self, workingdirectory: str) -> str:
1775
+ result: str = None
1350
1776
  if workingdirectory is None:
1351
- return os.getcwd()
1777
+ result = os.getcwd()
1352
1778
  else:
1353
- return GeneralUtilities.resolve_relative_path_from_current_working_directory(workingdirectory)
1779
+ if os.path.isabs(workingdirectory):
1780
+ result = workingdirectory
1781
+ else:
1782
+ result = GeneralUtilities.resolve_relative_path_from_current_working_directory(workingdirectory)
1783
+ if not os.path.isdir(result):
1784
+ raise ValueError(f"Working-directory '{workingdirectory}' does not exist.")
1785
+ return result
1354
1786
 
1355
1787
  @GeneralUtilities.check_arguments
1356
1788
  def verify_no_pending_mock_program_calls(self):
@@ -1405,13 +1837,28 @@ class ScriptCollectionCore:
1405
1837
  stderr: str
1406
1838
  pid: int
1407
1839
 
1840
+ @GeneralUtilities.check_arguments
1841
+ def run_with_epew(self, program: str, argument: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False,print_live_output:bool=False,encode_argument_in_base64:bool=False) -> tuple[int, str, str, int]:
1842
+ epew_argument=["-p",program ,"-w", working_directory]
1843
+ if encode_argument_in_base64:
1844
+ if arguments_for_log is None:
1845
+ arguments_for_log=epew_argument+["-a",f"\"{argument}\""]
1846
+ base64_string = base64.b64encode(argument.encode("utf-8")).decode("utf-8")
1847
+ epew_argument=epew_argument+["-a",base64_string,"-b"]
1848
+ else:
1849
+ epew_argument=epew_argument+["-a",argument]
1850
+ if arguments_for_log is None:
1851
+ arguments_for_log=epew_argument
1852
+ return self.run_program_argsasarray("epew", epew_argument, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive,print_live_output=print_live_output)
1853
+
1854
+
1408
1855
  # </run programs>
1409
1856
 
1410
1857
  @GeneralUtilities.check_arguments
1411
- def extract_archive_with_7z(self, unzip_program_file: str, zipfile: str, password: str, output_directory: str) -> None:
1858
+ def extract_archive_with_7z(self, unzip_program_file: str, zip_file: str, password: str, output_directory: str) -> None:
1412
1859
  password_set = not password is None
1413
- file_name = Path(zipfile).name
1414
- file_folder = os.path.dirname(zipfile)
1860
+ file_name = Path(zip_file).name
1861
+ file_folder = os.path.dirname(zip_file)
1415
1862
  argument = "x"
1416
1863
  if password_set:
1417
1864
  argument = f"{argument} -p\"{password}\""
@@ -1426,7 +1873,7 @@ class ScriptCollectionCore:
1426
1873
 
1427
1874
  @GeneralUtilities.check_arguments
1428
1875
  def system_time_equals_internet_time(self, maximal_tolerance_difference: timedelta) -> bool:
1429
- return abs(datetime.now() - self.get_internet_time()) < maximal_tolerance_difference
1876
+ return abs(GeneralUtilities.get_now() - self.get_internet_time()) < maximal_tolerance_difference
1430
1877
 
1431
1878
  @GeneralUtilities.check_arguments
1432
1879
  def system_time_equals_internet_time_with_default_tolerance(self) -> bool:
@@ -1462,6 +1909,7 @@ class ScriptCollectionCore:
1462
1909
 
1463
1910
  @GeneralUtilities.check_arguments
1464
1911
  def get_semver_version_from_gitversion(self, repository_folder: str) -> str:
1912
+ self.assert_is_git_repository(repository_folder)
1465
1913
  if (self.git_repository_has_commits(repository_folder)):
1466
1914
  result = self.get_version_from_gitversion(repository_folder, "MajorMinorPatch")
1467
1915
  if self.git_repository_has_uncommitted_changes(repository_folder):
@@ -1483,8 +1931,8 @@ class ScriptCollectionCore:
1483
1931
  @GeneralUtilities.check_arguments
1484
1932
  def get_version_from_gitversion(self, folder: str, variable: str) -> str:
1485
1933
  # called twice as workaround for issue 1877 in gitversion ( https://github.com/GitTools/GitVersion/issues/1877 )
1486
- result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder, verbosity=0)
1487
- result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder, verbosity=0)
1934
+ result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder)
1935
+ result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder)
1488
1936
  result = GeneralUtilities.strip_new_line_character(result[1])
1489
1937
 
1490
1938
  return result
@@ -1496,7 +1944,7 @@ class ScriptCollectionCore:
1496
1944
  if password is None:
1497
1945
  password = GeneralUtilities.generate_password()
1498
1946
  GeneralUtilities.ensure_directory_exists(folder)
1499
- self.run_program("openssl", f'req -new -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -days {days_until_expire} -nodes -x509 -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={name}/OU={subj_ou} -passout pass:{password} -keyout {name}.key -out {name}.crt', folder)
1947
+ self.run_program_argsasarray("openssl", ['req', '-new', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:prime256v1', '-days', str(days_until_expire), '-nodes', '-x509', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={name}/OU={subj_ou}', '-passout', f'pass:{password}', '-keyout', f'{name}.key', '-out', f'{name}.crt'], folder)
1500
1948
 
1501
1949
  @GeneralUtilities.check_arguments
1502
1950
  def generate_certificate(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
@@ -1505,9 +1953,9 @@ class ScriptCollectionCore:
1505
1953
  if password is None:
1506
1954
  password = GeneralUtilities.generate_password()
1507
1955
  rsa_key_length = 4096
1508
- self.run_program("openssl", f'genrsa -out {filename}.key {rsa_key_length}', folder)
1509
- self.run_program("openssl", f'req -new -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou} -x509 -key {filename}.key -out {filename}.unsigned.crt -days {days_until_expire}', folder)
1510
- self.run_program("openssl", f'pkcs12 -export -out {filename}.selfsigned.pfx -password pass:{password} -inkey {filename}.key -in {filename}.unsigned.crt', folder)
1956
+ self.run_program_argsasarray("openssl", ['genrsa', '-out', f'{filename}.key', f'{rsa_key_length}'], folder)
1957
+ self.run_program_argsasarray("openssl", ['req', '-new', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou}', '-x509', '-key', f'{filename}.key', '-out', f'{filename}.unsigned.crt', '-days', f'{days_until_expire}'], folder)
1958
+ self.run_program_argsasarray("openssl", ['pkcs12', '-export', '-out', f'{filename}.selfsigned.pfx', '-password', f'pass:{password}', '-inkey', f'{filename}.key', '-in', f'{filename}.unsigned.crt'], folder)
1511
1959
  GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.password"), password)
1512
1960
  GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.san.conf"), f"""[ req ]
1513
1961
  default_bits = {rsa_key_length}
@@ -1534,7 +1982,7 @@ DNS = {domain}
1534
1982
 
1535
1983
  @GeneralUtilities.check_arguments
1536
1984
  def generate_certificate_sign_request(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str) -> None:
1537
- self.run_program("openssl", f'req -new -subj /C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou} -key {filename}.key -out {filename}.csr -config {filename}.san.conf', folder)
1985
+ self.run_program_argsasarray("openssl", ['req', '-new', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou}', '-key', f'{filename}.key', f'-out', f'{filename}.csr', f'-config', f'{filename}.san.conf'], folder)
1538
1986
 
1539
1987
  @GeneralUtilities.check_arguments
1540
1988
  def sign_certificate(self, folder: str, ca_folder: str, ca_name: str, domain: str, filename: str, days_until_expire: int = None) -> None:
@@ -1543,11 +1991,12 @@ DNS = {domain}
1543
1991
  ca = os.path.join(ca_folder, ca_name)
1544
1992
  password_file = os.path.join(folder, f"{filename}.password")
1545
1993
  password = GeneralUtilities.read_text_from_file(password_file)
1546
- self.run_program("openssl", f'x509 -req -in {filename}.csr -CA {ca}.crt -CAkey {ca}.key -CAcreateserial -CAserial {ca}.srl -out {filename}.crt -days {days_until_expire} -sha256 -extensions v3_req -extfile {filename}.san.conf', folder)
1547
- self.run_program("openssl", f'pkcs12 -export -out {filename}.pfx -inkey {filename}.key -in {filename}.crt -password pass:{password}', folder)
1994
+ self.run_program_argsasarray("openssl", ['x509', '-req', '-in', f'{filename}.csr', '-CA', f'{ca}.crt', '-CAkey', f'{ca}.key', '-CAcreateserial', '-CAserial', f'{ca}.srl', '-out', f'{filename}.crt', '-days', str(days_until_expire), '-sha256', '-extensions', 'v3_req', '-extfile', f'{filename}.san.conf'], folder)
1995
+ self.run_program_argsasarray("openssl", ['pkcs12', '-export', '-out', f'{filename}.pfx', f'-inkey', f'{filename}.key', '-in', f'{filename}.crt', '-password', f'pass:{password}'], folder)
1548
1996
 
1549
1997
  @GeneralUtilities.check_arguments
1550
- def update_dependencies_of_python_in_requirementstxt_file(self, file: str, verbosity: int):
1998
+ def update_dependencies_of_python_in_requirementstxt_file(self, file: str, ignored_dependencies: list[str]):
1999
+ # TODO consider ignored_dependencies
1551
2000
  lines = GeneralUtilities.read_lines_from_file(file)
1552
2001
  new_lines = []
1553
2002
  for line in lines:
@@ -1565,7 +2014,8 @@ DNS = {domain}
1565
2014
  # (something like "cyclonedx-bom>=2.11.0" for example)
1566
2015
  package = line.split(">")[0]
1567
2016
  operator = ">=" if ">=" in line else ">"
1568
- response = requests.get(f'https://pypi.org/pypi/{package}/json', timeout=5)
2017
+ headers = {'Cache-Control': 'no-cache'}
2018
+ response = requests.get(f'https://pypi.org/pypi/{package}/json', timeout=5, headers=headers)
1569
2019
  latest_version = response.json()['info']['version']
1570
2020
  # TODO update only minor- and patch-version
1571
2021
  # TODO print info if there is a new major-version
@@ -1576,7 +2026,8 @@ DNS = {domain}
1576
2026
  raise ValueError(f'Unexpected line in requirements-file: "{line}"')
1577
2027
 
1578
2028
  @GeneralUtilities.check_arguments
1579
- def update_dependencies_of_python_in_setupcfg_file(self, setup_cfg_file: str, verbosity: int):
2029
+ def update_dependencies_of_python_in_setupcfg_file(self, setup_cfg_file: str, ignored_dependencies: list[str]):
2030
+ # TODO consider ignored_dependencies
1580
2031
  lines = GeneralUtilities.read_lines_from_file(setup_cfg_file)
1581
2032
  new_lines = []
1582
2033
  requirement_parsing_mode = False
@@ -1595,21 +2046,22 @@ DNS = {domain}
1595
2046
  GeneralUtilities.write_lines_to_file(setup_cfg_file, new_lines)
1596
2047
 
1597
2048
  @GeneralUtilities.check_arguments
1598
- def update_dependencies_of_dotnet_project(self, csproj_file: str, verbosity: int, ignored_dependencies: list[str]):
2049
+ def update_dependencies_of_dotnet_project(self, csproj_file: str, ignored_dependencies: list[str]):
1599
2050
  folder = os.path.dirname(csproj_file)
1600
2051
  csproj_filename = os.path.basename(csproj_file)
1601
- GeneralUtilities.write_message_to_stderr(f"Check for updates in {csproj_filename}")
1602
- result = self.run_program("dotnet", f"list {csproj_filename} package --outdated", folder)
1603
- for line in result[1].replace("\r", "").split("\n"):
2052
+ self.log.log(f"Check for updates in {csproj_filename}", LogLevel.Information)
2053
+ result = self.run_program_with_retry("dotnet", f"list {csproj_filename} package --outdated", folder, print_errors_as_information=True)
2054
+ for line in result[1].replace("\r", GeneralUtilities.empty_string).split("\n"):
1604
2055
  # Relevant output-lines are something like " > NJsonSchema 10.7.0 10.7.0 10.9.0"
1605
2056
  if ">" in line:
1606
- package_name = line.replace(">", "").strip().split(" ")[0]
1607
- if not (ignored_dependencies in package_name):
1608
- GeneralUtilities.write_message_to_stderr(f"Update package {package_name}")
1609
- self.run_program("dotnet", f"add {csproj_filename} package {package_name}", folder)
2057
+ package_name = line.replace(">", GeneralUtilities.empty_string).strip().split(" ")[0]
2058
+ if not (package_name in ignored_dependencies):
2059
+ self.log.log(f"Update package {package_name}...", LogLevel.Debug)
2060
+ time.sleep(1.1) # attempt to prevent rate-limit
2061
+ self.run_program_with_retry("dotnet", f"add {csproj_filename} package {package_name}", folder, print_errors_as_information=True)
1610
2062
 
1611
2063
  @GeneralUtilities.check_arguments
1612
- def create_deb_package(self, toolname: str, binary_folder: str, control_file_content: str, deb_output_folder: str, verbosity: int, permission_of_executable_file_as_octet_triple: int) -> None:
2064
+ def create_deb_package(self, toolname: str, binary_folder: str, control_file_content: str, deb_output_folder: str, permission_of_executable_file_as_octet_triple: int) -> None:
1613
2065
 
1614
2066
  # prepare
1615
2067
  GeneralUtilities.ensure_directory_exists(deb_output_folder)
@@ -1672,9 +2124,9 @@ chmod {permission} {link_file}
1672
2124
 
1673
2125
  # create debfile
1674
2126
  deb_filename = f"{toolname}.deb"
1675
- self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/control.tar.gz", "*"], packagecontent_control_folder, verbosity=verbosity)
1676
- self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/data.tar.gz", "*"], packagecontent_data_folder, verbosity=verbosity)
1677
- self.run_program_argsasarray("ar", ["r", deb_filename, "debian-binary", "control.tar.gz", "data.tar.gz"], packagecontent_entireresult_folder, verbosity=verbosity)
2127
+ self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/control.tar.gz", "*"], packagecontent_control_folder)
2128
+ self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/data.tar.gz", "*"], packagecontent_data_folder)
2129
+ self.run_program_argsasarray("ar", ["r", deb_filename, "debian-binary", "control.tar.gz", "data.tar.gz"], packagecontent_entireresult_folder)
1678
2130
  result_file = os.path.join(packagecontent_entireresult_folder, deb_filename)
1679
2131
  shutil.copy(result_file, os.path.join(deb_output_folder, deb_filename))
1680
2132
 
@@ -1683,7 +2135,7 @@ chmod {permission} {link_file}
1683
2135
 
1684
2136
  @GeneralUtilities.check_arguments
1685
2137
  def update_year_in_copyright_tags(self, file: str) -> None:
1686
- current_year = str(datetime.now().year)
2138
+ current_year = str(GeneralUtilities.get_now().year)
1687
2139
  lines = GeneralUtilities.read_lines_from_file(file)
1688
2140
  lines_result = []
1689
2141
  for line in lines:
@@ -1698,31 +2150,28 @@ chmod {permission} {link_file}
1698
2150
 
1699
2151
  @GeneralUtilities.check_arguments
1700
2152
  def update_year_in_first_line_of_file(self, file: str) -> None:
1701
- current_year = str(datetime.now().year)
2153
+ current_year = str(GeneralUtilities.get_now().year)
1702
2154
  lines = GeneralUtilities.read_lines_from_file(file)
1703
2155
  lines[0] = re.sub("\\d\\d\\d\\d", current_year, lines[0])
1704
2156
  GeneralUtilities.write_lines_to_file(file, lines)
1705
2157
 
1706
2158
  @GeneralUtilities.check_arguments
1707
- def get_external_ip(self, proxy: str) -> str:
1708
- information = self.get_externalnetworkinformation_as_json_string(proxy)
2159
+ def get_external_ip_address(self) -> str:
2160
+ information = self.get_externalnetworkinformation_as_json_string()
1709
2161
  parsed = json.loads(information)
1710
- return parsed.ip
2162
+ return parsed["IPAddress"]
1711
2163
 
1712
2164
  @GeneralUtilities.check_arguments
1713
- def get_country_of_external_ip(self, proxy: str) -> str:
1714
- information = self.get_externalnetworkinformation_as_json_string(proxy)
2165
+ def get_country_of_external_ip_address(self) -> str:
2166
+ information = self.get_externalnetworkinformation_as_json_string()
1715
2167
  parsed = json.loads(information)
1716
- return parsed.country
2168
+ return parsed["Country"]
1717
2169
 
1718
2170
  @GeneralUtilities.check_arguments
1719
- def get_externalnetworkinformation_as_json_string(self, proxy: str) -> str:
1720
- proxies = None
1721
- if GeneralUtilities.string_has_content(proxy):
1722
- proxies = {"http": proxy}
1723
- response = requests.get('https://ipinfo.io', proxies=proxies, timeout=5)
1724
- network_information_as_json_string = GeneralUtilities.bytes_to_string(
1725
- response.content)
2171
+ def get_externalnetworkinformation_as_json_string(self,clientinformation_link:str='https://clientinformation.anion327.de') -> str:
2172
+ headers = {'Cache-Control': 'no-cache'}
2173
+ response = requests.get(clientinformation_link, timeout=5, headers=headers)
2174
+ network_information_as_json_string = GeneralUtilities.bytes_to_string(response.content)
1726
2175
  return network_information_as_json_string
1727
2176
 
1728
2177
  @GeneralUtilities.check_arguments
@@ -1757,11 +2206,13 @@ chmod {permission} {link_file}
1757
2206
  """.replace("XDX", "ODO"))
1758
2207
 
1759
2208
  @GeneralUtilities.check_arguments
1760
- def generate_arc42_reference_template(self, repository: str, productname: str = None):
2209
+ def generate_arc42_reference_template(self, repository: str, productname: str = None, subfolder: str = None):
1761
2210
  productname: str
1762
2211
  if productname is None:
1763
2212
  productname = os.path.basename(repository)
1764
- reference_root_folder = f"{repository}/Other/Resources/Reference"
2213
+ if subfolder is None:
2214
+ subfolder = "Other/Reference"
2215
+ reference_root_folder = f"{repository}/{subfolder}"
1765
2216
  reference_content_folder = reference_root_folder + "/Technical"
1766
2217
  if os.path.isdir(reference_root_folder):
1767
2218
  raise ValueError(f"The folder '{reference_root_folder}' does already exist.")
@@ -1770,7 +2221,7 @@ chmod {permission} {link_file}
1770
2221
  main_reference_file = f"{reference_root_folder}/Reference.md"
1771
2222
  GeneralUtilities.ensure_file_exists(main_reference_file)
1772
2223
  GeneralUtilities.write_text_to_file(main_reference_file, f"""# {productname}
1773
-
2224
+
1774
2225
  TXDX add minimal service-description here.
1775
2226
 
1776
2227
  ## Technical documentation
@@ -1780,11 +2231,11 @@ TXDX add minimal service-description here.
1780
2231
 
1781
2232
  TXDX
1782
2233
 
1783
- # Quality goals
2234
+ ## Quality goals
1784
2235
 
1785
- TXDX
2236
+ TXDX
1786
2237
 
1787
- # Stakeholder
2238
+ ## Stakeholder
1788
2239
 
1789
2240
  | Name | How to contact | Reason |
1790
2241
  | ---- | -------------- | ------ |""")
@@ -1803,7 +2254,7 @@ TXDX
1803
2254
 
1804
2255
  ## Scope
1805
2256
 
1806
- TXDX""")
2257
+ TXDX""")
1807
2258
  self.__add_chapter(main_reference_file, reference_content_folder, 4, 'Solution Strategy', """TXDX""")
1808
2259
  self.__add_chapter(main_reference_file, reference_content_folder, 5, 'Building Block View', """TXDX""")
1809
2260
  self.__add_chapter(main_reference_file, reference_content_folder, 6, 'Runtime View', """TXDX""")
@@ -1817,8 +2268,7 @@ TXDX
1817
2268
 
1818
2269
  ## Deployment-proecsses
1819
2270
 
1820
- TXDX
1821
- """)
2271
+ TXDX""")
1822
2272
  self.__add_chapter(main_reference_file, reference_content_folder, 8, 'Crosscutting Concepts', """TXDX""")
1823
2273
  self.__add_chapter(main_reference_file, reference_content_folder, 9, 'Architectural Decisions', """## Decision-board
1824
2274
 
@@ -1861,5 +2311,231 @@ TXDX
1861
2311
  - [Repository](TXDX)
1862
2312
  - [Productive-System](TXDX)
1863
2313
  - [QualityCheck-system](TXDX)
2314
+ """.replace("XDX", "ODO"))
1864
2315
 
1865
- """)
2316
+ @GeneralUtilities.check_arguments
2317
+ def run_with_timeout(self, method, timeout_in_seconds: float) -> bool:
2318
+ # Returns true if the method was terminated due to a timeout
2319
+ # Returns false if the method terminates in the given time
2320
+ p = multiprocessing.Process(target=method)
2321
+ p.start()
2322
+ p.join(timeout_in_seconds)
2323
+ if p.is_alive():
2324
+ p.kill()
2325
+ p.join()
2326
+ return True
2327
+ else:
2328
+ return False
2329
+
2330
+ @GeneralUtilities.check_arguments
2331
+ def ensure_local_docker_network_exists(self, network_name: str) -> None:
2332
+ if not self.local_docker_network_exists(network_name):
2333
+ self.create_local_docker_network(network_name)
2334
+
2335
+ @GeneralUtilities.check_arguments
2336
+ def ensure_local_docker_network_does_not_exist(self, network_name: str) -> None:
2337
+ if self.local_docker_network_exists(network_name):
2338
+ self.remove_local_docker_network(network_name)
2339
+
2340
+ @GeneralUtilities.check_arguments
2341
+ def local_docker_network_exists(self, network_name: str) -> bool:
2342
+ return network_name in self.get_all_local_existing_docker_networks()
2343
+
2344
+ @GeneralUtilities.check_arguments
2345
+ def get_all_local_existing_docker_networks(self) -> list[str]:
2346
+ program_call_result = self.run_program("docker", "network list")
2347
+ std_out = program_call_result[1]
2348
+ std_out_lines = std_out.split("\n")[1:]
2349
+ result: list[str] = []
2350
+ for std_out_line in std_out_lines:
2351
+ normalized_line = ';'.join(std_out_line.split())
2352
+ splitted = normalized_line.split(";")
2353
+ result.append(splitted[1])
2354
+ return result
2355
+
2356
+ @GeneralUtilities.check_arguments
2357
+ def remove_local_docker_network(self, network_name: str) -> None:
2358
+ self.run_program("docker", f"network remove {network_name}")
2359
+
2360
+ @GeneralUtilities.check_arguments
2361
+ def create_local_docker_network(self, network_name: str) -> None:
2362
+ self.run_program("docker", f"network create {network_name}")
2363
+
2364
+ @GeneralUtilities.check_arguments
2365
+ def format_xml_file(self, file: str) -> None:
2366
+ encoding = "utf-8"
2367
+ element = ET.XML(GeneralUtilities.read_text_from_file(file, encoding))
2368
+ ET.indent(element)
2369
+ GeneralUtilities.write_text_to_file(file, ET.tostring(element, encoding="unicode"), encoding)
2370
+
2371
+ @GeneralUtilities.check_arguments
2372
+ def install_requirementstxt_file(self, requirements_txt_file: str):
2373
+ folder: str = os.path.dirname(requirements_txt_file)
2374
+ filename: str = os.path.basename(requirements_txt_file)
2375
+ self.run_program_argsasarray("pip", ["install", "-r", filename], folder)
2376
+
2377
+ @GeneralUtilities.check_arguments
2378
+ def ocr_analysis_of_folder(self, folder: str, serviceaddress: str, extensions: list[str], languages: list[str]) -> list[str]: # Returns a list of changed files due to ocr-analysis.
2379
+ GeneralUtilities.write_message_to_stdout("Starting OCR analysis of folder " + folder)
2380
+ supported_extensions = ['png', 'jpg', 'jpeg', 'tiff', 'bmp', 'gif', 'pdf', 'docx', 'doc', 'xlsx', 'xls', 'pptx', 'ppt']
2381
+ changes_files: list[str] = []
2382
+ if extensions is None:
2383
+ extensions = supported_extensions
2384
+ for file in GeneralUtilities.get_direct_files_of_folder(folder):
2385
+ file_lower = file.lower()
2386
+ for extension in extensions:
2387
+ if file_lower.endswith("."+extension):
2388
+ if self.ocr_analysis_of_file(file, serviceaddress, languages):
2389
+ changes_files.append(file)
2390
+ break
2391
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
2392
+ for file in self.ocr_analysis_of_folder(subfolder, serviceaddress, extensions, languages):
2393
+ changes_files.append(file)
2394
+ return changes_files
2395
+
2396
+ @GeneralUtilities.check_arguments
2397
+ def ocr_analysis_of_file(self, file: str, serviceaddress: str, languages: list[str]) -> bool: # Returns true if the ocr-file was generated or updated. Returns false if the existing ocr-file was not changed.
2398
+ GeneralUtilities.write_message_to_stdout("Do OCR analysis of file " + file)
2399
+ supported_extensions = ['png', 'jpg', 'jpeg', 'tiff', 'bmp', 'webp', 'gif', 'pdf', 'rtf', 'docx', 'doc', 'odt', 'xlsx', 'xls', 'ods', 'pptx', 'ppt', 'odp']
2400
+ for extension in supported_extensions:
2401
+ if file.lower().endswith("."+extension):
2402
+ raise ValueError(f"Extension '{extension}' is not supported. Supported extensions are: {', '.join(supported_extensions)}")
2403
+ target_file = file+".ocr.txt"
2404
+ hash_of_current_file: str = GeneralUtilities. get_sha256_of_file(file)
2405
+ if os.path.isfile(target_file):
2406
+ lines = GeneralUtilities.read_lines_from_file(target_file)
2407
+ previous_hash_of_current_file: str = lines[1].split(":")[1].strip()
2408
+ if hash_of_current_file == previous_hash_of_current_file:
2409
+ return False
2410
+ ocr_content = self.get_ocr_content_of_file(file, serviceaddress, languages)
2411
+ GeneralUtilities.ensure_file_exists(target_file)
2412
+ GeneralUtilities.write_text_to_file(file, f"""Name of file: \"{os.path.basename(file)}\""
2413
+ Hash of file: {hash_of_current_file}
2414
+ OCR-content:
2415
+ \"{ocr_content}\"""")
2416
+ return True
2417
+
2418
+ @GeneralUtilities.check_arguments
2419
+ def get_ocr_content_of_file(self, file: str, serviceaddress: str, languages: list[str]) -> str: # serviceaddress = None means local executable
2420
+ result: str = None
2421
+ extension = Path(file).suffix
2422
+ if serviceaddress is None:
2423
+ program_result = self.run_program_argsasarray("simpleocr", ["--File", file, "--Languages", "+".join(languages)] + languages)
2424
+ result = program_result[1]
2425
+ else:
2426
+ languages_for_url = '%2B'.join(languages)
2427
+ package_url: str = f"https://{serviceaddress}/GetOCRContent?languages={languages_for_url}&fileType={extension}"
2428
+ headers = {'Cache-Control': 'no-cache'}
2429
+ r = requests.put(package_url, timeout=5, headers=headers, data=GeneralUtilities.read_binary_from_file(file))
2430
+ if r.status_code != 200:
2431
+ raise ValueError(f"Checking for latest tor package resulted in HTTP-response-code {r.status_code}.")
2432
+ result = GeneralUtilities.bytes_to_string(r.content)
2433
+ return result
2434
+
2435
+ @GeneralUtilities.check_arguments
2436
+ def ocr_analysis_of_repository(self, folder: str, serviceaddress: str, extensions: list[str], languages: list[str]) -> None:
2437
+ self.assert_is_git_repository(folder)
2438
+ changed_files = self.ocr_analysis_of_folder(folder, serviceaddress, extensions, languages)
2439
+ for changed_ocr_file in changed_files:
2440
+ GeneralUtilities.assert_condition(changed_ocr_file.endswith(".ocr.txt"), f"File '{changed_ocr_file}' is not an OCR-file. It should end with '.ocr.txt'.")
2441
+ base_file = changed_ocr_file[:-len(".ocr.txt")]
2442
+ GeneralUtilities.assert_condition(os.path.isfile(base_file), f"Base file '{base_file}' does not exist. The OCR-file '{changed_ocr_file}' is not valid.")
2443
+ base_file_relative_path = os.path.relpath(base_file, folder)
2444
+ base_file_diff_program_result = self.run_program("git", f"diff --quiet -- \"{base_file_relative_path}\"", folder, throw_exception_if_exitcode_is_not_zero=False)
2445
+ has_staged_changes: bool = None
2446
+ if base_file_diff_program_result[0] == 0:
2447
+ has_staged_changes = False
2448
+ elif base_file_diff_program_result[0] == 1:
2449
+ has_staged_changes = True
2450
+ else:
2451
+ raise RuntimeError(f"Unexpected exit code {base_file_diff_program_result[0]} when checking for staged changes of file '{base_file_relative_path}'.")
2452
+ if has_staged_changes:
2453
+ changed_ocr_file_relative_path = os.path.relpath(changed_ocr_file, folder)
2454
+ self.run_program_argsasarray("git", ["add", changed_ocr_file_relative_path], folder)
2455
+
2456
+ @GeneralUtilities.check_arguments
2457
+ def update_timestamp_in_file(self, target_file: str) -> None:
2458
+ lines = GeneralUtilities.read_lines_from_file(target_file)
2459
+ new_lines = []
2460
+ prefix: str = "# last update: "
2461
+ for line in lines:
2462
+ if line.startswith(prefix):
2463
+ new_lines.append(prefix+GeneralUtilities.datetime_to_string_with_timezone(GeneralUtilities.get_now()))
2464
+ else:
2465
+ new_lines.append(line)
2466
+ GeneralUtilities.write_lines_to_file(target_file, new_lines)
2467
+
2468
+ def do_and_log_task(self, name_of_task: str, task):
2469
+ try:
2470
+ self.log.log(f"Start action \"{name_of_task}\".", LogLevel.Information)
2471
+ result = task()
2472
+ if result is None:
2473
+ result = 0
2474
+ return result
2475
+ except Exception as e:
2476
+ self.log.log_exception(f"Error while running action \"{name_of_task}\".", e, LogLevel.Error)
2477
+ return 1
2478
+ finally:
2479
+ self.log.log(f"Finished action \"{name_of_task}\".", LogLevel.Information)
2480
+
2481
+ def get_lines_of_code_with_default_excluded_patterns(self, repository: str) -> int:
2482
+ return self.get_lines_of_code(repository, self.default_excluded_patterns_for_loc)
2483
+
2484
+ default_excluded_patterns_for_loc: list[str] = [".txt", ".md", ".vscode", "Resources", "Reference", ".gitignore", ".gitattributes", "Other/Metrics"]
2485
+
2486
+ def get_lines_of_code(self, repository: str, excluded_pattern: list[str]) -> int:
2487
+ self.assert_is_git_repository(repository)
2488
+ result: int = 0
2489
+ self.log.log(f"Calculate lines of code in repository '{repository}' with excluded patterns: {', '.join(excluded_pattern)}",LogLevel.Debug)
2490
+ git_response = self.run_program("git", "ls-files", repository)
2491
+ files: list[str] = GeneralUtilities.string_to_lines(git_response[1])
2492
+ for file in files:
2493
+ if os.path.isfile(os.path.join(repository, file)):
2494
+ if self.__is_excluded_by_glob_pattern(file, excluded_pattern):
2495
+ self.log.log(f"File '{file}' is ignored because it matches an excluded pattern.",LogLevel.Diagnostic)
2496
+ else:
2497
+ full_file: str = os.path.join(repository, file)
2498
+ if GeneralUtilities.is_binary_file(full_file):
2499
+ self.log.log(f"File '{file}' is ignored because it is a binary-file.",LogLevel.Diagnostic)
2500
+ else:
2501
+ self.log.log(f"Count lines of file '{file}'.",LogLevel.Diagnostic)
2502
+ length = len(GeneralUtilities.read_nonempty_lines_from_file(full_file))
2503
+ result = result+length
2504
+ else:
2505
+ self.log.log(f"File '{file}' is ignored because it does not exist.",LogLevel.Diagnostic)
2506
+ return result
2507
+
2508
+ def __is_excluded_by_glob_pattern(self, file: str, excluded_patterns: list[str]) -> bool:
2509
+ for pattern in excluded_patterns:
2510
+ if fnmatch.fnmatch(file, f"*{pattern}*"):
2511
+ return True
2512
+ return False
2513
+
2514
+ @GeneralUtilities.check_arguments
2515
+ def create_zip_archive(self, folder:str,zip_file:str) -> None:
2516
+ GeneralUtilities.assert_folder_exists(folder)
2517
+ GeneralUtilities.assert_file_does_not_exist(zip_file)
2518
+ folder = os.path.abspath(folder)
2519
+ with zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED) as zipf:
2520
+ for root, _, files in os.walk(folder):
2521
+ for file in files:
2522
+ file_path = os.path.join(root, file)
2523
+ arcname = os.path.relpath(file_path, start=folder)
2524
+ zipf.write(file_path, arcname)
2525
+
2526
+ @GeneralUtilities.check_arguments
2527
+ def start_local_test_service(self, file: str):
2528
+ example_folder = os.path.dirname(file)
2529
+ docker_compose_file = os.path.join(example_folder, "docker-compose.yml")
2530
+ for service in self.get_services_from_yaml_file(docker_compose_file):
2531
+ self.kill_docker_container(service)
2532
+ example_name = os.path.basename(example_folder)
2533
+ title = f"Test{example_name}"
2534
+ self.run_program("docker", f"compose -p {title.lower()} up --detach", example_folder, title=title)
2535
+
2536
+ @GeneralUtilities.check_arguments
2537
+ def stop_local_test_service(self, file: str):
2538
+ example_folder = os.path.dirname(file)
2539
+ example_name = os.path.basename(example_folder)
2540
+ title = f"Test{example_name}"
2541
+ self.run_program("docker", f"compose -p {title.lower()} down", example_folder, title=title)