scriptcollection 4.2.81__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. ScriptCollection/AnionBuildPlatform.py +199 -0
  2. ScriptCollection/CertificateUpdater.py +149 -0
  3. ScriptCollection/Executables.py +921 -0
  4. ScriptCollection/GeneralUtilities.py +1589 -0
  5. ScriptCollection/HTTPMaintenanceOverheadHelper.py +36 -0
  6. ScriptCollection/OCIImages/AbstractImageHandler.py +38 -0
  7. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py +20 -0
  8. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py +20 -0
  9. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGeneric.py +20 -0
  10. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGenericV.py +20 -0
  11. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabCE.py +20 -0
  12. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabEE.py +20 -0
  13. ScriptCollection/OCIImages/ConcreteImageHandlers/__init__.py +0 -0
  14. ScriptCollection/OCIImages/OCIImageManager.py +190 -0
  15. ScriptCollection/OCIImages/__init__.py +0 -0
  16. ScriptCollection/ProcessesRunner.py +43 -0
  17. ScriptCollection/ProgramRunnerBase.py +47 -0
  18. ScriptCollection/ProgramRunnerMock.py +2 -0
  19. ScriptCollection/ProgramRunnerPopen.py +57 -0
  20. ScriptCollection/ProgramRunnerSudo.py +108 -0
  21. ScriptCollection/Resources/CultureChooser/CultureChooser.js +29 -0
  22. ScriptCollection/Resources/CultureChooser/index.html +15 -0
  23. ScriptCollection/Resources/MaintenanceSite/MaintenanceSite.html +15 -0
  24. ScriptCollection/SCLog.py +115 -0
  25. ScriptCollection/ScriptCollectionCore.py +3485 -0
  26. ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +192 -0
  27. ScriptCollection/TFCPS/Docker/__init__.py +0 -0
  28. ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py +8 -0
  29. ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py +6 -0
  30. ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py +7 -0
  31. ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +547 -0
  32. ScriptCollection/TFCPS/DotNet/__init__.py +0 -0
  33. ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +137 -0
  34. ScriptCollection/TFCPS/Flutter/__init__.py +0 -0
  35. ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py +72 -0
  36. ScriptCollection/TFCPS/Go/__init__.py +0 -0
  37. ScriptCollection/TFCPS/Maven/TFCPS_CodeUnitSpecific_Maven.py +42 -0
  38. ScriptCollection/TFCPS/Maven/__init__.py +0 -0
  39. ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +232 -0
  40. ScriptCollection/TFCPS/NodeJS/__init__.py +0 -0
  41. ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +239 -0
  42. ScriptCollection/TFCPS/Python/__init__.py +0 -0
  43. ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py +42 -0
  44. ScriptCollection/TFCPS/Rust/__init__.py +0 -0
  45. ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +433 -0
  46. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +135 -0
  47. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +301 -0
  48. ScriptCollection/TFCPS/TFCPS_CreateRelease.py +98 -0
  49. ScriptCollection/TFCPS/TFCPS_Generic.py +44 -0
  50. ScriptCollection/TFCPS/TFCPS_MergeToMain.py +128 -0
  51. ScriptCollection/TFCPS/TFCPS_MergeToStable.py +356 -0
  52. ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +48 -0
  53. ScriptCollection/TFCPS/TFCPS_Tools_General.py +1565 -0
  54. ScriptCollection/TFCPS/__init__.py +0 -0
  55. ScriptCollection/__init__.py +0 -0
  56. ScriptCollection/__pycache__/GeneralUtilities.cpython-311.pyc +0 -0
  57. ScriptCollection/__pycache__/__init__.cpython-311.pyc +0 -0
  58. scriptcollection-4.2.81.dist-info/METADATA +169 -0
  59. scriptcollection-4.2.81.dist-info/RECORD +62 -0
  60. scriptcollection-4.2.81.dist-info/WHEEL +5 -0
  61. scriptcollection-4.2.81.dist-info/entry_points.txt +67 -0
  62. scriptcollection-4.2.81.dist-info/top_level.txt +1 -0
@@ -0,0 +1,3485 @@
1
+ from datetime import timedelta, datetime
2
+ from functools import cmp_to_key
3
+ import json
4
+ import binascii
5
+ import filecmp
6
+ import hashlib
7
+ import multiprocessing
8
+ import time
9
+ from io import BytesIO
10
+ import itertools
11
+ import copy
12
+ import zipfile
13
+ import math
14
+ import base64
15
+ import os
16
+ from html.parser import HTMLParser
17
+ from queue import Queue, Empty
18
+ from concurrent.futures import ThreadPoolExecutor
19
+ import xml.etree.ElementTree as ET
20
+ from pathlib import Path
21
+ from subprocess import Popen
22
+ import re
23
+ import shutil
24
+ from typing import IO
25
+ import fnmatch
26
+ import uuid
27
+ import tempfile
28
+ import io
29
+ import requests
30
+ import ntplib
31
+ import yaml
32
+ import qrcode
33
+ import pycdlib
34
+ import send2trash
35
+ from pypdf import PdfReader, PdfWriter
36
+ from .GeneralUtilities import GeneralUtilities,Platform
37
+ from .ProgramRunnerBase import ProgramRunnerBase
38
+ from .ProgramRunnerPopen import ProgramRunnerPopen
39
+ from .SCLog import SCLog, LogLevel
40
+
41
+ version = "4.2.81"
42
+ __version__ = version
43
+
44
+ class VSCodeWorkspaceShellTask:
45
+ label:str
46
+ description:str#nullable
47
+ work_dir:str#nullable
48
+ command:str
49
+ aliases:list[str]
50
+ allow_custom_arguments:bool
51
+
52
+ def __init__(self,label:str,description:str,work_dir:str,command:str,aliases:list[str],allow_custom_arguments:bool):
53
+ GeneralUtilities.assert_not_null(label,"label")
54
+ self.label=label
55
+ self.description=description
56
+ self.work_dir=work_dir
57
+ GeneralUtilities.assert_not_null(command,"command")
58
+ self.command=command
59
+ GeneralUtilities.assert_not_null(aliases,"aliases")
60
+ self.aliases=aliases
61
+ GeneralUtilities.assert_not_null(allow_custom_arguments,"allow_custom_arguments")
62
+ self.allow_custom_arguments=allow_custom_arguments
63
+
64
+
65
+ def serialize_for_vscode(self)->str:
66
+ aliases=",".join([f"\"{GeneralUtilities.escape_json_string_value(alias)}\"" for alias in self.aliases])
67
+
68
+ cwd:str=None
69
+ if self.work_dir is None:
70
+ cwd=GeneralUtilities.empty_string
71
+ else:
72
+ cwd=f"\"cwd\": \"{GeneralUtilities.escape_json_string_value(self.work_dir)}\""
73
+
74
+ desc:str=None
75
+ if self.description is None:
76
+ desc=GeneralUtilities.empty_string
77
+ else:
78
+ desc=f"\"description\": \"{GeneralUtilities.escape_json_string_value(self.description)}\","
79
+
80
+ result=f""" {{
81
+ "label": "{GeneralUtilities.escape_json_string_value(self.label)}",
82
+ "command": "{GeneralUtilities.escape_json_string_value(self.command)}",
83
+ "type": "shell",
84
+ "options": {{
85
+ {cwd}
86
+ }},
87
+ "aliases": [
88
+ {aliases}
89
+ ],
90
+ {desc}
91
+ "allowcustomarguments": {str(self.allow_custom_arguments).lower()}
92
+ }}"""
93
+ return result
94
+
95
+ class VSCodeWorkspaceMariaDBConnection:
96
+ name:str
97
+ previewLimit:int=50
98
+ server:str
99
+ port:int
100
+ database:str
101
+ username:str
102
+ password:str
103
+
104
+ def __init__(self,name:str,server,port,database,username,password):
105
+ GeneralUtilities.assert_not_null(name,"name")
106
+ self.name=name
107
+ GeneralUtilities.assert_not_null(server,"server")
108
+ self.server=server
109
+ GeneralUtilities.assert_not_null(port,"port")
110
+ self.port=port
111
+ GeneralUtilities.assert_not_null(database,"database")
112
+ self.database=database
113
+ GeneralUtilities.assert_not_null(username,"username")
114
+ self.username=username
115
+ GeneralUtilities.assert_not_null(password,"password")
116
+ self.password=password
117
+
118
+ def serialize_for_vscode(self)->str:
119
+ result=f""" {{
120
+ "name": "{GeneralUtilities.escape_json_string_value(self.name)}",
121
+ "mysqlOptions": {{
122
+ "authProtocol": "default",
123
+ "enableSsl": "Disabled"
124
+ }},
125
+ "previewLimit": {self.previewLimit},
126
+ "server": "{GeneralUtilities.escape_json_string_value(self.server)}",
127
+ "port": {self.port},
128
+ "driver": "MySQL",
129
+ "database": "{GeneralUtilities.escape_json_string_value(self.database)}",
130
+ "username": "{GeneralUtilities.escape_json_string_value(self.username)}",
131
+ "password": "{GeneralUtilities.escape_json_string_value(self.password)}"
132
+ }}
133
+ """
134
+ return result
135
+
136
+
137
+ class VSCodeWorkspaceMongoDBConnection:
138
+ name:str
139
+ connection_string:str
140
+
141
+ def __init__(self,name:str,connection_string:str):
142
+ GeneralUtilities.assert_not_null(name,"name")
143
+ self.name=name
144
+ GeneralUtilities.assert_not_null(connection_string,"connection_string")
145
+ self.connection_string=connection_string
146
+
147
+ def serialize_for_vscode(self)->str:
148
+ result=f""" {{
149
+ "name": "{GeneralUtilities.escape_json_string_value(self.name)}",
150
+ "connectionString": "{GeneralUtilities.escape_json_string_value(self.connection_string)}"
151
+ }}
152
+ """
153
+ return result
154
+
155
+ class ScriptCollectionCore:
156
+
157
+ # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
158
+ # Do not change this value for productive environments.
159
+ mock_program_calls: bool = False#TODO remove this variable. When someone want to mock program-calls then the ProgramRunnerMock can be used instead
160
+ # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
161
+ execute_program_really_if_no_mock_call_is_defined: bool = False
162
+ __mocked_program_calls: list = None
163
+ program_runner: ProgramRunnerBase = None
164
+ call_program_runner_directly: bool = None
165
+ log: SCLog = None
166
+
167
+ def __init__(self):
168
+ self.program_runner = ProgramRunnerPopen()
169
+ self.call_program_runner_directly = None
170
+ self.__mocked_program_calls = list[ScriptCollectionCore.__MockProgramCall]()
171
+ self.log = SCLog(None, LogLevel.Warning, False)
172
+
173
+ @staticmethod
174
+ @GeneralUtilities.check_arguments
175
+ def get_scriptcollection_version() -> str:
176
+ return __version__
177
+
178
+ @GeneralUtilities.check_arguments
179
+ def get_scriptcollection_configuration_folder(self)->str:
180
+ user_folder = str(Path.home())
181
+ result = os.path.join(user_folder, ".ScriptCollection")
182
+ result=GeneralUtilities.normalize_path(result)
183
+ GeneralUtilities.ensure_directory_exists(result)
184
+ return result
185
+
186
+ def get_global_cache_folder(self)->str:
187
+ result = os.path.join(self.get_scriptcollection_configuration_folder(), "GlobalCache")
188
+ result=GeneralUtilities.normalize_path(result)
189
+ GeneralUtilities.ensure_directory_exists(result)
190
+ return result
191
+
192
+ def __get_docker_registry_credentials_file(self)->str:
193
+ result=os.path.join(self.get_global_cache_folder(),"RegistryCredentials.csv")
194
+ if not os.path.isfile(result):
195
+ GeneralUtilities.ensure_file_exists(result)
196
+ GeneralUtilities.write_lines_to_file(result,["RegistryName;Username;Password"])
197
+ return result
198
+
199
+ def __load_credentials_if_required_and_available(self,registry_url:str,registry_username:str,registry_password:str)->tuple[str,str]:
200
+ if registry_url.startswith("https://"):
201
+ registry_url=registry_url[len("https://"):]
202
+ if registry_password is None:
203
+ credential_file=self.__get_docker_registry_credentials_file()
204
+ lines=GeneralUtilities.read_nonempty_lines_from_file(credential_file)[1:]
205
+ for line in lines:
206
+ splitted=line.split(";")
207
+ registry=splitted[0]
208
+ username=splitted[1]
209
+ password=splitted[2]
210
+ if registry_url==registry and (registry_username is None or username==registry_username):
211
+ registry_username=username
212
+ registry_password=password
213
+ break
214
+ else:
215
+ GeneralUtilities.assert_not_null(registry_username)
216
+ return (registry_username,registry_password)
217
+
218
+ def __get_docker_registry_credentials(self)->list[tuple[str,str,str]]:
219
+ result=[]
220
+ credential_file=self.__get_docker_registry_credentials_file()
221
+ if os.path.isfile(credential_file):
222
+ lines=GeneralUtilities.read_nonempty_lines_from_file(credential_file)[1:]
223
+ for line in lines:
224
+ splitted=line.split(";")
225
+ registry=splitted[0]
226
+ username=splitted[1]
227
+ password=splitted[2]
228
+ result.append((registry,username,password))
229
+ return result
230
+
231
+ def registry_contains_image(self,registry_url:str,image:str,registry_username:str,registry_password:str)->bool:
232
+ """This function assumes that the registry is a custom deployed docker-registry (see https://hub.docker.com/_/registry )"""
233
+ try:
234
+ if "/" in image:
235
+ image=image.rsplit("/", 1)[-1]
236
+ registry_username,registry_password=self.__load_credentials_if_required_and_available(registry_url,registry_username,registry_password)
237
+ catalog_url = f"{registry_url}/v2/_catalog"
238
+ response = requests.get(catalog_url, auth=(registry_username, registry_password),timeout=20)
239
+ response.raise_for_status() # check if statuscode = 200
240
+ data = response.json()
241
+ # expected: {"repositories": ["nginx", "myapp"]}
242
+ images = data.get("repositories", [])
243
+ if not (image in images):
244
+ return False
245
+
246
+ if self.get_tags_of_images_from_registry(registry_url,image,registry_username,registry_password)<1:
247
+ return False
248
+
249
+ return True
250
+ except Exception:
251
+ return False
252
+
253
+ def docker_platform_to_slug(self,platform_value: Platform) -> str:
254
+ if platform_value == Platform.Linux_AMD64:
255
+ return "linux-amd64"
256
+ elif platform_value == Platform.Linux_ARM64:
257
+ return "linux-arm64"
258
+ raise ValueError(f"Unsupported platform: {platform_value}")
259
+
260
+ @GeneralUtilities.check_arguments
261
+ def add_image_to_custom_docker_image_registry(
262
+ self,
263
+ remote_hub: str,
264
+ imagename_on_remote_hub: str,
265
+ own_registry_address: str,
266
+ imagename_on_own_registry: str,
267
+ tag: str,
268
+ registry_username: str,
269
+ registry_password: str,
270
+ ) -> None:
271
+ registry_username, registry_password = self.__load_credentials_if_required_and_available(remote_hub, registry_username, registry_password)
272
+ source_address = f"{remote_hub}/{imagename_on_remote_hub}:{tag}"
273
+ target_address = f"{own_registry_address}/{imagename_on_own_registry}:{tag}"
274
+ self.run_program("docker", f"buildx imagetools create --tag {target_address} {source_address}")#this does pull and push for each platform
275
+
276
+
277
+ def get_tags_of_images_from_registry(self,registry_base_url:str,image:str,registry_username:str,registry_password:str)->list[str]:
278
+ """registry_base_url must be in the format 'https://myregistry.example.com'
279
+ This function assumes that the registry is a custom deployed docker-registry (see https://hub.docker.com/_/registry )"""
280
+ registry_username,registry_password=self.__load_credentials_if_required_and_available(registry_base_url,registry_username,registry_password)
281
+ if "/" in image:
282
+ image=image.rsplit("/", 1)[-1]
283
+ if not self.registry_contains_image(registry_base_url,image,registry_username,registry_password):
284
+ return []
285
+ tags_url = f"{registry_base_url}/v2/{image}/tags/list"
286
+ response = requests.get(tags_url, auth=(registry_username, registry_password),timeout=20)
287
+ response.raise_for_status() # check if statuscode = 200
288
+ data=response.json()
289
+ # expected: {"name":"myapp","tags":["1.2.22","1.2.21","1.2.20"]}
290
+ tags = data.get("tags", [])
291
+ return tags
292
+
293
+ def registry_contains_image_with_tag(self,registry_url:str,image:str,tag:str,registry_username:str,registry_password:str)->bool:
294
+ """This function assumes that the registry is a custom deployed docker-registry (see https://hub.docker.com/_/registry )"""
295
+ registry_username,registry_password=self.__load_credentials_if_required_and_available(registry_url,registry_username,registry_password)
296
+ if "/" in image:
297
+ image=image.rsplit("/", 1)[-1]
298
+ tags=self.get_tags_of_images_from_registry(registry_url,image,registry_username,registry_password)
299
+ if tags is None:
300
+ return False
301
+ else:
302
+ result = tag in tags
303
+ return result
304
+
305
+ def login_to_defined_docker_registries(self)->None:
306
+ registries=self.__get_docker_registry_credentials()
307
+ if len(registries)==0:
308
+ self.log.log("No docker registry credentials defined. Skipping docker login.",LogLevel.Debug)
309
+ else:
310
+ for registry,username,password in registries:
311
+ arg=f"login {registry} -u {username} -p {password}"
312
+ arg_for_log=f"login {registry} -u {username} -p ***"
313
+ self.run_program("docker",arg,arguments_for_log=arg_for_log,print_live_output=self.log.loglevel==LogLevel.Debug)
314
+
315
+ @GeneralUtilities.check_arguments
316
+ def python_file_has_errors(self, file: str, working_directory: str, treat_warnings_as_errors: bool = True) -> tuple[bool, list[str]]:
317
+ errors = list()
318
+ filename = os.path.relpath(file, working_directory)
319
+ if treat_warnings_as_errors:
320
+ errorsonly_argument = GeneralUtilities.empty_string
321
+ else:
322
+ errorsonly_argument = " --errors-only"
323
+ (exit_code, stdout, stderr, _) = self.run_program("pylint", filename + errorsonly_argument, working_directory, throw_exception_if_exitcode_is_not_zero=False)
324
+ if (exit_code != 0):
325
+ errors.append(f"Linting-issues of {file}:")
326
+ errors.append(f"Pylint-exitcode: {exit_code}")
327
+ for line in GeneralUtilities.string_to_lines(stdout):
328
+ errors.append(line)
329
+ for line in GeneralUtilities.string_to_lines(stderr):
330
+ errors.append(line)
331
+ return (True, errors)
332
+
333
+ return (False, errors)
334
+
335
+ @GeneralUtilities.check_arguments
336
+ def replace_version_in_dockerfile_file(self, dockerfile: str, new_version_value: str) -> None:
337
+ GeneralUtilities.write_text_to_file(dockerfile, re.sub("ARG Version=\"\\d+\\.\\d+\\.\\d+\"", f"ARG Version=\"{new_version_value}\"", GeneralUtilities.read_text_from_file(dockerfile)))
338
+
339
+ @GeneralUtilities.check_arguments
340
+ def replace_version_in_python_file(self, file: str, new_version_value: str):
341
+ GeneralUtilities.write_text_to_file(file, re.sub("version = \"\\d+\\.\\d+\\.\\d+\"", f"version = \"{new_version_value}\"", GeneralUtilities.read_text_from_file(file)))
342
+
343
+ @GeneralUtilities.check_arguments
344
+ def replace_version_in_ini_file(self, file: str, new_version_value: str):
345
+ GeneralUtilities.write_text_to_file(file, re.sub("version = \\d+\\.\\d+\\.\\d+", f"version = {new_version_value}", GeneralUtilities.read_text_from_file(file)))
346
+
347
+ @GeneralUtilities.check_arguments
348
+ def replace_version_in_nuspec_file(self, nuspec_file: str, new_version: str) -> None:
349
+ # TODO use XSLT instead
350
+ versionregex = "\\d+\\.\\d+\\.\\d+"
351
+ versiononlyregex = f"^{versionregex}$"
352
+ pattern = re.compile(versiononlyregex)
353
+ if pattern.match(new_version):
354
+ GeneralUtilities.write_text_to_file(nuspec_file, re.sub(f"<version>{versionregex}<\\/version>", f"<version>{new_version}</version>", GeneralUtilities.read_text_from_file(nuspec_file)))
355
+ else:
356
+ raise ValueError(f"Version '{new_version}' does not match version-regex '{versiononlyregex}'")
357
+
358
+ @GeneralUtilities.check_arguments
359
+ def replace_version_in_csproj_file(self, csproj_file: str, current_version: str):
360
+ versionregex = "\\d+\\.\\d+\\.\\d+"
361
+ versiononlyregex = f"^{versionregex}$"
362
+ pattern = re.compile(versiononlyregex)
363
+ if pattern.match(current_version):
364
+ for tag in ["Version", "AssemblyVersion", "FileVersion"]:
365
+ GeneralUtilities.write_text_to_file(csproj_file, re.sub(f"<{tag}>{versionregex}(.\\d+)?<\\/{tag}>", f"<{tag}>{current_version}</{tag}>", GeneralUtilities.read_text_from_file(csproj_file)))
366
+ else:
367
+ raise ValueError(f"Version '{current_version}' does not match version-regex '{versiononlyregex}'")
368
+
369
+ @GeneralUtilities.check_arguments
370
+ def push_nuget_build_artifact(self, nupkg_file: str, registry_address: str, api_key: str = None):
371
+ nupkg_file_name = os.path.basename(nupkg_file)
372
+ nupkg_file_folder = os.path.dirname(nupkg_file)
373
+ argument = f"nuget push {nupkg_file_name} --force-english-output --source {registry_address}"
374
+ if api_key is not None:
375
+ argument = f"{argument} --api-key {api_key}"
376
+ self.run_program("dotnet", argument, nupkg_file_folder)
377
+
378
+ @GeneralUtilities.check_arguments
379
+ def dotnet_build(self, folder: str, projectname: str, configuration: str):
380
+ self.run_program("dotnet", f"clean -c {configuration}", folder)
381
+ self.run_program("dotnet", f"build {projectname}/{projectname}.csproj -c {configuration}", folder)
382
+
383
+ @GeneralUtilities.check_arguments
384
+ def find_file_by_extension(self, folder: str, extension_without_dot: str):
385
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
386
+ result = [file for file in self.list_content(folder, True, False, False) if file.endswith(f".{extension_without_dot}")]
387
+ result_length = len(result)
388
+ if result_length == 0:
389
+ raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension_without_dot}'.")
390
+ if result_length == 1:
391
+ return result[0]
392
+ else:
393
+ raise ValueError(f"Multiple values available in folder '{folder}' with extension '{extension_without_dot}'.")
394
+
395
+ @GeneralUtilities.check_arguments
396
+ def find_last_file_by_extension(self, folder: str, extension_without_dot: str) -> str:
397
+ files: list[str] = GeneralUtilities.get_direct_files_of_folder(folder)
398
+ possible_results: list[str] = []
399
+ for file in files:
400
+ if file.endswith(f".{extension_without_dot}"):
401
+ possible_results.append(file)
402
+ result_length = len(possible_results)
403
+ if result_length == 0:
404
+ raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension_without_dot}'.")
405
+ else:
406
+ return possible_results[-1]
407
+
408
+ @GeneralUtilities.check_arguments
409
+ def commit_is_signed_by_key(self, repository_folder: str, revision_identifier: str, key: str) -> bool:
410
+ self.is_git_or_bare_git_repository(repository_folder)
411
+ result = self.run_program("git", f"verify-commit {revision_identifier}", repository_folder, throw_exception_if_exitcode_is_not_zero=False)
412
+ if (result[0] != 0):
413
+ return False
414
+ if (not GeneralUtilities.contains_line(result[1].splitlines(), f"gpg\\:\\ using\\ [A-Za-z0-9]+\\ key\\ [A-Za-z0-9]+{key}")):
415
+ # TODO check whether this works on machines where gpg is installed in another langauge than english
416
+ return False
417
+ if (not GeneralUtilities.contains_line(result[1].splitlines(), "gpg\\:\\ Good\\ signature\\ from")):
418
+ # TODO check whether this works on machines where gpg is installed in another langauge than english
419
+ return False
420
+ return True
421
+
422
+ @GeneralUtilities.check_arguments
423
+ def get_parent_commit_ids_of_commit(self, repository_folder: str, commit_id: str) -> str:
424
+ self.is_git_or_bare_git_repository(repository_folder)
425
+ return self.run_program("git", f'log --pretty=%P -n 1 "{commit_id}"', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string).split(" ")
426
+
427
+
428
+ @GeneralUtilities.check_arguments
429
+ def get_commit_ids_between_dates(self, repository_folder: str, since: datetime, until: datetime, ignore_commits_which_are_not_in_history_of_head: bool = True) -> None:
430
+ self.is_git_or_bare_git_repository(repository_folder)
431
+ since_as_string = self.__datetime_to_string_for_git(since)
432
+ until_as_string = self.__datetime_to_string_for_git(until)
433
+ result = filter(lambda line: not GeneralUtilities.string_is_none_or_whitespace(line), self.run_program("git", f'log --since "{since_as_string}" --until "{until_as_string}" --pretty=format:"%H" --no-patch', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].split("\n").replace("\r", GeneralUtilities.empty_string))
434
+ if ignore_commits_which_are_not_in_history_of_head:
435
+ result = [commit_id for commit_id in result if self.git_commit_is_ancestor(repository_folder, commit_id)]
436
+ return result
437
+
438
+ @GeneralUtilities.check_arguments
439
+ def __datetime_to_string_for_git(self, datetime_object: datetime) -> str:
440
+ return datetime_object.strftime('%Y-%m-%d %H:%M:%S')
441
+
442
+ @GeneralUtilities.check_arguments
443
+ def git_commit_is_ancestor(self, repository_folder: str, ancestor: str, descendant: str = "HEAD") -> bool:
444
+ self.is_git_or_bare_git_repository(repository_folder)
445
+ result = self.run_program_argsasarray("git", ["merge-base", "--is-ancestor", ancestor, descendant], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
446
+ exit_code = result[0]
447
+ if exit_code == 0:
448
+ return True
449
+ elif exit_code == 1:
450
+ return False
451
+ else:
452
+ raise ValueError(f'Can not calculate if {ancestor} is an ancestor of {descendant} in repository {repository_folder}. Outout of "{repository_folder}> git merge-base --is-ancestor {ancestor} {descendant}": Exitcode: {exit_code}; StdOut: {result[1]}; StdErr: {result[2]}.')
453
+
454
+ @GeneralUtilities.check_arguments
455
+ def __git_changes_helper(self, repository_folder: str, arguments_as_array: list[str]) -> bool:
456
+ self.assert_is_git_repository(repository_folder)
457
+ lines = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", arguments_as_array, repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1], False)
458
+ for line in lines:
459
+ if GeneralUtilities.string_has_content(line):
460
+ return True
461
+ return False
462
+
463
+ @GeneralUtilities.check_arguments
464
+ def git_repository_has_new_untracked_files(self, repository_folder: str):
465
+ self.assert_is_git_repository(repository_folder)
466
+ return self.__git_changes_helper(repository_folder, ["ls-files", "--exclude-standard", "--others"])
467
+
468
+ @GeneralUtilities.check_arguments
469
+ def git_repository_has_unstaged_changes_of_tracked_files(self, repository_folder: str):
470
+ self.assert_is_git_repository(repository_folder)
471
+ return self.__git_changes_helper(repository_folder, ["--no-pager", "diff"])
472
+
473
+ @GeneralUtilities.check_arguments
474
+ def git_repository_has_staged_changes(self, repository_folder: str):
475
+ self.assert_is_git_repository(repository_folder)
476
+ return self.__git_changes_helper(repository_folder, ["--no-pager", "diff", "--cached"])
477
+
478
+ @GeneralUtilities.check_arguments
479
+ def git_repository_has_uncommitted_changes(self, repository_folder: str) -> bool:
480
+ self.assert_is_git_repository(repository_folder)
481
+ if (self.git_repository_has_unstaged_changes(repository_folder)):
482
+ return True
483
+ if (self.git_repository_has_staged_changes(repository_folder)):
484
+ return True
485
+ return False
486
+
487
+ @GeneralUtilities.check_arguments
488
+ def git_repository_has_unstaged_changes(self, repository_folder: str) -> bool:
489
+ self.assert_is_git_repository(repository_folder)
490
+ if (self.git_repository_has_unstaged_changes_of_tracked_files(repository_folder)):
491
+ return True
492
+ if (self.git_repository_has_new_untracked_files(repository_folder)):
493
+ return True
494
+ return False
495
+
496
+ @GeneralUtilities.check_arguments
497
+ def git_get_commit_id(self, repository_folder: str, rev: str = "HEAD") -> str:
498
+ self.is_git_or_bare_git_repository(repository_folder)
499
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["rev-parse", "--verify", rev], repository_folder, throw_exception_if_exitcode_is_not_zero=True)
500
+ return result[1].replace('\n', '')
501
+
502
+ @GeneralUtilities.check_arguments
503
+ def git_get_commit_date(self, repository_folder: str, rev: str = "HEAD") -> datetime:
504
+ self.is_git_or_bare_git_repository(repository_folder)
505
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["log","-1","--format=%ci", rev], repository_folder, throw_exception_if_exitcode_is_not_zero=True)
506
+ date_as_string = result[1].replace('\n', '')
507
+ result = datetime.strptime(date_as_string, '%Y-%m-%d %H:%M:%S %z')
508
+ return result
509
+
510
+ @GeneralUtilities.check_arguments
511
+ def git_fetch_with_retry(self, folder: str, remotename: str = "--all", amount_of_attempts: int = 5) -> None:
512
+ GeneralUtilities.retry_action(lambda: self.git_fetch(folder, remotename), amount_of_attempts)
513
+
514
+ @GeneralUtilities.check_arguments
515
+ def git_fetch(self, folder: str, remotename: str = "--all") -> None:
516
+ self.assert_is_git_repository(folder)
517
+ self.run_program_argsasarray("git", ["fetch", remotename, "--tags", "--prune"], folder, throw_exception_if_exitcode_is_not_zero=True)
518
+
519
+ @GeneralUtilities.check_arguments
520
+ def git_fetch_in_bare_repository(self, folder: str, remotename, localbranch: str, remotebranch: str) -> None:
521
+ self.assert_is_git_repository(folder)
522
+ self.run_program_argsasarray("git", ["fetch", remotename, f"{remotebranch}:{localbranch}"], folder, throw_exception_if_exitcode_is_not_zero=True)
523
+
524
+ def branch_exists(self, folder: str, branchname: str) -> bool:
525
+ self.assert_is_git_repository(folder)
526
+ result = self.run_program_argsasarray("git", ["rev-parse", "--verify", branchname], folder, throw_exception_if_exitcode_is_not_zero=False)
527
+ return result[0] == 0
528
+
529
+ @GeneralUtilities.check_arguments
530
+ def git_remove_branch(self, folder: str, branchname: str) -> None:
531
+ self.assert_is_git_repository(folder)
532
+ self.run_program("git", f"branch -D {branchname}", folder, throw_exception_if_exitcode_is_not_zero=True)
533
+
534
+ @GeneralUtilities.check_arguments
535
+ def git_push_with_retry(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: LogLevel = LogLevel.Quiet, amount_of_attempts: int = 5) -> None:
536
+ GeneralUtilities.retry_action(lambda: self.git_push(folder, remotename, localbranchname, remotebranchname, forcepush, pushalltags, verbosity), amount_of_attempts)
537
+
538
+ @GeneralUtilities.check_arguments
539
+ def git_push(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: LogLevel = LogLevel.Quiet,resurse_submodules:bool=False) -> None:
540
+ self.is_git_or_bare_git_repository(folder)
541
+ argument = ["push"]
542
+ if resurse_submodules:
543
+ argument = argument + ["--recurse-submodules=on-demand"]
544
+ argument = argument + [remotename, f"{localbranchname}:{remotebranchname}"]
545
+ if (forcepush):
546
+ argument.append("--force")
547
+ if (pushalltags):
548
+ argument.append("--tags")
549
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True, print_errors_as_information=True)
550
+ return result[1].replace('\r', '').replace('\n', '')
551
+
552
+ @GeneralUtilities.check_arguments
553
+ def git_pull_with_retry(self, folder: str, remote: str, localbranchname: str, remotebranchname: str, force: bool = False, amount_of_attempts: int = 5) -> None:
554
+ GeneralUtilities.retry_action(lambda: self.git_pull(folder, remote, localbranchname, remotebranchname), amount_of_attempts)
555
+
556
+ @GeneralUtilities.check_arguments
557
+ def git_pull(self, folder: str, remote: str, localbranchname: str, remotebranchname: str, force: bool = False) -> None:
558
+ self.is_git_or_bare_git_repository(folder)
559
+ argument = f"pull {remote} {remotebranchname}:{localbranchname}"
560
+ if force:
561
+ argument = f"{argument} --force"
562
+ self.run_program("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
563
+
564
+ @GeneralUtilities.check_arguments
565
+ def git_list_remote_branches(self, folder: str, remote: str, fetch: bool) -> list[str]:
566
+ self.is_git_or_bare_git_repository(folder)
567
+ if fetch:
568
+ self.git_fetch(folder, remote)
569
+ run_program_result = self.run_program("git", f"branch -rl {remote}/*", folder, throw_exception_if_exitcode_is_not_zero=True)
570
+ output = GeneralUtilities.string_to_lines(run_program_result[1])
571
+ result = list[str]()
572
+ for item in output:
573
+ striped_item = item.strip()
574
+ if GeneralUtilities.string_has_content(striped_item):
575
+ branch: str = None
576
+ if " " in striped_item:
577
+ branch = striped_item.split(" ")[0]
578
+ else:
579
+ branch = striped_item
580
+ branchname = branch[len(remote)+1:]
581
+ if branchname != "HEAD":
582
+ result.append(branchname)
583
+ return result
584
+
585
+ @GeneralUtilities.check_arguments
586
+ def git_clone(self, clone_target_folder: str, remote_repository_path: str, include_submodules: bool = True, mirror: bool = False) -> None:
587
+ if (os.path.isdir(clone_target_folder)):
588
+ pass # TODO throw error
589
+ else:
590
+ args = ["clone", remote_repository_path, clone_target_folder]
591
+ if include_submodules:
592
+ args.append("--recurse-submodules")
593
+ args.append("--remote-submodules")
594
+ if mirror:
595
+ args.append("--mirror")
596
+ self.run_program_argsasarray("git", args, os.getcwd(), throw_exception_if_exitcode_is_not_zero=True)
597
+
598
+ @GeneralUtilities.check_arguments
599
+ def git_get_all_remote_names(self, directory: str) -> list[str]:
600
+ self.is_git_or_bare_git_repository(directory)
601
+ result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote"], directory, throw_exception_if_exitcode_is_not_zero=True)[1], False)
602
+ return result
603
+
604
+ @GeneralUtilities.check_arguments
605
+ def git_get_remote_url(self, directory: str, remote_name: str) -> str:
606
+ self.is_git_or_bare_git_repository(directory)
607
+ result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote", "get-url", remote_name], directory, throw_exception_if_exitcode_is_not_zero=True)[1], False)
608
+ return result[0].replace('\n', '')
609
+
610
+ @GeneralUtilities.check_arguments
611
+ def repository_has_remote_with_specific_name(self, directory: str, remote_name: str) -> bool:
612
+ self.is_git_or_bare_git_repository(directory)
613
+ return remote_name in self.git_get_all_remote_names(directory)
614
+
615
+ @GeneralUtilities.check_arguments
616
+ def git_add_or_set_remote_address(self, directory: str, remote_name: str, remote_address: str) -> None:
617
+ self.assert_is_git_repository(directory)
618
+ if (self.repository_has_remote_with_specific_name(directory, remote_name)):
619
+ self.run_program_argsasarray("git", ['remote', 'set-url', 'remote_name', remote_address], directory, throw_exception_if_exitcode_is_not_zero=True)
620
+ else:
621
+ self.run_program_argsasarray("git", ['remote', 'add', remote_name, remote_address], directory, throw_exception_if_exitcode_is_not_zero=True)
622
+
623
+ @GeneralUtilities.check_arguments
624
+ def git_stage_all_changes(self, directory: str) -> None:
625
+ self.assert_is_git_repository(directory)
626
+ self.run_program_argsasarray("git", ["add", "-A"], directory, throw_exception_if_exitcode_is_not_zero=True)
627
+
628
+ @GeneralUtilities.check_arguments
629
+ def git_unstage_all_changes(self, directory: str) -> None:
630
+ self.assert_is_git_repository(directory)
631
+ self.run_program_argsasarray("git", ["reset"], directory, throw_exception_if_exitcode_is_not_zero=True)
632
+ # TODO check if this will also be done for submodules
633
+
634
+ @GeneralUtilities.check_arguments
635
+ def git_stage_file(self, directory: str, file: str) -> None:
636
+ self.assert_is_git_repository(directory)
637
+ self.run_program_argsasarray("git", ['stage', file], directory, throw_exception_if_exitcode_is_not_zero=True)
638
+
639
+ @GeneralUtilities.check_arguments
640
+ def git_unstage_file(self, directory: str, file: str) -> None:
641
+ self.assert_is_git_repository(directory)
642
+ self.run_program_argsasarray("git", ['reset', file], directory, throw_exception_if_exitcode_is_not_zero=True)
643
+
644
+ @GeneralUtilities.check_arguments
645
+ def git_discard_unstaged_changes_of_file(self, directory: str, file: str) -> None:
646
+ """Caution: This method works really only for 'changed' files yet. So this method does not work properly for new or renamed files."""
647
+ self.assert_is_git_repository(directory)
648
+ self.run_program_argsasarray("git", ['checkout', file], directory, throw_exception_if_exitcode_is_not_zero=True)
649
+
650
+ @GeneralUtilities.check_arguments
651
+ def git_discard_all_unstaged_changes(self, directory: str) -> None:
652
+ """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
653
+ self.assert_is_git_repository(directory)
654
+ self.run_program_argsasarray("git", ['clean', '-df'], directory, throw_exception_if_exitcode_is_not_zero=True)
655
+ self.run_program_argsasarray("git", ['checkout', '.'], directory, throw_exception_if_exitcode_is_not_zero=True)
656
+ # TODO check if this will also be done for submodules
657
+
658
+ @GeneralUtilities.check_arguments
659
+ def git_commit(self, directory: str, message: str = "Saved changes.", author_name: str = None, author_email: str = None, stage_all_changes: bool = True, no_changes_behavior: int = 0,commit_message_body:str=None) -> str:
660
+ """no_changes_behavior=0 => No commit; no_changes_behavior=1 => Commit anyway; no_changes_behavior=2 => Exception"""
661
+ self.assert_is_git_repository(directory)
662
+ author_name = GeneralUtilities.str_none_safe(author_name).strip()
663
+ author_email = GeneralUtilities.str_none_safe(author_email).strip()
664
+ argument = ['commit', '--quiet', '--allow-empty', '--message', message]
665
+ if commit_message_body is not None:
666
+ argument.extend(['--message', commit_message_body])
667
+ if (GeneralUtilities.string_has_content(author_name)):
668
+ argument.append(f'--author="{author_name} <{author_email}>"')
669
+ git_repository_has_uncommitted_changes = self.git_repository_has_uncommitted_changes(directory)
670
+
671
+ if git_repository_has_uncommitted_changes:
672
+ do_commit = True
673
+ if stage_all_changes:
674
+ self.git_stage_all_changes(directory)
675
+ else:
676
+ if no_changes_behavior == 0:
677
+ self.log.log(f"Commit '{message}' will not be done because there are no changes to commit in repository '{directory}'", LogLevel.Debug)
678
+ do_commit = False
679
+ elif no_changes_behavior == 1:
680
+ self.log.log(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will be done anyway.", LogLevel.Debug)
681
+ do_commit = True
682
+ elif no_changes_behavior == 2:
683
+ raise RuntimeError(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will not be done.")
684
+ else:
685
+ raise ValueError(f"Unknown value for no_changes_behavior: {GeneralUtilities.str_none_safe(no_changes_behavior)}")
686
+
687
+ if do_commit:
688
+ self.log.log(f"Commit changes in '{directory}'", LogLevel.Information)
689
+ self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True)
690
+
691
+ return self.git_get_commit_id(directory)
692
+
693
+ def search_repository_folder(self,some_file_in_repository:str)->str:
694
+ current_path:str=os.path.dirname(some_file_in_repository)
695
+ enabled:bool=True
696
+ while enabled:
697
+ try:
698
+ current_path=GeneralUtilities.resolve_relative_path("..",current_path)
699
+ if self.is_git_repository(current_path):
700
+ return current_path
701
+ except:
702
+ enabled=False
703
+ raise ValueError(f"Can not find git-repository for folder \"{some_file_in_repository}\".")
704
+
705
+
706
+ @GeneralUtilities.check_arguments
707
+ def git_create_tag(self, directory: str, target_for_tag: str, tag: str, sign: bool = False, message: str = None) -> None:
708
+ self.is_git_or_bare_git_repository(directory)
709
+ argument = ["tag", tag, target_for_tag]
710
+ if sign:
711
+ if message is None:
712
+ message = f"Created {target_for_tag}"
713
+ argument.extend(["-s", '-m', message])
714
+ self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True)
715
+
716
+ @GeneralUtilities.check_arguments
717
+ def git_delete_tag(self, directory: str, tag: str) -> None:
718
+ self.is_git_or_bare_git_repository(directory)
719
+ self.run_program_argsasarray("git", ["tag", "--delete", tag], directory, throw_exception_if_exitcode_is_not_zero=True)
720
+
721
+ @GeneralUtilities.check_arguments
722
+ def git_checkout(self, directory: str, rev: str, undo_all_changes_after_checkout: bool = True, assert_no_uncommitted_changes: bool = True) -> None:
723
+ self.assert_is_git_repository(directory)
724
+ if assert_no_uncommitted_changes:
725
+ GeneralUtilities.assert_condition(not self.git_repository_has_uncommitted_changes(directory), f"Repository \"{directory}\" has uncommitted changes.")
726
+ self.run_program_argsasarray("git", ["checkout", rev], directory, throw_exception_if_exitcode_is_not_zero=True)
727
+ self.run_program_argsasarray("git", ["submodule", "update", "--recursive"], directory, throw_exception_if_exitcode_is_not_zero=True)
728
+ commit_id=self.git_get_commit_id(directory,"HEAD")
729
+ self.log.log(f"Checked out {commit_id} in \"{directory}\".", LogLevel.Debug)
730
+ if undo_all_changes_after_checkout:
731
+ self.git_undo_all_changes(directory)
732
+
733
+ @GeneralUtilities.check_arguments
734
+ def merge_repository(self, repository_folder: str, remote: str, branch: str):
735
+ GeneralUtilities.assert_condition(not self.git_repository_has_uncommitted_changes(repository_folder),f"Can not merge. There are uncommitted changes in \"{repository_folder}\".")
736
+ is_pullable: bool = self.git_commit_is_ancestor(repository_folder, branch, f"{remote}/{branch}")
737
+ if is_pullable:
738
+ self.git_pull(repository_folder, remote, branch, branch)
739
+ uncommitted_changes = self.git_repository_has_uncommitted_changes(repository_folder)
740
+ GeneralUtilities.assert_condition(not uncommitted_changes, f"Pulling remote \"{remote}\" in \"{repository_folder}\" caused new uncommitted files.")
741
+ self.git_checkout(repository_folder, branch)
742
+ self.git_fetch(repository_folder, remote)
743
+ self.git_merge(repository_folder, f"{remote}/{branch}", branch)
744
+ self.git_push_with_retry(repository_folder, remote, branch, branch)
745
+ self.git_checkout(repository_folder, branch)
746
+ #TODO opeional: checkfor merge conflicts and if there is one merge conflict print a warning
747
+
748
+ @GeneralUtilities.check_arguments
749
+ def git_merge_abort(self, directory: str) -> None:
750
+ self.assert_is_git_repository(directory)
751
+ self.run_program_argsasarray("git", ["merge", "--abort"], directory, throw_exception_if_exitcode_is_not_zero=True)
752
+
753
+ @GeneralUtilities.check_arguments
754
+ def git_merge(self, directory: str, sourcebranch: str, targetbranch: str, fastforward: bool = True, commit: bool = True, commit_message: str = None, undo_all_changes_after_checkout: bool = True, assert_no_uncommitted_changes: bool = True) -> str:
755
+ self.assert_is_git_repository(directory)
756
+ self.git_checkout(directory, targetbranch, undo_all_changes_after_checkout, assert_no_uncommitted_changes)
757
+ args = ["merge"]
758
+ if not commit:
759
+ args.append("--no-commit")
760
+ if not fastforward:
761
+ args.append("--no-ff")
762
+ if commit_message is not None:
763
+ args.append("-m")
764
+ args.append(commit_message)
765
+ args.append(sourcebranch)
766
+ self.run_program_argsasarray("git", args, directory, throw_exception_if_exitcode_is_not_zero=True)
767
+ self.run_program_argsasarray("git", ["submodule", "update"], directory, throw_exception_if_exitcode_is_not_zero=True)
768
+ return self.git_get_commit_id(directory)
769
+
770
+ @GeneralUtilities.check_arguments
771
+ def git_undo_all_changes(self, directory: str) -> None:
772
+ """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
773
+ self.assert_is_git_repository(directory)
774
+ self.git_unstage_all_changes(directory)
775
+ self.git_discard_all_unstaged_changes(directory)
776
+
777
+ @GeneralUtilities.check_arguments
778
+ def git_fetch_or_clone_all_in_directory(self, source_directory: str, target_directory: str) -> None:
779
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(source_directory):
780
+ foldername = os.path.basename(subfolder)
781
+ if self.is_git_repository(subfolder):
782
+ source_repository = subfolder
783
+ target_repository = os.path.join(target_directory, foldername)
784
+ if os.path.isdir(target_directory):
785
+ # fetch
786
+ self.git_fetch(target_directory)
787
+ else:
788
+ # clone
789
+ self.git_clone(target_repository, source_repository, include_submodules=True, mirror=True)
790
+
791
+ def get_git_submodules(self, directory: str) -> list[str]:
792
+ self.is_git_or_bare_git_repository(directory)
793
+ e = self.run_program("git", "submodule status", directory)
794
+ result = []
795
+ for submodule_line in GeneralUtilities.string_to_lines(e[1], False, True):
796
+ result.append(submodule_line.split(' ')[1])
797
+ return result
798
+
799
+ @GeneralUtilities.check_arguments
800
+ def file_is_git_ignored(self, file_in_repository: str, repositorybasefolder: str) -> None:
801
+ self.is_git_or_bare_git_repository(repositorybasefolder)
802
+ exit_code = self.run_program_argsasarray("git", ['check-ignore', file_in_repository], repositorybasefolder, throw_exception_if_exitcode_is_not_zero=False)[0]
803
+ if (exit_code == 0):
804
+ return True
805
+ if (exit_code == 1):
806
+ return False
807
+ raise ValueError(f"Unable to calculate whether '{file_in_repository}' in repository '{repositorybasefolder}' is ignored due to git-exitcode {exit_code}.")
808
+
809
+ @GeneralUtilities.check_arguments
810
+ def git_discard_all_changes(self, repository: str) -> None:
811
+ self.assert_is_git_repository(repository)
812
+ self.run_program_argsasarray("git", ["reset", "HEAD", "."], repository, throw_exception_if_exitcode_is_not_zero=True)
813
+ self.run_program_argsasarray("git", ["checkout", "."], repository, throw_exception_if_exitcode_is_not_zero=True)
814
+
815
+ @GeneralUtilities.check_arguments
816
+ def git_get_current_branch_name(self, repository: str) -> str:
817
+ self.assert_is_git_repository(repository)
818
+ result = self.run_program_argsasarray("git", ["rev-parse", "--abbrev-ref", "HEAD"], repository, throw_exception_if_exitcode_is_not_zero=True)
819
+ return result[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
820
+
821
+ @GeneralUtilities.check_arguments
822
+ def git_get_commitid_of_tag(self, repository: str, tag: str) -> str:
823
+ self.is_git_or_bare_git_repository(repository)
824
+ stdout = self.run_program_argsasarray("git", ["rev-list", "-n", "1", tag], repository)
825
+ result = stdout[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
826
+ return result
827
+
828
+ @GeneralUtilities.check_arguments
829
+ def git_get_tags(self, repository: str) -> list[str]:
830
+ self.is_git_or_bare_git_repository(repository)
831
+ tags = [line.replace("\r", GeneralUtilities.empty_string) for line in self.run_program_argsasarray(
832
+ "git", ["tag"], repository)[1].split("\n") if len(line) > 0]
833
+ return tags
834
+
835
+ @GeneralUtilities.check_arguments
836
+ def git_move_tags_to_another_branch(self, repository: str, tag_source_branch: str, tag_target_branch: str, sign: bool = False, message: str = None) -> None:
837
+ self.is_git_or_bare_git_repository(repository)
838
+ tags = self.git_get_tags(repository)
839
+ tags_count = len(tags)
840
+ counter = 0
841
+ for tag in tags:
842
+ counter = counter+1
843
+ self.log.log(f"Process tag {counter}/{tags_count}.", LogLevel.Information)
844
+ # tag is on source-branch
845
+ if self.git_commit_is_ancestor(repository, tag, tag_source_branch):
846
+ commit_id_old = self.git_get_commit_id(repository, tag)
847
+ commit_date: datetime = self.git_get_commit_date(repository, commit_id_old)
848
+ date_as_string = self.__datetime_to_string_for_git(commit_date)
849
+ search_commit_result = self.run_program_argsasarray("git", ["log", f'--after="{date_as_string}"', f'--before="{date_as_string}"', "--pretty=format:%H", tag_target_branch], repository, throw_exception_if_exitcode_is_not_zero=False)
850
+ if search_commit_result[0] != 0 or not GeneralUtilities.string_has_nonwhitespace_content(search_commit_result[1]):
851
+ raise ValueError(f"Can not calculate corresponding commit for tag '{tag}'.")
852
+ commit_id_new = search_commit_result[1]
853
+ self.git_delete_tag(repository, tag)
854
+ self.git_create_tag(repository, commit_id_new, tag, sign, message)
855
+
856
+ @GeneralUtilities.check_arguments
857
+ def get_current_git_branch_has_tag(self, repository_folder: str) -> bool:
858
+ self.is_git_or_bare_git_repository(repository_folder)
859
+ result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
860
+ return result[0] == 0
861
+
862
+ @GeneralUtilities.check_arguments
863
+ def get_latest_git_tag(self, repository_folder: str) -> str:
864
+ self.is_git_or_bare_git_repository(repository_folder)
865
+ result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder)
866
+ result = result[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
867
+ return result
868
+
869
+ @GeneralUtilities.check_arguments
870
+ def get_staged_or_committed_git_ignored_files(self, repository_folder: str) -> list[str]:
871
+ self.assert_is_git_repository(repository_folder)
872
+ temp_result = self.run_program_argsasarray("git", ["ls-files", "-i", "-c", "--exclude-standard"], repository_folder)
873
+ temp_result = temp_result[1].replace("\r", GeneralUtilities.empty_string)
874
+ result = [line for line in temp_result.split("\n") if len(line) > 0]
875
+ return result
876
+
877
+ @GeneralUtilities.check_arguments
878
+ def git_repository_has_commits(self, repository_folder: str) -> bool:
879
+ self.assert_is_git_repository(repository_folder)
880
+ return self.run_program_argsasarray("git", ["rev-parse", "--verify", "HEAD"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)[0] == 0
881
+
882
+ @GeneralUtilities.check_arguments
883
+ def run_git_command_in_repository_and_submodules(self, repository_folder: str, arguments: list[str],print_live_output:bool) -> None:
884
+ GeneralUtilities.assert_condition(self.is_git_or_bare_git_repository(repository_folder),f"\"{repository_folder}\" is not a git-repository.")
885
+ self.log.log("Run \"git "+" ".join(arguments)+f"\" in {repository_folder} and its submodules...",LogLevel.Debug)
886
+ self.run_program_argsasarray("git", arguments, repository_folder,print_live_output=print_live_output)
887
+ if not self.is_bare_git_repository(repository_folder) and 0<len(self.get_git_submodules(repository_folder)):
888
+ self.run_program_argsasarray("git", ["submodule", "foreach", "--recursive", "git"]+arguments, repository_folder,print_live_output=print_live_output)
889
+
890
+ @GeneralUtilities.check_arguments
891
+ def export_filemetadata(self, folder: str, target_file: str, encoding: str = "utf-8", filter_function=None) -> None:
892
+ folder = GeneralUtilities.resolve_relative_path_from_current_working_directory(folder)
893
+ lines = list()
894
+ path_prefix = len(folder)+1
895
+ items = dict()
896
+ for item in GeneralUtilities.get_all_folders_of_folder(folder):
897
+ items[item] = "d"
898
+ for item in GeneralUtilities.get_all_files_of_folder(folder):
899
+ items[item] = "f"
900
+ for file_or_folder, item_type in items.items():
901
+ truncated_file = file_or_folder[path_prefix:]
902
+ if (filter_function is None or filter_function(folder, truncated_file)):
903
+ owner_and_permisssion = self.get_file_owner_and_file_permission(file_or_folder)
904
+ user = owner_and_permisssion[0]
905
+ permissions = owner_and_permisssion[1]
906
+ lines.append(f"{truncated_file};{item_type};{user};{permissions}")
907
+ lines = sorted(lines, key=str.casefold)
908
+ with open(target_file, "w", encoding=encoding) as file_object:
909
+ file_object.write("\n".join(lines))
910
+
911
+ @GeneralUtilities.check_arguments
912
+ def escape_git_repositories_in_folder(self, folder: str) -> dict[str, str]:
913
+ return self.__escape_git_repositories_in_folder_internal(folder, dict[str, str]())
914
+
915
+ @GeneralUtilities.check_arguments
916
+ def __escape_git_repositories_in_folder_internal(self, folder: str, renamed_items: dict[str, str]) -> dict[str, str]:
917
+ for file in GeneralUtilities.get_direct_files_of_folder(folder):
918
+ filename = os.path.basename(file)
919
+ if ".git" in filename:
920
+ new_name = filename.replace(".git", ".gitx")
921
+ target = os.path.join(folder, new_name)
922
+ os.rename(file, target)
923
+ renamed_items[target] = file
924
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
925
+ foldername = os.path.basename(subfolder)
926
+ if ".git" in foldername:
927
+ new_name = foldername.replace(".git", ".gitx")
928
+ subfolder2 = os.path.join(str(Path(subfolder).parent), new_name)
929
+ os.rename(subfolder, subfolder2)
930
+ renamed_items[subfolder2] = subfolder
931
+ else:
932
+ subfolder2 = subfolder
933
+ self.__escape_git_repositories_in_folder_internal(subfolder2, renamed_items)
934
+ return renamed_items
935
+
936
+ @GeneralUtilities.check_arguments
937
+ def deescape_git_repositories_in_folder(self, renamed_items: dict[str, str]):
938
+ for renamed_item, original_name in renamed_items.items():
939
+ os.rename(renamed_item, original_name)
940
+
941
+ @GeneralUtilities.check_arguments
942
+ def is_git_repository(self, folder: str) -> bool:
943
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
944
+ folder=folder.replace("\\","/")
945
+ if folder.endswith("/"):
946
+ folder = folder[:-1]
947
+ if not self.is_folder(folder):
948
+ raise ValueError(f"Folder '{folder}' does not exist.")
949
+ git_folder_path = f"{folder}/.git"
950
+ return self.is_folder(git_folder_path) or self.is_file(git_folder_path)
951
+
952
+ @GeneralUtilities.check_arguments
953
+ def is_bare_git_repository(self, folder: str) -> bool:
954
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
955
+ if folder.endswith("/") or folder.endswith("\\"):
956
+ folder = folder[:-1]
957
+ if not self.is_folder(folder):
958
+ raise ValueError(f"Folder '{folder}' does not exist.")
959
+ return folder.endswith(".git")
960
+
961
+ @GeneralUtilities.check_arguments
962
+ def is_git_or_bare_git_repository(self, folder: str) -> bool:
963
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
964
+ return self.is_git_repository(folder) or self.is_bare_git_repository(folder)
965
+
966
+ @GeneralUtilities.check_arguments
967
+ def assert_is_git_repository(self, folder: str) -> str:
968
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
969
+ GeneralUtilities.assert_condition(self.is_git_repository(folder), f"'{folder}' is not a git-repository.")
970
+
971
+ @GeneralUtilities.check_arguments
972
+ def convert_git_repository_to_bare_repository(self, repository_folder: str):
973
+ repository_folder = repository_folder.replace("\\", "/")
974
+ self.assert_is_git_repository(repository_folder)
975
+ git_folder = repository_folder + "/.git"
976
+ if not self.is_folder(git_folder):
977
+ raise ValueError(f"Converting '{repository_folder}' to a bare repository not possible. The folder '{git_folder}' does not exist. Converting is currently only supported when the git-folder is a direct folder in a repository and not a reference to another location.")
978
+ target_folder: str = repository_folder + ".git"
979
+ GeneralUtilities.ensure_directory_exists(target_folder)
980
+ GeneralUtilities.move_content_of_folder(git_folder, target_folder)
981
+ GeneralUtilities.ensure_directory_does_not_exist(repository_folder)
982
+ self.run_program_argsasarray("git", ["config", "--bool", "core.bare", "true"], target_folder)
983
+
984
+ @GeneralUtilities.check_arguments
985
+ def assert_no_uncommitted_changes(self, repository_folder: str):
986
+ if self.git_repository_has_uncommitted_changes(repository_folder):
987
+ raise ValueError(f"Repository '{repository_folder}' has uncommitted changes.")
988
+
989
+ @GeneralUtilities.check_arguments
990
+ def list_content(self, path: str, include_files: bool, include_folder: bool, printonlynamewithoutpath: bool) -> list[str]:
991
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
992
+ result: list[str] = []
993
+ if self.program_runner.will_be_executed_locally():
994
+ if include_files:
995
+ result = result + GeneralUtilities.get_direct_files_of_folder(path)
996
+ if include_folder:
997
+ result = result + GeneralUtilities.get_direct_folders_of_folder(path)
998
+ else:
999
+ arguments = ["--path", path]
1000
+ if not include_files:
1001
+ arguments = arguments+["--excludefiles"]
1002
+ if not include_folder:
1003
+ arguments = arguments+["--excludedirectories"]
1004
+ if printonlynamewithoutpath:
1005
+ arguments = arguments+["--printonlynamewithoutpath"]
1006
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("sclistfoldercontent", arguments)
1007
+ if exit_code == 0:
1008
+ for line in stdout.split("\n"):
1009
+ normalized_line = line.replace("\r", "")
1010
+ result.append(normalized_line)
1011
+ else:
1012
+ raise ValueError(f"Fatal error occurrs while checking whether file '{path}' exists. StdErr: '{stderr}'")
1013
+ result = [item for item in result if GeneralUtilities.string_has_nonwhitespace_content(item)]
1014
+ return result
1015
+
1016
+ @GeneralUtilities.check_arguments
1017
+ def is_file(self, path: str) -> bool:
1018
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1019
+ if self.program_runner.will_be_executed_locally():
1020
+ return os.path.isfile(path) # works only locally, but much more performant than always running an external program
1021
+ else:
1022
+ exit_code, _, stderr, _ = self.run_program_argsasarray("scfileexists", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1023
+ if exit_code == 0:
1024
+ return True
1025
+ elif exit_code == 1:
1026
+ raise ValueError(f"Not calculatable whether file '{path}' exists. StdErr: '{stderr}'")
1027
+ elif exit_code == 2:
1028
+ return False
1029
+ raise ValueError(f"Fatal error occurrs while checking whether file '{path}' exists. StdErr: '{stderr}'")
1030
+
1031
+ @GeneralUtilities.check_arguments
1032
+ def get_size(self, path: str) -> int:
1033
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1034
+ if self.program_runner.will_be_executed_locally():
1035
+ return os.path.getsize(path) # works only locally, but much more performant than always running an external program
1036
+ else:
1037
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("scgetsize", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1038
+ if exit_code == 0:
1039
+ return int(stdout.replace("\r","").replace("\n","").strip())
1040
+ else:
1041
+ raise ValueError(f"Fatal error occurrs while checking whether file '{path}' exists. StdErr: '{stderr}'")
1042
+
1043
+ @GeneralUtilities.check_arguments
1044
+ def is_folder(self, path: str) -> bool:
1045
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1046
+ if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
1047
+ return os.path.isdir(path)
1048
+ else:
1049
+ exit_code, _, stderr, _ = self.run_program_argsasarray("scfolderexists", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1050
+ if exit_code == 0:
1051
+ return True
1052
+ elif exit_code == 1:
1053
+ raise ValueError(f"Not calculatable whether folder '{path}' exists. StdErr: '{stderr}'")
1054
+ elif exit_code == 2:
1055
+ return False
1056
+ raise ValueError(f"Fatal error occurrs while checking whether folder '{path}' exists. StdErr: '{stderr}'")
1057
+
1058
+ @GeneralUtilities.check_arguments
1059
+ def get_file_content(self, path: str, encoding: str = "utf-8") -> str:
1060
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1061
+ if self.program_runner.will_be_executed_locally():
1062
+ return GeneralUtilities.read_text_from_file(path, encoding)
1063
+ else:
1064
+ result = self.run_program_argsasarray("scprintfilecontent", ["--path", path, "--encofing", encoding]) # works platform-indepent
1065
+ return result[1].replace("\\n", "\n")
1066
+
1067
+ @GeneralUtilities.check_arguments
1068
+ def set_file_content(self, path: str, content: str, encoding: str = "utf-8") -> None:
1069
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1070
+ if self.program_runner.will_be_executed_locally():
1071
+ GeneralUtilities.write_text_to_file(path, content, encoding)
1072
+ else:
1073
+ content_bytes = content.encode('utf-8')
1074
+ base64_bytes = base64.b64encode(content_bytes)
1075
+ base64_string = base64_bytes.decode('utf-8')
1076
+ self.run_program_argsasarray("scsetfilecontent", ["--path", path, "--argumentisinbase64", "--content", base64_string]) # works platform-indepent
1077
+
1078
+ @GeneralUtilities.check_arguments
1079
+ def remove(self, path: str) -> None:
1080
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1081
+ if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
1082
+ if os.path.isdir(path):
1083
+ GeneralUtilities.ensure_directory_does_not_exist(path)
1084
+ if os.path.isfile(path):
1085
+ GeneralUtilities.ensure_file_does_not_exist(path)
1086
+ else:
1087
+ if self.is_file(path):
1088
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("scremovefile", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1089
+ if exit_code != 0:
1090
+ raise ValueError(f"Fatal error occurrs while removing file '{path}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
1091
+ if self.is_folder(path):
1092
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("scremovefolder", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1093
+ if exit_code != 0:
1094
+ raise ValueError(f"Fatal error occurrs while removing folder '{path}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
1095
+
1096
+ @GeneralUtilities.check_arguments
1097
+ def rename(self, source: str, target: str) -> None:
1098
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1099
+ if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
1100
+ os.rename(source, target)
1101
+ else:
1102
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("screname", ["--source", source, "--target", target], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1103
+ if exit_code != 0:
1104
+ raise ValueError(f"Fatal error occurrs while renaming '{source}' to '{target}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
1105
+
1106
+ @GeneralUtilities.check_arguments
1107
+ def copy(self, source: str, target: str) -> None:
1108
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1109
+ if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
1110
+ if os.path.isfile(target) or os.path.isdir(target):
1111
+ raise ValueError(f"Can not copy to '{target}' because the target already exists.")
1112
+ if os.path.isfile(source):
1113
+ shutil.copyfile(source, target)
1114
+ elif os.path.isdir(source):
1115
+ GeneralUtilities.ensure_directory_exists(target)
1116
+ GeneralUtilities.copy_content_of_folder(source, target)
1117
+ else:
1118
+ raise ValueError(f"'{source}' can not be copied because the path does not exist.")
1119
+ else:
1120
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("sccopy", ["--source", source, "--target", target], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1121
+ if exit_code != 0:
1122
+ raise ValueError(f"Fatal error occurrs while copying '{source}' to '{target}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
1123
+
1124
+ @GeneralUtilities.check_arguments
1125
+ def create_file(self, path: str, error_if_already_exists: bool, create_necessary_folder: bool) -> None:
1126
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1127
+ if self.program_runner.will_be_executed_locally():
1128
+ if not os.path.isabs(path):
1129
+ path = os.path.join(os.getcwd(), path)
1130
+
1131
+ if os.path.isfile(path) and error_if_already_exists:
1132
+ raise ValueError(f"File '{path}' already exists.")
1133
+
1134
+ # TODO maybe it should be checked if there is a folder with the same path which already exists.
1135
+
1136
+ folder = os.path.dirname(path)
1137
+
1138
+ if not os.path.isdir(folder):
1139
+ if create_necessary_folder:
1140
+ GeneralUtilities.ensure_directory_exists(folder) # TODO check if this also create nested folders if required
1141
+ else:
1142
+ raise ValueError(f"Folder '{folder}' does not exist.")
1143
+
1144
+ GeneralUtilities.ensure_file_exists(path)
1145
+ else:
1146
+ arguments = ["--path", path]
1147
+
1148
+ if error_if_already_exists:
1149
+ arguments = arguments+["--errorwhenexists"]
1150
+
1151
+ if create_necessary_folder:
1152
+ arguments = arguments+["--createnecessaryfolder"]
1153
+
1154
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("sccreatefile", arguments, throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1155
+ if exit_code != 0:
1156
+ raise ValueError(f"Fatal error occurrs while create file '{path}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
1157
+
1158
+ @GeneralUtilities.check_arguments
1159
+ def create_folder(self, path: str, error_if_already_exists: bool, create_necessary_folder: bool) -> None:
1160
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1161
+ if self.program_runner.will_be_executed_locally():
1162
+ if not os.path.isabs(path):
1163
+ path = os.path.join(os.getcwd(), path)
1164
+
1165
+ if os.path.isdir(path) and error_if_already_exists:
1166
+ raise ValueError(f"Folder '{path}' already exists.")
1167
+
1168
+ # TODO maybe it should be checked if there is a file with the same path which already exists.
1169
+
1170
+ folder = os.path.dirname(path)
1171
+
1172
+ if not os.path.isdir(folder):
1173
+ if create_necessary_folder:
1174
+ GeneralUtilities.ensure_directory_exists(folder) # TODO check if this also create nested folders if required
1175
+ else:
1176
+ raise ValueError(f"Folder '{folder}' does not exist.")
1177
+
1178
+ GeneralUtilities.ensure_directory_exists(path)
1179
+ else:
1180
+ arguments = ["--path", path]
1181
+
1182
+ if error_if_already_exists:
1183
+ arguments = arguments+["--errorwhenexists"]
1184
+
1185
+ if create_necessary_folder:
1186
+ arguments = arguments+["--createnecessaryfolder"]
1187
+
1188
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("sccreatefolder", arguments, throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1189
+ if exit_code != 0:
1190
+ raise ValueError(f"Fatal error occurrs while create folder '{path}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
1191
+
1192
+ @GeneralUtilities.check_arguments
1193
+ def __sort_fmd(self, line: str):
1194
+ splitted: list = line.split(";")
1195
+ filetype: str = splitted[1]
1196
+ if filetype == "d":
1197
+ return -1
1198
+ if filetype == "f":
1199
+ return 1
1200
+ return 0
1201
+
1202
+ @GeneralUtilities.check_arguments
1203
+ def restore_filemetadata(self, folder: str, source_file: str, strict=False, encoding: str = "utf-8", create_folder_is_not_exist: bool = True) -> None:
1204
+ lines = GeneralUtilities.read_lines_from_file(source_file, encoding)
1205
+ lines.sort(key=self.__sort_fmd)
1206
+ for line in lines:
1207
+ splitted: list = line.split(";")
1208
+ full_path_of_file_or_folder: str = os.path.join(folder, splitted[0])
1209
+ filetype: str = splitted[1]
1210
+ user: str = splitted[2]
1211
+ permissions: str = splitted[3]
1212
+ if filetype == "d" and create_folder_is_not_exist and not os.path.isdir(full_path_of_file_or_folder):
1213
+ GeneralUtilities.ensure_directory_exists(full_path_of_file_or_folder)
1214
+ if (filetype == "f" and os.path.isfile(full_path_of_file_or_folder)) or (filetype == "d" and os.path.isdir(full_path_of_file_or_folder)):
1215
+ self.set_owner(full_path_of_file_or_folder, user, os.name != 'nt')
1216
+ self.set_permission(full_path_of_file_or_folder, permissions)
1217
+ else:
1218
+ if strict:
1219
+ if filetype == "f":
1220
+ filetype_full = "File"
1221
+ elif filetype == "d":
1222
+ filetype_full = "Directory"
1223
+ else:
1224
+ raise ValueError(f"Unknown filetype: {GeneralUtilities.str_none_safe(filetype)}")
1225
+ raise ValueError(f"{filetype_full} '{full_path_of_file_or_folder}' does not exist")
1226
+
1227
+ @GeneralUtilities.check_arguments
1228
+ def __calculate_lengh_in_seconds(self, filename: str, folder: str) -> float:
1229
+ argument = ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', filename]
1230
+ result = self.run_program_argsasarray("ffprobe", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
1231
+ return float(result[1].replace('\n', ''))
1232
+
1233
+ @GeneralUtilities.check_arguments
1234
+ def __create_thumbnails(self, filename: str, fps: str, folder: str, tempname_for_thumbnails: str) -> list[str]:
1235
+ argument = ['-i', filename, '-r', fps, '-vf', 'scale=-1:120', '-vcodec', 'png', f'{tempname_for_thumbnails}-%002d.png']
1236
+ self.run_program_argsasarray("ffmpeg", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
1237
+ files = GeneralUtilities.get_direct_files_of_folder(folder)
1238
+ result: list[str] = []
1239
+ regex = "^"+re.escape(tempname_for_thumbnails)+"\\-\\d+\\.png$"
1240
+ regex_for_files = re.compile(regex)
1241
+ for file in files:
1242
+ filename = os.path.basename(file)
1243
+ if regex_for_files.match(filename):
1244
+ result.append(file)
1245
+ GeneralUtilities.assert_condition(0 < len(result), "No thumbnail-files found.")
1246
+ return result
1247
+
1248
+ @GeneralUtilities.check_arguments
1249
+ def __create_thumbnail(self, outputfilename: str, folder: str, length_in_seconds: float, tempname_for_thumbnails: str, amount_of_images: int) -> None:
1250
+ duration = timedelta(seconds=length_in_seconds)
1251
+ info = GeneralUtilities.timedelta_to_simple_string(duration)
1252
+ next_square_number = GeneralUtilities.get_next_square_number(amount_of_images)
1253
+ root = math.sqrt(next_square_number)
1254
+ rows: int = root # 5
1255
+ columns: int = root # math.ceil(amount_of_images/rows)
1256
+ argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{rows}x{columns}', f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
1257
+ self.run_program_argsasarray("montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
1258
+
1259
+ @GeneralUtilities.check_arguments
1260
+ def __create_thumbnail2(self, outputfilename: str, folder: str, length_in_seconds: float, rows: int, columns: int, tempname_for_thumbnails: str, amount_of_images: int) -> None:
1261
+ duration = timedelta(seconds=length_in_seconds)
1262
+ info = GeneralUtilities.timedelta_to_simple_string(duration)
1263
+ argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{rows}x{columns}', f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
1264
+ self.run_program_argsasarray("montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
1265
+
1266
+ @GeneralUtilities.check_arguments
1267
+ def __roundup(self, x: float, places: int) -> int:
1268
+ d = 10 ** places
1269
+ if x < 0:
1270
+ return math.floor(x * d) / d
1271
+ else:
1272
+ return math.ceil(x * d) / d
1273
+
1274
+ @GeneralUtilities.check_arguments
1275
+ def generate_thumbnail(self, file: str, frames_per_second: float, tempname_for_thumbnails: str = None, hook=None) -> None:
1276
+ if tempname_for_thumbnails is None:
1277
+ tempname_for_thumbnails = "t_"+str(uuid.uuid4())
1278
+
1279
+ file = GeneralUtilities.resolve_relative_path_from_current_working_directory(file)
1280
+ filename = os.path.basename(file)
1281
+ folder = os.path.dirname(file)
1282
+ filename_without_extension = Path(file).stem
1283
+ preview_files: list[str] = []
1284
+ try:
1285
+ length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
1286
+ # frames per second, example: frames_per_second="20fps" => 20 frames per second
1287
+ frames_per_second = self.__roundup(float(frames_per_second[:-3]), 2)
1288
+ frames_per_second_as_string = str(frames_per_second)
1289
+ preview_files = self.__create_thumbnails(filename, frames_per_second_as_string, folder, tempname_for_thumbnails)
1290
+ if hook is not None:
1291
+ hook(file, preview_files)
1292
+ actual_amounf_of_previewframes = len(preview_files)
1293
+ self.__create_thumbnail(filename_without_extension, folder, length_in_seconds, tempname_for_thumbnails, actual_amounf_of_previewframes)
1294
+ finally:
1295
+ for thumbnail_to_delete in preview_files:
1296
+ os.remove(thumbnail_to_delete)
1297
+
1298
+ @GeneralUtilities.check_arguments
1299
+ def generate_thumbnail_by_amount_of_pictures(self, file: str, amount_of_columns: int, amount_of_rows: int, tempname_for_thumbnails: str = None, hook=None) -> None:
1300
+ if tempname_for_thumbnails is None:
1301
+ tempname_for_thumbnails = "t_"+str(uuid.uuid4())
1302
+
1303
+ file = GeneralUtilities.resolve_relative_path_from_current_working_directory(file)
1304
+ filename = os.path.basename(file)
1305
+ folder = os.path.dirname(file)
1306
+ filename_without_extension = Path(file).stem
1307
+ preview_files: list[str] = []
1308
+ try:
1309
+ length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
1310
+ amounf_of_previewframes = int(amount_of_columns*amount_of_rows)
1311
+ frames_per_second_as_string = f"{amounf_of_previewframes-2}/{length_in_seconds}"
1312
+ preview_files = self.__create_thumbnails(filename, frames_per_second_as_string, folder, tempname_for_thumbnails)
1313
+ if hook is not None:
1314
+ hook(file, preview_files)
1315
+ actual_amounf_of_previewframes = len(preview_files)
1316
+ self.__create_thumbnail2(filename_without_extension, folder, length_in_seconds, amount_of_rows, amount_of_columns, tempname_for_thumbnails, actual_amounf_of_previewframes)
1317
+ finally:
1318
+ for thumbnail_to_delete in preview_files:
1319
+ os.remove(thumbnail_to_delete)
1320
+
1321
+ @GeneralUtilities.check_arguments
1322
+ def extract_pdf_pages(self, file: str, from_page: int, to_page: int, outputfile: str) -> None:
1323
+ pdf_reader: PdfReader = PdfReader(file)
1324
+ pdf_writer: PdfWriter = PdfWriter()
1325
+ start = from_page
1326
+ end = to_page
1327
+ while start <= end:
1328
+ pdf_writer.add_page(pdf_reader.pages[start-1])
1329
+ start += 1
1330
+ with open(outputfile, 'wb') as out:
1331
+ pdf_writer.write(out)
1332
+
1333
+ @GeneralUtilities.check_arguments
1334
+ def merge_pdf_files(self, files: list[str], outputfile: str) -> None:
1335
+ # TODO add wildcard-option
1336
+ pdfFileMerger: PdfWriter = PdfWriter()
1337
+ for file in files:
1338
+ with open(file, "rb") as f:
1339
+ pdfFileMerger.append(f)
1340
+ with open(outputfile, "wb") as output:
1341
+ pdfFileMerger.write(output)
1342
+ pdfFileMerger.close()
1343
+
1344
+ @GeneralUtilities.check_arguments
1345
+ def show_missing_files(self, folderA: str, folderB: str):
1346
+ for file in GeneralUtilities.get_missing_files(folderA, folderB):
1347
+ GeneralUtilities.write_message_to_stdout(file)
1348
+
1349
+ @GeneralUtilities.check_arguments
1350
+ def SCCreateEmptyFileWithSpecificSize(self, name: str, size_string: str) -> int:
1351
+ if size_string.isdigit():
1352
+ size = int(size_string)
1353
+ else:
1354
+ if len(size_string) >= 3:
1355
+ if (size_string.endswith("kb")):
1356
+ size = int(size_string[:-2]) * pow(10, 3)
1357
+ elif (size_string.endswith("mb")):
1358
+ size = int(size_string[:-2]) * pow(10, 6)
1359
+ elif (size_string.endswith("gb")):
1360
+ size = int(size_string[:-2]) * pow(10, 9)
1361
+ elif (size_string.endswith("kib")):
1362
+ size = int(size_string[:-3]) * pow(2, 10)
1363
+ elif (size_string.endswith("mib")):
1364
+ size = int(size_string[:-3]) * pow(2, 20)
1365
+ elif (size_string.endswith("gib")):
1366
+ size = int(size_string[:-3]) * pow(2, 30)
1367
+ else:
1368
+ self.log.log("Wrong format", LogLevel.Error)
1369
+ return 1
1370
+ else:
1371
+ self.log.log("Wrong format", LogLevel.Error)
1372
+ return 1
1373
+ with open(name, "wb") as f:
1374
+ f.seek(size-1)
1375
+ f.write(b"\0")
1376
+ return 0
1377
+
1378
+ @GeneralUtilities.check_arguments
1379
+ def SCCreateHashOfAllFiles(self, folder: str) -> None:
1380
+ for file in GeneralUtilities.absolute_file_paths(folder):
1381
+ with open(file+".sha256", "w+", encoding="utf-8") as f:
1382
+ f.write(GeneralUtilities.get_sha256_of_file(file))
1383
+
1384
+ @GeneralUtilities.check_arguments
1385
+ def SCCreateSimpleMergeWithoutRelease(self, repository: str, sourcebranch: str, targetbranch: str, remotename: str, remove_source_branch: bool) -> None:
1386
+ commitid = self.git_merge(repository, sourcebranch, targetbranch, False, True)
1387
+ self.git_merge(repository, targetbranch, sourcebranch, True, True)
1388
+ created_version = self.get_semver_version_from_gitversion(repository)
1389
+ self.git_create_tag(repository, commitid, f"v{created_version}", True)
1390
+ self.git_push(repository, remotename, targetbranch, targetbranch, False, True)
1391
+ if (GeneralUtilities.string_has_nonwhitespace_content(remotename)):
1392
+ self.git_push(repository, remotename, sourcebranch, sourcebranch, False, True)
1393
+ if (remove_source_branch):
1394
+ self.git_remove_branch(repository, sourcebranch)
1395
+
1396
+ @GeneralUtilities.check_arguments
1397
+ def sc_organize_lines_in_file(self, file: str, encoding: str="utf-8", sort: bool = False, remove_duplicated_lines: bool = False, ignore_first_line: bool = False, remove_empty_lines: bool = True, ignored_start_character: list = list()):
1398
+ GeneralUtilities.assert_file_exists(file)
1399
+
1400
+ # read file
1401
+ lines = GeneralUtilities.read_lines_from_file(file, encoding)
1402
+ if (len(lines) == 0):
1403
+ return
1404
+
1405
+ # store first line if desired
1406
+ if (ignore_first_line):
1407
+ first_line = lines.pop(0)
1408
+
1409
+ # remove empty lines if desired
1410
+ if remove_empty_lines:
1411
+ temp = lines
1412
+ lines = []
1413
+ for line in temp:
1414
+ if (not (GeneralUtilities.string_is_none_or_whitespace(line))):
1415
+ lines.append(line)
1416
+
1417
+ # remove duplicated lines if desired
1418
+ if remove_duplicated_lines:
1419
+ lines = GeneralUtilities.remove_duplicates(lines)
1420
+
1421
+ # sort lines if desired
1422
+ if sort:
1423
+ lines = sorted(lines, key=lambda singleline: self.__adapt_line_for_sorting(singleline, ignored_start_character))
1424
+
1425
+ # reinsert first line if required
1426
+ if ignore_first_line:
1427
+ lines.insert(0, first_line)
1428
+
1429
+ # write result to file
1430
+ GeneralUtilities.write_lines_to_file(file, lines, encoding)
1431
+
1432
+
1433
+ @GeneralUtilities.check_arguments
1434
+ def __adapt_line_for_sorting(self, line: str, ignored_start_characters: list):
1435
+ result = line.lower()
1436
+ while len(result) > 0 and result[0] in ignored_start_characters:
1437
+ result = result[1:]
1438
+ return result
1439
+
1440
+ @GeneralUtilities.check_arguments
1441
+ def SCGenerateSnkFiles(self, outputfolder, keysize=4096, amountofkeys=10) -> int:
1442
+ GeneralUtilities.ensure_directory_exists(outputfolder)
1443
+ for _ in range(amountofkeys):
1444
+ file = os.path.join(outputfolder, str(uuid.uuid4())+".snk")
1445
+ argument = f"-k {keysize} {file}"
1446
+ self.run_program("sn", argument, outputfolder)
1447
+
1448
+ @GeneralUtilities.check_arguments
1449
+ def __merge_files(self, sourcefile: str, targetfile: str) -> None:
1450
+ with open(sourcefile, "rb") as f:
1451
+ source_data = f.read()
1452
+ with open(targetfile, "ab") as fout:
1453
+ merge_separator = [0x0A]
1454
+ fout.write(bytes(merge_separator))
1455
+ fout.write(source_data)
1456
+
1457
+ @GeneralUtilities.check_arguments
1458
+ def __process_file(self, file: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
1459
+ new_filename = os.path.join(os.path.dirname(file), os.path.basename(file).replace(substringInFilename, newSubstringInFilename))
1460
+ if file != new_filename:
1461
+ if os.path.isfile(new_filename):
1462
+ if filecmp.cmp(file, new_filename):
1463
+ send2trash.send2trash(file)
1464
+ else:
1465
+ if conflictResolveMode == "ignore":
1466
+ pass
1467
+ elif conflictResolveMode == "preservenewest":
1468
+ if (os.path.getmtime(file) - os.path.getmtime(new_filename) > 0):
1469
+ send2trash.send2trash(file)
1470
+ else:
1471
+ send2trash.send2trash(new_filename)
1472
+ os.rename(file, new_filename)
1473
+ elif (conflictResolveMode == "merge"):
1474
+ self.__merge_files(file, new_filename)
1475
+ send2trash.send2trash(file)
1476
+ else:
1477
+ raise ValueError('Unknown conflict resolve mode')
1478
+ else:
1479
+ os.rename(file, new_filename)
1480
+
1481
+ @GeneralUtilities.check_arguments
1482
+ def SCReplaceSubstringsInFilenames(self, folder: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
1483
+ for file in GeneralUtilities.absolute_file_paths(folder):
1484
+ self.__process_file(file, substringInFilename, newSubstringInFilename, conflictResolveMode)
1485
+
1486
+ @GeneralUtilities.check_arguments
1487
+ def __check_file(self, file: str, searchstring: str) -> None:
1488
+ bytes_ascii = bytes(searchstring, "ascii")
1489
+ # often called "unicode-encoding"
1490
+ bytes_utf16 = bytes(searchstring, "utf-16")
1491
+ bytes_utf8 = bytes(searchstring, "utf-8")
1492
+ with open(file, mode='rb') as file_object:
1493
+ content = file_object.read()
1494
+ if bytes_ascii in content:
1495
+ GeneralUtilities.write_message_to_stdout(file)
1496
+ elif bytes_utf16 in content:
1497
+ GeneralUtilities.write_message_to_stdout(file)
1498
+ elif bytes_utf8 in content:
1499
+ GeneralUtilities.write_message_to_stdout(file)
1500
+
1501
+ @GeneralUtilities.check_arguments
1502
+ def SCSearchInFiles(self, folder: str, searchstring: str) -> None:
1503
+ for file in GeneralUtilities.absolute_file_paths(folder):
1504
+ self.__check_file(file, searchstring)
1505
+
1506
+ @GeneralUtilities.check_arguments
1507
+ def get_string_as_qr_code(self,string: str) -> None:
1508
+ qr = qrcode.QRCode()
1509
+ qr.add_data(string)
1510
+ f = io.StringIO()
1511
+ qr.print_ascii(out=f)
1512
+ f.seek(0)
1513
+ return f.read()
1514
+
1515
+ @GeneralUtilities.check_arguments
1516
+ def __print_qr_code_by_csv_line(self, displayname: str, website: str, emailaddress: str, key: str, period: str) -> None:
1517
+ qrcode_content = f"otpauth://totp/{website}:{emailaddress}?secret={key}&issuer={displayname}&period={period}"
1518
+ GeneralUtilities.write_message_to_stdout(f"{displayname} ({emailaddress}):")
1519
+ GeneralUtilities.write_message_to_stdout(qrcode_content)
1520
+ GeneralUtilities.write_message_to_stdout(self.get_string_as_qr_code(qrcode_content))
1521
+
1522
+ @GeneralUtilities.check_arguments
1523
+ def SCShow2FAAsQRCode(self, csvfile: str) -> None:
1524
+ lines = GeneralUtilities.read_csv_file(csvfile, True)
1525
+ lines.sort(key=lambda items: ''.join(items).lower())
1526
+ for line in lines:
1527
+ self.__print_qr_code_by_csv_line(line[0], line[1], line[2], line[3], line[4])
1528
+ GeneralUtilities.write_message_to_stdout(GeneralUtilities.get_longline())
1529
+
1530
+ @GeneralUtilities.check_arguments
1531
+ def SCCalculateBitcoinBlockHash(self, block_version_number: str, previousblockhash: str, transactionsmerkleroot: str, timestamp: str, target: str, nonce: str) -> str:
1532
+ # Example-values:
1533
+ # block_version_number: "00000020"
1534
+ # previousblockhash: "66720b99e07d284bd4fe67ff8c49a5db1dd8514fcdab61000000000000000000"
1535
+ # transactionsmerkleroot: "7829844f4c3a41a537b3131ca992643eaa9d093b2383e4cdc060ad7dc5481187"
1536
+ # timestamp: "51eb505a"
1537
+ # target: "c1910018"
1538
+ # nonce: "de19b302"
1539
+ header = str(block_version_number + previousblockhash + transactionsmerkleroot + timestamp + target + nonce)
1540
+ return binascii.hexlify(hashlib.sha256(hashlib.sha256(binascii.unhexlify(header)).digest()).digest()[::-1]).decode('utf-8')
1541
+
1542
+ @GeneralUtilities.check_arguments
1543
+ def SCChangeHashOfProgram(self, inputfile: str) -> None:
1544
+ valuetoappend = str(uuid.uuid4())
1545
+
1546
+ outputfile = inputfile + '.modified'
1547
+
1548
+ shutil.copy2(inputfile, outputfile)
1549
+ with open(outputfile, 'a', encoding="utf-8") as file:
1550
+ # TODO use rcedit for .exe-files instead of appending valuetoappend ( https://github.com/electron/rcedit/ )
1551
+ # background: you can retrieve the "original-filename" from the .exe-file like discussed here:
1552
+ # https://security.stackexchange.com/questions/210843/ is-it-possible-to-change-original-filename-of-an-exe
1553
+ # so removing the original filename with rcedit is probably a better way to make it more difficult to detect the programname.
1554
+ # this would obviously also change the hashvalue of the program so appending a whitespace is not required anymore.
1555
+ file.write(valuetoappend)
1556
+
1557
+ @GeneralUtilities.check_arguments
1558
+ def __adjust_folder_name(self, folder: str) -> str:
1559
+ result = os.path.dirname(folder).replace("\\", "/")
1560
+ if result == "/":
1561
+ return GeneralUtilities.empty_string
1562
+ else:
1563
+ return result
1564
+
1565
+ @GeneralUtilities.check_arguments
1566
+ def __create_iso(self, folder, iso_file) -> None:
1567
+ created_directories = []
1568
+ files_directory = "FILES"
1569
+ iso = pycdlib.PyCdlib()
1570
+ iso.new()
1571
+ files_directory = files_directory.upper()
1572
+ iso.add_directory("/" + files_directory)
1573
+ created_directories.append("/" + files_directory)
1574
+ for root, _, files in os.walk(folder):
1575
+ for file in files:
1576
+ full_path = os.path.join(root, file)
1577
+ with (open(full_path, "rb").read()) as text_io_wrapper:
1578
+ content = text_io_wrapper
1579
+ path_in_iso = '/' + files_directory + \
1580
+ self.__adjust_folder_name(full_path[len(folder)::1]).upper()
1581
+ if path_in_iso not in created_directories:
1582
+ iso.add_directory(path_in_iso)
1583
+ created_directories.append(path_in_iso)
1584
+ iso.add_fp(BytesIO(content), len(content), path_in_iso + '/' + file.upper() + ';1')
1585
+ iso.write(iso_file)
1586
+ iso.close()
1587
+
1588
+ @GeneralUtilities.check_arguments
1589
+ def SCCreateISOFileWithObfuscatedFiles(self, inputfolder: str, outputfile: str, printtableheadline, createisofile, extensions) -> None:
1590
+ if (os.path.isdir(inputfolder)):
1591
+ namemappingfile = "name_map.csv"
1592
+ files_directory = inputfolder
1593
+ files_directory_obf = f"{files_directory}_Obfuscated"
1594
+ self.SCObfuscateFilesFolder(
1595
+ inputfolder, printtableheadline, namemappingfile, extensions)
1596
+ os.rename(namemappingfile, os.path.join(
1597
+ files_directory_obf, namemappingfile))
1598
+ if createisofile:
1599
+ self.__create_iso(files_directory_obf, outputfile)
1600
+ shutil.rmtree(files_directory_obf)
1601
+ else:
1602
+ raise ValueError(f"Directory not found: '{inputfolder}'")
1603
+
1604
+ @GeneralUtilities.check_arguments
1605
+ def SCFilenameObfuscator(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
1606
+ obfuscate_all_files = extensions == "*"
1607
+ if (obfuscate_all_files):
1608
+ obfuscate_file_extensions = None
1609
+ else:
1610
+ obfuscate_file_extensions = extensions.split(",")
1611
+ if (os.path.isdir(inputfolder)):
1612
+ printtableheadline = GeneralUtilities.string_to_boolean(
1613
+ printtableheadline)
1614
+ files = []
1615
+ if not os.path.isfile(namemappingfile):
1616
+ with open(namemappingfile, "a", encoding="utf-8"):
1617
+ pass
1618
+ if printtableheadline:
1619
+ GeneralUtilities.append_line_to_file(
1620
+ namemappingfile, "Original filename;new filename;SHA2-hash of file")
1621
+ for file in GeneralUtilities.absolute_file_paths(inputfolder):
1622
+ if os.path.isfile(os.path.join(inputfolder, file)):
1623
+ if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
1624
+ files.append(file)
1625
+ for file in files:
1626
+ hash_value = GeneralUtilities.get_sha256_of_file(file)
1627
+ extension = Path(file).suffix
1628
+ new_file_name_without_path = str(uuid.uuid4())[0:8] + extension
1629
+ new_file_name = os.path.join(
1630
+ os.path.dirname(file), new_file_name_without_path)
1631
+ os.rename(file, new_file_name)
1632
+ GeneralUtilities.append_line_to_file(namemappingfile, os.path.basename(file) + ";" + new_file_name_without_path + ";" + hash_value)
1633
+ else:
1634
+ raise ValueError(f"Directory not found: '{inputfolder}'")
1635
+
1636
+ @GeneralUtilities.check_arguments
1637
+ def __extension_matchs(self, file: str, obfuscate_file_extensions) -> bool:
1638
+ for extension in obfuscate_file_extensions:
1639
+ if file.lower().endswith("."+extension.lower()):
1640
+ return True
1641
+ return False
1642
+
1643
+ @GeneralUtilities.check_arguments
1644
+ def SCHealthcheck(self, file: str) -> int:
1645
+ lines = GeneralUtilities.read_lines_from_file(file)
1646
+ for line in reversed(lines):
1647
+ if not GeneralUtilities.string_is_none_or_whitespace(line):
1648
+ if "RunningHealthy (" in line: # TODO use regex
1649
+ GeneralUtilities.write_message_to_stderr(f"Healthy running due to line '{line}' in file '{file}'.")
1650
+ return 0
1651
+ else:
1652
+ GeneralUtilities.write_message_to_stderr(f"Not healthy running due to line '{line}' in file '{file}'.")
1653
+ return 1
1654
+ GeneralUtilities.write_message_to_stderr(f"No valid line found for healthycheck in file '{file}'.")
1655
+ return 2
1656
+
1657
+ @GeneralUtilities.check_arguments
1658
+ def SCObfuscateFilesFolder(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
1659
+ obfuscate_all_files = extensions == "*"
1660
+ if (obfuscate_all_files):
1661
+ obfuscate_file_extensions = None
1662
+ else:
1663
+ if "," in extensions:
1664
+ obfuscate_file_extensions = extensions.split(",")
1665
+ else:
1666
+ obfuscate_file_extensions = [extensions]
1667
+ newd = inputfolder+"_Obfuscated"
1668
+ shutil.copytree(inputfolder, newd)
1669
+ inputfolder = newd
1670
+ if (os.path.isdir(inputfolder)):
1671
+ for file in GeneralUtilities.absolute_file_paths(inputfolder):
1672
+ if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
1673
+ self.SCChangeHashOfProgram(file)
1674
+ os.remove(file)
1675
+ os.rename(file + ".modified", file)
1676
+ self.SCFilenameObfuscator(inputfolder, printtableheadline, namemappingfile, extensions)
1677
+ else:
1678
+ raise ValueError(f"Directory not found: '{inputfolder}'")
1679
+
1680
+ @GeneralUtilities.check_arguments
1681
+ def get_services_from_yaml_file(self, yaml_file: str) -> list[str]:
1682
+ with open(yaml_file, encoding="utf-8") as stream:
1683
+ loaded = yaml.safe_load(stream)
1684
+ services = loaded["services"]
1685
+ result = list(services.keys())
1686
+ return result
1687
+
1688
+ @GeneralUtilities.check_arguments
1689
+ def kill_docker_container(self, container_name: str) -> None:
1690
+ self.run_program("docker", f"container rm -f {container_name}")
1691
+
1692
+ @GeneralUtilities.check_arguments
1693
+ def get_latest_apt_package_version_in_debian(self, image: str,package:str) -> str:
1694
+ #docker run --rm -it debian bash -c "apt update && apt list -a tor"
1695
+ output=self.run_with_epew("docker", f"run --rm -it {image} bash -c \"apt --color=false update && apt --color=false list -a tor\"",os.getcwd(),encode_argument_in_base64=True)
1696
+ stdout=output[1]
1697
+ version_lines=[line.strip() for line in GeneralUtilities.string_to_lines(stdout) if GeneralUtilities.string_has_nonwhitespace_content(line) and line.startswith(package+"/")]
1698
+ GeneralUtilities.assert_condition(0<len(version_lines), f"No version found for package '{package}' in image '{image}'.")
1699
+ versions = [version_line.split(" ")[1] for version_line in version_lines]
1700
+ def my_comparer(a, b) -> int:
1701
+ # return:
1702
+ # -1 → a < b
1703
+ # 0 → a = b
1704
+ # 1 → a > b
1705
+ # dpkg --compare-versions <a> lt <b> → exit 0 wenn a < b
1706
+ def dpkg_compare(op: str) -> bool:
1707
+ result = self.run_program_argsasarray("docker", [ "run", "--rm",image, "dpkg", "--compare-versions", a, op, b],throw_exception_if_exitcode_is_not_zero=False)
1708
+ GeneralUtilities.assert_condition(result[1]==GeneralUtilities.empty_string)
1709
+ GeneralUtilities.assert_condition(result[2]==GeneralUtilities.empty_string)
1710
+ return result[0] == 0
1711
+
1712
+ if dpkg_compare("lt"): # a < b
1713
+ return -1
1714
+ elif dpkg_compare("gt"): # a > b
1715
+ return 1
1716
+ else:
1717
+ return 0
1718
+ sorted_versions = sorted(versions, key=cmp_to_key(my_comparer))
1719
+ return sorted_versions[-1]
1720
+
1721
+ @GeneralUtilities.check_arguments
1722
+ def run_testcases_for_python_project(self, repository_folder: str):
1723
+ self.assert_is_git_repository(repository_folder)
1724
+ self.run_program("coverage", "run -m pytest", repository_folder)
1725
+ self.run_program("coverage", "xml", repository_folder)
1726
+ GeneralUtilities.ensure_directory_exists(os.path.join(repository_folder, "Other/TestCoverage"))
1727
+ coveragefile = os.path.join(repository_folder, "Other/TestCoverage/TestCoverage.xml")
1728
+ GeneralUtilities.ensure_file_does_not_exist(coveragefile)
1729
+ os.rename(os.path.join(repository_folder, "coverage.xml"), coveragefile)
1730
+
1731
+ @GeneralUtilities.check_arguments
1732
+ def get_file_permission(self, file: str) -> str:
1733
+ """This function returns an usual octet-triple, for example "700"."""
1734
+ ls_output: str = self.run_ls_for_folder(file)
1735
+ return self.__get_file_permission_helper(ls_output)
1736
+
1737
+ @GeneralUtilities.check_arguments
1738
+ def __get_file_permission_helper(self, permissions: str) -> str:
1739
+ return str(self.__to_octet(permissions[0:3])) + str(self.__to_octet(permissions[3:6]))+str(self.__to_octet(permissions[6:9]))
1740
+
1741
+ @GeneralUtilities.check_arguments
1742
+ def __to_octet(self, string: str) -> int:
1743
+ return int(self.__to_octet_helper(string[0]) + self.__to_octet_helper(string[1])+self.__to_octet_helper(string[2]), 2)
1744
+
1745
+ @GeneralUtilities.check_arguments
1746
+ def __to_octet_helper(self, string: str) -> str:
1747
+ if (string == "-"):
1748
+ return "0"
1749
+ else:
1750
+ return "1"
1751
+
1752
+ @GeneralUtilities.check_arguments
1753
+ def get_file_owner(self, file: str) -> str:
1754
+ """This function returns the user and the group in the format "user:group"."""
1755
+ ls_output: str = self.run_ls_for_folder(file)
1756
+ return self.__get_file_owner_helper(ls_output)
1757
+
1758
+ @GeneralUtilities.check_arguments
1759
+ def __get_file_owner_helper(self, ls_output: str) -> str:
1760
+ splitted = ls_output.split()
1761
+ return f"{splitted[2]}:{splitted[3]}"
1762
+
1763
+ @GeneralUtilities.check_arguments
1764
+ def get_file_owner_and_file_permission(self, file: str) -> str:
1765
+ ls_output: str = self.run_ls_for_folder(file)
1766
+ return [self.__get_file_owner_helper(ls_output), self.__get_file_permission_helper(ls_output)]
1767
+
1768
+ @GeneralUtilities.check_arguments
1769
+ def run_ls_for_folder(self, file_or_folder: str) -> str:
1770
+ file_or_folder = file_or_folder.replace("\\", "/")
1771
+ GeneralUtilities.assert_condition(os.path.isfile(file_or_folder) or os.path.isdir(file_or_folder), f"Can not execute 'ls -ld' because '{file_or_folder}' does not exist.")
1772
+ ls_result = self.run_program_argsasarray("ls", ["-ld", file_or_folder])
1773
+ GeneralUtilities.assert_condition(ls_result[0] == 0, f"'ls -ld {file_or_folder}' resulted in exitcode {str(ls_result[0])}. StdErr: {ls_result[2]}")
1774
+ GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(ls_result[1]), f"'ls -ld' of '{file_or_folder}' had an empty output. StdErr: '{ls_result[2]}'")
1775
+ output = ls_result[1]
1776
+ result = output.replace("\n", GeneralUtilities.empty_string)
1777
+ result = ' '.join(result.split()) # reduce multiple whitespaces to one
1778
+ return result
1779
+
1780
+ @GeneralUtilities.check_arguments
1781
+ def run_ls_for_folder_content(self, file_or_folder: str) -> list[str]:
1782
+ file_or_folder = file_or_folder.replace("\\", "/")
1783
+ GeneralUtilities.assert_condition(os.path.isfile(file_or_folder) or os.path.isdir(file_or_folder), f"Can not execute 'ls -la' because '{file_or_folder}' does not exist.")
1784
+ ls_result = self.run_program_argsasarray("ls", ["-la", file_or_folder])
1785
+ GeneralUtilities.assert_condition(ls_result[0] == 0, f"'ls -la {file_or_folder}' resulted in exitcode {str(ls_result[0])}. StdErr: {ls_result[2]}")
1786
+ GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(ls_result[1]), f"'ls -la' of '{file_or_folder}' had an empty output. StdErr: '{ls_result[2]}'")
1787
+ output = ls_result[1]
1788
+ result = output.split("\n")[3:] # skip the lines with "Total", "." and ".."
1789
+ result = [' '.join(line.split()) for line in result] # reduce multiple whitespaces to one
1790
+ return result
1791
+
1792
+ @GeneralUtilities.check_arguments
1793
+ def set_permission(self, file_or_folder: str, permissions: str, recursive: bool = False) -> None:
1794
+ """This function expects an usual octet-triple, for example "700"."""
1795
+ args = []
1796
+ if recursive:
1797
+ args.append("--recursive")
1798
+ args.append(permissions)
1799
+ args.append(file_or_folder)
1800
+ self.run_program_argsasarray("chmod", args)
1801
+
1802
+ @GeneralUtilities.check_arguments
1803
+ def set_owner(self, file_or_folder: str, owner: str, recursive: bool = False, follow_symlinks: bool = False) -> None:
1804
+ """This function expects the user and the group in the format "user:group"."""
1805
+ args = []
1806
+ if recursive:
1807
+ args.append("--recursive")
1808
+ if follow_symlinks:
1809
+ args.append("--no-dereference")
1810
+ args.append(owner)
1811
+ args.append(file_or_folder)
1812
+ self.run_program_argsasarray("chown", args)
1813
+
1814
+ # <run programs>
1815
+
1816
+ @GeneralUtilities.check_arguments
1817
+ def __run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> Popen:
1818
+ popen: Popen = self.program_runner.run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, custom_argument, interactive)
1819
+ return popen
1820
+
1821
+ @staticmethod
1822
+ def __enqueue_output(file: IO, queue: Queue):
1823
+ for line in iter(file.readline, ''):
1824
+ queue.put(line)
1825
+ file.close()
1826
+
1827
+ @staticmethod
1828
+ def __continue_process_reading(pid: int, p: Popen, q_stdout: Queue, q_stderr: Queue, reading_stdout_last_time_resulted_in_exception: bool, reading_stderr_last_time_resulted_in_exception: bool):
1829
+ if p.poll() is None:
1830
+ return True
1831
+
1832
+ # if reading_stdout_last_time_resulted_in_exception and reading_stderr_last_time_resulted_in_exception:
1833
+ # return False
1834
+
1835
+ if not q_stdout.empty():
1836
+ return True
1837
+
1838
+ if not q_stderr.empty():
1839
+ return True
1840
+
1841
+ return False
1842
+
1843
+ @staticmethod
1844
+ def __read_popen_pipes(p: Popen, print_live_output: bool, print_errors_as_information: bool, log: SCLog) -> tuple[list[str], list[str]]:
1845
+ p_id = p.pid
1846
+ with ThreadPoolExecutor(2) as pool:
1847
+ q_stdout = Queue()
1848
+ q_stderr = Queue()
1849
+
1850
+ pool.submit(ScriptCollectionCore.__enqueue_output, p.stdout, q_stdout)
1851
+ pool.submit(ScriptCollectionCore.__enqueue_output, p.stderr, q_stderr)
1852
+ reading_stdout_last_time_resulted_in_exception: bool = False
1853
+ reading_stderr_last_time_resulted_in_exception: bool = False
1854
+
1855
+ stdout_result: list[str] = []
1856
+ stderr_result: list[str] = []
1857
+
1858
+ while (ScriptCollectionCore.__continue_process_reading(p_id, p, q_stdout, q_stderr, reading_stdout_last_time_resulted_in_exception, reading_stderr_last_time_resulted_in_exception)):
1859
+ try:
1860
+ while not q_stdout.empty():
1861
+ out_line: str = q_stdout.get_nowait()
1862
+ out_line = out_line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
1863
+ if GeneralUtilities.string_has_content(out_line):
1864
+ stdout_result.append(out_line)
1865
+ reading_stdout_last_time_resulted_in_exception = False
1866
+ if print_live_output:
1867
+ loglevel = LogLevel.Information
1868
+ if out_line.startswith("Debug: "):
1869
+ loglevel = LogLevel.Debug
1870
+ out_line = out_line[len("Debug: "):]
1871
+ if out_line.startswith("Diagnostic: "):
1872
+ loglevel = LogLevel.Diagnostic
1873
+ out_line = out_line[len("Diagnostic: "):]
1874
+ log.log(out_line, loglevel)
1875
+ except Empty:
1876
+ reading_stdout_last_time_resulted_in_exception = True
1877
+
1878
+ try:
1879
+ while not q_stderr.empty():
1880
+ err_line: str = q_stderr.get_nowait()
1881
+ err_line = err_line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
1882
+ if GeneralUtilities.string_has_content(err_line):
1883
+ stderr_result.append(err_line)
1884
+ reading_stderr_last_time_resulted_in_exception = False
1885
+ if print_live_output:
1886
+ loglevel = LogLevel.Error
1887
+ if err_line.startswith("Warning: "):
1888
+ loglevel = LogLevel.Warning
1889
+ err_line = err_line[len("Warning: "):]
1890
+ if print_errors_as_information: # "errors" in "print_errors_as_information" means: all what is written to std-err
1891
+ loglevel = LogLevel.Information
1892
+ log.log(err_line, loglevel)
1893
+ except Empty:
1894
+ reading_stderr_last_time_resulted_in_exception = True
1895
+
1896
+ time.sleep(0.01) # this is required to not finish too early
1897
+
1898
+ return (stdout_result, stderr_result)
1899
+
1900
+ @GeneralUtilities.check_arguments
1901
+ def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False) -> tuple[int, str, str, int]:
1902
+ if self.call_program_runner_directly:
1903
+ return self.program_runner.run_program_argsasarray(program, arguments_as_array, working_directory, custom_argument, interactive)
1904
+ try:
1905
+ GeneralUtilities.assert_not_null(arguments_as_array,"arguments_as_array must not be null")
1906
+ arguments_as_str = ' '.join(arguments_as_array)
1907
+ mock_loader_result = self.__try_load_mock(program, arguments_as_str, working_directory)
1908
+ if mock_loader_result[0]:
1909
+ return mock_loader_result[1]
1910
+
1911
+ if self.program_runner.will_be_executed_locally():
1912
+ working_directory = self.__adapt_workingdirectory(working_directory)
1913
+
1914
+ if arguments_for_log is None or len(arguments_for_log)==0:
1915
+ arguments_for_log = arguments_as_array
1916
+
1917
+ cmd = f'{GeneralUtilities.str_none_safe(working_directory)}>{program}'
1918
+ if 0 < len(arguments_for_log):
1919
+ arguments_for_log_as_string: str = ' '.join([f'"{argument_for_log}"' for argument_for_log in arguments_for_log])
1920
+ cmd = f'{cmd} {arguments_for_log_as_string}'
1921
+
1922
+ if GeneralUtilities.string_is_none_or_whitespace(title):
1923
+ info_for_log = cmd
1924
+ else:
1925
+ info_for_log = title
1926
+
1927
+ self.log.log(f"Run '{info_for_log}'.", LogLevel.Debug)
1928
+
1929
+ exit_code: int = None
1930
+ stdout: str = GeneralUtilities.empty_string
1931
+ stderr: str = GeneralUtilities.empty_string
1932
+ pid: int = None
1933
+
1934
+ with self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive) as process:
1935
+
1936
+ if log_file is not None:
1937
+ GeneralUtilities.ensure_file_exists(log_file)
1938
+ pid = process.pid
1939
+
1940
+ outputs: tuple[list[str], list[str]] = ScriptCollectionCore.__read_popen_pipes(process, print_live_output, print_errors_as_information, self.log)
1941
+
1942
+ for out_line_plain in outputs[0]:
1943
+ if out_line_plain is not None:
1944
+ out_line: str = None
1945
+ if isinstance(out_line_plain, str):
1946
+ out_line = out_line_plain
1947
+ elif isinstance(out_line_plain, bytes):
1948
+ out_line = GeneralUtilities.bytes_to_string(out_line_plain)
1949
+ else:
1950
+ raise ValueError(f"Unknown type of output: {str(type(out_line_plain))}")
1951
+
1952
+ if out_line is not None and GeneralUtilities.string_has_content(out_line):
1953
+ if out_line.endswith("\n"):
1954
+ out_line = out_line[:-1]
1955
+ if 0 < len(stdout):
1956
+ stdout = stdout+"\n"
1957
+ stdout = stdout+out_line
1958
+ if log_file is not None:
1959
+ GeneralUtilities.append_line_to_file(log_file, out_line)
1960
+
1961
+ for err_line_plain in outputs[1]:
1962
+ if err_line_plain is not None:
1963
+ err_line: str = None
1964
+ if isinstance(err_line_plain, str):
1965
+ err_line = err_line_plain
1966
+ elif isinstance(err_line_plain, bytes):
1967
+ err_line = GeneralUtilities.bytes_to_string(err_line_plain)
1968
+ else:
1969
+ raise ValueError(f"Unknown type of output: {str(type(err_line_plain))}")
1970
+ if err_line is not None and GeneralUtilities.string_has_content(err_line):
1971
+ if err_line.endswith("\n"):
1972
+ err_line = err_line[:-1]
1973
+ if 0 < len(stderr):
1974
+ stderr = stderr+"\n"
1975
+ stderr = stderr+err_line
1976
+ if log_file is not None:
1977
+ GeneralUtilities.append_line_to_file(log_file, err_line)
1978
+
1979
+ exit_code = process.returncode
1980
+ GeneralUtilities.assert_condition(exit_code is not None, f"Exitcode of program-run of '{info_for_log}' is None.")
1981
+
1982
+ result_message = f"Program '{info_for_log}' resulted in exitcode {exit_code}."
1983
+
1984
+ self.log.log(result_message, LogLevel.Debug)
1985
+
1986
+ if throw_exception_if_exitcode_is_not_zero and exit_code != 0:
1987
+ raise ValueError(f"{result_message} (StdOut: '{stdout}', StdErr: '{stderr}')")
1988
+
1989
+ result = (exit_code, stdout, stderr, pid)
1990
+ return result
1991
+ except Exception as e:#pylint:disable=unused-variable, try-except-raise
1992
+ raise
1993
+
1994
+ # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
1995
+ @GeneralUtilities.check_arguments
1996
+ def run_program_with_retry(self, program: str, arguments: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: str = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False, amount_of_attempts: int = 5, delay_in_seconds: int = 2) -> tuple[int, str, str, int]:
1997
+ return GeneralUtilities.retry_action(lambda: self.run_program(program, arguments, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace,arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output), amount_of_attempts, delay_in_seconds=delay_in_seconds)
1998
+
1999
+ # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
2000
+ @GeneralUtilities.check_arguments
2001
+ def run_program(self, program: str, arguments: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: str = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False) -> tuple[int, str, str, int]:
2002
+ if self.call_program_runner_directly:
2003
+ return self.program_runner.run_program(program, arguments, working_directory, custom_argument, interactive)
2004
+ return self.run_program_argsasarray(program, GeneralUtilities.arguments_to_array(arguments), working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, GeneralUtilities.arguments_to_array(arguments_for_log), throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output)
2005
+
2006
+ # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
2007
+ @GeneralUtilities.check_arguments
2008
+ def run_program_argsasarray_with_retry(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False, amount_of_attempts: int = 5, delay_in_seconds: int = 2) -> tuple[int, str, str, int]:
2009
+ return GeneralUtilities.retry_action(lambda: self.run_program_argsasarray(program, arguments_as_array, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace,arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output), amount_of_attempts, delay_in_seconds=delay_in_seconds)
2010
+
2011
+
2012
+ # Return-values program_runner: Pid
2013
+ @GeneralUtilities.check_arguments
2014
+ def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
2015
+ if self.call_program_runner_directly:
2016
+ return self.program_runner.run_program_argsasarray_async(program, arguments_as_array, working_directory, custom_argument, interactive)
2017
+ mock_loader_result = self.__try_load_mock(program, ' '.join(arguments_as_array), working_directory)
2018
+ if mock_loader_result[0]:
2019
+ return mock_loader_result[1]
2020
+ process: Popen = self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
2021
+ return process.pid
2022
+
2023
+ # Return-values program_runner: Pid
2024
+ @GeneralUtilities.check_arguments
2025
+ def run_program_async(self, program: str, arguments: str = "", working_directory: str = None,print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
2026
+ if self.call_program_runner_directly:
2027
+ return self.program_runner.run_program_argsasarray_async(program, arguments, working_directory, custom_argument, interactive)
2028
+ return self.run_program_argsasarray_async(program, GeneralUtilities.arguments_to_array(arguments), working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
2029
+
2030
+ @GeneralUtilities.check_arguments
2031
+ def __try_load_mock(self, program: str, arguments: str, working_directory: str) -> tuple[bool, tuple[int, str, str, int]]:
2032
+ if self.mock_program_calls:
2033
+ try:
2034
+ return [True, self.__get_mock_program_call(program, arguments, working_directory)]
2035
+ except LookupError:
2036
+ if not self.execute_program_really_if_no_mock_call_is_defined:
2037
+ raise
2038
+ return [False, None]
2039
+
2040
+ @GeneralUtilities.check_arguments
2041
+ def __adapt_workingdirectory(self, workingdirectory: str) -> str:
2042
+ result: str = None
2043
+ if workingdirectory is None:
2044
+ result = os.getcwd()
2045
+ else:
2046
+ if os.path.isabs(workingdirectory):
2047
+ result = workingdirectory
2048
+ else:
2049
+ result = GeneralUtilities.resolve_relative_path_from_current_working_directory(workingdirectory)
2050
+ if not os.path.isdir(result):
2051
+ raise ValueError(f"Working-directory '{workingdirectory}' does not exist.")
2052
+ return result
2053
+
2054
+ @GeneralUtilities.check_arguments
2055
+ def verify_no_pending_mock_program_calls(self):
2056
+ if (len(self.__mocked_program_calls) > 0):
2057
+ raise AssertionError("The following mock-calls were not called:\n"+",\n ".join([self.__format_mock_program_call(r) for r in self.__mocked_program_calls]))
2058
+
2059
+ @GeneralUtilities.check_arguments
2060
+ def __format_mock_program_call(self, r) -> str:
2061
+ r: ScriptCollectionCore.__MockProgramCall = r
2062
+ return f"'{r.workingdirectory}>{r.program} {r.argument}' (" \
2063
+ f"exitcode: {GeneralUtilities.str_none_safe(str(r.exit_code))}, " \
2064
+ f"pid: {GeneralUtilities.str_none_safe(str(r.pid))}, "\
2065
+ f"stdout: {GeneralUtilities.str_none_safe(str(r.stdout))}, " \
2066
+ f"stderr: {GeneralUtilities.str_none_safe(str(r.stderr))})"
2067
+
2068
+ @GeneralUtilities.check_arguments
2069
+ def register_mock_program_call(self, program: str, argument: str, workingdirectory: str, result_exit_code: int, result_stdout: str, result_stderr: str, result_pid: int, amount_of_expected_calls=1):
2070
+ "This function is for test-purposes only"
2071
+ for _ in itertools.repeat(None, amount_of_expected_calls):
2072
+ mock_call = ScriptCollectionCore.__MockProgramCall()
2073
+ mock_call.program = program
2074
+ mock_call.argument = argument
2075
+ mock_call.workingdirectory = workingdirectory
2076
+ mock_call.exit_code = result_exit_code
2077
+ mock_call.stdout = result_stdout
2078
+ mock_call.stderr = result_stderr
2079
+ mock_call.pid = result_pid
2080
+ self.__mocked_program_calls.append(mock_call)
2081
+
2082
+ @GeneralUtilities.check_arguments
2083
+ def __get_mock_program_call(self, program: str, argument: str, workingdirectory: str):
2084
+ result: ScriptCollectionCore.__MockProgramCall = None
2085
+ for mock_call in self.__mocked_program_calls:
2086
+ if ((re.match(mock_call.program, program) is not None)
2087
+ and (re.match(mock_call.argument, argument) is not None)
2088
+ and (re.match(mock_call.workingdirectory, workingdirectory) is not None)):
2089
+ result = mock_call
2090
+ break
2091
+ if result is None:
2092
+ raise LookupError(f"Tried to execute mock-call '{workingdirectory}>{program} {argument}' but no mock-call was defined for that execution")
2093
+ else:
2094
+ self.__mocked_program_calls.remove(result)
2095
+ return (result.exit_code, result.stdout, result.stderr, result.pid)
2096
+
2097
+ @GeneralUtilities.check_arguments
2098
+ class __MockProgramCall:
2099
+ program: str
2100
+ argument: str
2101
+ workingdirectory: str
2102
+ exit_code: int
2103
+ stdout: str
2104
+ stderr: str
2105
+ pid: int
2106
+
2107
+ @GeneralUtilities.check_arguments
2108
+ def run_with_epew_with_retry(self, program: str, argument: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: str =None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False,print_live_output:bool=False,encode_argument_in_base64:bool=False, amount_of_attempts: int = 3, delay_in_seconds: int = 2) -> tuple[int, str, str, int]:
2109
+ return GeneralUtilities.retry_action(lambda: self.run_with_epew(program, argument, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace,arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output,encode_argument_in_base64), amount_of_attempts, delay_in_seconds=delay_in_seconds)
2110
+
2111
+ @GeneralUtilities.check_arguments
2112
+ def run_with_epew(self, program: str, argument: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: str =None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False,print_live_output:bool=False,encode_argument_in_base64:bool=False) -> tuple[int, str, str, int]:
2113
+ epew_argument:list[str]=["-p",program ,"-w", working_directory]
2114
+ if encode_argument_in_base64:
2115
+ if arguments_for_log is None:
2116
+ argument_escaped=argument.replace("\"", "\\\"")
2117
+ arguments_for_log=epew_argument+["-a",f"\"{argument_escaped}\""]
2118
+ base64_string = base64.b64encode(argument.encode("utf-8")).decode("utf-8")
2119
+ epew_argument=epew_argument+["-a",base64_string,"-b"]
2120
+ else:
2121
+ epew_argument=epew_argument+["-a",argument]
2122
+ if arguments_for_log is None:
2123
+ arguments_for_log=epew_argument
2124
+ return self.run_program_argsasarray("epew", epew_argument, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive,print_live_output=print_live_output)
2125
+
2126
+
2127
+ # </run programs>
2128
+
2129
+ @GeneralUtilities.check_arguments
2130
+ def extract_archive_with_7z(self, unzip_program_file: str, zip_file: str, password: str, output_directory: str) -> None:
2131
+ password_set = not password is None
2132
+ file_name = Path(zip_file).name
2133
+ file_folder = os.path.dirname(zip_file)
2134
+ argument = "x"
2135
+ if password_set:
2136
+ argument = f"{argument} -p\"{password}\""
2137
+ argument = f"{argument} -o {output_directory}"
2138
+ argument = f"{argument} {file_name}"
2139
+ return self.run_program(unzip_program_file, argument, file_folder)
2140
+
2141
+ @GeneralUtilities.check_arguments
2142
+ def get_internet_time(self) -> datetime:
2143
+ response = ntplib.NTPClient().request('pool.ntp.org')
2144
+ return datetime.fromtimestamp(response.tx_time)
2145
+
2146
+ @GeneralUtilities.check_arguments
2147
+ def system_time_equals_internet_time(self, maximal_tolerance_difference: timedelta) -> bool:
2148
+ return abs(GeneralUtilities.get_now() - self.get_internet_time()) < maximal_tolerance_difference
2149
+
2150
+ @GeneralUtilities.check_arguments
2151
+ def system_time_equals_internet_time_with_default_tolerance(self) -> bool:
2152
+ return self.system_time_equals_internet_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
2153
+
2154
+ @GeneralUtilities.check_arguments
2155
+ def check_system_time(self, maximal_tolerance_difference: timedelta):
2156
+ if not self.system_time_equals_internet_time(maximal_tolerance_difference):
2157
+ raise ValueError("System time may be wrong")
2158
+
2159
+ @GeneralUtilities.check_arguments
2160
+ def check_system_time_with_default_tolerance(self) -> None:
2161
+ self.check_system_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
2162
+
2163
+ @GeneralUtilities.check_arguments
2164
+ def __get_default_tolerance_for_system_time_equals_internet_time(self) -> timedelta:
2165
+ return timedelta(hours=0, minutes=0, seconds=3)
2166
+
2167
+ @GeneralUtilities.check_arguments
2168
+ def increment_version(self, input_version: str, increment_major: bool, increment_minor: bool, increment_patch: bool) -> str:
2169
+ splitted = input_version.split(".")
2170
+ GeneralUtilities.assert_condition(len(splitted) == 3, f"Version '{input_version}' does not have the 'major.minor.patch'-pattern.")
2171
+ major = int(splitted[0])
2172
+ minor = int(splitted[1])
2173
+ patch = int(splitted[2])
2174
+ if increment_major:
2175
+ major = major+1
2176
+ if increment_minor:
2177
+ minor = minor+1
2178
+ if increment_patch:
2179
+ patch = patch+1
2180
+ return f"{major}.{minor}.{patch}"
2181
+
2182
+ @GeneralUtilities.check_arguments
2183
+ def get_semver_version_from_gitversion(self, repository_folder: str) -> str:
2184
+ self.assert_is_git_repository(repository_folder)
2185
+ if (self.git_repository_has_commits(repository_folder)):
2186
+ result = self.get_version_from_gitversion(repository_folder, "MajorMinorPatch")
2187
+ if self.git_repository_has_uncommitted_changes(repository_folder):
2188
+ if self.get_current_git_branch_has_tag(repository_folder):
2189
+ id_of_latest_tag = self.git_get_commit_id(repository_folder, self.get_latest_git_tag(repository_folder))
2190
+ current_commit = self.git_get_commit_id(repository_folder)
2191
+ current_commit_is_on_latest_tag = id_of_latest_tag == current_commit
2192
+ if current_commit_is_on_latest_tag:
2193
+ result = self.increment_version(result, False, False, True)
2194
+ else:
2195
+ result = "0.1.0"
2196
+ return result
2197
+
2198
+ @staticmethod
2199
+ @GeneralUtilities.check_arguments
2200
+ def is_patch_version(version_string: str) -> bool:
2201
+ return not version_string.endswith(".0")
2202
+
2203
+ @GeneralUtilities.check_arguments
2204
+ def get_version_from_gitversion(self, folder: str, variable: str) -> str:
2205
+ # called twice as workaround for issue 1877 in gitversion ( https://github.com/GitTools/GitVersion/issues/1877 )
2206
+ result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder)
2207
+ result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder)
2208
+ result = GeneralUtilities.strip_new_line_character(result[1])
2209
+
2210
+ return result
2211
+
2212
+ @GeneralUtilities.check_arguments
2213
+ def generate_certificate_authority(self, folder: str, name: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
2214
+ if days_until_expire is None:
2215
+ days_until_expire = 1825
2216
+ if password is None:
2217
+ password = GeneralUtilities.generate_password()
2218
+ GeneralUtilities.ensure_directory_exists(folder)
2219
+ self.run_program_argsasarray("openssl", ['req', '-new', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:prime256v1', '-days', str(days_until_expire), '-nodes', '-x509', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={name}/OU={subj_ou}', '-passout', f'pass:{password}', '-keyout', f'{name}.key', '-out', f'{name}.crt'], folder)
2220
+
2221
+ @GeneralUtilities.check_arguments
2222
+ def generate_certificate(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
2223
+ if days_until_expire is None:
2224
+ days_until_expire = 397
2225
+ if password is None:
2226
+ password = GeneralUtilities.generate_password()
2227
+ rsa_key_length = 4096
2228
+ self.run_program_argsasarray("openssl", ['genrsa', '-out', f'{filename}.key', f'{rsa_key_length}'], folder)
2229
+ self.run_program_argsasarray("openssl", ['req', '-new', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou}', '-x509', '-key', f'{filename}.key', '-out', f'{filename}.unsigned.crt', '-days', f'{days_until_expire}'], folder)
2230
+ self.run_program_argsasarray("openssl", ['pkcs12', '-export', '-out', f'{filename}.selfsigned.pfx', '-password', f'pass:{password}', '-inkey', f'{filename}.key', '-in', f'{filename}.unsigned.crt'], folder)
2231
+ GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.password"), password)
2232
+ GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.san.conf"), f"""[ req ]
2233
+ default_bits = {rsa_key_length}
2234
+ distinguished_name = req_distinguished_name
2235
+ req_extensions = v3_req
2236
+ default_md = sha256
2237
+ dirstring_type = nombstr
2238
+ prompt = no
2239
+
2240
+ [ req_distinguished_name ]
2241
+ countryName = {subj_c}
2242
+ stateOrProvinceName = {subj_st}
2243
+ localityName = {subj_l}
2244
+ organizationName = {subj_o}
2245
+ organizationUnit = {subj_ou}
2246
+ commonName = {domain}
2247
+
2248
+ [v3_req]
2249
+ subjectAltName = @subject_alt_name
2250
+
2251
+ [ subject_alt_name ]
2252
+ DNS = {domain}
2253
+ """)
2254
+
2255
+ @GeneralUtilities.check_arguments
2256
+ def generate_certificate_sign_request(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str) -> None:
2257
+ self.run_program_argsasarray("openssl", ['req', '-new', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou}', '-key', f'{filename}.key', f'-out', f'{filename}.csr', f'-config', f'{filename}.san.conf'], folder)
2258
+
2259
+ @GeneralUtilities.check_arguments
2260
+ def sign_certificate(self, folder: str, ca_folder: str, ca_name: str, domain: str, filename: str, days_until_expire: int = None) -> None:
2261
+ if days_until_expire is None:
2262
+ days_until_expire = 397
2263
+ ca = os.path.join(ca_folder, ca_name)
2264
+ password_file = os.path.join(folder, f"{filename}.password")
2265
+ password = GeneralUtilities.read_text_from_file(password_file)
2266
+ self.run_program_argsasarray("openssl", ['x509', '-req', '-in', f'{filename}.csr', '-CA', f'{ca}.crt', '-CAkey', f'{ca}.key', '-CAcreateserial', '-CAserial', f'{ca}.srl', '-out', f'{filename}.crt', '-days', str(days_until_expire), '-sha256', '-extensions', 'v3_req', '-extfile', f'{filename}.san.conf'], folder)
2267
+ self.run_program_argsasarray("openssl", ['pkcs12', '-export', '-out', f'{filename}.pfx', f'-inkey', f'{filename}.key', '-in', f'{filename}.crt', '-password', f'pass:{password}'], folder)
2268
+
2269
+ @GeneralUtilities.check_arguments
2270
+ def update_dependencies_of_python_in_requirementstxt_file(self, file: str, ignored_dependencies: list[str]):
2271
+ # TODO consider ignored_dependencies
2272
+ lines = GeneralUtilities.read_lines_from_file(file)
2273
+ new_lines = []
2274
+ for line in lines:
2275
+ if GeneralUtilities.string_has_content(line):
2276
+ new_lines.append(self.__get_updated_line_for_python_requirements(line.strip()))
2277
+ GeneralUtilities.write_lines_to_file(file, new_lines)
2278
+
2279
+ @GeneralUtilities.check_arguments
2280
+ def __get_updated_line_for_python_requirements(self, line: str) -> str:
2281
+ if "==" in line or "<" in line:
2282
+ return line
2283
+ elif ">" in line:
2284
+ try:
2285
+ # line is something like "cyclonedx-bom>=2.0.2" and the function must return with the updated version
2286
+ # (something like "cyclonedx-bom>=2.11.0" for example)
2287
+ package = line.split(">")[0]
2288
+ operator = ">=" if ">=" in line else ">"
2289
+ headers = {'Cache-Control': 'no-cache'}
2290
+ response = requests.get(f'https://pypi.org/pypi/{package}/json', timeout=5, headers=headers)
2291
+ latest_version = response.json()['info']['version']
2292
+ # TODO update only minor- and patch-version
2293
+ # TODO print info if there is a new major-version
2294
+ return package+operator+latest_version
2295
+ except:
2296
+ return line
2297
+ else:
2298
+ raise ValueError(f'Unexpected line in requirements-file: "{line}"')
2299
+
2300
+ @GeneralUtilities.check_arguments
2301
+ def update_dependencies_of_python_in_setupcfg_file(self, setup_cfg_file: str, ignored_dependencies: list[str]):
2302
+ # TODO consider ignored_dependencies
2303
+ lines = GeneralUtilities.read_lines_from_file(setup_cfg_file)
2304
+ new_lines = []
2305
+ requirement_parsing_mode = False
2306
+ for line in lines:
2307
+ new_line = line
2308
+ if (requirement_parsing_mode):
2309
+ if ("<" in line or "=" in line or ">" in line):
2310
+ updated_line = f" {self.__get_updated_line_for_python_requirements(line.strip())}"
2311
+ new_line = updated_line
2312
+ else:
2313
+ requirement_parsing_mode = False
2314
+ else:
2315
+ if line.startswith("install_requires ="):
2316
+ requirement_parsing_mode = True
2317
+ new_lines.append(new_line)
2318
+ GeneralUtilities.write_lines_to_file(setup_cfg_file, new_lines)
2319
+
2320
+ @GeneralUtilities.check_arguments
2321
+ def update_dependencies_of_dotnet_project(self, csproj_file: str, ignored_dependencies: list[str]):
2322
+ folder = os.path.dirname(csproj_file)
2323
+ csproj_filename = os.path.basename(csproj_file)
2324
+ self.log.log(f"Check for updates in {csproj_filename}", LogLevel.Information)
2325
+ result = self.run_program_with_retry("dotnet", f"list {csproj_filename} package --outdated", folder, print_errors_as_information=True)
2326
+ for line in result[1].replace("\r", GeneralUtilities.empty_string).split("\n"):
2327
+ # Relevant output-lines are something like " > NJsonSchema 10.7.0 10.7.0 10.9.0"
2328
+ if ">" in line:
2329
+ package_name = line.replace(">", GeneralUtilities.empty_string).strip().split(" ")[0]
2330
+ if not (package_name in ignored_dependencies):
2331
+ self.log.log(f"Update package {package_name}...", LogLevel.Debug)
2332
+ time.sleep(1.1) # attempt to prevent rate-limit
2333
+ self.run_program_with_retry("dotnet", f"add {csproj_filename} package {package_name}", folder, print_errors_as_information=True)
2334
+
2335
+ @GeneralUtilities.check_arguments
2336
+ def create_deb_package(self, toolname: str, binary_folder: str, control_file_content: str, deb_output_folder: str, permission_of_executable_file_as_octet_triple: int) -> None:
2337
+
2338
+ # prepare
2339
+ GeneralUtilities.ensure_directory_exists(deb_output_folder)
2340
+ temp_folder = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
2341
+ GeneralUtilities.ensure_directory_exists(temp_folder)
2342
+ bin_folder = binary_folder
2343
+ tool_content_folder_name = toolname+"Content"
2344
+
2345
+ # create folder
2346
+ GeneralUtilities.ensure_directory_exists(temp_folder)
2347
+ control_content_folder_name = "controlcontent"
2348
+ packagecontent_control_folder = os.path.join(temp_folder, control_content_folder_name)
2349
+ GeneralUtilities.ensure_directory_exists(packagecontent_control_folder)
2350
+ data_content_folder_name = "datacontent"
2351
+ packagecontent_data_folder = os.path.join(temp_folder, data_content_folder_name)
2352
+ GeneralUtilities.ensure_directory_exists(packagecontent_data_folder)
2353
+ entireresult_content_folder_name = "entireresultcontent"
2354
+ packagecontent_entireresult_folder = os.path.join(temp_folder, entireresult_content_folder_name)
2355
+ GeneralUtilities.ensure_directory_exists(packagecontent_entireresult_folder)
2356
+
2357
+ # create "debian-binary"-file
2358
+ debianbinary_file = os.path.join(packagecontent_entireresult_folder, "debian-binary")
2359
+ GeneralUtilities.ensure_file_exists(debianbinary_file)
2360
+ GeneralUtilities.write_text_to_file(debianbinary_file, "2.0\n")
2361
+
2362
+ # create control-content
2363
+
2364
+ # conffiles
2365
+ conffiles_file = os.path.join(packagecontent_control_folder, "conffiles")
2366
+ GeneralUtilities.ensure_file_exists(conffiles_file)
2367
+
2368
+ # postinst-script
2369
+ postinst_file = os.path.join(packagecontent_control_folder, "postinst")
2370
+ GeneralUtilities.ensure_file_exists(postinst_file)
2371
+ exe_file = f"/usr/bin/{tool_content_folder_name}/{toolname}"
2372
+ link_file = f"/usr/bin/{toolname.lower()}"
2373
+ permission = str(permission_of_executable_file_as_octet_triple)
2374
+ GeneralUtilities.write_text_to_file(postinst_file, f"""#!/bin/sh
2375
+ ln -s {exe_file} {link_file}
2376
+ chmod {permission} {exe_file}
2377
+ chmod {permission} {link_file}
2378
+ """)
2379
+
2380
+ # control
2381
+ control_file = os.path.join(packagecontent_control_folder, "control")
2382
+ GeneralUtilities.ensure_file_exists(control_file)
2383
+ GeneralUtilities.write_text_to_file(control_file, control_file_content)
2384
+
2385
+ # md5sums
2386
+ md5sums_file = os.path.join(packagecontent_control_folder, "md5sums")
2387
+ GeneralUtilities.ensure_file_exists(md5sums_file)
2388
+
2389
+ # create data-content
2390
+
2391
+ # copy binaries
2392
+ usr_bin_folder = os.path.join(packagecontent_data_folder, "usr/bin")
2393
+ GeneralUtilities.ensure_directory_exists(usr_bin_folder)
2394
+ usr_bin_content_folder = os.path.join(usr_bin_folder, tool_content_folder_name)
2395
+ GeneralUtilities.copy_content_of_folder(bin_folder, usr_bin_content_folder)
2396
+
2397
+ # create debfile
2398
+ deb_filename = f"{toolname}.deb"
2399
+ self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/control.tar.gz", "*"], packagecontent_control_folder)
2400
+ self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/data.tar.gz", "*"], packagecontent_data_folder)
2401
+ self.run_program_argsasarray("ar", ["r", deb_filename, "debian-binary", "control.tar.gz", "data.tar.gz"], packagecontent_entireresult_folder)
2402
+ result_file = os.path.join(packagecontent_entireresult_folder, deb_filename)
2403
+ shutil.copy(result_file, os.path.join(deb_output_folder, deb_filename))
2404
+
2405
+ # cleanup
2406
+ GeneralUtilities.ensure_directory_does_not_exist(temp_folder)
2407
+
2408
+ @GeneralUtilities.check_arguments
2409
+ def update_year_in_copyright_tags(self, file: str) -> None:
2410
+ current_year = str(GeneralUtilities.get_now().year)
2411
+ lines = GeneralUtilities.read_lines_from_file(file)
2412
+ lines_result = []
2413
+ for line in lines:
2414
+ if match := re.search("(.*<[Cc]opyright>.*)\\d\\d\\d\\d(.*<\\/[Cc]opyright>.*)", line):
2415
+ part1 = match.group(1)
2416
+ part2 = match.group(2)
2417
+ adapted = part1+current_year+part2
2418
+ else:
2419
+ adapted = line
2420
+ lines_result.append(adapted)
2421
+ GeneralUtilities.write_lines_to_file(file, lines_result)
2422
+
2423
+ @GeneralUtilities.check_arguments
2424
+ def update_year_in_first_line_of_file(self, file: str) -> None:
2425
+ current_year = str(GeneralUtilities.get_now().year)
2426
+ lines = GeneralUtilities.read_lines_from_file(file)
2427
+ lines[0] = re.sub("\\d\\d\\d\\d", current_year, lines[0])
2428
+ GeneralUtilities.write_lines_to_file(file, lines)
2429
+
2430
+ @GeneralUtilities.check_arguments
2431
+ def get_external_ip_address(self) -> str:
2432
+ information = self.get_externalnetworkinformation_as_json_string()
2433
+ parsed = json.loads(information)
2434
+ return parsed["IPAddress"]
2435
+
2436
+ @GeneralUtilities.check_arguments
2437
+ def get_country_of_external_ip_address(self) -> str:
2438
+ information = self.get_externalnetworkinformation_as_json_string()
2439
+ parsed = json.loads(information)
2440
+ return parsed["Country"]
2441
+
2442
+ @GeneralUtilities.check_arguments
2443
+ def get_externalnetworkinformation_as_json_string(self,clientinformation_link:str='https://clientinformation.anion327.de') -> str:
2444
+ headers = {'Cache-Control': 'no-cache'}
2445
+ response = requests.get(clientinformation_link, timeout=5, headers=headers)
2446
+ network_information_as_json_string = GeneralUtilities.bytes_to_string(response.content)
2447
+ return network_information_as_json_string
2448
+
2449
+ @GeneralUtilities.check_arguments
2450
+ def change_file_extensions(self, folder: str, from_extension: str, to_extension: str, recursive: bool, ignore_case: bool) -> None:
2451
+ extension_to_compare: str = None
2452
+ if ignore_case:
2453
+ extension_to_compare = from_extension.lower()
2454
+ else:
2455
+ extension_to_compare = from_extension
2456
+ for file in GeneralUtilities.get_direct_files_of_folder(folder):
2457
+ if (ignore_case and file.lower().endswith(f".{extension_to_compare}") or not ignore_case and file.endswith(f".{extension_to_compare}")):
2458
+ p = Path(file)
2459
+ p.rename(p.with_suffix('.'+to_extension))
2460
+ if recursive:
2461
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
2462
+ self.change_file_extensions(subfolder, from_extension, to_extension, recursive, ignore_case)
2463
+
2464
+ @GeneralUtilities.check_arguments
2465
+ def __add_chapter(self, main_reference_file, reference_content_folder, number: int, chaptertitle: str, content: str = None):
2466
+ if content is None:
2467
+ content = "TXDX add content here"
2468
+ filename = str(number).zfill(2)+"_"+chaptertitle.replace(' ', '-')
2469
+ file = f"{reference_content_folder}/{filename}.md"
2470
+ full_title = f"{number}. {chaptertitle}"
2471
+
2472
+ GeneralUtilities.append_line_to_file(main_reference_file, f"- [{full_title}](./{filename}.md)")
2473
+
2474
+ GeneralUtilities.ensure_file_exists(file)
2475
+ GeneralUtilities.write_text_to_file(file, f"""# {full_title}
2476
+
2477
+ {content}
2478
+ """.replace("XDX", "ODO"))
2479
+
2480
+ @GeneralUtilities.check_arguments
2481
+ def generate_arc42_reference_template(self, repository: str, productname: str = None, subfolder: str = None):
2482
+ productname: str
2483
+ if productname is None:
2484
+ productname = os.path.basename(repository)
2485
+ if subfolder is None:
2486
+ subfolder = "Other/Reference"
2487
+ reference_root_folder = f"{repository}/{subfolder}"
2488
+ reference_content_folder = reference_root_folder + "/Technical"
2489
+ if os.path.isdir(reference_root_folder):
2490
+ raise ValueError(f"The folder '{reference_root_folder}' does already exist.")
2491
+ GeneralUtilities.ensure_directory_exists(reference_root_folder)
2492
+ GeneralUtilities.ensure_directory_exists(reference_content_folder)
2493
+ main_reference_file = f"{reference_root_folder}/Reference.md"
2494
+ GeneralUtilities.ensure_file_exists(main_reference_file)
2495
+ GeneralUtilities.write_text_to_file(main_reference_file, f"""# {productname}
2496
+
2497
+ TXDX add minimal service-description here.
2498
+
2499
+ ## Technical documentation
2500
+
2501
+ """.replace("XDX", "ODO"))
2502
+ self.__add_chapter(main_reference_file, reference_content_folder, 1, 'Introduction and Goals', """## Overview
2503
+
2504
+ TXDX
2505
+
2506
+ ## Quality goals
2507
+
2508
+ TXDX
2509
+
2510
+ ## Stakeholder
2511
+
2512
+ | Name | How to contact | Reason |
2513
+ | ---- | -------------- | ------ |""")
2514
+ self.__add_chapter(main_reference_file, reference_content_folder, 2, 'Constraints', """## Technical constraints
2515
+
2516
+ | Constraint-identifier | Constraint | Reason |
2517
+ | --------------------- | ---------- | ------ |
2518
+
2519
+ ## Organizational constraints
2520
+
2521
+ | Constraint-identifier | Constraint | Reason |
2522
+ | --------------------- | ---------- | ------ |""")
2523
+ self.__add_chapter(main_reference_file, reference_content_folder, 3, 'Context and Scope', """## Context
2524
+
2525
+ TXDX
2526
+
2527
+ ## Scope
2528
+
2529
+ TXDX""")
2530
+ self.__add_chapter(main_reference_file, reference_content_folder, 4, 'Solution Strategy', """TXDX""")
2531
+ self.__add_chapter(main_reference_file, reference_content_folder, 5, 'Building Block View', """TXDX""")
2532
+ self.__add_chapter(main_reference_file, reference_content_folder, 6, 'Runtime View', """TXDX""")
2533
+ self.__add_chapter(main_reference_file, reference_content_folder, 7, 'Deployment View', """## Infrastructure-overview
2534
+
2535
+ TXDX
2536
+
2537
+ ## Infrastructure-requirements
2538
+
2539
+ TXDX
2540
+
2541
+ ## Deployment-proecsses
2542
+
2543
+ TXDX""")
2544
+ self.__add_chapter(main_reference_file, reference_content_folder, 8, 'Crosscutting Concepts', """TXDX""")
2545
+ self.__add_chapter(main_reference_file, reference_content_folder, 9, 'Architectural Decisions', """## Decision-board
2546
+
2547
+ | Decision-identifier | Date | Decision | Reason and notes |
2548
+ | ------------------- | ---- | -------- | ---------------- |""") # empty because there are no decsions yet
2549
+ self.__add_chapter(main_reference_file, reference_content_folder, 10, 'Quality Requirements', """TXDX""")
2550
+ self.__add_chapter(main_reference_file, reference_content_folder, 11, 'Risks and Technical Debt', """## Risks
2551
+
2552
+ Currently there are no known risks.
2553
+
2554
+ ## Technical debts
2555
+
2556
+ Currently there are no technical depts.""")
2557
+ self.__add_chapter(main_reference_file, reference_content_folder, 12, 'Glossary', """## Terms
2558
+
2559
+ | Term | Meaning |
2560
+ | ---- | ------- |
2561
+
2562
+ ## Abbreviations
2563
+
2564
+ | Abbreviation | Meaning |
2565
+ | ------------ | ------- |""")
2566
+
2567
+ GeneralUtilities.append_to_file(main_reference_file, """
2568
+
2569
+ ## Responsibilities
2570
+
2571
+ | Responsibility | Name and contact-information |
2572
+ | --------------- | ---------------------------- |
2573
+ | Pdocut-owner | TXDX |
2574
+ | Product-manager | TXDX |
2575
+ | Support | TXDX |
2576
+
2577
+ ## License & Pricing
2578
+
2579
+ TXDX
2580
+
2581
+ ## External resources
2582
+
2583
+ - [Repository](TXDX)
2584
+ - [Productive-System](TXDX)
2585
+ - [QualityCheck-system](TXDX)
2586
+ """.replace("XDX", "ODO"))
2587
+
2588
+ @GeneralUtilities.check_arguments
2589
+ def run_with_timeout(self, method, timeout_in_seconds: float) -> bool:
2590
+ # Returns true if the method was terminated due to a timeout
2591
+ # Returns false if the method terminates in the given time
2592
+ p = multiprocessing.Process(target=method)
2593
+ p.start()
2594
+ p.join(timeout_in_seconds)
2595
+ if p.is_alive():
2596
+ p.kill()
2597
+ p.join()
2598
+ return True
2599
+ else:
2600
+ return False
2601
+
2602
+ @GeneralUtilities.check_arguments
2603
+ def ensure_local_docker_network_exists(self, network_name: str) -> None:
2604
+ if not self.local_docker_network_exists(network_name):
2605
+ self.create_local_docker_network(network_name)
2606
+
2607
+ @GeneralUtilities.check_arguments
2608
+ def ensure_local_docker_network_does_not_exist(self, network_name: str) -> None:
2609
+ if self.local_docker_network_exists(network_name):
2610
+ self.remove_local_docker_network(network_name)
2611
+
2612
+ @GeneralUtilities.check_arguments
2613
+ def local_docker_network_exists(self, network_name: str) -> bool:
2614
+ return network_name in self.get_all_local_existing_docker_networks()
2615
+
2616
+ @GeneralUtilities.check_arguments
2617
+ def get_all_local_existing_docker_networks(self) -> list[str]:
2618
+ program_call_result = self.run_program("docker", "network list")
2619
+ std_out = program_call_result[1]
2620
+ std_out_lines = std_out.split("\n")[1:]
2621
+ result: list[str] = []
2622
+ for std_out_line in std_out_lines:
2623
+ normalized_line = ';'.join(std_out_line.split())
2624
+ splitted = normalized_line.split(";")
2625
+ result.append(splitted[1])
2626
+ return result
2627
+
2628
+ @GeneralUtilities.check_arguments
2629
+ def remove_local_docker_network(self, network_name: str) -> None:
2630
+ self.run_program("docker", f"network remove {network_name}")
2631
+
2632
+ @GeneralUtilities.check_arguments
2633
+ def create_local_docker_network(self, network_name: str) -> None:
2634
+ self.run_program("docker", f"network create {network_name}")
2635
+
2636
+ @GeneralUtilities.check_arguments
2637
+ def format_xml_file(self, file: str,add_xml_declaration:bool=True) -> None:
2638
+ encoding = "utf-8"
2639
+ element = ET.XML(GeneralUtilities.read_text_from_file(file, encoding))
2640
+ def trim_texts(elem: ET.Element):
2641
+ if elem.text:
2642
+ elem.text = elem.text.strip()
2643
+ if elem.tail:
2644
+ elem.tail = elem.tail.strip()
2645
+ for child in elem:
2646
+ trim_texts(child)
2647
+ trim_texts(element)
2648
+ ET.indent(element)
2649
+ content = ET.tostring(element, xml_declaration=add_xml_declaration, encoding="unicode")
2650
+ GeneralUtilities.write_text_to_file(file, content.rstrip("\n") + "\n", encoding)
2651
+
2652
+ @GeneralUtilities.check_arguments
2653
+ def format_html_file(self, file: str, add_html_declaration: bool = False) -> None:
2654
+ encoding = "utf-8"
2655
+ content = GeneralUtilities.read_text_from_file(file, encoding)
2656
+ content=self.format_html_content(content, add_html_declaration)
2657
+ GeneralUtilities.write_text_to_file(file, content, encoding)
2658
+
2659
+ @GeneralUtilities.check_arguments
2660
+ def format_html_content(self, content: str, add_html_declaration: bool = False) -> str:
2661
+
2662
+ VOID_ELEMENTS = {"area", "base", "br", "col", "embed", "hr", "img", "input",
2663
+ "link", "meta", "param", "source", "track", "wbr"}
2664
+
2665
+ class _Node:
2666
+ __slots__ = ("tag", "attrs", "children", "text", "is_void", "raw")
2667
+ def __init__(self, tag=None, attrs=(), text=None, is_void=False, raw=None):
2668
+ self.tag = tag
2669
+ self.attrs = list(attrs)
2670
+ self.children = []
2671
+ self.text = text
2672
+ self.is_void = is_void
2673
+ self.raw = raw
2674
+
2675
+ class _Builder(HTMLParser):
2676
+ def __init__(self):
2677
+ super().__init__(convert_charrefs=False)
2678
+ self.root = _Node()
2679
+ self.stack = [self.root]
2680
+
2681
+ def _top(self):
2682
+ return self.stack[-1]
2683
+
2684
+ def handle_starttag(self, tag, attrs):
2685
+ raw = self.get_starttag_text()
2686
+ raw_lower = raw.lower()
2687
+ original_attrs = []
2688
+ pos = 1 + len(tag)
2689
+ for lc_name, value in attrs:
2690
+ idx = raw_lower.index(lc_name, pos)
2691
+ original_attrs.append((raw[idx:idx + len(lc_name)], value))
2692
+ pos = idx + len(lc_name)
2693
+ node = _Node(tag=tag, attrs=original_attrs, is_void=tag.lower() in VOID_ELEMENTS)
2694
+ self._top().children.append(node)
2695
+ if not node.is_void:
2696
+ self.stack.append(node)
2697
+
2698
+ def handle_endtag(self, tag):
2699
+ if len(self.stack) > 1 and self.stack[-1].tag == tag:
2700
+ self.stack.pop()
2701
+
2702
+ def handle_data(self, data):
2703
+ t = " ".join(data.split())
2704
+ if t:
2705
+ self._top().children.append(_Node(text=t))
2706
+
2707
+ def handle_entityref(self, name):
2708
+ self._top().children.append(_Node(text=f"&{name};"))
2709
+
2710
+ def handle_charref(self, name):
2711
+ self._top().children.append(_Node(text=f"&#{name};"))
2712
+
2713
+ def handle_comment(self, data):
2714
+ self._top().children.append(_Node(raw=f"<!--{data}-->"))
2715
+
2716
+ def handle_decl(self, decl):
2717
+ self._top().children.append(_Node(raw=f"<!{decl}>"))
2718
+
2719
+ def handle_pi(self, data):
2720
+ self._top().children.append(_Node(raw=f"<?{data}>"))
2721
+
2722
+ _angular_exprs: list[str] = []
2723
+
2724
+ def _protect_angular(m: re.Match) -> str:
2725
+ idx = len(_angular_exprs)
2726
+ _angular_exprs.append(m.group(0))
2727
+ return f"__ANGEXPR{idx}__"
2728
+
2729
+ protected = re.sub(r'\{\{[\s\S]*?\}\}', _protect_angular, content)
2730
+ builder = _Builder()
2731
+ builder.feed(protected)
2732
+ ind = " "
2733
+
2734
+ def _serialize(node: _Node, depth: int) -> list:
2735
+ prefix = ind * depth
2736
+ if node.raw is not None:
2737
+ return [prefix + node.raw]
2738
+ if node.text is not None:
2739
+ return [node.text]
2740
+ if node.tag is None:
2741
+ out = []
2742
+ for c in node.children:
2743
+ out.extend(_serialize(c, depth))
2744
+ return out
2745
+ attr_str = "".join(f" {n}" if v is None else f' {n}="{v}"' for n, v in node.attrs)
2746
+ if node.is_void:
2747
+ return [f"{prefix}<{node.tag}{attr_str}>"]
2748
+ has_elem_children = any(c.tag is not None for c in node.children)
2749
+ if not has_elem_children:
2750
+ inner = "".join(c.text or "" for c in node.children)
2751
+ return [f"{prefix}<{node.tag}{attr_str}>{inner}</{node.tag}>"]
2752
+ lines = [f"{prefix}<{node.tag}{attr_str}>"]
2753
+ for c in node.children:
2754
+ child_lines = _serialize(c, depth + 1)
2755
+ if c.text is not None:
2756
+ lines.append(ind * (depth + 1) + child_lines[0])
2757
+ else:
2758
+ lines.extend(child_lines)
2759
+ lines.append(f"{prefix}</{node.tag}>")
2760
+ return lines
2761
+
2762
+ result = "\n".join(line for line in _serialize(builder.root, 0) if line.strip())
2763
+ for i, expr in enumerate(_angular_exprs):
2764
+ result = result.replace(f"__ANGEXPR{i}__", expr)
2765
+ if add_html_declaration and not result.lstrip().startswith("<!DOCTYPE"):
2766
+ result = "<!DOCTYPE html>\n" + result
2767
+ return result
2768
+
2769
+ @GeneralUtilities.check_arguments
2770
+ def get_pip_index_url_arguments_from_local_cache(self)->list[str]:
2771
+ arguments=[]
2772
+ pip_folder=GeneralUtilities.normalize_path(self.get_global_cache_folder()+"/Pip")
2773
+ if os.path.isdir(pip_folder):
2774
+ main_index_file=GeneralUtilities.normalize_path(os.path.join(pip_folder, "MainIndex.txt"))
2775
+ if os.path.isfile(main_index_file):
2776
+ lines=GeneralUtilities.read_nonempty_lines_from_file(main_index_file)
2777
+ url=[line for line in lines if line.startswith("IndexURL: ")][0].split(":")[1].strip()
2778
+ arguments.append("--index-url")
2779
+ arguments.append(url)
2780
+ extra_index_folder=GeneralUtilities.normalize_path(os.path.join(pip_folder, "ExtraIndexURLs"))
2781
+ if os.path.isdir(extra_index_folder):
2782
+ index_files=GeneralUtilities.get_direct_files_of_folder(extra_index_folder)
2783
+ if len(index_files) > 0:
2784
+ for indexurl_file in index_files:
2785
+ lines=GeneralUtilities.read_nonempty_lines_from_file(indexurl_file)
2786
+ url=[line for line in lines if line.startswith("IndexURL: ")][0].split(":")[1].strip()
2787
+ arguments.append("--extra-index-url")
2788
+ arguments.append(url)
2789
+ return arguments
2790
+
2791
+ @GeneralUtilities.check_arguments
2792
+ def install_requirementstxt_file(self, requirements_txt_file: str):
2793
+ folder: str = os.path.dirname(requirements_txt_file)
2794
+ filename: str = os.path.basename(requirements_txt_file)
2795
+ arguments= ["install", "-r", filename]
2796
+ for argument in self.get_pip_index_url_arguments_from_local_cache():
2797
+ arguments.append(argument)
2798
+ self.run_program_argsasarray("pip", arguments, folder,print_live_output=self.log.loglevel==LogLevel.Debug)
2799
+
2800
+ @GeneralUtilities.check_arguments
2801
+ def ocr_analysis_of_folder_using_local_docker_image(self, folder: str, extensions: list[str], languages: list[str],base_folder_for_entry: str,ignore_pattern:list[str] ) -> list[str]: # Returns a list of changed files due to ocr-analysis.
2802
+ #TODO start docker server
2803
+ serviceaddress:str=None#TODO
2804
+ self.ocr_analysis_of_folder(folder, serviceaddress, extensions, languages, base_folder_for_entry,ignore_pattern)
2805
+ #TODO stop docker server
2806
+
2807
+ @GeneralUtilities.check_arguments
2808
+ def ocr_analysis_of_folder(self, folder: str, serviceaddress: str, extensions: list[str], languages: list[str],base_folder_for_entry: str,ignore_pattern:list[str] ) -> list[str]: # Returns a list of changed files due to ocr-analysis.
2809
+ supported_extensions = ['png', 'jpg', 'jpeg', 'tiff', 'bmp', 'gif', 'pdf', 'docx', 'doc', 'xlsx', 'xls', 'pptx', 'ppt']
2810
+ changes_files: list[str] = []
2811
+ if base_folder_for_entry is None:
2812
+ base_folder_for_entry=folder
2813
+ if extensions is None:
2814
+ extensions = supported_extensions
2815
+ for file in GeneralUtilities.get_direct_files_of_folder(folder):
2816
+ if GeneralUtilities.is_ignored_by_glob_pattern(os.path.dirname(file),file,ignore_pattern):
2817
+ continue
2818
+ file_lower = file.lower()
2819
+ for extension in extensions:
2820
+ if file_lower.endswith("."+extension):
2821
+ if self.ocr_analysis_of_file(file, serviceaddress, languages,base_folder_for_entry):
2822
+ changes_files.append(file)
2823
+ break
2824
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
2825
+ if GeneralUtilities.is_ignored_by_glob_pattern(os.path.dirname(subfolder),subfolder,ignore_pattern):
2826
+ continue
2827
+ for file in self.ocr_analysis_of_folder(subfolder, serviceaddress, extensions, languages,base_folder_for_entry+"/"+os.path.basename(subfolder), ignore_pattern):
2828
+ changes_files.append(file)
2829
+ return changes_files
2830
+
2831
+
2832
+ @GeneralUtilities.check_arguments
2833
+ def __it_supported_extension(self, file: str, supported_extensions: list[str]) -> bool:
2834
+ file_lower = file.lower()
2835
+ for extension in supported_extensions:
2836
+ if file_lower.endswith("."+extension):
2837
+ return True
2838
+ return False
2839
+
2840
+ @GeneralUtilities.check_arguments
2841
+ def ocr_analysis_of_file(self, file: str, serviceaddress: str, languages: list[str], readable_folder_entry:str ) -> bool: # Returns true if the ocr-file was generated or updated. Returns false if the existing ocr-file was not changed.
2842
+ supported_extensions = ['png', 'jpg', 'jpeg', 'tiff', 'bmp', 'webp', 'gif', 'pdf', 'rtf', 'docx', 'doc', 'odt', 'xlsx', 'xls', 'ods', 'pptx', 'ppt', 'odp']
2843
+ if not self.__it_supported_extension(file, supported_extensions):
2844
+ raise ValueError(f"File '{file}' is not supported due to unsupported extension. Supported extensions are: {', '.join(supported_extensions)}")
2845
+ target_file = file+".ocr.txt"
2846
+ hash_of_current_file: str = GeneralUtilities.get_sha256_of_file(file)
2847
+ try:
2848
+ if os.path.isfile(target_file):
2849
+ lines = GeneralUtilities.read_lines_from_file(target_file)
2850
+ previous_hash_of_current_file: str = lines[1].split(":")[1].strip()
2851
+ if hash_of_current_file == previous_hash_of_current_file:
2852
+ return False
2853
+ except:
2854
+ pass
2855
+ GeneralUtilities.write_message_to_stdout(f"Starting OCR-analysis of file \"{file}\"...")
2856
+ ocr_content = self.get_ocr_content_of_file(file, serviceaddress, languages)
2857
+ GeneralUtilities.ensure_file_exists(target_file)
2858
+ if readable_folder_entry is None:
2859
+ readable_folder_entry="."
2860
+ readable_folder_entry=readable_folder_entry.replace("\\", "/")
2861
+ GeneralUtilities.write_text_to_file(target_file, f"""Name of file: \"{readable_folder_entry}/{os.path.basename(file)}\""
2862
+ Hash of file: {hash_of_current_file}
2863
+ OCR-content:
2864
+ {ocr_content}""")
2865
+ return True
2866
+
2867
+ @GeneralUtilities.check_arguments
2868
+ def get_ocr_content_of_file(self, file: str, serviceaddress: str, languages: list[str]) -> str:
2869
+ result: str = None
2870
+ extension = Path(file).suffix[1:]
2871
+ mime_types = {
2872
+ "pdf": "application/pdf",
2873
+ "png": "image/png",
2874
+ "jpg": "image/jpeg",
2875
+ "jpeg": "image/jpeg",
2876
+ "txt": "text/plain",
2877
+ "json": "application/json",
2878
+ "doc": "application/msword",
2879
+ "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
2880
+ "xls": "application/vnd.ms-excel",
2881
+ "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
2882
+ }
2883
+ if serviceaddress is None:
2884
+ server_url_file:str= GeneralUtilities.normalize_path(f"{str(Path.home())}/.ScriptCollection/OCR/ServiceURL.txt")
2885
+ if os.path.isfile(server_url_file):
2886
+ for line in GeneralUtilities.read_nonempty_lines_from_file(server_url_file):
2887
+ if not line.startswith("#"):
2888
+ serviceaddress = line.strip()
2889
+ break
2890
+ GeneralUtilities.assert_not_null(serviceaddress, "ocr-service-address must not be null.")
2891
+ mime_type = mime_types.get(extension.lower(), "application/octet-stream")
2892
+ service_url: str = f"{serviceaddress}/API/v1/SimpleOCR/GetOCRContent?mimeType={mime_type}"
2893
+ for language in languages:
2894
+ service_url = service_url + f"&languages={language}"
2895
+ headers = {'Cache-Control': 'no-cache'}
2896
+ with open(file, "rb") as f:
2897
+ files_to_analyse = {
2898
+ "fileContent": (os.path.basename(file), f, mime_type)
2899
+ }
2900
+ r = requests.put(service_url, timeout=3600, headers=headers, files=files_to_analyse,verify=True)
2901
+ if r.status_code != 200:
2902
+ if r.status_code == 400:
2903
+ return f"Could not calculate ocr-content for file \"{file}\". File may be broken."
2904
+ else:
2905
+ raise ValueError(f"Retrieving ocr-content for file \"{file}\" resulted in HTTP-response-code {r.status_code}.")
2906
+
2907
+ result = GeneralUtilities.bytes_to_string(r.content)
2908
+ return result
2909
+
2910
+ @GeneralUtilities.check_arguments
2911
+ def ocr_analysis_of_repository(self, folder: str, serviceaddress: str, extensions: list[str], languages: list[str]) -> None:
2912
+ self.assert_is_git_repository(folder)
2913
+ self.ocr_analysis_of_folder(folder, serviceaddress, extensions, languages,".",[".git"])
2914
+
2915
+ @GeneralUtilities.check_arguments
2916
+ def update_timestamp_in_file(self, target_file: str) -> None:
2917
+ lines = GeneralUtilities.read_lines_from_file(target_file)
2918
+ new_lines = []
2919
+ prefix: str = "# last update: "
2920
+ for line in lines:
2921
+ if line.startswith(prefix):
2922
+ new_lines.append(prefix+GeneralUtilities.datetime_to_string_for_readable_entry(GeneralUtilities.get_now(),False))
2923
+ else:
2924
+ new_lines.append(line)
2925
+ GeneralUtilities.write_lines_to_file(target_file, new_lines)
2926
+
2927
+ @GeneralUtilities.check_arguments
2928
+ def do_and_log_task(self, name_of_task: str, task,log_end_of_Task:bool=True)->int:
2929
+ try:
2930
+ self.log.log(f"Start action \"{name_of_task}\".", LogLevel.Information)
2931
+ result = task()
2932
+ if result is None:
2933
+ result = 0
2934
+ return result
2935
+ except Exception as e:
2936
+ self.log.log_exception(f"Error while running action \"{name_of_task}\".", e, LogLevel.Error)
2937
+ return 1
2938
+ finally:
2939
+ if log_end_of_Task:
2940
+ self.log.log(f"Finished action \"{name_of_task}\".", LogLevel.Information)
2941
+
2942
+
2943
+ default_excluded_patterns_for_loc: list[str] = ["**.txt", "**.md", "**.svg", "**.xlf", "**.vscode", "**/Resources/**", "**/Reference/**", ".gitignore", ".gitattributes", "Other/Metrics/**"]
2944
+
2945
+ @GeneralUtilities.check_arguments
2946
+ def get_lines_of_code_with_default_excluded_patterns(self, repository: str) -> int:
2947
+ return self.get_lines_of_code(repository, self.default_excluded_patterns_for_loc)
2948
+
2949
+ @GeneralUtilities.check_arguments
2950
+ def get_lines_of_code(self, repository: str, excluded_pattern: list[str]) -> int:
2951
+ self.assert_is_git_repository(repository)
2952
+ result: int = 0
2953
+ self.log.log(f"Calculate lines of code in repository '{repository}' with excluded patterns: {', '.join(excluded_pattern)}",LogLevel.Debug)
2954
+ git_response = self.run_program("git", "ls-files", repository)
2955
+ files: list[str] = GeneralUtilities.string_to_lines(git_response[1])
2956
+ for file in files:
2957
+ if os.path.isfile(os.path.join(repository, file)):
2958
+ if self.__is_excluded_by_glob_pattern(file, excluded_pattern):
2959
+ self.log.log(f"File '{file}' is ignored because it matches an excluded pattern.",LogLevel.Diagnostic)
2960
+ else:
2961
+ full_file: str = os.path.join(repository, file)
2962
+ if GeneralUtilities.is_binary_file(full_file):
2963
+ self.log.log(f"File '{file}' is ignored because it is a binary-file.",LogLevel.Diagnostic)
2964
+ else:
2965
+ self.log.log(f"Count lines of file '{file}'.",LogLevel.Diagnostic)
2966
+ length = len(GeneralUtilities.read_nonempty_lines_from_file(full_file))
2967
+ result = result+length
2968
+ else:
2969
+ self.log.log(f"File '{file}' is ignored because it does not exist.",LogLevel.Diagnostic)
2970
+ return result
2971
+
2972
+ @GeneralUtilities.check_arguments
2973
+ def __is_excluded_by_glob_pattern(self, file: str, excluded_patterns: list[str]) -> bool:
2974
+ for pattern in excluded_patterns:
2975
+ if fnmatch.fnmatch(file, pattern):
2976
+ return True
2977
+ return False
2978
+
2979
+ @GeneralUtilities.check_arguments
2980
+ def create_zip_archive(self, folder:str,zip_file:str) -> None:
2981
+ GeneralUtilities.assert_folder_exists(folder)
2982
+ GeneralUtilities.assert_file_does_not_exist(zip_file)
2983
+ folder = os.path.abspath(folder)
2984
+ with zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED) as zipf:
2985
+ for root, _, files in os.walk(folder):
2986
+ for file in files:
2987
+ file_path = os.path.join(root, file)
2988
+ arcname = os.path.relpath(file_path, start=folder)
2989
+ zipf.write(file_path, arcname)
2990
+
2991
+ @GeneralUtilities.check_arguments
2992
+ def start_local_test_service(self, file: str):
2993
+ example_folder = os.path.dirname(file)
2994
+ docker_compose_file = os.path.join(example_folder, "docker-compose.yml")
2995
+ for service in self.get_services_from_yaml_file(docker_compose_file):
2996
+ self.kill_docker_container(service)
2997
+ example_name = os.path.basename(example_folder)
2998
+ title = f"Test{example_name}"
2999
+ argument=f"compose -p {title.lower()}"
3000
+ if os.path.isfile(os.path.join(example_folder,"Parameters.env")):
3001
+ argument=argument+" --env-file Parameters.env"
3002
+ argument=argument+" up --detach"
3003
+ self.run_program("docker", argument, example_folder, title=title,print_live_output=True)
3004
+
3005
+ @GeneralUtilities.check_arguments
3006
+ def stop_local_test_service(self, file: str):
3007
+ example_folder = os.path.dirname(file)
3008
+ example_name = os.path.basename(example_folder)
3009
+ title = f"Test{example_name}"
3010
+ self.run_program("docker", f"compose -p {title.lower()} down", example_folder, title=title,print_live_output=True)
3011
+
3012
+ @GeneralUtilities.check_arguments
3013
+ def generate_chart_diagram(self,source_file:str,target_file:str):
3014
+ workingfolder=os.path.dirname(source_file)
3015
+ argument=f"\"{source_file}\" \"{target_file}\""
3016
+ loglevelMap = {
3017
+ LogLevel.Error: "error",
3018
+ LogLevel.Warning: "warn",
3019
+ LogLevel.Information: "info",
3020
+ LogLevel.Debug: "debug",
3021
+ }
3022
+ if self.log.loglevel==LogLevel.Debug:
3023
+ argument=f"-l {loglevelMap[self.log.loglevel]} {argument}"
3024
+ self.run_with_epew("vl2svg",argument,workingfolder,encode_argument_in_base64=True)
3025
+ #this uses vega-light. to use vega "vg2svg" should be used instead.
3026
+
3027
+ @GeneralUtilities.check_arguments
3028
+ def inspect_container(self, container_name: str) :
3029
+ program_result = self.run_program(
3030
+ "docker",
3031
+ f"inspect {container_name}",
3032
+ throw_exception_if_exitcode_is_not_zero=True
3033
+ )
3034
+ stdout=program_result[1]
3035
+
3036
+ data = json.loads(stdout)
3037
+ GeneralUtilities.assert_condition(len(data)==1,f"Unexpected array-length of docker-inspect-output for container \"{container_name}\".")
3038
+ return data[0]
3039
+
3040
+ @GeneralUtilities.check_arguments
3041
+ def container_is_exists(self,container_name:str)->bool:
3042
+ program_result = self.run_program(
3043
+ "docker",
3044
+ f"inspect {container_name}",
3045
+ throw_exception_if_exitcode_is_not_zero=False
3046
+ )
3047
+ return program_result[0]==0
3048
+
3049
+ @GeneralUtilities.check_arguments
3050
+ def container_is_running(self,container_name:str)->bool:
3051
+ data = self.inspect_container( container_name)
3052
+ if data is None:
3053
+ return False
3054
+
3055
+ return data["State"]["Status"] == "running"
3056
+
3057
+ @GeneralUtilities.check_arguments
3058
+ def container_is_healthy(self,container_name:str)->bool:
3059
+ data = self.inspect_container( container_name)
3060
+ if data is None:
3061
+ return False
3062
+
3063
+ state = data["State"]
3064
+ health = state.get("Health")
3065
+
3066
+ if health is None:
3067
+ return False # kein HEALTHCHECK definiert
3068
+
3069
+ return health["Status"] == "healthy"
3070
+
3071
+ @GeneralUtilities.check_arguments
3072
+ def get_output_of_container(self,container_name:str)->str:
3073
+
3074
+ program_result= self.run_program_argsasarray(
3075
+ "docker",
3076
+ ["logs",container_name],
3077
+ throw_exception_if_exitcode_is_not_zero=False
3078
+ )
3079
+ exit_code=program_result[0]
3080
+ stdout=program_result[1]
3081
+ stderr=program_result[2]
3082
+ if exit_code != 0:
3083
+ return ""
3084
+
3085
+ return stdout+"\n"+stderr
3086
+
3087
+ @GeneralUtilities.check_arguments
3088
+ def container_is_running_and_healthy(self,container_name:str)->bool:
3089
+ if not self.container_is_exists(container_name):
3090
+ return False
3091
+ if not self.container_is_running(container_name):
3092
+ return False
3093
+ if not self.container_is_healthy(container_name):
3094
+ return False
3095
+ return True
3096
+
3097
+ def reclaim_space_from_docker(self,remove_containers:bool,remove_volumes:bool,remove_images:bool, amount_of_attempts: int = 5):
3098
+ self.log.log("Reclaim disk space from docker...",LogLevel.Debug)
3099
+ if remove_containers:
3100
+ self.run_program_with_retry("docker","container prune -f",amount_of_attempts=amount_of_attempts)
3101
+ if remove_volumes:
3102
+ self.run_program_with_retry("docker","volume prune -f",amount_of_attempts=amount_of_attempts)
3103
+ if remove_images:
3104
+ self.run_program_with_retry("docker","image prune -a -f",amount_of_attempts=amount_of_attempts)
3105
+ self.run_program_with_retry("docker","builder prune -a -f",amount_of_attempts=amount_of_attempts)
3106
+ self.run_program_with_retry("docker","buildx prune -a -f",amount_of_attempts=amount_of_attempts,throw_exception_if_exitcode_is_not_zero=False) # buildx prune is not available on every machine.
3107
+ self.run_program_with_retry("docker","system df",print_live_output=self.log.loglevel==LogLevel.Debug,amount_of_attempts=amount_of_attempts)
3108
+
3109
+ @GeneralUtilities.check_arguments
3110
+ def get_docker_networks(self)->list[str]:
3111
+ program_result=self.run_program("docker","network list")
3112
+ result=[]
3113
+ lines=program_result[1].split("\n")[1:]
3114
+ for line in lines:
3115
+ splitted=[item for item in line.split(' ') if GeneralUtilities.string_has_content(item)]
3116
+ result.append(splitted[1].replace("\n","").replace("\r","").strip())
3117
+ return result
3118
+
3119
+ @GeneralUtilities.check_arguments
3120
+ def ensure_docker_network_is_available(self,network_name:str):
3121
+ #TODO add cli-script to call this function
3122
+ if not (network_name in self.get_docker_networks()):
3123
+ self.run_program("docker",f"network create {network_name}")
3124
+
3125
+ @GeneralUtilities.check_arguments
3126
+ def ensure_docker_network_is_not_available(self,network_name:str):
3127
+ #TODO add cli-script to call this function
3128
+ if network_name in self.get_docker_networks():
3129
+ self.run_program("docker",f"network rm {network_name}")
3130
+
3131
+ @GeneralUtilities.check_arguments
3132
+ def get_available_cultures_for_angular_app(self,angular_json_file:str)->list[str]:
3133
+ languages = ["en"]
3134
+ with open(angular_json_file, "r", encoding="utf-8") as f:
3135
+ data = json.load(f)
3136
+ for project in data.get("projects", {}).values():
3137
+ i18n = project.get("i18n", {})
3138
+ locales = i18n.get("locales", {})
3139
+ languages.extend(locales.keys())
3140
+
3141
+ languages=list(languages)
3142
+ return languages
3143
+
3144
+ @GeneralUtilities.check_arguments
3145
+ def parse_tasks_from_codeworkspace_file(self,code_workspace_file:str)->list[VSCodeWorkspaceShellTask]:
3146
+ result=[]
3147
+ jsoncontent = json.loads(GeneralUtilities.read_text_from_file(code_workspace_file))
3148
+ tasks = jsoncontent["tasks"]["tasks"]
3149
+ for task in tasks:
3150
+ if task["type"] == "shell":
3151
+ label: str = task["label"]
3152
+ name: str = GeneralUtilities.to_pascal_case(label)
3153
+ command:str= task["command"]
3154
+ work_dir:str = None
3155
+
3156
+ if "options" in task:
3157
+ options = task["options"]
3158
+ if "cwd" in options:
3159
+ work_dir = options["cwd"]
3160
+ work_dir = work_dir.replace("${workspaceFolder}", ".")
3161
+
3162
+ command_with_args = command
3163
+ if "args" in task:
3164
+ args = task["args"]
3165
+ if len(args) > 1:
3166
+ command_with_args = f"{command_with_args} {' '.join(args)}"
3167
+
3168
+ description: str =None
3169
+ if "description" in task:
3170
+ description = f'{label} ({task["description"]})'
3171
+ else:
3172
+ description = label
3173
+
3174
+
3175
+ alias_list:list[str]=[]
3176
+ name_lower=name.lower()
3177
+ if name!=name.lower():
3178
+ alias_list.append(name_lower)
3179
+
3180
+ if "aliases" in task:
3181
+ aliases = task["aliases"]
3182
+ for alias in aliases:
3183
+ alias_list.append(alias)
3184
+
3185
+ allow_custom_arguments:bool=False
3186
+ if "allowcustomarguments" in task:
3187
+ allow_custom_arguments = task["allowcustomarguments"]
3188
+
3189
+ result.append(VSCodeWorkspaceShellTask(name, description, work_dir, command_with_args, alias_list, allow_custom_arguments))
3190
+ return result
3191
+
3192
+ @GeneralUtilities.check_arguments
3193
+ def parse_mongodbconnection_from_codeworkspace_file(self,code_workspace_file:str)->list[VSCodeWorkspaceMongoDBConnection]:
3194
+ result=[]
3195
+ jsoncontent = json.loads(GeneralUtilities.read_text_from_file(code_workspace_file))
3196
+ settings=jsoncontent["settings"]
3197
+ if "mdb.presetConnections" in settings:
3198
+ connections = settings["mdb.presetConnections"]
3199
+ for connection in connections:
3200
+ result.append(VSCodeWorkspaceMongoDBConnection(connection["name"],connection["connectionString"]))
3201
+ return result
3202
+
3203
+ @GeneralUtilities.check_arguments
3204
+ def parse_sqlconnection_from_codeworkspace_file(self,code_workspace_file:str)->list[VSCodeWorkspaceMariaDBConnection]:
3205
+ result=[]
3206
+ jsoncontent = json.loads(GeneralUtilities.read_text_from_file(code_workspace_file))
3207
+ settings=jsoncontent["settings"]
3208
+ if "sqltools.connections" in settings:
3209
+ connections = settings["sqltools.connections"]
3210
+ for connection in connections:
3211
+ result.append(VSCodeWorkspaceMariaDBConnection(connection["name"],connection["server"],connection["port"],connection["database"],connection["username"],connection["password"]))
3212
+ return result
3213
+
3214
+ @GeneralUtilities.check_arguments
3215
+ def is_xliff2_file(self,file: str) -> bool:
3216
+ tree = ET.parse(file)
3217
+ root = tree.getroot()
3218
+
3219
+ tag = root.tag # "{urn:oasis:names:tc:xliff:document:2.0}xliff"
3220
+
3221
+ if tag.startswith("{"):
3222
+ namespace, localname = tag[1:].split("}", 1)
3223
+ else:
3224
+ namespace = None
3225
+ localname = tag
3226
+
3227
+ if localname != "xliff":
3228
+ return False
3229
+
3230
+ if namespace != "urn:oasis:names:tc:xliff:document:2.0":
3231
+ return False
3232
+
3233
+ if root.get("version") != "2.0":
3234
+ return False
3235
+
3236
+ return True
3237
+
3238
+ @GeneralUtilities.check_arguments
3239
+ def __sync_xlf2_files(self,base_file:ET.ElementTree, language_files:dict [
3240
+ str,#filepath
3241
+ ET.ElementTree#parsed file
3242
+ ]):
3243
+ """This function assumes that all files are valid xliff2 files and that the base file is the reference for syncing.
3244
+ This function adds new entries from the base file to the language files if they do not already exist using the value from base_file.
3245
+ This function removes entries from the language files if they do not exist in the base file anymore.
3246
+ In the end the updated language files are written to the disk. The base file is not changed."""
3247
+ #The file which was parsed looks like:
3248
+ #<?xml version="1.0" encoding="UTF-8" ?>
3249
+ #<xliff version="2.0" xmlns="urn:oasis:names:tc:xliff:document:2.0" srcLang="en">
3250
+ # <file id="ngi18n" original="ng.template">
3251
+ # <unit id="logingreeting">
3252
+ # <notes>
3253
+ # <note category="location">src/app/modules/home-page/login-form/login-form.component.html:2,4</note>
3254
+ # </notes>
3255
+ # <segment>
3256
+ # <source>Welcome back, please login</source>
3257
+ # </segment>
3258
+ # </unit>
3259
+ # <unit id="username">
3260
+ # <notes>
3261
+ # <note category="location">src/app/modules/home-page/login-form/login-form.component.html:5,6</note>
3262
+ # </notes>
3263
+ # <segment>
3264
+ # <source>Username</source>
3265
+ # </segment>
3266
+ # </unit>
3267
+ # <unit id="password">
3268
+ # <notes>
3269
+ # <note category="location">src/app/modules/home-page/login-form/login-form.component.html:12,13</note>
3270
+ # </notes>
3271
+ # <segment>
3272
+ # <source>Password</source>
3273
+ # </segment>
3274
+ # </unit>
3275
+ # </file>
3276
+ #</xliff>
3277
+
3278
+ NS = "urn:oasis:names:tc:xliff:document:2.0"
3279
+ NSMAP = {"x": NS}
3280
+ base_root = base_file.getroot()
3281
+ base_file_element = base_root.find("x:file", namespaces=NSMAP)
3282
+ if base_file_element is None:
3283
+ raise ValueError("Invalid XLIFF base file: <file> element not found")
3284
+
3285
+ # Collect base units
3286
+ base_units = {
3287
+ unit.get("id"): unit
3288
+ for unit in base_file_element.findall("x:unit", namespaces=NSMAP)
3289
+ }
3290
+ base_ids = set(base_units.keys())
3291
+ for filepath, lang_tree in language_files.items():
3292
+ lang_root = lang_tree.getroot()
3293
+ lang_file_element = lang_root.find("x:file", namespaces=NSMAP)
3294
+ if lang_file_element is None:
3295
+ raise ValueError(f"{filepath}: <file> element not found")
3296
+
3297
+ # Collect language units
3298
+ lang_units = {
3299
+ unit.get("id"): unit
3300
+ for unit in lang_file_element.findall("x:unit", namespaces=NSMAP)
3301
+ }
3302
+ lang_ids = set(lang_units.keys())
3303
+
3304
+ # Remove obsolete units
3305
+ obsolete_ids = lang_ids - base_ids
3306
+ for unit_id in obsolete_ids:
3307
+ lang_file_element.remove(lang_units[unit_id])
3308
+
3309
+ # Add missing units
3310
+ missing_ids = base_ids - lang_ids
3311
+ for unit_id in missing_ids:
3312
+ new_unit = copy.deepcopy(base_units[unit_id])
3313
+ lang_file_element.append(new_unit)
3314
+
3315
+ # Reorder units to match base order
3316
+ current_units = {
3317
+ unit.get("id"): unit
3318
+ for unit in lang_file_element.findall("x:unit", namespaces=NSMAP)
3319
+ }
3320
+ for unit in list(lang_file_element.findall("x:unit", namespaces=NSMAP)):
3321
+ lang_file_element.remove(unit)
3322
+ for unit_id in base_units.keys():
3323
+ if unit_id in current_units:
3324
+ lang_file_element.append(current_units[unit_id])
3325
+
3326
+ #TODO if a translation-unit has the "new"-attribute: set its value from the fallback-language. (if the culture contains a "-": e. g. take value from "de" as fallback-value for "de-AT"; or else: take value from base_file as fallback-value)
3327
+
3328
+ # Write file back to disk
3329
+ ET.register_namespace("", NS) # Ensure default namespace is declared without prefix
3330
+ Path(filepath).write_bytes(
3331
+ ET.tostring(
3332
+ lang_tree.getroot(),
3333
+ xml_declaration=True,
3334
+ encoding="UTF-8"
3335
+ )
3336
+ )
3337
+ ScriptCollectionCore().format_xml_file(filepath)
3338
+
3339
+ @GeneralUtilities.check_arguments
3340
+ def sync_xlf2_files(self,prefix:str, languages:list[str], folder:str):
3341
+ #languages=["de", "fr"] for example. the default-language (usually english) must not be included.
3342
+ base_file=os.path.join(folder, f"{prefix}.xlf")
3343
+ base_file_xml:ET.ElementTree=ET.parse(base_file)
3344
+ GeneralUtilities.assert_condition(self.is_xliff2_file(base_file), f"The base file '{base_file}' is not a valid XLIFF 2.0 file.")
3345
+ GeneralUtilities.assert_file_exists(base_file)
3346
+ if len(languages)==0:
3347
+ raise ValueError("No files provided for syncing.")
3348
+ if len(languages)==1:
3349
+ return
3350
+ language_files_list=[os.path.join(folder, f"{prefix}.{language}.xlf") for language in languages]
3351
+ language_files_with_content:dict[str,ET.ElementTree]=dict()
3352
+ for language_file in language_files_list:
3353
+ GeneralUtilities.assert_file_exists(language_file)
3354
+ GeneralUtilities.assert_condition(self.is_xliff2_file(language_file), f"The base file '{base_file}' is not a valid XLIFF 2.0 file.")
3355
+ language_files_with_content[language_file]=ET.parse(language_file)
3356
+
3357
+ #sync existing files
3358
+ self.__sync_xlf2_files(base_file_xml, language_files_with_content)
3359
+
3360
+
3361
+ @GeneralUtilities.check_arguments
3362
+ def translate_xlf_files_in_folder(self, folder: str, base_language: str, libre_translate_api_server: str):
3363
+ """Translates all .xlf files directly in the given folder (non-recursive)."""
3364
+ pattern = re.compile(r'^.+\.[a-z]{2,3}\.xlf$')
3365
+ for filename in os.listdir(folder):
3366
+ if not pattern.match(filename):
3367
+ continue
3368
+ file_path = os.path.join(folder, filename)
3369
+ self.translate_xlf_file(file_path, base_language, libre_translate_api_server)
3370
+
3371
+ @GeneralUtilities.check_arguments
3372
+ def translate_xlf_file(self, file: str, base_language: str, libre_translate_api_server: str):
3373
+ """
3374
+ Translates all segments with state='initial' in a XLIFF 2.0 file.
3375
+ The target language is extracted from the filename (e.g. 'messages.es.xlf' -> 'es').
3376
+ """
3377
+ ns_uri = "urn:oasis:names:tc:xliff:document:2.0"
3378
+ ns = {"xliff": ns_uri}
3379
+
3380
+ filename = os.path.basename(file)
3381
+ parts = filename.split(".")
3382
+ if len(parts) < 3:
3383
+ raise ValueError(f"Cannot extract language from filename: {filename}")
3384
+ target_language = parts[-2]
3385
+
3386
+ tree = ET.parse(file)
3387
+ root = tree.getroot()
3388
+
3389
+ for segment in root.findall(".//xliff:segment", ns):
3390
+ if segment.get("state", "initial") != "initial":
3391
+ continue
3392
+ source_el = segment.find("xliff:source", ns)
3393
+ if source_el is None or not source_el.text:
3394
+ continue
3395
+
3396
+ translated_text = self.translate(source_el.text, base_language, target_language, libre_translate_api_server)
3397
+
3398
+ target_el = segment.find("xliff:target", ns)
3399
+ if target_el is None:
3400
+ target_el = ET.SubElement(segment, f"{{{ns_uri}}}target")
3401
+
3402
+ target_el.text = translated_text
3403
+ segment.set("state", "translated")
3404
+
3405
+ ET.register_namespace("", ns_uri)
3406
+ xml_bytes = ET.tostring(root, encoding="utf-8", xml_declaration=True)
3407
+ with open(file, "wb") as f:
3408
+ f.write(xml_bytes)
3409
+
3410
+ @GeneralUtilities.check_arguments
3411
+ def translate(self, content: str, source_language: str, target_language: str, libre_translate_api_server: str) -> str:
3412
+ """Translates text using the LibreTranslate API."""
3413
+ url = f"{libre_translate_api_server.rstrip('/')}/translate"
3414
+ if "-" in source_language:
3415
+ source_language=source_language.split("-")[0]
3416
+ if "-" in target_language:
3417
+ target_language=target_language.split("-")[0]
3418
+ payload = {
3419
+ "q": content,
3420
+ "source": source_language,
3421
+ "target": target_language,
3422
+ "format": "text"
3423
+ }
3424
+ response = requests.post(url, json=payload, timeout=30)
3425
+ response.raise_for_status()
3426
+ return response.json()["translatedText"]
3427
+
3428
+ @GeneralUtilities.check_arguments
3429
+ def detect_language(self, content: str, libre_translate_api_server: str) -> str:
3430
+ """Detects the language of the given text using the LibreTranslate API."""
3431
+ url = f"{libre_translate_api_server.rstrip('/')}/detect"
3432
+ payload = {"q": content}
3433
+ response = requests.post(url, json=payload, timeout=30)
3434
+ response.raise_for_status()
3435
+ results = response.json()
3436
+ if not results:
3437
+ raise ValueError("Language detection returned no results.")
3438
+ return results[0]["language"]
3439
+
3440
+ @GeneralUtilities.check_arguments
3441
+ def get_all_files_in_git_repository(self,repository_folder:str,ignore_ignored_files:bool=True,include_submodules: bool = True) -> list[str]:
3442
+ """Returns a list of all files in a git-repository."""
3443
+ cmd = ["ls-files", "--cached"]
3444
+ if ignore_ignored_files:
3445
+ cmd.append("--exclude-standard")
3446
+ if include_submodules:
3447
+ cmd.append("--recurse-submodules")
3448
+ output=self.run_program_argsasarray("git", cmd,repository_folder)
3449
+ files = [ GeneralUtilities.normalize_path("./" + line) for line in output[1].splitlines()if line.strip() ]
3450
+ return files
3451
+
3452
+ @GeneralUtilities.check_arguments
3453
+ def write_file_list_for_repository(self,repository_folder:str,target_file:str="./FileList.txt",ignore_ignored_files:bool=True,include_submodules: bool = True) -> None:
3454
+ if os.path.isabs(target_file):
3455
+ target_file=GeneralUtilities.resolve_relative_path(target_file,repository_folder)
3456
+ target_file=GeneralUtilities.normalize_path(target_file)
3457
+ files=[path.replace("\\","/") for path in self.get_all_files_in_git_repository(repository_folder,ignore_ignored_files,include_submodules)]
3458
+ GeneralUtilities.ensure_file_exists(target_file)
3459
+ GeneralUtilities.write_lines_to_file(target_file, files)
3460
+
3461
+ @GeneralUtilities.check_arguments
3462
+ def get_all_commits_in_git_repository(self,repository_folder:str,include_all_heads:bool=False) -> list[str]:
3463
+ """Returns a textual visualization of all commits in a git-repository."""
3464
+ #do 'git log --reverse --all --pretty=format:"%ci | %H | %cn <%ce> | %d | %s"'
3465
+ args = ["log", "--reverse", "--pretty=format:%ci | %H | %cn <%ce> | %D | %s"]
3466
+ if include_all_heads:
3467
+ args.append("--all")
3468
+ result=self.run_program_argsasarray("git", args, repository_folder, throw_exception_if_exitcode_is_not_zero=True)
3469
+ output= result[1]
3470
+ result=output.splitlines()
3471
+ return result
3472
+
3473
+ @GeneralUtilities.check_arguments
3474
+ def write_commit_list_for_repository(self,repository_folder:str,target_file:str,include_all_heads:bool=False) -> None:
3475
+ if os.path.isabs(target_file):
3476
+ target_file=GeneralUtilities.resolve_relative_path(target_file,repository_folder)
3477
+ target_file=GeneralUtilities.normalize_path(target_file)
3478
+ commits=self.get_all_commits_in_git_repository(repository_folder, include_all_heads)
3479
+ GeneralUtilities.ensure_file_exists(target_file)
3480
+ GeneralUtilities.write_lines_to_file(target_file, commits)
3481
+
3482
+ @GeneralUtilities.check_arguments
3483
+ def is_runnning_in_container(self) ->bool:
3484
+ """this function is based on a convention and does not do a real check."""
3485
+ return os.environ.get("ISRUNNINGINCONTAINER") == "true"