scriptcollection 4.2.81__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/AnionBuildPlatform.py +199 -0
- ScriptCollection/CertificateUpdater.py +149 -0
- ScriptCollection/Executables.py +921 -0
- ScriptCollection/GeneralUtilities.py +1589 -0
- ScriptCollection/HTTPMaintenanceOverheadHelper.py +36 -0
- ScriptCollection/OCIImages/AbstractImageHandler.py +38 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGeneric.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGenericV.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabCE.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabEE.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/__init__.py +0 -0
- ScriptCollection/OCIImages/OCIImageManager.py +190 -0
- ScriptCollection/OCIImages/__init__.py +0 -0
- ScriptCollection/ProcessesRunner.py +43 -0
- ScriptCollection/ProgramRunnerBase.py +47 -0
- ScriptCollection/ProgramRunnerMock.py +2 -0
- ScriptCollection/ProgramRunnerPopen.py +57 -0
- ScriptCollection/ProgramRunnerSudo.py +108 -0
- ScriptCollection/Resources/CultureChooser/CultureChooser.js +29 -0
- ScriptCollection/Resources/CultureChooser/index.html +15 -0
- ScriptCollection/Resources/MaintenanceSite/MaintenanceSite.html +15 -0
- ScriptCollection/SCLog.py +115 -0
- ScriptCollection/ScriptCollectionCore.py +3485 -0
- ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +192 -0
- ScriptCollection/TFCPS/Docker/__init__.py +0 -0
- ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py +8 -0
- ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py +6 -0
- ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py +7 -0
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +547 -0
- ScriptCollection/TFCPS/DotNet/__init__.py +0 -0
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +137 -0
- ScriptCollection/TFCPS/Flutter/__init__.py +0 -0
- ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py +72 -0
- ScriptCollection/TFCPS/Go/__init__.py +0 -0
- ScriptCollection/TFCPS/Maven/TFCPS_CodeUnitSpecific_Maven.py +42 -0
- ScriptCollection/TFCPS/Maven/__init__.py +0 -0
- ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +232 -0
- ScriptCollection/TFCPS/NodeJS/__init__.py +0 -0
- ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +239 -0
- ScriptCollection/TFCPS/Python/__init__.py +0 -0
- ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py +42 -0
- ScriptCollection/TFCPS/Rust/__init__.py +0 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +433 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +135 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +301 -0
- ScriptCollection/TFCPS/TFCPS_CreateRelease.py +98 -0
- ScriptCollection/TFCPS/TFCPS_Generic.py +44 -0
- ScriptCollection/TFCPS/TFCPS_MergeToMain.py +128 -0
- ScriptCollection/TFCPS/TFCPS_MergeToStable.py +356 -0
- ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +48 -0
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +1565 -0
- ScriptCollection/TFCPS/__init__.py +0 -0
- ScriptCollection/__init__.py +0 -0
- ScriptCollection/__pycache__/GeneralUtilities.cpython-311.pyc +0 -0
- ScriptCollection/__pycache__/__init__.cpython-311.pyc +0 -0
- scriptcollection-4.2.81.dist-info/METADATA +169 -0
- scriptcollection-4.2.81.dist-info/RECORD +62 -0
- scriptcollection-4.2.81.dist-info/WHEEL +5 -0
- scriptcollection-4.2.81.dist-info/entry_points.txt +67 -0
- scriptcollection-4.2.81.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,3485 @@
|
|
|
1
|
+
from datetime import timedelta, datetime
|
|
2
|
+
from functools import cmp_to_key
|
|
3
|
+
import json
|
|
4
|
+
import binascii
|
|
5
|
+
import filecmp
|
|
6
|
+
import hashlib
|
|
7
|
+
import multiprocessing
|
|
8
|
+
import time
|
|
9
|
+
from io import BytesIO
|
|
10
|
+
import itertools
|
|
11
|
+
import copy
|
|
12
|
+
import zipfile
|
|
13
|
+
import math
|
|
14
|
+
import base64
|
|
15
|
+
import os
|
|
16
|
+
from html.parser import HTMLParser
|
|
17
|
+
from queue import Queue, Empty
|
|
18
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
19
|
+
import xml.etree.ElementTree as ET
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from subprocess import Popen
|
|
22
|
+
import re
|
|
23
|
+
import shutil
|
|
24
|
+
from typing import IO
|
|
25
|
+
import fnmatch
|
|
26
|
+
import uuid
|
|
27
|
+
import tempfile
|
|
28
|
+
import io
|
|
29
|
+
import requests
|
|
30
|
+
import ntplib
|
|
31
|
+
import yaml
|
|
32
|
+
import qrcode
|
|
33
|
+
import pycdlib
|
|
34
|
+
import send2trash
|
|
35
|
+
from pypdf import PdfReader, PdfWriter
|
|
36
|
+
from .GeneralUtilities import GeneralUtilities,Platform
|
|
37
|
+
from .ProgramRunnerBase import ProgramRunnerBase
|
|
38
|
+
from .ProgramRunnerPopen import ProgramRunnerPopen
|
|
39
|
+
from .SCLog import SCLog, LogLevel
|
|
40
|
+
|
|
41
|
+
version = "4.2.81"
|
|
42
|
+
__version__ = version
|
|
43
|
+
|
|
44
|
+
class VSCodeWorkspaceShellTask:
|
|
45
|
+
label:str
|
|
46
|
+
description:str#nullable
|
|
47
|
+
work_dir:str#nullable
|
|
48
|
+
command:str
|
|
49
|
+
aliases:list[str]
|
|
50
|
+
allow_custom_arguments:bool
|
|
51
|
+
|
|
52
|
+
def __init__(self,label:str,description:str,work_dir:str,command:str,aliases:list[str],allow_custom_arguments:bool):
|
|
53
|
+
GeneralUtilities.assert_not_null(label,"label")
|
|
54
|
+
self.label=label
|
|
55
|
+
self.description=description
|
|
56
|
+
self.work_dir=work_dir
|
|
57
|
+
GeneralUtilities.assert_not_null(command,"command")
|
|
58
|
+
self.command=command
|
|
59
|
+
GeneralUtilities.assert_not_null(aliases,"aliases")
|
|
60
|
+
self.aliases=aliases
|
|
61
|
+
GeneralUtilities.assert_not_null(allow_custom_arguments,"allow_custom_arguments")
|
|
62
|
+
self.allow_custom_arguments=allow_custom_arguments
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def serialize_for_vscode(self)->str:
|
|
66
|
+
aliases=",".join([f"\"{GeneralUtilities.escape_json_string_value(alias)}\"" for alias in self.aliases])
|
|
67
|
+
|
|
68
|
+
cwd:str=None
|
|
69
|
+
if self.work_dir is None:
|
|
70
|
+
cwd=GeneralUtilities.empty_string
|
|
71
|
+
else:
|
|
72
|
+
cwd=f"\"cwd\": \"{GeneralUtilities.escape_json_string_value(self.work_dir)}\""
|
|
73
|
+
|
|
74
|
+
desc:str=None
|
|
75
|
+
if self.description is None:
|
|
76
|
+
desc=GeneralUtilities.empty_string
|
|
77
|
+
else:
|
|
78
|
+
desc=f"\"description\": \"{GeneralUtilities.escape_json_string_value(self.description)}\","
|
|
79
|
+
|
|
80
|
+
result=f""" {{
|
|
81
|
+
"label": "{GeneralUtilities.escape_json_string_value(self.label)}",
|
|
82
|
+
"command": "{GeneralUtilities.escape_json_string_value(self.command)}",
|
|
83
|
+
"type": "shell",
|
|
84
|
+
"options": {{
|
|
85
|
+
{cwd}
|
|
86
|
+
}},
|
|
87
|
+
"aliases": [
|
|
88
|
+
{aliases}
|
|
89
|
+
],
|
|
90
|
+
{desc}
|
|
91
|
+
"allowcustomarguments": {str(self.allow_custom_arguments).lower()}
|
|
92
|
+
}}"""
|
|
93
|
+
return result
|
|
94
|
+
|
|
95
|
+
class VSCodeWorkspaceMariaDBConnection:
|
|
96
|
+
name:str
|
|
97
|
+
previewLimit:int=50
|
|
98
|
+
server:str
|
|
99
|
+
port:int
|
|
100
|
+
database:str
|
|
101
|
+
username:str
|
|
102
|
+
password:str
|
|
103
|
+
|
|
104
|
+
def __init__(self,name:str,server,port,database,username,password):
|
|
105
|
+
GeneralUtilities.assert_not_null(name,"name")
|
|
106
|
+
self.name=name
|
|
107
|
+
GeneralUtilities.assert_not_null(server,"server")
|
|
108
|
+
self.server=server
|
|
109
|
+
GeneralUtilities.assert_not_null(port,"port")
|
|
110
|
+
self.port=port
|
|
111
|
+
GeneralUtilities.assert_not_null(database,"database")
|
|
112
|
+
self.database=database
|
|
113
|
+
GeneralUtilities.assert_not_null(username,"username")
|
|
114
|
+
self.username=username
|
|
115
|
+
GeneralUtilities.assert_not_null(password,"password")
|
|
116
|
+
self.password=password
|
|
117
|
+
|
|
118
|
+
def serialize_for_vscode(self)->str:
|
|
119
|
+
result=f""" {{
|
|
120
|
+
"name": "{GeneralUtilities.escape_json_string_value(self.name)}",
|
|
121
|
+
"mysqlOptions": {{
|
|
122
|
+
"authProtocol": "default",
|
|
123
|
+
"enableSsl": "Disabled"
|
|
124
|
+
}},
|
|
125
|
+
"previewLimit": {self.previewLimit},
|
|
126
|
+
"server": "{GeneralUtilities.escape_json_string_value(self.server)}",
|
|
127
|
+
"port": {self.port},
|
|
128
|
+
"driver": "MySQL",
|
|
129
|
+
"database": "{GeneralUtilities.escape_json_string_value(self.database)}",
|
|
130
|
+
"username": "{GeneralUtilities.escape_json_string_value(self.username)}",
|
|
131
|
+
"password": "{GeneralUtilities.escape_json_string_value(self.password)}"
|
|
132
|
+
}}
|
|
133
|
+
"""
|
|
134
|
+
return result
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class VSCodeWorkspaceMongoDBConnection:
|
|
138
|
+
name:str
|
|
139
|
+
connection_string:str
|
|
140
|
+
|
|
141
|
+
def __init__(self,name:str,connection_string:str):
|
|
142
|
+
GeneralUtilities.assert_not_null(name,"name")
|
|
143
|
+
self.name=name
|
|
144
|
+
GeneralUtilities.assert_not_null(connection_string,"connection_string")
|
|
145
|
+
self.connection_string=connection_string
|
|
146
|
+
|
|
147
|
+
def serialize_for_vscode(self)->str:
|
|
148
|
+
result=f""" {{
|
|
149
|
+
"name": "{GeneralUtilities.escape_json_string_value(self.name)}",
|
|
150
|
+
"connectionString": "{GeneralUtilities.escape_json_string_value(self.connection_string)}"
|
|
151
|
+
}}
|
|
152
|
+
"""
|
|
153
|
+
return result
|
|
154
|
+
|
|
155
|
+
class ScriptCollectionCore:
|
|
156
|
+
|
|
157
|
+
# The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
|
|
158
|
+
# Do not change this value for productive environments.
|
|
159
|
+
mock_program_calls: bool = False#TODO remove this variable. When someone want to mock program-calls then the ProgramRunnerMock can be used instead
|
|
160
|
+
# The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
|
|
161
|
+
execute_program_really_if_no_mock_call_is_defined: bool = False
|
|
162
|
+
__mocked_program_calls: list = None
|
|
163
|
+
program_runner: ProgramRunnerBase = None
|
|
164
|
+
call_program_runner_directly: bool = None
|
|
165
|
+
log: SCLog = None
|
|
166
|
+
|
|
167
|
+
def __init__(self):
|
|
168
|
+
self.program_runner = ProgramRunnerPopen()
|
|
169
|
+
self.call_program_runner_directly = None
|
|
170
|
+
self.__mocked_program_calls = list[ScriptCollectionCore.__MockProgramCall]()
|
|
171
|
+
self.log = SCLog(None, LogLevel.Warning, False)
|
|
172
|
+
|
|
173
|
+
@staticmethod
|
|
174
|
+
@GeneralUtilities.check_arguments
|
|
175
|
+
def get_scriptcollection_version() -> str:
|
|
176
|
+
return __version__
|
|
177
|
+
|
|
178
|
+
@GeneralUtilities.check_arguments
|
|
179
|
+
def get_scriptcollection_configuration_folder(self)->str:
|
|
180
|
+
user_folder = str(Path.home())
|
|
181
|
+
result = os.path.join(user_folder, ".ScriptCollection")
|
|
182
|
+
result=GeneralUtilities.normalize_path(result)
|
|
183
|
+
GeneralUtilities.ensure_directory_exists(result)
|
|
184
|
+
return result
|
|
185
|
+
|
|
186
|
+
def get_global_cache_folder(self)->str:
|
|
187
|
+
result = os.path.join(self.get_scriptcollection_configuration_folder(), "GlobalCache")
|
|
188
|
+
result=GeneralUtilities.normalize_path(result)
|
|
189
|
+
GeneralUtilities.ensure_directory_exists(result)
|
|
190
|
+
return result
|
|
191
|
+
|
|
192
|
+
def __get_docker_registry_credentials_file(self)->str:
|
|
193
|
+
result=os.path.join(self.get_global_cache_folder(),"RegistryCredentials.csv")
|
|
194
|
+
if not os.path.isfile(result):
|
|
195
|
+
GeneralUtilities.ensure_file_exists(result)
|
|
196
|
+
GeneralUtilities.write_lines_to_file(result,["RegistryName;Username;Password"])
|
|
197
|
+
return result
|
|
198
|
+
|
|
199
|
+
def __load_credentials_if_required_and_available(self,registry_url:str,registry_username:str,registry_password:str)->tuple[str,str]:
|
|
200
|
+
if registry_url.startswith("https://"):
|
|
201
|
+
registry_url=registry_url[len("https://"):]
|
|
202
|
+
if registry_password is None:
|
|
203
|
+
credential_file=self.__get_docker_registry_credentials_file()
|
|
204
|
+
lines=GeneralUtilities.read_nonempty_lines_from_file(credential_file)[1:]
|
|
205
|
+
for line in lines:
|
|
206
|
+
splitted=line.split(";")
|
|
207
|
+
registry=splitted[0]
|
|
208
|
+
username=splitted[1]
|
|
209
|
+
password=splitted[2]
|
|
210
|
+
if registry_url==registry and (registry_username is None or username==registry_username):
|
|
211
|
+
registry_username=username
|
|
212
|
+
registry_password=password
|
|
213
|
+
break
|
|
214
|
+
else:
|
|
215
|
+
GeneralUtilities.assert_not_null(registry_username)
|
|
216
|
+
return (registry_username,registry_password)
|
|
217
|
+
|
|
218
|
+
def __get_docker_registry_credentials(self)->list[tuple[str,str,str]]:
|
|
219
|
+
result=[]
|
|
220
|
+
credential_file=self.__get_docker_registry_credentials_file()
|
|
221
|
+
if os.path.isfile(credential_file):
|
|
222
|
+
lines=GeneralUtilities.read_nonempty_lines_from_file(credential_file)[1:]
|
|
223
|
+
for line in lines:
|
|
224
|
+
splitted=line.split(";")
|
|
225
|
+
registry=splitted[0]
|
|
226
|
+
username=splitted[1]
|
|
227
|
+
password=splitted[2]
|
|
228
|
+
result.append((registry,username,password))
|
|
229
|
+
return result
|
|
230
|
+
|
|
231
|
+
def registry_contains_image(self,registry_url:str,image:str,registry_username:str,registry_password:str)->bool:
|
|
232
|
+
"""This function assumes that the registry is a custom deployed docker-registry (see https://hub.docker.com/_/registry )"""
|
|
233
|
+
try:
|
|
234
|
+
if "/" in image:
|
|
235
|
+
image=image.rsplit("/", 1)[-1]
|
|
236
|
+
registry_username,registry_password=self.__load_credentials_if_required_and_available(registry_url,registry_username,registry_password)
|
|
237
|
+
catalog_url = f"{registry_url}/v2/_catalog"
|
|
238
|
+
response = requests.get(catalog_url, auth=(registry_username, registry_password),timeout=20)
|
|
239
|
+
response.raise_for_status() # check if statuscode = 200
|
|
240
|
+
data = response.json()
|
|
241
|
+
# expected: {"repositories": ["nginx", "myapp"]}
|
|
242
|
+
images = data.get("repositories", [])
|
|
243
|
+
if not (image in images):
|
|
244
|
+
return False
|
|
245
|
+
|
|
246
|
+
if self.get_tags_of_images_from_registry(registry_url,image,registry_username,registry_password)<1:
|
|
247
|
+
return False
|
|
248
|
+
|
|
249
|
+
return True
|
|
250
|
+
except Exception:
|
|
251
|
+
return False
|
|
252
|
+
|
|
253
|
+
def docker_platform_to_slug(self,platform_value: Platform) -> str:
|
|
254
|
+
if platform_value == Platform.Linux_AMD64:
|
|
255
|
+
return "linux-amd64"
|
|
256
|
+
elif platform_value == Platform.Linux_ARM64:
|
|
257
|
+
return "linux-arm64"
|
|
258
|
+
raise ValueError(f"Unsupported platform: {platform_value}")
|
|
259
|
+
|
|
260
|
+
@GeneralUtilities.check_arguments
|
|
261
|
+
def add_image_to_custom_docker_image_registry(
|
|
262
|
+
self,
|
|
263
|
+
remote_hub: str,
|
|
264
|
+
imagename_on_remote_hub: str,
|
|
265
|
+
own_registry_address: str,
|
|
266
|
+
imagename_on_own_registry: str,
|
|
267
|
+
tag: str,
|
|
268
|
+
registry_username: str,
|
|
269
|
+
registry_password: str,
|
|
270
|
+
) -> None:
|
|
271
|
+
registry_username, registry_password = self.__load_credentials_if_required_and_available(remote_hub, registry_username, registry_password)
|
|
272
|
+
source_address = f"{remote_hub}/{imagename_on_remote_hub}:{tag}"
|
|
273
|
+
target_address = f"{own_registry_address}/{imagename_on_own_registry}:{tag}"
|
|
274
|
+
self.run_program("docker", f"buildx imagetools create --tag {target_address} {source_address}")#this does pull and push for each platform
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def get_tags_of_images_from_registry(self,registry_base_url:str,image:str,registry_username:str,registry_password:str)->list[str]:
|
|
278
|
+
"""registry_base_url must be in the format 'https://myregistry.example.com'
|
|
279
|
+
This function assumes that the registry is a custom deployed docker-registry (see https://hub.docker.com/_/registry )"""
|
|
280
|
+
registry_username,registry_password=self.__load_credentials_if_required_and_available(registry_base_url,registry_username,registry_password)
|
|
281
|
+
if "/" in image:
|
|
282
|
+
image=image.rsplit("/", 1)[-1]
|
|
283
|
+
if not self.registry_contains_image(registry_base_url,image,registry_username,registry_password):
|
|
284
|
+
return []
|
|
285
|
+
tags_url = f"{registry_base_url}/v2/{image}/tags/list"
|
|
286
|
+
response = requests.get(tags_url, auth=(registry_username, registry_password),timeout=20)
|
|
287
|
+
response.raise_for_status() # check if statuscode = 200
|
|
288
|
+
data=response.json()
|
|
289
|
+
# expected: {"name":"myapp","tags":["1.2.22","1.2.21","1.2.20"]}
|
|
290
|
+
tags = data.get("tags", [])
|
|
291
|
+
return tags
|
|
292
|
+
|
|
293
|
+
def registry_contains_image_with_tag(self,registry_url:str,image:str,tag:str,registry_username:str,registry_password:str)->bool:
|
|
294
|
+
"""This function assumes that the registry is a custom deployed docker-registry (see https://hub.docker.com/_/registry )"""
|
|
295
|
+
registry_username,registry_password=self.__load_credentials_if_required_and_available(registry_url,registry_username,registry_password)
|
|
296
|
+
if "/" in image:
|
|
297
|
+
image=image.rsplit("/", 1)[-1]
|
|
298
|
+
tags=self.get_tags_of_images_from_registry(registry_url,image,registry_username,registry_password)
|
|
299
|
+
if tags is None:
|
|
300
|
+
return False
|
|
301
|
+
else:
|
|
302
|
+
result = tag in tags
|
|
303
|
+
return result
|
|
304
|
+
|
|
305
|
+
def login_to_defined_docker_registries(self)->None:
|
|
306
|
+
registries=self.__get_docker_registry_credentials()
|
|
307
|
+
if len(registries)==0:
|
|
308
|
+
self.log.log("No docker registry credentials defined. Skipping docker login.",LogLevel.Debug)
|
|
309
|
+
else:
|
|
310
|
+
for registry,username,password in registries:
|
|
311
|
+
arg=f"login {registry} -u {username} -p {password}"
|
|
312
|
+
arg_for_log=f"login {registry} -u {username} -p ***"
|
|
313
|
+
self.run_program("docker",arg,arguments_for_log=arg_for_log,print_live_output=self.log.loglevel==LogLevel.Debug)
|
|
314
|
+
|
|
315
|
+
@GeneralUtilities.check_arguments
|
|
316
|
+
def python_file_has_errors(self, file: str, working_directory: str, treat_warnings_as_errors: bool = True) -> tuple[bool, list[str]]:
|
|
317
|
+
errors = list()
|
|
318
|
+
filename = os.path.relpath(file, working_directory)
|
|
319
|
+
if treat_warnings_as_errors:
|
|
320
|
+
errorsonly_argument = GeneralUtilities.empty_string
|
|
321
|
+
else:
|
|
322
|
+
errorsonly_argument = " --errors-only"
|
|
323
|
+
(exit_code, stdout, stderr, _) = self.run_program("pylint", filename + errorsonly_argument, working_directory, throw_exception_if_exitcode_is_not_zero=False)
|
|
324
|
+
if (exit_code != 0):
|
|
325
|
+
errors.append(f"Linting-issues of {file}:")
|
|
326
|
+
errors.append(f"Pylint-exitcode: {exit_code}")
|
|
327
|
+
for line in GeneralUtilities.string_to_lines(stdout):
|
|
328
|
+
errors.append(line)
|
|
329
|
+
for line in GeneralUtilities.string_to_lines(stderr):
|
|
330
|
+
errors.append(line)
|
|
331
|
+
return (True, errors)
|
|
332
|
+
|
|
333
|
+
return (False, errors)
|
|
334
|
+
|
|
335
|
+
@GeneralUtilities.check_arguments
|
|
336
|
+
def replace_version_in_dockerfile_file(self, dockerfile: str, new_version_value: str) -> None:
|
|
337
|
+
GeneralUtilities.write_text_to_file(dockerfile, re.sub("ARG Version=\"\\d+\\.\\d+\\.\\d+\"", f"ARG Version=\"{new_version_value}\"", GeneralUtilities.read_text_from_file(dockerfile)))
|
|
338
|
+
|
|
339
|
+
@GeneralUtilities.check_arguments
|
|
340
|
+
def replace_version_in_python_file(self, file: str, new_version_value: str):
|
|
341
|
+
GeneralUtilities.write_text_to_file(file, re.sub("version = \"\\d+\\.\\d+\\.\\d+\"", f"version = \"{new_version_value}\"", GeneralUtilities.read_text_from_file(file)))
|
|
342
|
+
|
|
343
|
+
@GeneralUtilities.check_arguments
|
|
344
|
+
def replace_version_in_ini_file(self, file: str, new_version_value: str):
|
|
345
|
+
GeneralUtilities.write_text_to_file(file, re.sub("version = \\d+\\.\\d+\\.\\d+", f"version = {new_version_value}", GeneralUtilities.read_text_from_file(file)))
|
|
346
|
+
|
|
347
|
+
@GeneralUtilities.check_arguments
|
|
348
|
+
def replace_version_in_nuspec_file(self, nuspec_file: str, new_version: str) -> None:
|
|
349
|
+
# TODO use XSLT instead
|
|
350
|
+
versionregex = "\\d+\\.\\d+\\.\\d+"
|
|
351
|
+
versiononlyregex = f"^{versionregex}$"
|
|
352
|
+
pattern = re.compile(versiononlyregex)
|
|
353
|
+
if pattern.match(new_version):
|
|
354
|
+
GeneralUtilities.write_text_to_file(nuspec_file, re.sub(f"<version>{versionregex}<\\/version>", f"<version>{new_version}</version>", GeneralUtilities.read_text_from_file(nuspec_file)))
|
|
355
|
+
else:
|
|
356
|
+
raise ValueError(f"Version '{new_version}' does not match version-regex '{versiononlyregex}'")
|
|
357
|
+
|
|
358
|
+
@GeneralUtilities.check_arguments
|
|
359
|
+
def replace_version_in_csproj_file(self, csproj_file: str, current_version: str):
|
|
360
|
+
versionregex = "\\d+\\.\\d+\\.\\d+"
|
|
361
|
+
versiononlyregex = f"^{versionregex}$"
|
|
362
|
+
pattern = re.compile(versiononlyregex)
|
|
363
|
+
if pattern.match(current_version):
|
|
364
|
+
for tag in ["Version", "AssemblyVersion", "FileVersion"]:
|
|
365
|
+
GeneralUtilities.write_text_to_file(csproj_file, re.sub(f"<{tag}>{versionregex}(.\\d+)?<\\/{tag}>", f"<{tag}>{current_version}</{tag}>", GeneralUtilities.read_text_from_file(csproj_file)))
|
|
366
|
+
else:
|
|
367
|
+
raise ValueError(f"Version '{current_version}' does not match version-regex '{versiononlyregex}'")
|
|
368
|
+
|
|
369
|
+
@GeneralUtilities.check_arguments
|
|
370
|
+
def push_nuget_build_artifact(self, nupkg_file: str, registry_address: str, api_key: str = None):
|
|
371
|
+
nupkg_file_name = os.path.basename(nupkg_file)
|
|
372
|
+
nupkg_file_folder = os.path.dirname(nupkg_file)
|
|
373
|
+
argument = f"nuget push {nupkg_file_name} --force-english-output --source {registry_address}"
|
|
374
|
+
if api_key is not None:
|
|
375
|
+
argument = f"{argument} --api-key {api_key}"
|
|
376
|
+
self.run_program("dotnet", argument, nupkg_file_folder)
|
|
377
|
+
|
|
378
|
+
@GeneralUtilities.check_arguments
|
|
379
|
+
def dotnet_build(self, folder: str, projectname: str, configuration: str):
|
|
380
|
+
self.run_program("dotnet", f"clean -c {configuration}", folder)
|
|
381
|
+
self.run_program("dotnet", f"build {projectname}/{projectname}.csproj -c {configuration}", folder)
|
|
382
|
+
|
|
383
|
+
@GeneralUtilities.check_arguments
|
|
384
|
+
def find_file_by_extension(self, folder: str, extension_without_dot: str):
|
|
385
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
386
|
+
result = [file for file in self.list_content(folder, True, False, False) if file.endswith(f".{extension_without_dot}")]
|
|
387
|
+
result_length = len(result)
|
|
388
|
+
if result_length == 0:
|
|
389
|
+
raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension_without_dot}'.")
|
|
390
|
+
if result_length == 1:
|
|
391
|
+
return result[0]
|
|
392
|
+
else:
|
|
393
|
+
raise ValueError(f"Multiple values available in folder '{folder}' with extension '{extension_without_dot}'.")
|
|
394
|
+
|
|
395
|
+
@GeneralUtilities.check_arguments
|
|
396
|
+
def find_last_file_by_extension(self, folder: str, extension_without_dot: str) -> str:
|
|
397
|
+
files: list[str] = GeneralUtilities.get_direct_files_of_folder(folder)
|
|
398
|
+
possible_results: list[str] = []
|
|
399
|
+
for file in files:
|
|
400
|
+
if file.endswith(f".{extension_without_dot}"):
|
|
401
|
+
possible_results.append(file)
|
|
402
|
+
result_length = len(possible_results)
|
|
403
|
+
if result_length == 0:
|
|
404
|
+
raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension_without_dot}'.")
|
|
405
|
+
else:
|
|
406
|
+
return possible_results[-1]
|
|
407
|
+
|
|
408
|
+
@GeneralUtilities.check_arguments
|
|
409
|
+
def commit_is_signed_by_key(self, repository_folder: str, revision_identifier: str, key: str) -> bool:
|
|
410
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
411
|
+
result = self.run_program("git", f"verify-commit {revision_identifier}", repository_folder, throw_exception_if_exitcode_is_not_zero=False)
|
|
412
|
+
if (result[0] != 0):
|
|
413
|
+
return False
|
|
414
|
+
if (not GeneralUtilities.contains_line(result[1].splitlines(), f"gpg\\:\\ using\\ [A-Za-z0-9]+\\ key\\ [A-Za-z0-9]+{key}")):
|
|
415
|
+
# TODO check whether this works on machines where gpg is installed in another langauge than english
|
|
416
|
+
return False
|
|
417
|
+
if (not GeneralUtilities.contains_line(result[1].splitlines(), "gpg\\:\\ Good\\ signature\\ from")):
|
|
418
|
+
# TODO check whether this works on machines where gpg is installed in another langauge than english
|
|
419
|
+
return False
|
|
420
|
+
return True
|
|
421
|
+
|
|
422
|
+
@GeneralUtilities.check_arguments
|
|
423
|
+
def get_parent_commit_ids_of_commit(self, repository_folder: str, commit_id: str) -> str:
|
|
424
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
425
|
+
return self.run_program("git", f'log --pretty=%P -n 1 "{commit_id}"', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string).split(" ")
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
@GeneralUtilities.check_arguments
|
|
429
|
+
def get_commit_ids_between_dates(self, repository_folder: str, since: datetime, until: datetime, ignore_commits_which_are_not_in_history_of_head: bool = True) -> None:
|
|
430
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
431
|
+
since_as_string = self.__datetime_to_string_for_git(since)
|
|
432
|
+
until_as_string = self.__datetime_to_string_for_git(until)
|
|
433
|
+
result = filter(lambda line: not GeneralUtilities.string_is_none_or_whitespace(line), self.run_program("git", f'log --since "{since_as_string}" --until "{until_as_string}" --pretty=format:"%H" --no-patch', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].split("\n").replace("\r", GeneralUtilities.empty_string))
|
|
434
|
+
if ignore_commits_which_are_not_in_history_of_head:
|
|
435
|
+
result = [commit_id for commit_id in result if self.git_commit_is_ancestor(repository_folder, commit_id)]
|
|
436
|
+
return result
|
|
437
|
+
|
|
438
|
+
@GeneralUtilities.check_arguments
|
|
439
|
+
def __datetime_to_string_for_git(self, datetime_object: datetime) -> str:
|
|
440
|
+
return datetime_object.strftime('%Y-%m-%d %H:%M:%S')
|
|
441
|
+
|
|
442
|
+
@GeneralUtilities.check_arguments
|
|
443
|
+
def git_commit_is_ancestor(self, repository_folder: str, ancestor: str, descendant: str = "HEAD") -> bool:
|
|
444
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
445
|
+
result = self.run_program_argsasarray("git", ["merge-base", "--is-ancestor", ancestor, descendant], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
|
|
446
|
+
exit_code = result[0]
|
|
447
|
+
if exit_code == 0:
|
|
448
|
+
return True
|
|
449
|
+
elif exit_code == 1:
|
|
450
|
+
return False
|
|
451
|
+
else:
|
|
452
|
+
raise ValueError(f'Can not calculate if {ancestor} is an ancestor of {descendant} in repository {repository_folder}. Outout of "{repository_folder}> git merge-base --is-ancestor {ancestor} {descendant}": Exitcode: {exit_code}; StdOut: {result[1]}; StdErr: {result[2]}.')
|
|
453
|
+
|
|
454
|
+
@GeneralUtilities.check_arguments
|
|
455
|
+
def __git_changes_helper(self, repository_folder: str, arguments_as_array: list[str]) -> bool:
|
|
456
|
+
self.assert_is_git_repository(repository_folder)
|
|
457
|
+
lines = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", arguments_as_array, repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1], False)
|
|
458
|
+
for line in lines:
|
|
459
|
+
if GeneralUtilities.string_has_content(line):
|
|
460
|
+
return True
|
|
461
|
+
return False
|
|
462
|
+
|
|
463
|
+
@GeneralUtilities.check_arguments
|
|
464
|
+
def git_repository_has_new_untracked_files(self, repository_folder: str):
|
|
465
|
+
self.assert_is_git_repository(repository_folder)
|
|
466
|
+
return self.__git_changes_helper(repository_folder, ["ls-files", "--exclude-standard", "--others"])
|
|
467
|
+
|
|
468
|
+
@GeneralUtilities.check_arguments
|
|
469
|
+
def git_repository_has_unstaged_changes_of_tracked_files(self, repository_folder: str):
|
|
470
|
+
self.assert_is_git_repository(repository_folder)
|
|
471
|
+
return self.__git_changes_helper(repository_folder, ["--no-pager", "diff"])
|
|
472
|
+
|
|
473
|
+
@GeneralUtilities.check_arguments
|
|
474
|
+
def git_repository_has_staged_changes(self, repository_folder: str):
|
|
475
|
+
self.assert_is_git_repository(repository_folder)
|
|
476
|
+
return self.__git_changes_helper(repository_folder, ["--no-pager", "diff", "--cached"])
|
|
477
|
+
|
|
478
|
+
@GeneralUtilities.check_arguments
|
|
479
|
+
def git_repository_has_uncommitted_changes(self, repository_folder: str) -> bool:
|
|
480
|
+
self.assert_is_git_repository(repository_folder)
|
|
481
|
+
if (self.git_repository_has_unstaged_changes(repository_folder)):
|
|
482
|
+
return True
|
|
483
|
+
if (self.git_repository_has_staged_changes(repository_folder)):
|
|
484
|
+
return True
|
|
485
|
+
return False
|
|
486
|
+
|
|
487
|
+
@GeneralUtilities.check_arguments
|
|
488
|
+
def git_repository_has_unstaged_changes(self, repository_folder: str) -> bool:
|
|
489
|
+
self.assert_is_git_repository(repository_folder)
|
|
490
|
+
if (self.git_repository_has_unstaged_changes_of_tracked_files(repository_folder)):
|
|
491
|
+
return True
|
|
492
|
+
if (self.git_repository_has_new_untracked_files(repository_folder)):
|
|
493
|
+
return True
|
|
494
|
+
return False
|
|
495
|
+
|
|
496
|
+
@GeneralUtilities.check_arguments
|
|
497
|
+
def git_get_commit_id(self, repository_folder: str, rev: str = "HEAD") -> str:
|
|
498
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
499
|
+
result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["rev-parse", "--verify", rev], repository_folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
500
|
+
return result[1].replace('\n', '')
|
|
501
|
+
|
|
502
|
+
@GeneralUtilities.check_arguments
|
|
503
|
+
def git_get_commit_date(self, repository_folder: str, rev: str = "HEAD") -> datetime:
|
|
504
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
505
|
+
result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["log","-1","--format=%ci", rev], repository_folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
506
|
+
date_as_string = result[1].replace('\n', '')
|
|
507
|
+
result = datetime.strptime(date_as_string, '%Y-%m-%d %H:%M:%S %z')
|
|
508
|
+
return result
|
|
509
|
+
|
|
510
|
+
@GeneralUtilities.check_arguments
|
|
511
|
+
def git_fetch_with_retry(self, folder: str, remotename: str = "--all", amount_of_attempts: int = 5) -> None:
|
|
512
|
+
GeneralUtilities.retry_action(lambda: self.git_fetch(folder, remotename), amount_of_attempts)
|
|
513
|
+
|
|
514
|
+
@GeneralUtilities.check_arguments
|
|
515
|
+
def git_fetch(self, folder: str, remotename: str = "--all") -> None:
|
|
516
|
+
self.assert_is_git_repository(folder)
|
|
517
|
+
self.run_program_argsasarray("git", ["fetch", remotename, "--tags", "--prune"], folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
518
|
+
|
|
519
|
+
@GeneralUtilities.check_arguments
|
|
520
|
+
def git_fetch_in_bare_repository(self, folder: str, remotename, localbranch: str, remotebranch: str) -> None:
|
|
521
|
+
self.assert_is_git_repository(folder)
|
|
522
|
+
self.run_program_argsasarray("git", ["fetch", remotename, f"{remotebranch}:{localbranch}"], folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
523
|
+
|
|
524
|
+
def branch_exists(self, folder: str, branchname: str) -> bool:
|
|
525
|
+
self.assert_is_git_repository(folder)
|
|
526
|
+
result = self.run_program_argsasarray("git", ["rev-parse", "--verify", branchname], folder, throw_exception_if_exitcode_is_not_zero=False)
|
|
527
|
+
return result[0] == 0
|
|
528
|
+
|
|
529
|
+
@GeneralUtilities.check_arguments
|
|
530
|
+
def git_remove_branch(self, folder: str, branchname: str) -> None:
|
|
531
|
+
self.assert_is_git_repository(folder)
|
|
532
|
+
self.run_program("git", f"branch -D {branchname}", folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
533
|
+
|
|
534
|
+
@GeneralUtilities.check_arguments
|
|
535
|
+
def git_push_with_retry(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: LogLevel = LogLevel.Quiet, amount_of_attempts: int = 5) -> None:
|
|
536
|
+
GeneralUtilities.retry_action(lambda: self.git_push(folder, remotename, localbranchname, remotebranchname, forcepush, pushalltags, verbosity), amount_of_attempts)
|
|
537
|
+
|
|
538
|
+
@GeneralUtilities.check_arguments
|
|
539
|
+
def git_push(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: LogLevel = LogLevel.Quiet,resurse_submodules:bool=False) -> None:
|
|
540
|
+
self.is_git_or_bare_git_repository(folder)
|
|
541
|
+
argument = ["push"]
|
|
542
|
+
if resurse_submodules:
|
|
543
|
+
argument = argument + ["--recurse-submodules=on-demand"]
|
|
544
|
+
argument = argument + [remotename, f"{localbranchname}:{remotebranchname}"]
|
|
545
|
+
if (forcepush):
|
|
546
|
+
argument.append("--force")
|
|
547
|
+
if (pushalltags):
|
|
548
|
+
argument.append("--tags")
|
|
549
|
+
result: tuple[int, str, str, int] = self.run_program_argsasarray("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True, print_errors_as_information=True)
|
|
550
|
+
return result[1].replace('\r', '').replace('\n', '')
|
|
551
|
+
|
|
552
|
+
@GeneralUtilities.check_arguments
|
|
553
|
+
def git_pull_with_retry(self, folder: str, remote: str, localbranchname: str, remotebranchname: str, force: bool = False, amount_of_attempts: int = 5) -> None:
|
|
554
|
+
GeneralUtilities.retry_action(lambda: self.git_pull(folder, remote, localbranchname, remotebranchname), amount_of_attempts)
|
|
555
|
+
|
|
556
|
+
@GeneralUtilities.check_arguments
|
|
557
|
+
def git_pull(self, folder: str, remote: str, localbranchname: str, remotebranchname: str, force: bool = False) -> None:
|
|
558
|
+
self.is_git_or_bare_git_repository(folder)
|
|
559
|
+
argument = f"pull {remote} {remotebranchname}:{localbranchname}"
|
|
560
|
+
if force:
|
|
561
|
+
argument = f"{argument} --force"
|
|
562
|
+
self.run_program("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
563
|
+
|
|
564
|
+
@GeneralUtilities.check_arguments
|
|
565
|
+
def git_list_remote_branches(self, folder: str, remote: str, fetch: bool) -> list[str]:
|
|
566
|
+
self.is_git_or_bare_git_repository(folder)
|
|
567
|
+
if fetch:
|
|
568
|
+
self.git_fetch(folder, remote)
|
|
569
|
+
run_program_result = self.run_program("git", f"branch -rl {remote}/*", folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
570
|
+
output = GeneralUtilities.string_to_lines(run_program_result[1])
|
|
571
|
+
result = list[str]()
|
|
572
|
+
for item in output:
|
|
573
|
+
striped_item = item.strip()
|
|
574
|
+
if GeneralUtilities.string_has_content(striped_item):
|
|
575
|
+
branch: str = None
|
|
576
|
+
if " " in striped_item:
|
|
577
|
+
branch = striped_item.split(" ")[0]
|
|
578
|
+
else:
|
|
579
|
+
branch = striped_item
|
|
580
|
+
branchname = branch[len(remote)+1:]
|
|
581
|
+
if branchname != "HEAD":
|
|
582
|
+
result.append(branchname)
|
|
583
|
+
return result
|
|
584
|
+
|
|
585
|
+
@GeneralUtilities.check_arguments
|
|
586
|
+
def git_clone(self, clone_target_folder: str, remote_repository_path: str, include_submodules: bool = True, mirror: bool = False) -> None:
|
|
587
|
+
if (os.path.isdir(clone_target_folder)):
|
|
588
|
+
pass # TODO throw error
|
|
589
|
+
else:
|
|
590
|
+
args = ["clone", remote_repository_path, clone_target_folder]
|
|
591
|
+
if include_submodules:
|
|
592
|
+
args.append("--recurse-submodules")
|
|
593
|
+
args.append("--remote-submodules")
|
|
594
|
+
if mirror:
|
|
595
|
+
args.append("--mirror")
|
|
596
|
+
self.run_program_argsasarray("git", args, os.getcwd(), throw_exception_if_exitcode_is_not_zero=True)
|
|
597
|
+
|
|
598
|
+
@GeneralUtilities.check_arguments
|
|
599
|
+
def git_get_all_remote_names(self, directory: str) -> list[str]:
|
|
600
|
+
self.is_git_or_bare_git_repository(directory)
|
|
601
|
+
result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote"], directory, throw_exception_if_exitcode_is_not_zero=True)[1], False)
|
|
602
|
+
return result
|
|
603
|
+
|
|
604
|
+
@GeneralUtilities.check_arguments
|
|
605
|
+
def git_get_remote_url(self, directory: str, remote_name: str) -> str:
|
|
606
|
+
self.is_git_or_bare_git_repository(directory)
|
|
607
|
+
result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote", "get-url", remote_name], directory, throw_exception_if_exitcode_is_not_zero=True)[1], False)
|
|
608
|
+
return result[0].replace('\n', '')
|
|
609
|
+
|
|
610
|
+
@GeneralUtilities.check_arguments
|
|
611
|
+
def repository_has_remote_with_specific_name(self, directory: str, remote_name: str) -> bool:
|
|
612
|
+
self.is_git_or_bare_git_repository(directory)
|
|
613
|
+
return remote_name in self.git_get_all_remote_names(directory)
|
|
614
|
+
|
|
615
|
+
@GeneralUtilities.check_arguments
|
|
616
|
+
def git_add_or_set_remote_address(self, directory: str, remote_name: str, remote_address: str) -> None:
|
|
617
|
+
self.assert_is_git_repository(directory)
|
|
618
|
+
if (self.repository_has_remote_with_specific_name(directory, remote_name)):
|
|
619
|
+
self.run_program_argsasarray("git", ['remote', 'set-url', 'remote_name', remote_address], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
620
|
+
else:
|
|
621
|
+
self.run_program_argsasarray("git", ['remote', 'add', remote_name, remote_address], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
622
|
+
|
|
623
|
+
@GeneralUtilities.check_arguments
|
|
624
|
+
def git_stage_all_changes(self, directory: str) -> None:
|
|
625
|
+
self.assert_is_git_repository(directory)
|
|
626
|
+
self.run_program_argsasarray("git", ["add", "-A"], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
627
|
+
|
|
628
|
+
@GeneralUtilities.check_arguments
|
|
629
|
+
def git_unstage_all_changes(self, directory: str) -> None:
|
|
630
|
+
self.assert_is_git_repository(directory)
|
|
631
|
+
self.run_program_argsasarray("git", ["reset"], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
632
|
+
# TODO check if this will also be done for submodules
|
|
633
|
+
|
|
634
|
+
@GeneralUtilities.check_arguments
|
|
635
|
+
def git_stage_file(self, directory: str, file: str) -> None:
|
|
636
|
+
self.assert_is_git_repository(directory)
|
|
637
|
+
self.run_program_argsasarray("git", ['stage', file], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
638
|
+
|
|
639
|
+
@GeneralUtilities.check_arguments
|
|
640
|
+
def git_unstage_file(self, directory: str, file: str) -> None:
|
|
641
|
+
self.assert_is_git_repository(directory)
|
|
642
|
+
self.run_program_argsasarray("git", ['reset', file], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
643
|
+
|
|
644
|
+
@GeneralUtilities.check_arguments
|
|
645
|
+
def git_discard_unstaged_changes_of_file(self, directory: str, file: str) -> None:
|
|
646
|
+
"""Caution: This method works really only for 'changed' files yet. So this method does not work properly for new or renamed files."""
|
|
647
|
+
self.assert_is_git_repository(directory)
|
|
648
|
+
self.run_program_argsasarray("git", ['checkout', file], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
649
|
+
|
|
650
|
+
@GeneralUtilities.check_arguments
|
|
651
|
+
def git_discard_all_unstaged_changes(self, directory: str) -> None:
|
|
652
|
+
"""Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
|
|
653
|
+
self.assert_is_git_repository(directory)
|
|
654
|
+
self.run_program_argsasarray("git", ['clean', '-df'], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
655
|
+
self.run_program_argsasarray("git", ['checkout', '.'], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
656
|
+
# TODO check if this will also be done for submodules
|
|
657
|
+
|
|
658
|
+
@GeneralUtilities.check_arguments
|
|
659
|
+
def git_commit(self, directory: str, message: str = "Saved changes.", author_name: str = None, author_email: str = None, stage_all_changes: bool = True, no_changes_behavior: int = 0,commit_message_body:str=None) -> str:
|
|
660
|
+
"""no_changes_behavior=0 => No commit; no_changes_behavior=1 => Commit anyway; no_changes_behavior=2 => Exception"""
|
|
661
|
+
self.assert_is_git_repository(directory)
|
|
662
|
+
author_name = GeneralUtilities.str_none_safe(author_name).strip()
|
|
663
|
+
author_email = GeneralUtilities.str_none_safe(author_email).strip()
|
|
664
|
+
argument = ['commit', '--quiet', '--allow-empty', '--message', message]
|
|
665
|
+
if commit_message_body is not None:
|
|
666
|
+
argument.extend(['--message', commit_message_body])
|
|
667
|
+
if (GeneralUtilities.string_has_content(author_name)):
|
|
668
|
+
argument.append(f'--author="{author_name} <{author_email}>"')
|
|
669
|
+
git_repository_has_uncommitted_changes = self.git_repository_has_uncommitted_changes(directory)
|
|
670
|
+
|
|
671
|
+
if git_repository_has_uncommitted_changes:
|
|
672
|
+
do_commit = True
|
|
673
|
+
if stage_all_changes:
|
|
674
|
+
self.git_stage_all_changes(directory)
|
|
675
|
+
else:
|
|
676
|
+
if no_changes_behavior == 0:
|
|
677
|
+
self.log.log(f"Commit '{message}' will not be done because there are no changes to commit in repository '{directory}'", LogLevel.Debug)
|
|
678
|
+
do_commit = False
|
|
679
|
+
elif no_changes_behavior == 1:
|
|
680
|
+
self.log.log(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will be done anyway.", LogLevel.Debug)
|
|
681
|
+
do_commit = True
|
|
682
|
+
elif no_changes_behavior == 2:
|
|
683
|
+
raise RuntimeError(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will not be done.")
|
|
684
|
+
else:
|
|
685
|
+
raise ValueError(f"Unknown value for no_changes_behavior: {GeneralUtilities.str_none_safe(no_changes_behavior)}")
|
|
686
|
+
|
|
687
|
+
if do_commit:
|
|
688
|
+
self.log.log(f"Commit changes in '{directory}'", LogLevel.Information)
|
|
689
|
+
self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
690
|
+
|
|
691
|
+
return self.git_get_commit_id(directory)
|
|
692
|
+
|
|
693
|
+
def search_repository_folder(self,some_file_in_repository:str)->str:
|
|
694
|
+
current_path:str=os.path.dirname(some_file_in_repository)
|
|
695
|
+
enabled:bool=True
|
|
696
|
+
while enabled:
|
|
697
|
+
try:
|
|
698
|
+
current_path=GeneralUtilities.resolve_relative_path("..",current_path)
|
|
699
|
+
if self.is_git_repository(current_path):
|
|
700
|
+
return current_path
|
|
701
|
+
except:
|
|
702
|
+
enabled=False
|
|
703
|
+
raise ValueError(f"Can not find git-repository for folder \"{some_file_in_repository}\".")
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
@GeneralUtilities.check_arguments
|
|
707
|
+
def git_create_tag(self, directory: str, target_for_tag: str, tag: str, sign: bool = False, message: str = None) -> None:
|
|
708
|
+
self.is_git_or_bare_git_repository(directory)
|
|
709
|
+
argument = ["tag", tag, target_for_tag]
|
|
710
|
+
if sign:
|
|
711
|
+
if message is None:
|
|
712
|
+
message = f"Created {target_for_tag}"
|
|
713
|
+
argument.extend(["-s", '-m', message])
|
|
714
|
+
self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
715
|
+
|
|
716
|
+
@GeneralUtilities.check_arguments
|
|
717
|
+
def git_delete_tag(self, directory: str, tag: str) -> None:
|
|
718
|
+
self.is_git_or_bare_git_repository(directory)
|
|
719
|
+
self.run_program_argsasarray("git", ["tag", "--delete", tag], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
720
|
+
|
|
721
|
+
@GeneralUtilities.check_arguments
|
|
722
|
+
def git_checkout(self, directory: str, rev: str, undo_all_changes_after_checkout: bool = True, assert_no_uncommitted_changes: bool = True) -> None:
|
|
723
|
+
self.assert_is_git_repository(directory)
|
|
724
|
+
if assert_no_uncommitted_changes:
|
|
725
|
+
GeneralUtilities.assert_condition(not self.git_repository_has_uncommitted_changes(directory), f"Repository \"{directory}\" has uncommitted changes.")
|
|
726
|
+
self.run_program_argsasarray("git", ["checkout", rev], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
727
|
+
self.run_program_argsasarray("git", ["submodule", "update", "--recursive"], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
728
|
+
commit_id=self.git_get_commit_id(directory,"HEAD")
|
|
729
|
+
self.log.log(f"Checked out {commit_id} in \"{directory}\".", LogLevel.Debug)
|
|
730
|
+
if undo_all_changes_after_checkout:
|
|
731
|
+
self.git_undo_all_changes(directory)
|
|
732
|
+
|
|
733
|
+
@GeneralUtilities.check_arguments
|
|
734
|
+
def merge_repository(self, repository_folder: str, remote: str, branch: str):
|
|
735
|
+
GeneralUtilities.assert_condition(not self.git_repository_has_uncommitted_changes(repository_folder),f"Can not merge. There are uncommitted changes in \"{repository_folder}\".")
|
|
736
|
+
is_pullable: bool = self.git_commit_is_ancestor(repository_folder, branch, f"{remote}/{branch}")
|
|
737
|
+
if is_pullable:
|
|
738
|
+
self.git_pull(repository_folder, remote, branch, branch)
|
|
739
|
+
uncommitted_changes = self.git_repository_has_uncommitted_changes(repository_folder)
|
|
740
|
+
GeneralUtilities.assert_condition(not uncommitted_changes, f"Pulling remote \"{remote}\" in \"{repository_folder}\" caused new uncommitted files.")
|
|
741
|
+
self.git_checkout(repository_folder, branch)
|
|
742
|
+
self.git_fetch(repository_folder, remote)
|
|
743
|
+
self.git_merge(repository_folder, f"{remote}/{branch}", branch)
|
|
744
|
+
self.git_push_with_retry(repository_folder, remote, branch, branch)
|
|
745
|
+
self.git_checkout(repository_folder, branch)
|
|
746
|
+
#TODO opeional: checkfor merge conflicts and if there is one merge conflict print a warning
|
|
747
|
+
|
|
748
|
+
@GeneralUtilities.check_arguments
|
|
749
|
+
def git_merge_abort(self, directory: str) -> None:
|
|
750
|
+
self.assert_is_git_repository(directory)
|
|
751
|
+
self.run_program_argsasarray("git", ["merge", "--abort"], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
752
|
+
|
|
753
|
+
@GeneralUtilities.check_arguments
|
|
754
|
+
def git_merge(self, directory: str, sourcebranch: str, targetbranch: str, fastforward: bool = True, commit: bool = True, commit_message: str = None, undo_all_changes_after_checkout: bool = True, assert_no_uncommitted_changes: bool = True) -> str:
|
|
755
|
+
self.assert_is_git_repository(directory)
|
|
756
|
+
self.git_checkout(directory, targetbranch, undo_all_changes_after_checkout, assert_no_uncommitted_changes)
|
|
757
|
+
args = ["merge"]
|
|
758
|
+
if not commit:
|
|
759
|
+
args.append("--no-commit")
|
|
760
|
+
if not fastforward:
|
|
761
|
+
args.append("--no-ff")
|
|
762
|
+
if commit_message is not None:
|
|
763
|
+
args.append("-m")
|
|
764
|
+
args.append(commit_message)
|
|
765
|
+
args.append(sourcebranch)
|
|
766
|
+
self.run_program_argsasarray("git", args, directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
767
|
+
self.run_program_argsasarray("git", ["submodule", "update"], directory, throw_exception_if_exitcode_is_not_zero=True)
|
|
768
|
+
return self.git_get_commit_id(directory)
|
|
769
|
+
|
|
770
|
+
@GeneralUtilities.check_arguments
|
|
771
|
+
def git_undo_all_changes(self, directory: str) -> None:
|
|
772
|
+
"""Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
|
|
773
|
+
self.assert_is_git_repository(directory)
|
|
774
|
+
self.git_unstage_all_changes(directory)
|
|
775
|
+
self.git_discard_all_unstaged_changes(directory)
|
|
776
|
+
|
|
777
|
+
@GeneralUtilities.check_arguments
|
|
778
|
+
def git_fetch_or_clone_all_in_directory(self, source_directory: str, target_directory: str) -> None:
|
|
779
|
+
for subfolder in GeneralUtilities.get_direct_folders_of_folder(source_directory):
|
|
780
|
+
foldername = os.path.basename(subfolder)
|
|
781
|
+
if self.is_git_repository(subfolder):
|
|
782
|
+
source_repository = subfolder
|
|
783
|
+
target_repository = os.path.join(target_directory, foldername)
|
|
784
|
+
if os.path.isdir(target_directory):
|
|
785
|
+
# fetch
|
|
786
|
+
self.git_fetch(target_directory)
|
|
787
|
+
else:
|
|
788
|
+
# clone
|
|
789
|
+
self.git_clone(target_repository, source_repository, include_submodules=True, mirror=True)
|
|
790
|
+
|
|
791
|
+
def get_git_submodules(self, directory: str) -> list[str]:
|
|
792
|
+
self.is_git_or_bare_git_repository(directory)
|
|
793
|
+
e = self.run_program("git", "submodule status", directory)
|
|
794
|
+
result = []
|
|
795
|
+
for submodule_line in GeneralUtilities.string_to_lines(e[1], False, True):
|
|
796
|
+
result.append(submodule_line.split(' ')[1])
|
|
797
|
+
return result
|
|
798
|
+
|
|
799
|
+
@GeneralUtilities.check_arguments
|
|
800
|
+
def file_is_git_ignored(self, file_in_repository: str, repositorybasefolder: str) -> None:
|
|
801
|
+
self.is_git_or_bare_git_repository(repositorybasefolder)
|
|
802
|
+
exit_code = self.run_program_argsasarray("git", ['check-ignore', file_in_repository], repositorybasefolder, throw_exception_if_exitcode_is_not_zero=False)[0]
|
|
803
|
+
if (exit_code == 0):
|
|
804
|
+
return True
|
|
805
|
+
if (exit_code == 1):
|
|
806
|
+
return False
|
|
807
|
+
raise ValueError(f"Unable to calculate whether '{file_in_repository}' in repository '{repositorybasefolder}' is ignored due to git-exitcode {exit_code}.")
|
|
808
|
+
|
|
809
|
+
@GeneralUtilities.check_arguments
|
|
810
|
+
def git_discard_all_changes(self, repository: str) -> None:
|
|
811
|
+
self.assert_is_git_repository(repository)
|
|
812
|
+
self.run_program_argsasarray("git", ["reset", "HEAD", "."], repository, throw_exception_if_exitcode_is_not_zero=True)
|
|
813
|
+
self.run_program_argsasarray("git", ["checkout", "."], repository, throw_exception_if_exitcode_is_not_zero=True)
|
|
814
|
+
|
|
815
|
+
@GeneralUtilities.check_arguments
|
|
816
|
+
def git_get_current_branch_name(self, repository: str) -> str:
|
|
817
|
+
self.assert_is_git_repository(repository)
|
|
818
|
+
result = self.run_program_argsasarray("git", ["rev-parse", "--abbrev-ref", "HEAD"], repository, throw_exception_if_exitcode_is_not_zero=True)
|
|
819
|
+
return result[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
|
|
820
|
+
|
|
821
|
+
@GeneralUtilities.check_arguments
|
|
822
|
+
def git_get_commitid_of_tag(self, repository: str, tag: str) -> str:
|
|
823
|
+
self.is_git_or_bare_git_repository(repository)
|
|
824
|
+
stdout = self.run_program_argsasarray("git", ["rev-list", "-n", "1", tag], repository)
|
|
825
|
+
result = stdout[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
|
|
826
|
+
return result
|
|
827
|
+
|
|
828
|
+
@GeneralUtilities.check_arguments
|
|
829
|
+
def git_get_tags(self, repository: str) -> list[str]:
|
|
830
|
+
self.is_git_or_bare_git_repository(repository)
|
|
831
|
+
tags = [line.replace("\r", GeneralUtilities.empty_string) for line in self.run_program_argsasarray(
|
|
832
|
+
"git", ["tag"], repository)[1].split("\n") if len(line) > 0]
|
|
833
|
+
return tags
|
|
834
|
+
|
|
835
|
+
@GeneralUtilities.check_arguments
|
|
836
|
+
def git_move_tags_to_another_branch(self, repository: str, tag_source_branch: str, tag_target_branch: str, sign: bool = False, message: str = None) -> None:
|
|
837
|
+
self.is_git_or_bare_git_repository(repository)
|
|
838
|
+
tags = self.git_get_tags(repository)
|
|
839
|
+
tags_count = len(tags)
|
|
840
|
+
counter = 0
|
|
841
|
+
for tag in tags:
|
|
842
|
+
counter = counter+1
|
|
843
|
+
self.log.log(f"Process tag {counter}/{tags_count}.", LogLevel.Information)
|
|
844
|
+
# tag is on source-branch
|
|
845
|
+
if self.git_commit_is_ancestor(repository, tag, tag_source_branch):
|
|
846
|
+
commit_id_old = self.git_get_commit_id(repository, tag)
|
|
847
|
+
commit_date: datetime = self.git_get_commit_date(repository, commit_id_old)
|
|
848
|
+
date_as_string = self.__datetime_to_string_for_git(commit_date)
|
|
849
|
+
search_commit_result = self.run_program_argsasarray("git", ["log", f'--after="{date_as_string}"', f'--before="{date_as_string}"', "--pretty=format:%H", tag_target_branch], repository, throw_exception_if_exitcode_is_not_zero=False)
|
|
850
|
+
if search_commit_result[0] != 0 or not GeneralUtilities.string_has_nonwhitespace_content(search_commit_result[1]):
|
|
851
|
+
raise ValueError(f"Can not calculate corresponding commit for tag '{tag}'.")
|
|
852
|
+
commit_id_new = search_commit_result[1]
|
|
853
|
+
self.git_delete_tag(repository, tag)
|
|
854
|
+
self.git_create_tag(repository, commit_id_new, tag, sign, message)
|
|
855
|
+
|
|
856
|
+
@GeneralUtilities.check_arguments
|
|
857
|
+
def get_current_git_branch_has_tag(self, repository_folder: str) -> bool:
|
|
858
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
859
|
+
result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
|
|
860
|
+
return result[0] == 0
|
|
861
|
+
|
|
862
|
+
@GeneralUtilities.check_arguments
|
|
863
|
+
def get_latest_git_tag(self, repository_folder: str) -> str:
|
|
864
|
+
self.is_git_or_bare_git_repository(repository_folder)
|
|
865
|
+
result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder)
|
|
866
|
+
result = result[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
|
|
867
|
+
return result
|
|
868
|
+
|
|
869
|
+
@GeneralUtilities.check_arguments
|
|
870
|
+
def get_staged_or_committed_git_ignored_files(self, repository_folder: str) -> list[str]:
|
|
871
|
+
self.assert_is_git_repository(repository_folder)
|
|
872
|
+
temp_result = self.run_program_argsasarray("git", ["ls-files", "-i", "-c", "--exclude-standard"], repository_folder)
|
|
873
|
+
temp_result = temp_result[1].replace("\r", GeneralUtilities.empty_string)
|
|
874
|
+
result = [line for line in temp_result.split("\n") if len(line) > 0]
|
|
875
|
+
return result
|
|
876
|
+
|
|
877
|
+
@GeneralUtilities.check_arguments
|
|
878
|
+
def git_repository_has_commits(self, repository_folder: str) -> bool:
|
|
879
|
+
self.assert_is_git_repository(repository_folder)
|
|
880
|
+
return self.run_program_argsasarray("git", ["rev-parse", "--verify", "HEAD"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)[0] == 0
|
|
881
|
+
|
|
882
|
+
@GeneralUtilities.check_arguments
|
|
883
|
+
def run_git_command_in_repository_and_submodules(self, repository_folder: str, arguments: list[str],print_live_output:bool) -> None:
|
|
884
|
+
GeneralUtilities.assert_condition(self.is_git_or_bare_git_repository(repository_folder),f"\"{repository_folder}\" is not a git-repository.")
|
|
885
|
+
self.log.log("Run \"git "+" ".join(arguments)+f"\" in {repository_folder} and its submodules...",LogLevel.Debug)
|
|
886
|
+
self.run_program_argsasarray("git", arguments, repository_folder,print_live_output=print_live_output)
|
|
887
|
+
if not self.is_bare_git_repository(repository_folder) and 0<len(self.get_git_submodules(repository_folder)):
|
|
888
|
+
self.run_program_argsasarray("git", ["submodule", "foreach", "--recursive", "git"]+arguments, repository_folder,print_live_output=print_live_output)
|
|
889
|
+
|
|
890
|
+
@GeneralUtilities.check_arguments
|
|
891
|
+
def export_filemetadata(self, folder: str, target_file: str, encoding: str = "utf-8", filter_function=None) -> None:
|
|
892
|
+
folder = GeneralUtilities.resolve_relative_path_from_current_working_directory(folder)
|
|
893
|
+
lines = list()
|
|
894
|
+
path_prefix = len(folder)+1
|
|
895
|
+
items = dict()
|
|
896
|
+
for item in GeneralUtilities.get_all_folders_of_folder(folder):
|
|
897
|
+
items[item] = "d"
|
|
898
|
+
for item in GeneralUtilities.get_all_files_of_folder(folder):
|
|
899
|
+
items[item] = "f"
|
|
900
|
+
for file_or_folder, item_type in items.items():
|
|
901
|
+
truncated_file = file_or_folder[path_prefix:]
|
|
902
|
+
if (filter_function is None or filter_function(folder, truncated_file)):
|
|
903
|
+
owner_and_permisssion = self.get_file_owner_and_file_permission(file_or_folder)
|
|
904
|
+
user = owner_and_permisssion[0]
|
|
905
|
+
permissions = owner_and_permisssion[1]
|
|
906
|
+
lines.append(f"{truncated_file};{item_type};{user};{permissions}")
|
|
907
|
+
lines = sorted(lines, key=str.casefold)
|
|
908
|
+
with open(target_file, "w", encoding=encoding) as file_object:
|
|
909
|
+
file_object.write("\n".join(lines))
|
|
910
|
+
|
|
911
|
+
@GeneralUtilities.check_arguments
|
|
912
|
+
def escape_git_repositories_in_folder(self, folder: str) -> dict[str, str]:
|
|
913
|
+
return self.__escape_git_repositories_in_folder_internal(folder, dict[str, str]())
|
|
914
|
+
|
|
915
|
+
@GeneralUtilities.check_arguments
|
|
916
|
+
def __escape_git_repositories_in_folder_internal(self, folder: str, renamed_items: dict[str, str]) -> dict[str, str]:
|
|
917
|
+
for file in GeneralUtilities.get_direct_files_of_folder(folder):
|
|
918
|
+
filename = os.path.basename(file)
|
|
919
|
+
if ".git" in filename:
|
|
920
|
+
new_name = filename.replace(".git", ".gitx")
|
|
921
|
+
target = os.path.join(folder, new_name)
|
|
922
|
+
os.rename(file, target)
|
|
923
|
+
renamed_items[target] = file
|
|
924
|
+
for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
|
|
925
|
+
foldername = os.path.basename(subfolder)
|
|
926
|
+
if ".git" in foldername:
|
|
927
|
+
new_name = foldername.replace(".git", ".gitx")
|
|
928
|
+
subfolder2 = os.path.join(str(Path(subfolder).parent), new_name)
|
|
929
|
+
os.rename(subfolder, subfolder2)
|
|
930
|
+
renamed_items[subfolder2] = subfolder
|
|
931
|
+
else:
|
|
932
|
+
subfolder2 = subfolder
|
|
933
|
+
self.__escape_git_repositories_in_folder_internal(subfolder2, renamed_items)
|
|
934
|
+
return renamed_items
|
|
935
|
+
|
|
936
|
+
@GeneralUtilities.check_arguments
|
|
937
|
+
def deescape_git_repositories_in_folder(self, renamed_items: dict[str, str]):
|
|
938
|
+
for renamed_item, original_name in renamed_items.items():
|
|
939
|
+
os.rename(renamed_item, original_name)
|
|
940
|
+
|
|
941
|
+
@GeneralUtilities.check_arguments
|
|
942
|
+
def is_git_repository(self, folder: str) -> bool:
|
|
943
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
944
|
+
folder=folder.replace("\\","/")
|
|
945
|
+
if folder.endswith("/"):
|
|
946
|
+
folder = folder[:-1]
|
|
947
|
+
if not self.is_folder(folder):
|
|
948
|
+
raise ValueError(f"Folder '{folder}' does not exist.")
|
|
949
|
+
git_folder_path = f"{folder}/.git"
|
|
950
|
+
return self.is_folder(git_folder_path) or self.is_file(git_folder_path)
|
|
951
|
+
|
|
952
|
+
@GeneralUtilities.check_arguments
|
|
953
|
+
def is_bare_git_repository(self, folder: str) -> bool:
|
|
954
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
955
|
+
if folder.endswith("/") or folder.endswith("\\"):
|
|
956
|
+
folder = folder[:-1]
|
|
957
|
+
if not self.is_folder(folder):
|
|
958
|
+
raise ValueError(f"Folder '{folder}' does not exist.")
|
|
959
|
+
return folder.endswith(".git")
|
|
960
|
+
|
|
961
|
+
@GeneralUtilities.check_arguments
|
|
962
|
+
def is_git_or_bare_git_repository(self, folder: str) -> bool:
|
|
963
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
964
|
+
return self.is_git_repository(folder) or self.is_bare_git_repository(folder)
|
|
965
|
+
|
|
966
|
+
@GeneralUtilities.check_arguments
|
|
967
|
+
def assert_is_git_repository(self, folder: str) -> str:
|
|
968
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
969
|
+
GeneralUtilities.assert_condition(self.is_git_repository(folder), f"'{folder}' is not a git-repository.")
|
|
970
|
+
|
|
971
|
+
@GeneralUtilities.check_arguments
|
|
972
|
+
def convert_git_repository_to_bare_repository(self, repository_folder: str):
|
|
973
|
+
repository_folder = repository_folder.replace("\\", "/")
|
|
974
|
+
self.assert_is_git_repository(repository_folder)
|
|
975
|
+
git_folder = repository_folder + "/.git"
|
|
976
|
+
if not self.is_folder(git_folder):
|
|
977
|
+
raise ValueError(f"Converting '{repository_folder}' to a bare repository not possible. The folder '{git_folder}' does not exist. Converting is currently only supported when the git-folder is a direct folder in a repository and not a reference to another location.")
|
|
978
|
+
target_folder: str = repository_folder + ".git"
|
|
979
|
+
GeneralUtilities.ensure_directory_exists(target_folder)
|
|
980
|
+
GeneralUtilities.move_content_of_folder(git_folder, target_folder)
|
|
981
|
+
GeneralUtilities.ensure_directory_does_not_exist(repository_folder)
|
|
982
|
+
self.run_program_argsasarray("git", ["config", "--bool", "core.bare", "true"], target_folder)
|
|
983
|
+
|
|
984
|
+
@GeneralUtilities.check_arguments
|
|
985
|
+
def assert_no_uncommitted_changes(self, repository_folder: str):
|
|
986
|
+
if self.git_repository_has_uncommitted_changes(repository_folder):
|
|
987
|
+
raise ValueError(f"Repository '{repository_folder}' has uncommitted changes.")
|
|
988
|
+
|
|
989
|
+
@GeneralUtilities.check_arguments
|
|
990
|
+
def list_content(self, path: str, include_files: bool, include_folder: bool, printonlynamewithoutpath: bool) -> list[str]:
|
|
991
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
992
|
+
result: list[str] = []
|
|
993
|
+
if self.program_runner.will_be_executed_locally():
|
|
994
|
+
if include_files:
|
|
995
|
+
result = result + GeneralUtilities.get_direct_files_of_folder(path)
|
|
996
|
+
if include_folder:
|
|
997
|
+
result = result + GeneralUtilities.get_direct_folders_of_folder(path)
|
|
998
|
+
else:
|
|
999
|
+
arguments = ["--path", path]
|
|
1000
|
+
if not include_files:
|
|
1001
|
+
arguments = arguments+["--excludefiles"]
|
|
1002
|
+
if not include_folder:
|
|
1003
|
+
arguments = arguments+["--excludedirectories"]
|
|
1004
|
+
if printonlynamewithoutpath:
|
|
1005
|
+
arguments = arguments+["--printonlynamewithoutpath"]
|
|
1006
|
+
exit_code, stdout, stderr, _ = self.run_program_argsasarray("sclistfoldercontent", arguments)
|
|
1007
|
+
if exit_code == 0:
|
|
1008
|
+
for line in stdout.split("\n"):
|
|
1009
|
+
normalized_line = line.replace("\r", "")
|
|
1010
|
+
result.append(normalized_line)
|
|
1011
|
+
else:
|
|
1012
|
+
raise ValueError(f"Fatal error occurrs while checking whether file '{path}' exists. StdErr: '{stderr}'")
|
|
1013
|
+
result = [item for item in result if GeneralUtilities.string_has_nonwhitespace_content(item)]
|
|
1014
|
+
return result
|
|
1015
|
+
|
|
1016
|
+
@GeneralUtilities.check_arguments
|
|
1017
|
+
def is_file(self, path: str) -> bool:
|
|
1018
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
1019
|
+
if self.program_runner.will_be_executed_locally():
|
|
1020
|
+
return os.path.isfile(path) # works only locally, but much more performant than always running an external program
|
|
1021
|
+
else:
|
|
1022
|
+
exit_code, _, stderr, _ = self.run_program_argsasarray("scfileexists", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
1023
|
+
if exit_code == 0:
|
|
1024
|
+
return True
|
|
1025
|
+
elif exit_code == 1:
|
|
1026
|
+
raise ValueError(f"Not calculatable whether file '{path}' exists. StdErr: '{stderr}'")
|
|
1027
|
+
elif exit_code == 2:
|
|
1028
|
+
return False
|
|
1029
|
+
raise ValueError(f"Fatal error occurrs while checking whether file '{path}' exists. StdErr: '{stderr}'")
|
|
1030
|
+
|
|
1031
|
+
@GeneralUtilities.check_arguments
|
|
1032
|
+
def get_size(self, path: str) -> int:
|
|
1033
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
1034
|
+
if self.program_runner.will_be_executed_locally():
|
|
1035
|
+
return os.path.getsize(path) # works only locally, but much more performant than always running an external program
|
|
1036
|
+
else:
|
|
1037
|
+
exit_code, stdout, stderr, _ = self.run_program_argsasarray("scgetsize", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
1038
|
+
if exit_code == 0:
|
|
1039
|
+
return int(stdout.replace("\r","").replace("\n","").strip())
|
|
1040
|
+
else:
|
|
1041
|
+
raise ValueError(f"Fatal error occurrs while checking whether file '{path}' exists. StdErr: '{stderr}'")
|
|
1042
|
+
|
|
1043
|
+
@GeneralUtilities.check_arguments
|
|
1044
|
+
def is_folder(self, path: str) -> bool:
|
|
1045
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
1046
|
+
if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
|
|
1047
|
+
return os.path.isdir(path)
|
|
1048
|
+
else:
|
|
1049
|
+
exit_code, _, stderr, _ = self.run_program_argsasarray("scfolderexists", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
1050
|
+
if exit_code == 0:
|
|
1051
|
+
return True
|
|
1052
|
+
elif exit_code == 1:
|
|
1053
|
+
raise ValueError(f"Not calculatable whether folder '{path}' exists. StdErr: '{stderr}'")
|
|
1054
|
+
elif exit_code == 2:
|
|
1055
|
+
return False
|
|
1056
|
+
raise ValueError(f"Fatal error occurrs while checking whether folder '{path}' exists. StdErr: '{stderr}'")
|
|
1057
|
+
|
|
1058
|
+
@GeneralUtilities.check_arguments
|
|
1059
|
+
def get_file_content(self, path: str, encoding: str = "utf-8") -> str:
|
|
1060
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
1061
|
+
if self.program_runner.will_be_executed_locally():
|
|
1062
|
+
return GeneralUtilities.read_text_from_file(path, encoding)
|
|
1063
|
+
else:
|
|
1064
|
+
result = self.run_program_argsasarray("scprintfilecontent", ["--path", path, "--encofing", encoding]) # works platform-indepent
|
|
1065
|
+
return result[1].replace("\\n", "\n")
|
|
1066
|
+
|
|
1067
|
+
@GeneralUtilities.check_arguments
|
|
1068
|
+
def set_file_content(self, path: str, content: str, encoding: str = "utf-8") -> None:
|
|
1069
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
1070
|
+
if self.program_runner.will_be_executed_locally():
|
|
1071
|
+
GeneralUtilities.write_text_to_file(path, content, encoding)
|
|
1072
|
+
else:
|
|
1073
|
+
content_bytes = content.encode('utf-8')
|
|
1074
|
+
base64_bytes = base64.b64encode(content_bytes)
|
|
1075
|
+
base64_string = base64_bytes.decode('utf-8')
|
|
1076
|
+
self.run_program_argsasarray("scsetfilecontent", ["--path", path, "--argumentisinbase64", "--content", base64_string]) # works platform-indepent
|
|
1077
|
+
|
|
1078
|
+
@GeneralUtilities.check_arguments
|
|
1079
|
+
def remove(self, path: str) -> None:
|
|
1080
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
1081
|
+
if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
|
|
1082
|
+
if os.path.isdir(path):
|
|
1083
|
+
GeneralUtilities.ensure_directory_does_not_exist(path)
|
|
1084
|
+
if os.path.isfile(path):
|
|
1085
|
+
GeneralUtilities.ensure_file_does_not_exist(path)
|
|
1086
|
+
else:
|
|
1087
|
+
if self.is_file(path):
|
|
1088
|
+
exit_code, stdout, stderr, _ = self.run_program_argsasarray("scremovefile", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
1089
|
+
if exit_code != 0:
|
|
1090
|
+
raise ValueError(f"Fatal error occurrs while removing file '{path}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
|
|
1091
|
+
if self.is_folder(path):
|
|
1092
|
+
exit_code, stdout, stderr, _ = self.run_program_argsasarray("scremovefolder", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
1093
|
+
if exit_code != 0:
|
|
1094
|
+
raise ValueError(f"Fatal error occurrs while removing folder '{path}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
|
|
1095
|
+
|
|
1096
|
+
@GeneralUtilities.check_arguments
|
|
1097
|
+
def rename(self, source: str, target: str) -> None:
|
|
1098
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
1099
|
+
if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
|
|
1100
|
+
os.rename(source, target)
|
|
1101
|
+
else:
|
|
1102
|
+
exit_code, stdout, stderr, _ = self.run_program_argsasarray("screname", ["--source", source, "--target", target], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
1103
|
+
if exit_code != 0:
|
|
1104
|
+
raise ValueError(f"Fatal error occurrs while renaming '{source}' to '{target}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
|
|
1105
|
+
|
|
1106
|
+
@GeneralUtilities.check_arguments
|
|
1107
|
+
def copy(self, source: str, target: str) -> None:
|
|
1108
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
1109
|
+
if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
|
|
1110
|
+
if os.path.isfile(target) or os.path.isdir(target):
|
|
1111
|
+
raise ValueError(f"Can not copy to '{target}' because the target already exists.")
|
|
1112
|
+
if os.path.isfile(source):
|
|
1113
|
+
shutil.copyfile(source, target)
|
|
1114
|
+
elif os.path.isdir(source):
|
|
1115
|
+
GeneralUtilities.ensure_directory_exists(target)
|
|
1116
|
+
GeneralUtilities.copy_content_of_folder(source, target)
|
|
1117
|
+
else:
|
|
1118
|
+
raise ValueError(f"'{source}' can not be copied because the path does not exist.")
|
|
1119
|
+
else:
|
|
1120
|
+
exit_code, stdout, stderr, _ = self.run_program_argsasarray("sccopy", ["--source", source, "--target", target], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
1121
|
+
if exit_code != 0:
|
|
1122
|
+
raise ValueError(f"Fatal error occurrs while copying '{source}' to '{target}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
|
|
1123
|
+
|
|
1124
|
+
@GeneralUtilities.check_arguments
|
|
1125
|
+
def create_file(self, path: str, error_if_already_exists: bool, create_necessary_folder: bool) -> None:
|
|
1126
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
1127
|
+
if self.program_runner.will_be_executed_locally():
|
|
1128
|
+
if not os.path.isabs(path):
|
|
1129
|
+
path = os.path.join(os.getcwd(), path)
|
|
1130
|
+
|
|
1131
|
+
if os.path.isfile(path) and error_if_already_exists:
|
|
1132
|
+
raise ValueError(f"File '{path}' already exists.")
|
|
1133
|
+
|
|
1134
|
+
# TODO maybe it should be checked if there is a folder with the same path which already exists.
|
|
1135
|
+
|
|
1136
|
+
folder = os.path.dirname(path)
|
|
1137
|
+
|
|
1138
|
+
if not os.path.isdir(folder):
|
|
1139
|
+
if create_necessary_folder:
|
|
1140
|
+
GeneralUtilities.ensure_directory_exists(folder) # TODO check if this also create nested folders if required
|
|
1141
|
+
else:
|
|
1142
|
+
raise ValueError(f"Folder '{folder}' does not exist.")
|
|
1143
|
+
|
|
1144
|
+
GeneralUtilities.ensure_file_exists(path)
|
|
1145
|
+
else:
|
|
1146
|
+
arguments = ["--path", path]
|
|
1147
|
+
|
|
1148
|
+
if error_if_already_exists:
|
|
1149
|
+
arguments = arguments+["--errorwhenexists"]
|
|
1150
|
+
|
|
1151
|
+
if create_necessary_folder:
|
|
1152
|
+
arguments = arguments+["--createnecessaryfolder"]
|
|
1153
|
+
|
|
1154
|
+
exit_code, stdout, stderr, _ = self.run_program_argsasarray("sccreatefile", arguments, throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
1155
|
+
if exit_code != 0:
|
|
1156
|
+
raise ValueError(f"Fatal error occurrs while create file '{path}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
|
|
1157
|
+
|
|
1158
|
+
@GeneralUtilities.check_arguments
|
|
1159
|
+
def create_folder(self, path: str, error_if_already_exists: bool, create_necessary_folder: bool) -> None:
|
|
1160
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
1161
|
+
if self.program_runner.will_be_executed_locally():
|
|
1162
|
+
if not os.path.isabs(path):
|
|
1163
|
+
path = os.path.join(os.getcwd(), path)
|
|
1164
|
+
|
|
1165
|
+
if os.path.isdir(path) and error_if_already_exists:
|
|
1166
|
+
raise ValueError(f"Folder '{path}' already exists.")
|
|
1167
|
+
|
|
1168
|
+
# TODO maybe it should be checked if there is a file with the same path which already exists.
|
|
1169
|
+
|
|
1170
|
+
folder = os.path.dirname(path)
|
|
1171
|
+
|
|
1172
|
+
if not os.path.isdir(folder):
|
|
1173
|
+
if create_necessary_folder:
|
|
1174
|
+
GeneralUtilities.ensure_directory_exists(folder) # TODO check if this also create nested folders if required
|
|
1175
|
+
else:
|
|
1176
|
+
raise ValueError(f"Folder '{folder}' does not exist.")
|
|
1177
|
+
|
|
1178
|
+
GeneralUtilities.ensure_directory_exists(path)
|
|
1179
|
+
else:
|
|
1180
|
+
arguments = ["--path", path]
|
|
1181
|
+
|
|
1182
|
+
if error_if_already_exists:
|
|
1183
|
+
arguments = arguments+["--errorwhenexists"]
|
|
1184
|
+
|
|
1185
|
+
if create_necessary_folder:
|
|
1186
|
+
arguments = arguments+["--createnecessaryfolder"]
|
|
1187
|
+
|
|
1188
|
+
exit_code, stdout, stderr, _ = self.run_program_argsasarray("sccreatefolder", arguments, throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
|
|
1189
|
+
if exit_code != 0:
|
|
1190
|
+
raise ValueError(f"Fatal error occurrs while create folder '{path}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
|
|
1191
|
+
|
|
1192
|
+
@GeneralUtilities.check_arguments
|
|
1193
|
+
def __sort_fmd(self, line: str):
|
|
1194
|
+
splitted: list = line.split(";")
|
|
1195
|
+
filetype: str = splitted[1]
|
|
1196
|
+
if filetype == "d":
|
|
1197
|
+
return -1
|
|
1198
|
+
if filetype == "f":
|
|
1199
|
+
return 1
|
|
1200
|
+
return 0
|
|
1201
|
+
|
|
1202
|
+
@GeneralUtilities.check_arguments
|
|
1203
|
+
def restore_filemetadata(self, folder: str, source_file: str, strict=False, encoding: str = "utf-8", create_folder_is_not_exist: bool = True) -> None:
|
|
1204
|
+
lines = GeneralUtilities.read_lines_from_file(source_file, encoding)
|
|
1205
|
+
lines.sort(key=self.__sort_fmd)
|
|
1206
|
+
for line in lines:
|
|
1207
|
+
splitted: list = line.split(";")
|
|
1208
|
+
full_path_of_file_or_folder: str = os.path.join(folder, splitted[0])
|
|
1209
|
+
filetype: str = splitted[1]
|
|
1210
|
+
user: str = splitted[2]
|
|
1211
|
+
permissions: str = splitted[3]
|
|
1212
|
+
if filetype == "d" and create_folder_is_not_exist and not os.path.isdir(full_path_of_file_or_folder):
|
|
1213
|
+
GeneralUtilities.ensure_directory_exists(full_path_of_file_or_folder)
|
|
1214
|
+
if (filetype == "f" and os.path.isfile(full_path_of_file_or_folder)) or (filetype == "d" and os.path.isdir(full_path_of_file_or_folder)):
|
|
1215
|
+
self.set_owner(full_path_of_file_or_folder, user, os.name != 'nt')
|
|
1216
|
+
self.set_permission(full_path_of_file_or_folder, permissions)
|
|
1217
|
+
else:
|
|
1218
|
+
if strict:
|
|
1219
|
+
if filetype == "f":
|
|
1220
|
+
filetype_full = "File"
|
|
1221
|
+
elif filetype == "d":
|
|
1222
|
+
filetype_full = "Directory"
|
|
1223
|
+
else:
|
|
1224
|
+
raise ValueError(f"Unknown filetype: {GeneralUtilities.str_none_safe(filetype)}")
|
|
1225
|
+
raise ValueError(f"{filetype_full} '{full_path_of_file_or_folder}' does not exist")
|
|
1226
|
+
|
|
1227
|
+
@GeneralUtilities.check_arguments
|
|
1228
|
+
def __calculate_lengh_in_seconds(self, filename: str, folder: str) -> float:
|
|
1229
|
+
argument = ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', filename]
|
|
1230
|
+
result = self.run_program_argsasarray("ffprobe", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
1231
|
+
return float(result[1].replace('\n', ''))
|
|
1232
|
+
|
|
1233
|
+
@GeneralUtilities.check_arguments
|
|
1234
|
+
def __create_thumbnails(self, filename: str, fps: str, folder: str, tempname_for_thumbnails: str) -> list[str]:
|
|
1235
|
+
argument = ['-i', filename, '-r', fps, '-vf', 'scale=-1:120', '-vcodec', 'png', f'{tempname_for_thumbnails}-%002d.png']
|
|
1236
|
+
self.run_program_argsasarray("ffmpeg", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
1237
|
+
files = GeneralUtilities.get_direct_files_of_folder(folder)
|
|
1238
|
+
result: list[str] = []
|
|
1239
|
+
regex = "^"+re.escape(tempname_for_thumbnails)+"\\-\\d+\\.png$"
|
|
1240
|
+
regex_for_files = re.compile(regex)
|
|
1241
|
+
for file in files:
|
|
1242
|
+
filename = os.path.basename(file)
|
|
1243
|
+
if regex_for_files.match(filename):
|
|
1244
|
+
result.append(file)
|
|
1245
|
+
GeneralUtilities.assert_condition(0 < len(result), "No thumbnail-files found.")
|
|
1246
|
+
return result
|
|
1247
|
+
|
|
1248
|
+
@GeneralUtilities.check_arguments
|
|
1249
|
+
def __create_thumbnail(self, outputfilename: str, folder: str, length_in_seconds: float, tempname_for_thumbnails: str, amount_of_images: int) -> None:
|
|
1250
|
+
duration = timedelta(seconds=length_in_seconds)
|
|
1251
|
+
info = GeneralUtilities.timedelta_to_simple_string(duration)
|
|
1252
|
+
next_square_number = GeneralUtilities.get_next_square_number(amount_of_images)
|
|
1253
|
+
root = math.sqrt(next_square_number)
|
|
1254
|
+
rows: int = root # 5
|
|
1255
|
+
columns: int = root # math.ceil(amount_of_images/rows)
|
|
1256
|
+
argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{rows}x{columns}', f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
|
|
1257
|
+
self.run_program_argsasarray("montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
1258
|
+
|
|
1259
|
+
@GeneralUtilities.check_arguments
|
|
1260
|
+
def __create_thumbnail2(self, outputfilename: str, folder: str, length_in_seconds: float, rows: int, columns: int, tempname_for_thumbnails: str, amount_of_images: int) -> None:
|
|
1261
|
+
duration = timedelta(seconds=length_in_seconds)
|
|
1262
|
+
info = GeneralUtilities.timedelta_to_simple_string(duration)
|
|
1263
|
+
argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{rows}x{columns}', f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
|
|
1264
|
+
self.run_program_argsasarray("montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
1265
|
+
|
|
1266
|
+
@GeneralUtilities.check_arguments
|
|
1267
|
+
def __roundup(self, x: float, places: int) -> int:
|
|
1268
|
+
d = 10 ** places
|
|
1269
|
+
if x < 0:
|
|
1270
|
+
return math.floor(x * d) / d
|
|
1271
|
+
else:
|
|
1272
|
+
return math.ceil(x * d) / d
|
|
1273
|
+
|
|
1274
|
+
@GeneralUtilities.check_arguments
|
|
1275
|
+
def generate_thumbnail(self, file: str, frames_per_second: float, tempname_for_thumbnails: str = None, hook=None) -> None:
|
|
1276
|
+
if tempname_for_thumbnails is None:
|
|
1277
|
+
tempname_for_thumbnails = "t_"+str(uuid.uuid4())
|
|
1278
|
+
|
|
1279
|
+
file = GeneralUtilities.resolve_relative_path_from_current_working_directory(file)
|
|
1280
|
+
filename = os.path.basename(file)
|
|
1281
|
+
folder = os.path.dirname(file)
|
|
1282
|
+
filename_without_extension = Path(file).stem
|
|
1283
|
+
preview_files: list[str] = []
|
|
1284
|
+
try:
|
|
1285
|
+
length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
|
|
1286
|
+
# frames per second, example: frames_per_second="20fps" => 20 frames per second
|
|
1287
|
+
frames_per_second = self.__roundup(float(frames_per_second[:-3]), 2)
|
|
1288
|
+
frames_per_second_as_string = str(frames_per_second)
|
|
1289
|
+
preview_files = self.__create_thumbnails(filename, frames_per_second_as_string, folder, tempname_for_thumbnails)
|
|
1290
|
+
if hook is not None:
|
|
1291
|
+
hook(file, preview_files)
|
|
1292
|
+
actual_amounf_of_previewframes = len(preview_files)
|
|
1293
|
+
self.__create_thumbnail(filename_without_extension, folder, length_in_seconds, tempname_for_thumbnails, actual_amounf_of_previewframes)
|
|
1294
|
+
finally:
|
|
1295
|
+
for thumbnail_to_delete in preview_files:
|
|
1296
|
+
os.remove(thumbnail_to_delete)
|
|
1297
|
+
|
|
1298
|
+
@GeneralUtilities.check_arguments
|
|
1299
|
+
def generate_thumbnail_by_amount_of_pictures(self, file: str, amount_of_columns: int, amount_of_rows: int, tempname_for_thumbnails: str = None, hook=None) -> None:
|
|
1300
|
+
if tempname_for_thumbnails is None:
|
|
1301
|
+
tempname_for_thumbnails = "t_"+str(uuid.uuid4())
|
|
1302
|
+
|
|
1303
|
+
file = GeneralUtilities.resolve_relative_path_from_current_working_directory(file)
|
|
1304
|
+
filename = os.path.basename(file)
|
|
1305
|
+
folder = os.path.dirname(file)
|
|
1306
|
+
filename_without_extension = Path(file).stem
|
|
1307
|
+
preview_files: list[str] = []
|
|
1308
|
+
try:
|
|
1309
|
+
length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
|
|
1310
|
+
amounf_of_previewframes = int(amount_of_columns*amount_of_rows)
|
|
1311
|
+
frames_per_second_as_string = f"{amounf_of_previewframes-2}/{length_in_seconds}"
|
|
1312
|
+
preview_files = self.__create_thumbnails(filename, frames_per_second_as_string, folder, tempname_for_thumbnails)
|
|
1313
|
+
if hook is not None:
|
|
1314
|
+
hook(file, preview_files)
|
|
1315
|
+
actual_amounf_of_previewframes = len(preview_files)
|
|
1316
|
+
self.__create_thumbnail2(filename_without_extension, folder, length_in_seconds, amount_of_rows, amount_of_columns, tempname_for_thumbnails, actual_amounf_of_previewframes)
|
|
1317
|
+
finally:
|
|
1318
|
+
for thumbnail_to_delete in preview_files:
|
|
1319
|
+
os.remove(thumbnail_to_delete)
|
|
1320
|
+
|
|
1321
|
+
@GeneralUtilities.check_arguments
|
|
1322
|
+
def extract_pdf_pages(self, file: str, from_page: int, to_page: int, outputfile: str) -> None:
|
|
1323
|
+
pdf_reader: PdfReader = PdfReader(file)
|
|
1324
|
+
pdf_writer: PdfWriter = PdfWriter()
|
|
1325
|
+
start = from_page
|
|
1326
|
+
end = to_page
|
|
1327
|
+
while start <= end:
|
|
1328
|
+
pdf_writer.add_page(pdf_reader.pages[start-1])
|
|
1329
|
+
start += 1
|
|
1330
|
+
with open(outputfile, 'wb') as out:
|
|
1331
|
+
pdf_writer.write(out)
|
|
1332
|
+
|
|
1333
|
+
@GeneralUtilities.check_arguments
|
|
1334
|
+
def merge_pdf_files(self, files: list[str], outputfile: str) -> None:
|
|
1335
|
+
# TODO add wildcard-option
|
|
1336
|
+
pdfFileMerger: PdfWriter = PdfWriter()
|
|
1337
|
+
for file in files:
|
|
1338
|
+
with open(file, "rb") as f:
|
|
1339
|
+
pdfFileMerger.append(f)
|
|
1340
|
+
with open(outputfile, "wb") as output:
|
|
1341
|
+
pdfFileMerger.write(output)
|
|
1342
|
+
pdfFileMerger.close()
|
|
1343
|
+
|
|
1344
|
+
@GeneralUtilities.check_arguments
|
|
1345
|
+
def show_missing_files(self, folderA: str, folderB: str):
|
|
1346
|
+
for file in GeneralUtilities.get_missing_files(folderA, folderB):
|
|
1347
|
+
GeneralUtilities.write_message_to_stdout(file)
|
|
1348
|
+
|
|
1349
|
+
@GeneralUtilities.check_arguments
|
|
1350
|
+
def SCCreateEmptyFileWithSpecificSize(self, name: str, size_string: str) -> int:
|
|
1351
|
+
if size_string.isdigit():
|
|
1352
|
+
size = int(size_string)
|
|
1353
|
+
else:
|
|
1354
|
+
if len(size_string) >= 3:
|
|
1355
|
+
if (size_string.endswith("kb")):
|
|
1356
|
+
size = int(size_string[:-2]) * pow(10, 3)
|
|
1357
|
+
elif (size_string.endswith("mb")):
|
|
1358
|
+
size = int(size_string[:-2]) * pow(10, 6)
|
|
1359
|
+
elif (size_string.endswith("gb")):
|
|
1360
|
+
size = int(size_string[:-2]) * pow(10, 9)
|
|
1361
|
+
elif (size_string.endswith("kib")):
|
|
1362
|
+
size = int(size_string[:-3]) * pow(2, 10)
|
|
1363
|
+
elif (size_string.endswith("mib")):
|
|
1364
|
+
size = int(size_string[:-3]) * pow(2, 20)
|
|
1365
|
+
elif (size_string.endswith("gib")):
|
|
1366
|
+
size = int(size_string[:-3]) * pow(2, 30)
|
|
1367
|
+
else:
|
|
1368
|
+
self.log.log("Wrong format", LogLevel.Error)
|
|
1369
|
+
return 1
|
|
1370
|
+
else:
|
|
1371
|
+
self.log.log("Wrong format", LogLevel.Error)
|
|
1372
|
+
return 1
|
|
1373
|
+
with open(name, "wb") as f:
|
|
1374
|
+
f.seek(size-1)
|
|
1375
|
+
f.write(b"\0")
|
|
1376
|
+
return 0
|
|
1377
|
+
|
|
1378
|
+
@GeneralUtilities.check_arguments
|
|
1379
|
+
def SCCreateHashOfAllFiles(self, folder: str) -> None:
|
|
1380
|
+
for file in GeneralUtilities.absolute_file_paths(folder):
|
|
1381
|
+
with open(file+".sha256", "w+", encoding="utf-8") as f:
|
|
1382
|
+
f.write(GeneralUtilities.get_sha256_of_file(file))
|
|
1383
|
+
|
|
1384
|
+
@GeneralUtilities.check_arguments
|
|
1385
|
+
def SCCreateSimpleMergeWithoutRelease(self, repository: str, sourcebranch: str, targetbranch: str, remotename: str, remove_source_branch: bool) -> None:
|
|
1386
|
+
commitid = self.git_merge(repository, sourcebranch, targetbranch, False, True)
|
|
1387
|
+
self.git_merge(repository, targetbranch, sourcebranch, True, True)
|
|
1388
|
+
created_version = self.get_semver_version_from_gitversion(repository)
|
|
1389
|
+
self.git_create_tag(repository, commitid, f"v{created_version}", True)
|
|
1390
|
+
self.git_push(repository, remotename, targetbranch, targetbranch, False, True)
|
|
1391
|
+
if (GeneralUtilities.string_has_nonwhitespace_content(remotename)):
|
|
1392
|
+
self.git_push(repository, remotename, sourcebranch, sourcebranch, False, True)
|
|
1393
|
+
if (remove_source_branch):
|
|
1394
|
+
self.git_remove_branch(repository, sourcebranch)
|
|
1395
|
+
|
|
1396
|
+
@GeneralUtilities.check_arguments
|
|
1397
|
+
def sc_organize_lines_in_file(self, file: str, encoding: str="utf-8", sort: bool = False, remove_duplicated_lines: bool = False, ignore_first_line: bool = False, remove_empty_lines: bool = True, ignored_start_character: list = list()):
|
|
1398
|
+
GeneralUtilities.assert_file_exists(file)
|
|
1399
|
+
|
|
1400
|
+
# read file
|
|
1401
|
+
lines = GeneralUtilities.read_lines_from_file(file, encoding)
|
|
1402
|
+
if (len(lines) == 0):
|
|
1403
|
+
return
|
|
1404
|
+
|
|
1405
|
+
# store first line if desired
|
|
1406
|
+
if (ignore_first_line):
|
|
1407
|
+
first_line = lines.pop(0)
|
|
1408
|
+
|
|
1409
|
+
# remove empty lines if desired
|
|
1410
|
+
if remove_empty_lines:
|
|
1411
|
+
temp = lines
|
|
1412
|
+
lines = []
|
|
1413
|
+
for line in temp:
|
|
1414
|
+
if (not (GeneralUtilities.string_is_none_or_whitespace(line))):
|
|
1415
|
+
lines.append(line)
|
|
1416
|
+
|
|
1417
|
+
# remove duplicated lines if desired
|
|
1418
|
+
if remove_duplicated_lines:
|
|
1419
|
+
lines = GeneralUtilities.remove_duplicates(lines)
|
|
1420
|
+
|
|
1421
|
+
# sort lines if desired
|
|
1422
|
+
if sort:
|
|
1423
|
+
lines = sorted(lines, key=lambda singleline: self.__adapt_line_for_sorting(singleline, ignored_start_character))
|
|
1424
|
+
|
|
1425
|
+
# reinsert first line if required
|
|
1426
|
+
if ignore_first_line:
|
|
1427
|
+
lines.insert(0, first_line)
|
|
1428
|
+
|
|
1429
|
+
# write result to file
|
|
1430
|
+
GeneralUtilities.write_lines_to_file(file, lines, encoding)
|
|
1431
|
+
|
|
1432
|
+
|
|
1433
|
+
@GeneralUtilities.check_arguments
|
|
1434
|
+
def __adapt_line_for_sorting(self, line: str, ignored_start_characters: list):
|
|
1435
|
+
result = line.lower()
|
|
1436
|
+
while len(result) > 0 and result[0] in ignored_start_characters:
|
|
1437
|
+
result = result[1:]
|
|
1438
|
+
return result
|
|
1439
|
+
|
|
1440
|
+
@GeneralUtilities.check_arguments
|
|
1441
|
+
def SCGenerateSnkFiles(self, outputfolder, keysize=4096, amountofkeys=10) -> int:
|
|
1442
|
+
GeneralUtilities.ensure_directory_exists(outputfolder)
|
|
1443
|
+
for _ in range(amountofkeys):
|
|
1444
|
+
file = os.path.join(outputfolder, str(uuid.uuid4())+".snk")
|
|
1445
|
+
argument = f"-k {keysize} {file}"
|
|
1446
|
+
self.run_program("sn", argument, outputfolder)
|
|
1447
|
+
|
|
1448
|
+
@GeneralUtilities.check_arguments
|
|
1449
|
+
def __merge_files(self, sourcefile: str, targetfile: str) -> None:
|
|
1450
|
+
with open(sourcefile, "rb") as f:
|
|
1451
|
+
source_data = f.read()
|
|
1452
|
+
with open(targetfile, "ab") as fout:
|
|
1453
|
+
merge_separator = [0x0A]
|
|
1454
|
+
fout.write(bytes(merge_separator))
|
|
1455
|
+
fout.write(source_data)
|
|
1456
|
+
|
|
1457
|
+
@GeneralUtilities.check_arguments
|
|
1458
|
+
def __process_file(self, file: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
|
|
1459
|
+
new_filename = os.path.join(os.path.dirname(file), os.path.basename(file).replace(substringInFilename, newSubstringInFilename))
|
|
1460
|
+
if file != new_filename:
|
|
1461
|
+
if os.path.isfile(new_filename):
|
|
1462
|
+
if filecmp.cmp(file, new_filename):
|
|
1463
|
+
send2trash.send2trash(file)
|
|
1464
|
+
else:
|
|
1465
|
+
if conflictResolveMode == "ignore":
|
|
1466
|
+
pass
|
|
1467
|
+
elif conflictResolveMode == "preservenewest":
|
|
1468
|
+
if (os.path.getmtime(file) - os.path.getmtime(new_filename) > 0):
|
|
1469
|
+
send2trash.send2trash(file)
|
|
1470
|
+
else:
|
|
1471
|
+
send2trash.send2trash(new_filename)
|
|
1472
|
+
os.rename(file, new_filename)
|
|
1473
|
+
elif (conflictResolveMode == "merge"):
|
|
1474
|
+
self.__merge_files(file, new_filename)
|
|
1475
|
+
send2trash.send2trash(file)
|
|
1476
|
+
else:
|
|
1477
|
+
raise ValueError('Unknown conflict resolve mode')
|
|
1478
|
+
else:
|
|
1479
|
+
os.rename(file, new_filename)
|
|
1480
|
+
|
|
1481
|
+
@GeneralUtilities.check_arguments
|
|
1482
|
+
def SCReplaceSubstringsInFilenames(self, folder: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
|
|
1483
|
+
for file in GeneralUtilities.absolute_file_paths(folder):
|
|
1484
|
+
self.__process_file(file, substringInFilename, newSubstringInFilename, conflictResolveMode)
|
|
1485
|
+
|
|
1486
|
+
@GeneralUtilities.check_arguments
|
|
1487
|
+
def __check_file(self, file: str, searchstring: str) -> None:
|
|
1488
|
+
bytes_ascii = bytes(searchstring, "ascii")
|
|
1489
|
+
# often called "unicode-encoding"
|
|
1490
|
+
bytes_utf16 = bytes(searchstring, "utf-16")
|
|
1491
|
+
bytes_utf8 = bytes(searchstring, "utf-8")
|
|
1492
|
+
with open(file, mode='rb') as file_object:
|
|
1493
|
+
content = file_object.read()
|
|
1494
|
+
if bytes_ascii in content:
|
|
1495
|
+
GeneralUtilities.write_message_to_stdout(file)
|
|
1496
|
+
elif bytes_utf16 in content:
|
|
1497
|
+
GeneralUtilities.write_message_to_stdout(file)
|
|
1498
|
+
elif bytes_utf8 in content:
|
|
1499
|
+
GeneralUtilities.write_message_to_stdout(file)
|
|
1500
|
+
|
|
1501
|
+
@GeneralUtilities.check_arguments
|
|
1502
|
+
def SCSearchInFiles(self, folder: str, searchstring: str) -> None:
|
|
1503
|
+
for file in GeneralUtilities.absolute_file_paths(folder):
|
|
1504
|
+
self.__check_file(file, searchstring)
|
|
1505
|
+
|
|
1506
|
+
@GeneralUtilities.check_arguments
|
|
1507
|
+
def get_string_as_qr_code(self,string: str) -> None:
|
|
1508
|
+
qr = qrcode.QRCode()
|
|
1509
|
+
qr.add_data(string)
|
|
1510
|
+
f = io.StringIO()
|
|
1511
|
+
qr.print_ascii(out=f)
|
|
1512
|
+
f.seek(0)
|
|
1513
|
+
return f.read()
|
|
1514
|
+
|
|
1515
|
+
@GeneralUtilities.check_arguments
|
|
1516
|
+
def __print_qr_code_by_csv_line(self, displayname: str, website: str, emailaddress: str, key: str, period: str) -> None:
|
|
1517
|
+
qrcode_content = f"otpauth://totp/{website}:{emailaddress}?secret={key}&issuer={displayname}&period={period}"
|
|
1518
|
+
GeneralUtilities.write_message_to_stdout(f"{displayname} ({emailaddress}):")
|
|
1519
|
+
GeneralUtilities.write_message_to_stdout(qrcode_content)
|
|
1520
|
+
GeneralUtilities.write_message_to_stdout(self.get_string_as_qr_code(qrcode_content))
|
|
1521
|
+
|
|
1522
|
+
@GeneralUtilities.check_arguments
|
|
1523
|
+
def SCShow2FAAsQRCode(self, csvfile: str) -> None:
|
|
1524
|
+
lines = GeneralUtilities.read_csv_file(csvfile, True)
|
|
1525
|
+
lines.sort(key=lambda items: ''.join(items).lower())
|
|
1526
|
+
for line in lines:
|
|
1527
|
+
self.__print_qr_code_by_csv_line(line[0], line[1], line[2], line[3], line[4])
|
|
1528
|
+
GeneralUtilities.write_message_to_stdout(GeneralUtilities.get_longline())
|
|
1529
|
+
|
|
1530
|
+
@GeneralUtilities.check_arguments
|
|
1531
|
+
def SCCalculateBitcoinBlockHash(self, block_version_number: str, previousblockhash: str, transactionsmerkleroot: str, timestamp: str, target: str, nonce: str) -> str:
|
|
1532
|
+
# Example-values:
|
|
1533
|
+
# block_version_number: "00000020"
|
|
1534
|
+
# previousblockhash: "66720b99e07d284bd4fe67ff8c49a5db1dd8514fcdab61000000000000000000"
|
|
1535
|
+
# transactionsmerkleroot: "7829844f4c3a41a537b3131ca992643eaa9d093b2383e4cdc060ad7dc5481187"
|
|
1536
|
+
# timestamp: "51eb505a"
|
|
1537
|
+
# target: "c1910018"
|
|
1538
|
+
# nonce: "de19b302"
|
|
1539
|
+
header = str(block_version_number + previousblockhash + transactionsmerkleroot + timestamp + target + nonce)
|
|
1540
|
+
return binascii.hexlify(hashlib.sha256(hashlib.sha256(binascii.unhexlify(header)).digest()).digest()[::-1]).decode('utf-8')
|
|
1541
|
+
|
|
1542
|
+
@GeneralUtilities.check_arguments
|
|
1543
|
+
def SCChangeHashOfProgram(self, inputfile: str) -> None:
|
|
1544
|
+
valuetoappend = str(uuid.uuid4())
|
|
1545
|
+
|
|
1546
|
+
outputfile = inputfile + '.modified'
|
|
1547
|
+
|
|
1548
|
+
shutil.copy2(inputfile, outputfile)
|
|
1549
|
+
with open(outputfile, 'a', encoding="utf-8") as file:
|
|
1550
|
+
# TODO use rcedit for .exe-files instead of appending valuetoappend ( https://github.com/electron/rcedit/ )
|
|
1551
|
+
# background: you can retrieve the "original-filename" from the .exe-file like discussed here:
|
|
1552
|
+
# https://security.stackexchange.com/questions/210843/ is-it-possible-to-change-original-filename-of-an-exe
|
|
1553
|
+
# so removing the original filename with rcedit is probably a better way to make it more difficult to detect the programname.
|
|
1554
|
+
# this would obviously also change the hashvalue of the program so appending a whitespace is not required anymore.
|
|
1555
|
+
file.write(valuetoappend)
|
|
1556
|
+
|
|
1557
|
+
@GeneralUtilities.check_arguments
|
|
1558
|
+
def __adjust_folder_name(self, folder: str) -> str:
|
|
1559
|
+
result = os.path.dirname(folder).replace("\\", "/")
|
|
1560
|
+
if result == "/":
|
|
1561
|
+
return GeneralUtilities.empty_string
|
|
1562
|
+
else:
|
|
1563
|
+
return result
|
|
1564
|
+
|
|
1565
|
+
@GeneralUtilities.check_arguments
|
|
1566
|
+
def __create_iso(self, folder, iso_file) -> None:
|
|
1567
|
+
created_directories = []
|
|
1568
|
+
files_directory = "FILES"
|
|
1569
|
+
iso = pycdlib.PyCdlib()
|
|
1570
|
+
iso.new()
|
|
1571
|
+
files_directory = files_directory.upper()
|
|
1572
|
+
iso.add_directory("/" + files_directory)
|
|
1573
|
+
created_directories.append("/" + files_directory)
|
|
1574
|
+
for root, _, files in os.walk(folder):
|
|
1575
|
+
for file in files:
|
|
1576
|
+
full_path = os.path.join(root, file)
|
|
1577
|
+
with (open(full_path, "rb").read()) as text_io_wrapper:
|
|
1578
|
+
content = text_io_wrapper
|
|
1579
|
+
path_in_iso = '/' + files_directory + \
|
|
1580
|
+
self.__adjust_folder_name(full_path[len(folder)::1]).upper()
|
|
1581
|
+
if path_in_iso not in created_directories:
|
|
1582
|
+
iso.add_directory(path_in_iso)
|
|
1583
|
+
created_directories.append(path_in_iso)
|
|
1584
|
+
iso.add_fp(BytesIO(content), len(content), path_in_iso + '/' + file.upper() + ';1')
|
|
1585
|
+
iso.write(iso_file)
|
|
1586
|
+
iso.close()
|
|
1587
|
+
|
|
1588
|
+
@GeneralUtilities.check_arguments
|
|
1589
|
+
def SCCreateISOFileWithObfuscatedFiles(self, inputfolder: str, outputfile: str, printtableheadline, createisofile, extensions) -> None:
|
|
1590
|
+
if (os.path.isdir(inputfolder)):
|
|
1591
|
+
namemappingfile = "name_map.csv"
|
|
1592
|
+
files_directory = inputfolder
|
|
1593
|
+
files_directory_obf = f"{files_directory}_Obfuscated"
|
|
1594
|
+
self.SCObfuscateFilesFolder(
|
|
1595
|
+
inputfolder, printtableheadline, namemappingfile, extensions)
|
|
1596
|
+
os.rename(namemappingfile, os.path.join(
|
|
1597
|
+
files_directory_obf, namemappingfile))
|
|
1598
|
+
if createisofile:
|
|
1599
|
+
self.__create_iso(files_directory_obf, outputfile)
|
|
1600
|
+
shutil.rmtree(files_directory_obf)
|
|
1601
|
+
else:
|
|
1602
|
+
raise ValueError(f"Directory not found: '{inputfolder}'")
|
|
1603
|
+
|
|
1604
|
+
@GeneralUtilities.check_arguments
|
|
1605
|
+
def SCFilenameObfuscator(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
|
|
1606
|
+
obfuscate_all_files = extensions == "*"
|
|
1607
|
+
if (obfuscate_all_files):
|
|
1608
|
+
obfuscate_file_extensions = None
|
|
1609
|
+
else:
|
|
1610
|
+
obfuscate_file_extensions = extensions.split(",")
|
|
1611
|
+
if (os.path.isdir(inputfolder)):
|
|
1612
|
+
printtableheadline = GeneralUtilities.string_to_boolean(
|
|
1613
|
+
printtableheadline)
|
|
1614
|
+
files = []
|
|
1615
|
+
if not os.path.isfile(namemappingfile):
|
|
1616
|
+
with open(namemappingfile, "a", encoding="utf-8"):
|
|
1617
|
+
pass
|
|
1618
|
+
if printtableheadline:
|
|
1619
|
+
GeneralUtilities.append_line_to_file(
|
|
1620
|
+
namemappingfile, "Original filename;new filename;SHA2-hash of file")
|
|
1621
|
+
for file in GeneralUtilities.absolute_file_paths(inputfolder):
|
|
1622
|
+
if os.path.isfile(os.path.join(inputfolder, file)):
|
|
1623
|
+
if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
|
|
1624
|
+
files.append(file)
|
|
1625
|
+
for file in files:
|
|
1626
|
+
hash_value = GeneralUtilities.get_sha256_of_file(file)
|
|
1627
|
+
extension = Path(file).suffix
|
|
1628
|
+
new_file_name_without_path = str(uuid.uuid4())[0:8] + extension
|
|
1629
|
+
new_file_name = os.path.join(
|
|
1630
|
+
os.path.dirname(file), new_file_name_without_path)
|
|
1631
|
+
os.rename(file, new_file_name)
|
|
1632
|
+
GeneralUtilities.append_line_to_file(namemappingfile, os.path.basename(file) + ";" + new_file_name_without_path + ";" + hash_value)
|
|
1633
|
+
else:
|
|
1634
|
+
raise ValueError(f"Directory not found: '{inputfolder}'")
|
|
1635
|
+
|
|
1636
|
+
@GeneralUtilities.check_arguments
|
|
1637
|
+
def __extension_matchs(self, file: str, obfuscate_file_extensions) -> bool:
|
|
1638
|
+
for extension in obfuscate_file_extensions:
|
|
1639
|
+
if file.lower().endswith("."+extension.lower()):
|
|
1640
|
+
return True
|
|
1641
|
+
return False
|
|
1642
|
+
|
|
1643
|
+
@GeneralUtilities.check_arguments
|
|
1644
|
+
def SCHealthcheck(self, file: str) -> int:
|
|
1645
|
+
lines = GeneralUtilities.read_lines_from_file(file)
|
|
1646
|
+
for line in reversed(lines):
|
|
1647
|
+
if not GeneralUtilities.string_is_none_or_whitespace(line):
|
|
1648
|
+
if "RunningHealthy (" in line: # TODO use regex
|
|
1649
|
+
GeneralUtilities.write_message_to_stderr(f"Healthy running due to line '{line}' in file '{file}'.")
|
|
1650
|
+
return 0
|
|
1651
|
+
else:
|
|
1652
|
+
GeneralUtilities.write_message_to_stderr(f"Not healthy running due to line '{line}' in file '{file}'.")
|
|
1653
|
+
return 1
|
|
1654
|
+
GeneralUtilities.write_message_to_stderr(f"No valid line found for healthycheck in file '{file}'.")
|
|
1655
|
+
return 2
|
|
1656
|
+
|
|
1657
|
+
@GeneralUtilities.check_arguments
|
|
1658
|
+
def SCObfuscateFilesFolder(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
|
|
1659
|
+
obfuscate_all_files = extensions == "*"
|
|
1660
|
+
if (obfuscate_all_files):
|
|
1661
|
+
obfuscate_file_extensions = None
|
|
1662
|
+
else:
|
|
1663
|
+
if "," in extensions:
|
|
1664
|
+
obfuscate_file_extensions = extensions.split(",")
|
|
1665
|
+
else:
|
|
1666
|
+
obfuscate_file_extensions = [extensions]
|
|
1667
|
+
newd = inputfolder+"_Obfuscated"
|
|
1668
|
+
shutil.copytree(inputfolder, newd)
|
|
1669
|
+
inputfolder = newd
|
|
1670
|
+
if (os.path.isdir(inputfolder)):
|
|
1671
|
+
for file in GeneralUtilities.absolute_file_paths(inputfolder):
|
|
1672
|
+
if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
|
|
1673
|
+
self.SCChangeHashOfProgram(file)
|
|
1674
|
+
os.remove(file)
|
|
1675
|
+
os.rename(file + ".modified", file)
|
|
1676
|
+
self.SCFilenameObfuscator(inputfolder, printtableheadline, namemappingfile, extensions)
|
|
1677
|
+
else:
|
|
1678
|
+
raise ValueError(f"Directory not found: '{inputfolder}'")
|
|
1679
|
+
|
|
1680
|
+
@GeneralUtilities.check_arguments
|
|
1681
|
+
def get_services_from_yaml_file(self, yaml_file: str) -> list[str]:
|
|
1682
|
+
with open(yaml_file, encoding="utf-8") as stream:
|
|
1683
|
+
loaded = yaml.safe_load(stream)
|
|
1684
|
+
services = loaded["services"]
|
|
1685
|
+
result = list(services.keys())
|
|
1686
|
+
return result
|
|
1687
|
+
|
|
1688
|
+
@GeneralUtilities.check_arguments
|
|
1689
|
+
def kill_docker_container(self, container_name: str) -> None:
|
|
1690
|
+
self.run_program("docker", f"container rm -f {container_name}")
|
|
1691
|
+
|
|
1692
|
+
@GeneralUtilities.check_arguments
|
|
1693
|
+
def get_latest_apt_package_version_in_debian(self, image: str,package:str) -> str:
|
|
1694
|
+
#docker run --rm -it debian bash -c "apt update && apt list -a tor"
|
|
1695
|
+
output=self.run_with_epew("docker", f"run --rm -it {image} bash -c \"apt --color=false update && apt --color=false list -a tor\"",os.getcwd(),encode_argument_in_base64=True)
|
|
1696
|
+
stdout=output[1]
|
|
1697
|
+
version_lines=[line.strip() for line in GeneralUtilities.string_to_lines(stdout) if GeneralUtilities.string_has_nonwhitespace_content(line) and line.startswith(package+"/")]
|
|
1698
|
+
GeneralUtilities.assert_condition(0<len(version_lines), f"No version found for package '{package}' in image '{image}'.")
|
|
1699
|
+
versions = [version_line.split(" ")[1] for version_line in version_lines]
|
|
1700
|
+
def my_comparer(a, b) -> int:
|
|
1701
|
+
# return:
|
|
1702
|
+
# -1 → a < b
|
|
1703
|
+
# 0 → a = b
|
|
1704
|
+
# 1 → a > b
|
|
1705
|
+
# dpkg --compare-versions <a> lt <b> → exit 0 wenn a < b
|
|
1706
|
+
def dpkg_compare(op: str) -> bool:
|
|
1707
|
+
result = self.run_program_argsasarray("docker", [ "run", "--rm",image, "dpkg", "--compare-versions", a, op, b],throw_exception_if_exitcode_is_not_zero=False)
|
|
1708
|
+
GeneralUtilities.assert_condition(result[1]==GeneralUtilities.empty_string)
|
|
1709
|
+
GeneralUtilities.assert_condition(result[2]==GeneralUtilities.empty_string)
|
|
1710
|
+
return result[0] == 0
|
|
1711
|
+
|
|
1712
|
+
if dpkg_compare("lt"): # a < b
|
|
1713
|
+
return -1
|
|
1714
|
+
elif dpkg_compare("gt"): # a > b
|
|
1715
|
+
return 1
|
|
1716
|
+
else:
|
|
1717
|
+
return 0
|
|
1718
|
+
sorted_versions = sorted(versions, key=cmp_to_key(my_comparer))
|
|
1719
|
+
return sorted_versions[-1]
|
|
1720
|
+
|
|
1721
|
+
@GeneralUtilities.check_arguments
|
|
1722
|
+
def run_testcases_for_python_project(self, repository_folder: str):
|
|
1723
|
+
self.assert_is_git_repository(repository_folder)
|
|
1724
|
+
self.run_program("coverage", "run -m pytest", repository_folder)
|
|
1725
|
+
self.run_program("coverage", "xml", repository_folder)
|
|
1726
|
+
GeneralUtilities.ensure_directory_exists(os.path.join(repository_folder, "Other/TestCoverage"))
|
|
1727
|
+
coveragefile = os.path.join(repository_folder, "Other/TestCoverage/TestCoverage.xml")
|
|
1728
|
+
GeneralUtilities.ensure_file_does_not_exist(coveragefile)
|
|
1729
|
+
os.rename(os.path.join(repository_folder, "coverage.xml"), coveragefile)
|
|
1730
|
+
|
|
1731
|
+
@GeneralUtilities.check_arguments
|
|
1732
|
+
def get_file_permission(self, file: str) -> str:
|
|
1733
|
+
"""This function returns an usual octet-triple, for example "700"."""
|
|
1734
|
+
ls_output: str = self.run_ls_for_folder(file)
|
|
1735
|
+
return self.__get_file_permission_helper(ls_output)
|
|
1736
|
+
|
|
1737
|
+
@GeneralUtilities.check_arguments
|
|
1738
|
+
def __get_file_permission_helper(self, permissions: str) -> str:
|
|
1739
|
+
return str(self.__to_octet(permissions[0:3])) + str(self.__to_octet(permissions[3:6]))+str(self.__to_octet(permissions[6:9]))
|
|
1740
|
+
|
|
1741
|
+
@GeneralUtilities.check_arguments
|
|
1742
|
+
def __to_octet(self, string: str) -> int:
|
|
1743
|
+
return int(self.__to_octet_helper(string[0]) + self.__to_octet_helper(string[1])+self.__to_octet_helper(string[2]), 2)
|
|
1744
|
+
|
|
1745
|
+
@GeneralUtilities.check_arguments
|
|
1746
|
+
def __to_octet_helper(self, string: str) -> str:
|
|
1747
|
+
if (string == "-"):
|
|
1748
|
+
return "0"
|
|
1749
|
+
else:
|
|
1750
|
+
return "1"
|
|
1751
|
+
|
|
1752
|
+
@GeneralUtilities.check_arguments
|
|
1753
|
+
def get_file_owner(self, file: str) -> str:
|
|
1754
|
+
"""This function returns the user and the group in the format "user:group"."""
|
|
1755
|
+
ls_output: str = self.run_ls_for_folder(file)
|
|
1756
|
+
return self.__get_file_owner_helper(ls_output)
|
|
1757
|
+
|
|
1758
|
+
@GeneralUtilities.check_arguments
|
|
1759
|
+
def __get_file_owner_helper(self, ls_output: str) -> str:
|
|
1760
|
+
splitted = ls_output.split()
|
|
1761
|
+
return f"{splitted[2]}:{splitted[3]}"
|
|
1762
|
+
|
|
1763
|
+
@GeneralUtilities.check_arguments
|
|
1764
|
+
def get_file_owner_and_file_permission(self, file: str) -> str:
|
|
1765
|
+
ls_output: str = self.run_ls_for_folder(file)
|
|
1766
|
+
return [self.__get_file_owner_helper(ls_output), self.__get_file_permission_helper(ls_output)]
|
|
1767
|
+
|
|
1768
|
+
@GeneralUtilities.check_arguments
|
|
1769
|
+
def run_ls_for_folder(self, file_or_folder: str) -> str:
|
|
1770
|
+
file_or_folder = file_or_folder.replace("\\", "/")
|
|
1771
|
+
GeneralUtilities.assert_condition(os.path.isfile(file_or_folder) or os.path.isdir(file_or_folder), f"Can not execute 'ls -ld' because '{file_or_folder}' does not exist.")
|
|
1772
|
+
ls_result = self.run_program_argsasarray("ls", ["-ld", file_or_folder])
|
|
1773
|
+
GeneralUtilities.assert_condition(ls_result[0] == 0, f"'ls -ld {file_or_folder}' resulted in exitcode {str(ls_result[0])}. StdErr: {ls_result[2]}")
|
|
1774
|
+
GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(ls_result[1]), f"'ls -ld' of '{file_or_folder}' had an empty output. StdErr: '{ls_result[2]}'")
|
|
1775
|
+
output = ls_result[1]
|
|
1776
|
+
result = output.replace("\n", GeneralUtilities.empty_string)
|
|
1777
|
+
result = ' '.join(result.split()) # reduce multiple whitespaces to one
|
|
1778
|
+
return result
|
|
1779
|
+
|
|
1780
|
+
@GeneralUtilities.check_arguments
|
|
1781
|
+
def run_ls_for_folder_content(self, file_or_folder: str) -> list[str]:
|
|
1782
|
+
file_or_folder = file_or_folder.replace("\\", "/")
|
|
1783
|
+
GeneralUtilities.assert_condition(os.path.isfile(file_or_folder) or os.path.isdir(file_or_folder), f"Can not execute 'ls -la' because '{file_or_folder}' does not exist.")
|
|
1784
|
+
ls_result = self.run_program_argsasarray("ls", ["-la", file_or_folder])
|
|
1785
|
+
GeneralUtilities.assert_condition(ls_result[0] == 0, f"'ls -la {file_or_folder}' resulted in exitcode {str(ls_result[0])}. StdErr: {ls_result[2]}")
|
|
1786
|
+
GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(ls_result[1]), f"'ls -la' of '{file_or_folder}' had an empty output. StdErr: '{ls_result[2]}'")
|
|
1787
|
+
output = ls_result[1]
|
|
1788
|
+
result = output.split("\n")[3:] # skip the lines with "Total", "." and ".."
|
|
1789
|
+
result = [' '.join(line.split()) for line in result] # reduce multiple whitespaces to one
|
|
1790
|
+
return result
|
|
1791
|
+
|
|
1792
|
+
@GeneralUtilities.check_arguments
|
|
1793
|
+
def set_permission(self, file_or_folder: str, permissions: str, recursive: bool = False) -> None:
|
|
1794
|
+
"""This function expects an usual octet-triple, for example "700"."""
|
|
1795
|
+
args = []
|
|
1796
|
+
if recursive:
|
|
1797
|
+
args.append("--recursive")
|
|
1798
|
+
args.append(permissions)
|
|
1799
|
+
args.append(file_or_folder)
|
|
1800
|
+
self.run_program_argsasarray("chmod", args)
|
|
1801
|
+
|
|
1802
|
+
@GeneralUtilities.check_arguments
|
|
1803
|
+
def set_owner(self, file_or_folder: str, owner: str, recursive: bool = False, follow_symlinks: bool = False) -> None:
|
|
1804
|
+
"""This function expects the user and the group in the format "user:group"."""
|
|
1805
|
+
args = []
|
|
1806
|
+
if recursive:
|
|
1807
|
+
args.append("--recursive")
|
|
1808
|
+
if follow_symlinks:
|
|
1809
|
+
args.append("--no-dereference")
|
|
1810
|
+
args.append(owner)
|
|
1811
|
+
args.append(file_or_folder)
|
|
1812
|
+
self.run_program_argsasarray("chown", args)
|
|
1813
|
+
|
|
1814
|
+
# <run programs>
|
|
1815
|
+
|
|
1816
|
+
@GeneralUtilities.check_arguments
|
|
1817
|
+
def __run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> Popen:
|
|
1818
|
+
popen: Popen = self.program_runner.run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, custom_argument, interactive)
|
|
1819
|
+
return popen
|
|
1820
|
+
|
|
1821
|
+
@staticmethod
|
|
1822
|
+
def __enqueue_output(file: IO, queue: Queue):
|
|
1823
|
+
for line in iter(file.readline, ''):
|
|
1824
|
+
queue.put(line)
|
|
1825
|
+
file.close()
|
|
1826
|
+
|
|
1827
|
+
@staticmethod
|
|
1828
|
+
def __continue_process_reading(pid: int, p: Popen, q_stdout: Queue, q_stderr: Queue, reading_stdout_last_time_resulted_in_exception: bool, reading_stderr_last_time_resulted_in_exception: bool):
|
|
1829
|
+
if p.poll() is None:
|
|
1830
|
+
return True
|
|
1831
|
+
|
|
1832
|
+
# if reading_stdout_last_time_resulted_in_exception and reading_stderr_last_time_resulted_in_exception:
|
|
1833
|
+
# return False
|
|
1834
|
+
|
|
1835
|
+
if not q_stdout.empty():
|
|
1836
|
+
return True
|
|
1837
|
+
|
|
1838
|
+
if not q_stderr.empty():
|
|
1839
|
+
return True
|
|
1840
|
+
|
|
1841
|
+
return False
|
|
1842
|
+
|
|
1843
|
+
@staticmethod
|
|
1844
|
+
def __read_popen_pipes(p: Popen, print_live_output: bool, print_errors_as_information: bool, log: SCLog) -> tuple[list[str], list[str]]:
|
|
1845
|
+
p_id = p.pid
|
|
1846
|
+
with ThreadPoolExecutor(2) as pool:
|
|
1847
|
+
q_stdout = Queue()
|
|
1848
|
+
q_stderr = Queue()
|
|
1849
|
+
|
|
1850
|
+
pool.submit(ScriptCollectionCore.__enqueue_output, p.stdout, q_stdout)
|
|
1851
|
+
pool.submit(ScriptCollectionCore.__enqueue_output, p.stderr, q_stderr)
|
|
1852
|
+
reading_stdout_last_time_resulted_in_exception: bool = False
|
|
1853
|
+
reading_stderr_last_time_resulted_in_exception: bool = False
|
|
1854
|
+
|
|
1855
|
+
stdout_result: list[str] = []
|
|
1856
|
+
stderr_result: list[str] = []
|
|
1857
|
+
|
|
1858
|
+
while (ScriptCollectionCore.__continue_process_reading(p_id, p, q_stdout, q_stderr, reading_stdout_last_time_resulted_in_exception, reading_stderr_last_time_resulted_in_exception)):
|
|
1859
|
+
try:
|
|
1860
|
+
while not q_stdout.empty():
|
|
1861
|
+
out_line: str = q_stdout.get_nowait()
|
|
1862
|
+
out_line = out_line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
|
|
1863
|
+
if GeneralUtilities.string_has_content(out_line):
|
|
1864
|
+
stdout_result.append(out_line)
|
|
1865
|
+
reading_stdout_last_time_resulted_in_exception = False
|
|
1866
|
+
if print_live_output:
|
|
1867
|
+
loglevel = LogLevel.Information
|
|
1868
|
+
if out_line.startswith("Debug: "):
|
|
1869
|
+
loglevel = LogLevel.Debug
|
|
1870
|
+
out_line = out_line[len("Debug: "):]
|
|
1871
|
+
if out_line.startswith("Diagnostic: "):
|
|
1872
|
+
loglevel = LogLevel.Diagnostic
|
|
1873
|
+
out_line = out_line[len("Diagnostic: "):]
|
|
1874
|
+
log.log(out_line, loglevel)
|
|
1875
|
+
except Empty:
|
|
1876
|
+
reading_stdout_last_time_resulted_in_exception = True
|
|
1877
|
+
|
|
1878
|
+
try:
|
|
1879
|
+
while not q_stderr.empty():
|
|
1880
|
+
err_line: str = q_stderr.get_nowait()
|
|
1881
|
+
err_line = err_line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
|
|
1882
|
+
if GeneralUtilities.string_has_content(err_line):
|
|
1883
|
+
stderr_result.append(err_line)
|
|
1884
|
+
reading_stderr_last_time_resulted_in_exception = False
|
|
1885
|
+
if print_live_output:
|
|
1886
|
+
loglevel = LogLevel.Error
|
|
1887
|
+
if err_line.startswith("Warning: "):
|
|
1888
|
+
loglevel = LogLevel.Warning
|
|
1889
|
+
err_line = err_line[len("Warning: "):]
|
|
1890
|
+
if print_errors_as_information: # "errors" in "print_errors_as_information" means: all what is written to std-err
|
|
1891
|
+
loglevel = LogLevel.Information
|
|
1892
|
+
log.log(err_line, loglevel)
|
|
1893
|
+
except Empty:
|
|
1894
|
+
reading_stderr_last_time_resulted_in_exception = True
|
|
1895
|
+
|
|
1896
|
+
time.sleep(0.01) # this is required to not finish too early
|
|
1897
|
+
|
|
1898
|
+
return (stdout_result, stderr_result)
|
|
1899
|
+
|
|
1900
|
+
@GeneralUtilities.check_arguments
|
|
1901
|
+
def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False) -> tuple[int, str, str, int]:
|
|
1902
|
+
if self.call_program_runner_directly:
|
|
1903
|
+
return self.program_runner.run_program_argsasarray(program, arguments_as_array, working_directory, custom_argument, interactive)
|
|
1904
|
+
try:
|
|
1905
|
+
GeneralUtilities.assert_not_null(arguments_as_array,"arguments_as_array must not be null")
|
|
1906
|
+
arguments_as_str = ' '.join(arguments_as_array)
|
|
1907
|
+
mock_loader_result = self.__try_load_mock(program, arguments_as_str, working_directory)
|
|
1908
|
+
if mock_loader_result[0]:
|
|
1909
|
+
return mock_loader_result[1]
|
|
1910
|
+
|
|
1911
|
+
if self.program_runner.will_be_executed_locally():
|
|
1912
|
+
working_directory = self.__adapt_workingdirectory(working_directory)
|
|
1913
|
+
|
|
1914
|
+
if arguments_for_log is None or len(arguments_for_log)==0:
|
|
1915
|
+
arguments_for_log = arguments_as_array
|
|
1916
|
+
|
|
1917
|
+
cmd = f'{GeneralUtilities.str_none_safe(working_directory)}>{program}'
|
|
1918
|
+
if 0 < len(arguments_for_log):
|
|
1919
|
+
arguments_for_log_as_string: str = ' '.join([f'"{argument_for_log}"' for argument_for_log in arguments_for_log])
|
|
1920
|
+
cmd = f'{cmd} {arguments_for_log_as_string}'
|
|
1921
|
+
|
|
1922
|
+
if GeneralUtilities.string_is_none_or_whitespace(title):
|
|
1923
|
+
info_for_log = cmd
|
|
1924
|
+
else:
|
|
1925
|
+
info_for_log = title
|
|
1926
|
+
|
|
1927
|
+
self.log.log(f"Run '{info_for_log}'.", LogLevel.Debug)
|
|
1928
|
+
|
|
1929
|
+
exit_code: int = None
|
|
1930
|
+
stdout: str = GeneralUtilities.empty_string
|
|
1931
|
+
stderr: str = GeneralUtilities.empty_string
|
|
1932
|
+
pid: int = None
|
|
1933
|
+
|
|
1934
|
+
with self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive) as process:
|
|
1935
|
+
|
|
1936
|
+
if log_file is not None:
|
|
1937
|
+
GeneralUtilities.ensure_file_exists(log_file)
|
|
1938
|
+
pid = process.pid
|
|
1939
|
+
|
|
1940
|
+
outputs: tuple[list[str], list[str]] = ScriptCollectionCore.__read_popen_pipes(process, print_live_output, print_errors_as_information, self.log)
|
|
1941
|
+
|
|
1942
|
+
for out_line_plain in outputs[0]:
|
|
1943
|
+
if out_line_plain is not None:
|
|
1944
|
+
out_line: str = None
|
|
1945
|
+
if isinstance(out_line_plain, str):
|
|
1946
|
+
out_line = out_line_plain
|
|
1947
|
+
elif isinstance(out_line_plain, bytes):
|
|
1948
|
+
out_line = GeneralUtilities.bytes_to_string(out_line_plain)
|
|
1949
|
+
else:
|
|
1950
|
+
raise ValueError(f"Unknown type of output: {str(type(out_line_plain))}")
|
|
1951
|
+
|
|
1952
|
+
if out_line is not None and GeneralUtilities.string_has_content(out_line):
|
|
1953
|
+
if out_line.endswith("\n"):
|
|
1954
|
+
out_line = out_line[:-1]
|
|
1955
|
+
if 0 < len(stdout):
|
|
1956
|
+
stdout = stdout+"\n"
|
|
1957
|
+
stdout = stdout+out_line
|
|
1958
|
+
if log_file is not None:
|
|
1959
|
+
GeneralUtilities.append_line_to_file(log_file, out_line)
|
|
1960
|
+
|
|
1961
|
+
for err_line_plain in outputs[1]:
|
|
1962
|
+
if err_line_plain is not None:
|
|
1963
|
+
err_line: str = None
|
|
1964
|
+
if isinstance(err_line_plain, str):
|
|
1965
|
+
err_line = err_line_plain
|
|
1966
|
+
elif isinstance(err_line_plain, bytes):
|
|
1967
|
+
err_line = GeneralUtilities.bytes_to_string(err_line_plain)
|
|
1968
|
+
else:
|
|
1969
|
+
raise ValueError(f"Unknown type of output: {str(type(err_line_plain))}")
|
|
1970
|
+
if err_line is not None and GeneralUtilities.string_has_content(err_line):
|
|
1971
|
+
if err_line.endswith("\n"):
|
|
1972
|
+
err_line = err_line[:-1]
|
|
1973
|
+
if 0 < len(stderr):
|
|
1974
|
+
stderr = stderr+"\n"
|
|
1975
|
+
stderr = stderr+err_line
|
|
1976
|
+
if log_file is not None:
|
|
1977
|
+
GeneralUtilities.append_line_to_file(log_file, err_line)
|
|
1978
|
+
|
|
1979
|
+
exit_code = process.returncode
|
|
1980
|
+
GeneralUtilities.assert_condition(exit_code is not None, f"Exitcode of program-run of '{info_for_log}' is None.")
|
|
1981
|
+
|
|
1982
|
+
result_message = f"Program '{info_for_log}' resulted in exitcode {exit_code}."
|
|
1983
|
+
|
|
1984
|
+
self.log.log(result_message, LogLevel.Debug)
|
|
1985
|
+
|
|
1986
|
+
if throw_exception_if_exitcode_is_not_zero and exit_code != 0:
|
|
1987
|
+
raise ValueError(f"{result_message} (StdOut: '{stdout}', StdErr: '{stderr}')")
|
|
1988
|
+
|
|
1989
|
+
result = (exit_code, stdout, stderr, pid)
|
|
1990
|
+
return result
|
|
1991
|
+
except Exception as e:#pylint:disable=unused-variable, try-except-raise
|
|
1992
|
+
raise
|
|
1993
|
+
|
|
1994
|
+
# Return-values program_runner: Exitcode, StdOut, StdErr, Pid
|
|
1995
|
+
@GeneralUtilities.check_arguments
|
|
1996
|
+
def run_program_with_retry(self, program: str, arguments: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: str = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False, amount_of_attempts: int = 5, delay_in_seconds: int = 2) -> tuple[int, str, str, int]:
|
|
1997
|
+
return GeneralUtilities.retry_action(lambda: self.run_program(program, arguments, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace,arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output), amount_of_attempts, delay_in_seconds=delay_in_seconds)
|
|
1998
|
+
|
|
1999
|
+
# Return-values program_runner: Exitcode, StdOut, StdErr, Pid
|
|
2000
|
+
@GeneralUtilities.check_arguments
|
|
2001
|
+
def run_program(self, program: str, arguments: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: str = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False) -> tuple[int, str, str, int]:
|
|
2002
|
+
if self.call_program_runner_directly:
|
|
2003
|
+
return self.program_runner.run_program(program, arguments, working_directory, custom_argument, interactive)
|
|
2004
|
+
return self.run_program_argsasarray(program, GeneralUtilities.arguments_to_array(arguments), working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, GeneralUtilities.arguments_to_array(arguments_for_log), throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output)
|
|
2005
|
+
|
|
2006
|
+
# Return-values program_runner: Exitcode, StdOut, StdErr, Pid
|
|
2007
|
+
@GeneralUtilities.check_arguments
|
|
2008
|
+
def run_program_argsasarray_with_retry(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False, amount_of_attempts: int = 5, delay_in_seconds: int = 2) -> tuple[int, str, str, int]:
|
|
2009
|
+
return GeneralUtilities.retry_action(lambda: self.run_program_argsasarray(program, arguments_as_array, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace,arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output), amount_of_attempts, delay_in_seconds=delay_in_seconds)
|
|
2010
|
+
|
|
2011
|
+
|
|
2012
|
+
# Return-values program_runner: Pid
|
|
2013
|
+
@GeneralUtilities.check_arguments
|
|
2014
|
+
def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
|
|
2015
|
+
if self.call_program_runner_directly:
|
|
2016
|
+
return self.program_runner.run_program_argsasarray_async(program, arguments_as_array, working_directory, custom_argument, interactive)
|
|
2017
|
+
mock_loader_result = self.__try_load_mock(program, ' '.join(arguments_as_array), working_directory)
|
|
2018
|
+
if mock_loader_result[0]:
|
|
2019
|
+
return mock_loader_result[1]
|
|
2020
|
+
process: Popen = self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
|
|
2021
|
+
return process.pid
|
|
2022
|
+
|
|
2023
|
+
# Return-values program_runner: Pid
|
|
2024
|
+
@GeneralUtilities.check_arguments
|
|
2025
|
+
def run_program_async(self, program: str, arguments: str = "", working_directory: str = None,print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
|
|
2026
|
+
if self.call_program_runner_directly:
|
|
2027
|
+
return self.program_runner.run_program_argsasarray_async(program, arguments, working_directory, custom_argument, interactive)
|
|
2028
|
+
return self.run_program_argsasarray_async(program, GeneralUtilities.arguments_to_array(arguments), working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
|
|
2029
|
+
|
|
2030
|
+
@GeneralUtilities.check_arguments
|
|
2031
|
+
def __try_load_mock(self, program: str, arguments: str, working_directory: str) -> tuple[bool, tuple[int, str, str, int]]:
|
|
2032
|
+
if self.mock_program_calls:
|
|
2033
|
+
try:
|
|
2034
|
+
return [True, self.__get_mock_program_call(program, arguments, working_directory)]
|
|
2035
|
+
except LookupError:
|
|
2036
|
+
if not self.execute_program_really_if_no_mock_call_is_defined:
|
|
2037
|
+
raise
|
|
2038
|
+
return [False, None]
|
|
2039
|
+
|
|
2040
|
+
@GeneralUtilities.check_arguments
|
|
2041
|
+
def __adapt_workingdirectory(self, workingdirectory: str) -> str:
|
|
2042
|
+
result: str = None
|
|
2043
|
+
if workingdirectory is None:
|
|
2044
|
+
result = os.getcwd()
|
|
2045
|
+
else:
|
|
2046
|
+
if os.path.isabs(workingdirectory):
|
|
2047
|
+
result = workingdirectory
|
|
2048
|
+
else:
|
|
2049
|
+
result = GeneralUtilities.resolve_relative_path_from_current_working_directory(workingdirectory)
|
|
2050
|
+
if not os.path.isdir(result):
|
|
2051
|
+
raise ValueError(f"Working-directory '{workingdirectory}' does not exist.")
|
|
2052
|
+
return result
|
|
2053
|
+
|
|
2054
|
+
@GeneralUtilities.check_arguments
|
|
2055
|
+
def verify_no_pending_mock_program_calls(self):
|
|
2056
|
+
if (len(self.__mocked_program_calls) > 0):
|
|
2057
|
+
raise AssertionError("The following mock-calls were not called:\n"+",\n ".join([self.__format_mock_program_call(r) for r in self.__mocked_program_calls]))
|
|
2058
|
+
|
|
2059
|
+
@GeneralUtilities.check_arguments
|
|
2060
|
+
def __format_mock_program_call(self, r) -> str:
|
|
2061
|
+
r: ScriptCollectionCore.__MockProgramCall = r
|
|
2062
|
+
return f"'{r.workingdirectory}>{r.program} {r.argument}' (" \
|
|
2063
|
+
f"exitcode: {GeneralUtilities.str_none_safe(str(r.exit_code))}, " \
|
|
2064
|
+
f"pid: {GeneralUtilities.str_none_safe(str(r.pid))}, "\
|
|
2065
|
+
f"stdout: {GeneralUtilities.str_none_safe(str(r.stdout))}, " \
|
|
2066
|
+
f"stderr: {GeneralUtilities.str_none_safe(str(r.stderr))})"
|
|
2067
|
+
|
|
2068
|
+
@GeneralUtilities.check_arguments
|
|
2069
|
+
def register_mock_program_call(self, program: str, argument: str, workingdirectory: str, result_exit_code: int, result_stdout: str, result_stderr: str, result_pid: int, amount_of_expected_calls=1):
|
|
2070
|
+
"This function is for test-purposes only"
|
|
2071
|
+
for _ in itertools.repeat(None, amount_of_expected_calls):
|
|
2072
|
+
mock_call = ScriptCollectionCore.__MockProgramCall()
|
|
2073
|
+
mock_call.program = program
|
|
2074
|
+
mock_call.argument = argument
|
|
2075
|
+
mock_call.workingdirectory = workingdirectory
|
|
2076
|
+
mock_call.exit_code = result_exit_code
|
|
2077
|
+
mock_call.stdout = result_stdout
|
|
2078
|
+
mock_call.stderr = result_stderr
|
|
2079
|
+
mock_call.pid = result_pid
|
|
2080
|
+
self.__mocked_program_calls.append(mock_call)
|
|
2081
|
+
|
|
2082
|
+
@GeneralUtilities.check_arguments
|
|
2083
|
+
def __get_mock_program_call(self, program: str, argument: str, workingdirectory: str):
|
|
2084
|
+
result: ScriptCollectionCore.__MockProgramCall = None
|
|
2085
|
+
for mock_call in self.__mocked_program_calls:
|
|
2086
|
+
if ((re.match(mock_call.program, program) is not None)
|
|
2087
|
+
and (re.match(mock_call.argument, argument) is not None)
|
|
2088
|
+
and (re.match(mock_call.workingdirectory, workingdirectory) is not None)):
|
|
2089
|
+
result = mock_call
|
|
2090
|
+
break
|
|
2091
|
+
if result is None:
|
|
2092
|
+
raise LookupError(f"Tried to execute mock-call '{workingdirectory}>{program} {argument}' but no mock-call was defined for that execution")
|
|
2093
|
+
else:
|
|
2094
|
+
self.__mocked_program_calls.remove(result)
|
|
2095
|
+
return (result.exit_code, result.stdout, result.stderr, result.pid)
|
|
2096
|
+
|
|
2097
|
+
@GeneralUtilities.check_arguments
|
|
2098
|
+
class __MockProgramCall:
|
|
2099
|
+
program: str
|
|
2100
|
+
argument: str
|
|
2101
|
+
workingdirectory: str
|
|
2102
|
+
exit_code: int
|
|
2103
|
+
stdout: str
|
|
2104
|
+
stderr: str
|
|
2105
|
+
pid: int
|
|
2106
|
+
|
|
2107
|
+
@GeneralUtilities.check_arguments
|
|
2108
|
+
def run_with_epew_with_retry(self, program: str, argument: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: str =None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False,print_live_output:bool=False,encode_argument_in_base64:bool=False, amount_of_attempts: int = 3, delay_in_seconds: int = 2) -> tuple[int, str, str, int]:
|
|
2109
|
+
return GeneralUtilities.retry_action(lambda: self.run_with_epew(program, argument, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace,arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output,encode_argument_in_base64), amount_of_attempts, delay_in_seconds=delay_in_seconds)
|
|
2110
|
+
|
|
2111
|
+
@GeneralUtilities.check_arguments
|
|
2112
|
+
def run_with_epew(self, program: str, argument: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: str =None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False,print_live_output:bool=False,encode_argument_in_base64:bool=False) -> tuple[int, str, str, int]:
|
|
2113
|
+
epew_argument:list[str]=["-p",program ,"-w", working_directory]
|
|
2114
|
+
if encode_argument_in_base64:
|
|
2115
|
+
if arguments_for_log is None:
|
|
2116
|
+
argument_escaped=argument.replace("\"", "\\\"")
|
|
2117
|
+
arguments_for_log=epew_argument+["-a",f"\"{argument_escaped}\""]
|
|
2118
|
+
base64_string = base64.b64encode(argument.encode("utf-8")).decode("utf-8")
|
|
2119
|
+
epew_argument=epew_argument+["-a",base64_string,"-b"]
|
|
2120
|
+
else:
|
|
2121
|
+
epew_argument=epew_argument+["-a",argument]
|
|
2122
|
+
if arguments_for_log is None:
|
|
2123
|
+
arguments_for_log=epew_argument
|
|
2124
|
+
return self.run_program_argsasarray("epew", epew_argument, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive,print_live_output=print_live_output)
|
|
2125
|
+
|
|
2126
|
+
|
|
2127
|
+
# </run programs>
|
|
2128
|
+
|
|
2129
|
+
@GeneralUtilities.check_arguments
|
|
2130
|
+
def extract_archive_with_7z(self, unzip_program_file: str, zip_file: str, password: str, output_directory: str) -> None:
|
|
2131
|
+
password_set = not password is None
|
|
2132
|
+
file_name = Path(zip_file).name
|
|
2133
|
+
file_folder = os.path.dirname(zip_file)
|
|
2134
|
+
argument = "x"
|
|
2135
|
+
if password_set:
|
|
2136
|
+
argument = f"{argument} -p\"{password}\""
|
|
2137
|
+
argument = f"{argument} -o {output_directory}"
|
|
2138
|
+
argument = f"{argument} {file_name}"
|
|
2139
|
+
return self.run_program(unzip_program_file, argument, file_folder)
|
|
2140
|
+
|
|
2141
|
+
@GeneralUtilities.check_arguments
|
|
2142
|
+
def get_internet_time(self) -> datetime:
|
|
2143
|
+
response = ntplib.NTPClient().request('pool.ntp.org')
|
|
2144
|
+
return datetime.fromtimestamp(response.tx_time)
|
|
2145
|
+
|
|
2146
|
+
@GeneralUtilities.check_arguments
|
|
2147
|
+
def system_time_equals_internet_time(self, maximal_tolerance_difference: timedelta) -> bool:
|
|
2148
|
+
return abs(GeneralUtilities.get_now() - self.get_internet_time()) < maximal_tolerance_difference
|
|
2149
|
+
|
|
2150
|
+
@GeneralUtilities.check_arguments
|
|
2151
|
+
def system_time_equals_internet_time_with_default_tolerance(self) -> bool:
|
|
2152
|
+
return self.system_time_equals_internet_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
|
|
2153
|
+
|
|
2154
|
+
@GeneralUtilities.check_arguments
|
|
2155
|
+
def check_system_time(self, maximal_tolerance_difference: timedelta):
|
|
2156
|
+
if not self.system_time_equals_internet_time(maximal_tolerance_difference):
|
|
2157
|
+
raise ValueError("System time may be wrong")
|
|
2158
|
+
|
|
2159
|
+
@GeneralUtilities.check_arguments
|
|
2160
|
+
def check_system_time_with_default_tolerance(self) -> None:
|
|
2161
|
+
self.check_system_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
|
|
2162
|
+
|
|
2163
|
+
@GeneralUtilities.check_arguments
|
|
2164
|
+
def __get_default_tolerance_for_system_time_equals_internet_time(self) -> timedelta:
|
|
2165
|
+
return timedelta(hours=0, minutes=0, seconds=3)
|
|
2166
|
+
|
|
2167
|
+
@GeneralUtilities.check_arguments
|
|
2168
|
+
def increment_version(self, input_version: str, increment_major: bool, increment_minor: bool, increment_patch: bool) -> str:
|
|
2169
|
+
splitted = input_version.split(".")
|
|
2170
|
+
GeneralUtilities.assert_condition(len(splitted) == 3, f"Version '{input_version}' does not have the 'major.minor.patch'-pattern.")
|
|
2171
|
+
major = int(splitted[0])
|
|
2172
|
+
minor = int(splitted[1])
|
|
2173
|
+
patch = int(splitted[2])
|
|
2174
|
+
if increment_major:
|
|
2175
|
+
major = major+1
|
|
2176
|
+
if increment_minor:
|
|
2177
|
+
minor = minor+1
|
|
2178
|
+
if increment_patch:
|
|
2179
|
+
patch = patch+1
|
|
2180
|
+
return f"{major}.{minor}.{patch}"
|
|
2181
|
+
|
|
2182
|
+
@GeneralUtilities.check_arguments
|
|
2183
|
+
def get_semver_version_from_gitversion(self, repository_folder: str) -> str:
|
|
2184
|
+
self.assert_is_git_repository(repository_folder)
|
|
2185
|
+
if (self.git_repository_has_commits(repository_folder)):
|
|
2186
|
+
result = self.get_version_from_gitversion(repository_folder, "MajorMinorPatch")
|
|
2187
|
+
if self.git_repository_has_uncommitted_changes(repository_folder):
|
|
2188
|
+
if self.get_current_git_branch_has_tag(repository_folder):
|
|
2189
|
+
id_of_latest_tag = self.git_get_commit_id(repository_folder, self.get_latest_git_tag(repository_folder))
|
|
2190
|
+
current_commit = self.git_get_commit_id(repository_folder)
|
|
2191
|
+
current_commit_is_on_latest_tag = id_of_latest_tag == current_commit
|
|
2192
|
+
if current_commit_is_on_latest_tag:
|
|
2193
|
+
result = self.increment_version(result, False, False, True)
|
|
2194
|
+
else:
|
|
2195
|
+
result = "0.1.0"
|
|
2196
|
+
return result
|
|
2197
|
+
|
|
2198
|
+
@staticmethod
|
|
2199
|
+
@GeneralUtilities.check_arguments
|
|
2200
|
+
def is_patch_version(version_string: str) -> bool:
|
|
2201
|
+
return not version_string.endswith(".0")
|
|
2202
|
+
|
|
2203
|
+
@GeneralUtilities.check_arguments
|
|
2204
|
+
def get_version_from_gitversion(self, folder: str, variable: str) -> str:
|
|
2205
|
+
# called twice as workaround for issue 1877 in gitversion ( https://github.com/GitTools/GitVersion/issues/1877 )
|
|
2206
|
+
result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder)
|
|
2207
|
+
result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder)
|
|
2208
|
+
result = GeneralUtilities.strip_new_line_character(result[1])
|
|
2209
|
+
|
|
2210
|
+
return result
|
|
2211
|
+
|
|
2212
|
+
@GeneralUtilities.check_arguments
|
|
2213
|
+
def generate_certificate_authority(self, folder: str, name: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
|
|
2214
|
+
if days_until_expire is None:
|
|
2215
|
+
days_until_expire = 1825
|
|
2216
|
+
if password is None:
|
|
2217
|
+
password = GeneralUtilities.generate_password()
|
|
2218
|
+
GeneralUtilities.ensure_directory_exists(folder)
|
|
2219
|
+
self.run_program_argsasarray("openssl", ['req', '-new', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:prime256v1', '-days', str(days_until_expire), '-nodes', '-x509', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={name}/OU={subj_ou}', '-passout', f'pass:{password}', '-keyout', f'{name}.key', '-out', f'{name}.crt'], folder)
|
|
2220
|
+
|
|
2221
|
+
@GeneralUtilities.check_arguments
|
|
2222
|
+
def generate_certificate(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
|
|
2223
|
+
if days_until_expire is None:
|
|
2224
|
+
days_until_expire = 397
|
|
2225
|
+
if password is None:
|
|
2226
|
+
password = GeneralUtilities.generate_password()
|
|
2227
|
+
rsa_key_length = 4096
|
|
2228
|
+
self.run_program_argsasarray("openssl", ['genrsa', '-out', f'{filename}.key', f'{rsa_key_length}'], folder)
|
|
2229
|
+
self.run_program_argsasarray("openssl", ['req', '-new', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou}', '-x509', '-key', f'{filename}.key', '-out', f'{filename}.unsigned.crt', '-days', f'{days_until_expire}'], folder)
|
|
2230
|
+
self.run_program_argsasarray("openssl", ['pkcs12', '-export', '-out', f'{filename}.selfsigned.pfx', '-password', f'pass:{password}', '-inkey', f'{filename}.key', '-in', f'{filename}.unsigned.crt'], folder)
|
|
2231
|
+
GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.password"), password)
|
|
2232
|
+
GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.san.conf"), f"""[ req ]
|
|
2233
|
+
default_bits = {rsa_key_length}
|
|
2234
|
+
distinguished_name = req_distinguished_name
|
|
2235
|
+
req_extensions = v3_req
|
|
2236
|
+
default_md = sha256
|
|
2237
|
+
dirstring_type = nombstr
|
|
2238
|
+
prompt = no
|
|
2239
|
+
|
|
2240
|
+
[ req_distinguished_name ]
|
|
2241
|
+
countryName = {subj_c}
|
|
2242
|
+
stateOrProvinceName = {subj_st}
|
|
2243
|
+
localityName = {subj_l}
|
|
2244
|
+
organizationName = {subj_o}
|
|
2245
|
+
organizationUnit = {subj_ou}
|
|
2246
|
+
commonName = {domain}
|
|
2247
|
+
|
|
2248
|
+
[v3_req]
|
|
2249
|
+
subjectAltName = @subject_alt_name
|
|
2250
|
+
|
|
2251
|
+
[ subject_alt_name ]
|
|
2252
|
+
DNS = {domain}
|
|
2253
|
+
""")
|
|
2254
|
+
|
|
2255
|
+
@GeneralUtilities.check_arguments
|
|
2256
|
+
def generate_certificate_sign_request(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str) -> None:
|
|
2257
|
+
self.run_program_argsasarray("openssl", ['req', '-new', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou}', '-key', f'{filename}.key', f'-out', f'{filename}.csr', f'-config', f'{filename}.san.conf'], folder)
|
|
2258
|
+
|
|
2259
|
+
@GeneralUtilities.check_arguments
|
|
2260
|
+
def sign_certificate(self, folder: str, ca_folder: str, ca_name: str, domain: str, filename: str, days_until_expire: int = None) -> None:
|
|
2261
|
+
if days_until_expire is None:
|
|
2262
|
+
days_until_expire = 397
|
|
2263
|
+
ca = os.path.join(ca_folder, ca_name)
|
|
2264
|
+
password_file = os.path.join(folder, f"{filename}.password")
|
|
2265
|
+
password = GeneralUtilities.read_text_from_file(password_file)
|
|
2266
|
+
self.run_program_argsasarray("openssl", ['x509', '-req', '-in', f'{filename}.csr', '-CA', f'{ca}.crt', '-CAkey', f'{ca}.key', '-CAcreateserial', '-CAserial', f'{ca}.srl', '-out', f'{filename}.crt', '-days', str(days_until_expire), '-sha256', '-extensions', 'v3_req', '-extfile', f'{filename}.san.conf'], folder)
|
|
2267
|
+
self.run_program_argsasarray("openssl", ['pkcs12', '-export', '-out', f'{filename}.pfx', f'-inkey', f'{filename}.key', '-in', f'{filename}.crt', '-password', f'pass:{password}'], folder)
|
|
2268
|
+
|
|
2269
|
+
@GeneralUtilities.check_arguments
|
|
2270
|
+
def update_dependencies_of_python_in_requirementstxt_file(self, file: str, ignored_dependencies: list[str]):
|
|
2271
|
+
# TODO consider ignored_dependencies
|
|
2272
|
+
lines = GeneralUtilities.read_lines_from_file(file)
|
|
2273
|
+
new_lines = []
|
|
2274
|
+
for line in lines:
|
|
2275
|
+
if GeneralUtilities.string_has_content(line):
|
|
2276
|
+
new_lines.append(self.__get_updated_line_for_python_requirements(line.strip()))
|
|
2277
|
+
GeneralUtilities.write_lines_to_file(file, new_lines)
|
|
2278
|
+
|
|
2279
|
+
@GeneralUtilities.check_arguments
|
|
2280
|
+
def __get_updated_line_for_python_requirements(self, line: str) -> str:
|
|
2281
|
+
if "==" in line or "<" in line:
|
|
2282
|
+
return line
|
|
2283
|
+
elif ">" in line:
|
|
2284
|
+
try:
|
|
2285
|
+
# line is something like "cyclonedx-bom>=2.0.2" and the function must return with the updated version
|
|
2286
|
+
# (something like "cyclonedx-bom>=2.11.0" for example)
|
|
2287
|
+
package = line.split(">")[0]
|
|
2288
|
+
operator = ">=" if ">=" in line else ">"
|
|
2289
|
+
headers = {'Cache-Control': 'no-cache'}
|
|
2290
|
+
response = requests.get(f'https://pypi.org/pypi/{package}/json', timeout=5, headers=headers)
|
|
2291
|
+
latest_version = response.json()['info']['version']
|
|
2292
|
+
# TODO update only minor- and patch-version
|
|
2293
|
+
# TODO print info if there is a new major-version
|
|
2294
|
+
return package+operator+latest_version
|
|
2295
|
+
except:
|
|
2296
|
+
return line
|
|
2297
|
+
else:
|
|
2298
|
+
raise ValueError(f'Unexpected line in requirements-file: "{line}"')
|
|
2299
|
+
|
|
2300
|
+
@GeneralUtilities.check_arguments
|
|
2301
|
+
def update_dependencies_of_python_in_setupcfg_file(self, setup_cfg_file: str, ignored_dependencies: list[str]):
|
|
2302
|
+
# TODO consider ignored_dependencies
|
|
2303
|
+
lines = GeneralUtilities.read_lines_from_file(setup_cfg_file)
|
|
2304
|
+
new_lines = []
|
|
2305
|
+
requirement_parsing_mode = False
|
|
2306
|
+
for line in lines:
|
|
2307
|
+
new_line = line
|
|
2308
|
+
if (requirement_parsing_mode):
|
|
2309
|
+
if ("<" in line or "=" in line or ">" in line):
|
|
2310
|
+
updated_line = f" {self.__get_updated_line_for_python_requirements(line.strip())}"
|
|
2311
|
+
new_line = updated_line
|
|
2312
|
+
else:
|
|
2313
|
+
requirement_parsing_mode = False
|
|
2314
|
+
else:
|
|
2315
|
+
if line.startswith("install_requires ="):
|
|
2316
|
+
requirement_parsing_mode = True
|
|
2317
|
+
new_lines.append(new_line)
|
|
2318
|
+
GeneralUtilities.write_lines_to_file(setup_cfg_file, new_lines)
|
|
2319
|
+
|
|
2320
|
+
@GeneralUtilities.check_arguments
|
|
2321
|
+
def update_dependencies_of_dotnet_project(self, csproj_file: str, ignored_dependencies: list[str]):
|
|
2322
|
+
folder = os.path.dirname(csproj_file)
|
|
2323
|
+
csproj_filename = os.path.basename(csproj_file)
|
|
2324
|
+
self.log.log(f"Check for updates in {csproj_filename}", LogLevel.Information)
|
|
2325
|
+
result = self.run_program_with_retry("dotnet", f"list {csproj_filename} package --outdated", folder, print_errors_as_information=True)
|
|
2326
|
+
for line in result[1].replace("\r", GeneralUtilities.empty_string).split("\n"):
|
|
2327
|
+
# Relevant output-lines are something like " > NJsonSchema 10.7.0 10.7.0 10.9.0"
|
|
2328
|
+
if ">" in line:
|
|
2329
|
+
package_name = line.replace(">", GeneralUtilities.empty_string).strip().split(" ")[0]
|
|
2330
|
+
if not (package_name in ignored_dependencies):
|
|
2331
|
+
self.log.log(f"Update package {package_name}...", LogLevel.Debug)
|
|
2332
|
+
time.sleep(1.1) # attempt to prevent rate-limit
|
|
2333
|
+
self.run_program_with_retry("dotnet", f"add {csproj_filename} package {package_name}", folder, print_errors_as_information=True)
|
|
2334
|
+
|
|
2335
|
+
@GeneralUtilities.check_arguments
|
|
2336
|
+
def create_deb_package(self, toolname: str, binary_folder: str, control_file_content: str, deb_output_folder: str, permission_of_executable_file_as_octet_triple: int) -> None:
|
|
2337
|
+
|
|
2338
|
+
# prepare
|
|
2339
|
+
GeneralUtilities.ensure_directory_exists(deb_output_folder)
|
|
2340
|
+
temp_folder = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
|
|
2341
|
+
GeneralUtilities.ensure_directory_exists(temp_folder)
|
|
2342
|
+
bin_folder = binary_folder
|
|
2343
|
+
tool_content_folder_name = toolname+"Content"
|
|
2344
|
+
|
|
2345
|
+
# create folder
|
|
2346
|
+
GeneralUtilities.ensure_directory_exists(temp_folder)
|
|
2347
|
+
control_content_folder_name = "controlcontent"
|
|
2348
|
+
packagecontent_control_folder = os.path.join(temp_folder, control_content_folder_name)
|
|
2349
|
+
GeneralUtilities.ensure_directory_exists(packagecontent_control_folder)
|
|
2350
|
+
data_content_folder_name = "datacontent"
|
|
2351
|
+
packagecontent_data_folder = os.path.join(temp_folder, data_content_folder_name)
|
|
2352
|
+
GeneralUtilities.ensure_directory_exists(packagecontent_data_folder)
|
|
2353
|
+
entireresult_content_folder_name = "entireresultcontent"
|
|
2354
|
+
packagecontent_entireresult_folder = os.path.join(temp_folder, entireresult_content_folder_name)
|
|
2355
|
+
GeneralUtilities.ensure_directory_exists(packagecontent_entireresult_folder)
|
|
2356
|
+
|
|
2357
|
+
# create "debian-binary"-file
|
|
2358
|
+
debianbinary_file = os.path.join(packagecontent_entireresult_folder, "debian-binary")
|
|
2359
|
+
GeneralUtilities.ensure_file_exists(debianbinary_file)
|
|
2360
|
+
GeneralUtilities.write_text_to_file(debianbinary_file, "2.0\n")
|
|
2361
|
+
|
|
2362
|
+
# create control-content
|
|
2363
|
+
|
|
2364
|
+
# conffiles
|
|
2365
|
+
conffiles_file = os.path.join(packagecontent_control_folder, "conffiles")
|
|
2366
|
+
GeneralUtilities.ensure_file_exists(conffiles_file)
|
|
2367
|
+
|
|
2368
|
+
# postinst-script
|
|
2369
|
+
postinst_file = os.path.join(packagecontent_control_folder, "postinst")
|
|
2370
|
+
GeneralUtilities.ensure_file_exists(postinst_file)
|
|
2371
|
+
exe_file = f"/usr/bin/{tool_content_folder_name}/{toolname}"
|
|
2372
|
+
link_file = f"/usr/bin/{toolname.lower()}"
|
|
2373
|
+
permission = str(permission_of_executable_file_as_octet_triple)
|
|
2374
|
+
GeneralUtilities.write_text_to_file(postinst_file, f"""#!/bin/sh
|
|
2375
|
+
ln -s {exe_file} {link_file}
|
|
2376
|
+
chmod {permission} {exe_file}
|
|
2377
|
+
chmod {permission} {link_file}
|
|
2378
|
+
""")
|
|
2379
|
+
|
|
2380
|
+
# control
|
|
2381
|
+
control_file = os.path.join(packagecontent_control_folder, "control")
|
|
2382
|
+
GeneralUtilities.ensure_file_exists(control_file)
|
|
2383
|
+
GeneralUtilities.write_text_to_file(control_file, control_file_content)
|
|
2384
|
+
|
|
2385
|
+
# md5sums
|
|
2386
|
+
md5sums_file = os.path.join(packagecontent_control_folder, "md5sums")
|
|
2387
|
+
GeneralUtilities.ensure_file_exists(md5sums_file)
|
|
2388
|
+
|
|
2389
|
+
# create data-content
|
|
2390
|
+
|
|
2391
|
+
# copy binaries
|
|
2392
|
+
usr_bin_folder = os.path.join(packagecontent_data_folder, "usr/bin")
|
|
2393
|
+
GeneralUtilities.ensure_directory_exists(usr_bin_folder)
|
|
2394
|
+
usr_bin_content_folder = os.path.join(usr_bin_folder, tool_content_folder_name)
|
|
2395
|
+
GeneralUtilities.copy_content_of_folder(bin_folder, usr_bin_content_folder)
|
|
2396
|
+
|
|
2397
|
+
# create debfile
|
|
2398
|
+
deb_filename = f"{toolname}.deb"
|
|
2399
|
+
self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/control.tar.gz", "*"], packagecontent_control_folder)
|
|
2400
|
+
self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/data.tar.gz", "*"], packagecontent_data_folder)
|
|
2401
|
+
self.run_program_argsasarray("ar", ["r", deb_filename, "debian-binary", "control.tar.gz", "data.tar.gz"], packagecontent_entireresult_folder)
|
|
2402
|
+
result_file = os.path.join(packagecontent_entireresult_folder, deb_filename)
|
|
2403
|
+
shutil.copy(result_file, os.path.join(deb_output_folder, deb_filename))
|
|
2404
|
+
|
|
2405
|
+
# cleanup
|
|
2406
|
+
GeneralUtilities.ensure_directory_does_not_exist(temp_folder)
|
|
2407
|
+
|
|
2408
|
+
@GeneralUtilities.check_arguments
|
|
2409
|
+
def update_year_in_copyright_tags(self, file: str) -> None:
|
|
2410
|
+
current_year = str(GeneralUtilities.get_now().year)
|
|
2411
|
+
lines = GeneralUtilities.read_lines_from_file(file)
|
|
2412
|
+
lines_result = []
|
|
2413
|
+
for line in lines:
|
|
2414
|
+
if match := re.search("(.*<[Cc]opyright>.*)\\d\\d\\d\\d(.*<\\/[Cc]opyright>.*)", line):
|
|
2415
|
+
part1 = match.group(1)
|
|
2416
|
+
part2 = match.group(2)
|
|
2417
|
+
adapted = part1+current_year+part2
|
|
2418
|
+
else:
|
|
2419
|
+
adapted = line
|
|
2420
|
+
lines_result.append(adapted)
|
|
2421
|
+
GeneralUtilities.write_lines_to_file(file, lines_result)
|
|
2422
|
+
|
|
2423
|
+
@GeneralUtilities.check_arguments
|
|
2424
|
+
def update_year_in_first_line_of_file(self, file: str) -> None:
|
|
2425
|
+
current_year = str(GeneralUtilities.get_now().year)
|
|
2426
|
+
lines = GeneralUtilities.read_lines_from_file(file)
|
|
2427
|
+
lines[0] = re.sub("\\d\\d\\d\\d", current_year, lines[0])
|
|
2428
|
+
GeneralUtilities.write_lines_to_file(file, lines)
|
|
2429
|
+
|
|
2430
|
+
@GeneralUtilities.check_arguments
|
|
2431
|
+
def get_external_ip_address(self) -> str:
|
|
2432
|
+
information = self.get_externalnetworkinformation_as_json_string()
|
|
2433
|
+
parsed = json.loads(information)
|
|
2434
|
+
return parsed["IPAddress"]
|
|
2435
|
+
|
|
2436
|
+
@GeneralUtilities.check_arguments
|
|
2437
|
+
def get_country_of_external_ip_address(self) -> str:
|
|
2438
|
+
information = self.get_externalnetworkinformation_as_json_string()
|
|
2439
|
+
parsed = json.loads(information)
|
|
2440
|
+
return parsed["Country"]
|
|
2441
|
+
|
|
2442
|
+
@GeneralUtilities.check_arguments
|
|
2443
|
+
def get_externalnetworkinformation_as_json_string(self,clientinformation_link:str='https://clientinformation.anion327.de') -> str:
|
|
2444
|
+
headers = {'Cache-Control': 'no-cache'}
|
|
2445
|
+
response = requests.get(clientinformation_link, timeout=5, headers=headers)
|
|
2446
|
+
network_information_as_json_string = GeneralUtilities.bytes_to_string(response.content)
|
|
2447
|
+
return network_information_as_json_string
|
|
2448
|
+
|
|
2449
|
+
@GeneralUtilities.check_arguments
|
|
2450
|
+
def change_file_extensions(self, folder: str, from_extension: str, to_extension: str, recursive: bool, ignore_case: bool) -> None:
|
|
2451
|
+
extension_to_compare: str = None
|
|
2452
|
+
if ignore_case:
|
|
2453
|
+
extension_to_compare = from_extension.lower()
|
|
2454
|
+
else:
|
|
2455
|
+
extension_to_compare = from_extension
|
|
2456
|
+
for file in GeneralUtilities.get_direct_files_of_folder(folder):
|
|
2457
|
+
if (ignore_case and file.lower().endswith(f".{extension_to_compare}") or not ignore_case and file.endswith(f".{extension_to_compare}")):
|
|
2458
|
+
p = Path(file)
|
|
2459
|
+
p.rename(p.with_suffix('.'+to_extension))
|
|
2460
|
+
if recursive:
|
|
2461
|
+
for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
|
|
2462
|
+
self.change_file_extensions(subfolder, from_extension, to_extension, recursive, ignore_case)
|
|
2463
|
+
|
|
2464
|
+
@GeneralUtilities.check_arguments
|
|
2465
|
+
def __add_chapter(self, main_reference_file, reference_content_folder, number: int, chaptertitle: str, content: str = None):
|
|
2466
|
+
if content is None:
|
|
2467
|
+
content = "TXDX add content here"
|
|
2468
|
+
filename = str(number).zfill(2)+"_"+chaptertitle.replace(' ', '-')
|
|
2469
|
+
file = f"{reference_content_folder}/{filename}.md"
|
|
2470
|
+
full_title = f"{number}. {chaptertitle}"
|
|
2471
|
+
|
|
2472
|
+
GeneralUtilities.append_line_to_file(main_reference_file, f"- [{full_title}](./{filename}.md)")
|
|
2473
|
+
|
|
2474
|
+
GeneralUtilities.ensure_file_exists(file)
|
|
2475
|
+
GeneralUtilities.write_text_to_file(file, f"""# {full_title}
|
|
2476
|
+
|
|
2477
|
+
{content}
|
|
2478
|
+
""".replace("XDX", "ODO"))
|
|
2479
|
+
|
|
2480
|
+
@GeneralUtilities.check_arguments
|
|
2481
|
+
def generate_arc42_reference_template(self, repository: str, productname: str = None, subfolder: str = None):
|
|
2482
|
+
productname: str
|
|
2483
|
+
if productname is None:
|
|
2484
|
+
productname = os.path.basename(repository)
|
|
2485
|
+
if subfolder is None:
|
|
2486
|
+
subfolder = "Other/Reference"
|
|
2487
|
+
reference_root_folder = f"{repository}/{subfolder}"
|
|
2488
|
+
reference_content_folder = reference_root_folder + "/Technical"
|
|
2489
|
+
if os.path.isdir(reference_root_folder):
|
|
2490
|
+
raise ValueError(f"The folder '{reference_root_folder}' does already exist.")
|
|
2491
|
+
GeneralUtilities.ensure_directory_exists(reference_root_folder)
|
|
2492
|
+
GeneralUtilities.ensure_directory_exists(reference_content_folder)
|
|
2493
|
+
main_reference_file = f"{reference_root_folder}/Reference.md"
|
|
2494
|
+
GeneralUtilities.ensure_file_exists(main_reference_file)
|
|
2495
|
+
GeneralUtilities.write_text_to_file(main_reference_file, f"""# {productname}
|
|
2496
|
+
|
|
2497
|
+
TXDX add minimal service-description here.
|
|
2498
|
+
|
|
2499
|
+
## Technical documentation
|
|
2500
|
+
|
|
2501
|
+
""".replace("XDX", "ODO"))
|
|
2502
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 1, 'Introduction and Goals', """## Overview
|
|
2503
|
+
|
|
2504
|
+
TXDX
|
|
2505
|
+
|
|
2506
|
+
## Quality goals
|
|
2507
|
+
|
|
2508
|
+
TXDX
|
|
2509
|
+
|
|
2510
|
+
## Stakeholder
|
|
2511
|
+
|
|
2512
|
+
| Name | How to contact | Reason |
|
|
2513
|
+
| ---- | -------------- | ------ |""")
|
|
2514
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 2, 'Constraints', """## Technical constraints
|
|
2515
|
+
|
|
2516
|
+
| Constraint-identifier | Constraint | Reason |
|
|
2517
|
+
| --------------------- | ---------- | ------ |
|
|
2518
|
+
|
|
2519
|
+
## Organizational constraints
|
|
2520
|
+
|
|
2521
|
+
| Constraint-identifier | Constraint | Reason |
|
|
2522
|
+
| --------------------- | ---------- | ------ |""")
|
|
2523
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 3, 'Context and Scope', """## Context
|
|
2524
|
+
|
|
2525
|
+
TXDX
|
|
2526
|
+
|
|
2527
|
+
## Scope
|
|
2528
|
+
|
|
2529
|
+
TXDX""")
|
|
2530
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 4, 'Solution Strategy', """TXDX""")
|
|
2531
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 5, 'Building Block View', """TXDX""")
|
|
2532
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 6, 'Runtime View', """TXDX""")
|
|
2533
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 7, 'Deployment View', """## Infrastructure-overview
|
|
2534
|
+
|
|
2535
|
+
TXDX
|
|
2536
|
+
|
|
2537
|
+
## Infrastructure-requirements
|
|
2538
|
+
|
|
2539
|
+
TXDX
|
|
2540
|
+
|
|
2541
|
+
## Deployment-proecsses
|
|
2542
|
+
|
|
2543
|
+
TXDX""")
|
|
2544
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 8, 'Crosscutting Concepts', """TXDX""")
|
|
2545
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 9, 'Architectural Decisions', """## Decision-board
|
|
2546
|
+
|
|
2547
|
+
| Decision-identifier | Date | Decision | Reason and notes |
|
|
2548
|
+
| ------------------- | ---- | -------- | ---------------- |""") # empty because there are no decsions yet
|
|
2549
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 10, 'Quality Requirements', """TXDX""")
|
|
2550
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 11, 'Risks and Technical Debt', """## Risks
|
|
2551
|
+
|
|
2552
|
+
Currently there are no known risks.
|
|
2553
|
+
|
|
2554
|
+
## Technical debts
|
|
2555
|
+
|
|
2556
|
+
Currently there are no technical depts.""")
|
|
2557
|
+
self.__add_chapter(main_reference_file, reference_content_folder, 12, 'Glossary', """## Terms
|
|
2558
|
+
|
|
2559
|
+
| Term | Meaning |
|
|
2560
|
+
| ---- | ------- |
|
|
2561
|
+
|
|
2562
|
+
## Abbreviations
|
|
2563
|
+
|
|
2564
|
+
| Abbreviation | Meaning |
|
|
2565
|
+
| ------------ | ------- |""")
|
|
2566
|
+
|
|
2567
|
+
GeneralUtilities.append_to_file(main_reference_file, """
|
|
2568
|
+
|
|
2569
|
+
## Responsibilities
|
|
2570
|
+
|
|
2571
|
+
| Responsibility | Name and contact-information |
|
|
2572
|
+
| --------------- | ---------------------------- |
|
|
2573
|
+
| Pdocut-owner | TXDX |
|
|
2574
|
+
| Product-manager | TXDX |
|
|
2575
|
+
| Support | TXDX |
|
|
2576
|
+
|
|
2577
|
+
## License & Pricing
|
|
2578
|
+
|
|
2579
|
+
TXDX
|
|
2580
|
+
|
|
2581
|
+
## External resources
|
|
2582
|
+
|
|
2583
|
+
- [Repository](TXDX)
|
|
2584
|
+
- [Productive-System](TXDX)
|
|
2585
|
+
- [QualityCheck-system](TXDX)
|
|
2586
|
+
""".replace("XDX", "ODO"))
|
|
2587
|
+
|
|
2588
|
+
@GeneralUtilities.check_arguments
|
|
2589
|
+
def run_with_timeout(self, method, timeout_in_seconds: float) -> bool:
|
|
2590
|
+
# Returns true if the method was terminated due to a timeout
|
|
2591
|
+
# Returns false if the method terminates in the given time
|
|
2592
|
+
p = multiprocessing.Process(target=method)
|
|
2593
|
+
p.start()
|
|
2594
|
+
p.join(timeout_in_seconds)
|
|
2595
|
+
if p.is_alive():
|
|
2596
|
+
p.kill()
|
|
2597
|
+
p.join()
|
|
2598
|
+
return True
|
|
2599
|
+
else:
|
|
2600
|
+
return False
|
|
2601
|
+
|
|
2602
|
+
@GeneralUtilities.check_arguments
|
|
2603
|
+
def ensure_local_docker_network_exists(self, network_name: str) -> None:
|
|
2604
|
+
if not self.local_docker_network_exists(network_name):
|
|
2605
|
+
self.create_local_docker_network(network_name)
|
|
2606
|
+
|
|
2607
|
+
@GeneralUtilities.check_arguments
|
|
2608
|
+
def ensure_local_docker_network_does_not_exist(self, network_name: str) -> None:
|
|
2609
|
+
if self.local_docker_network_exists(network_name):
|
|
2610
|
+
self.remove_local_docker_network(network_name)
|
|
2611
|
+
|
|
2612
|
+
@GeneralUtilities.check_arguments
|
|
2613
|
+
def local_docker_network_exists(self, network_name: str) -> bool:
|
|
2614
|
+
return network_name in self.get_all_local_existing_docker_networks()
|
|
2615
|
+
|
|
2616
|
+
@GeneralUtilities.check_arguments
|
|
2617
|
+
def get_all_local_existing_docker_networks(self) -> list[str]:
|
|
2618
|
+
program_call_result = self.run_program("docker", "network list")
|
|
2619
|
+
std_out = program_call_result[1]
|
|
2620
|
+
std_out_lines = std_out.split("\n")[1:]
|
|
2621
|
+
result: list[str] = []
|
|
2622
|
+
for std_out_line in std_out_lines:
|
|
2623
|
+
normalized_line = ';'.join(std_out_line.split())
|
|
2624
|
+
splitted = normalized_line.split(";")
|
|
2625
|
+
result.append(splitted[1])
|
|
2626
|
+
return result
|
|
2627
|
+
|
|
2628
|
+
@GeneralUtilities.check_arguments
|
|
2629
|
+
def remove_local_docker_network(self, network_name: str) -> None:
|
|
2630
|
+
self.run_program("docker", f"network remove {network_name}")
|
|
2631
|
+
|
|
2632
|
+
@GeneralUtilities.check_arguments
|
|
2633
|
+
def create_local_docker_network(self, network_name: str) -> None:
|
|
2634
|
+
self.run_program("docker", f"network create {network_name}")
|
|
2635
|
+
|
|
2636
|
+
@GeneralUtilities.check_arguments
|
|
2637
|
+
def format_xml_file(self, file: str,add_xml_declaration:bool=True) -> None:
|
|
2638
|
+
encoding = "utf-8"
|
|
2639
|
+
element = ET.XML(GeneralUtilities.read_text_from_file(file, encoding))
|
|
2640
|
+
def trim_texts(elem: ET.Element):
|
|
2641
|
+
if elem.text:
|
|
2642
|
+
elem.text = elem.text.strip()
|
|
2643
|
+
if elem.tail:
|
|
2644
|
+
elem.tail = elem.tail.strip()
|
|
2645
|
+
for child in elem:
|
|
2646
|
+
trim_texts(child)
|
|
2647
|
+
trim_texts(element)
|
|
2648
|
+
ET.indent(element)
|
|
2649
|
+
content = ET.tostring(element, xml_declaration=add_xml_declaration, encoding="unicode")
|
|
2650
|
+
GeneralUtilities.write_text_to_file(file, content.rstrip("\n") + "\n", encoding)
|
|
2651
|
+
|
|
2652
|
+
@GeneralUtilities.check_arguments
|
|
2653
|
+
def format_html_file(self, file: str, add_html_declaration: bool = False) -> None:
|
|
2654
|
+
encoding = "utf-8"
|
|
2655
|
+
content = GeneralUtilities.read_text_from_file(file, encoding)
|
|
2656
|
+
content=self.format_html_content(content, add_html_declaration)
|
|
2657
|
+
GeneralUtilities.write_text_to_file(file, content, encoding)
|
|
2658
|
+
|
|
2659
|
+
@GeneralUtilities.check_arguments
|
|
2660
|
+
def format_html_content(self, content: str, add_html_declaration: bool = False) -> str:
|
|
2661
|
+
|
|
2662
|
+
VOID_ELEMENTS = {"area", "base", "br", "col", "embed", "hr", "img", "input",
|
|
2663
|
+
"link", "meta", "param", "source", "track", "wbr"}
|
|
2664
|
+
|
|
2665
|
+
class _Node:
|
|
2666
|
+
__slots__ = ("tag", "attrs", "children", "text", "is_void", "raw")
|
|
2667
|
+
def __init__(self, tag=None, attrs=(), text=None, is_void=False, raw=None):
|
|
2668
|
+
self.tag = tag
|
|
2669
|
+
self.attrs = list(attrs)
|
|
2670
|
+
self.children = []
|
|
2671
|
+
self.text = text
|
|
2672
|
+
self.is_void = is_void
|
|
2673
|
+
self.raw = raw
|
|
2674
|
+
|
|
2675
|
+
class _Builder(HTMLParser):
|
|
2676
|
+
def __init__(self):
|
|
2677
|
+
super().__init__(convert_charrefs=False)
|
|
2678
|
+
self.root = _Node()
|
|
2679
|
+
self.stack = [self.root]
|
|
2680
|
+
|
|
2681
|
+
def _top(self):
|
|
2682
|
+
return self.stack[-1]
|
|
2683
|
+
|
|
2684
|
+
def handle_starttag(self, tag, attrs):
|
|
2685
|
+
raw = self.get_starttag_text()
|
|
2686
|
+
raw_lower = raw.lower()
|
|
2687
|
+
original_attrs = []
|
|
2688
|
+
pos = 1 + len(tag)
|
|
2689
|
+
for lc_name, value in attrs:
|
|
2690
|
+
idx = raw_lower.index(lc_name, pos)
|
|
2691
|
+
original_attrs.append((raw[idx:idx + len(lc_name)], value))
|
|
2692
|
+
pos = idx + len(lc_name)
|
|
2693
|
+
node = _Node(tag=tag, attrs=original_attrs, is_void=tag.lower() in VOID_ELEMENTS)
|
|
2694
|
+
self._top().children.append(node)
|
|
2695
|
+
if not node.is_void:
|
|
2696
|
+
self.stack.append(node)
|
|
2697
|
+
|
|
2698
|
+
def handle_endtag(self, tag):
|
|
2699
|
+
if len(self.stack) > 1 and self.stack[-1].tag == tag:
|
|
2700
|
+
self.stack.pop()
|
|
2701
|
+
|
|
2702
|
+
def handle_data(self, data):
|
|
2703
|
+
t = " ".join(data.split())
|
|
2704
|
+
if t:
|
|
2705
|
+
self._top().children.append(_Node(text=t))
|
|
2706
|
+
|
|
2707
|
+
def handle_entityref(self, name):
|
|
2708
|
+
self._top().children.append(_Node(text=f"&{name};"))
|
|
2709
|
+
|
|
2710
|
+
def handle_charref(self, name):
|
|
2711
|
+
self._top().children.append(_Node(text=f"&#{name};"))
|
|
2712
|
+
|
|
2713
|
+
def handle_comment(self, data):
|
|
2714
|
+
self._top().children.append(_Node(raw=f"<!--{data}-->"))
|
|
2715
|
+
|
|
2716
|
+
def handle_decl(self, decl):
|
|
2717
|
+
self._top().children.append(_Node(raw=f"<!{decl}>"))
|
|
2718
|
+
|
|
2719
|
+
def handle_pi(self, data):
|
|
2720
|
+
self._top().children.append(_Node(raw=f"<?{data}>"))
|
|
2721
|
+
|
|
2722
|
+
_angular_exprs: list[str] = []
|
|
2723
|
+
|
|
2724
|
+
def _protect_angular(m: re.Match) -> str:
|
|
2725
|
+
idx = len(_angular_exprs)
|
|
2726
|
+
_angular_exprs.append(m.group(0))
|
|
2727
|
+
return f"__ANGEXPR{idx}__"
|
|
2728
|
+
|
|
2729
|
+
protected = re.sub(r'\{\{[\s\S]*?\}\}', _protect_angular, content)
|
|
2730
|
+
builder = _Builder()
|
|
2731
|
+
builder.feed(protected)
|
|
2732
|
+
ind = " "
|
|
2733
|
+
|
|
2734
|
+
def _serialize(node: _Node, depth: int) -> list:
|
|
2735
|
+
prefix = ind * depth
|
|
2736
|
+
if node.raw is not None:
|
|
2737
|
+
return [prefix + node.raw]
|
|
2738
|
+
if node.text is not None:
|
|
2739
|
+
return [node.text]
|
|
2740
|
+
if node.tag is None:
|
|
2741
|
+
out = []
|
|
2742
|
+
for c in node.children:
|
|
2743
|
+
out.extend(_serialize(c, depth))
|
|
2744
|
+
return out
|
|
2745
|
+
attr_str = "".join(f" {n}" if v is None else f' {n}="{v}"' for n, v in node.attrs)
|
|
2746
|
+
if node.is_void:
|
|
2747
|
+
return [f"{prefix}<{node.tag}{attr_str}>"]
|
|
2748
|
+
has_elem_children = any(c.tag is not None for c in node.children)
|
|
2749
|
+
if not has_elem_children:
|
|
2750
|
+
inner = "".join(c.text or "" for c in node.children)
|
|
2751
|
+
return [f"{prefix}<{node.tag}{attr_str}>{inner}</{node.tag}>"]
|
|
2752
|
+
lines = [f"{prefix}<{node.tag}{attr_str}>"]
|
|
2753
|
+
for c in node.children:
|
|
2754
|
+
child_lines = _serialize(c, depth + 1)
|
|
2755
|
+
if c.text is not None:
|
|
2756
|
+
lines.append(ind * (depth + 1) + child_lines[0])
|
|
2757
|
+
else:
|
|
2758
|
+
lines.extend(child_lines)
|
|
2759
|
+
lines.append(f"{prefix}</{node.tag}>")
|
|
2760
|
+
return lines
|
|
2761
|
+
|
|
2762
|
+
result = "\n".join(line for line in _serialize(builder.root, 0) if line.strip())
|
|
2763
|
+
for i, expr in enumerate(_angular_exprs):
|
|
2764
|
+
result = result.replace(f"__ANGEXPR{i}__", expr)
|
|
2765
|
+
if add_html_declaration and not result.lstrip().startswith("<!DOCTYPE"):
|
|
2766
|
+
result = "<!DOCTYPE html>\n" + result
|
|
2767
|
+
return result
|
|
2768
|
+
|
|
2769
|
+
@GeneralUtilities.check_arguments
|
|
2770
|
+
def get_pip_index_url_arguments_from_local_cache(self)->list[str]:
|
|
2771
|
+
arguments=[]
|
|
2772
|
+
pip_folder=GeneralUtilities.normalize_path(self.get_global_cache_folder()+"/Pip")
|
|
2773
|
+
if os.path.isdir(pip_folder):
|
|
2774
|
+
main_index_file=GeneralUtilities.normalize_path(os.path.join(pip_folder, "MainIndex.txt"))
|
|
2775
|
+
if os.path.isfile(main_index_file):
|
|
2776
|
+
lines=GeneralUtilities.read_nonempty_lines_from_file(main_index_file)
|
|
2777
|
+
url=[line for line in lines if line.startswith("IndexURL: ")][0].split(":")[1].strip()
|
|
2778
|
+
arguments.append("--index-url")
|
|
2779
|
+
arguments.append(url)
|
|
2780
|
+
extra_index_folder=GeneralUtilities.normalize_path(os.path.join(pip_folder, "ExtraIndexURLs"))
|
|
2781
|
+
if os.path.isdir(extra_index_folder):
|
|
2782
|
+
index_files=GeneralUtilities.get_direct_files_of_folder(extra_index_folder)
|
|
2783
|
+
if len(index_files) > 0:
|
|
2784
|
+
for indexurl_file in index_files:
|
|
2785
|
+
lines=GeneralUtilities.read_nonempty_lines_from_file(indexurl_file)
|
|
2786
|
+
url=[line for line in lines if line.startswith("IndexURL: ")][0].split(":")[1].strip()
|
|
2787
|
+
arguments.append("--extra-index-url")
|
|
2788
|
+
arguments.append(url)
|
|
2789
|
+
return arguments
|
|
2790
|
+
|
|
2791
|
+
@GeneralUtilities.check_arguments
|
|
2792
|
+
def install_requirementstxt_file(self, requirements_txt_file: str):
|
|
2793
|
+
folder: str = os.path.dirname(requirements_txt_file)
|
|
2794
|
+
filename: str = os.path.basename(requirements_txt_file)
|
|
2795
|
+
arguments= ["install", "-r", filename]
|
|
2796
|
+
for argument in self.get_pip_index_url_arguments_from_local_cache():
|
|
2797
|
+
arguments.append(argument)
|
|
2798
|
+
self.run_program_argsasarray("pip", arguments, folder,print_live_output=self.log.loglevel==LogLevel.Debug)
|
|
2799
|
+
|
|
2800
|
+
@GeneralUtilities.check_arguments
|
|
2801
|
+
def ocr_analysis_of_folder_using_local_docker_image(self, folder: str, extensions: list[str], languages: list[str],base_folder_for_entry: str,ignore_pattern:list[str] ) -> list[str]: # Returns a list of changed files due to ocr-analysis.
|
|
2802
|
+
#TODO start docker server
|
|
2803
|
+
serviceaddress:str=None#TODO
|
|
2804
|
+
self.ocr_analysis_of_folder(folder, serviceaddress, extensions, languages, base_folder_for_entry,ignore_pattern)
|
|
2805
|
+
#TODO stop docker server
|
|
2806
|
+
|
|
2807
|
+
@GeneralUtilities.check_arguments
|
|
2808
|
+
def ocr_analysis_of_folder(self, folder: str, serviceaddress: str, extensions: list[str], languages: list[str],base_folder_for_entry: str,ignore_pattern:list[str] ) -> list[str]: # Returns a list of changed files due to ocr-analysis.
|
|
2809
|
+
supported_extensions = ['png', 'jpg', 'jpeg', 'tiff', 'bmp', 'gif', 'pdf', 'docx', 'doc', 'xlsx', 'xls', 'pptx', 'ppt']
|
|
2810
|
+
changes_files: list[str] = []
|
|
2811
|
+
if base_folder_for_entry is None:
|
|
2812
|
+
base_folder_for_entry=folder
|
|
2813
|
+
if extensions is None:
|
|
2814
|
+
extensions = supported_extensions
|
|
2815
|
+
for file in GeneralUtilities.get_direct_files_of_folder(folder):
|
|
2816
|
+
if GeneralUtilities.is_ignored_by_glob_pattern(os.path.dirname(file),file,ignore_pattern):
|
|
2817
|
+
continue
|
|
2818
|
+
file_lower = file.lower()
|
|
2819
|
+
for extension in extensions:
|
|
2820
|
+
if file_lower.endswith("."+extension):
|
|
2821
|
+
if self.ocr_analysis_of_file(file, serviceaddress, languages,base_folder_for_entry):
|
|
2822
|
+
changes_files.append(file)
|
|
2823
|
+
break
|
|
2824
|
+
for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
|
|
2825
|
+
if GeneralUtilities.is_ignored_by_glob_pattern(os.path.dirname(subfolder),subfolder,ignore_pattern):
|
|
2826
|
+
continue
|
|
2827
|
+
for file in self.ocr_analysis_of_folder(subfolder, serviceaddress, extensions, languages,base_folder_for_entry+"/"+os.path.basename(subfolder), ignore_pattern):
|
|
2828
|
+
changes_files.append(file)
|
|
2829
|
+
return changes_files
|
|
2830
|
+
|
|
2831
|
+
|
|
2832
|
+
@GeneralUtilities.check_arguments
|
|
2833
|
+
def __it_supported_extension(self, file: str, supported_extensions: list[str]) -> bool:
|
|
2834
|
+
file_lower = file.lower()
|
|
2835
|
+
for extension in supported_extensions:
|
|
2836
|
+
if file_lower.endswith("."+extension):
|
|
2837
|
+
return True
|
|
2838
|
+
return False
|
|
2839
|
+
|
|
2840
|
+
@GeneralUtilities.check_arguments
|
|
2841
|
+
def ocr_analysis_of_file(self, file: str, serviceaddress: str, languages: list[str], readable_folder_entry:str ) -> bool: # Returns true if the ocr-file was generated or updated. Returns false if the existing ocr-file was not changed.
|
|
2842
|
+
supported_extensions = ['png', 'jpg', 'jpeg', 'tiff', 'bmp', 'webp', 'gif', 'pdf', 'rtf', 'docx', 'doc', 'odt', 'xlsx', 'xls', 'ods', 'pptx', 'ppt', 'odp']
|
|
2843
|
+
if not self.__it_supported_extension(file, supported_extensions):
|
|
2844
|
+
raise ValueError(f"File '{file}' is not supported due to unsupported extension. Supported extensions are: {', '.join(supported_extensions)}")
|
|
2845
|
+
target_file = file+".ocr.txt"
|
|
2846
|
+
hash_of_current_file: str = GeneralUtilities.get_sha256_of_file(file)
|
|
2847
|
+
try:
|
|
2848
|
+
if os.path.isfile(target_file):
|
|
2849
|
+
lines = GeneralUtilities.read_lines_from_file(target_file)
|
|
2850
|
+
previous_hash_of_current_file: str = lines[1].split(":")[1].strip()
|
|
2851
|
+
if hash_of_current_file == previous_hash_of_current_file:
|
|
2852
|
+
return False
|
|
2853
|
+
except:
|
|
2854
|
+
pass
|
|
2855
|
+
GeneralUtilities.write_message_to_stdout(f"Starting OCR-analysis of file \"{file}\"...")
|
|
2856
|
+
ocr_content = self.get_ocr_content_of_file(file, serviceaddress, languages)
|
|
2857
|
+
GeneralUtilities.ensure_file_exists(target_file)
|
|
2858
|
+
if readable_folder_entry is None:
|
|
2859
|
+
readable_folder_entry="."
|
|
2860
|
+
readable_folder_entry=readable_folder_entry.replace("\\", "/")
|
|
2861
|
+
GeneralUtilities.write_text_to_file(target_file, f"""Name of file: \"{readable_folder_entry}/{os.path.basename(file)}\""
|
|
2862
|
+
Hash of file: {hash_of_current_file}
|
|
2863
|
+
OCR-content:
|
|
2864
|
+
{ocr_content}""")
|
|
2865
|
+
return True
|
|
2866
|
+
|
|
2867
|
+
@GeneralUtilities.check_arguments
|
|
2868
|
+
def get_ocr_content_of_file(self, file: str, serviceaddress: str, languages: list[str]) -> str:
|
|
2869
|
+
result: str = None
|
|
2870
|
+
extension = Path(file).suffix[1:]
|
|
2871
|
+
mime_types = {
|
|
2872
|
+
"pdf": "application/pdf",
|
|
2873
|
+
"png": "image/png",
|
|
2874
|
+
"jpg": "image/jpeg",
|
|
2875
|
+
"jpeg": "image/jpeg",
|
|
2876
|
+
"txt": "text/plain",
|
|
2877
|
+
"json": "application/json",
|
|
2878
|
+
"doc": "application/msword",
|
|
2879
|
+
"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
2880
|
+
"xls": "application/vnd.ms-excel",
|
|
2881
|
+
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
2882
|
+
}
|
|
2883
|
+
if serviceaddress is None:
|
|
2884
|
+
server_url_file:str= GeneralUtilities.normalize_path(f"{str(Path.home())}/.ScriptCollection/OCR/ServiceURL.txt")
|
|
2885
|
+
if os.path.isfile(server_url_file):
|
|
2886
|
+
for line in GeneralUtilities.read_nonempty_lines_from_file(server_url_file):
|
|
2887
|
+
if not line.startswith("#"):
|
|
2888
|
+
serviceaddress = line.strip()
|
|
2889
|
+
break
|
|
2890
|
+
GeneralUtilities.assert_not_null(serviceaddress, "ocr-service-address must not be null.")
|
|
2891
|
+
mime_type = mime_types.get(extension.lower(), "application/octet-stream")
|
|
2892
|
+
service_url: str = f"{serviceaddress}/API/v1/SimpleOCR/GetOCRContent?mimeType={mime_type}"
|
|
2893
|
+
for language in languages:
|
|
2894
|
+
service_url = service_url + f"&languages={language}"
|
|
2895
|
+
headers = {'Cache-Control': 'no-cache'}
|
|
2896
|
+
with open(file, "rb") as f:
|
|
2897
|
+
files_to_analyse = {
|
|
2898
|
+
"fileContent": (os.path.basename(file), f, mime_type)
|
|
2899
|
+
}
|
|
2900
|
+
r = requests.put(service_url, timeout=3600, headers=headers, files=files_to_analyse,verify=True)
|
|
2901
|
+
if r.status_code != 200:
|
|
2902
|
+
if r.status_code == 400:
|
|
2903
|
+
return f"Could not calculate ocr-content for file \"{file}\". File may be broken."
|
|
2904
|
+
else:
|
|
2905
|
+
raise ValueError(f"Retrieving ocr-content for file \"{file}\" resulted in HTTP-response-code {r.status_code}.")
|
|
2906
|
+
|
|
2907
|
+
result = GeneralUtilities.bytes_to_string(r.content)
|
|
2908
|
+
return result
|
|
2909
|
+
|
|
2910
|
+
@GeneralUtilities.check_arguments
|
|
2911
|
+
def ocr_analysis_of_repository(self, folder: str, serviceaddress: str, extensions: list[str], languages: list[str]) -> None:
|
|
2912
|
+
self.assert_is_git_repository(folder)
|
|
2913
|
+
self.ocr_analysis_of_folder(folder, serviceaddress, extensions, languages,".",[".git"])
|
|
2914
|
+
|
|
2915
|
+
@GeneralUtilities.check_arguments
|
|
2916
|
+
def update_timestamp_in_file(self, target_file: str) -> None:
|
|
2917
|
+
lines = GeneralUtilities.read_lines_from_file(target_file)
|
|
2918
|
+
new_lines = []
|
|
2919
|
+
prefix: str = "# last update: "
|
|
2920
|
+
for line in lines:
|
|
2921
|
+
if line.startswith(prefix):
|
|
2922
|
+
new_lines.append(prefix+GeneralUtilities.datetime_to_string_for_readable_entry(GeneralUtilities.get_now(),False))
|
|
2923
|
+
else:
|
|
2924
|
+
new_lines.append(line)
|
|
2925
|
+
GeneralUtilities.write_lines_to_file(target_file, new_lines)
|
|
2926
|
+
|
|
2927
|
+
@GeneralUtilities.check_arguments
|
|
2928
|
+
def do_and_log_task(self, name_of_task: str, task,log_end_of_Task:bool=True)->int:
|
|
2929
|
+
try:
|
|
2930
|
+
self.log.log(f"Start action \"{name_of_task}\".", LogLevel.Information)
|
|
2931
|
+
result = task()
|
|
2932
|
+
if result is None:
|
|
2933
|
+
result = 0
|
|
2934
|
+
return result
|
|
2935
|
+
except Exception as e:
|
|
2936
|
+
self.log.log_exception(f"Error while running action \"{name_of_task}\".", e, LogLevel.Error)
|
|
2937
|
+
return 1
|
|
2938
|
+
finally:
|
|
2939
|
+
if log_end_of_Task:
|
|
2940
|
+
self.log.log(f"Finished action \"{name_of_task}\".", LogLevel.Information)
|
|
2941
|
+
|
|
2942
|
+
|
|
2943
|
+
default_excluded_patterns_for_loc: list[str] = ["**.txt", "**.md", "**.svg", "**.xlf", "**.vscode", "**/Resources/**", "**/Reference/**", ".gitignore", ".gitattributes", "Other/Metrics/**"]
|
|
2944
|
+
|
|
2945
|
+
@GeneralUtilities.check_arguments
|
|
2946
|
+
def get_lines_of_code_with_default_excluded_patterns(self, repository: str) -> int:
|
|
2947
|
+
return self.get_lines_of_code(repository, self.default_excluded_patterns_for_loc)
|
|
2948
|
+
|
|
2949
|
+
@GeneralUtilities.check_arguments
|
|
2950
|
+
def get_lines_of_code(self, repository: str, excluded_pattern: list[str]) -> int:
|
|
2951
|
+
self.assert_is_git_repository(repository)
|
|
2952
|
+
result: int = 0
|
|
2953
|
+
self.log.log(f"Calculate lines of code in repository '{repository}' with excluded patterns: {', '.join(excluded_pattern)}",LogLevel.Debug)
|
|
2954
|
+
git_response = self.run_program("git", "ls-files", repository)
|
|
2955
|
+
files: list[str] = GeneralUtilities.string_to_lines(git_response[1])
|
|
2956
|
+
for file in files:
|
|
2957
|
+
if os.path.isfile(os.path.join(repository, file)):
|
|
2958
|
+
if self.__is_excluded_by_glob_pattern(file, excluded_pattern):
|
|
2959
|
+
self.log.log(f"File '{file}' is ignored because it matches an excluded pattern.",LogLevel.Diagnostic)
|
|
2960
|
+
else:
|
|
2961
|
+
full_file: str = os.path.join(repository, file)
|
|
2962
|
+
if GeneralUtilities.is_binary_file(full_file):
|
|
2963
|
+
self.log.log(f"File '{file}' is ignored because it is a binary-file.",LogLevel.Diagnostic)
|
|
2964
|
+
else:
|
|
2965
|
+
self.log.log(f"Count lines of file '{file}'.",LogLevel.Diagnostic)
|
|
2966
|
+
length = len(GeneralUtilities.read_nonempty_lines_from_file(full_file))
|
|
2967
|
+
result = result+length
|
|
2968
|
+
else:
|
|
2969
|
+
self.log.log(f"File '{file}' is ignored because it does not exist.",LogLevel.Diagnostic)
|
|
2970
|
+
return result
|
|
2971
|
+
|
|
2972
|
+
@GeneralUtilities.check_arguments
|
|
2973
|
+
def __is_excluded_by_glob_pattern(self, file: str, excluded_patterns: list[str]) -> bool:
|
|
2974
|
+
for pattern in excluded_patterns:
|
|
2975
|
+
if fnmatch.fnmatch(file, pattern):
|
|
2976
|
+
return True
|
|
2977
|
+
return False
|
|
2978
|
+
|
|
2979
|
+
@GeneralUtilities.check_arguments
|
|
2980
|
+
def create_zip_archive(self, folder:str,zip_file:str) -> None:
|
|
2981
|
+
GeneralUtilities.assert_folder_exists(folder)
|
|
2982
|
+
GeneralUtilities.assert_file_does_not_exist(zip_file)
|
|
2983
|
+
folder = os.path.abspath(folder)
|
|
2984
|
+
with zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED) as zipf:
|
|
2985
|
+
for root, _, files in os.walk(folder):
|
|
2986
|
+
for file in files:
|
|
2987
|
+
file_path = os.path.join(root, file)
|
|
2988
|
+
arcname = os.path.relpath(file_path, start=folder)
|
|
2989
|
+
zipf.write(file_path, arcname)
|
|
2990
|
+
|
|
2991
|
+
@GeneralUtilities.check_arguments
|
|
2992
|
+
def start_local_test_service(self, file: str):
|
|
2993
|
+
example_folder = os.path.dirname(file)
|
|
2994
|
+
docker_compose_file = os.path.join(example_folder, "docker-compose.yml")
|
|
2995
|
+
for service in self.get_services_from_yaml_file(docker_compose_file):
|
|
2996
|
+
self.kill_docker_container(service)
|
|
2997
|
+
example_name = os.path.basename(example_folder)
|
|
2998
|
+
title = f"Test{example_name}"
|
|
2999
|
+
argument=f"compose -p {title.lower()}"
|
|
3000
|
+
if os.path.isfile(os.path.join(example_folder,"Parameters.env")):
|
|
3001
|
+
argument=argument+" --env-file Parameters.env"
|
|
3002
|
+
argument=argument+" up --detach"
|
|
3003
|
+
self.run_program("docker", argument, example_folder, title=title,print_live_output=True)
|
|
3004
|
+
|
|
3005
|
+
@GeneralUtilities.check_arguments
|
|
3006
|
+
def stop_local_test_service(self, file: str):
|
|
3007
|
+
example_folder = os.path.dirname(file)
|
|
3008
|
+
example_name = os.path.basename(example_folder)
|
|
3009
|
+
title = f"Test{example_name}"
|
|
3010
|
+
self.run_program("docker", f"compose -p {title.lower()} down", example_folder, title=title,print_live_output=True)
|
|
3011
|
+
|
|
3012
|
+
@GeneralUtilities.check_arguments
|
|
3013
|
+
def generate_chart_diagram(self,source_file:str,target_file:str):
|
|
3014
|
+
workingfolder=os.path.dirname(source_file)
|
|
3015
|
+
argument=f"\"{source_file}\" \"{target_file}\""
|
|
3016
|
+
loglevelMap = {
|
|
3017
|
+
LogLevel.Error: "error",
|
|
3018
|
+
LogLevel.Warning: "warn",
|
|
3019
|
+
LogLevel.Information: "info",
|
|
3020
|
+
LogLevel.Debug: "debug",
|
|
3021
|
+
}
|
|
3022
|
+
if self.log.loglevel==LogLevel.Debug:
|
|
3023
|
+
argument=f"-l {loglevelMap[self.log.loglevel]} {argument}"
|
|
3024
|
+
self.run_with_epew("vl2svg",argument,workingfolder,encode_argument_in_base64=True)
|
|
3025
|
+
#this uses vega-light. to use vega "vg2svg" should be used instead.
|
|
3026
|
+
|
|
3027
|
+
@GeneralUtilities.check_arguments
|
|
3028
|
+
def inspect_container(self, container_name: str) :
|
|
3029
|
+
program_result = self.run_program(
|
|
3030
|
+
"docker",
|
|
3031
|
+
f"inspect {container_name}",
|
|
3032
|
+
throw_exception_if_exitcode_is_not_zero=True
|
|
3033
|
+
)
|
|
3034
|
+
stdout=program_result[1]
|
|
3035
|
+
|
|
3036
|
+
data = json.loads(stdout)
|
|
3037
|
+
GeneralUtilities.assert_condition(len(data)==1,f"Unexpected array-length of docker-inspect-output for container \"{container_name}\".")
|
|
3038
|
+
return data[0]
|
|
3039
|
+
|
|
3040
|
+
@GeneralUtilities.check_arguments
|
|
3041
|
+
def container_is_exists(self,container_name:str)->bool:
|
|
3042
|
+
program_result = self.run_program(
|
|
3043
|
+
"docker",
|
|
3044
|
+
f"inspect {container_name}",
|
|
3045
|
+
throw_exception_if_exitcode_is_not_zero=False
|
|
3046
|
+
)
|
|
3047
|
+
return program_result[0]==0
|
|
3048
|
+
|
|
3049
|
+
@GeneralUtilities.check_arguments
|
|
3050
|
+
def container_is_running(self,container_name:str)->bool:
|
|
3051
|
+
data = self.inspect_container( container_name)
|
|
3052
|
+
if data is None:
|
|
3053
|
+
return False
|
|
3054
|
+
|
|
3055
|
+
return data["State"]["Status"] == "running"
|
|
3056
|
+
|
|
3057
|
+
@GeneralUtilities.check_arguments
|
|
3058
|
+
def container_is_healthy(self,container_name:str)->bool:
|
|
3059
|
+
data = self.inspect_container( container_name)
|
|
3060
|
+
if data is None:
|
|
3061
|
+
return False
|
|
3062
|
+
|
|
3063
|
+
state = data["State"]
|
|
3064
|
+
health = state.get("Health")
|
|
3065
|
+
|
|
3066
|
+
if health is None:
|
|
3067
|
+
return False # kein HEALTHCHECK definiert
|
|
3068
|
+
|
|
3069
|
+
return health["Status"] == "healthy"
|
|
3070
|
+
|
|
3071
|
+
@GeneralUtilities.check_arguments
|
|
3072
|
+
def get_output_of_container(self,container_name:str)->str:
|
|
3073
|
+
|
|
3074
|
+
program_result= self.run_program_argsasarray(
|
|
3075
|
+
"docker",
|
|
3076
|
+
["logs",container_name],
|
|
3077
|
+
throw_exception_if_exitcode_is_not_zero=False
|
|
3078
|
+
)
|
|
3079
|
+
exit_code=program_result[0]
|
|
3080
|
+
stdout=program_result[1]
|
|
3081
|
+
stderr=program_result[2]
|
|
3082
|
+
if exit_code != 0:
|
|
3083
|
+
return ""
|
|
3084
|
+
|
|
3085
|
+
return stdout+"\n"+stderr
|
|
3086
|
+
|
|
3087
|
+
@GeneralUtilities.check_arguments
|
|
3088
|
+
def container_is_running_and_healthy(self,container_name:str)->bool:
|
|
3089
|
+
if not self.container_is_exists(container_name):
|
|
3090
|
+
return False
|
|
3091
|
+
if not self.container_is_running(container_name):
|
|
3092
|
+
return False
|
|
3093
|
+
if not self.container_is_healthy(container_name):
|
|
3094
|
+
return False
|
|
3095
|
+
return True
|
|
3096
|
+
|
|
3097
|
+
def reclaim_space_from_docker(self,remove_containers:bool,remove_volumes:bool,remove_images:bool, amount_of_attempts: int = 5):
|
|
3098
|
+
self.log.log("Reclaim disk space from docker...",LogLevel.Debug)
|
|
3099
|
+
if remove_containers:
|
|
3100
|
+
self.run_program_with_retry("docker","container prune -f",amount_of_attempts=amount_of_attempts)
|
|
3101
|
+
if remove_volumes:
|
|
3102
|
+
self.run_program_with_retry("docker","volume prune -f",amount_of_attempts=amount_of_attempts)
|
|
3103
|
+
if remove_images:
|
|
3104
|
+
self.run_program_with_retry("docker","image prune -a -f",amount_of_attempts=amount_of_attempts)
|
|
3105
|
+
self.run_program_with_retry("docker","builder prune -a -f",amount_of_attempts=amount_of_attempts)
|
|
3106
|
+
self.run_program_with_retry("docker","buildx prune -a -f",amount_of_attempts=amount_of_attempts,throw_exception_if_exitcode_is_not_zero=False) # buildx prune is not available on every machine.
|
|
3107
|
+
self.run_program_with_retry("docker","system df",print_live_output=self.log.loglevel==LogLevel.Debug,amount_of_attempts=amount_of_attempts)
|
|
3108
|
+
|
|
3109
|
+
@GeneralUtilities.check_arguments
|
|
3110
|
+
def get_docker_networks(self)->list[str]:
|
|
3111
|
+
program_result=self.run_program("docker","network list")
|
|
3112
|
+
result=[]
|
|
3113
|
+
lines=program_result[1].split("\n")[1:]
|
|
3114
|
+
for line in lines:
|
|
3115
|
+
splitted=[item for item in line.split(' ') if GeneralUtilities.string_has_content(item)]
|
|
3116
|
+
result.append(splitted[1].replace("\n","").replace("\r","").strip())
|
|
3117
|
+
return result
|
|
3118
|
+
|
|
3119
|
+
@GeneralUtilities.check_arguments
|
|
3120
|
+
def ensure_docker_network_is_available(self,network_name:str):
|
|
3121
|
+
#TODO add cli-script to call this function
|
|
3122
|
+
if not (network_name in self.get_docker_networks()):
|
|
3123
|
+
self.run_program("docker",f"network create {network_name}")
|
|
3124
|
+
|
|
3125
|
+
@GeneralUtilities.check_arguments
|
|
3126
|
+
def ensure_docker_network_is_not_available(self,network_name:str):
|
|
3127
|
+
#TODO add cli-script to call this function
|
|
3128
|
+
if network_name in self.get_docker_networks():
|
|
3129
|
+
self.run_program("docker",f"network rm {network_name}")
|
|
3130
|
+
|
|
3131
|
+
@GeneralUtilities.check_arguments
|
|
3132
|
+
def get_available_cultures_for_angular_app(self,angular_json_file:str)->list[str]:
|
|
3133
|
+
languages = ["en"]
|
|
3134
|
+
with open(angular_json_file, "r", encoding="utf-8") as f:
|
|
3135
|
+
data = json.load(f)
|
|
3136
|
+
for project in data.get("projects", {}).values():
|
|
3137
|
+
i18n = project.get("i18n", {})
|
|
3138
|
+
locales = i18n.get("locales", {})
|
|
3139
|
+
languages.extend(locales.keys())
|
|
3140
|
+
|
|
3141
|
+
languages=list(languages)
|
|
3142
|
+
return languages
|
|
3143
|
+
|
|
3144
|
+
@GeneralUtilities.check_arguments
|
|
3145
|
+
def parse_tasks_from_codeworkspace_file(self,code_workspace_file:str)->list[VSCodeWorkspaceShellTask]:
|
|
3146
|
+
result=[]
|
|
3147
|
+
jsoncontent = json.loads(GeneralUtilities.read_text_from_file(code_workspace_file))
|
|
3148
|
+
tasks = jsoncontent["tasks"]["tasks"]
|
|
3149
|
+
for task in tasks:
|
|
3150
|
+
if task["type"] == "shell":
|
|
3151
|
+
label: str = task["label"]
|
|
3152
|
+
name: str = GeneralUtilities.to_pascal_case(label)
|
|
3153
|
+
command:str= task["command"]
|
|
3154
|
+
work_dir:str = None
|
|
3155
|
+
|
|
3156
|
+
if "options" in task:
|
|
3157
|
+
options = task["options"]
|
|
3158
|
+
if "cwd" in options:
|
|
3159
|
+
work_dir = options["cwd"]
|
|
3160
|
+
work_dir = work_dir.replace("${workspaceFolder}", ".")
|
|
3161
|
+
|
|
3162
|
+
command_with_args = command
|
|
3163
|
+
if "args" in task:
|
|
3164
|
+
args = task["args"]
|
|
3165
|
+
if len(args) > 1:
|
|
3166
|
+
command_with_args = f"{command_with_args} {' '.join(args)}"
|
|
3167
|
+
|
|
3168
|
+
description: str =None
|
|
3169
|
+
if "description" in task:
|
|
3170
|
+
description = f'{label} ({task["description"]})'
|
|
3171
|
+
else:
|
|
3172
|
+
description = label
|
|
3173
|
+
|
|
3174
|
+
|
|
3175
|
+
alias_list:list[str]=[]
|
|
3176
|
+
name_lower=name.lower()
|
|
3177
|
+
if name!=name.lower():
|
|
3178
|
+
alias_list.append(name_lower)
|
|
3179
|
+
|
|
3180
|
+
if "aliases" in task:
|
|
3181
|
+
aliases = task["aliases"]
|
|
3182
|
+
for alias in aliases:
|
|
3183
|
+
alias_list.append(alias)
|
|
3184
|
+
|
|
3185
|
+
allow_custom_arguments:bool=False
|
|
3186
|
+
if "allowcustomarguments" in task:
|
|
3187
|
+
allow_custom_arguments = task["allowcustomarguments"]
|
|
3188
|
+
|
|
3189
|
+
result.append(VSCodeWorkspaceShellTask(name, description, work_dir, command_with_args, alias_list, allow_custom_arguments))
|
|
3190
|
+
return result
|
|
3191
|
+
|
|
3192
|
+
@GeneralUtilities.check_arguments
|
|
3193
|
+
def parse_mongodbconnection_from_codeworkspace_file(self,code_workspace_file:str)->list[VSCodeWorkspaceMongoDBConnection]:
|
|
3194
|
+
result=[]
|
|
3195
|
+
jsoncontent = json.loads(GeneralUtilities.read_text_from_file(code_workspace_file))
|
|
3196
|
+
settings=jsoncontent["settings"]
|
|
3197
|
+
if "mdb.presetConnections" in settings:
|
|
3198
|
+
connections = settings["mdb.presetConnections"]
|
|
3199
|
+
for connection in connections:
|
|
3200
|
+
result.append(VSCodeWorkspaceMongoDBConnection(connection["name"],connection["connectionString"]))
|
|
3201
|
+
return result
|
|
3202
|
+
|
|
3203
|
+
@GeneralUtilities.check_arguments
|
|
3204
|
+
def parse_sqlconnection_from_codeworkspace_file(self,code_workspace_file:str)->list[VSCodeWorkspaceMariaDBConnection]:
|
|
3205
|
+
result=[]
|
|
3206
|
+
jsoncontent = json.loads(GeneralUtilities.read_text_from_file(code_workspace_file))
|
|
3207
|
+
settings=jsoncontent["settings"]
|
|
3208
|
+
if "sqltools.connections" in settings:
|
|
3209
|
+
connections = settings["sqltools.connections"]
|
|
3210
|
+
for connection in connections:
|
|
3211
|
+
result.append(VSCodeWorkspaceMariaDBConnection(connection["name"],connection["server"],connection["port"],connection["database"],connection["username"],connection["password"]))
|
|
3212
|
+
return result
|
|
3213
|
+
|
|
3214
|
+
@GeneralUtilities.check_arguments
|
|
3215
|
+
def is_xliff2_file(self,file: str) -> bool:
|
|
3216
|
+
tree = ET.parse(file)
|
|
3217
|
+
root = tree.getroot()
|
|
3218
|
+
|
|
3219
|
+
tag = root.tag # "{urn:oasis:names:tc:xliff:document:2.0}xliff"
|
|
3220
|
+
|
|
3221
|
+
if tag.startswith("{"):
|
|
3222
|
+
namespace, localname = tag[1:].split("}", 1)
|
|
3223
|
+
else:
|
|
3224
|
+
namespace = None
|
|
3225
|
+
localname = tag
|
|
3226
|
+
|
|
3227
|
+
if localname != "xliff":
|
|
3228
|
+
return False
|
|
3229
|
+
|
|
3230
|
+
if namespace != "urn:oasis:names:tc:xliff:document:2.0":
|
|
3231
|
+
return False
|
|
3232
|
+
|
|
3233
|
+
if root.get("version") != "2.0":
|
|
3234
|
+
return False
|
|
3235
|
+
|
|
3236
|
+
return True
|
|
3237
|
+
|
|
3238
|
+
@GeneralUtilities.check_arguments
|
|
3239
|
+
def __sync_xlf2_files(self,base_file:ET.ElementTree, language_files:dict [
|
|
3240
|
+
str,#filepath
|
|
3241
|
+
ET.ElementTree#parsed file
|
|
3242
|
+
]):
|
|
3243
|
+
"""This function assumes that all files are valid xliff2 files and that the base file is the reference for syncing.
|
|
3244
|
+
This function adds new entries from the base file to the language files if they do not already exist using the value from base_file.
|
|
3245
|
+
This function removes entries from the language files if they do not exist in the base file anymore.
|
|
3246
|
+
In the end the updated language files are written to the disk. The base file is not changed."""
|
|
3247
|
+
#The file which was parsed looks like:
|
|
3248
|
+
#<?xml version="1.0" encoding="UTF-8" ?>
|
|
3249
|
+
#<xliff version="2.0" xmlns="urn:oasis:names:tc:xliff:document:2.0" srcLang="en">
|
|
3250
|
+
# <file id="ngi18n" original="ng.template">
|
|
3251
|
+
# <unit id="logingreeting">
|
|
3252
|
+
# <notes>
|
|
3253
|
+
# <note category="location">src/app/modules/home-page/login-form/login-form.component.html:2,4</note>
|
|
3254
|
+
# </notes>
|
|
3255
|
+
# <segment>
|
|
3256
|
+
# <source>Welcome back, please login</source>
|
|
3257
|
+
# </segment>
|
|
3258
|
+
# </unit>
|
|
3259
|
+
# <unit id="username">
|
|
3260
|
+
# <notes>
|
|
3261
|
+
# <note category="location">src/app/modules/home-page/login-form/login-form.component.html:5,6</note>
|
|
3262
|
+
# </notes>
|
|
3263
|
+
# <segment>
|
|
3264
|
+
# <source>Username</source>
|
|
3265
|
+
# </segment>
|
|
3266
|
+
# </unit>
|
|
3267
|
+
# <unit id="password">
|
|
3268
|
+
# <notes>
|
|
3269
|
+
# <note category="location">src/app/modules/home-page/login-form/login-form.component.html:12,13</note>
|
|
3270
|
+
# </notes>
|
|
3271
|
+
# <segment>
|
|
3272
|
+
# <source>Password</source>
|
|
3273
|
+
# </segment>
|
|
3274
|
+
# </unit>
|
|
3275
|
+
# </file>
|
|
3276
|
+
#</xliff>
|
|
3277
|
+
|
|
3278
|
+
NS = "urn:oasis:names:tc:xliff:document:2.0"
|
|
3279
|
+
NSMAP = {"x": NS}
|
|
3280
|
+
base_root = base_file.getroot()
|
|
3281
|
+
base_file_element = base_root.find("x:file", namespaces=NSMAP)
|
|
3282
|
+
if base_file_element is None:
|
|
3283
|
+
raise ValueError("Invalid XLIFF base file: <file> element not found")
|
|
3284
|
+
|
|
3285
|
+
# Collect base units
|
|
3286
|
+
base_units = {
|
|
3287
|
+
unit.get("id"): unit
|
|
3288
|
+
for unit in base_file_element.findall("x:unit", namespaces=NSMAP)
|
|
3289
|
+
}
|
|
3290
|
+
base_ids = set(base_units.keys())
|
|
3291
|
+
for filepath, lang_tree in language_files.items():
|
|
3292
|
+
lang_root = lang_tree.getroot()
|
|
3293
|
+
lang_file_element = lang_root.find("x:file", namespaces=NSMAP)
|
|
3294
|
+
if lang_file_element is None:
|
|
3295
|
+
raise ValueError(f"{filepath}: <file> element not found")
|
|
3296
|
+
|
|
3297
|
+
# Collect language units
|
|
3298
|
+
lang_units = {
|
|
3299
|
+
unit.get("id"): unit
|
|
3300
|
+
for unit in lang_file_element.findall("x:unit", namespaces=NSMAP)
|
|
3301
|
+
}
|
|
3302
|
+
lang_ids = set(lang_units.keys())
|
|
3303
|
+
|
|
3304
|
+
# Remove obsolete units
|
|
3305
|
+
obsolete_ids = lang_ids - base_ids
|
|
3306
|
+
for unit_id in obsolete_ids:
|
|
3307
|
+
lang_file_element.remove(lang_units[unit_id])
|
|
3308
|
+
|
|
3309
|
+
# Add missing units
|
|
3310
|
+
missing_ids = base_ids - lang_ids
|
|
3311
|
+
for unit_id in missing_ids:
|
|
3312
|
+
new_unit = copy.deepcopy(base_units[unit_id])
|
|
3313
|
+
lang_file_element.append(new_unit)
|
|
3314
|
+
|
|
3315
|
+
# Reorder units to match base order
|
|
3316
|
+
current_units = {
|
|
3317
|
+
unit.get("id"): unit
|
|
3318
|
+
for unit in lang_file_element.findall("x:unit", namespaces=NSMAP)
|
|
3319
|
+
}
|
|
3320
|
+
for unit in list(lang_file_element.findall("x:unit", namespaces=NSMAP)):
|
|
3321
|
+
lang_file_element.remove(unit)
|
|
3322
|
+
for unit_id in base_units.keys():
|
|
3323
|
+
if unit_id in current_units:
|
|
3324
|
+
lang_file_element.append(current_units[unit_id])
|
|
3325
|
+
|
|
3326
|
+
#TODO if a translation-unit has the "new"-attribute: set its value from the fallback-language. (if the culture contains a "-": e. g. take value from "de" as fallback-value for "de-AT"; or else: take value from base_file as fallback-value)
|
|
3327
|
+
|
|
3328
|
+
# Write file back to disk
|
|
3329
|
+
ET.register_namespace("", NS) # Ensure default namespace is declared without prefix
|
|
3330
|
+
Path(filepath).write_bytes(
|
|
3331
|
+
ET.tostring(
|
|
3332
|
+
lang_tree.getroot(),
|
|
3333
|
+
xml_declaration=True,
|
|
3334
|
+
encoding="UTF-8"
|
|
3335
|
+
)
|
|
3336
|
+
)
|
|
3337
|
+
ScriptCollectionCore().format_xml_file(filepath)
|
|
3338
|
+
|
|
3339
|
+
@GeneralUtilities.check_arguments
|
|
3340
|
+
def sync_xlf2_files(self,prefix:str, languages:list[str], folder:str):
|
|
3341
|
+
#languages=["de", "fr"] for example. the default-language (usually english) must not be included.
|
|
3342
|
+
base_file=os.path.join(folder, f"{prefix}.xlf")
|
|
3343
|
+
base_file_xml:ET.ElementTree=ET.parse(base_file)
|
|
3344
|
+
GeneralUtilities.assert_condition(self.is_xliff2_file(base_file), f"The base file '{base_file}' is not a valid XLIFF 2.0 file.")
|
|
3345
|
+
GeneralUtilities.assert_file_exists(base_file)
|
|
3346
|
+
if len(languages)==0:
|
|
3347
|
+
raise ValueError("No files provided for syncing.")
|
|
3348
|
+
if len(languages)==1:
|
|
3349
|
+
return
|
|
3350
|
+
language_files_list=[os.path.join(folder, f"{prefix}.{language}.xlf") for language in languages]
|
|
3351
|
+
language_files_with_content:dict[str,ET.ElementTree]=dict()
|
|
3352
|
+
for language_file in language_files_list:
|
|
3353
|
+
GeneralUtilities.assert_file_exists(language_file)
|
|
3354
|
+
GeneralUtilities.assert_condition(self.is_xliff2_file(language_file), f"The base file '{base_file}' is not a valid XLIFF 2.0 file.")
|
|
3355
|
+
language_files_with_content[language_file]=ET.parse(language_file)
|
|
3356
|
+
|
|
3357
|
+
#sync existing files
|
|
3358
|
+
self.__sync_xlf2_files(base_file_xml, language_files_with_content)
|
|
3359
|
+
|
|
3360
|
+
|
|
3361
|
+
@GeneralUtilities.check_arguments
|
|
3362
|
+
def translate_xlf_files_in_folder(self, folder: str, base_language: str, libre_translate_api_server: str):
|
|
3363
|
+
"""Translates all .xlf files directly in the given folder (non-recursive)."""
|
|
3364
|
+
pattern = re.compile(r'^.+\.[a-z]{2,3}\.xlf$')
|
|
3365
|
+
for filename in os.listdir(folder):
|
|
3366
|
+
if not pattern.match(filename):
|
|
3367
|
+
continue
|
|
3368
|
+
file_path = os.path.join(folder, filename)
|
|
3369
|
+
self.translate_xlf_file(file_path, base_language, libre_translate_api_server)
|
|
3370
|
+
|
|
3371
|
+
@GeneralUtilities.check_arguments
|
|
3372
|
+
def translate_xlf_file(self, file: str, base_language: str, libre_translate_api_server: str):
|
|
3373
|
+
"""
|
|
3374
|
+
Translates all segments with state='initial' in a XLIFF 2.0 file.
|
|
3375
|
+
The target language is extracted from the filename (e.g. 'messages.es.xlf' -> 'es').
|
|
3376
|
+
"""
|
|
3377
|
+
ns_uri = "urn:oasis:names:tc:xliff:document:2.0"
|
|
3378
|
+
ns = {"xliff": ns_uri}
|
|
3379
|
+
|
|
3380
|
+
filename = os.path.basename(file)
|
|
3381
|
+
parts = filename.split(".")
|
|
3382
|
+
if len(parts) < 3:
|
|
3383
|
+
raise ValueError(f"Cannot extract language from filename: {filename}")
|
|
3384
|
+
target_language = parts[-2]
|
|
3385
|
+
|
|
3386
|
+
tree = ET.parse(file)
|
|
3387
|
+
root = tree.getroot()
|
|
3388
|
+
|
|
3389
|
+
for segment in root.findall(".//xliff:segment", ns):
|
|
3390
|
+
if segment.get("state", "initial") != "initial":
|
|
3391
|
+
continue
|
|
3392
|
+
source_el = segment.find("xliff:source", ns)
|
|
3393
|
+
if source_el is None or not source_el.text:
|
|
3394
|
+
continue
|
|
3395
|
+
|
|
3396
|
+
translated_text = self.translate(source_el.text, base_language, target_language, libre_translate_api_server)
|
|
3397
|
+
|
|
3398
|
+
target_el = segment.find("xliff:target", ns)
|
|
3399
|
+
if target_el is None:
|
|
3400
|
+
target_el = ET.SubElement(segment, f"{{{ns_uri}}}target")
|
|
3401
|
+
|
|
3402
|
+
target_el.text = translated_text
|
|
3403
|
+
segment.set("state", "translated")
|
|
3404
|
+
|
|
3405
|
+
ET.register_namespace("", ns_uri)
|
|
3406
|
+
xml_bytes = ET.tostring(root, encoding="utf-8", xml_declaration=True)
|
|
3407
|
+
with open(file, "wb") as f:
|
|
3408
|
+
f.write(xml_bytes)
|
|
3409
|
+
|
|
3410
|
+
@GeneralUtilities.check_arguments
|
|
3411
|
+
def translate(self, content: str, source_language: str, target_language: str, libre_translate_api_server: str) -> str:
|
|
3412
|
+
"""Translates text using the LibreTranslate API."""
|
|
3413
|
+
url = f"{libre_translate_api_server.rstrip('/')}/translate"
|
|
3414
|
+
if "-" in source_language:
|
|
3415
|
+
source_language=source_language.split("-")[0]
|
|
3416
|
+
if "-" in target_language:
|
|
3417
|
+
target_language=target_language.split("-")[0]
|
|
3418
|
+
payload = {
|
|
3419
|
+
"q": content,
|
|
3420
|
+
"source": source_language,
|
|
3421
|
+
"target": target_language,
|
|
3422
|
+
"format": "text"
|
|
3423
|
+
}
|
|
3424
|
+
response = requests.post(url, json=payload, timeout=30)
|
|
3425
|
+
response.raise_for_status()
|
|
3426
|
+
return response.json()["translatedText"]
|
|
3427
|
+
|
|
3428
|
+
@GeneralUtilities.check_arguments
|
|
3429
|
+
def detect_language(self, content: str, libre_translate_api_server: str) -> str:
|
|
3430
|
+
"""Detects the language of the given text using the LibreTranslate API."""
|
|
3431
|
+
url = f"{libre_translate_api_server.rstrip('/')}/detect"
|
|
3432
|
+
payload = {"q": content}
|
|
3433
|
+
response = requests.post(url, json=payload, timeout=30)
|
|
3434
|
+
response.raise_for_status()
|
|
3435
|
+
results = response.json()
|
|
3436
|
+
if not results:
|
|
3437
|
+
raise ValueError("Language detection returned no results.")
|
|
3438
|
+
return results[0]["language"]
|
|
3439
|
+
|
|
3440
|
+
@GeneralUtilities.check_arguments
|
|
3441
|
+
def get_all_files_in_git_repository(self,repository_folder:str,ignore_ignored_files:bool=True,include_submodules: bool = True) -> list[str]:
|
|
3442
|
+
"""Returns a list of all files in a git-repository."""
|
|
3443
|
+
cmd = ["ls-files", "--cached"]
|
|
3444
|
+
if ignore_ignored_files:
|
|
3445
|
+
cmd.append("--exclude-standard")
|
|
3446
|
+
if include_submodules:
|
|
3447
|
+
cmd.append("--recurse-submodules")
|
|
3448
|
+
output=self.run_program_argsasarray("git", cmd,repository_folder)
|
|
3449
|
+
files = [ GeneralUtilities.normalize_path("./" + line) for line in output[1].splitlines()if line.strip() ]
|
|
3450
|
+
return files
|
|
3451
|
+
|
|
3452
|
+
@GeneralUtilities.check_arguments
|
|
3453
|
+
def write_file_list_for_repository(self,repository_folder:str,target_file:str="./FileList.txt",ignore_ignored_files:bool=True,include_submodules: bool = True) -> None:
|
|
3454
|
+
if os.path.isabs(target_file):
|
|
3455
|
+
target_file=GeneralUtilities.resolve_relative_path(target_file,repository_folder)
|
|
3456
|
+
target_file=GeneralUtilities.normalize_path(target_file)
|
|
3457
|
+
files=[path.replace("\\","/") for path in self.get_all_files_in_git_repository(repository_folder,ignore_ignored_files,include_submodules)]
|
|
3458
|
+
GeneralUtilities.ensure_file_exists(target_file)
|
|
3459
|
+
GeneralUtilities.write_lines_to_file(target_file, files)
|
|
3460
|
+
|
|
3461
|
+
@GeneralUtilities.check_arguments
|
|
3462
|
+
def get_all_commits_in_git_repository(self,repository_folder:str,include_all_heads:bool=False) -> list[str]:
|
|
3463
|
+
"""Returns a textual visualization of all commits in a git-repository."""
|
|
3464
|
+
#do 'git log --reverse --all --pretty=format:"%ci | %H | %cn <%ce> | %d | %s"'
|
|
3465
|
+
args = ["log", "--reverse", "--pretty=format:%ci | %H | %cn <%ce> | %D | %s"]
|
|
3466
|
+
if include_all_heads:
|
|
3467
|
+
args.append("--all")
|
|
3468
|
+
result=self.run_program_argsasarray("git", args, repository_folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
3469
|
+
output= result[1]
|
|
3470
|
+
result=output.splitlines()
|
|
3471
|
+
return result
|
|
3472
|
+
|
|
3473
|
+
@GeneralUtilities.check_arguments
|
|
3474
|
+
def write_commit_list_for_repository(self,repository_folder:str,target_file:str,include_all_heads:bool=False) -> None:
|
|
3475
|
+
if os.path.isabs(target_file):
|
|
3476
|
+
target_file=GeneralUtilities.resolve_relative_path(target_file,repository_folder)
|
|
3477
|
+
target_file=GeneralUtilities.normalize_path(target_file)
|
|
3478
|
+
commits=self.get_all_commits_in_git_repository(repository_folder, include_all_heads)
|
|
3479
|
+
GeneralUtilities.ensure_file_exists(target_file)
|
|
3480
|
+
GeneralUtilities.write_lines_to_file(target_file, commits)
|
|
3481
|
+
|
|
3482
|
+
@GeneralUtilities.check_arguments
|
|
3483
|
+
def is_runnning_in_container(self) ->bool:
|
|
3484
|
+
"""this function is based on a convention and does not do a real check."""
|
|
3485
|
+
return os.environ.get("ISRUNNINGINCONTAINER") == "true"
|