scriptcollection 4.2.81__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/AnionBuildPlatform.py +199 -0
- ScriptCollection/CertificateUpdater.py +149 -0
- ScriptCollection/Executables.py +921 -0
- ScriptCollection/GeneralUtilities.py +1589 -0
- ScriptCollection/HTTPMaintenanceOverheadHelper.py +36 -0
- ScriptCollection/OCIImages/AbstractImageHandler.py +38 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGeneric.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGenericV.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabCE.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabEE.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/__init__.py +0 -0
- ScriptCollection/OCIImages/OCIImageManager.py +190 -0
- ScriptCollection/OCIImages/__init__.py +0 -0
- ScriptCollection/ProcessesRunner.py +43 -0
- ScriptCollection/ProgramRunnerBase.py +47 -0
- ScriptCollection/ProgramRunnerMock.py +2 -0
- ScriptCollection/ProgramRunnerPopen.py +57 -0
- ScriptCollection/ProgramRunnerSudo.py +108 -0
- ScriptCollection/Resources/CultureChooser/CultureChooser.js +29 -0
- ScriptCollection/Resources/CultureChooser/index.html +15 -0
- ScriptCollection/Resources/MaintenanceSite/MaintenanceSite.html +15 -0
- ScriptCollection/SCLog.py +115 -0
- ScriptCollection/ScriptCollectionCore.py +3485 -0
- ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +192 -0
- ScriptCollection/TFCPS/Docker/__init__.py +0 -0
- ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py +8 -0
- ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py +6 -0
- ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py +7 -0
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +547 -0
- ScriptCollection/TFCPS/DotNet/__init__.py +0 -0
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +137 -0
- ScriptCollection/TFCPS/Flutter/__init__.py +0 -0
- ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py +72 -0
- ScriptCollection/TFCPS/Go/__init__.py +0 -0
- ScriptCollection/TFCPS/Maven/TFCPS_CodeUnitSpecific_Maven.py +42 -0
- ScriptCollection/TFCPS/Maven/__init__.py +0 -0
- ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +232 -0
- ScriptCollection/TFCPS/NodeJS/__init__.py +0 -0
- ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +239 -0
- ScriptCollection/TFCPS/Python/__init__.py +0 -0
- ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py +42 -0
- ScriptCollection/TFCPS/Rust/__init__.py +0 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +433 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +135 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +301 -0
- ScriptCollection/TFCPS/TFCPS_CreateRelease.py +98 -0
- ScriptCollection/TFCPS/TFCPS_Generic.py +44 -0
- ScriptCollection/TFCPS/TFCPS_MergeToMain.py +128 -0
- ScriptCollection/TFCPS/TFCPS_MergeToStable.py +356 -0
- ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +48 -0
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +1565 -0
- ScriptCollection/TFCPS/__init__.py +0 -0
- ScriptCollection/__init__.py +0 -0
- ScriptCollection/__pycache__/GeneralUtilities.cpython-311.pyc +0 -0
- ScriptCollection/__pycache__/__init__.cpython-311.pyc +0 -0
- scriptcollection-4.2.81.dist-info/METADATA +169 -0
- scriptcollection-4.2.81.dist-info/RECORD +62 -0
- scriptcollection-4.2.81.dist-info/WHEEL +5 -0
- scriptcollection-4.2.81.dist-info/entry_points.txt +67 -0
- scriptcollection-4.2.81.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1565 @@
|
|
|
1
|
+
from datetime import datetime,timezone
|
|
2
|
+
from graphlib import TopologicalSorter
|
|
3
|
+
import math
|
|
4
|
+
import os
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import shutil
|
|
7
|
+
import zipfile
|
|
8
|
+
import tarfile
|
|
9
|
+
import time
|
|
10
|
+
import re
|
|
11
|
+
import sys
|
|
12
|
+
import json
|
|
13
|
+
import tempfile
|
|
14
|
+
import uuid
|
|
15
|
+
import urllib.request
|
|
16
|
+
from packaging import version
|
|
17
|
+
import requests
|
|
18
|
+
from lxml import etree
|
|
19
|
+
from ..GeneralUtilities import GeneralUtilities,Platform
|
|
20
|
+
from ..ScriptCollectionCore import ScriptCollectionCore,VSCodeWorkspaceShellTask
|
|
21
|
+
from ..SCLog import LogLevel
|
|
22
|
+
from ..OCIImages.AbstractImageHandler import AbstractImageHandler
|
|
23
|
+
from ..OCIImages.OCIImageManager import OCIImageManager
|
|
24
|
+
|
|
25
|
+
class TFCPS_Tools_General:
|
|
26
|
+
|
|
27
|
+
__sc:ScriptCollectionCore=None
|
|
28
|
+
oci_image_manager:OCIImageManager=None
|
|
29
|
+
|
|
30
|
+
def __init__(self,sc:ScriptCollectionCore):
|
|
31
|
+
self.__sc=sc
|
|
32
|
+
self.oci_image_manager=OCIImageManager(self.__sc)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@GeneralUtilities.check_arguments
|
|
36
|
+
def codeunit_is_enabled(self, codeunit_file: str) -> bool:
|
|
37
|
+
root: etree._ElementTree = etree.parse(codeunit_file)
|
|
38
|
+
return GeneralUtilities.string_to_boolean(str(root.xpath('//cps:codeunit/@enabled', namespaces={'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'})[0]))
|
|
39
|
+
|
|
40
|
+
@GeneralUtilities.check_arguments
|
|
41
|
+
def ensure_cyclonedxcli_is_available(self,enforce_update:bool) -> str:
|
|
42
|
+
local_resource_name="CycloneDXCLI"
|
|
43
|
+
self.ensure_file_from_github_assets_is_available_with_retry("CycloneDX", "cyclonedx-cli", local_resource_name,"cyclonedx-linux-arm64",lambda latest_version: "cyclonedx-linux-arm64",enforce_update=enforce_update)
|
|
44
|
+
self.ensure_file_from_github_assets_is_available_with_retry("CycloneDX", "cyclonedx-cli", local_resource_name,"cyclonedx-linux-x64",lambda latest_version: "cyclonedx-linux-x64",enforce_update=enforce_update)
|
|
45
|
+
self.ensure_file_from_github_assets_is_available_with_retry("CycloneDX", "cyclonedx-cli", local_resource_name,"cyclonedx-win-arm64.exe",lambda latest_version: "cyclonedx-win-arm64.exe",enforce_update=enforce_update)
|
|
46
|
+
self.ensure_file_from_github_assets_is_available_with_retry("CycloneDX", "cyclonedx-cli", local_resource_name, "cyclonedx-win-x64.exe",lambda latest_version: "cyclonedx-win-x64.exe",enforce_update=enforce_update)
|
|
47
|
+
|
|
48
|
+
resource_folder =os.path.join( self.__sc.get_global_cache_folder(),"Tools",local_resource_name)
|
|
49
|
+
|
|
50
|
+
is_x64:bool=GeneralUtilities.current_system_is_x64()
|
|
51
|
+
is_arm:bool=GeneralUtilities.current_system_is_arm64()
|
|
52
|
+
if GeneralUtilities.current_system_is_windows():
|
|
53
|
+
if is_x64:
|
|
54
|
+
return os.path.join(resource_folder, "cyclonedx-win-x64.exe")
|
|
55
|
+
elif is_arm:
|
|
56
|
+
return os.path.join(resource_folder, "cyclonedx-win-arm64.exe")
|
|
57
|
+
else:
|
|
58
|
+
raise ValueError("Unsupported architecture for cyclonedx-cli on windows.")
|
|
59
|
+
elif GeneralUtilities.current_system_is_linux():
|
|
60
|
+
if is_x64:
|
|
61
|
+
return os.path.join(resource_folder, "cyclonedx-linux-x64")
|
|
62
|
+
elif is_arm:
|
|
63
|
+
return os.path.join(resource_folder, "cyclonedx-linux-arm64")
|
|
64
|
+
else:
|
|
65
|
+
raise ValueError("Unsupported architecture for cyclonedx-cli on linux.")
|
|
66
|
+
else:
|
|
67
|
+
raise ValueError("Unsupported operating system for cyclonedx-cli.")
|
|
68
|
+
|
|
69
|
+
@GeneralUtilities.check_arguments
|
|
70
|
+
def ensure_file_from_github_assets_is_available_with_retry(self, githubuser: str, githubprojectname: str, local_resource_name: str, local_filename: str, get_filename_on_github, amount_of_attempts: int = 5,enforce_update:bool=False) -> str:
|
|
71
|
+
return GeneralUtilities.retry_action(lambda: self.ensure_file_from_github_assets_is_available(githubuser, githubprojectname, local_resource_name, local_filename, get_filename_on_github,enforce_update), amount_of_attempts)
|
|
72
|
+
|
|
73
|
+
@GeneralUtilities.check_arguments
|
|
74
|
+
def ensure_file_from_github_assets_is_available(self,githubuser: str, githubprojectname: str, local_resource_name: str, local_filename: str, get_filename_on_github,enforce_update:bool) -> str:
|
|
75
|
+
#TODO use or remove target_folder-parameter
|
|
76
|
+
resource_folder =os.path.join( self.__sc.get_global_cache_folder(),"Tools",local_resource_name)
|
|
77
|
+
file = f"{resource_folder}/{local_filename}"
|
|
78
|
+
file_exists = os.path.isfile(file)
|
|
79
|
+
if not file_exists:
|
|
80
|
+
self.__sc.log.log(f"Download Asset \"{githubuser}/{githubprojectname}: {local_resource_name}\" from GitHub to global cache...", LogLevel.Information)
|
|
81
|
+
GeneralUtilities.ensure_folder_exists_and_is_empty(resource_folder)
|
|
82
|
+
headers = { 'User-Agent': 'Mozilla/5.0'}
|
|
83
|
+
self.__add_github_api_key_if_available(headers)
|
|
84
|
+
url = f"https://api.github.com/repos/{githubuser}/{githubprojectname}/releases/latest"
|
|
85
|
+
self.__sc.log.log(f"Download \"{url}\"...", LogLevel.Debug)
|
|
86
|
+
time.sleep(2)
|
|
87
|
+
response = requests.get(url, headers=headers, allow_redirects=True, timeout=(10, 10))
|
|
88
|
+
response_json=response.json()
|
|
89
|
+
latest_version = response_json["tag_name"]
|
|
90
|
+
filename_on_github = get_filename_on_github(latest_version)
|
|
91
|
+
link = f"https://github.com/{githubuser}/{githubprojectname}/releases/download/{latest_version}/{filename_on_github}"
|
|
92
|
+
time.sleep(2)
|
|
93
|
+
with requests.get(link, headers=headers, stream=True, allow_redirects=True, timeout=(5, 600)) as r:
|
|
94
|
+
r.raise_for_status()
|
|
95
|
+
total_size = int(r.headers.get("Content-Length", 0))
|
|
96
|
+
downloaded = 0
|
|
97
|
+
with open(file, "wb") as f:
|
|
98
|
+
for chunk in r.iter_content(chunk_size=8192):
|
|
99
|
+
f.write(chunk)
|
|
100
|
+
show_progress: bool = False
|
|
101
|
+
if show_progress:
|
|
102
|
+
downloaded += len(chunk)
|
|
103
|
+
if total_size:
|
|
104
|
+
percent = downloaded / total_size * 100
|
|
105
|
+
sys.stdout.write(f"\rDownload: {percent:.2f}%")
|
|
106
|
+
sys.stdout.flush()
|
|
107
|
+
self.__sc.log.log(f"Downloaded \"{url}\".", LogLevel.Diagnostic)
|
|
108
|
+
GeneralUtilities.assert_file_exists(file)
|
|
109
|
+
return file
|
|
110
|
+
|
|
111
|
+
def __add_github_api_key_if_available(self, headers: dict):
|
|
112
|
+
token = os.getenv("GITHUB_TOKEN")
|
|
113
|
+
if token is not None:
|
|
114
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
115
|
+
else:
|
|
116
|
+
user_folder = str(Path.home())
|
|
117
|
+
github_token_file: str = str(os.path.join(user_folder, ".github", "token.txt"))
|
|
118
|
+
if os.path.isfile(github_token_file):
|
|
119
|
+
token = GeneralUtilities.read_text_from_file(github_token_file)
|
|
120
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
121
|
+
return headers
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@GeneralUtilities.check_arguments
|
|
125
|
+
def is_codeunit_folder(self, codeunit_folder: str) -> bool:
|
|
126
|
+
repo_folder = GeneralUtilities.resolve_relative_path("..", codeunit_folder)
|
|
127
|
+
if not self.__sc.is_git_repository(repo_folder):
|
|
128
|
+
return False
|
|
129
|
+
codeunit_name = os.path.basename(codeunit_folder)
|
|
130
|
+
codeunit_file: str = os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml")
|
|
131
|
+
if not os.path.isfile(codeunit_file):
|
|
132
|
+
return False
|
|
133
|
+
return True
|
|
134
|
+
|
|
135
|
+
@GeneralUtilities.check_arguments
|
|
136
|
+
def assert_is_codeunit_folder(self, codeunit_folder: str) -> str:
|
|
137
|
+
repo_folder = GeneralUtilities.resolve_relative_path("..", codeunit_folder)
|
|
138
|
+
if not self.__sc.is_git_repository(repo_folder):
|
|
139
|
+
raise ValueError(f"'{codeunit_folder}' can not be a valid codeunit-folder because '{repo_folder}' is not a git-repository.")
|
|
140
|
+
codeunit_name = os.path.basename(codeunit_folder)
|
|
141
|
+
codeunit_file: str = os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml")
|
|
142
|
+
if not os.path.isfile(codeunit_file):
|
|
143
|
+
raise ValueError(f"'{codeunit_folder}' is no codeunit-folder because '{codeunit_file}' does not exist.")
|
|
144
|
+
|
|
145
|
+
@GeneralUtilities.check_arguments
|
|
146
|
+
def get_codeunits(self, repository_folder: str, ignore_disabled_codeunits: bool = True) -> list[str]:
|
|
147
|
+
codeunits_with_dependent_codeunits: dict[str, set[str]] = dict[str, set[str]]()
|
|
148
|
+
subfolders = GeneralUtilities.get_direct_folders_of_folder(repository_folder)
|
|
149
|
+
for subfolder in subfolders:
|
|
150
|
+
codeunit_name: str = os.path.basename(subfolder)
|
|
151
|
+
codeunit_file = os.path.join(subfolder, f"{codeunit_name}.codeunit.xml")
|
|
152
|
+
if os.path.exists(codeunit_file):
|
|
153
|
+
if ignore_disabled_codeunits and not self.codeunit_is_enabled(codeunit_file):
|
|
154
|
+
continue
|
|
155
|
+
codeunits_with_dependent_codeunits[codeunit_name] = self.get_dependent_code_units(codeunit_file)
|
|
156
|
+
sorted_codeunits = self._internal_get_sorted_codeunits_by_dict(codeunits_with_dependent_codeunits)
|
|
157
|
+
#TODO show warning somehow for enabled codeunits which depends on ignored codeunits
|
|
158
|
+
return sorted_codeunits
|
|
159
|
+
|
|
160
|
+
@GeneralUtilities.check_arguments
|
|
161
|
+
def repository_has_codeunits(self, repository: str, ignore_disabled_codeunits: bool = True) -> bool:
|
|
162
|
+
return 0<len(self.get_codeunits(repository, ignore_disabled_codeunits))
|
|
163
|
+
|
|
164
|
+
@GeneralUtilities.check_arguments
|
|
165
|
+
def get_dependent_code_units(self, codeunit_file: str) -> list[str]:
|
|
166
|
+
root: etree._ElementTree = etree.parse(codeunit_file)
|
|
167
|
+
result = set(root.xpath('//cps:dependentcodeunit/text()', namespaces={'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'}))
|
|
168
|
+
result = sorted(result)
|
|
169
|
+
return result
|
|
170
|
+
|
|
171
|
+
@GeneralUtilities.check_arguments
|
|
172
|
+
def _internal_get_sorted_codeunits_by_dict(self, codeunits: dict[str, set[str]]) -> list[str]:
|
|
173
|
+
sorted_codeunits = {
|
|
174
|
+
node: sorted(codeunits[node])
|
|
175
|
+
for node in sorted(codeunits)
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
ts = TopologicalSorter()
|
|
179
|
+
for node, deps in sorted_codeunits.items():
|
|
180
|
+
ts.add(node, *deps)
|
|
181
|
+
|
|
182
|
+
result_typed = list(ts.static_order())
|
|
183
|
+
result = [str(item) for item in result_typed]
|
|
184
|
+
return result
|
|
185
|
+
|
|
186
|
+
@GeneralUtilities.check_arguments
|
|
187
|
+
def get_unsupported_versions(self, repository_folder: str, moment: datetime) -> list[tuple[str, datetime, datetime]]:
|
|
188
|
+
self.__sc.assert_is_git_repository(repository_folder)
|
|
189
|
+
result: list[tuple[str, datetime, datetime]] = list[tuple[str, datetime, datetime]]()
|
|
190
|
+
for entry in self.get_versions(repository_folder):
|
|
191
|
+
if not (entry[1] <= moment and moment <= entry[2]):
|
|
192
|
+
result.append(entry)
|
|
193
|
+
return result
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
@GeneralUtilities.check_arguments
|
|
197
|
+
def get_versions(self, repository_folder: str) -> list[tuple[str, datetime, datetime]]:
|
|
198
|
+
self.__sc.assert_is_git_repository(repository_folder)
|
|
199
|
+
folder = os.path.join(repository_folder, "Other", "Resources", "Support")
|
|
200
|
+
file = os.path.join(folder, "InformationAboutSupportedVersions.csv")
|
|
201
|
+
result: list[(str, datetime, datetime)] = list[(str, datetime, datetime)]()
|
|
202
|
+
if not os.path.isfile(file):
|
|
203
|
+
return result
|
|
204
|
+
entries = GeneralUtilities.read_csv_file(file, True)
|
|
205
|
+
for entry in entries:
|
|
206
|
+
d1 = GeneralUtilities.string_to_datetime(entry[1])
|
|
207
|
+
if d1.tzinfo is None:
|
|
208
|
+
d1 = d1.replace(tzinfo=timezone.utc)
|
|
209
|
+
d2 = GeneralUtilities.string_to_datetime(entry[2])
|
|
210
|
+
if d2.tzinfo is None:
|
|
211
|
+
d2 = d2.replace(tzinfo=timezone.utc)
|
|
212
|
+
result.append((entry[0], d1, d2))
|
|
213
|
+
return result
|
|
214
|
+
|
|
215
|
+
@GeneralUtilities.check_arguments
|
|
216
|
+
def dependent_codeunit_exists(self, repository: str, codeunit: str) -> None:
|
|
217
|
+
codeunit_file = f"{repository}/{codeunit}/{codeunit}.codeunit.xml"
|
|
218
|
+
return os.path.isfile(codeunit_file)
|
|
219
|
+
|
|
220
|
+
@GeneralUtilities.check_arguments
|
|
221
|
+
def get_all_authors_and_committers_of_repository(self, repository_folder: str, subfolder: str = None) -> list[tuple[str, str]]:
|
|
222
|
+
self.__sc.is_git_or_bare_git_repository(repository_folder)
|
|
223
|
+
space_character = "_"
|
|
224
|
+
if subfolder is None:
|
|
225
|
+
subfolder_argument = GeneralUtilities.empty_string
|
|
226
|
+
else:
|
|
227
|
+
subfolder_argument = f" -- {subfolder}"
|
|
228
|
+
log_result = self.__sc.run_program("git", f'log --pretty=%aN{space_character}%aE%n%cN{space_character}%cE HEAD{subfolder_argument}', repository_folder)
|
|
229
|
+
plain_content: list[str] = list(
|
|
230
|
+
set([line for line in log_result[1].split("\n") if len(line) > 0]))
|
|
231
|
+
result: list[tuple[str, str]] = []
|
|
232
|
+
for item in plain_content:
|
|
233
|
+
if len(re.findall(space_character, item)) == 1:
|
|
234
|
+
splitted = item.split(space_character)
|
|
235
|
+
result.append((splitted[0], splitted[1]))
|
|
236
|
+
else:
|
|
237
|
+
raise ValueError(f'Unexpected author: "{item}"')
|
|
238
|
+
return result
|
|
239
|
+
|
|
240
|
+
@GeneralUtilities.check_arguments
|
|
241
|
+
def copy_artifacts_from_dependent_code_units(self, repo_folder: str, codeunit_name: str) -> None:
|
|
242
|
+
codeunit_file = os.path.join(repo_folder, codeunit_name, codeunit_name + ".codeunit.xml")
|
|
243
|
+
dependent_codeunits = self.get_dependent_code_units(codeunit_file)
|
|
244
|
+
if len(dependent_codeunits) > 0:
|
|
245
|
+
self.__sc.log.log(f"Get dependent artifacts for codeunit {codeunit_name}.")
|
|
246
|
+
dependent_codeunits_folder = os.path.join(repo_folder, codeunit_name, "Other", "Resources", "DependentCodeUnits")
|
|
247
|
+
GeneralUtilities.ensure_directory_does_not_exist(dependent_codeunits_folder)
|
|
248
|
+
for dependent_codeunit in dependent_codeunits:
|
|
249
|
+
target_folder = os.path.join(dependent_codeunits_folder, dependent_codeunit)
|
|
250
|
+
GeneralUtilities.ensure_directory_does_not_exist(target_folder)
|
|
251
|
+
other_folder = os.path.join(repo_folder, dependent_codeunit, "Other")
|
|
252
|
+
artifacts_folder = os.path.join(other_folder, "Artifacts")
|
|
253
|
+
GeneralUtilities.ensure_directory_exists(artifacts_folder)
|
|
254
|
+
GeneralUtilities.copy_content_of_folder(artifacts_folder,target_folder)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
@GeneralUtilities.check_arguments
|
|
258
|
+
def write_version_to_codeunit_file(self, codeunit_file: str, current_version: str) -> None:
|
|
259
|
+
versionregex = "\\d+\\.\\d+\\.\\d+"
|
|
260
|
+
versiononlyregex = f"^{versionregex}$"
|
|
261
|
+
pattern = re.compile(versiononlyregex)
|
|
262
|
+
if pattern.match(current_version):
|
|
263
|
+
GeneralUtilities.write_text_to_file(codeunit_file, re.sub(f"<cps:version>{versionregex}<\\/cps:version>", f"<cps:version>{current_version}</cps:version>", GeneralUtilities.read_text_from_file(codeunit_file)))
|
|
264
|
+
else:
|
|
265
|
+
raise ValueError(f"Version '{current_version}' does not match version-regex '{versiononlyregex}'.")
|
|
266
|
+
|
|
267
|
+
@GeneralUtilities.check_arguments
|
|
268
|
+
def set_default_constants(self, codeunit_folder: str) -> None:
|
|
269
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
270
|
+
self.set_constant_for_curenttimestamp(codeunit_folder)
|
|
271
|
+
self.set_constant_for_commitid(codeunit_folder)
|
|
272
|
+
self.set_constant_for_commitdate(codeunit_folder)
|
|
273
|
+
self.set_constant_for_codeunitname(codeunit_folder)
|
|
274
|
+
self.set_constant_for_codeunitversion(codeunit_folder)
|
|
275
|
+
self.set_constant_for_codeunitmajorversion(codeunit_folder)
|
|
276
|
+
self.set_constant_for_description(codeunit_folder)
|
|
277
|
+
|
|
278
|
+
@GeneralUtilities.check_arguments
|
|
279
|
+
def set_constant_for_curenttimestamp(self, codeunit_folder: str) -> None:
|
|
280
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
281
|
+
timestamp = GeneralUtilities.datetime_to_string_for_logfile_entry(GeneralUtilities.get_now().astimezone(timezone.utc),False)
|
|
282
|
+
self.set_constant(codeunit_folder, "CurrentTimestamp", timestamp)
|
|
283
|
+
|
|
284
|
+
@GeneralUtilities.check_arguments
|
|
285
|
+
def set_constant_for_commitid(self, codeunit_folder: str) -> None:
|
|
286
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
287
|
+
repository = GeneralUtilities.resolve_relative_path("..", codeunit_folder)
|
|
288
|
+
commit_id = self.__sc.git_get_commit_id(repository)
|
|
289
|
+
self.set_constant(codeunit_folder, "CommitId", commit_id)
|
|
290
|
+
|
|
291
|
+
@GeneralUtilities.check_arguments
|
|
292
|
+
def set_constant_for_commitdate(self, codeunit_folder: str) -> None:
|
|
293
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
294
|
+
repository = GeneralUtilities.resolve_relative_path("..", codeunit_folder)
|
|
295
|
+
commit_date: datetime = self.__sc.git_get_commit_date(repository)
|
|
296
|
+
self.set_constant(codeunit_folder, "CommitDate", GeneralUtilities.datetime_to_string(commit_date))
|
|
297
|
+
|
|
298
|
+
@GeneralUtilities.check_arguments
|
|
299
|
+
def set_constant_for_codeunitname(self, codeunit_folder: str) -> None:
|
|
300
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
301
|
+
codeunit_name: str = os.path.basename(codeunit_folder)
|
|
302
|
+
self.set_constant(codeunit_folder, "CodeUnitName", codeunit_name)
|
|
303
|
+
|
|
304
|
+
@GeneralUtilities.check_arguments
|
|
305
|
+
def set_constant_for_codeunitversion(self, codeunit_folder: str) -> None:
|
|
306
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
307
|
+
codeunit_version: str = self.get_version_of_codeunit(os.path.join(codeunit_folder,f"{os.path.basename(codeunit_folder)}.codeunit.xml"))
|
|
308
|
+
self.set_constant(codeunit_folder, "CodeUnitVersion", codeunit_version)
|
|
309
|
+
|
|
310
|
+
@GeneralUtilities.check_arguments
|
|
311
|
+
def set_constant_for_codeunitmajorversion(self, codeunit_folder: str) -> None:
|
|
312
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
313
|
+
major_version = int(self.get_version_of_codeunit(os.path.join(codeunit_folder,f"{os.path.basename(codeunit_folder)}.codeunit.xml")).split(".")[0])
|
|
314
|
+
self.set_constant(codeunit_folder, "CodeUnitMajorVersion", str(major_version))
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
@GeneralUtilities.check_arguments
|
|
318
|
+
def get_version_of_codeunit(self,codeunit_file:str) -> None:
|
|
319
|
+
codeunit_file_content:str=GeneralUtilities.read_text_from_file(codeunit_file)
|
|
320
|
+
return self.get_version_of_codeunit_filecontent(codeunit_file_content)
|
|
321
|
+
|
|
322
|
+
@GeneralUtilities.check_arguments
|
|
323
|
+
def get_version_of_codeunit_filecontent(self,file_content:str) -> None:
|
|
324
|
+
root: etree._ElementTree = etree.fromstring(file_content.encode("utf-8"))
|
|
325
|
+
result = str(root.xpath('//cps:version/text()', namespaces={'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'})[0])
|
|
326
|
+
return result
|
|
327
|
+
|
|
328
|
+
@GeneralUtilities.check_arguments
|
|
329
|
+
def set_constant_for_description(self, codeunit_folder: str) -> None:
|
|
330
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
331
|
+
codeunit_file:str=os.path.join(codeunit_folder,f"{os.path.basename(codeunit_folder)}.codeunit.xml")
|
|
332
|
+
codeunit_description: str = self.get_codeunit_description(codeunit_file)
|
|
333
|
+
self.set_constant(codeunit_folder, "CodeUnitDescription", codeunit_description)
|
|
334
|
+
|
|
335
|
+
@GeneralUtilities.check_arguments
|
|
336
|
+
def get_codeunit_description(self,codeunit_file:str) -> bool:
|
|
337
|
+
root: etree._ElementTree = etree.parse(codeunit_file)
|
|
338
|
+
return str(root.xpath('//cps:properties/@description', namespaces={'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'})[0])
|
|
339
|
+
|
|
340
|
+
@GeneralUtilities.check_arguments
|
|
341
|
+
def set_constant(self, codeunit_folder: str, constantname: str, constant_value: str, documentationsummary: str = None, constants_valuefile: str = None) -> None:
|
|
342
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
343
|
+
if documentationsummary is None:
|
|
344
|
+
documentationsummary = GeneralUtilities.empty_string
|
|
345
|
+
constants_folder = os.path.join(codeunit_folder, "Other", "Resources", "Constants")
|
|
346
|
+
GeneralUtilities.ensure_directory_exists(constants_folder)
|
|
347
|
+
constants_metafile = os.path.join(constants_folder, f"{constantname}.constant.xml")
|
|
348
|
+
if constants_valuefile is None:
|
|
349
|
+
constants_valuefile_folder = constants_folder
|
|
350
|
+
constants_valuefile_name = f"{constantname}.value.txt"
|
|
351
|
+
constants_valuefiler_reference = f"./{constants_valuefile_name}"
|
|
352
|
+
else:
|
|
353
|
+
constants_valuefile_folder = os.path.dirname(constants_valuefile)
|
|
354
|
+
constants_valuefile_name = os.path.basename(constants_valuefile)
|
|
355
|
+
constants_valuefiler_reference = os.path.join(constants_valuefile_folder, constants_valuefile_name)
|
|
356
|
+
|
|
357
|
+
# TODO implement usage of self.reference_latest_version_of_xsd_when_generating_xml
|
|
358
|
+
GeneralUtilities.write_text_to_file(constants_metafile, f"""<?xml version="1.0" encoding="UTF-8" ?>
|
|
359
|
+
<cps:constant xmlns:cps="https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure" constantspecificationversion="1.1.0"
|
|
360
|
+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/raw/main/Conventions/RepositoryStructure/CommonProjectStructure/constant.xsd">
|
|
361
|
+
<cps:name>{constantname}</cps:name>
|
|
362
|
+
<cps:documentationsummary>{documentationsummary}</cps:documentationsummary>
|
|
363
|
+
<cps:path>{constants_valuefiler_reference}</cps:path>
|
|
364
|
+
</cps:constant>""")
|
|
365
|
+
# TODO validate generated xml against xsd
|
|
366
|
+
GeneralUtilities.write_text_to_file(os.path.join(constants_valuefile_folder, constants_valuefile_name), constant_value)
|
|
367
|
+
|
|
368
|
+
@GeneralUtilities.check_arguments
|
|
369
|
+
def get_constant_value(self, source_codeunit_folder: str, constant_name: str) -> str:
|
|
370
|
+
self.assert_is_codeunit_folder(source_codeunit_folder)
|
|
371
|
+
value_file_relative = self.__get_constant_helper(source_codeunit_folder, constant_name, "path")
|
|
372
|
+
value_file = GeneralUtilities.resolve_relative_path(value_file_relative, os.path.join(source_codeunit_folder, "Other", "Resources", "Constants"))
|
|
373
|
+
return GeneralUtilities.read_text_from_file(value_file)
|
|
374
|
+
|
|
375
|
+
@GeneralUtilities.check_arguments
|
|
376
|
+
def get_constant_documentation(self, source_codeunit_folder: str, constant_name: str) -> str:
|
|
377
|
+
self.assert_is_codeunit_folder(source_codeunit_folder)
|
|
378
|
+
return self.__get_constant_helper(source_codeunit_folder, constant_name, "documentationsummary")
|
|
379
|
+
|
|
380
|
+
@GeneralUtilities.check_arguments
|
|
381
|
+
def __get_constant_helper(self, source_codeunit_folder: str, constant_name: str, propertyname: str) -> str:
|
|
382
|
+
self.assert_is_codeunit_folder(source_codeunit_folder)
|
|
383
|
+
root: etree._ElementTree = etree.parse(os.path.join(source_codeunit_folder, "Other", "Resources", "Constants", f"{constant_name}.constant.xml"))
|
|
384
|
+
results = root.xpath(f'//cps:{propertyname}/text()', namespaces={
|
|
385
|
+
'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'
|
|
386
|
+
})
|
|
387
|
+
length = len(results)
|
|
388
|
+
if (length == 0):
|
|
389
|
+
return ""
|
|
390
|
+
elif length == 1:
|
|
391
|
+
return results[0]
|
|
392
|
+
else:
|
|
393
|
+
raise ValueError("Too many results found.")
|
|
394
|
+
|
|
395
|
+
@GeneralUtilities.check_arguments
|
|
396
|
+
def copy_licence_file(self, codeunit_folder: str) -> None:
|
|
397
|
+
folder_of_current_file = os.path.join(codeunit_folder,"Other")
|
|
398
|
+
license_file = GeneralUtilities.resolve_relative_path("../../License.txt", folder_of_current_file)
|
|
399
|
+
target_folder = GeneralUtilities.resolve_relative_path("Artifacts/License", folder_of_current_file)
|
|
400
|
+
GeneralUtilities.ensure_directory_exists(target_folder)
|
|
401
|
+
shutil.copy(license_file, target_folder)
|
|
402
|
+
|
|
403
|
+
@GeneralUtilities.check_arguments
|
|
404
|
+
def generate_diff_report(self, repository_folder: str, codeunit_name: str, current_version: str) -> None:
|
|
405
|
+
#TODO refactor this. if new changes (committed or uncommitted) since last git-tag: diff-report from last tag to "now". if no new changes (curren-commit==commit on a vx.y-tag): take diff from last tag to this tag
|
|
406
|
+
self.__sc.assert_is_git_repository(repository_folder)
|
|
407
|
+
codeunit_folder = os.path.join(repository_folder, codeunit_name)
|
|
408
|
+
target_folder = GeneralUtilities.resolve_relative_path("Other/Artifacts/DiffReport", codeunit_folder)
|
|
409
|
+
GeneralUtilities.ensure_directory_does_not_exist(target_folder)
|
|
410
|
+
GeneralUtilities.ensure_directory_exists(target_folder)
|
|
411
|
+
target_file_light = os.path.join(target_folder, "DiffReport.html").replace("\\", "/")
|
|
412
|
+
target_file_dark = os.path.join(target_folder, "DiffReportDark.html").replace("\\", "/")
|
|
413
|
+
src = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" # hash/id of empty git-tree
|
|
414
|
+
src_prefix = "Begin"
|
|
415
|
+
if self.__sc.get_current_git_branch_has_tag(repository_folder):
|
|
416
|
+
latest_tag = self.__sc.get_latest_git_tag(repository_folder)
|
|
417
|
+
src = self.__sc.git_get_commit_id(repository_folder, latest_tag)
|
|
418
|
+
src_prefix = latest_tag
|
|
419
|
+
dst = "HEAD"
|
|
420
|
+
dst_prefix = f"v{current_version}"
|
|
421
|
+
|
|
422
|
+
temp_file = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
|
|
423
|
+
try:
|
|
424
|
+
GeneralUtilities.ensure_file_does_not_exist(temp_file)
|
|
425
|
+
GeneralUtilities.write_text_to_file(temp_file, self.__sc.run_program("git", f'--no-pager diff --src-prefix={src_prefix}/ --dst-prefix={dst_prefix}/ {src} {dst} -- {codeunit_name}', repository_folder)[1])
|
|
426
|
+
styles:dict[str,str]={
|
|
427
|
+
"default":target_file_light,
|
|
428
|
+
"github-dark":target_file_dark
|
|
429
|
+
}
|
|
430
|
+
for style,target_file in styles.items():
|
|
431
|
+
self.__sc.run_program_argsasarray("pygmentize", ['-l', 'diff', '-f', 'html', '-O', 'full', '-o', target_file, '-P', f'style={style}', temp_file], repository_folder)
|
|
432
|
+
finally:
|
|
433
|
+
GeneralUtilities.ensure_file_does_not_exist(temp_file)
|
|
434
|
+
|
|
435
|
+
@GeneralUtilities.check_arguments
|
|
436
|
+
def get_version_of_project(self,repositoryfolder:str) -> str:
|
|
437
|
+
self.__sc.assert_is_git_repository(repositoryfolder)
|
|
438
|
+
return self.__sc.get_semver_version_from_gitversion(repositoryfolder)
|
|
439
|
+
|
|
440
|
+
@GeneralUtilities.check_arguments
|
|
441
|
+
def __try_calculate_changelog_message(self, repositoryfolder: str):
|
|
442
|
+
self.__sc.assert_is_git_repository(repositoryfolder)
|
|
443
|
+
message = self.__sc.run_program("git", "log -1 --pretty=%B", repositoryfolder)[1]
|
|
444
|
+
message = message.strip()
|
|
445
|
+
if len(message) == 0:
|
|
446
|
+
raise ValueError("No commit message found.")
|
|
447
|
+
return message
|
|
448
|
+
|
|
449
|
+
@GeneralUtilities.check_arguments
|
|
450
|
+
def create_changelog_entry(self, repositoryfolder: str, message: str, commit: bool, force: bool):
|
|
451
|
+
self.__sc.assert_is_git_repository(repositoryfolder)
|
|
452
|
+
if message is None:
|
|
453
|
+
try:
|
|
454
|
+
message=self.__try_calculate_changelog_message(repositoryfolder)
|
|
455
|
+
except:
|
|
456
|
+
message="Update."
|
|
457
|
+
random_file = os.path.join(repositoryfolder, str(uuid.uuid4()))
|
|
458
|
+
try:
|
|
459
|
+
if force and not self.__sc.git_repository_has_uncommitted_changes(repositoryfolder):
|
|
460
|
+
GeneralUtilities.ensure_file_exists(random_file)
|
|
461
|
+
current_version = self.get_version_of_project(repositoryfolder)
|
|
462
|
+
changelog_file = os.path.join(repositoryfolder, "Other", "Resources", "Changelog", f"v{current_version}.md")
|
|
463
|
+
if os.path.isfile(changelog_file):
|
|
464
|
+
self.__sc.log.log(f"Changelog-file '{changelog_file}' already exists.")
|
|
465
|
+
else:
|
|
466
|
+
GeneralUtilities.ensure_file_exists(changelog_file)
|
|
467
|
+
GeneralUtilities.write_text_to_file(changelog_file, f"""# Release notes
|
|
468
|
+
|
|
469
|
+
## Changes
|
|
470
|
+
|
|
471
|
+
- {message}
|
|
472
|
+
""")
|
|
473
|
+
finally:
|
|
474
|
+
GeneralUtilities.ensure_file_does_not_exist(random_file)
|
|
475
|
+
if commit:
|
|
476
|
+
self.__sc.git_commit(repositoryfolder, f"Added changelog-file for v{current_version}.")
|
|
477
|
+
|
|
478
|
+
@GeneralUtilities.check_arguments
|
|
479
|
+
def merge_sbom_file_from_dependent_codeunit_into_this(self,codeunit_folder: str, codeunitname:str,dependent_codeunit_name: str,use_cache:bool) -> None:
|
|
480
|
+
repository_folder = GeneralUtilities.resolve_relative_path("..", codeunit_folder)
|
|
481
|
+
dependent_codeunit_folder = os.path.join(repository_folder, dependent_codeunit_name).replace("\\", "/")
|
|
482
|
+
codeunit_file:str=os.path.join(codeunit_folder,f"{codeunitname}.codeunit.xml")
|
|
483
|
+
dependent_codeunit_file:str=os.path.join(dependent_codeunit_folder,f"{dependent_codeunit_name}.codeunit.xml")
|
|
484
|
+
sbom_file = f"{repository_folder}/{codeunitname}/Other/Artifacts/BOM/{codeunitname}.{self.get_version_of_codeunit(codeunit_file)}.sbom.xml"
|
|
485
|
+
dependent_sbom_file = f"{repository_folder}/{dependent_codeunit_name}/Other/Artifacts/BOM/{dependent_codeunit_name}.{self.get_version_of_codeunit(dependent_codeunit_file)}.sbom.xml"
|
|
486
|
+
self.merge_sbom_file(repository_folder, dependent_sbom_file, sbom_file,use_cache)
|
|
487
|
+
|
|
488
|
+
@GeneralUtilities.check_arguments
|
|
489
|
+
def merge_sbom_file(self, repository_folder: str, source_sbom_file_relative: str, target_sbom_file_relative: str,use_cache:bool) -> None:
|
|
490
|
+
GeneralUtilities.assert_file_exists(os.path.join(repository_folder, source_sbom_file_relative))
|
|
491
|
+
GeneralUtilities.assert_file_exists(os.path.join(repository_folder, target_sbom_file_relative))
|
|
492
|
+
target_original_sbom_file_relative = os.path.dirname(target_sbom_file_relative)+"/"+os.path.basename(target_sbom_file_relative)+".original.xml"
|
|
493
|
+
os.rename(os.path.join(repository_folder, target_sbom_file_relative), os.path.join(repository_folder, target_original_sbom_file_relative))
|
|
494
|
+
|
|
495
|
+
cyclonedx_exe:str=self.ensure_cyclonedxcli_is_available(not use_cache)
|
|
496
|
+
self.__sc.run_program(cyclonedx_exe, f"merge --input-files {source_sbom_file_relative} {target_original_sbom_file_relative} --output-file {target_sbom_file_relative}", repository_folder)
|
|
497
|
+
GeneralUtilities.ensure_file_does_not_exist(os.path.join(repository_folder, target_original_sbom_file_relative))
|
|
498
|
+
self.__sc.format_xml_file(os.path.join(repository_folder, target_sbom_file_relative))
|
|
499
|
+
|
|
500
|
+
@GeneralUtilities.check_arguments
|
|
501
|
+
def codeunit_has_testable_sourcecode(self,codeunit_file:str) -> bool:
|
|
502
|
+
self.assert_is_codeunit_folder(os.path.dirname(codeunit_file))
|
|
503
|
+
root: etree._ElementTree = etree.parse(codeunit_file)
|
|
504
|
+
return GeneralUtilities.string_to_boolean(str(root.xpath('//cps:properties/@codeunithastestablesourcecode', namespaces={'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'})[0]))
|
|
505
|
+
|
|
506
|
+
@GeneralUtilities.check_arguments
|
|
507
|
+
def codeunit_has_updatable_dependencies(self,codeunit_file:str) -> bool:
|
|
508
|
+
self.assert_is_codeunit_folder(os.path.dirname(codeunit_file))
|
|
509
|
+
root: etree._ElementTree = etree.parse(codeunit_file)
|
|
510
|
+
return GeneralUtilities.string_to_boolean(str(root.xpath('//cps:properties/@codeunithasupdatabledependencies', namespaces={'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'})[0]))
|
|
511
|
+
|
|
512
|
+
@GeneralUtilities.check_arguments
|
|
513
|
+
def get_codeunit_owner_emailaddress(self,codeunit_file:str) -> None:
|
|
514
|
+
self.assert_is_codeunit_folder(os.path.dirname(codeunit_file))
|
|
515
|
+
namespaces = {'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure', 'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
|
|
516
|
+
root: etree._ElementTree = etree.parse(codeunit_file)
|
|
517
|
+
result = root.xpath('//cps:codeunit/cps:codeunitowneremailaddress/text()', namespaces=namespaces)[0]
|
|
518
|
+
return result
|
|
519
|
+
|
|
520
|
+
@GeneralUtilities.check_arguments
|
|
521
|
+
def get_codeunit_owner_name(self,codeunit_file:str) -> None:
|
|
522
|
+
self.assert_is_codeunit_folder(os.path.dirname(codeunit_file))
|
|
523
|
+
namespaces = {'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure', 'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
|
|
524
|
+
root: etree._ElementTree = etree.parse(codeunit_file)
|
|
525
|
+
result = root.xpath('//cps:codeunit/cps:codeunitownername/text()', namespaces=namespaces)[0]
|
|
526
|
+
return result
|
|
527
|
+
|
|
528
|
+
@GeneralUtilities.check_arguments
|
|
529
|
+
def generate_svg_files_from_plantuml_files_for_repository(self, repository_folder: str,use_cache:bool) -> None:
|
|
530
|
+
self.__sc.log.log("Generate svg-files from plantuml-files...")
|
|
531
|
+
self.__sc.assert_is_git_repository(repository_folder)
|
|
532
|
+
plantuml_jar_file=self.ensure_plantuml_is_available(not use_cache)
|
|
533
|
+
target_folder = os.path.join(repository_folder, "Other", "Reference")
|
|
534
|
+
self.__generate_svg_files_from_plantuml(target_folder, plantuml_jar_file)
|
|
535
|
+
|
|
536
|
+
@GeneralUtilities.check_arguments
|
|
537
|
+
def generate_svg_files_from_plantuml_files_for_codeunit(self, codeunit_folder: str,use_cache:bool) -> None:
|
|
538
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
539
|
+
plantuml_jar_file=self.ensure_plantuml_is_available(not use_cache)
|
|
540
|
+
target_folder = os.path.join(codeunit_folder, "Other", "Reference")
|
|
541
|
+
self.__generate_svg_files_from_plantuml(target_folder, plantuml_jar_file)
|
|
542
|
+
|
|
543
|
+
@GeneralUtilities.check_arguments
|
|
544
|
+
def ensure_plantuml_is_available(self,enforce_update:bool) -> str:
|
|
545
|
+
return self.ensure_file_from_github_assets_is_available_with_retry("plantuml", "plantuml", "PlantUML", "plantuml.jar", lambda latest_version: "plantuml.jar",enforce_update=enforce_update)
|
|
546
|
+
|
|
547
|
+
@GeneralUtilities.check_arguments
|
|
548
|
+
def __generate_svg_files_from_plantuml(self, diagrams_files_folder: str, plantuml_jar_file: str) -> None:
|
|
549
|
+
for file in GeneralUtilities.get_all_files_of_folder(diagrams_files_folder):
|
|
550
|
+
if file.endswith(".plantuml"):
|
|
551
|
+
output_filename = self.get_output_filename_for_plantuml_filename(file)
|
|
552
|
+
argument = ['-jar',plantuml_jar_file, '-tsvg', os.path.basename(file)]
|
|
553
|
+
folder = os.path.dirname(file)
|
|
554
|
+
self.__sc.run_program_argsasarray("java", argument, folder)
|
|
555
|
+
result_file = folder+"/" + output_filename
|
|
556
|
+
GeneralUtilities.assert_file_exists(result_file)
|
|
557
|
+
self.__sc.format_xml_file(result_file)
|
|
558
|
+
|
|
559
|
+
@GeneralUtilities.check_arguments
|
|
560
|
+
def get_output_filename_for_plantuml_filename(self, plantuml_file: str) -> str:
|
|
561
|
+
for line in GeneralUtilities.read_lines_from_file(plantuml_file):
|
|
562
|
+
prefix = "@startuml "
|
|
563
|
+
if line.startswith(prefix):
|
|
564
|
+
title = line[len(prefix):]
|
|
565
|
+
return title+".svg"
|
|
566
|
+
return Path(plantuml_file).stem+".svg"
|
|
567
|
+
|
|
568
|
+
@GeneralUtilities.check_arguments
|
|
569
|
+
def generate_codeunits_overview_diagram(self, repository_folder: str) -> None:
|
|
570
|
+
self.__sc.log.log("Generate Codeunits-overview-diagram...")
|
|
571
|
+
self.__sc.assert_is_git_repository(repository_folder)
|
|
572
|
+
project_name: str = os.path.basename(repository_folder)
|
|
573
|
+
target_folder = os.path.join(repository_folder, "Other", "Reference", "Technical", "Diagrams")
|
|
574
|
+
GeneralUtilities.ensure_directory_exists(target_folder)
|
|
575
|
+
target_file = os.path.join(target_folder, "CodeUnits-Overview.plantuml")
|
|
576
|
+
lines = ["@startuml CodeUnits-Overview"]
|
|
577
|
+
lines.append(f"title CodeUnits of {project_name}")
|
|
578
|
+
|
|
579
|
+
codeunits = self.get_codeunits(repository_folder)
|
|
580
|
+
for codeunitname in codeunits:
|
|
581
|
+
codeunit_file: str = os.path.join(repository_folder, codeunitname, f"{codeunitname}.codeunit.xml")
|
|
582
|
+
|
|
583
|
+
description = self.get_codeunit_description(codeunit_file)
|
|
584
|
+
|
|
585
|
+
lines.append(GeneralUtilities.empty_string)
|
|
586
|
+
lines.append(f"[{codeunitname}]")
|
|
587
|
+
lines.append(f"note as {codeunitname}Note")
|
|
588
|
+
lines.append(f" {description}")
|
|
589
|
+
lines.append(f"end note")
|
|
590
|
+
lines.append(f"{codeunitname} .. {codeunitname}Note")
|
|
591
|
+
|
|
592
|
+
lines.append(GeneralUtilities.empty_string)
|
|
593
|
+
for codeunitname in codeunits:
|
|
594
|
+
codeunit_file: str = os.path.join(repository_folder, codeunitname, f"{codeunitname}.codeunit.xml")
|
|
595
|
+
dependent_codeunits = self.get_dependent_code_units(codeunit_file)
|
|
596
|
+
for dependent_codeunit in dependent_codeunits:
|
|
597
|
+
lines.append(f"{codeunitname} --> {dependent_codeunit}")
|
|
598
|
+
|
|
599
|
+
lines.append(GeneralUtilities.empty_string)
|
|
600
|
+
lines.append("@enduml")
|
|
601
|
+
|
|
602
|
+
GeneralUtilities.write_lines_to_file(target_file, lines)
|
|
603
|
+
|
|
604
|
+
@GeneralUtilities.check_arguments
|
|
605
|
+
def ensure_trufflehog_is_available(self,enforce_update:bool=False) -> dict[str,str]:
|
|
606
|
+
def download_and_extract(osname: str, osname_in_github_asset: str, extension: str):
|
|
607
|
+
resource_name: str = f"TruffleHog_{osname}"
|
|
608
|
+
zip_filename: str = f"{resource_name}.{extension}"
|
|
609
|
+
target_folder_unextracted = os.path.join(self.__sc.get_global_cache_folder(),"Tools",resource_name+"_Unextracted")
|
|
610
|
+
target_folder_extracted = os.path.join(self.__sc.get_global_cache_folder(),"Tools",resource_name)
|
|
611
|
+
update:bool=not os.path.isdir(target_folder_extracted) or GeneralUtilities.folder_is_empty(target_folder_extracted) or enforce_update
|
|
612
|
+
if update:
|
|
613
|
+
downloaded_file=self.ensure_file_from_github_assets_is_available_with_retry( "trufflesecurity", "trufflehog", resource_name+"_Unextracted", zip_filename, lambda latest_version: f"trufflehog_{latest_version[1:]}_{osname_in_github_asset}_amd64.tar.gz",enforce_update=enforce_update)
|
|
614
|
+
#TODO add option to also download arm-version
|
|
615
|
+
local_zip_file: str = downloaded_file
|
|
616
|
+
GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder_extracted)
|
|
617
|
+
if extension == "zip":
|
|
618
|
+
with zipfile.ZipFile(local_zip_file, 'r') as zip_ref:
|
|
619
|
+
zip_ref.extractall(target_folder_extracted)
|
|
620
|
+
elif extension == "tar.gz":
|
|
621
|
+
with tarfile.open(local_zip_file, "r:gz") as tar:
|
|
622
|
+
tar.extractall(path=target_folder_extracted)
|
|
623
|
+
else:
|
|
624
|
+
raise ValueError(f"Unknown extension: \"{extension}\"")
|
|
625
|
+
GeneralUtilities.ensure_directory_does_not_exist(target_folder_unextracted)
|
|
626
|
+
GeneralUtilities.assert_folder_exists(target_folder_extracted)
|
|
627
|
+
executable=[f for f in GeneralUtilities.get_all_files_of_folder(target_folder_extracted) if os.path.basename(f).startswith("trufflehog")][0]
|
|
628
|
+
return executable
|
|
629
|
+
|
|
630
|
+
result=dict[str,str]()
|
|
631
|
+
result["Windows"]=download_and_extract("Windows", "windows", "tar.gz")
|
|
632
|
+
result["Linux"]=download_and_extract("Linux", "linux", "tar.gz")
|
|
633
|
+
result["MacOS"]=download_and_extract("MacOS", "darwin", "tar.gz")
|
|
634
|
+
return result
|
|
635
|
+
|
|
636
|
+
@GeneralUtilities.check_arguments
|
|
637
|
+
def generate_tasksfile_from_workspace_file(self, repository_folder: str, append_cli_args_at_end: bool = False) -> None:
|
|
638
|
+
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
639
|
+
if self.__sc.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
|
|
640
|
+
self.__sc.log.log("Generate taskfile from code-workspace-file...")
|
|
641
|
+
self.__sc.assert_is_git_repository(repository_folder)
|
|
642
|
+
workspace_file: str = self.__sc.find_file_by_extension(repository_folder, "code-workspace")
|
|
643
|
+
task_file: str = repository_folder + "/Taskfile.yml"
|
|
644
|
+
lines: list[str] = [
|
|
645
|
+
"version: '3'", GeneralUtilities.empty_string,
|
|
646
|
+
"tasks:", GeneralUtilities.empty_string,
|
|
647
|
+
]
|
|
648
|
+
tasks = self.__sc.parse_tasks_from_codeworkspace_file(workspace_file)
|
|
649
|
+
tasks.sort(key=lambda task: task.label, reverse=False)
|
|
650
|
+
for t in tasks:
|
|
651
|
+
task:VSCodeWorkspaceShellTask = t
|
|
652
|
+
lines.append(f" {GeneralUtilities.escape_yaml_property_value(task.label)}:")
|
|
653
|
+
if task.description is not None:
|
|
654
|
+
lines.append(f' desc: "{GeneralUtilities.escape_yaml_string_value(task.description)}"')
|
|
655
|
+
lines.append(' silent: true')
|
|
656
|
+
if task.work_dir is not None:
|
|
657
|
+
lines.append(f' dir: "{GeneralUtilities.escape_yaml_string_value(task.work_dir)}"')
|
|
658
|
+
lines.append(" cmds:")
|
|
659
|
+
command=GeneralUtilities.escape_yaml_string_value(task.command)
|
|
660
|
+
if task.allow_custom_arguments:
|
|
661
|
+
command=command+" {{.CLI_ARGS}}"
|
|
662
|
+
lines.append(f' - "{command}"')
|
|
663
|
+
if task.aliases!=None and len(task.aliases) > 0:
|
|
664
|
+
lines.append(" aliases:")
|
|
665
|
+
for alias in task.aliases:
|
|
666
|
+
lines.append(f' - {GeneralUtilities.escape_yaml_property_value(alias)}')
|
|
667
|
+
lines.append(GeneralUtilities.empty_string)
|
|
668
|
+
self.__sc.set_file_content(task_file, "\n".join(lines))
|
|
669
|
+
else:
|
|
670
|
+
self.__sc.run_program("scgeneratetasksfilefromworkspacefile", f"--repositoryfolder {repository_folder}")
|
|
671
|
+
|
|
672
|
+
@GeneralUtilities.check_arguments
|
|
673
|
+
def ensure_androidappbundletool_is_available(self, target_folder: str,enforce_update:bool) -> str:
|
|
674
|
+
return self.ensure_file_from_github_assets_is_available_with_retry( "google", "bundletool", "AndroidAppBundleTool", "bundletool.jar", lambda latest_version: f"bundletool-all-{latest_version}.jar",enforce_update=enforce_update)
|
|
675
|
+
|
|
676
|
+
@GeneralUtilities.check_arguments
|
|
677
|
+
def ensure_mediamtx_is_available(self, target_folder: str,enforce_update:bool) -> None:
|
|
678
|
+
def download_and_extract(osname: str, osname_in_github_asset: str, extension: str,architecture:Platform):
|
|
679
|
+
resource_name: str = f"MediaMTX_{GeneralUtilities.platform_to_dash_str(architecture)}"
|
|
680
|
+
resource_folder: str = os.path.join(target_folder, "Other", "Resources", resource_name)
|
|
681
|
+
target_folder_extracted = os.path.join(resource_folder, "MediaMTX")
|
|
682
|
+
update:bool=not os.path.isdir(target_folder_extracted) or GeneralUtilities.folder_is_empty(target_folder_extracted) or enforce_update
|
|
683
|
+
if update:
|
|
684
|
+
platform_str:str=None
|
|
685
|
+
match architecture:
|
|
686
|
+
case Platform.Windows_AMD64:
|
|
687
|
+
platform_str = "windows_amd64"
|
|
688
|
+
case Platform.Linux_ARM64:
|
|
689
|
+
platform_str = "linux_arm64"
|
|
690
|
+
case Platform.Linux_AMD64:
|
|
691
|
+
platform_str = "linux_amd64"
|
|
692
|
+
case Platform.MacOS_ARM64:
|
|
693
|
+
platform_str = "darwin_arm64"
|
|
694
|
+
case _:
|
|
695
|
+
raise ValueError(f"Unknown platform: {str(architecture)}")
|
|
696
|
+
|
|
697
|
+
resource_filename_name_remote:str=f"mediamtx_{platform_str}.{extension}"
|
|
698
|
+
resource_name_local:str=f"MediaMTCX_{platform_str}"
|
|
699
|
+
global_cache_file=os.path.join( self.__sc.get_global_cache_folder(),"Tools",resource_name_local,resource_filename_name_remote)
|
|
700
|
+
if (not os.path.isfile( global_cache_file )) or enforce_update:
|
|
701
|
+
self.ensure_file_from_github_assets_is_available_with_retry( "bluenviron", "mediamtx", resource_name_local, resource_filename_name_remote, lambda latest_version: f"mediamtx_{latest_version}_{platform_str}.{extension}",enforce_update=enforce_update)
|
|
702
|
+
GeneralUtilities.assert_file_exists(global_cache_file)
|
|
703
|
+
GeneralUtilities.assert_file_exists(global_cache_file)
|
|
704
|
+
GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder_extracted)
|
|
705
|
+
if extension == "zip":
|
|
706
|
+
with zipfile.ZipFile(global_cache_file, 'r') as zip_ref:
|
|
707
|
+
zip_ref.extractall(target_folder_extracted)
|
|
708
|
+
elif extension == "tar.gz":
|
|
709
|
+
with tarfile.open(global_cache_file, "r:gz") as tar:
|
|
710
|
+
tar.extractall(path=target_folder_extracted)
|
|
711
|
+
else:
|
|
712
|
+
raise ValueError(f"Unknown extension: \"{extension}\"")
|
|
713
|
+
|
|
714
|
+
download_and_extract("Windows", "windows", "zip",Platform.Windows_AMD64)
|
|
715
|
+
download_and_extract("Linux", "linux", "tar.gz",Platform.Linux_AMD64)
|
|
716
|
+
download_and_extract("Linux", "linux", "tar.gz",Platform.Linux_ARM64)
|
|
717
|
+
download_and_extract("MacOS", "darwin", "tar.gz",Platform.MacOS_ARM64)
|
|
718
|
+
|
|
719
|
+
@GeneralUtilities.check_arguments
|
|
720
|
+
def clone_repository_as_resource(self, local_repository_folder: str, remote_repository_link: str, resource_name: str, repository_subname: str = None,use_cache:bool=True) -> None:
|
|
721
|
+
self.__sc.log.log(f'Clone resource {resource_name}...')
|
|
722
|
+
resrepo_commit_id_folder: str = os.path.join(local_repository_folder, "Other", "Resources", f"{resource_name}Version")
|
|
723
|
+
resrepo_commit_id_file: str = os.path.join(resrepo_commit_id_folder, f"{resource_name}Version.txt")
|
|
724
|
+
latest_version: str = GeneralUtilities.read_text_from_file(resrepo_commit_id_file)
|
|
725
|
+
resrepo_data_folder: str = os.path.join(local_repository_folder, "Other", "Resources", resource_name).replace("\\", "/")
|
|
726
|
+
current_version: str = None
|
|
727
|
+
resrepo_data_version: str = os.path.join(resrepo_data_folder, f"{resource_name}Version.txt")
|
|
728
|
+
if os.path.isdir(resrepo_data_folder):
|
|
729
|
+
if os.path.isfile(resrepo_data_version):
|
|
730
|
+
current_version = GeneralUtilities.read_text_from_file(resrepo_data_version)
|
|
731
|
+
if (current_version is None) or (current_version != latest_version):
|
|
732
|
+
target_folder: str = resrepo_data_folder
|
|
733
|
+
if repository_subname is not None:
|
|
734
|
+
target_folder = f"{resrepo_data_folder}/{repository_subname}"
|
|
735
|
+
|
|
736
|
+
update:bool=not os.path.isdir(target_folder) or GeneralUtilities.folder_is_empty(target_folder) or not use_cache
|
|
737
|
+
if update:
|
|
738
|
+
self.__sc.log.log(f"Clone {remote_repository_link} as resource...", LogLevel.Information)
|
|
739
|
+
GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder)
|
|
740
|
+
self.__sc.run_program("git", f"clone --recurse-submodules {remote_repository_link} {target_folder}")
|
|
741
|
+
self.__sc.run_program("git", f"checkout {latest_version}", target_folder)
|
|
742
|
+
GeneralUtilities.write_text_to_file(resrepo_data_version, latest_version)
|
|
743
|
+
|
|
744
|
+
git_folders: list[str] = []
|
|
745
|
+
git_files: list[str] = []
|
|
746
|
+
for dirpath, dirnames, filenames in os.walk(target_folder):
|
|
747
|
+
for dirname in dirnames:
|
|
748
|
+
if dirname == ".git":
|
|
749
|
+
full_path = os.path.join(dirpath, dirname)
|
|
750
|
+
git_folders.append(full_path)
|
|
751
|
+
for filename in filenames:
|
|
752
|
+
if filename == ".git":
|
|
753
|
+
full_path = os.path.join(dirpath, filename)
|
|
754
|
+
git_files.append(full_path)
|
|
755
|
+
for git_folder in git_folders:
|
|
756
|
+
if os.path.isdir(git_folder):
|
|
757
|
+
GeneralUtilities.ensure_directory_does_not_exist(git_folder)
|
|
758
|
+
for git_file in git_files:
|
|
759
|
+
if os.path.isdir(git_file):
|
|
760
|
+
GeneralUtilities.ensure_file_does_not_exist(git_file)
|
|
761
|
+
|
|
762
|
+
@GeneralUtilities.check_arguments
|
|
763
|
+
def ensure_certificate_authority_for_development_purposes_is_generated(self, product_folder: str):
|
|
764
|
+
product_name: str = os.path.basename(product_folder)
|
|
765
|
+
now = GeneralUtilities.get_now()
|
|
766
|
+
ca_name = f"{product_name}CA_{now.year:04}{now.month:02}{now.day:02}{now.hour:02}{now.min:02}{now.second:02}"
|
|
767
|
+
ca_folder = os.path.join(product_folder, "Other", "Resources", "CA")
|
|
768
|
+
generate_certificate = True
|
|
769
|
+
if os.path.isdir(ca_folder):
|
|
770
|
+
ca_files = [file for file in GeneralUtilities.get_direct_files_of_folder(ca_folder) if file.endswith(".crt")]
|
|
771
|
+
if len(ca_files) > 0:
|
|
772
|
+
ca_file = ca_files[-1] # pylint:disable=unused-variable
|
|
773
|
+
certificate_is_valid = True # TODO check if certificate is really valid
|
|
774
|
+
generate_certificate = not certificate_is_valid
|
|
775
|
+
if generate_certificate:
|
|
776
|
+
self.__sc.generate_certificate_authority(ca_folder, ca_name, "DE", "SubjST", "SubjL", "SubjO", "SubjOU")
|
|
777
|
+
# TODO add switch to auto-install the script if desired
|
|
778
|
+
# for windows: powershell Import-Certificate -FilePath MyProjectCA_20241121000236.crt -CertStoreLocation 'Cert:\CurrentUser\Root'
|
|
779
|
+
# for linux: (TODO)
|
|
780
|
+
|
|
781
|
+
@GeneralUtilities.check_arguments
|
|
782
|
+
def generate_certificate_for_development_purposes_for_product(self, repository_folder: str):
|
|
783
|
+
self.__sc.assert_is_git_repository(repository_folder)
|
|
784
|
+
product_name = os.path.basename(repository_folder)
|
|
785
|
+
ca_folder: str = os.path.join(repository_folder, "Other", "Resources", "CA")
|
|
786
|
+
self.__generate_certificate_for_development_purposes(product_name, os.path.join(repository_folder, "Other", "Resources"), ca_folder, None)
|
|
787
|
+
|
|
788
|
+
@GeneralUtilities.check_arguments
|
|
789
|
+
def __generate_certificate_for_development_purposes(self, service_name: str, resources_folder: str, ca_folder: str, domain: str = None):
|
|
790
|
+
if domain is None:
|
|
791
|
+
domain = f"{service_name}.test.local"
|
|
792
|
+
domain = domain.lower()
|
|
793
|
+
resource_name: str = "DevelopmentCertificate"
|
|
794
|
+
certificate_folder: str = os.path.join(resources_folder, resource_name)
|
|
795
|
+
|
|
796
|
+
resource_content_filename: str = service_name+resource_name
|
|
797
|
+
certificate_file = os.path.join(certificate_folder, f"{domain}.crt")
|
|
798
|
+
unsignedcertificate_file = os.path.join(certificate_folder, f"{domain}.unsigned.crt")
|
|
799
|
+
certificate_exists = os.path.exists(certificate_file)
|
|
800
|
+
if certificate_exists:
|
|
801
|
+
certificate_expired = GeneralUtilities.certificate_is_expired(certificate_file)
|
|
802
|
+
generate_new_certificate = certificate_expired
|
|
803
|
+
else:
|
|
804
|
+
generate_new_certificate = True
|
|
805
|
+
if generate_new_certificate:
|
|
806
|
+
GeneralUtilities.ensure_directory_does_not_exist(certificate_folder)
|
|
807
|
+
GeneralUtilities.ensure_directory_exists(certificate_folder)
|
|
808
|
+
self.__sc.log.log("Generate TLS-certificate for development-purposes...")
|
|
809
|
+
self.__sc.generate_certificate(certificate_folder, domain, resource_content_filename, "DE", "SubjST", "SubjL", "SubjO", "SubjOU")
|
|
810
|
+
self.__sc.generate_certificate_sign_request(certificate_folder, domain, resource_content_filename, "DE", "SubjST", "SubjL", "SubjO", "SubjOU")
|
|
811
|
+
ca_name = os.path.basename(self.__sc.find_last_file_by_extension(ca_folder, "crt"))[:-4]
|
|
812
|
+
self.__sc.sign_certificate(certificate_folder, ca_folder, ca_name, domain, resource_content_filename)
|
|
813
|
+
GeneralUtilities.ensure_file_does_not_exist(unsignedcertificate_file)
|
|
814
|
+
self.__sc.log.log("Finished generating TLS-certificate for development-purposes...",LogLevel.Debug)
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
@GeneralUtilities.check_arguments
|
|
818
|
+
def do_npm_install(self, package_json_folder: str, npm_force: bool,use_cache:bool) -> None:
|
|
819
|
+
target_folder:str=os.path.join(package_json_folder,"node_modules")
|
|
820
|
+
update:bool=not os.path.isdir(target_folder) or GeneralUtilities.folder_is_empty(target_folder) or not use_cache
|
|
821
|
+
if update:
|
|
822
|
+
self.__sc.log.log("Do npm-install...")
|
|
823
|
+
argument1 = "install"
|
|
824
|
+
if npm_force:
|
|
825
|
+
argument1 = f"{argument1} --force"
|
|
826
|
+
self.__sc.run_with_epew("npm", argument1, package_json_folder)
|
|
827
|
+
|
|
828
|
+
argument2 = "install --package-lock-only"
|
|
829
|
+
if npm_force:
|
|
830
|
+
argument2 = f"{argument2} --force"
|
|
831
|
+
self.__sc.run_with_epew("npm", argument2, package_json_folder)
|
|
832
|
+
|
|
833
|
+
argument3 = "clean-install"
|
|
834
|
+
if npm_force:
|
|
835
|
+
argument3 = f"{argument3} --force"
|
|
836
|
+
self.__sc.run_with_epew("npm", argument3, package_json_folder)
|
|
837
|
+
|
|
838
|
+
@staticmethod
|
|
839
|
+
@GeneralUtilities.check_arguments
|
|
840
|
+
def sort_reference_folder(folder1: str, folder2: str) -> int:
|
|
841
|
+
"""Returns a value greater than 0 if and only if folder1 has a base-folder-name with a with a higher version than the base-folder-name of folder2.
|
|
842
|
+
Returns a value lower than 0 if and only if folder1 has a base-folder-name with a with a lower version than the base-folder-name of folder2.
|
|
843
|
+
Returns 0 if both values are equal."""
|
|
844
|
+
if (folder1 == folder2):
|
|
845
|
+
return 0
|
|
846
|
+
|
|
847
|
+
version_identifier_1 = os.path.basename(folder1)
|
|
848
|
+
if version_identifier_1 == "Latest":
|
|
849
|
+
return -1
|
|
850
|
+
version_identifier_1 = version_identifier_1[1:]
|
|
851
|
+
|
|
852
|
+
version_identifier_2 = os.path.basename(folder2)
|
|
853
|
+
if version_identifier_2 == "Latest":
|
|
854
|
+
return 1
|
|
855
|
+
version_identifier_2 = version_identifier_2[1:]
|
|
856
|
+
|
|
857
|
+
if version.parse(version_identifier_1) < version.parse(version_identifier_2):
|
|
858
|
+
return -1
|
|
859
|
+
elif version.parse(version_identifier_1) > version.parse(version_identifier_2):
|
|
860
|
+
return 1
|
|
861
|
+
else:
|
|
862
|
+
return 0
|
|
863
|
+
|
|
864
|
+
@GeneralUtilities.check_arguments
|
|
865
|
+
def t4_transform(self, codeunit_folder: str, ignore_git_ignored_files: bool ,use_cache:bool):
|
|
866
|
+
grylib_dll:str=self.__ensure_grylibrary_is_available(use_cache)
|
|
867
|
+
repository_folder: str = os.path.dirname(codeunit_folder)
|
|
868
|
+
codeunitname: str = os.path.basename(codeunit_folder)
|
|
869
|
+
codeunit_folder = os.path.join(repository_folder, codeunitname)
|
|
870
|
+
for search_result in Path(codeunit_folder).glob('**/*.tt'):
|
|
871
|
+
tt_file = str(search_result)
|
|
872
|
+
relative_path_to_tt_file_from_repository = str(Path(tt_file).relative_to(repository_folder))
|
|
873
|
+
if (not ignore_git_ignored_files) or (ignore_git_ignored_files and not self.__sc.file_is_git_ignored(relative_path_to_tt_file_from_repository, repository_folder)):
|
|
874
|
+
relative_path_to_tt_file_from_codeunit_file = str(Path(tt_file).relative_to(codeunit_folder))
|
|
875
|
+
argument = [f"--parameter=repositoryFolder={repository_folder}", f"--parameter=codeUnitName={codeunitname}", f"--parameter=gryLibraryDLLFile={grylib_dll}", relative_path_to_tt_file_from_codeunit_file]
|
|
876
|
+
self.__sc.run_program_argsasarray("t4", argument, codeunit_folder)
|
|
877
|
+
|
|
878
|
+
@GeneralUtilities.check_arguments
|
|
879
|
+
def __ensure_grylibrary_is_available(self, use_cache:bool) -> None:
|
|
880
|
+
grylibrary_folder =os.path.join( self.__sc.get_global_cache_folder(),"Tools","GRYLibrary")
|
|
881
|
+
grylibrary_dll_file = os.path.join(grylibrary_folder, "BuildResult_DotNet_win-x64", "GRYLibrary.dll")
|
|
882
|
+
grylibrary_dll_file_exists = os.path.isfile(grylibrary_dll_file)
|
|
883
|
+
if not os.path.isfile(grylibrary_dll_file):
|
|
884
|
+
self.__sc.log.log("Download GRYLibrary to global cache...",LogLevel.Information)
|
|
885
|
+
grylibrary_latest_codeunit_file = "https://raw.githubusercontent.com/anionDev/GRYLibrary/stable/GRYLibrary/GRYLibrary.codeunit.xml"
|
|
886
|
+
with urllib.request.urlopen(grylibrary_latest_codeunit_file) as url_result:
|
|
887
|
+
grylibrary_latest_version = self.get_version_of_codeunit_filecontent(url_result.read().decode("utf-8"))
|
|
888
|
+
if grylibrary_dll_file_exists:
|
|
889
|
+
grylibrary_existing_codeunit_file = os.path.join(grylibrary_folder, "SourceCode", "GRYLibrary.codeunit.xml")
|
|
890
|
+
grylibrary_existing_codeunit_version = self.get_version_of_codeunit(grylibrary_existing_codeunit_file)
|
|
891
|
+
if grylibrary_existing_codeunit_version != grylibrary_latest_version:
|
|
892
|
+
GeneralUtilities.ensure_directory_does_not_exist(grylibrary_folder)
|
|
893
|
+
GeneralUtilities.ensure_directory_does_not_exist(grylibrary_folder)
|
|
894
|
+
GeneralUtilities.ensure_directory_exists(grylibrary_folder)
|
|
895
|
+
archive_name = f"GRYLibrary.v{grylibrary_latest_version}.Artifacts.zip"
|
|
896
|
+
archive_download_link = f"https://github.com/anionDev/GRYLibrary/releases/download/v{grylibrary_latest_version}/{archive_name}"
|
|
897
|
+
archive_file = os.path.join(grylibrary_folder, archive_name)
|
|
898
|
+
urllib.request.urlretrieve(archive_download_link, archive_file)
|
|
899
|
+
with zipfile.ZipFile(archive_file, 'r') as zip_ref:
|
|
900
|
+
zip_ref.extractall(grylibrary_folder)
|
|
901
|
+
GeneralUtilities.ensure_file_does_not_exist(archive_file)
|
|
902
|
+
GeneralUtilities.assert_file_exists(grylibrary_dll_file)
|
|
903
|
+
return grylibrary_dll_file
|
|
904
|
+
|
|
905
|
+
@GeneralUtilities.check_arguments
|
|
906
|
+
def ensure_ffmpeg_is_available(self, codeunit_folder: str,use_cache:bool) -> None:
|
|
907
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
908
|
+
ffmpeg_folder = os.path.join(codeunit_folder, "Other", "Resources", "FFMPEG")
|
|
909
|
+
internet_connection_is_available = GeneralUtilities.internet_connection_is_available()
|
|
910
|
+
exe_file = f"{ffmpeg_folder}/ffmpeg.exe"
|
|
911
|
+
exe_file_exists = os.path.isfile(exe_file)
|
|
912
|
+
update:bool=(not exe_file_exists) or (not use_cache)
|
|
913
|
+
if update:
|
|
914
|
+
if internet_connection_is_available: # Load/Update
|
|
915
|
+
GeneralUtilities.ensure_directory_does_not_exist(ffmpeg_folder)
|
|
916
|
+
GeneralUtilities.ensure_directory_exists(ffmpeg_folder)
|
|
917
|
+
ffmpeg_temp_folder = ffmpeg_folder+"Temp"
|
|
918
|
+
GeneralUtilities.ensure_directory_does_not_exist(ffmpeg_temp_folder)
|
|
919
|
+
GeneralUtilities.ensure_directory_exists(ffmpeg_temp_folder)
|
|
920
|
+
zip_file_on_disk = os.path.join(ffmpeg_temp_folder, "ffmpeg.zip")
|
|
921
|
+
original_zip_filename = "ffmpeg-master-latest-win64-gpl-shared"
|
|
922
|
+
zip_link = f"https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/{original_zip_filename}.zip"
|
|
923
|
+
urllib.request.urlretrieve(zip_link, zip_file_on_disk)
|
|
924
|
+
shutil.unpack_archive(zip_file_on_disk, ffmpeg_temp_folder)
|
|
925
|
+
bin_folder_source = os.path.join(ffmpeg_temp_folder, "ffmpeg-master-latest-win64-gpl-shared/bin")
|
|
926
|
+
bin_folder_target = ffmpeg_folder
|
|
927
|
+
GeneralUtilities.copy_content_of_folder(bin_folder_source, bin_folder_target)
|
|
928
|
+
GeneralUtilities.ensure_directory_does_not_exist(ffmpeg_temp_folder)
|
|
929
|
+
else:
|
|
930
|
+
if exe_file_exists:
|
|
931
|
+
self.__sc.log.log("Can not check for updates of FFMPEG due to missing internet-connection.")
|
|
932
|
+
else:
|
|
933
|
+
raise ValueError("Can not download FFMPEG.")
|
|
934
|
+
|
|
935
|
+
@GeneralUtilities.check_arguments
|
|
936
|
+
def set_constants_for_certificate_private_information(self, codeunit_folder: str) -> None:
|
|
937
|
+
"""Expects a certificate-resource and generates a constant for its sensitive information in hex-format"""
|
|
938
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
939
|
+
repo_name:str=os.path.basename(GeneralUtilities.resolve_relative_path("..",codeunit_folder))
|
|
940
|
+
resource_name: str = "DevelopmentCertificate"
|
|
941
|
+
filename: str = repo_name+"DevelopmentCertificate"
|
|
942
|
+
self.generate_constant_from_resource_by_filename(codeunit_folder, resource_name, f"{filename}.pfx", "PFX")
|
|
943
|
+
self.generate_constant_from_resource_by_filename(codeunit_folder, resource_name, f"{filename}.password", "Password")
|
|
944
|
+
|
|
945
|
+
@GeneralUtilities.check_arguments
|
|
946
|
+
def generate_constant_from_resource_by_filename(self, codeunit_folder: str, resource_name: str, filename: str, constant_name: str) -> None:
|
|
947
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
948
|
+
certificate_resource_folder = GeneralUtilities.resolve_relative_path(f"Other/Resources/{resource_name}", codeunit_folder)
|
|
949
|
+
resource_file = os.path.join(certificate_resource_folder, filename)
|
|
950
|
+
resource_file_content = GeneralUtilities.read_binary_from_file(resource_file)
|
|
951
|
+
resource_file_as_hex = resource_file_content.hex()
|
|
952
|
+
self.set_constant(codeunit_folder, f"{resource_name}{constant_name}Hex", resource_file_as_hex)
|
|
953
|
+
|
|
954
|
+
@GeneralUtilities.check_arguments
|
|
955
|
+
def get_resource_from_global_resource(self, codeunit_folder: str, resource_name: str):
|
|
956
|
+
repository_folder: str = GeneralUtilities.resolve_relative_path("..", codeunit_folder)
|
|
957
|
+
source_folder: str = os.path.join(repository_folder, "Other", "Resources", resource_name)
|
|
958
|
+
target_folder: str = os.path.join(codeunit_folder, "Other", "Resources", resource_name)
|
|
959
|
+
GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder)
|
|
960
|
+
GeneralUtilities.copy_content_of_folder(source_folder, target_folder)
|
|
961
|
+
|
|
962
|
+
|
|
963
|
+
@GeneralUtilities.check_arguments
|
|
964
|
+
def merge_packages(self,coverage_file:str,package_name:str) -> None:
|
|
965
|
+
tree = etree.parse(coverage_file)
|
|
966
|
+
root = tree.getroot()
|
|
967
|
+
packages = root.findall("./packages/package")
|
|
968
|
+
all_classes = []
|
|
969
|
+
for pkg in packages:
|
|
970
|
+
pkg_name:str=pkg.get("name")
|
|
971
|
+
if len(packages)==1 or ( pkg_name==package_name or pkg_name.startswith(f"{package_name}.")):
|
|
972
|
+
classes = pkg.find("classes")
|
|
973
|
+
if classes is not None:
|
|
974
|
+
all_classes.extend(classes.findall("class"))
|
|
975
|
+
new_package = etree.Element("package", name=package_name)
|
|
976
|
+
new_classes = etree.SubElement(new_package, "classes")
|
|
977
|
+
for cls in all_classes:
|
|
978
|
+
new_classes.append(cls)
|
|
979
|
+
packages_node = root.find("./packages")
|
|
980
|
+
packages_node.clear()
|
|
981
|
+
packages_node.append(new_package)
|
|
982
|
+
tree.write(coverage_file, pretty_print=True, xml_declaration=True, encoding="UTF-8")
|
|
983
|
+
self.calculate_entire_line_rate(coverage_file)
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
@GeneralUtilities.check_arguments
|
|
987
|
+
def calculate_entire_line_rate(self,coverage_file:str) -> None:
|
|
988
|
+
tree = etree.parse(coverage_file)
|
|
989
|
+
root = tree.getroot()
|
|
990
|
+
package = root.find("./packages/package")
|
|
991
|
+
if package is None:
|
|
992
|
+
raise RuntimeError("No <package>-Element found")
|
|
993
|
+
|
|
994
|
+
line_elements = package.findall(".//line")
|
|
995
|
+
|
|
996
|
+
amount_of_lines = 0
|
|
997
|
+
amount_of_hited_lines = 0
|
|
998
|
+
|
|
999
|
+
for line in line_elements:
|
|
1000
|
+
amount_of_lines += 1
|
|
1001
|
+
hits = int(line.get("hits", "0"))
|
|
1002
|
+
if hits > 0:
|
|
1003
|
+
amount_of_hited_lines += 1
|
|
1004
|
+
line_rate = amount_of_hited_lines / amount_of_lines if amount_of_lines > 0 else 0.0
|
|
1005
|
+
package.set("line-rate", str(line_rate))
|
|
1006
|
+
tree.write(coverage_file, pretty_print=True, xml_declaration=True, encoding="UTF-8")
|
|
1007
|
+
|
|
1008
|
+
|
|
1009
|
+
@GeneralUtilities.check_arguments
|
|
1010
|
+
def generate_api_client_from_dependent_codeunit_with_default_properties(self, codeunit_folder:str, name_of_api_providing_codeunit: str, target_subfolder_in_codeunit: str,language:str,use_cache:bool) -> None:
|
|
1011
|
+
self.generate_api_client_from_dependent_codeunit(codeunit_folder,name_of_api_providing_codeunit,target_subfolder_in_codeunit,language,use_cache,["models","apis"])
|
|
1012
|
+
|
|
1013
|
+
@GeneralUtilities.check_arguments
|
|
1014
|
+
def generate_api_client_from_dependent_codeunit(self, codeunit_folder:str, name_of_api_providing_codeunit: str, target_subfolder_in_codeunit: str,language:str,use_cache:bool,properties:list[str]) -> None:
|
|
1015
|
+
openapigenerator_jar_file = self.ensure_openapigenerator_is_available(use_cache)
|
|
1016
|
+
openapi_spec_file = os.path.join(codeunit_folder, "Other", "Resources", "DependentCodeUnits", name_of_api_providing_codeunit, "APISpecification", f"{name_of_api_providing_codeunit}.latest.api.json")
|
|
1017
|
+
target_folder = os.path.join(codeunit_folder, target_subfolder_in_codeunit)
|
|
1018
|
+
GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder)
|
|
1019
|
+
argument=f'-jar {openapigenerator_jar_file} generate -i {openapi_spec_file} -g {language} -o {target_folder}'
|
|
1020
|
+
for property_value in properties:
|
|
1021
|
+
argument=f"{argument} --global-property {property_value}"
|
|
1022
|
+
self.__sc.run_program("java",argument , codeunit_folder)
|
|
1023
|
+
|
|
1024
|
+
@GeneralUtilities.check_arguments
|
|
1025
|
+
def replace_version_in_packagejson_file(self, packagejson_file: str, codeunit_version: str) -> None:
|
|
1026
|
+
encoding = "utf-8"
|
|
1027
|
+
with open(packagejson_file, encoding=encoding) as f:
|
|
1028
|
+
data = json.load(f)
|
|
1029
|
+
data['version'] = codeunit_version
|
|
1030
|
+
with open(packagejson_file, 'w', encoding=encoding) as f:
|
|
1031
|
+
json.dump(data, f, indent=2)
|
|
1032
|
+
GeneralUtilities.write_text_to_file(packagejson_file, GeneralUtilities.read_text_from_file(packagejson_file).replace("\r", ""))
|
|
1033
|
+
|
|
1034
|
+
@GeneralUtilities.check_arguments
|
|
1035
|
+
def ensure_openapigenerator_is_available(self,use_cache:bool) -> None:
|
|
1036
|
+
openapigenerator_folder = os.path.join(self.__sc.get_global_cache_folder(), "Tools", "OpenAPIGenerator")
|
|
1037
|
+
filename = "open-api-generator.jar"
|
|
1038
|
+
jar_file = f"{openapigenerator_folder}/{filename}"
|
|
1039
|
+
jar_file_exists = os.path.isfile(jar_file)
|
|
1040
|
+
update:bool=not jar_file_exists or not use_cache
|
|
1041
|
+
if update:
|
|
1042
|
+
self.__sc.log.log("Download OpenAPIGeneratorCLI...",LogLevel.Debug)
|
|
1043
|
+
used_version ="7.16.0"#TODO retrieve latest version
|
|
1044
|
+
download_link = f"https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/{used_version}/openapi-generator-cli-{used_version}.jar"
|
|
1045
|
+
GeneralUtilities.ensure_directory_does_not_exist(openapigenerator_folder)
|
|
1046
|
+
GeneralUtilities.ensure_directory_exists(openapigenerator_folder)
|
|
1047
|
+
urllib.request.urlretrieve(download_link, jar_file)
|
|
1048
|
+
GeneralUtilities.assert_file_exists(jar_file)
|
|
1049
|
+
return jar_file
|
|
1050
|
+
|
|
1051
|
+
@GeneralUtilities.check_arguments
|
|
1052
|
+
def standardized_tasks_update_version_in_docker_examples(self, codeunit_folder:str, codeunit_version:str) -> None:
|
|
1053
|
+
codeunit_name = os.path.basename(codeunit_folder)
|
|
1054
|
+
codeunit_name_lower = codeunit_name.lower()
|
|
1055
|
+
examples_folder = GeneralUtilities.resolve_relative_path("Other/Reference/ReferenceContent/Examples", codeunit_folder)
|
|
1056
|
+
for example_folder in GeneralUtilities.get_direct_folders_of_folder(examples_folder):
|
|
1057
|
+
docker_compose_file = os.path.join(example_folder, "docker-compose.yml")
|
|
1058
|
+
if os.path.isfile(docker_compose_file):
|
|
1059
|
+
filecontent = GeneralUtilities.read_text_from_file(docker_compose_file)
|
|
1060
|
+
replaced = re.sub(f'image:\\s+{codeunit_name_lower}:\\d+\\.\\d+\\.\\d+', f"image: {codeunit_name_lower}:{codeunit_version}", filecontent)
|
|
1061
|
+
GeneralUtilities.write_text_to_file(docker_compose_file, replaced)
|
|
1062
|
+
|
|
1063
|
+
@GeneralUtilities.check_arguments
|
|
1064
|
+
def set_version_of_openapigenerator(self, codeunit_folder: str, used_version: str = None) -> None:
|
|
1065
|
+
target_folder: str = os.path.join(codeunit_folder, "Other", "Resources", "Dependencies", "OpenAPIGenerator")
|
|
1066
|
+
version_file = os.path.join(target_folder, "Version.txt")
|
|
1067
|
+
GeneralUtilities.ensure_directory_exists(target_folder)
|
|
1068
|
+
GeneralUtilities.ensure_file_exists(version_file)
|
|
1069
|
+
GeneralUtilities.write_text_to_file(version_file, used_version)
|
|
1070
|
+
|
|
1071
|
+
@GeneralUtilities.check_arguments
|
|
1072
|
+
def get_latest_version_of_openapigenerator(self) -> None:
|
|
1073
|
+
headers = {'Cache-Control': 'no-cache'}
|
|
1074
|
+
self.__add_github_api_key_if_available(headers)
|
|
1075
|
+
response = requests.get(f"https://api.github.com/repos/OpenAPITools/openapi-generator/releases", headers=headers, timeout=(10, 10))
|
|
1076
|
+
latest_version = response.json()["tag_name"]
|
|
1077
|
+
return latest_version
|
|
1078
|
+
|
|
1079
|
+
@GeneralUtilities.check_arguments
|
|
1080
|
+
def update_images_in_example_with_default_excluded(self, codeunit_folder: str,custom_updater:AbstractImageHandler):
|
|
1081
|
+
self.update_images_in_example(codeunit_folder,[],custom_updater)
|
|
1082
|
+
|
|
1083
|
+
@GeneralUtilities.check_arguments
|
|
1084
|
+
def update_images_in_example(self, codeunit_folder: str,excluded:list[str],custom_updater:AbstractImageHandler):
|
|
1085
|
+
#only the version of the project itself must be updated. dependencies like postgresql or adminer for example should be updated by the usual used-image-update-mechanism
|
|
1086
|
+
dockercomposefile: str = f"{codeunit_folder}\\Other\\Reference\\ReferenceContent\\Examples\\MinimalDockerComposeFile\\docker-compose.yml"
|
|
1087
|
+
GeneralUtilities.assert_file_exists(dockercomposefile)
|
|
1088
|
+
#TODO update images in docker-compose files
|
|
1089
|
+
|
|
1090
|
+
@GeneralUtilities.check_arguments
|
|
1091
|
+
def push_wheel_build_artifact(self, push_build_artifacts_file, codeunitname, repository: str, apikey: str, gpg_identity: str, repository_folder_name: str,verbosity:LogLevel) -> None:
|
|
1092
|
+
folder_of_this_file = os.path.dirname(push_build_artifacts_file)
|
|
1093
|
+
repository_folder = GeneralUtilities.resolve_relative_path(f"..{os.path.sep}../Submodules{os.path.sep}{repository_folder_name}", folder_of_this_file)
|
|
1094
|
+
wheel_file = self.get_wheel_file(repository_folder, codeunitname)
|
|
1095
|
+
self.__standardized_tasks_push_wheel_file_to_registry(wheel_file, apikey, repository, gpg_identity,verbosity)
|
|
1096
|
+
|
|
1097
|
+
@GeneralUtilities.check_arguments
|
|
1098
|
+
def get_wheel_file(self, repository_folder: str, codeunit_name: str) -> str:
|
|
1099
|
+
self.__sc.assert_is_git_repository(repository_folder)
|
|
1100
|
+
return self.__sc.find_file_by_extension(os.path.join(repository_folder, codeunit_name,"Other","Artifacts", "BuildResult_Wheel"), "whl")
|
|
1101
|
+
|
|
1102
|
+
@GeneralUtilities.check_arguments
|
|
1103
|
+
def __standardized_tasks_push_wheel_file_to_registry(self, wheel_file: str, api_key: str, repository: str, gpg_identity: str,verbosity:LogLevel) -> None:
|
|
1104
|
+
# repository-value when PyPi should be used: "pypi"
|
|
1105
|
+
# gpg_identity-value when wheel-file should not be signed: None
|
|
1106
|
+
folder = os.path.dirname(wheel_file)
|
|
1107
|
+
filename = os.path.basename(wheel_file)
|
|
1108
|
+
|
|
1109
|
+
if gpg_identity is None:
|
|
1110
|
+
gpg_identity_argument = GeneralUtilities.empty_string
|
|
1111
|
+
else:
|
|
1112
|
+
gpg_identity_argument = GeneralUtilities.empty_string # f" --sign --identity {gpg_identity}"
|
|
1113
|
+
# disabled due to https://blog.pypi.org/posts/2023-05-23-removing-pgp/
|
|
1114
|
+
|
|
1115
|
+
if int(LogLevel.Information)<int(verbosity):
|
|
1116
|
+
verbose_argument = " --verbose"
|
|
1117
|
+
else:
|
|
1118
|
+
verbose_argument = GeneralUtilities.empty_string
|
|
1119
|
+
|
|
1120
|
+
twine_argument = f"upload{gpg_identity_argument} --repository {repository} --non-interactive {filename} --disable-progress-bar"
|
|
1121
|
+
twine_argument = f"{twine_argument} --username __token__ --password {api_key}{verbose_argument}"
|
|
1122
|
+
self.__sc.run_program("twine", twine_argument, folder, throw_exception_if_exitcode_is_not_zero=True)
|
|
1123
|
+
|
|
1124
|
+
@GeneralUtilities.check_arguments
|
|
1125
|
+
def push_nuget_build_artifact(self, push_script_file: str, repository_folder_name: str, codeunitname: str, registry_address: str,api_key: str):
|
|
1126
|
+
build_artifact_folder = GeneralUtilities.resolve_relative_path(f"../../Submodules/{repository_folder_name}/{codeunitname}/Other/Artifacts/BuildResult_NuGet", os.path.dirname(push_script_file))
|
|
1127
|
+
self.__sc.push_nuget_build_artifact(self.__sc.find_file_by_extension(build_artifact_folder, "nupkg"), registry_address, api_key)
|
|
1128
|
+
|
|
1129
|
+
@GeneralUtilities.check_arguments
|
|
1130
|
+
def suport_information_exists(self, repository_folder: str, version_of_product: str) -> bool:
|
|
1131
|
+
self.__sc.assert_is_git_repository(repository_folder)
|
|
1132
|
+
folder = os.path.join(repository_folder, "Other", "Resources", "Support")
|
|
1133
|
+
file = os.path.join(folder, "InformationAboutSupportedVersions.csv")
|
|
1134
|
+
if not os.path.isfile(file):
|
|
1135
|
+
return False
|
|
1136
|
+
entries = GeneralUtilities.read_csv_file(file, True)
|
|
1137
|
+
for entry in entries:
|
|
1138
|
+
if entry[0] == version_of_product:
|
|
1139
|
+
return True
|
|
1140
|
+
return False
|
|
1141
|
+
|
|
1142
|
+
@GeneralUtilities.check_arguments
|
|
1143
|
+
def mark_current_version_as_supported(self, repository_folder: str, version_of_product: str, supported_from: datetime, supported_until: datetime):
|
|
1144
|
+
self.__sc.assert_is_git_repository(repository_folder)
|
|
1145
|
+
if self.suport_information_exists(repository_folder, version_of_product):
|
|
1146
|
+
raise ValueError(f"Version-support for v{version_of_product} already defined.")
|
|
1147
|
+
folder = os.path.join(repository_folder, "Other", "Resources", "Support")
|
|
1148
|
+
GeneralUtilities.ensure_directory_exists(folder)
|
|
1149
|
+
file = os.path.join(folder, "InformationAboutSupportedVersions.csv")
|
|
1150
|
+
if not os.path.isfile(file):
|
|
1151
|
+
GeneralUtilities.ensure_file_exists(file)
|
|
1152
|
+
GeneralUtilities.append_line_to_file(file, "Version;SupportBegin;SupportEnd")
|
|
1153
|
+
GeneralUtilities.append_line_to_file(file, f"{version_of_product};{GeneralUtilities.datetime_to_string(supported_from)};{GeneralUtilities.datetime_to_string(supported_until)}")
|
|
1154
|
+
|
|
1155
|
+
|
|
1156
|
+
@GeneralUtilities.check_arguments
|
|
1157
|
+
def add_github_release(self, productname: str, projectversion: str, build_artifacts_folder: str, github_username: str, repository_folder: str, additional_attached_files: list[str]) -> None:
|
|
1158
|
+
self.__sc.assert_is_git_repository(repository_folder)
|
|
1159
|
+
self.__sc.log.log(f"Create GitHub-release for {productname}...")
|
|
1160
|
+
github_repo = f"{github_username}/{productname}"
|
|
1161
|
+
artifact_files = []
|
|
1162
|
+
codeunits = self.get_codeunits(repository_folder)
|
|
1163
|
+
for codeunit in codeunits:
|
|
1164
|
+
artifact_files.append(self.__sc.find_file_by_extension(f"{build_artifacts_folder}/{productname}/{projectversion}/{codeunit}", "Artifacts.zip"))
|
|
1165
|
+
if additional_attached_files is not None:
|
|
1166
|
+
for additional_attached_file in additional_attached_files:
|
|
1167
|
+
artifact_files.append(additional_attached_file)
|
|
1168
|
+
changelog_file = os.path.join(repository_folder, "Other", "Resources", "Changelog", f"v{projectversion}.md")
|
|
1169
|
+
self.__sc.run_program_argsasarray("gh", ["release", "create", f"v{projectversion}", "--repo", github_repo, "--notes-file", changelog_file, "--title", f"Release v{projectversion}"]+artifact_files)
|
|
1170
|
+
|
|
1171
|
+
@GeneralUtilities.check_arguments
|
|
1172
|
+
def get_dependency_version_in_resources_folder(self, resources_folder:str, dependency_name: str) ->str:
|
|
1173
|
+
dependency_folder = os.path.join(resources_folder, "Dependencies", dependency_name)
|
|
1174
|
+
version_file = os.path.join(dependency_folder, "Version.txt")
|
|
1175
|
+
if not os.path.isfile(version_file):
|
|
1176
|
+
raise ValueError(f"Version-file for dependency {dependency_name} does not exist. Expected location: {version_file}")
|
|
1177
|
+
return GeneralUtilities.read_text_from_file(version_file)
|
|
1178
|
+
|
|
1179
|
+
@GeneralUtilities.check_arguments
|
|
1180
|
+
def update_dependency_in_resources_folder(self, update_dependencies_file, dependency_name: str, latest_version_function: str) -> None:
|
|
1181
|
+
dependency_folder = GeneralUtilities.resolve_relative_path(f"../Resources/Dependencies/{dependency_name}", update_dependencies_file)
|
|
1182
|
+
version_file = os.path.join(dependency_folder, "Version.txt")
|
|
1183
|
+
version_file_exists = os.path.isfile(version_file)
|
|
1184
|
+
write_to_file = False
|
|
1185
|
+
if version_file_exists:
|
|
1186
|
+
current_version = GeneralUtilities.read_text_from_file(version_file)
|
|
1187
|
+
if current_version != latest_version_function:
|
|
1188
|
+
write_to_file = True
|
|
1189
|
+
else:
|
|
1190
|
+
GeneralUtilities.ensure_directory_exists(dependency_folder)
|
|
1191
|
+
GeneralUtilities.ensure_file_exists(version_file)
|
|
1192
|
+
write_to_file = True
|
|
1193
|
+
if write_to_file:
|
|
1194
|
+
GeneralUtilities.write_text_to_file(version_file, latest_version_function)
|
|
1195
|
+
|
|
1196
|
+
@GeneralUtilities.check_arguments
|
|
1197
|
+
def push_docker_build_artifact(self, push_artifacts_file: str, registry: str, push_readme: bool, repository_folder_name: str, remote_image_name: str = None) -> None:
|
|
1198
|
+
folder_of_this_file = os.path.dirname(push_artifacts_file)
|
|
1199
|
+
filename = os.path.basename(push_artifacts_file)
|
|
1200
|
+
codeunitname_regex: str = "([a-zA-Z0-9]+)"
|
|
1201
|
+
filename_regex: str = f"PushArtifacts\\.{codeunitname_regex}\\.py"
|
|
1202
|
+
if match := re.search(filename_regex, filename, re.IGNORECASE):
|
|
1203
|
+
codeunitname = match.group(1)
|
|
1204
|
+
else:
|
|
1205
|
+
raise ValueError(f"Expected push-artifacts-file to match the regex \"{filename_regex}\" where \"{codeunitname_regex}\" represents the codeunit-name.")
|
|
1206
|
+
repository_folder = GeneralUtilities.resolve_relative_path(f"..{os.path.sep}..{os.path.sep}Submodules{os.path.sep}{repository_folder_name}", folder_of_this_file)
|
|
1207
|
+
codeunit_folder = os.path.join(repository_folder, codeunitname)
|
|
1208
|
+
artifacts_folder = os.path.join(repository_folder,codeunitname, "Other", "Artifacts")
|
|
1209
|
+
applicationimage_folder = os.path.join(artifacts_folder, "BuildResult_OCIImage")
|
|
1210
|
+
codeunit_version = self.get_version_of_codeunit(os.path.join(codeunit_folder, f"{codeunitname}.codeunit.xml"))
|
|
1211
|
+
if remote_image_name is None:
|
|
1212
|
+
remote_image_name = codeunitname.lower()
|
|
1213
|
+
tar_files=[f for f in GeneralUtilities.get_direct_files_of_folder(applicationimage_folder) if f.endswith(".tar")]
|
|
1214
|
+
target_image_address=f"{registry}/{remote_image_name}"
|
|
1215
|
+
tar_files_with_platforms: list[tuple[str, str, str]] = []
|
|
1216
|
+
for tar_file in tar_files:
|
|
1217
|
+
filename=os.path.basename(tar_file)#filename looks like "{codeunitname}_v{codeunitversion}_{GeneralUtilities.platform_to_dash_str(platform)}.tar"
|
|
1218
|
+
platform_of_file:Platform=self.platform_from_filename(filename)#GeneralUtilities.platform_from_dash_str( filename.split("_")[-1].split(".")[0])
|
|
1219
|
+
platform_os_in_docker_format :str = None
|
|
1220
|
+
platform_arch_in_docker_format :str = None
|
|
1221
|
+
if platform_of_file==Platform.Windows_AMD64:
|
|
1222
|
+
raise NotImplementedError("Building docker images for Windows is not implemented yet.")
|
|
1223
|
+
elif platform_of_file==Platform.Linux_AMD64:
|
|
1224
|
+
platform_os_in_docker_format = "linux"
|
|
1225
|
+
platform_arch_in_docker_format = "amd64"
|
|
1226
|
+
elif platform_of_file==Platform.Linux_ARM64:
|
|
1227
|
+
platform_os_in_docker_format = "linux"
|
|
1228
|
+
platform_arch_in_docker_format = "arm64"
|
|
1229
|
+
elif platform_of_file==Platform.MacOS_ARM64:
|
|
1230
|
+
raise NotImplementedError("Building docker images for MacOS is not implemented yet.")
|
|
1231
|
+
else:
|
|
1232
|
+
raise ValueError(f"Unsupported platform {platform_of_file} extracted from filename {filename}.")
|
|
1233
|
+
tar_files_with_platforms.append((tar_file, platform_os_in_docker_format, platform_arch_in_docker_format))
|
|
1234
|
+
self.push_docker_build_artifact_as_multi_arch_artifact(tar_files_with_platforms,target_image_address, "v"+codeunit_version)
|
|
1235
|
+
self.push_docker_build_artifact_as_multi_arch_artifact(tar_files_with_platforms,target_image_address, "latest")
|
|
1236
|
+
if push_readme:
|
|
1237
|
+
GeneralUtilities.assert_file_exists(os.path.join(codeunit_folder, "ReadMe.md"))
|
|
1238
|
+
self.__sc.run_program_with_retry("docker-pushrm", target_image_address, codeunit_folder)
|
|
1239
|
+
|
|
1240
|
+
def push_docker_build_artifact_as_multi_arch_artifact(self,tar_files: list[tuple[str, str, str]], image_address: str, tag: str):
|
|
1241
|
+
"""
|
|
1242
|
+
tar_files: list of (tar_path, os, arch) tuples
|
|
1243
|
+
for example [
|
|
1244
|
+
("MyApp.Linux.arm64.tar", "linux", "arm64"),
|
|
1245
|
+
("MyApp.Linux.amd64.tar", "linux", "amd64")
|
|
1246
|
+
]
|
|
1247
|
+
image_address for example: "myregistry.example.com/myapp"
|
|
1248
|
+
tag for example: "1.0.0"
|
|
1249
|
+
"""
|
|
1250
|
+
GeneralUtilities.write_message_to_stdout(f"Creating multi-arch artifact {image_address}:{tag}...")
|
|
1251
|
+
arch_tags = []
|
|
1252
|
+
for tar_path, os_name, arch in tar_files:
|
|
1253
|
+
arch_tag = f"{image_address}:{tag}-{os_name}-{arch}"
|
|
1254
|
+
arch_tags.append(arch_tag)
|
|
1255
|
+
# Load tar → local image
|
|
1256
|
+
GeneralUtilities.write_message_to_stdout(f"Loading {tar_path}...")
|
|
1257
|
+
result = self.__sc.run_program_argsasarray("docker",[ "load", "-i", tar_path])
|
|
1258
|
+
# docker load outputs: "Loaded image: sha256:abc123..." or "Loaded image ID: ..."
|
|
1259
|
+
# we need the loaded image ID
|
|
1260
|
+
loaded_id = None
|
|
1261
|
+
for line in GeneralUtilities.string_to_lines(result[1]):
|
|
1262
|
+
if "Loaded image" in line:
|
|
1263
|
+
loaded_id = line.split(":", 1)[1].strip()
|
|
1264
|
+
break
|
|
1265
|
+
if not loaded_id:
|
|
1266
|
+
raise RuntimeError(f"Could not determine loaded image from output: \"{result[1]}\"")
|
|
1267
|
+
# Retag + push
|
|
1268
|
+
self.__sc.run_program_argsasarray("docker",[ "tag", loaded_id, arch_tag])
|
|
1269
|
+
self.__sc.run_program_argsasarray("docker",[ "push", arch_tag])
|
|
1270
|
+
# Create multi-arch manifest
|
|
1271
|
+
final_tag = f"{image_address}:{tag}"
|
|
1272
|
+
self.__sc.run_program_argsasarray("docker", [ "buildx", "imagetools", "create", "--tag", final_tag] + arch_tags)
|
|
1273
|
+
|
|
1274
|
+
@GeneralUtilities.check_arguments
|
|
1275
|
+
def platform_from_filename(self,filename: str) -> Platform:
|
|
1276
|
+
match = re.search(r'_([^_]+)\.tar', filename)
|
|
1277
|
+
if match:
|
|
1278
|
+
return GeneralUtilities.platform_from_dash_str(match.group(1))
|
|
1279
|
+
else:
|
|
1280
|
+
raise ValueError(f"Cannot extract platform from filename: \"{filename}\"")
|
|
1281
|
+
|
|
1282
|
+
def prepare_building_codeunits(self,repository_folder:str,use_cache:bool,generate_development_certificate:bool):
|
|
1283
|
+
if generate_development_certificate:
|
|
1284
|
+
self.ensure_certificate_authority_for_development_purposes_is_generated(repository_folder)
|
|
1285
|
+
self.generate_certificate_for_development_purposes_for_product(repository_folder)
|
|
1286
|
+
self.generate_tasksfile_from_workspace_file(repository_folder)
|
|
1287
|
+
self.generate_codeunits_overview_diagram(repository_folder)
|
|
1288
|
+
self.generate_svg_files_from_plantuml_files_for_repository(repository_folder,use_cache)
|
|
1289
|
+
|
|
1290
|
+
@GeneralUtilities.check_arguments
|
|
1291
|
+
def copy_product_resource_to_codeunit_resource_folder(self, codeunit_folder: str, resourcename: str) -> None:
|
|
1292
|
+
repository_folder = GeneralUtilities.resolve_relative_path(f"..", codeunit_folder)
|
|
1293
|
+
self.__sc.assert_is_git_repository(repository_folder)
|
|
1294
|
+
src_folder = GeneralUtilities.resolve_relative_path(f"Other/Resources/{resourcename}", repository_folder)
|
|
1295
|
+
GeneralUtilities.assert_condition(os.path.isdir(src_folder), f"Required product-resource {resourcename} does not exist. Expected folder: {src_folder}")
|
|
1296
|
+
trg_folder = GeneralUtilities.resolve_relative_path(f"Other/Resources/{resourcename}", codeunit_folder)
|
|
1297
|
+
GeneralUtilities.ensure_directory_does_not_exist(trg_folder)
|
|
1298
|
+
GeneralUtilities.ensure_directory_exists(trg_folder)
|
|
1299
|
+
GeneralUtilities.copy_content_of_folder(src_folder, trg_folder)
|
|
1300
|
+
|
|
1301
|
+
@GeneralUtilities.check_arguments
|
|
1302
|
+
def ensure_containers_are_not_running(self, container_names_to_remove:list[str]) -> None:
|
|
1303
|
+
for container_name in container_names_to_remove:
|
|
1304
|
+
self.__sc.log.log(f"Ensure container {container_name} does not exist...")
|
|
1305
|
+
self.__sc.run_program("docker", f"container rm -f {container_name}", throw_exception_if_exitcode_is_not_zero=False)
|
|
1306
|
+
|
|
1307
|
+
@GeneralUtilities.check_arguments
|
|
1308
|
+
def load_docker_image(self, oci_image_artifacts_folder:str,platform_for_image:Platform) -> None:
|
|
1309
|
+
for file in GeneralUtilities.get_direct_files_of_folder(oci_image_artifacts_folder):
|
|
1310
|
+
if file.endswith(f"_{GeneralUtilities.platform_to_dash_str(platform_for_image)}.tar"):
|
|
1311
|
+
image_filename = file
|
|
1312
|
+
self.__sc.log.log(f"Load docker-image {image_filename}...")
|
|
1313
|
+
self.__sc.run_program("docker", f"load -i {image_filename}", oci_image_artifacts_folder)
|
|
1314
|
+
return
|
|
1315
|
+
raise ValueError(f"No docker-image found for platform {GeneralUtilities.platform_to_dash_str(platform_for_image)} in folder {oci_image_artifacts_folder}.")
|
|
1316
|
+
|
|
1317
|
+
@GeneralUtilities.check_arguments
|
|
1318
|
+
def start_dockerfile_example(self, current_file: str,remove_old_container: bool, remove_volumes_folder: bool, env_file: str) -> None:
|
|
1319
|
+
container_names_to_remove:list[str]=[]
|
|
1320
|
+
folder_of_current_file = os.path.dirname(current_file)
|
|
1321
|
+
if remove_old_container:
|
|
1322
|
+
docker_compose_file = f"{folder_of_current_file}/docker-compose.yml"
|
|
1323
|
+
lines = GeneralUtilities.read_lines_from_file(docker_compose_file)
|
|
1324
|
+
for line in lines:
|
|
1325
|
+
if match := re.search("container_name:\\s*'?([^']+)'?", line):
|
|
1326
|
+
container_names_to_remove.append(match.group(1))
|
|
1327
|
+
self.__sc.log.log(f"Ensure container of {docker_compose_file} do not exist...")
|
|
1328
|
+
oci_image_artifacts_folder = GeneralUtilities.resolve_relative_path("../../../../Artifacts/BuildResult_OCIImage", folder_of_current_file)
|
|
1329
|
+
self.ensure_containers_are_not_running(container_names_to_remove)
|
|
1330
|
+
platform_for_image :Platform=None
|
|
1331
|
+
if GeneralUtilities.current_system_is_x64():
|
|
1332
|
+
platform_for_image=Platform.Linux_AMD64
|
|
1333
|
+
elif GeneralUtilities.current_system_is_arm64():
|
|
1334
|
+
platform_for_image=Platform.Linux_ARM64
|
|
1335
|
+
else:
|
|
1336
|
+
raise ValueError("Unsupported platform for docker-image. Only AMD64 and ARM64 are supported.")
|
|
1337
|
+
self.load_docker_image(oci_image_artifacts_folder,platform_for_image)
|
|
1338
|
+
example_name = os.path.basename(folder_of_current_file)
|
|
1339
|
+
codeunit_name = os.path.basename(GeneralUtilities.resolve_relative_path("../../../../..", folder_of_current_file))
|
|
1340
|
+
if remove_volumes_folder:
|
|
1341
|
+
volumes_folder = os.path.join(folder_of_current_file, "Volumes")
|
|
1342
|
+
self.__sc.log.log(f"Ensure volumes-folder '{volumes_folder}' does not exist...")
|
|
1343
|
+
GeneralUtilities.ensure_directory_does_not_exist(volumes_folder)
|
|
1344
|
+
GeneralUtilities.ensure_directory_exists(volumes_folder)
|
|
1345
|
+
docker_project_name = f"{codeunit_name}_{example_name}".lower()
|
|
1346
|
+
self.__sc.log.log("Start docker-container...")
|
|
1347
|
+
argument = f"compose --project-name {docker_project_name}"
|
|
1348
|
+
if env_file is not None:
|
|
1349
|
+
argument = f"{argument} --env-file {env_file}"
|
|
1350
|
+
argument = f"{argument} up --detach"
|
|
1351
|
+
self.__sc.run_program("docker", argument, folder_of_current_file)
|
|
1352
|
+
|
|
1353
|
+
@GeneralUtilities.check_arguments
|
|
1354
|
+
def ensure_env_file_is_generated(self, current_file: str, env_file_name: str, env_values: dict[str, str]):
|
|
1355
|
+
folder = os.path.dirname(current_file)
|
|
1356
|
+
env_file = os.path.join(folder, env_file_name)
|
|
1357
|
+
GeneralUtilities.ensure_file_exists(env_file)
|
|
1358
|
+
lines = []
|
|
1359
|
+
for key, value in env_values.items():
|
|
1360
|
+
lines.append(f"{key}={value}")
|
|
1361
|
+
GeneralUtilities.write_lines_to_file(env_file, lines)
|
|
1362
|
+
|
|
1363
|
+
@GeneralUtilities.check_arguments
|
|
1364
|
+
def stop_dockerfile_example(self, current_file: str, remove_old_container: bool, remove_volumes_folder: bool) -> None:
|
|
1365
|
+
folder = os.path.dirname(current_file)
|
|
1366
|
+
example_name = os.path.basename(folder)
|
|
1367
|
+
codeunit_name = os.path.basename(GeneralUtilities.resolve_relative_path("../../../../..", folder))
|
|
1368
|
+
docker_project_name = f"{codeunit_name}_{example_name}".lower()
|
|
1369
|
+
self.__sc.log.log("Stop docker-container...")
|
|
1370
|
+
self.__sc.run_program("docker", f"compose --project-name {docker_project_name} down", folder)
|
|
1371
|
+
if remove_old_container:
|
|
1372
|
+
pass#TODO
|
|
1373
|
+
if remove_volumes_folder:
|
|
1374
|
+
pass#TODO
|
|
1375
|
+
|
|
1376
|
+
@GeneralUtilities.check_arguments
|
|
1377
|
+
def update_submodule(self, repository_folder: str, submodule_name: str, local_branch: str = "main", remote_branch: str = "main", remote: str = "origin"):
|
|
1378
|
+
submodule_folder = GeneralUtilities.resolve_relative_path("Other/Resources/Submodules/"+submodule_name, repository_folder)
|
|
1379
|
+
self.__sc.git_fetch(submodule_folder, remote)
|
|
1380
|
+
self.__sc.git_checkout(submodule_folder, local_branch)
|
|
1381
|
+
self.__sc.git_pull(submodule_folder, remote, local_branch, remote_branch, True)
|
|
1382
|
+
current_version = self.__sc.get_semver_version_from_gitversion(repository_folder)
|
|
1383
|
+
changelog_file = os.path.join(repository_folder, "Other", "Resources", "Changelog", f"v{current_version}.md")
|
|
1384
|
+
if (not os.path.isfile(changelog_file)):
|
|
1385
|
+
GeneralUtilities.ensure_file_exists(changelog_file)
|
|
1386
|
+
GeneralUtilities.write_text_to_file(changelog_file, """# Release notes
|
|
1387
|
+
|
|
1388
|
+
## Changes
|
|
1389
|
+
|
|
1390
|
+
- Updated geo-ip-database.
|
|
1391
|
+
""")
|
|
1392
|
+
|
|
1393
|
+
def set_latest_version_for_clone_repository_as_resource(self,repository_folder:str, resourcename: str, github_link: str, branch: str = "main"):
|
|
1394
|
+
resrepo_commit_id_folder: str = os.path.join(repository_folder, "Other", "Resources", f"{resourcename}Version")
|
|
1395
|
+
resrepo_commit_id_file: str = os.path.join(resrepo_commit_id_folder, f"{resourcename}Version.txt")
|
|
1396
|
+
current_version: str = GeneralUtilities.read_text_from_file(resrepo_commit_id_file)
|
|
1397
|
+
|
|
1398
|
+
stdOut = [l.split("\t") for l in GeneralUtilities.string_to_lines(self.__sc.run_program("git", f"ls-remote {github_link}")[1])]
|
|
1399
|
+
stdOut = [l for l in stdOut if l[1] == f"refs/heads/{branch}"]
|
|
1400
|
+
GeneralUtilities.assert_condition(len(stdOut) == 1)
|
|
1401
|
+
latest_version: str = stdOut[0][0]
|
|
1402
|
+
if current_version != latest_version:
|
|
1403
|
+
GeneralUtilities.write_text_to_file(resrepo_commit_id_file, latest_version)
|
|
1404
|
+
|
|
1405
|
+
@GeneralUtilities.check_arguments
|
|
1406
|
+
def get_dependencies_which_are_ignored_from_updates(self, codeunit_folder: str) -> list[str]:
|
|
1407
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
1408
|
+
namespaces = {'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure', 'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
|
|
1409
|
+
codeunit_name = os.path.basename(codeunit_folder)
|
|
1410
|
+
codeunit_file = os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml")
|
|
1411
|
+
root: etree._ElementTree = etree.parse(codeunit_file)
|
|
1412
|
+
ignoreddependencies = root.xpath('//cps:codeunit/cps:properties/cps:updatesettings/cps:ignoreddependencies/cps:ignoreddependency', namespaces=namespaces)
|
|
1413
|
+
result = [x.text.replace("\\n", GeneralUtilities.empty_string).replace("\\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string).replace("\r", GeneralUtilities.empty_string).strip() for x in ignoreddependencies]
|
|
1414
|
+
return result
|
|
1415
|
+
|
|
1416
|
+
@GeneralUtilities.check_arguments
|
|
1417
|
+
def update_dependencies_of_package_json(self, folder_of_package_json: str) -> None:#TODO this should probably be implemented in TFCPS_CodeUnitSpecific_NodeJS_Functions
|
|
1418
|
+
#TODO move this to TFCPS_CodeUnitSpecific_NodeJS_Functions
|
|
1419
|
+
if self.is_codeunit_folder(folder_of_package_json):
|
|
1420
|
+
ignored_dependencies = self.get_dependencies_which_are_ignored_from_updates(folder_of_package_json)
|
|
1421
|
+
else:
|
|
1422
|
+
ignored_dependencies = []
|
|
1423
|
+
# TODO consider ignored_dependencies
|
|
1424
|
+
result = self.__sc.run_with_epew("npm", "outdated", folder_of_package_json, throw_exception_if_exitcode_is_not_zero=False)
|
|
1425
|
+
if result[0] == 0:
|
|
1426
|
+
return # all dependencies up to date
|
|
1427
|
+
elif result[0] == 1:
|
|
1428
|
+
package_json_content = None
|
|
1429
|
+
package_json_file = f"{folder_of_package_json}/package.json"
|
|
1430
|
+
with open(package_json_file, "r", encoding="utf-8") as package_json_file_object:
|
|
1431
|
+
package_json_content = json.load(package_json_file_object)
|
|
1432
|
+
lines = GeneralUtilities.string_to_lines(result[1])[1:][:-1]
|
|
1433
|
+
for line in lines:
|
|
1434
|
+
normalized_line_splitted = ' '.join(line.split()).split(" ")
|
|
1435
|
+
package = normalized_line_splitted[0]
|
|
1436
|
+
latest_version = normalized_line_splitted[3]
|
|
1437
|
+
if package in package_json_content["dependencies"]:
|
|
1438
|
+
package_json_content["dependencies"][package] = latest_version
|
|
1439
|
+
if package in package_json_content["devDependencies"]:
|
|
1440
|
+
package_json_content["devDependencies"][package] = latest_version
|
|
1441
|
+
with open(package_json_file, "w", encoding="utf-8") as package_json_file_object:
|
|
1442
|
+
json.dump(package_json_content, package_json_file_object, indent=4)
|
|
1443
|
+
GeneralUtilities.write_text_to_file(package_json_file, GeneralUtilities.read_text_from_file(package_json_file).replace("\r", ""))
|
|
1444
|
+
self.do_npm_install(folder_of_package_json, True,True)#TODO use_cache might be dangerous here
|
|
1445
|
+
else:
|
|
1446
|
+
self.__sc.log.log("Update dependencies resulted in an error.", LogLevel.Error)
|
|
1447
|
+
|
|
1448
|
+
|
|
1449
|
+
@GeneralUtilities.check_arguments
|
|
1450
|
+
def get_resource_from_submodule_with_default_ignore_pattern(self,codeunit_folder:str,submodule_name:str,resource_name:str):
|
|
1451
|
+
self.get_resource_from_submodule(codeunit_folder,submodule_name,resource_name,["**.git","**.gitmodules"])
|
|
1452
|
+
|
|
1453
|
+
@GeneralUtilities.check_arguments
|
|
1454
|
+
def get_resource_from_submodule(self,codeunit_folder:str,submodule_name:str,resource_name:str,ignore_patterns:list[str]):
|
|
1455
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
1456
|
+
repository=os.path.dirname(codeunit_folder)
|
|
1457
|
+
source_folder=os.path.join(repository,"Other","Resources","Submodules",submodule_name)
|
|
1458
|
+
GeneralUtilities.assert_folder_exists(source_folder)
|
|
1459
|
+
target_folder=os.path.join(codeunit_folder,"Other","Resources",resource_name)
|
|
1460
|
+
GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder)
|
|
1461
|
+
GeneralUtilities.copy_content_of_folder(source_folder,target_folder,True,ignore_patterns)
|
|
1462
|
+
|
|
1463
|
+
|
|
1464
|
+
@GeneralUtilities.check_arguments
|
|
1465
|
+
def pull_images_of_test_services(self,repository_folder:str,env_variables:dict[str,str]):
|
|
1466
|
+
if env_variables is None:
|
|
1467
|
+
env_variables={}
|
|
1468
|
+
for image in self.oci_image_manager.get_used_images_in_repository(repository_folder):
|
|
1469
|
+
env_variables[f"image_{image.lower()}"]=self.oci_image_manager.get_registry_address_for_image(repository_folder,image)+":"+self.oci_image_manager.get_tag_for_image(repository_folder,image, True)
|
|
1470
|
+
test_services=GeneralUtilities.get_direct_folders_of_folder(os.path.join(repository_folder,"Other","Resources","LocalTestServices"))
|
|
1471
|
+
if len(test_services)==0:
|
|
1472
|
+
return
|
|
1473
|
+
self.__sc.log.log("Pull images for local test-services...")
|
|
1474
|
+
self.__sc.login_to_defined_docker_registries()
|
|
1475
|
+
for test_service_folder in test_services:
|
|
1476
|
+
test_service_name=os.path.basename(test_service_folder)
|
|
1477
|
+
self.__sc.log.log(f"Pull images for test-service {test_service_name}...")
|
|
1478
|
+
arguments=f"compose -f docker-compose.yml"
|
|
1479
|
+
env_variables_file=os.path.join(test_service_folder,"Parameters.env")
|
|
1480
|
+
GeneralUtilities.ensure_file_exists(env_variables_file)
|
|
1481
|
+
lines=[]
|
|
1482
|
+
for k,v in env_variables.items():
|
|
1483
|
+
lines=lines+[f"{k}={v}"]
|
|
1484
|
+
GeneralUtilities.write_lines_to_file(env_variables_file,lines)
|
|
1485
|
+
arguments=arguments + " --env-file Parameters.env pull --quiet"
|
|
1486
|
+
arguments_for_log=arguments
|
|
1487
|
+
self.__sc.run_program_with_retry("docker",arguments,test_service_folder,arguments_for_log=arguments_for_log,print_live_output=self.__sc.log.loglevel==LogLevel.Debug)
|
|
1488
|
+
|
|
1489
|
+
def load_deb_control_file_content(self, file: str, codeunitname: str, codeunitversion: str, installedsize: int, maintainername: str, maintaineremail: str, description: str) -> str:
|
|
1490
|
+
content = GeneralUtilities.read_text_from_file(file)
|
|
1491
|
+
content = GeneralUtilities.replace_variable_in_string(content, "codeunitname", codeunitname)
|
|
1492
|
+
content = GeneralUtilities.replace_variable_in_string(content, "codeunitversion", codeunitversion)
|
|
1493
|
+
content = GeneralUtilities.replace_variable_in_string(content, "installedsize", str(installedsize))
|
|
1494
|
+
content = GeneralUtilities.replace_variable_in_string(content, "maintainername", maintainername)
|
|
1495
|
+
content = GeneralUtilities.replace_variable_in_string(content, "maintaineremail", maintaineremail)
|
|
1496
|
+
content = GeneralUtilities.replace_variable_in_string(content, "description", description)
|
|
1497
|
+
return content
|
|
1498
|
+
|
|
1499
|
+
def calculate_deb_package_size(self, binary_folder: str) -> int:
|
|
1500
|
+
size_in_bytes = 0
|
|
1501
|
+
for file in GeneralUtilities.get_all_files_of_folder(binary_folder):
|
|
1502
|
+
size_in_bytes = size_in_bytes+os.path.getsize(file)
|
|
1503
|
+
result = math.ceil(size_in_bytes/1024)
|
|
1504
|
+
return result
|
|
1505
|
+
|
|
1506
|
+
def create_deb_package_for_artifact(self,codeunit_folder: str, maintainername: str, maintaineremail: str, description: str) -> None:
|
|
1507
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
1508
|
+
codeunit_name = os.path.basename(codeunit_folder)
|
|
1509
|
+
binary_folder = GeneralUtilities.resolve_relative_path("Other/Artifacts/BuildResult_DotNet_linux-x64", codeunit_folder)
|
|
1510
|
+
deb_output_folder = GeneralUtilities.resolve_relative_path("Other/Artifacts/BuildResult_Deb", codeunit_folder)
|
|
1511
|
+
control_file = GeneralUtilities.resolve_relative_path("Other/Build/DebControlFile.txt", codeunit_folder)
|
|
1512
|
+
installedsize = self.calculate_deb_package_size(binary_folder)
|
|
1513
|
+
control_file_content = self.load_deb_control_file_content(control_file, codeunit_name, self.get_version_of_codeunit(os.path.join(codeunit_folder,f"{codeunit_name}.codeunit.xml")), installedsize, maintainername, maintaineremail, description)
|
|
1514
|
+
self.__sc.create_deb_package(codeunit_name, binary_folder, control_file_content, deb_output_folder, 555)
|
|
1515
|
+
|
|
1516
|
+
@GeneralUtilities.check_arguments
|
|
1517
|
+
def create_zip_file_for_artifact(self, codeunit_folder: str, artifact_source_name: str, name_of_new_artifact: str) -> None:
|
|
1518
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
1519
|
+
src_artifact_folder = GeneralUtilities.resolve_relative_path(f"Other/Artifacts/{artifact_source_name}", codeunit_folder)
|
|
1520
|
+
shutil.make_archive(name_of_new_artifact, 'zip', src_artifact_folder)
|
|
1521
|
+
archive_file = os.path.join(os.getcwd(), f"{name_of_new_artifact}.zip")
|
|
1522
|
+
target_folder = GeneralUtilities.resolve_relative_path(f"Other/Artifacts/{name_of_new_artifact}", codeunit_folder)
|
|
1523
|
+
GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder)
|
|
1524
|
+
shutil.move(archive_file, target_folder)
|
|
1525
|
+
|
|
1526
|
+
def generate_winget_zip_manifest(self, codeunit_folder: str, artifact_name_of_zip: str):
|
|
1527
|
+
self.assert_is_codeunit_folder(codeunit_folder)
|
|
1528
|
+
codeunit_name = os.path.basename(codeunit_folder)
|
|
1529
|
+
codeunit_version = self.get_version_of_codeunit(os.path.join(codeunit_folder,f"{codeunit_name}.codeunit.xml"))
|
|
1530
|
+
build_folder = os.path.join(codeunit_folder, "Other", "Build")
|
|
1531
|
+
artifacts_folder = os.path.join(codeunit_folder, "Other", "Artifacts", artifact_name_of_zip)
|
|
1532
|
+
manifest_folder = os.path.join(codeunit_folder, "Other", "Artifacts", "WinGet-Manifest")
|
|
1533
|
+
GeneralUtilities.assert_folder_exists(artifacts_folder)
|
|
1534
|
+
artifacts_file = self.__sc.find_file_by_extension(artifacts_folder, "zip")
|
|
1535
|
+
winget_template_file = os.path.join(build_folder, "WinGet-Template.yaml")
|
|
1536
|
+
winget_manifest_file = os.path.join(manifest_folder, "WinGet-Manifest.yaml")
|
|
1537
|
+
GeneralUtilities.assert_file_exists(winget_template_file)
|
|
1538
|
+
GeneralUtilities.ensure_directory_exists(manifest_folder)
|
|
1539
|
+
GeneralUtilities.ensure_file_exists(winget_manifest_file)
|
|
1540
|
+
manifest_content = GeneralUtilities.read_text_from_file(winget_template_file)
|
|
1541
|
+
manifest_content = GeneralUtilities.replace_variable_in_string(manifest_content, "version", codeunit_version)
|
|
1542
|
+
manifest_content = GeneralUtilities.replace_variable_in_string(manifest_content, "sha256_hashvalue", GeneralUtilities.get_sha256_of_file(artifacts_file))
|
|
1543
|
+
GeneralUtilities.write_text_to_file(winget_manifest_file, manifest_content)
|
|
1544
|
+
|
|
1545
|
+
def download_file(self,source:str,target:str):
|
|
1546
|
+
GeneralUtilities.ensure_directory_exists(os.path.dirname(target))
|
|
1547
|
+
GeneralUtilities.ensure_file_exists(target)
|
|
1548
|
+
response = requests.get(source, timeout=30)
|
|
1549
|
+
response.raise_for_status()
|
|
1550
|
+
with open(target, "wb") as f:
|
|
1551
|
+
f.write(response.content)
|
|
1552
|
+
|
|
1553
|
+
def try_update_basic_codeunitreference_from_examples_repository(self,codeunit_folder:str,example_codeunit_name: str):
|
|
1554
|
+
source=f"https://raw.githubusercontent.com/anionDev/CommonProjectStructureExamples/refs/heads/main/{example_codeunit_name}/Other/Reference/ReferenceContent/HowToBuild.md"
|
|
1555
|
+
target=f"{codeunit_folder}/Other/Reference/ReferenceContent/HowToBuild.md"
|
|
1556
|
+
self.download_file(source,target)
|
|
1557
|
+
|
|
1558
|
+
def try_update_basic_repositoryreference_from_examples_repository(self,repository_folder:str):
|
|
1559
|
+
source=f"https://raw.githubusercontent.com/anionDev/CommonProjectStructureExamples/refs/heads/main/Other/Reference/RepositoryStructure.mdd"
|
|
1560
|
+
target=f"{repository_folder}/Other/Reference/RepositoryStructure.md"
|
|
1561
|
+
self.download_file(source,target)
|
|
1562
|
+
|
|
1563
|
+
|
|
1564
|
+
def update_dependent_oci_images(self,repo:str):
|
|
1565
|
+
self.oci_image_manager.update_default_tag_for_images_in_image_definitions_file(repo,True)
|