scriptcollection 4.2.81__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. ScriptCollection/AnionBuildPlatform.py +199 -0
  2. ScriptCollection/CertificateUpdater.py +149 -0
  3. ScriptCollection/Executables.py +921 -0
  4. ScriptCollection/GeneralUtilities.py +1589 -0
  5. ScriptCollection/HTTPMaintenanceOverheadHelper.py +36 -0
  6. ScriptCollection/OCIImages/AbstractImageHandler.py +38 -0
  7. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py +20 -0
  8. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py +20 -0
  9. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGeneric.py +20 -0
  10. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGenericV.py +20 -0
  11. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabCE.py +20 -0
  12. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabEE.py +20 -0
  13. ScriptCollection/OCIImages/ConcreteImageHandlers/__init__.py +0 -0
  14. ScriptCollection/OCIImages/OCIImageManager.py +190 -0
  15. ScriptCollection/OCIImages/__init__.py +0 -0
  16. ScriptCollection/ProcessesRunner.py +43 -0
  17. ScriptCollection/ProgramRunnerBase.py +47 -0
  18. ScriptCollection/ProgramRunnerMock.py +2 -0
  19. ScriptCollection/ProgramRunnerPopen.py +57 -0
  20. ScriptCollection/ProgramRunnerSudo.py +108 -0
  21. ScriptCollection/Resources/CultureChooser/CultureChooser.js +29 -0
  22. ScriptCollection/Resources/CultureChooser/index.html +15 -0
  23. ScriptCollection/Resources/MaintenanceSite/MaintenanceSite.html +15 -0
  24. ScriptCollection/SCLog.py +115 -0
  25. ScriptCollection/ScriptCollectionCore.py +3485 -0
  26. ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +192 -0
  27. ScriptCollection/TFCPS/Docker/__init__.py +0 -0
  28. ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py +8 -0
  29. ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py +6 -0
  30. ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py +7 -0
  31. ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +547 -0
  32. ScriptCollection/TFCPS/DotNet/__init__.py +0 -0
  33. ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +137 -0
  34. ScriptCollection/TFCPS/Flutter/__init__.py +0 -0
  35. ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py +72 -0
  36. ScriptCollection/TFCPS/Go/__init__.py +0 -0
  37. ScriptCollection/TFCPS/Maven/TFCPS_CodeUnitSpecific_Maven.py +42 -0
  38. ScriptCollection/TFCPS/Maven/__init__.py +0 -0
  39. ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +232 -0
  40. ScriptCollection/TFCPS/NodeJS/__init__.py +0 -0
  41. ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +239 -0
  42. ScriptCollection/TFCPS/Python/__init__.py +0 -0
  43. ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py +42 -0
  44. ScriptCollection/TFCPS/Rust/__init__.py +0 -0
  45. ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +433 -0
  46. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +135 -0
  47. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +301 -0
  48. ScriptCollection/TFCPS/TFCPS_CreateRelease.py +98 -0
  49. ScriptCollection/TFCPS/TFCPS_Generic.py +44 -0
  50. ScriptCollection/TFCPS/TFCPS_MergeToMain.py +128 -0
  51. ScriptCollection/TFCPS/TFCPS_MergeToStable.py +356 -0
  52. ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +48 -0
  53. ScriptCollection/TFCPS/TFCPS_Tools_General.py +1565 -0
  54. ScriptCollection/TFCPS/__init__.py +0 -0
  55. ScriptCollection/__init__.py +0 -0
  56. ScriptCollection/__pycache__/GeneralUtilities.cpython-311.pyc +0 -0
  57. ScriptCollection/__pycache__/__init__.cpython-311.pyc +0 -0
  58. scriptcollection-4.2.81.dist-info/METADATA +169 -0
  59. scriptcollection-4.2.81.dist-info/RECORD +62 -0
  60. scriptcollection-4.2.81.dist-info/WHEEL +5 -0
  61. scriptcollection-4.2.81.dist-info/entry_points.txt +67 -0
  62. scriptcollection-4.2.81.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1565 @@
1
+ from datetime import datetime,timezone
2
+ from graphlib import TopologicalSorter
3
+ import math
4
+ import os
5
+ from pathlib import Path
6
+ import shutil
7
+ import zipfile
8
+ import tarfile
9
+ import time
10
+ import re
11
+ import sys
12
+ import json
13
+ import tempfile
14
+ import uuid
15
+ import urllib.request
16
+ from packaging import version
17
+ import requests
18
+ from lxml import etree
19
+ from ..GeneralUtilities import GeneralUtilities,Platform
20
+ from ..ScriptCollectionCore import ScriptCollectionCore,VSCodeWorkspaceShellTask
21
+ from ..SCLog import LogLevel
22
+ from ..OCIImages.AbstractImageHandler import AbstractImageHandler
23
+ from ..OCIImages.OCIImageManager import OCIImageManager
24
+
25
+ class TFCPS_Tools_General:
26
+
27
+ __sc:ScriptCollectionCore=None
28
+ oci_image_manager:OCIImageManager=None
29
+
30
+ def __init__(self,sc:ScriptCollectionCore):
31
+ self.__sc=sc
32
+ self.oci_image_manager=OCIImageManager(self.__sc)
33
+
34
+
35
+ @GeneralUtilities.check_arguments
36
+ def codeunit_is_enabled(self, codeunit_file: str) -> bool:
37
+ root: etree._ElementTree = etree.parse(codeunit_file)
38
+ return GeneralUtilities.string_to_boolean(str(root.xpath('//cps:codeunit/@enabled', namespaces={'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'})[0]))
39
+
40
+ @GeneralUtilities.check_arguments
41
+ def ensure_cyclonedxcli_is_available(self,enforce_update:bool) -> str:
42
+ local_resource_name="CycloneDXCLI"
43
+ self.ensure_file_from_github_assets_is_available_with_retry("CycloneDX", "cyclonedx-cli", local_resource_name,"cyclonedx-linux-arm64",lambda latest_version: "cyclonedx-linux-arm64",enforce_update=enforce_update)
44
+ self.ensure_file_from_github_assets_is_available_with_retry("CycloneDX", "cyclonedx-cli", local_resource_name,"cyclonedx-linux-x64",lambda latest_version: "cyclonedx-linux-x64",enforce_update=enforce_update)
45
+ self.ensure_file_from_github_assets_is_available_with_retry("CycloneDX", "cyclonedx-cli", local_resource_name,"cyclonedx-win-arm64.exe",lambda latest_version: "cyclonedx-win-arm64.exe",enforce_update=enforce_update)
46
+ self.ensure_file_from_github_assets_is_available_with_retry("CycloneDX", "cyclonedx-cli", local_resource_name, "cyclonedx-win-x64.exe",lambda latest_version: "cyclonedx-win-x64.exe",enforce_update=enforce_update)
47
+
48
+ resource_folder =os.path.join( self.__sc.get_global_cache_folder(),"Tools",local_resource_name)
49
+
50
+ is_x64:bool=GeneralUtilities.current_system_is_x64()
51
+ is_arm:bool=GeneralUtilities.current_system_is_arm64()
52
+ if GeneralUtilities.current_system_is_windows():
53
+ if is_x64:
54
+ return os.path.join(resource_folder, "cyclonedx-win-x64.exe")
55
+ elif is_arm:
56
+ return os.path.join(resource_folder, "cyclonedx-win-arm64.exe")
57
+ else:
58
+ raise ValueError("Unsupported architecture for cyclonedx-cli on windows.")
59
+ elif GeneralUtilities.current_system_is_linux():
60
+ if is_x64:
61
+ return os.path.join(resource_folder, "cyclonedx-linux-x64")
62
+ elif is_arm:
63
+ return os.path.join(resource_folder, "cyclonedx-linux-arm64")
64
+ else:
65
+ raise ValueError("Unsupported architecture for cyclonedx-cli on linux.")
66
+ else:
67
+ raise ValueError("Unsupported operating system for cyclonedx-cli.")
68
+
69
+ @GeneralUtilities.check_arguments
70
+ def ensure_file_from_github_assets_is_available_with_retry(self, githubuser: str, githubprojectname: str, local_resource_name: str, local_filename: str, get_filename_on_github, amount_of_attempts: int = 5,enforce_update:bool=False) -> str:
71
+ return GeneralUtilities.retry_action(lambda: self.ensure_file_from_github_assets_is_available(githubuser, githubprojectname, local_resource_name, local_filename, get_filename_on_github,enforce_update), amount_of_attempts)
72
+
73
+ @GeneralUtilities.check_arguments
74
+ def ensure_file_from_github_assets_is_available(self,githubuser: str, githubprojectname: str, local_resource_name: str, local_filename: str, get_filename_on_github,enforce_update:bool) -> str:
75
+ #TODO use or remove target_folder-parameter
76
+ resource_folder =os.path.join( self.__sc.get_global_cache_folder(),"Tools",local_resource_name)
77
+ file = f"{resource_folder}/{local_filename}"
78
+ file_exists = os.path.isfile(file)
79
+ if not file_exists:
80
+ self.__sc.log.log(f"Download Asset \"{githubuser}/{githubprojectname}: {local_resource_name}\" from GitHub to global cache...", LogLevel.Information)
81
+ GeneralUtilities.ensure_folder_exists_and_is_empty(resource_folder)
82
+ headers = { 'User-Agent': 'Mozilla/5.0'}
83
+ self.__add_github_api_key_if_available(headers)
84
+ url = f"https://api.github.com/repos/{githubuser}/{githubprojectname}/releases/latest"
85
+ self.__sc.log.log(f"Download \"{url}\"...", LogLevel.Debug)
86
+ time.sleep(2)
87
+ response = requests.get(url, headers=headers, allow_redirects=True, timeout=(10, 10))
88
+ response_json=response.json()
89
+ latest_version = response_json["tag_name"]
90
+ filename_on_github = get_filename_on_github(latest_version)
91
+ link = f"https://github.com/{githubuser}/{githubprojectname}/releases/download/{latest_version}/{filename_on_github}"
92
+ time.sleep(2)
93
+ with requests.get(link, headers=headers, stream=True, allow_redirects=True, timeout=(5, 600)) as r:
94
+ r.raise_for_status()
95
+ total_size = int(r.headers.get("Content-Length", 0))
96
+ downloaded = 0
97
+ with open(file, "wb") as f:
98
+ for chunk in r.iter_content(chunk_size=8192):
99
+ f.write(chunk)
100
+ show_progress: bool = False
101
+ if show_progress:
102
+ downloaded += len(chunk)
103
+ if total_size:
104
+ percent = downloaded / total_size * 100
105
+ sys.stdout.write(f"\rDownload: {percent:.2f}%")
106
+ sys.stdout.flush()
107
+ self.__sc.log.log(f"Downloaded \"{url}\".", LogLevel.Diagnostic)
108
+ GeneralUtilities.assert_file_exists(file)
109
+ return file
110
+
111
+ def __add_github_api_key_if_available(self, headers: dict):
112
+ token = os.getenv("GITHUB_TOKEN")
113
+ if token is not None:
114
+ headers["Authorization"] = f"Bearer {token}"
115
+ else:
116
+ user_folder = str(Path.home())
117
+ github_token_file: str = str(os.path.join(user_folder, ".github", "token.txt"))
118
+ if os.path.isfile(github_token_file):
119
+ token = GeneralUtilities.read_text_from_file(github_token_file)
120
+ headers["Authorization"] = f"Bearer {token}"
121
+ return headers
122
+
123
+
124
+ @GeneralUtilities.check_arguments
125
+ def is_codeunit_folder(self, codeunit_folder: str) -> bool:
126
+ repo_folder = GeneralUtilities.resolve_relative_path("..", codeunit_folder)
127
+ if not self.__sc.is_git_repository(repo_folder):
128
+ return False
129
+ codeunit_name = os.path.basename(codeunit_folder)
130
+ codeunit_file: str = os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml")
131
+ if not os.path.isfile(codeunit_file):
132
+ return False
133
+ return True
134
+
135
+ @GeneralUtilities.check_arguments
136
+ def assert_is_codeunit_folder(self, codeunit_folder: str) -> str:
137
+ repo_folder = GeneralUtilities.resolve_relative_path("..", codeunit_folder)
138
+ if not self.__sc.is_git_repository(repo_folder):
139
+ raise ValueError(f"'{codeunit_folder}' can not be a valid codeunit-folder because '{repo_folder}' is not a git-repository.")
140
+ codeunit_name = os.path.basename(codeunit_folder)
141
+ codeunit_file: str = os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml")
142
+ if not os.path.isfile(codeunit_file):
143
+ raise ValueError(f"'{codeunit_folder}' is no codeunit-folder because '{codeunit_file}' does not exist.")
144
+
145
+ @GeneralUtilities.check_arguments
146
+ def get_codeunits(self, repository_folder: str, ignore_disabled_codeunits: bool = True) -> list[str]:
147
+ codeunits_with_dependent_codeunits: dict[str, set[str]] = dict[str, set[str]]()
148
+ subfolders = GeneralUtilities.get_direct_folders_of_folder(repository_folder)
149
+ for subfolder in subfolders:
150
+ codeunit_name: str = os.path.basename(subfolder)
151
+ codeunit_file = os.path.join(subfolder, f"{codeunit_name}.codeunit.xml")
152
+ if os.path.exists(codeunit_file):
153
+ if ignore_disabled_codeunits and not self.codeunit_is_enabled(codeunit_file):
154
+ continue
155
+ codeunits_with_dependent_codeunits[codeunit_name] = self.get_dependent_code_units(codeunit_file)
156
+ sorted_codeunits = self._internal_get_sorted_codeunits_by_dict(codeunits_with_dependent_codeunits)
157
+ #TODO show warning somehow for enabled codeunits which depends on ignored codeunits
158
+ return sorted_codeunits
159
+
160
+ @GeneralUtilities.check_arguments
161
+ def repository_has_codeunits(self, repository: str, ignore_disabled_codeunits: bool = True) -> bool:
162
+ return 0<len(self.get_codeunits(repository, ignore_disabled_codeunits))
163
+
164
+ @GeneralUtilities.check_arguments
165
+ def get_dependent_code_units(self, codeunit_file: str) -> list[str]:
166
+ root: etree._ElementTree = etree.parse(codeunit_file)
167
+ result = set(root.xpath('//cps:dependentcodeunit/text()', namespaces={'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'}))
168
+ result = sorted(result)
169
+ return result
170
+
171
+ @GeneralUtilities.check_arguments
172
+ def _internal_get_sorted_codeunits_by_dict(self, codeunits: dict[str, set[str]]) -> list[str]:
173
+ sorted_codeunits = {
174
+ node: sorted(codeunits[node])
175
+ for node in sorted(codeunits)
176
+ }
177
+
178
+ ts = TopologicalSorter()
179
+ for node, deps in sorted_codeunits.items():
180
+ ts.add(node, *deps)
181
+
182
+ result_typed = list(ts.static_order())
183
+ result = [str(item) for item in result_typed]
184
+ return result
185
+
186
+ @GeneralUtilities.check_arguments
187
+ def get_unsupported_versions(self, repository_folder: str, moment: datetime) -> list[tuple[str, datetime, datetime]]:
188
+ self.__sc.assert_is_git_repository(repository_folder)
189
+ result: list[tuple[str, datetime, datetime]] = list[tuple[str, datetime, datetime]]()
190
+ for entry in self.get_versions(repository_folder):
191
+ if not (entry[1] <= moment and moment <= entry[2]):
192
+ result.append(entry)
193
+ return result
194
+
195
+
196
+ @GeneralUtilities.check_arguments
197
+ def get_versions(self, repository_folder: str) -> list[tuple[str, datetime, datetime]]:
198
+ self.__sc.assert_is_git_repository(repository_folder)
199
+ folder = os.path.join(repository_folder, "Other", "Resources", "Support")
200
+ file = os.path.join(folder, "InformationAboutSupportedVersions.csv")
201
+ result: list[(str, datetime, datetime)] = list[(str, datetime, datetime)]()
202
+ if not os.path.isfile(file):
203
+ return result
204
+ entries = GeneralUtilities.read_csv_file(file, True)
205
+ for entry in entries:
206
+ d1 = GeneralUtilities.string_to_datetime(entry[1])
207
+ if d1.tzinfo is None:
208
+ d1 = d1.replace(tzinfo=timezone.utc)
209
+ d2 = GeneralUtilities.string_to_datetime(entry[2])
210
+ if d2.tzinfo is None:
211
+ d2 = d2.replace(tzinfo=timezone.utc)
212
+ result.append((entry[0], d1, d2))
213
+ return result
214
+
215
+ @GeneralUtilities.check_arguments
216
+ def dependent_codeunit_exists(self, repository: str, codeunit: str) -> None:
217
+ codeunit_file = f"{repository}/{codeunit}/{codeunit}.codeunit.xml"
218
+ return os.path.isfile(codeunit_file)
219
+
220
+ @GeneralUtilities.check_arguments
221
+ def get_all_authors_and_committers_of_repository(self, repository_folder: str, subfolder: str = None) -> list[tuple[str, str]]:
222
+ self.__sc.is_git_or_bare_git_repository(repository_folder)
223
+ space_character = "_"
224
+ if subfolder is None:
225
+ subfolder_argument = GeneralUtilities.empty_string
226
+ else:
227
+ subfolder_argument = f" -- {subfolder}"
228
+ log_result = self.__sc.run_program("git", f'log --pretty=%aN{space_character}%aE%n%cN{space_character}%cE HEAD{subfolder_argument}', repository_folder)
229
+ plain_content: list[str] = list(
230
+ set([line for line in log_result[1].split("\n") if len(line) > 0]))
231
+ result: list[tuple[str, str]] = []
232
+ for item in plain_content:
233
+ if len(re.findall(space_character, item)) == 1:
234
+ splitted = item.split(space_character)
235
+ result.append((splitted[0], splitted[1]))
236
+ else:
237
+ raise ValueError(f'Unexpected author: "{item}"')
238
+ return result
239
+
240
+ @GeneralUtilities.check_arguments
241
+ def copy_artifacts_from_dependent_code_units(self, repo_folder: str, codeunit_name: str) -> None:
242
+ codeunit_file = os.path.join(repo_folder, codeunit_name, codeunit_name + ".codeunit.xml")
243
+ dependent_codeunits = self.get_dependent_code_units(codeunit_file)
244
+ if len(dependent_codeunits) > 0:
245
+ self.__sc.log.log(f"Get dependent artifacts for codeunit {codeunit_name}.")
246
+ dependent_codeunits_folder = os.path.join(repo_folder, codeunit_name, "Other", "Resources", "DependentCodeUnits")
247
+ GeneralUtilities.ensure_directory_does_not_exist(dependent_codeunits_folder)
248
+ for dependent_codeunit in dependent_codeunits:
249
+ target_folder = os.path.join(dependent_codeunits_folder, dependent_codeunit)
250
+ GeneralUtilities.ensure_directory_does_not_exist(target_folder)
251
+ other_folder = os.path.join(repo_folder, dependent_codeunit, "Other")
252
+ artifacts_folder = os.path.join(other_folder, "Artifacts")
253
+ GeneralUtilities.ensure_directory_exists(artifacts_folder)
254
+ GeneralUtilities.copy_content_of_folder(artifacts_folder,target_folder)
255
+
256
+
257
+ @GeneralUtilities.check_arguments
258
+ def write_version_to_codeunit_file(self, codeunit_file: str, current_version: str) -> None:
259
+ versionregex = "\\d+\\.\\d+\\.\\d+"
260
+ versiononlyregex = f"^{versionregex}$"
261
+ pattern = re.compile(versiononlyregex)
262
+ if pattern.match(current_version):
263
+ GeneralUtilities.write_text_to_file(codeunit_file, re.sub(f"<cps:version>{versionregex}<\\/cps:version>", f"<cps:version>{current_version}</cps:version>", GeneralUtilities.read_text_from_file(codeunit_file)))
264
+ else:
265
+ raise ValueError(f"Version '{current_version}' does not match version-regex '{versiononlyregex}'.")
266
+
267
+ @GeneralUtilities.check_arguments
268
+ def set_default_constants(self, codeunit_folder: str) -> None:
269
+ self.assert_is_codeunit_folder(codeunit_folder)
270
+ self.set_constant_for_curenttimestamp(codeunit_folder)
271
+ self.set_constant_for_commitid(codeunit_folder)
272
+ self.set_constant_for_commitdate(codeunit_folder)
273
+ self.set_constant_for_codeunitname(codeunit_folder)
274
+ self.set_constant_for_codeunitversion(codeunit_folder)
275
+ self.set_constant_for_codeunitmajorversion(codeunit_folder)
276
+ self.set_constant_for_description(codeunit_folder)
277
+
278
+ @GeneralUtilities.check_arguments
279
+ def set_constant_for_curenttimestamp(self, codeunit_folder: str) -> None:
280
+ self.assert_is_codeunit_folder(codeunit_folder)
281
+ timestamp = GeneralUtilities.datetime_to_string_for_logfile_entry(GeneralUtilities.get_now().astimezone(timezone.utc),False)
282
+ self.set_constant(codeunit_folder, "CurrentTimestamp", timestamp)
283
+
284
+ @GeneralUtilities.check_arguments
285
+ def set_constant_for_commitid(self, codeunit_folder: str) -> None:
286
+ self.assert_is_codeunit_folder(codeunit_folder)
287
+ repository = GeneralUtilities.resolve_relative_path("..", codeunit_folder)
288
+ commit_id = self.__sc.git_get_commit_id(repository)
289
+ self.set_constant(codeunit_folder, "CommitId", commit_id)
290
+
291
+ @GeneralUtilities.check_arguments
292
+ def set_constant_for_commitdate(self, codeunit_folder: str) -> None:
293
+ self.assert_is_codeunit_folder(codeunit_folder)
294
+ repository = GeneralUtilities.resolve_relative_path("..", codeunit_folder)
295
+ commit_date: datetime = self.__sc.git_get_commit_date(repository)
296
+ self.set_constant(codeunit_folder, "CommitDate", GeneralUtilities.datetime_to_string(commit_date))
297
+
298
+ @GeneralUtilities.check_arguments
299
+ def set_constant_for_codeunitname(self, codeunit_folder: str) -> None:
300
+ self.assert_is_codeunit_folder(codeunit_folder)
301
+ codeunit_name: str = os.path.basename(codeunit_folder)
302
+ self.set_constant(codeunit_folder, "CodeUnitName", codeunit_name)
303
+
304
+ @GeneralUtilities.check_arguments
305
+ def set_constant_for_codeunitversion(self, codeunit_folder: str) -> None:
306
+ self.assert_is_codeunit_folder(codeunit_folder)
307
+ codeunit_version: str = self.get_version_of_codeunit(os.path.join(codeunit_folder,f"{os.path.basename(codeunit_folder)}.codeunit.xml"))
308
+ self.set_constant(codeunit_folder, "CodeUnitVersion", codeunit_version)
309
+
310
+ @GeneralUtilities.check_arguments
311
+ def set_constant_for_codeunitmajorversion(self, codeunit_folder: str) -> None:
312
+ self.assert_is_codeunit_folder(codeunit_folder)
313
+ major_version = int(self.get_version_of_codeunit(os.path.join(codeunit_folder,f"{os.path.basename(codeunit_folder)}.codeunit.xml")).split(".")[0])
314
+ self.set_constant(codeunit_folder, "CodeUnitMajorVersion", str(major_version))
315
+
316
+
317
+ @GeneralUtilities.check_arguments
318
+ def get_version_of_codeunit(self,codeunit_file:str) -> None:
319
+ codeunit_file_content:str=GeneralUtilities.read_text_from_file(codeunit_file)
320
+ return self.get_version_of_codeunit_filecontent(codeunit_file_content)
321
+
322
+ @GeneralUtilities.check_arguments
323
+ def get_version_of_codeunit_filecontent(self,file_content:str) -> None:
324
+ root: etree._ElementTree = etree.fromstring(file_content.encode("utf-8"))
325
+ result = str(root.xpath('//cps:version/text()', namespaces={'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'})[0])
326
+ return result
327
+
328
+ @GeneralUtilities.check_arguments
329
+ def set_constant_for_description(self, codeunit_folder: str) -> None:
330
+ self.assert_is_codeunit_folder(codeunit_folder)
331
+ codeunit_file:str=os.path.join(codeunit_folder,f"{os.path.basename(codeunit_folder)}.codeunit.xml")
332
+ codeunit_description: str = self.get_codeunit_description(codeunit_file)
333
+ self.set_constant(codeunit_folder, "CodeUnitDescription", codeunit_description)
334
+
335
+ @GeneralUtilities.check_arguments
336
+ def get_codeunit_description(self,codeunit_file:str) -> bool:
337
+ root: etree._ElementTree = etree.parse(codeunit_file)
338
+ return str(root.xpath('//cps:properties/@description', namespaces={'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'})[0])
339
+
340
+ @GeneralUtilities.check_arguments
341
+ def set_constant(self, codeunit_folder: str, constantname: str, constant_value: str, documentationsummary: str = None, constants_valuefile: str = None) -> None:
342
+ self.assert_is_codeunit_folder(codeunit_folder)
343
+ if documentationsummary is None:
344
+ documentationsummary = GeneralUtilities.empty_string
345
+ constants_folder = os.path.join(codeunit_folder, "Other", "Resources", "Constants")
346
+ GeneralUtilities.ensure_directory_exists(constants_folder)
347
+ constants_metafile = os.path.join(constants_folder, f"{constantname}.constant.xml")
348
+ if constants_valuefile is None:
349
+ constants_valuefile_folder = constants_folder
350
+ constants_valuefile_name = f"{constantname}.value.txt"
351
+ constants_valuefiler_reference = f"./{constants_valuefile_name}"
352
+ else:
353
+ constants_valuefile_folder = os.path.dirname(constants_valuefile)
354
+ constants_valuefile_name = os.path.basename(constants_valuefile)
355
+ constants_valuefiler_reference = os.path.join(constants_valuefile_folder, constants_valuefile_name)
356
+
357
+ # TODO implement usage of self.reference_latest_version_of_xsd_when_generating_xml
358
+ GeneralUtilities.write_text_to_file(constants_metafile, f"""<?xml version="1.0" encoding="UTF-8" ?>
359
+ <cps:constant xmlns:cps="https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure" constantspecificationversion="1.1.0"
360
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/raw/main/Conventions/RepositoryStructure/CommonProjectStructure/constant.xsd">
361
+ <cps:name>{constantname}</cps:name>
362
+ <cps:documentationsummary>{documentationsummary}</cps:documentationsummary>
363
+ <cps:path>{constants_valuefiler_reference}</cps:path>
364
+ </cps:constant>""")
365
+ # TODO validate generated xml against xsd
366
+ GeneralUtilities.write_text_to_file(os.path.join(constants_valuefile_folder, constants_valuefile_name), constant_value)
367
+
368
+ @GeneralUtilities.check_arguments
369
+ def get_constant_value(self, source_codeunit_folder: str, constant_name: str) -> str:
370
+ self.assert_is_codeunit_folder(source_codeunit_folder)
371
+ value_file_relative = self.__get_constant_helper(source_codeunit_folder, constant_name, "path")
372
+ value_file = GeneralUtilities.resolve_relative_path(value_file_relative, os.path.join(source_codeunit_folder, "Other", "Resources", "Constants"))
373
+ return GeneralUtilities.read_text_from_file(value_file)
374
+
375
+ @GeneralUtilities.check_arguments
376
+ def get_constant_documentation(self, source_codeunit_folder: str, constant_name: str) -> str:
377
+ self.assert_is_codeunit_folder(source_codeunit_folder)
378
+ return self.__get_constant_helper(source_codeunit_folder, constant_name, "documentationsummary")
379
+
380
+ @GeneralUtilities.check_arguments
381
+ def __get_constant_helper(self, source_codeunit_folder: str, constant_name: str, propertyname: str) -> str:
382
+ self.assert_is_codeunit_folder(source_codeunit_folder)
383
+ root: etree._ElementTree = etree.parse(os.path.join(source_codeunit_folder, "Other", "Resources", "Constants", f"{constant_name}.constant.xml"))
384
+ results = root.xpath(f'//cps:{propertyname}/text()', namespaces={
385
+ 'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'
386
+ })
387
+ length = len(results)
388
+ if (length == 0):
389
+ return ""
390
+ elif length == 1:
391
+ return results[0]
392
+ else:
393
+ raise ValueError("Too many results found.")
394
+
395
+ @GeneralUtilities.check_arguments
396
+ def copy_licence_file(self, codeunit_folder: str) -> None:
397
+ folder_of_current_file = os.path.join(codeunit_folder,"Other")
398
+ license_file = GeneralUtilities.resolve_relative_path("../../License.txt", folder_of_current_file)
399
+ target_folder = GeneralUtilities.resolve_relative_path("Artifacts/License", folder_of_current_file)
400
+ GeneralUtilities.ensure_directory_exists(target_folder)
401
+ shutil.copy(license_file, target_folder)
402
+
403
+ @GeneralUtilities.check_arguments
404
+ def generate_diff_report(self, repository_folder: str, codeunit_name: str, current_version: str) -> None:
405
+ #TODO refactor this. if new changes (committed or uncommitted) since last git-tag: diff-report from last tag to "now". if no new changes (curren-commit==commit on a vx.y-tag): take diff from last tag to this tag
406
+ self.__sc.assert_is_git_repository(repository_folder)
407
+ codeunit_folder = os.path.join(repository_folder, codeunit_name)
408
+ target_folder = GeneralUtilities.resolve_relative_path("Other/Artifacts/DiffReport", codeunit_folder)
409
+ GeneralUtilities.ensure_directory_does_not_exist(target_folder)
410
+ GeneralUtilities.ensure_directory_exists(target_folder)
411
+ target_file_light = os.path.join(target_folder, "DiffReport.html").replace("\\", "/")
412
+ target_file_dark = os.path.join(target_folder, "DiffReportDark.html").replace("\\", "/")
413
+ src = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" # hash/id of empty git-tree
414
+ src_prefix = "Begin"
415
+ if self.__sc.get_current_git_branch_has_tag(repository_folder):
416
+ latest_tag = self.__sc.get_latest_git_tag(repository_folder)
417
+ src = self.__sc.git_get_commit_id(repository_folder, latest_tag)
418
+ src_prefix = latest_tag
419
+ dst = "HEAD"
420
+ dst_prefix = f"v{current_version}"
421
+
422
+ temp_file = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
423
+ try:
424
+ GeneralUtilities.ensure_file_does_not_exist(temp_file)
425
+ GeneralUtilities.write_text_to_file(temp_file, self.__sc.run_program("git", f'--no-pager diff --src-prefix={src_prefix}/ --dst-prefix={dst_prefix}/ {src} {dst} -- {codeunit_name}', repository_folder)[1])
426
+ styles:dict[str,str]={
427
+ "default":target_file_light,
428
+ "github-dark":target_file_dark
429
+ }
430
+ for style,target_file in styles.items():
431
+ self.__sc.run_program_argsasarray("pygmentize", ['-l', 'diff', '-f', 'html', '-O', 'full', '-o', target_file, '-P', f'style={style}', temp_file], repository_folder)
432
+ finally:
433
+ GeneralUtilities.ensure_file_does_not_exist(temp_file)
434
+
435
+ @GeneralUtilities.check_arguments
436
+ def get_version_of_project(self,repositoryfolder:str) -> str:
437
+ self.__sc.assert_is_git_repository(repositoryfolder)
438
+ return self.__sc.get_semver_version_from_gitversion(repositoryfolder)
439
+
440
+ @GeneralUtilities.check_arguments
441
+ def __try_calculate_changelog_message(self, repositoryfolder: str):
442
+ self.__sc.assert_is_git_repository(repositoryfolder)
443
+ message = self.__sc.run_program("git", "log -1 --pretty=%B", repositoryfolder)[1]
444
+ message = message.strip()
445
+ if len(message) == 0:
446
+ raise ValueError("No commit message found.")
447
+ return message
448
+
449
+ @GeneralUtilities.check_arguments
450
+ def create_changelog_entry(self, repositoryfolder: str, message: str, commit: bool, force: bool):
451
+ self.__sc.assert_is_git_repository(repositoryfolder)
452
+ if message is None:
453
+ try:
454
+ message=self.__try_calculate_changelog_message(repositoryfolder)
455
+ except:
456
+ message="Update."
457
+ random_file = os.path.join(repositoryfolder, str(uuid.uuid4()))
458
+ try:
459
+ if force and not self.__sc.git_repository_has_uncommitted_changes(repositoryfolder):
460
+ GeneralUtilities.ensure_file_exists(random_file)
461
+ current_version = self.get_version_of_project(repositoryfolder)
462
+ changelog_file = os.path.join(repositoryfolder, "Other", "Resources", "Changelog", f"v{current_version}.md")
463
+ if os.path.isfile(changelog_file):
464
+ self.__sc.log.log(f"Changelog-file '{changelog_file}' already exists.")
465
+ else:
466
+ GeneralUtilities.ensure_file_exists(changelog_file)
467
+ GeneralUtilities.write_text_to_file(changelog_file, f"""# Release notes
468
+
469
+ ## Changes
470
+
471
+ - {message}
472
+ """)
473
+ finally:
474
+ GeneralUtilities.ensure_file_does_not_exist(random_file)
475
+ if commit:
476
+ self.__sc.git_commit(repositoryfolder, f"Added changelog-file for v{current_version}.")
477
+
478
+ @GeneralUtilities.check_arguments
479
+ def merge_sbom_file_from_dependent_codeunit_into_this(self,codeunit_folder: str, codeunitname:str,dependent_codeunit_name: str,use_cache:bool) -> None:
480
+ repository_folder = GeneralUtilities.resolve_relative_path("..", codeunit_folder)
481
+ dependent_codeunit_folder = os.path.join(repository_folder, dependent_codeunit_name).replace("\\", "/")
482
+ codeunit_file:str=os.path.join(codeunit_folder,f"{codeunitname}.codeunit.xml")
483
+ dependent_codeunit_file:str=os.path.join(dependent_codeunit_folder,f"{dependent_codeunit_name}.codeunit.xml")
484
+ sbom_file = f"{repository_folder}/{codeunitname}/Other/Artifacts/BOM/{codeunitname}.{self.get_version_of_codeunit(codeunit_file)}.sbom.xml"
485
+ dependent_sbom_file = f"{repository_folder}/{dependent_codeunit_name}/Other/Artifacts/BOM/{dependent_codeunit_name}.{self.get_version_of_codeunit(dependent_codeunit_file)}.sbom.xml"
486
+ self.merge_sbom_file(repository_folder, dependent_sbom_file, sbom_file,use_cache)
487
+
488
+ @GeneralUtilities.check_arguments
489
+ def merge_sbom_file(self, repository_folder: str, source_sbom_file_relative: str, target_sbom_file_relative: str,use_cache:bool) -> None:
490
+ GeneralUtilities.assert_file_exists(os.path.join(repository_folder, source_sbom_file_relative))
491
+ GeneralUtilities.assert_file_exists(os.path.join(repository_folder, target_sbom_file_relative))
492
+ target_original_sbom_file_relative = os.path.dirname(target_sbom_file_relative)+"/"+os.path.basename(target_sbom_file_relative)+".original.xml"
493
+ os.rename(os.path.join(repository_folder, target_sbom_file_relative), os.path.join(repository_folder, target_original_sbom_file_relative))
494
+
495
+ cyclonedx_exe:str=self.ensure_cyclonedxcli_is_available(not use_cache)
496
+ self.__sc.run_program(cyclonedx_exe, f"merge --input-files {source_sbom_file_relative} {target_original_sbom_file_relative} --output-file {target_sbom_file_relative}", repository_folder)
497
+ GeneralUtilities.ensure_file_does_not_exist(os.path.join(repository_folder, target_original_sbom_file_relative))
498
+ self.__sc.format_xml_file(os.path.join(repository_folder, target_sbom_file_relative))
499
+
500
+ @GeneralUtilities.check_arguments
501
+ def codeunit_has_testable_sourcecode(self,codeunit_file:str) -> bool:
502
+ self.assert_is_codeunit_folder(os.path.dirname(codeunit_file))
503
+ root: etree._ElementTree = etree.parse(codeunit_file)
504
+ return GeneralUtilities.string_to_boolean(str(root.xpath('//cps:properties/@codeunithastestablesourcecode', namespaces={'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'})[0]))
505
+
506
+ @GeneralUtilities.check_arguments
507
+ def codeunit_has_updatable_dependencies(self,codeunit_file:str) -> bool:
508
+ self.assert_is_codeunit_folder(os.path.dirname(codeunit_file))
509
+ root: etree._ElementTree = etree.parse(codeunit_file)
510
+ return GeneralUtilities.string_to_boolean(str(root.xpath('//cps:properties/@codeunithasupdatabledependencies', namespaces={'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'})[0]))
511
+
512
+ @GeneralUtilities.check_arguments
513
+ def get_codeunit_owner_emailaddress(self,codeunit_file:str) -> None:
514
+ self.assert_is_codeunit_folder(os.path.dirname(codeunit_file))
515
+ namespaces = {'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure', 'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
516
+ root: etree._ElementTree = etree.parse(codeunit_file)
517
+ result = root.xpath('//cps:codeunit/cps:codeunitowneremailaddress/text()', namespaces=namespaces)[0]
518
+ return result
519
+
520
+ @GeneralUtilities.check_arguments
521
+ def get_codeunit_owner_name(self,codeunit_file:str) -> None:
522
+ self.assert_is_codeunit_folder(os.path.dirname(codeunit_file))
523
+ namespaces = {'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure', 'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
524
+ root: etree._ElementTree = etree.parse(codeunit_file)
525
+ result = root.xpath('//cps:codeunit/cps:codeunitownername/text()', namespaces=namespaces)[0]
526
+ return result
527
+
528
+ @GeneralUtilities.check_arguments
529
+ def generate_svg_files_from_plantuml_files_for_repository(self, repository_folder: str,use_cache:bool) -> None:
530
+ self.__sc.log.log("Generate svg-files from plantuml-files...")
531
+ self.__sc.assert_is_git_repository(repository_folder)
532
+ plantuml_jar_file=self.ensure_plantuml_is_available(not use_cache)
533
+ target_folder = os.path.join(repository_folder, "Other", "Reference")
534
+ self.__generate_svg_files_from_plantuml(target_folder, plantuml_jar_file)
535
+
536
+ @GeneralUtilities.check_arguments
537
+ def generate_svg_files_from_plantuml_files_for_codeunit(self, codeunit_folder: str,use_cache:bool) -> None:
538
+ self.assert_is_codeunit_folder(codeunit_folder)
539
+ plantuml_jar_file=self.ensure_plantuml_is_available(not use_cache)
540
+ target_folder = os.path.join(codeunit_folder, "Other", "Reference")
541
+ self.__generate_svg_files_from_plantuml(target_folder, plantuml_jar_file)
542
+
543
+ @GeneralUtilities.check_arguments
544
+ def ensure_plantuml_is_available(self,enforce_update:bool) -> str:
545
+ return self.ensure_file_from_github_assets_is_available_with_retry("plantuml", "plantuml", "PlantUML", "plantuml.jar", lambda latest_version: "plantuml.jar",enforce_update=enforce_update)
546
+
547
+ @GeneralUtilities.check_arguments
548
+ def __generate_svg_files_from_plantuml(self, diagrams_files_folder: str, plantuml_jar_file: str) -> None:
549
+ for file in GeneralUtilities.get_all_files_of_folder(diagrams_files_folder):
550
+ if file.endswith(".plantuml"):
551
+ output_filename = self.get_output_filename_for_plantuml_filename(file)
552
+ argument = ['-jar',plantuml_jar_file, '-tsvg', os.path.basename(file)]
553
+ folder = os.path.dirname(file)
554
+ self.__sc.run_program_argsasarray("java", argument, folder)
555
+ result_file = folder+"/" + output_filename
556
+ GeneralUtilities.assert_file_exists(result_file)
557
+ self.__sc.format_xml_file(result_file)
558
+
559
+ @GeneralUtilities.check_arguments
560
+ def get_output_filename_for_plantuml_filename(self, plantuml_file: str) -> str:
561
+ for line in GeneralUtilities.read_lines_from_file(plantuml_file):
562
+ prefix = "@startuml "
563
+ if line.startswith(prefix):
564
+ title = line[len(prefix):]
565
+ return title+".svg"
566
+ return Path(plantuml_file).stem+".svg"
567
+
568
+ @GeneralUtilities.check_arguments
569
+ def generate_codeunits_overview_diagram(self, repository_folder: str) -> None:
570
+ self.__sc.log.log("Generate Codeunits-overview-diagram...")
571
+ self.__sc.assert_is_git_repository(repository_folder)
572
+ project_name: str = os.path.basename(repository_folder)
573
+ target_folder = os.path.join(repository_folder, "Other", "Reference", "Technical", "Diagrams")
574
+ GeneralUtilities.ensure_directory_exists(target_folder)
575
+ target_file = os.path.join(target_folder, "CodeUnits-Overview.plantuml")
576
+ lines = ["@startuml CodeUnits-Overview"]
577
+ lines.append(f"title CodeUnits of {project_name}")
578
+
579
+ codeunits = self.get_codeunits(repository_folder)
580
+ for codeunitname in codeunits:
581
+ codeunit_file: str = os.path.join(repository_folder, codeunitname, f"{codeunitname}.codeunit.xml")
582
+
583
+ description = self.get_codeunit_description(codeunit_file)
584
+
585
+ lines.append(GeneralUtilities.empty_string)
586
+ lines.append(f"[{codeunitname}]")
587
+ lines.append(f"note as {codeunitname}Note")
588
+ lines.append(f" {description}")
589
+ lines.append(f"end note")
590
+ lines.append(f"{codeunitname} .. {codeunitname}Note")
591
+
592
+ lines.append(GeneralUtilities.empty_string)
593
+ for codeunitname in codeunits:
594
+ codeunit_file: str = os.path.join(repository_folder, codeunitname, f"{codeunitname}.codeunit.xml")
595
+ dependent_codeunits = self.get_dependent_code_units(codeunit_file)
596
+ for dependent_codeunit in dependent_codeunits:
597
+ lines.append(f"{codeunitname} --> {dependent_codeunit}")
598
+
599
+ lines.append(GeneralUtilities.empty_string)
600
+ lines.append("@enduml")
601
+
602
+ GeneralUtilities.write_lines_to_file(target_file, lines)
603
+
604
+ @GeneralUtilities.check_arguments
605
+ def ensure_trufflehog_is_available(self,enforce_update:bool=False) -> dict[str,str]:
606
+ def download_and_extract(osname: str, osname_in_github_asset: str, extension: str):
607
+ resource_name: str = f"TruffleHog_{osname}"
608
+ zip_filename: str = f"{resource_name}.{extension}"
609
+ target_folder_unextracted = os.path.join(self.__sc.get_global_cache_folder(),"Tools",resource_name+"_Unextracted")
610
+ target_folder_extracted = os.path.join(self.__sc.get_global_cache_folder(),"Tools",resource_name)
611
+ update:bool=not os.path.isdir(target_folder_extracted) or GeneralUtilities.folder_is_empty(target_folder_extracted) or enforce_update
612
+ if update:
613
+ downloaded_file=self.ensure_file_from_github_assets_is_available_with_retry( "trufflesecurity", "trufflehog", resource_name+"_Unextracted", zip_filename, lambda latest_version: f"trufflehog_{latest_version[1:]}_{osname_in_github_asset}_amd64.tar.gz",enforce_update=enforce_update)
614
+ #TODO add option to also download arm-version
615
+ local_zip_file: str = downloaded_file
616
+ GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder_extracted)
617
+ if extension == "zip":
618
+ with zipfile.ZipFile(local_zip_file, 'r') as zip_ref:
619
+ zip_ref.extractall(target_folder_extracted)
620
+ elif extension == "tar.gz":
621
+ with tarfile.open(local_zip_file, "r:gz") as tar:
622
+ tar.extractall(path=target_folder_extracted)
623
+ else:
624
+ raise ValueError(f"Unknown extension: \"{extension}\"")
625
+ GeneralUtilities.ensure_directory_does_not_exist(target_folder_unextracted)
626
+ GeneralUtilities.assert_folder_exists(target_folder_extracted)
627
+ executable=[f for f in GeneralUtilities.get_all_files_of_folder(target_folder_extracted) if os.path.basename(f).startswith("trufflehog")][0]
628
+ return executable
629
+
630
+ result=dict[str,str]()
631
+ result["Windows"]=download_and_extract("Windows", "windows", "tar.gz")
632
+ result["Linux"]=download_and_extract("Linux", "linux", "tar.gz")
633
+ result["MacOS"]=download_and_extract("MacOS", "darwin", "tar.gz")
634
+ return result
635
+
636
+ @GeneralUtilities.check_arguments
637
+ def generate_tasksfile_from_workspace_file(self, repository_folder: str, append_cli_args_at_end: bool = False) -> None:
638
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
639
+ if self.__sc.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
640
+ self.__sc.log.log("Generate taskfile from code-workspace-file...")
641
+ self.__sc.assert_is_git_repository(repository_folder)
642
+ workspace_file: str = self.__sc.find_file_by_extension(repository_folder, "code-workspace")
643
+ task_file: str = repository_folder + "/Taskfile.yml"
644
+ lines: list[str] = [
645
+ "version: '3'", GeneralUtilities.empty_string,
646
+ "tasks:", GeneralUtilities.empty_string,
647
+ ]
648
+ tasks = self.__sc.parse_tasks_from_codeworkspace_file(workspace_file)
649
+ tasks.sort(key=lambda task: task.label, reverse=False)
650
+ for t in tasks:
651
+ task:VSCodeWorkspaceShellTask = t
652
+ lines.append(f" {GeneralUtilities.escape_yaml_property_value(task.label)}:")
653
+ if task.description is not None:
654
+ lines.append(f' desc: "{GeneralUtilities.escape_yaml_string_value(task.description)}"')
655
+ lines.append(' silent: true')
656
+ if task.work_dir is not None:
657
+ lines.append(f' dir: "{GeneralUtilities.escape_yaml_string_value(task.work_dir)}"')
658
+ lines.append(" cmds:")
659
+ command=GeneralUtilities.escape_yaml_string_value(task.command)
660
+ if task.allow_custom_arguments:
661
+ command=command+" {{.CLI_ARGS}}"
662
+ lines.append(f' - "{command}"')
663
+ if task.aliases!=None and len(task.aliases) > 0:
664
+ lines.append(" aliases:")
665
+ for alias in task.aliases:
666
+ lines.append(f' - {GeneralUtilities.escape_yaml_property_value(alias)}')
667
+ lines.append(GeneralUtilities.empty_string)
668
+ self.__sc.set_file_content(task_file, "\n".join(lines))
669
+ else:
670
+ self.__sc.run_program("scgeneratetasksfilefromworkspacefile", f"--repositoryfolder {repository_folder}")
671
+
672
+ @GeneralUtilities.check_arguments
673
+ def ensure_androidappbundletool_is_available(self, target_folder: str,enforce_update:bool) -> str:
674
+ return self.ensure_file_from_github_assets_is_available_with_retry( "google", "bundletool", "AndroidAppBundleTool", "bundletool.jar", lambda latest_version: f"bundletool-all-{latest_version}.jar",enforce_update=enforce_update)
675
+
676
+ @GeneralUtilities.check_arguments
677
+ def ensure_mediamtx_is_available(self, target_folder: str,enforce_update:bool) -> None:
678
+ def download_and_extract(osname: str, osname_in_github_asset: str, extension: str,architecture:Platform):
679
+ resource_name: str = f"MediaMTX_{GeneralUtilities.platform_to_dash_str(architecture)}"
680
+ resource_folder: str = os.path.join(target_folder, "Other", "Resources", resource_name)
681
+ target_folder_extracted = os.path.join(resource_folder, "MediaMTX")
682
+ update:bool=not os.path.isdir(target_folder_extracted) or GeneralUtilities.folder_is_empty(target_folder_extracted) or enforce_update
683
+ if update:
684
+ platform_str:str=None
685
+ match architecture:
686
+ case Platform.Windows_AMD64:
687
+ platform_str = "windows_amd64"
688
+ case Platform.Linux_ARM64:
689
+ platform_str = "linux_arm64"
690
+ case Platform.Linux_AMD64:
691
+ platform_str = "linux_amd64"
692
+ case Platform.MacOS_ARM64:
693
+ platform_str = "darwin_arm64"
694
+ case _:
695
+ raise ValueError(f"Unknown platform: {str(architecture)}")
696
+
697
+ resource_filename_name_remote:str=f"mediamtx_{platform_str}.{extension}"
698
+ resource_name_local:str=f"MediaMTCX_{platform_str}"
699
+ global_cache_file=os.path.join( self.__sc.get_global_cache_folder(),"Tools",resource_name_local,resource_filename_name_remote)
700
+ if (not os.path.isfile( global_cache_file )) or enforce_update:
701
+ self.ensure_file_from_github_assets_is_available_with_retry( "bluenviron", "mediamtx", resource_name_local, resource_filename_name_remote, lambda latest_version: f"mediamtx_{latest_version}_{platform_str}.{extension}",enforce_update=enforce_update)
702
+ GeneralUtilities.assert_file_exists(global_cache_file)
703
+ GeneralUtilities.assert_file_exists(global_cache_file)
704
+ GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder_extracted)
705
+ if extension == "zip":
706
+ with zipfile.ZipFile(global_cache_file, 'r') as zip_ref:
707
+ zip_ref.extractall(target_folder_extracted)
708
+ elif extension == "tar.gz":
709
+ with tarfile.open(global_cache_file, "r:gz") as tar:
710
+ tar.extractall(path=target_folder_extracted)
711
+ else:
712
+ raise ValueError(f"Unknown extension: \"{extension}\"")
713
+
714
+ download_and_extract("Windows", "windows", "zip",Platform.Windows_AMD64)
715
+ download_and_extract("Linux", "linux", "tar.gz",Platform.Linux_AMD64)
716
+ download_and_extract("Linux", "linux", "tar.gz",Platform.Linux_ARM64)
717
+ download_and_extract("MacOS", "darwin", "tar.gz",Platform.MacOS_ARM64)
718
+
719
+ @GeneralUtilities.check_arguments
720
+ def clone_repository_as_resource(self, local_repository_folder: str, remote_repository_link: str, resource_name: str, repository_subname: str = None,use_cache:bool=True) -> None:
721
+ self.__sc.log.log(f'Clone resource {resource_name}...')
722
+ resrepo_commit_id_folder: str = os.path.join(local_repository_folder, "Other", "Resources", f"{resource_name}Version")
723
+ resrepo_commit_id_file: str = os.path.join(resrepo_commit_id_folder, f"{resource_name}Version.txt")
724
+ latest_version: str = GeneralUtilities.read_text_from_file(resrepo_commit_id_file)
725
+ resrepo_data_folder: str = os.path.join(local_repository_folder, "Other", "Resources", resource_name).replace("\\", "/")
726
+ current_version: str = None
727
+ resrepo_data_version: str = os.path.join(resrepo_data_folder, f"{resource_name}Version.txt")
728
+ if os.path.isdir(resrepo_data_folder):
729
+ if os.path.isfile(resrepo_data_version):
730
+ current_version = GeneralUtilities.read_text_from_file(resrepo_data_version)
731
+ if (current_version is None) or (current_version != latest_version):
732
+ target_folder: str = resrepo_data_folder
733
+ if repository_subname is not None:
734
+ target_folder = f"{resrepo_data_folder}/{repository_subname}"
735
+
736
+ update:bool=not os.path.isdir(target_folder) or GeneralUtilities.folder_is_empty(target_folder) or not use_cache
737
+ if update:
738
+ self.__sc.log.log(f"Clone {remote_repository_link} as resource...", LogLevel.Information)
739
+ GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder)
740
+ self.__sc.run_program("git", f"clone --recurse-submodules {remote_repository_link} {target_folder}")
741
+ self.__sc.run_program("git", f"checkout {latest_version}", target_folder)
742
+ GeneralUtilities.write_text_to_file(resrepo_data_version, latest_version)
743
+
744
+ git_folders: list[str] = []
745
+ git_files: list[str] = []
746
+ for dirpath, dirnames, filenames in os.walk(target_folder):
747
+ for dirname in dirnames:
748
+ if dirname == ".git":
749
+ full_path = os.path.join(dirpath, dirname)
750
+ git_folders.append(full_path)
751
+ for filename in filenames:
752
+ if filename == ".git":
753
+ full_path = os.path.join(dirpath, filename)
754
+ git_files.append(full_path)
755
+ for git_folder in git_folders:
756
+ if os.path.isdir(git_folder):
757
+ GeneralUtilities.ensure_directory_does_not_exist(git_folder)
758
+ for git_file in git_files:
759
+ if os.path.isdir(git_file):
760
+ GeneralUtilities.ensure_file_does_not_exist(git_file)
761
+
762
+ @GeneralUtilities.check_arguments
763
+ def ensure_certificate_authority_for_development_purposes_is_generated(self, product_folder: str):
764
+ product_name: str = os.path.basename(product_folder)
765
+ now = GeneralUtilities.get_now()
766
+ ca_name = f"{product_name}CA_{now.year:04}{now.month:02}{now.day:02}{now.hour:02}{now.min:02}{now.second:02}"
767
+ ca_folder = os.path.join(product_folder, "Other", "Resources", "CA")
768
+ generate_certificate = True
769
+ if os.path.isdir(ca_folder):
770
+ ca_files = [file for file in GeneralUtilities.get_direct_files_of_folder(ca_folder) if file.endswith(".crt")]
771
+ if len(ca_files) > 0:
772
+ ca_file = ca_files[-1] # pylint:disable=unused-variable
773
+ certificate_is_valid = True # TODO check if certificate is really valid
774
+ generate_certificate = not certificate_is_valid
775
+ if generate_certificate:
776
+ self.__sc.generate_certificate_authority(ca_folder, ca_name, "DE", "SubjST", "SubjL", "SubjO", "SubjOU")
777
+ # TODO add switch to auto-install the script if desired
778
+ # for windows: powershell Import-Certificate -FilePath MyProjectCA_20241121000236.crt -CertStoreLocation 'Cert:\CurrentUser\Root'
779
+ # for linux: (TODO)
780
+
781
+ @GeneralUtilities.check_arguments
782
+ def generate_certificate_for_development_purposes_for_product(self, repository_folder: str):
783
+ self.__sc.assert_is_git_repository(repository_folder)
784
+ product_name = os.path.basename(repository_folder)
785
+ ca_folder: str = os.path.join(repository_folder, "Other", "Resources", "CA")
786
+ self.__generate_certificate_for_development_purposes(product_name, os.path.join(repository_folder, "Other", "Resources"), ca_folder, None)
787
+
788
+ @GeneralUtilities.check_arguments
789
+ def __generate_certificate_for_development_purposes(self, service_name: str, resources_folder: str, ca_folder: str, domain: str = None):
790
+ if domain is None:
791
+ domain = f"{service_name}.test.local"
792
+ domain = domain.lower()
793
+ resource_name: str = "DevelopmentCertificate"
794
+ certificate_folder: str = os.path.join(resources_folder, resource_name)
795
+
796
+ resource_content_filename: str = service_name+resource_name
797
+ certificate_file = os.path.join(certificate_folder, f"{domain}.crt")
798
+ unsignedcertificate_file = os.path.join(certificate_folder, f"{domain}.unsigned.crt")
799
+ certificate_exists = os.path.exists(certificate_file)
800
+ if certificate_exists:
801
+ certificate_expired = GeneralUtilities.certificate_is_expired(certificate_file)
802
+ generate_new_certificate = certificate_expired
803
+ else:
804
+ generate_new_certificate = True
805
+ if generate_new_certificate:
806
+ GeneralUtilities.ensure_directory_does_not_exist(certificate_folder)
807
+ GeneralUtilities.ensure_directory_exists(certificate_folder)
808
+ self.__sc.log.log("Generate TLS-certificate for development-purposes...")
809
+ self.__sc.generate_certificate(certificate_folder, domain, resource_content_filename, "DE", "SubjST", "SubjL", "SubjO", "SubjOU")
810
+ self.__sc.generate_certificate_sign_request(certificate_folder, domain, resource_content_filename, "DE", "SubjST", "SubjL", "SubjO", "SubjOU")
811
+ ca_name = os.path.basename(self.__sc.find_last_file_by_extension(ca_folder, "crt"))[:-4]
812
+ self.__sc.sign_certificate(certificate_folder, ca_folder, ca_name, domain, resource_content_filename)
813
+ GeneralUtilities.ensure_file_does_not_exist(unsignedcertificate_file)
814
+ self.__sc.log.log("Finished generating TLS-certificate for development-purposes...",LogLevel.Debug)
815
+
816
+
817
+ @GeneralUtilities.check_arguments
818
+ def do_npm_install(self, package_json_folder: str, npm_force: bool,use_cache:bool) -> None:
819
+ target_folder:str=os.path.join(package_json_folder,"node_modules")
820
+ update:bool=not os.path.isdir(target_folder) or GeneralUtilities.folder_is_empty(target_folder) or not use_cache
821
+ if update:
822
+ self.__sc.log.log("Do npm-install...")
823
+ argument1 = "install"
824
+ if npm_force:
825
+ argument1 = f"{argument1} --force"
826
+ self.__sc.run_with_epew("npm", argument1, package_json_folder)
827
+
828
+ argument2 = "install --package-lock-only"
829
+ if npm_force:
830
+ argument2 = f"{argument2} --force"
831
+ self.__sc.run_with_epew("npm", argument2, package_json_folder)
832
+
833
+ argument3 = "clean-install"
834
+ if npm_force:
835
+ argument3 = f"{argument3} --force"
836
+ self.__sc.run_with_epew("npm", argument3, package_json_folder)
837
+
838
+ @staticmethod
839
+ @GeneralUtilities.check_arguments
840
+ def sort_reference_folder(folder1: str, folder2: str) -> int:
841
+ """Returns a value greater than 0 if and only if folder1 has a base-folder-name with a with a higher version than the base-folder-name of folder2.
842
+ Returns a value lower than 0 if and only if folder1 has a base-folder-name with a with a lower version than the base-folder-name of folder2.
843
+ Returns 0 if both values are equal."""
844
+ if (folder1 == folder2):
845
+ return 0
846
+
847
+ version_identifier_1 = os.path.basename(folder1)
848
+ if version_identifier_1 == "Latest":
849
+ return -1
850
+ version_identifier_1 = version_identifier_1[1:]
851
+
852
+ version_identifier_2 = os.path.basename(folder2)
853
+ if version_identifier_2 == "Latest":
854
+ return 1
855
+ version_identifier_2 = version_identifier_2[1:]
856
+
857
+ if version.parse(version_identifier_1) < version.parse(version_identifier_2):
858
+ return -1
859
+ elif version.parse(version_identifier_1) > version.parse(version_identifier_2):
860
+ return 1
861
+ else:
862
+ return 0
863
+
864
+ @GeneralUtilities.check_arguments
865
+ def t4_transform(self, codeunit_folder: str, ignore_git_ignored_files: bool ,use_cache:bool):
866
+ grylib_dll:str=self.__ensure_grylibrary_is_available(use_cache)
867
+ repository_folder: str = os.path.dirname(codeunit_folder)
868
+ codeunitname: str = os.path.basename(codeunit_folder)
869
+ codeunit_folder = os.path.join(repository_folder, codeunitname)
870
+ for search_result in Path(codeunit_folder).glob('**/*.tt'):
871
+ tt_file = str(search_result)
872
+ relative_path_to_tt_file_from_repository = str(Path(tt_file).relative_to(repository_folder))
873
+ if (not ignore_git_ignored_files) or (ignore_git_ignored_files and not self.__sc.file_is_git_ignored(relative_path_to_tt_file_from_repository, repository_folder)):
874
+ relative_path_to_tt_file_from_codeunit_file = str(Path(tt_file).relative_to(codeunit_folder))
875
+ argument = [f"--parameter=repositoryFolder={repository_folder}", f"--parameter=codeUnitName={codeunitname}", f"--parameter=gryLibraryDLLFile={grylib_dll}", relative_path_to_tt_file_from_codeunit_file]
876
+ self.__sc.run_program_argsasarray("t4", argument, codeunit_folder)
877
+
878
+ @GeneralUtilities.check_arguments
879
+ def __ensure_grylibrary_is_available(self, use_cache:bool) -> None:
880
+ grylibrary_folder =os.path.join( self.__sc.get_global_cache_folder(),"Tools","GRYLibrary")
881
+ grylibrary_dll_file = os.path.join(grylibrary_folder, "BuildResult_DotNet_win-x64", "GRYLibrary.dll")
882
+ grylibrary_dll_file_exists = os.path.isfile(grylibrary_dll_file)
883
+ if not os.path.isfile(grylibrary_dll_file):
884
+ self.__sc.log.log("Download GRYLibrary to global cache...",LogLevel.Information)
885
+ grylibrary_latest_codeunit_file = "https://raw.githubusercontent.com/anionDev/GRYLibrary/stable/GRYLibrary/GRYLibrary.codeunit.xml"
886
+ with urllib.request.urlopen(grylibrary_latest_codeunit_file) as url_result:
887
+ grylibrary_latest_version = self.get_version_of_codeunit_filecontent(url_result.read().decode("utf-8"))
888
+ if grylibrary_dll_file_exists:
889
+ grylibrary_existing_codeunit_file = os.path.join(grylibrary_folder, "SourceCode", "GRYLibrary.codeunit.xml")
890
+ grylibrary_existing_codeunit_version = self.get_version_of_codeunit(grylibrary_existing_codeunit_file)
891
+ if grylibrary_existing_codeunit_version != grylibrary_latest_version:
892
+ GeneralUtilities.ensure_directory_does_not_exist(grylibrary_folder)
893
+ GeneralUtilities.ensure_directory_does_not_exist(grylibrary_folder)
894
+ GeneralUtilities.ensure_directory_exists(grylibrary_folder)
895
+ archive_name = f"GRYLibrary.v{grylibrary_latest_version}.Artifacts.zip"
896
+ archive_download_link = f"https://github.com/anionDev/GRYLibrary/releases/download/v{grylibrary_latest_version}/{archive_name}"
897
+ archive_file = os.path.join(grylibrary_folder, archive_name)
898
+ urllib.request.urlretrieve(archive_download_link, archive_file)
899
+ with zipfile.ZipFile(archive_file, 'r') as zip_ref:
900
+ zip_ref.extractall(grylibrary_folder)
901
+ GeneralUtilities.ensure_file_does_not_exist(archive_file)
902
+ GeneralUtilities.assert_file_exists(grylibrary_dll_file)
903
+ return grylibrary_dll_file
904
+
905
+ @GeneralUtilities.check_arguments
906
+ def ensure_ffmpeg_is_available(self, codeunit_folder: str,use_cache:bool) -> None:
907
+ self.assert_is_codeunit_folder(codeunit_folder)
908
+ ffmpeg_folder = os.path.join(codeunit_folder, "Other", "Resources", "FFMPEG")
909
+ internet_connection_is_available = GeneralUtilities.internet_connection_is_available()
910
+ exe_file = f"{ffmpeg_folder}/ffmpeg.exe"
911
+ exe_file_exists = os.path.isfile(exe_file)
912
+ update:bool=(not exe_file_exists) or (not use_cache)
913
+ if update:
914
+ if internet_connection_is_available: # Load/Update
915
+ GeneralUtilities.ensure_directory_does_not_exist(ffmpeg_folder)
916
+ GeneralUtilities.ensure_directory_exists(ffmpeg_folder)
917
+ ffmpeg_temp_folder = ffmpeg_folder+"Temp"
918
+ GeneralUtilities.ensure_directory_does_not_exist(ffmpeg_temp_folder)
919
+ GeneralUtilities.ensure_directory_exists(ffmpeg_temp_folder)
920
+ zip_file_on_disk = os.path.join(ffmpeg_temp_folder, "ffmpeg.zip")
921
+ original_zip_filename = "ffmpeg-master-latest-win64-gpl-shared"
922
+ zip_link = f"https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/{original_zip_filename}.zip"
923
+ urllib.request.urlretrieve(zip_link, zip_file_on_disk)
924
+ shutil.unpack_archive(zip_file_on_disk, ffmpeg_temp_folder)
925
+ bin_folder_source = os.path.join(ffmpeg_temp_folder, "ffmpeg-master-latest-win64-gpl-shared/bin")
926
+ bin_folder_target = ffmpeg_folder
927
+ GeneralUtilities.copy_content_of_folder(bin_folder_source, bin_folder_target)
928
+ GeneralUtilities.ensure_directory_does_not_exist(ffmpeg_temp_folder)
929
+ else:
930
+ if exe_file_exists:
931
+ self.__sc.log.log("Can not check for updates of FFMPEG due to missing internet-connection.")
932
+ else:
933
+ raise ValueError("Can not download FFMPEG.")
934
+
935
+ @GeneralUtilities.check_arguments
936
+ def set_constants_for_certificate_private_information(self, codeunit_folder: str) -> None:
937
+ """Expects a certificate-resource and generates a constant for its sensitive information in hex-format"""
938
+ self.assert_is_codeunit_folder(codeunit_folder)
939
+ repo_name:str=os.path.basename(GeneralUtilities.resolve_relative_path("..",codeunit_folder))
940
+ resource_name: str = "DevelopmentCertificate"
941
+ filename: str = repo_name+"DevelopmentCertificate"
942
+ self.generate_constant_from_resource_by_filename(codeunit_folder, resource_name, f"{filename}.pfx", "PFX")
943
+ self.generate_constant_from_resource_by_filename(codeunit_folder, resource_name, f"{filename}.password", "Password")
944
+
945
+ @GeneralUtilities.check_arguments
946
+ def generate_constant_from_resource_by_filename(self, codeunit_folder: str, resource_name: str, filename: str, constant_name: str) -> None:
947
+ self.assert_is_codeunit_folder(codeunit_folder)
948
+ certificate_resource_folder = GeneralUtilities.resolve_relative_path(f"Other/Resources/{resource_name}", codeunit_folder)
949
+ resource_file = os.path.join(certificate_resource_folder, filename)
950
+ resource_file_content = GeneralUtilities.read_binary_from_file(resource_file)
951
+ resource_file_as_hex = resource_file_content.hex()
952
+ self.set_constant(codeunit_folder, f"{resource_name}{constant_name}Hex", resource_file_as_hex)
953
+
954
+ @GeneralUtilities.check_arguments
955
+ def get_resource_from_global_resource(self, codeunit_folder: str, resource_name: str):
956
+ repository_folder: str = GeneralUtilities.resolve_relative_path("..", codeunit_folder)
957
+ source_folder: str = os.path.join(repository_folder, "Other", "Resources", resource_name)
958
+ target_folder: str = os.path.join(codeunit_folder, "Other", "Resources", resource_name)
959
+ GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder)
960
+ GeneralUtilities.copy_content_of_folder(source_folder, target_folder)
961
+
962
+
963
+ @GeneralUtilities.check_arguments
964
+ def merge_packages(self,coverage_file:str,package_name:str) -> None:
965
+ tree = etree.parse(coverage_file)
966
+ root = tree.getroot()
967
+ packages = root.findall("./packages/package")
968
+ all_classes = []
969
+ for pkg in packages:
970
+ pkg_name:str=pkg.get("name")
971
+ if len(packages)==1 or ( pkg_name==package_name or pkg_name.startswith(f"{package_name}.")):
972
+ classes = pkg.find("classes")
973
+ if classes is not None:
974
+ all_classes.extend(classes.findall("class"))
975
+ new_package = etree.Element("package", name=package_name)
976
+ new_classes = etree.SubElement(new_package, "classes")
977
+ for cls in all_classes:
978
+ new_classes.append(cls)
979
+ packages_node = root.find("./packages")
980
+ packages_node.clear()
981
+ packages_node.append(new_package)
982
+ tree.write(coverage_file, pretty_print=True, xml_declaration=True, encoding="UTF-8")
983
+ self.calculate_entire_line_rate(coverage_file)
984
+
985
+
986
+ @GeneralUtilities.check_arguments
987
+ def calculate_entire_line_rate(self,coverage_file:str) -> None:
988
+ tree = etree.parse(coverage_file)
989
+ root = tree.getroot()
990
+ package = root.find("./packages/package")
991
+ if package is None:
992
+ raise RuntimeError("No <package>-Element found")
993
+
994
+ line_elements = package.findall(".//line")
995
+
996
+ amount_of_lines = 0
997
+ amount_of_hited_lines = 0
998
+
999
+ for line in line_elements:
1000
+ amount_of_lines += 1
1001
+ hits = int(line.get("hits", "0"))
1002
+ if hits > 0:
1003
+ amount_of_hited_lines += 1
1004
+ line_rate = amount_of_hited_lines / amount_of_lines if amount_of_lines > 0 else 0.0
1005
+ package.set("line-rate", str(line_rate))
1006
+ tree.write(coverage_file, pretty_print=True, xml_declaration=True, encoding="UTF-8")
1007
+
1008
+
1009
+ @GeneralUtilities.check_arguments
1010
+ def generate_api_client_from_dependent_codeunit_with_default_properties(self, codeunit_folder:str, name_of_api_providing_codeunit: str, target_subfolder_in_codeunit: str,language:str,use_cache:bool) -> None:
1011
+ self.generate_api_client_from_dependent_codeunit(codeunit_folder,name_of_api_providing_codeunit,target_subfolder_in_codeunit,language,use_cache,["models","apis"])
1012
+
1013
+ @GeneralUtilities.check_arguments
1014
+ def generate_api_client_from_dependent_codeunit(self, codeunit_folder:str, name_of_api_providing_codeunit: str, target_subfolder_in_codeunit: str,language:str,use_cache:bool,properties:list[str]) -> None:
1015
+ openapigenerator_jar_file = self.ensure_openapigenerator_is_available(use_cache)
1016
+ openapi_spec_file = os.path.join(codeunit_folder, "Other", "Resources", "DependentCodeUnits", name_of_api_providing_codeunit, "APISpecification", f"{name_of_api_providing_codeunit}.latest.api.json")
1017
+ target_folder = os.path.join(codeunit_folder, target_subfolder_in_codeunit)
1018
+ GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder)
1019
+ argument=f'-jar {openapigenerator_jar_file} generate -i {openapi_spec_file} -g {language} -o {target_folder}'
1020
+ for property_value in properties:
1021
+ argument=f"{argument} --global-property {property_value}"
1022
+ self.__sc.run_program("java",argument , codeunit_folder)
1023
+
1024
+ @GeneralUtilities.check_arguments
1025
+ def replace_version_in_packagejson_file(self, packagejson_file: str, codeunit_version: str) -> None:
1026
+ encoding = "utf-8"
1027
+ with open(packagejson_file, encoding=encoding) as f:
1028
+ data = json.load(f)
1029
+ data['version'] = codeunit_version
1030
+ with open(packagejson_file, 'w', encoding=encoding) as f:
1031
+ json.dump(data, f, indent=2)
1032
+ GeneralUtilities.write_text_to_file(packagejson_file, GeneralUtilities.read_text_from_file(packagejson_file).replace("\r", ""))
1033
+
1034
+ @GeneralUtilities.check_arguments
1035
+ def ensure_openapigenerator_is_available(self,use_cache:bool) -> None:
1036
+ openapigenerator_folder = os.path.join(self.__sc.get_global_cache_folder(), "Tools", "OpenAPIGenerator")
1037
+ filename = "open-api-generator.jar"
1038
+ jar_file = f"{openapigenerator_folder}/{filename}"
1039
+ jar_file_exists = os.path.isfile(jar_file)
1040
+ update:bool=not jar_file_exists or not use_cache
1041
+ if update:
1042
+ self.__sc.log.log("Download OpenAPIGeneratorCLI...",LogLevel.Debug)
1043
+ used_version ="7.16.0"#TODO retrieve latest version
1044
+ download_link = f"https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/{used_version}/openapi-generator-cli-{used_version}.jar"
1045
+ GeneralUtilities.ensure_directory_does_not_exist(openapigenerator_folder)
1046
+ GeneralUtilities.ensure_directory_exists(openapigenerator_folder)
1047
+ urllib.request.urlretrieve(download_link, jar_file)
1048
+ GeneralUtilities.assert_file_exists(jar_file)
1049
+ return jar_file
1050
+
1051
+ @GeneralUtilities.check_arguments
1052
+ def standardized_tasks_update_version_in_docker_examples(self, codeunit_folder:str, codeunit_version:str) -> None:
1053
+ codeunit_name = os.path.basename(codeunit_folder)
1054
+ codeunit_name_lower = codeunit_name.lower()
1055
+ examples_folder = GeneralUtilities.resolve_relative_path("Other/Reference/ReferenceContent/Examples", codeunit_folder)
1056
+ for example_folder in GeneralUtilities.get_direct_folders_of_folder(examples_folder):
1057
+ docker_compose_file = os.path.join(example_folder, "docker-compose.yml")
1058
+ if os.path.isfile(docker_compose_file):
1059
+ filecontent = GeneralUtilities.read_text_from_file(docker_compose_file)
1060
+ replaced = re.sub(f'image:\\s+{codeunit_name_lower}:\\d+\\.\\d+\\.\\d+', f"image: {codeunit_name_lower}:{codeunit_version}", filecontent)
1061
+ GeneralUtilities.write_text_to_file(docker_compose_file, replaced)
1062
+
1063
+ @GeneralUtilities.check_arguments
1064
+ def set_version_of_openapigenerator(self, codeunit_folder: str, used_version: str = None) -> None:
1065
+ target_folder: str = os.path.join(codeunit_folder, "Other", "Resources", "Dependencies", "OpenAPIGenerator")
1066
+ version_file = os.path.join(target_folder, "Version.txt")
1067
+ GeneralUtilities.ensure_directory_exists(target_folder)
1068
+ GeneralUtilities.ensure_file_exists(version_file)
1069
+ GeneralUtilities.write_text_to_file(version_file, used_version)
1070
+
1071
+ @GeneralUtilities.check_arguments
1072
+ def get_latest_version_of_openapigenerator(self) -> None:
1073
+ headers = {'Cache-Control': 'no-cache'}
1074
+ self.__add_github_api_key_if_available(headers)
1075
+ response = requests.get(f"https://api.github.com/repos/OpenAPITools/openapi-generator/releases", headers=headers, timeout=(10, 10))
1076
+ latest_version = response.json()["tag_name"]
1077
+ return latest_version
1078
+
1079
+ @GeneralUtilities.check_arguments
1080
+ def update_images_in_example_with_default_excluded(self, codeunit_folder: str,custom_updater:AbstractImageHandler):
1081
+ self.update_images_in_example(codeunit_folder,[],custom_updater)
1082
+
1083
+ @GeneralUtilities.check_arguments
1084
+ def update_images_in_example(self, codeunit_folder: str,excluded:list[str],custom_updater:AbstractImageHandler):
1085
+ #only the version of the project itself must be updated. dependencies like postgresql or adminer for example should be updated by the usual used-image-update-mechanism
1086
+ dockercomposefile: str = f"{codeunit_folder}\\Other\\Reference\\ReferenceContent\\Examples\\MinimalDockerComposeFile\\docker-compose.yml"
1087
+ GeneralUtilities.assert_file_exists(dockercomposefile)
1088
+ #TODO update images in docker-compose files
1089
+
1090
+ @GeneralUtilities.check_arguments
1091
+ def push_wheel_build_artifact(self, push_build_artifacts_file, codeunitname, repository: str, apikey: str, gpg_identity: str, repository_folder_name: str,verbosity:LogLevel) -> None:
1092
+ folder_of_this_file = os.path.dirname(push_build_artifacts_file)
1093
+ repository_folder = GeneralUtilities.resolve_relative_path(f"..{os.path.sep}../Submodules{os.path.sep}{repository_folder_name}", folder_of_this_file)
1094
+ wheel_file = self.get_wheel_file(repository_folder, codeunitname)
1095
+ self.__standardized_tasks_push_wheel_file_to_registry(wheel_file, apikey, repository, gpg_identity,verbosity)
1096
+
1097
+ @GeneralUtilities.check_arguments
1098
+ def get_wheel_file(self, repository_folder: str, codeunit_name: str) -> str:
1099
+ self.__sc.assert_is_git_repository(repository_folder)
1100
+ return self.__sc.find_file_by_extension(os.path.join(repository_folder, codeunit_name,"Other","Artifacts", "BuildResult_Wheel"), "whl")
1101
+
1102
+ @GeneralUtilities.check_arguments
1103
+ def __standardized_tasks_push_wheel_file_to_registry(self, wheel_file: str, api_key: str, repository: str, gpg_identity: str,verbosity:LogLevel) -> None:
1104
+ # repository-value when PyPi should be used: "pypi"
1105
+ # gpg_identity-value when wheel-file should not be signed: None
1106
+ folder = os.path.dirname(wheel_file)
1107
+ filename = os.path.basename(wheel_file)
1108
+
1109
+ if gpg_identity is None:
1110
+ gpg_identity_argument = GeneralUtilities.empty_string
1111
+ else:
1112
+ gpg_identity_argument = GeneralUtilities.empty_string # f" --sign --identity {gpg_identity}"
1113
+ # disabled due to https://blog.pypi.org/posts/2023-05-23-removing-pgp/
1114
+
1115
+ if int(LogLevel.Information)<int(verbosity):
1116
+ verbose_argument = " --verbose"
1117
+ else:
1118
+ verbose_argument = GeneralUtilities.empty_string
1119
+
1120
+ twine_argument = f"upload{gpg_identity_argument} --repository {repository} --non-interactive {filename} --disable-progress-bar"
1121
+ twine_argument = f"{twine_argument} --username __token__ --password {api_key}{verbose_argument}"
1122
+ self.__sc.run_program("twine", twine_argument, folder, throw_exception_if_exitcode_is_not_zero=True)
1123
+
1124
+ @GeneralUtilities.check_arguments
1125
+ def push_nuget_build_artifact(self, push_script_file: str, repository_folder_name: str, codeunitname: str, registry_address: str,api_key: str):
1126
+ build_artifact_folder = GeneralUtilities.resolve_relative_path(f"../../Submodules/{repository_folder_name}/{codeunitname}/Other/Artifacts/BuildResult_NuGet", os.path.dirname(push_script_file))
1127
+ self.__sc.push_nuget_build_artifact(self.__sc.find_file_by_extension(build_artifact_folder, "nupkg"), registry_address, api_key)
1128
+
1129
+ @GeneralUtilities.check_arguments
1130
+ def suport_information_exists(self, repository_folder: str, version_of_product: str) -> bool:
1131
+ self.__sc.assert_is_git_repository(repository_folder)
1132
+ folder = os.path.join(repository_folder, "Other", "Resources", "Support")
1133
+ file = os.path.join(folder, "InformationAboutSupportedVersions.csv")
1134
+ if not os.path.isfile(file):
1135
+ return False
1136
+ entries = GeneralUtilities.read_csv_file(file, True)
1137
+ for entry in entries:
1138
+ if entry[0] == version_of_product:
1139
+ return True
1140
+ return False
1141
+
1142
+ @GeneralUtilities.check_arguments
1143
+ def mark_current_version_as_supported(self, repository_folder: str, version_of_product: str, supported_from: datetime, supported_until: datetime):
1144
+ self.__sc.assert_is_git_repository(repository_folder)
1145
+ if self.suport_information_exists(repository_folder, version_of_product):
1146
+ raise ValueError(f"Version-support for v{version_of_product} already defined.")
1147
+ folder = os.path.join(repository_folder, "Other", "Resources", "Support")
1148
+ GeneralUtilities.ensure_directory_exists(folder)
1149
+ file = os.path.join(folder, "InformationAboutSupportedVersions.csv")
1150
+ if not os.path.isfile(file):
1151
+ GeneralUtilities.ensure_file_exists(file)
1152
+ GeneralUtilities.append_line_to_file(file, "Version;SupportBegin;SupportEnd")
1153
+ GeneralUtilities.append_line_to_file(file, f"{version_of_product};{GeneralUtilities.datetime_to_string(supported_from)};{GeneralUtilities.datetime_to_string(supported_until)}")
1154
+
1155
+
1156
+ @GeneralUtilities.check_arguments
1157
+ def add_github_release(self, productname: str, projectversion: str, build_artifacts_folder: str, github_username: str, repository_folder: str, additional_attached_files: list[str]) -> None:
1158
+ self.__sc.assert_is_git_repository(repository_folder)
1159
+ self.__sc.log.log(f"Create GitHub-release for {productname}...")
1160
+ github_repo = f"{github_username}/{productname}"
1161
+ artifact_files = []
1162
+ codeunits = self.get_codeunits(repository_folder)
1163
+ for codeunit in codeunits:
1164
+ artifact_files.append(self.__sc.find_file_by_extension(f"{build_artifacts_folder}/{productname}/{projectversion}/{codeunit}", "Artifacts.zip"))
1165
+ if additional_attached_files is not None:
1166
+ for additional_attached_file in additional_attached_files:
1167
+ artifact_files.append(additional_attached_file)
1168
+ changelog_file = os.path.join(repository_folder, "Other", "Resources", "Changelog", f"v{projectversion}.md")
1169
+ self.__sc.run_program_argsasarray("gh", ["release", "create", f"v{projectversion}", "--repo", github_repo, "--notes-file", changelog_file, "--title", f"Release v{projectversion}"]+artifact_files)
1170
+
1171
+ @GeneralUtilities.check_arguments
1172
+ def get_dependency_version_in_resources_folder(self, resources_folder:str, dependency_name: str) ->str:
1173
+ dependency_folder = os.path.join(resources_folder, "Dependencies", dependency_name)
1174
+ version_file = os.path.join(dependency_folder, "Version.txt")
1175
+ if not os.path.isfile(version_file):
1176
+ raise ValueError(f"Version-file for dependency {dependency_name} does not exist. Expected location: {version_file}")
1177
+ return GeneralUtilities.read_text_from_file(version_file)
1178
+
1179
+ @GeneralUtilities.check_arguments
1180
+ def update_dependency_in_resources_folder(self, update_dependencies_file, dependency_name: str, latest_version_function: str) -> None:
1181
+ dependency_folder = GeneralUtilities.resolve_relative_path(f"../Resources/Dependencies/{dependency_name}", update_dependencies_file)
1182
+ version_file = os.path.join(dependency_folder, "Version.txt")
1183
+ version_file_exists = os.path.isfile(version_file)
1184
+ write_to_file = False
1185
+ if version_file_exists:
1186
+ current_version = GeneralUtilities.read_text_from_file(version_file)
1187
+ if current_version != latest_version_function:
1188
+ write_to_file = True
1189
+ else:
1190
+ GeneralUtilities.ensure_directory_exists(dependency_folder)
1191
+ GeneralUtilities.ensure_file_exists(version_file)
1192
+ write_to_file = True
1193
+ if write_to_file:
1194
+ GeneralUtilities.write_text_to_file(version_file, latest_version_function)
1195
+
1196
+ @GeneralUtilities.check_arguments
1197
+ def push_docker_build_artifact(self, push_artifacts_file: str, registry: str, push_readme: bool, repository_folder_name: str, remote_image_name: str = None) -> None:
1198
+ folder_of_this_file = os.path.dirname(push_artifacts_file)
1199
+ filename = os.path.basename(push_artifacts_file)
1200
+ codeunitname_regex: str = "([a-zA-Z0-9]+)"
1201
+ filename_regex: str = f"PushArtifacts\\.{codeunitname_regex}\\.py"
1202
+ if match := re.search(filename_regex, filename, re.IGNORECASE):
1203
+ codeunitname = match.group(1)
1204
+ else:
1205
+ raise ValueError(f"Expected push-artifacts-file to match the regex \"{filename_regex}\" where \"{codeunitname_regex}\" represents the codeunit-name.")
1206
+ repository_folder = GeneralUtilities.resolve_relative_path(f"..{os.path.sep}..{os.path.sep}Submodules{os.path.sep}{repository_folder_name}", folder_of_this_file)
1207
+ codeunit_folder = os.path.join(repository_folder, codeunitname)
1208
+ artifacts_folder = os.path.join(repository_folder,codeunitname, "Other", "Artifacts")
1209
+ applicationimage_folder = os.path.join(artifacts_folder, "BuildResult_OCIImage")
1210
+ codeunit_version = self.get_version_of_codeunit(os.path.join(codeunit_folder, f"{codeunitname}.codeunit.xml"))
1211
+ if remote_image_name is None:
1212
+ remote_image_name = codeunitname.lower()
1213
+ tar_files=[f for f in GeneralUtilities.get_direct_files_of_folder(applicationimage_folder) if f.endswith(".tar")]
1214
+ target_image_address=f"{registry}/{remote_image_name}"
1215
+ tar_files_with_platforms: list[tuple[str, str, str]] = []
1216
+ for tar_file in tar_files:
1217
+ filename=os.path.basename(tar_file)#filename looks like "{codeunitname}_v{codeunitversion}_{GeneralUtilities.platform_to_dash_str(platform)}.tar"
1218
+ platform_of_file:Platform=self.platform_from_filename(filename)#GeneralUtilities.platform_from_dash_str( filename.split("_")[-1].split(".")[0])
1219
+ platform_os_in_docker_format :str = None
1220
+ platform_arch_in_docker_format :str = None
1221
+ if platform_of_file==Platform.Windows_AMD64:
1222
+ raise NotImplementedError("Building docker images for Windows is not implemented yet.")
1223
+ elif platform_of_file==Platform.Linux_AMD64:
1224
+ platform_os_in_docker_format = "linux"
1225
+ platform_arch_in_docker_format = "amd64"
1226
+ elif platform_of_file==Platform.Linux_ARM64:
1227
+ platform_os_in_docker_format = "linux"
1228
+ platform_arch_in_docker_format = "arm64"
1229
+ elif platform_of_file==Platform.MacOS_ARM64:
1230
+ raise NotImplementedError("Building docker images for MacOS is not implemented yet.")
1231
+ else:
1232
+ raise ValueError(f"Unsupported platform {platform_of_file} extracted from filename {filename}.")
1233
+ tar_files_with_platforms.append((tar_file, platform_os_in_docker_format, platform_arch_in_docker_format))
1234
+ self.push_docker_build_artifact_as_multi_arch_artifact(tar_files_with_platforms,target_image_address, "v"+codeunit_version)
1235
+ self.push_docker_build_artifact_as_multi_arch_artifact(tar_files_with_platforms,target_image_address, "latest")
1236
+ if push_readme:
1237
+ GeneralUtilities.assert_file_exists(os.path.join(codeunit_folder, "ReadMe.md"))
1238
+ self.__sc.run_program_with_retry("docker-pushrm", target_image_address, codeunit_folder)
1239
+
1240
+ def push_docker_build_artifact_as_multi_arch_artifact(self,tar_files: list[tuple[str, str, str]], image_address: str, tag: str):
1241
+ """
1242
+ tar_files: list of (tar_path, os, arch) tuples
1243
+ for example [
1244
+ ("MyApp.Linux.arm64.tar", "linux", "arm64"),
1245
+ ("MyApp.Linux.amd64.tar", "linux", "amd64")
1246
+ ]
1247
+ image_address for example: "myregistry.example.com/myapp"
1248
+ tag for example: "1.0.0"
1249
+ """
1250
+ GeneralUtilities.write_message_to_stdout(f"Creating multi-arch artifact {image_address}:{tag}...")
1251
+ arch_tags = []
1252
+ for tar_path, os_name, arch in tar_files:
1253
+ arch_tag = f"{image_address}:{tag}-{os_name}-{arch}"
1254
+ arch_tags.append(arch_tag)
1255
+ # Load tar → local image
1256
+ GeneralUtilities.write_message_to_stdout(f"Loading {tar_path}...")
1257
+ result = self.__sc.run_program_argsasarray("docker",[ "load", "-i", tar_path])
1258
+ # docker load outputs: "Loaded image: sha256:abc123..." or "Loaded image ID: ..."
1259
+ # we need the loaded image ID
1260
+ loaded_id = None
1261
+ for line in GeneralUtilities.string_to_lines(result[1]):
1262
+ if "Loaded image" in line:
1263
+ loaded_id = line.split(":", 1)[1].strip()
1264
+ break
1265
+ if not loaded_id:
1266
+ raise RuntimeError(f"Could not determine loaded image from output: \"{result[1]}\"")
1267
+ # Retag + push
1268
+ self.__sc.run_program_argsasarray("docker",[ "tag", loaded_id, arch_tag])
1269
+ self.__sc.run_program_argsasarray("docker",[ "push", arch_tag])
1270
+ # Create multi-arch manifest
1271
+ final_tag = f"{image_address}:{tag}"
1272
+ self.__sc.run_program_argsasarray("docker", [ "buildx", "imagetools", "create", "--tag", final_tag] + arch_tags)
1273
+
1274
+ @GeneralUtilities.check_arguments
1275
+ def platform_from_filename(self,filename: str) -> Platform:
1276
+ match = re.search(r'_([^_]+)\.tar', filename)
1277
+ if match:
1278
+ return GeneralUtilities.platform_from_dash_str(match.group(1))
1279
+ else:
1280
+ raise ValueError(f"Cannot extract platform from filename: \"{filename}\"")
1281
+
1282
+ def prepare_building_codeunits(self,repository_folder:str,use_cache:bool,generate_development_certificate:bool):
1283
+ if generate_development_certificate:
1284
+ self.ensure_certificate_authority_for_development_purposes_is_generated(repository_folder)
1285
+ self.generate_certificate_for_development_purposes_for_product(repository_folder)
1286
+ self.generate_tasksfile_from_workspace_file(repository_folder)
1287
+ self.generate_codeunits_overview_diagram(repository_folder)
1288
+ self.generate_svg_files_from_plantuml_files_for_repository(repository_folder,use_cache)
1289
+
1290
+ @GeneralUtilities.check_arguments
1291
+ def copy_product_resource_to_codeunit_resource_folder(self, codeunit_folder: str, resourcename: str) -> None:
1292
+ repository_folder = GeneralUtilities.resolve_relative_path(f"..", codeunit_folder)
1293
+ self.__sc.assert_is_git_repository(repository_folder)
1294
+ src_folder = GeneralUtilities.resolve_relative_path(f"Other/Resources/{resourcename}", repository_folder)
1295
+ GeneralUtilities.assert_condition(os.path.isdir(src_folder), f"Required product-resource {resourcename} does not exist. Expected folder: {src_folder}")
1296
+ trg_folder = GeneralUtilities.resolve_relative_path(f"Other/Resources/{resourcename}", codeunit_folder)
1297
+ GeneralUtilities.ensure_directory_does_not_exist(trg_folder)
1298
+ GeneralUtilities.ensure_directory_exists(trg_folder)
1299
+ GeneralUtilities.copy_content_of_folder(src_folder, trg_folder)
1300
+
1301
+ @GeneralUtilities.check_arguments
1302
+ def ensure_containers_are_not_running(self, container_names_to_remove:list[str]) -> None:
1303
+ for container_name in container_names_to_remove:
1304
+ self.__sc.log.log(f"Ensure container {container_name} does not exist...")
1305
+ self.__sc.run_program("docker", f"container rm -f {container_name}", throw_exception_if_exitcode_is_not_zero=False)
1306
+
1307
+ @GeneralUtilities.check_arguments
1308
+ def load_docker_image(self, oci_image_artifacts_folder:str,platform_for_image:Platform) -> None:
1309
+ for file in GeneralUtilities.get_direct_files_of_folder(oci_image_artifacts_folder):
1310
+ if file.endswith(f"_{GeneralUtilities.platform_to_dash_str(platform_for_image)}.tar"):
1311
+ image_filename = file
1312
+ self.__sc.log.log(f"Load docker-image {image_filename}...")
1313
+ self.__sc.run_program("docker", f"load -i {image_filename}", oci_image_artifacts_folder)
1314
+ return
1315
+ raise ValueError(f"No docker-image found for platform {GeneralUtilities.platform_to_dash_str(platform_for_image)} in folder {oci_image_artifacts_folder}.")
1316
+
1317
+ @GeneralUtilities.check_arguments
1318
+ def start_dockerfile_example(self, current_file: str,remove_old_container: bool, remove_volumes_folder: bool, env_file: str) -> None:
1319
+ container_names_to_remove:list[str]=[]
1320
+ folder_of_current_file = os.path.dirname(current_file)
1321
+ if remove_old_container:
1322
+ docker_compose_file = f"{folder_of_current_file}/docker-compose.yml"
1323
+ lines = GeneralUtilities.read_lines_from_file(docker_compose_file)
1324
+ for line in lines:
1325
+ if match := re.search("container_name:\\s*'?([^']+)'?", line):
1326
+ container_names_to_remove.append(match.group(1))
1327
+ self.__sc.log.log(f"Ensure container of {docker_compose_file} do not exist...")
1328
+ oci_image_artifacts_folder = GeneralUtilities.resolve_relative_path("../../../../Artifacts/BuildResult_OCIImage", folder_of_current_file)
1329
+ self.ensure_containers_are_not_running(container_names_to_remove)
1330
+ platform_for_image :Platform=None
1331
+ if GeneralUtilities.current_system_is_x64():
1332
+ platform_for_image=Platform.Linux_AMD64
1333
+ elif GeneralUtilities.current_system_is_arm64():
1334
+ platform_for_image=Platform.Linux_ARM64
1335
+ else:
1336
+ raise ValueError("Unsupported platform for docker-image. Only AMD64 and ARM64 are supported.")
1337
+ self.load_docker_image(oci_image_artifacts_folder,platform_for_image)
1338
+ example_name = os.path.basename(folder_of_current_file)
1339
+ codeunit_name = os.path.basename(GeneralUtilities.resolve_relative_path("../../../../..", folder_of_current_file))
1340
+ if remove_volumes_folder:
1341
+ volumes_folder = os.path.join(folder_of_current_file, "Volumes")
1342
+ self.__sc.log.log(f"Ensure volumes-folder '{volumes_folder}' does not exist...")
1343
+ GeneralUtilities.ensure_directory_does_not_exist(volumes_folder)
1344
+ GeneralUtilities.ensure_directory_exists(volumes_folder)
1345
+ docker_project_name = f"{codeunit_name}_{example_name}".lower()
1346
+ self.__sc.log.log("Start docker-container...")
1347
+ argument = f"compose --project-name {docker_project_name}"
1348
+ if env_file is not None:
1349
+ argument = f"{argument} --env-file {env_file}"
1350
+ argument = f"{argument} up --detach"
1351
+ self.__sc.run_program("docker", argument, folder_of_current_file)
1352
+
1353
+ @GeneralUtilities.check_arguments
1354
+ def ensure_env_file_is_generated(self, current_file: str, env_file_name: str, env_values: dict[str, str]):
1355
+ folder = os.path.dirname(current_file)
1356
+ env_file = os.path.join(folder, env_file_name)
1357
+ GeneralUtilities.ensure_file_exists(env_file)
1358
+ lines = []
1359
+ for key, value in env_values.items():
1360
+ lines.append(f"{key}={value}")
1361
+ GeneralUtilities.write_lines_to_file(env_file, lines)
1362
+
1363
+ @GeneralUtilities.check_arguments
1364
+ def stop_dockerfile_example(self, current_file: str, remove_old_container: bool, remove_volumes_folder: bool) -> None:
1365
+ folder = os.path.dirname(current_file)
1366
+ example_name = os.path.basename(folder)
1367
+ codeunit_name = os.path.basename(GeneralUtilities.resolve_relative_path("../../../../..", folder))
1368
+ docker_project_name = f"{codeunit_name}_{example_name}".lower()
1369
+ self.__sc.log.log("Stop docker-container...")
1370
+ self.__sc.run_program("docker", f"compose --project-name {docker_project_name} down", folder)
1371
+ if remove_old_container:
1372
+ pass#TODO
1373
+ if remove_volumes_folder:
1374
+ pass#TODO
1375
+
1376
+ @GeneralUtilities.check_arguments
1377
+ def update_submodule(self, repository_folder: str, submodule_name: str, local_branch: str = "main", remote_branch: str = "main", remote: str = "origin"):
1378
+ submodule_folder = GeneralUtilities.resolve_relative_path("Other/Resources/Submodules/"+submodule_name, repository_folder)
1379
+ self.__sc.git_fetch(submodule_folder, remote)
1380
+ self.__sc.git_checkout(submodule_folder, local_branch)
1381
+ self.__sc.git_pull(submodule_folder, remote, local_branch, remote_branch, True)
1382
+ current_version = self.__sc.get_semver_version_from_gitversion(repository_folder)
1383
+ changelog_file = os.path.join(repository_folder, "Other", "Resources", "Changelog", f"v{current_version}.md")
1384
+ if (not os.path.isfile(changelog_file)):
1385
+ GeneralUtilities.ensure_file_exists(changelog_file)
1386
+ GeneralUtilities.write_text_to_file(changelog_file, """# Release notes
1387
+
1388
+ ## Changes
1389
+
1390
+ - Updated geo-ip-database.
1391
+ """)
1392
+
1393
+ def set_latest_version_for_clone_repository_as_resource(self,repository_folder:str, resourcename: str, github_link: str, branch: str = "main"):
1394
+ resrepo_commit_id_folder: str = os.path.join(repository_folder, "Other", "Resources", f"{resourcename}Version")
1395
+ resrepo_commit_id_file: str = os.path.join(resrepo_commit_id_folder, f"{resourcename}Version.txt")
1396
+ current_version: str = GeneralUtilities.read_text_from_file(resrepo_commit_id_file)
1397
+
1398
+ stdOut = [l.split("\t") for l in GeneralUtilities.string_to_lines(self.__sc.run_program("git", f"ls-remote {github_link}")[1])]
1399
+ stdOut = [l for l in stdOut if l[1] == f"refs/heads/{branch}"]
1400
+ GeneralUtilities.assert_condition(len(stdOut) == 1)
1401
+ latest_version: str = stdOut[0][0]
1402
+ if current_version != latest_version:
1403
+ GeneralUtilities.write_text_to_file(resrepo_commit_id_file, latest_version)
1404
+
1405
+ @GeneralUtilities.check_arguments
1406
+ def get_dependencies_which_are_ignored_from_updates(self, codeunit_folder: str) -> list[str]:
1407
+ self.assert_is_codeunit_folder(codeunit_folder)
1408
+ namespaces = {'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure', 'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
1409
+ codeunit_name = os.path.basename(codeunit_folder)
1410
+ codeunit_file = os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml")
1411
+ root: etree._ElementTree = etree.parse(codeunit_file)
1412
+ ignoreddependencies = root.xpath('//cps:codeunit/cps:properties/cps:updatesettings/cps:ignoreddependencies/cps:ignoreddependency', namespaces=namespaces)
1413
+ result = [x.text.replace("\\n", GeneralUtilities.empty_string).replace("\\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string).replace("\r", GeneralUtilities.empty_string).strip() for x in ignoreddependencies]
1414
+ return result
1415
+
1416
+ @GeneralUtilities.check_arguments
1417
+ def update_dependencies_of_package_json(self, folder_of_package_json: str) -> None:#TODO this should probably be implemented in TFCPS_CodeUnitSpecific_NodeJS_Functions
1418
+ #TODO move this to TFCPS_CodeUnitSpecific_NodeJS_Functions
1419
+ if self.is_codeunit_folder(folder_of_package_json):
1420
+ ignored_dependencies = self.get_dependencies_which_are_ignored_from_updates(folder_of_package_json)
1421
+ else:
1422
+ ignored_dependencies = []
1423
+ # TODO consider ignored_dependencies
1424
+ result = self.__sc.run_with_epew("npm", "outdated", folder_of_package_json, throw_exception_if_exitcode_is_not_zero=False)
1425
+ if result[0] == 0:
1426
+ return # all dependencies up to date
1427
+ elif result[0] == 1:
1428
+ package_json_content = None
1429
+ package_json_file = f"{folder_of_package_json}/package.json"
1430
+ with open(package_json_file, "r", encoding="utf-8") as package_json_file_object:
1431
+ package_json_content = json.load(package_json_file_object)
1432
+ lines = GeneralUtilities.string_to_lines(result[1])[1:][:-1]
1433
+ for line in lines:
1434
+ normalized_line_splitted = ' '.join(line.split()).split(" ")
1435
+ package = normalized_line_splitted[0]
1436
+ latest_version = normalized_line_splitted[3]
1437
+ if package in package_json_content["dependencies"]:
1438
+ package_json_content["dependencies"][package] = latest_version
1439
+ if package in package_json_content["devDependencies"]:
1440
+ package_json_content["devDependencies"][package] = latest_version
1441
+ with open(package_json_file, "w", encoding="utf-8") as package_json_file_object:
1442
+ json.dump(package_json_content, package_json_file_object, indent=4)
1443
+ GeneralUtilities.write_text_to_file(package_json_file, GeneralUtilities.read_text_from_file(package_json_file).replace("\r", ""))
1444
+ self.do_npm_install(folder_of_package_json, True,True)#TODO use_cache might be dangerous here
1445
+ else:
1446
+ self.__sc.log.log("Update dependencies resulted in an error.", LogLevel.Error)
1447
+
1448
+
1449
+ @GeneralUtilities.check_arguments
1450
+ def get_resource_from_submodule_with_default_ignore_pattern(self,codeunit_folder:str,submodule_name:str,resource_name:str):
1451
+ self.get_resource_from_submodule(codeunit_folder,submodule_name,resource_name,["**.git","**.gitmodules"])
1452
+
1453
+ @GeneralUtilities.check_arguments
1454
+ def get_resource_from_submodule(self,codeunit_folder:str,submodule_name:str,resource_name:str,ignore_patterns:list[str]):
1455
+ self.assert_is_codeunit_folder(codeunit_folder)
1456
+ repository=os.path.dirname(codeunit_folder)
1457
+ source_folder=os.path.join(repository,"Other","Resources","Submodules",submodule_name)
1458
+ GeneralUtilities.assert_folder_exists(source_folder)
1459
+ target_folder=os.path.join(codeunit_folder,"Other","Resources",resource_name)
1460
+ GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder)
1461
+ GeneralUtilities.copy_content_of_folder(source_folder,target_folder,True,ignore_patterns)
1462
+
1463
+
1464
+ @GeneralUtilities.check_arguments
1465
+ def pull_images_of_test_services(self,repository_folder:str,env_variables:dict[str,str]):
1466
+ if env_variables is None:
1467
+ env_variables={}
1468
+ for image in self.oci_image_manager.get_used_images_in_repository(repository_folder):
1469
+ env_variables[f"image_{image.lower()}"]=self.oci_image_manager.get_registry_address_for_image(repository_folder,image)+":"+self.oci_image_manager.get_tag_for_image(repository_folder,image, True)
1470
+ test_services=GeneralUtilities.get_direct_folders_of_folder(os.path.join(repository_folder,"Other","Resources","LocalTestServices"))
1471
+ if len(test_services)==0:
1472
+ return
1473
+ self.__sc.log.log("Pull images for local test-services...")
1474
+ self.__sc.login_to_defined_docker_registries()
1475
+ for test_service_folder in test_services:
1476
+ test_service_name=os.path.basename(test_service_folder)
1477
+ self.__sc.log.log(f"Pull images for test-service {test_service_name}...")
1478
+ arguments=f"compose -f docker-compose.yml"
1479
+ env_variables_file=os.path.join(test_service_folder,"Parameters.env")
1480
+ GeneralUtilities.ensure_file_exists(env_variables_file)
1481
+ lines=[]
1482
+ for k,v in env_variables.items():
1483
+ lines=lines+[f"{k}={v}"]
1484
+ GeneralUtilities.write_lines_to_file(env_variables_file,lines)
1485
+ arguments=arguments + " --env-file Parameters.env pull --quiet"
1486
+ arguments_for_log=arguments
1487
+ self.__sc.run_program_with_retry("docker",arguments,test_service_folder,arguments_for_log=arguments_for_log,print_live_output=self.__sc.log.loglevel==LogLevel.Debug)
1488
+
1489
+ def load_deb_control_file_content(self, file: str, codeunitname: str, codeunitversion: str, installedsize: int, maintainername: str, maintaineremail: str, description: str) -> str:
1490
+ content = GeneralUtilities.read_text_from_file(file)
1491
+ content = GeneralUtilities.replace_variable_in_string(content, "codeunitname", codeunitname)
1492
+ content = GeneralUtilities.replace_variable_in_string(content, "codeunitversion", codeunitversion)
1493
+ content = GeneralUtilities.replace_variable_in_string(content, "installedsize", str(installedsize))
1494
+ content = GeneralUtilities.replace_variable_in_string(content, "maintainername", maintainername)
1495
+ content = GeneralUtilities.replace_variable_in_string(content, "maintaineremail", maintaineremail)
1496
+ content = GeneralUtilities.replace_variable_in_string(content, "description", description)
1497
+ return content
1498
+
1499
+ def calculate_deb_package_size(self, binary_folder: str) -> int:
1500
+ size_in_bytes = 0
1501
+ for file in GeneralUtilities.get_all_files_of_folder(binary_folder):
1502
+ size_in_bytes = size_in_bytes+os.path.getsize(file)
1503
+ result = math.ceil(size_in_bytes/1024)
1504
+ return result
1505
+
1506
+ def create_deb_package_for_artifact(self,codeunit_folder: str, maintainername: str, maintaineremail: str, description: str) -> None:
1507
+ self.assert_is_codeunit_folder(codeunit_folder)
1508
+ codeunit_name = os.path.basename(codeunit_folder)
1509
+ binary_folder = GeneralUtilities.resolve_relative_path("Other/Artifacts/BuildResult_DotNet_linux-x64", codeunit_folder)
1510
+ deb_output_folder = GeneralUtilities.resolve_relative_path("Other/Artifacts/BuildResult_Deb", codeunit_folder)
1511
+ control_file = GeneralUtilities.resolve_relative_path("Other/Build/DebControlFile.txt", codeunit_folder)
1512
+ installedsize = self.calculate_deb_package_size(binary_folder)
1513
+ control_file_content = self.load_deb_control_file_content(control_file, codeunit_name, self.get_version_of_codeunit(os.path.join(codeunit_folder,f"{codeunit_name}.codeunit.xml")), installedsize, maintainername, maintaineremail, description)
1514
+ self.__sc.create_deb_package(codeunit_name, binary_folder, control_file_content, deb_output_folder, 555)
1515
+
1516
+ @GeneralUtilities.check_arguments
1517
+ def create_zip_file_for_artifact(self, codeunit_folder: str, artifact_source_name: str, name_of_new_artifact: str) -> None:
1518
+ self.assert_is_codeunit_folder(codeunit_folder)
1519
+ src_artifact_folder = GeneralUtilities.resolve_relative_path(f"Other/Artifacts/{artifact_source_name}", codeunit_folder)
1520
+ shutil.make_archive(name_of_new_artifact, 'zip', src_artifact_folder)
1521
+ archive_file = os.path.join(os.getcwd(), f"{name_of_new_artifact}.zip")
1522
+ target_folder = GeneralUtilities.resolve_relative_path(f"Other/Artifacts/{name_of_new_artifact}", codeunit_folder)
1523
+ GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder)
1524
+ shutil.move(archive_file, target_folder)
1525
+
1526
+ def generate_winget_zip_manifest(self, codeunit_folder: str, artifact_name_of_zip: str):
1527
+ self.assert_is_codeunit_folder(codeunit_folder)
1528
+ codeunit_name = os.path.basename(codeunit_folder)
1529
+ codeunit_version = self.get_version_of_codeunit(os.path.join(codeunit_folder,f"{codeunit_name}.codeunit.xml"))
1530
+ build_folder = os.path.join(codeunit_folder, "Other", "Build")
1531
+ artifacts_folder = os.path.join(codeunit_folder, "Other", "Artifacts", artifact_name_of_zip)
1532
+ manifest_folder = os.path.join(codeunit_folder, "Other", "Artifacts", "WinGet-Manifest")
1533
+ GeneralUtilities.assert_folder_exists(artifacts_folder)
1534
+ artifacts_file = self.__sc.find_file_by_extension(artifacts_folder, "zip")
1535
+ winget_template_file = os.path.join(build_folder, "WinGet-Template.yaml")
1536
+ winget_manifest_file = os.path.join(manifest_folder, "WinGet-Manifest.yaml")
1537
+ GeneralUtilities.assert_file_exists(winget_template_file)
1538
+ GeneralUtilities.ensure_directory_exists(manifest_folder)
1539
+ GeneralUtilities.ensure_file_exists(winget_manifest_file)
1540
+ manifest_content = GeneralUtilities.read_text_from_file(winget_template_file)
1541
+ manifest_content = GeneralUtilities.replace_variable_in_string(manifest_content, "version", codeunit_version)
1542
+ manifest_content = GeneralUtilities.replace_variable_in_string(manifest_content, "sha256_hashvalue", GeneralUtilities.get_sha256_of_file(artifacts_file))
1543
+ GeneralUtilities.write_text_to_file(winget_manifest_file, manifest_content)
1544
+
1545
+ def download_file(self,source:str,target:str):
1546
+ GeneralUtilities.ensure_directory_exists(os.path.dirname(target))
1547
+ GeneralUtilities.ensure_file_exists(target)
1548
+ response = requests.get(source, timeout=30)
1549
+ response.raise_for_status()
1550
+ with open(target, "wb") as f:
1551
+ f.write(response.content)
1552
+
1553
+ def try_update_basic_codeunitreference_from_examples_repository(self,codeunit_folder:str,example_codeunit_name: str):
1554
+ source=f"https://raw.githubusercontent.com/anionDev/CommonProjectStructureExamples/refs/heads/main/{example_codeunit_name}/Other/Reference/ReferenceContent/HowToBuild.md"
1555
+ target=f"{codeunit_folder}/Other/Reference/ReferenceContent/HowToBuild.md"
1556
+ self.download_file(source,target)
1557
+
1558
+ def try_update_basic_repositoryreference_from_examples_repository(self,repository_folder:str):
1559
+ source=f"https://raw.githubusercontent.com/anionDev/CommonProjectStructureExamples/refs/heads/main/Other/Reference/RepositoryStructure.mdd"
1560
+ target=f"{repository_folder}/Other/Reference/RepositoryStructure.md"
1561
+ self.download_file(source,target)
1562
+
1563
+
1564
+ def update_dependent_oci_images(self,repo:str):
1565
+ self.oci_image_manager.update_default_tag_for_images_in_image_definitions_file(repo,True)