ScriptCollection 4.3.63__py3-none-any.whl → 4.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. ScriptCollection/AnionBuildPlatform.py +205 -205
  2. ScriptCollection/CertificateUpdater.py +157 -155
  3. ScriptCollection/Executables.py +4 -2
  4. ScriptCollection/GeneralUtilities.py +1696 -1696
  5. ScriptCollection/OCIImages/OCIImageManager.py +278 -278
  6. ScriptCollection/ProcessesRunner.py +43 -43
  7. ScriptCollection/ProgramRunnerSudo.py +110 -110
  8. ScriptCollection/Resources/CultureChooser/CultureChooser.js +0 -0
  9. ScriptCollection/Resources/CultureChooser/index.html +0 -0
  10. ScriptCollection/Resources/Dependencies/JRE/Version.txt +0 -0
  11. ScriptCollection/Resources/Dependencies/OpenAPIGenerator/Version.txt +0 -0
  12. ScriptCollection/Resources/Dependencies/PlantUML/Version.txt +0 -0
  13. ScriptCollection/Resources/Fonts/DejaVuSans-Bold.ttf +0 -0
  14. ScriptCollection/Resources/Fonts/DejaVuSans-BoldOblique.ttf +0 -0
  15. ScriptCollection/Resources/Fonts/DejaVuSans-Oblique.ttf +0 -0
  16. ScriptCollection/Resources/Fonts/DejaVuSans.ttf +0 -0
  17. ScriptCollection/Resources/Fonts/LICENSE.txt +0 -0
  18. ScriptCollection/Resources/MaintenanceSite/MaintenanceSite.html +0 -0
  19. ScriptCollection/SCLog.py +115 -115
  20. ScriptCollection/ScriptCollectionCore.py +4372 -4292
  21. ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +8 -2
  22. ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +716 -709
  23. ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +235 -235
  24. ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +258 -258
  25. ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +524 -524
  26. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +157 -155
  27. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +691 -672
  28. ScriptCollection/TFCPS/TFCPS_CreateRelease.py +98 -98
  29. ScriptCollection/TFCPS/TFCPS_Generic.py +44 -44
  30. ScriptCollection/TFCPS/TFCPS_MergeToMain.py +132 -131
  31. ScriptCollection/TFCPS/TFCPS_MergeToStable.py +358 -358
  32. ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +48 -48
  33. ScriptCollection/TFCPS/TFCPS_Tools_General.py +38 -8
  34. {scriptcollection-4.3.63.dist-info → scriptcollection-4.4.0.dist-info}/METADATA +1 -1
  35. {scriptcollection-4.3.63.dist-info → scriptcollection-4.4.0.dist-info}/RECORD +27 -27
  36. {scriptcollection-4.3.63.dist-info → scriptcollection-4.4.0.dist-info}/WHEEL +0 -0
  37. {scriptcollection-4.3.63.dist-info → scriptcollection-4.4.0.dist-info}/entry_points.txt +0 -0
  38. {scriptcollection-4.3.63.dist-info → scriptcollection-4.4.0.dist-info}/top_level.txt +0 -0
@@ -1,672 +1,691 @@
1
- import os
2
- import json
3
- import re
4
- import socket
5
- from datetime import datetime, timedelta,timezone
6
- import xmlschema
7
- import yaml
8
- from packaging.version import Version
9
- from ..GeneralUtilities import GeneralUtilities
10
- from ..ScriptCollectionCore import ScriptCollectionCore
11
- from ..SCLog import LogLevel
12
- from .TFCPS_BuildCodeUnitsHook import TFCPS_BuildCodeUnitsHook
13
- from .TFCPS_CodeUnit_BuildCodeUnit import TFCPS_CodeUnit_BuildCodeUnit
14
- from .TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
15
- from .TFCPS_Tools_General import TFCPS_Tools_General
16
-
17
-
18
- class TFCPS_UpdateDependenciesHook(TFCPS_BuildCodeUnitsHook):
19
- """Updates the dependencies of the repository and of its codeunits while the codeunits are built.
20
- The dependency-update is implemented as a hook of the regular codeunit-build (and not as an own reduced build-process, which is how
21
- it was implemented before) so that it runs with exactly the same preparation as a normal build: the required environment-variables
22
- are set, the custom pre-codeunit-build-script runs and therefore the package-sources which are needed to resolve the dependencies of
23
- a codeunit are available. Without that preparation an update fails as soon as a codeunit has a dependency which is not available on a
24
- public package-source - which was the case for an update inside a build-container."""
25
-
26
- __sc: ScriptCollectionCore = None
27
- __tfcps_tools_general: TFCPS_Tools_General = None
28
-
29
- def __init__(self, sc: ScriptCollectionCore, tfcps_tools_general: TFCPS_Tools_General):
30
- self.__sc = sc
31
- self.__tfcps_tools_general = tfcps_tools_general
32
-
33
- @GeneralUtilities.check_arguments
34
- def run_after_preparation(self, repository: str) -> None:
35
- update_dependencies_script_folder: str = os.path.join(repository, "Other", "Scripts")
36
- if os.path.isfile(os.path.join(update_dependencies_script_folder, "UpdateDependencies.py")):
37
- self.__sc.log.log("Update dependencies of the repository...")
38
- self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", update_dependencies_script_folder)
39
-
40
- @GeneralUtilities.check_arguments
41
- def run_after_codeunit_was_built(self, codeunit_build: TFCPS_CodeUnit_BuildCodeUnit) -> None:
42
- codeunit_name: str = codeunit_build.codeunit_name
43
- codeunit_folder: str = codeunit_build.codeunit_folder
44
- repository: str = codeunit_build.repository_folder
45
- if not self.__tfcps_tools_general.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml")):
46
- return
47
- self.__sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
48
- self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", os.path.join(codeunit_folder, "Other"))
49
- if self.__sc.git_repository_has_uncommitted_changes(repository):
50
- #the update changed something, so it has to be verified that the codeunit is still buildable with the updated dependencies.
51
- self.__sc.log.log(f"Build codeunit {codeunit_name} again to verify it is still buildable with the updated dependencies...")
52
- codeunit_build.build_codeunit()
53
-
54
- class TFCPS_CodeUnit_BuildCodeUnits:
55
- repository:str=None
56
- tfcps_tools_general:TFCPS_Tools_General=None
57
- sc:ScriptCollectionCore=None
58
- target_environment_type:str=None
59
- additionalargumentsfile:str=None
60
- __use_cache:bool = None
61
- __is_pre_merge:bool = None
62
- __assert_no_new_changes:bool = None
63
- __add_ready_to_merge_flag:bool = None
64
-
65
- def __init__(self,repository:str,loglevel:LogLevel,target_environment_type:str,additionalargumentsfile:str,use_cache:bool,is_pre_merge:bool,assertnonewchanges:bool,add_ready_to_merge_flag:bool=False):
66
- self.sc=ScriptCollectionCore()
67
- self.sc.log.loglevel=loglevel
68
- self.__use_cache=use_cache
69
- self.sc.assert_is_git_repository(repository)
70
- self.repository=repository
71
- self.tfcps_tools_general:TFCPS_Tools_General=TFCPS_Tools_General(self.sc)
72
- allowed_target_environment_types=["Development","QualityCheck","Productive"]
73
- GeneralUtilities.assert_condition(target_environment_type in allowed_target_environment_types,"Unknown target-environment-type. Allowed values are: "+", ".join(allowed_target_environment_types))
74
- self.target_environment_type=target_environment_type
75
- self.additionalargumentsfile=additionalargumentsfile
76
- self.__is_pre_merge=is_pre_merge
77
- self.__assert_no_new_changes=assertnonewchanges
78
- self.__add_ready_to_merge_flag=add_ready_to_merge_flag
79
-
80
- @GeneralUtilities.check_arguments
81
- def build_codeunits(self, hook: TFCPS_BuildCodeUnitsHook = None) -> None:
82
- """Builds all codeunits of the repository. The optional hook allows a caller inside ScriptCollection to run additional actions
83
- between the regular build-steps (see TFCPS_BuildCodeUnitsHook); a caller which only wants to build passes nothing."""
84
- if hook is None:
85
- hook = TFCPS_BuildCodeUnitsHook()
86
- self.sc.log.log(GeneralUtilities.get_line())
87
- start_time:datetime=GeneralUtilities.get_now()
88
- ready_to_merge_file=os.path.join(self.repository,".ScriptCollection",".IsReadyToMerge")
89
- error_occurred=False
90
- try:
91
- #assert that the product-information-file exists
92
- product_information_file = os.path.join(self.repository, ".ScriptCollection", "ProductInformation.xml")
93
- GeneralUtilities.assert_file_exists(product_information_file, f"The file '{product_information_file}' does not exist.")
94
-
95
-
96
- #when the build runs inside a container, ensure the used SCBuilder-image is at least the version required by this repository (defined in .ScriptCollection/OCIImages/ImageDefinition.csv)
97
- if self.sc.is_runnning_in_container():
98
- scbuilder_version_environment_value = os.environ.get("SCBuilderVersion")
99
- GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(scbuilder_version_environment_value), "The environment-variable 'SCBuilderVersion' is not set although the build runs inside a container (environment-variable 'ISRUNNINGINCONTAINER' is 'true').")
100
- required_scbuilder_version = Version(self.tfcps_tools_general.oci_image_manager.get_tag_for_image(self.repository, "SCBuilder"))
101
- version_match = re.search(r"\d+(\.\d+)+", scbuilder_version_environment_value)
102
- GeneralUtilities.assert_condition(version_match is not None, f"The environment-variable 'SCBuilderVersion' (value: '{scbuilder_version_environment_value}') does not contain a version-string.")
103
- actual_scbuilder_version = Version(version_match.group(0))
104
- GeneralUtilities.assert_condition(actual_scbuilder_version >= required_scbuilder_version, f"The used SCBuilder-version {actual_scbuilder_version} is older than the version {required_scbuilder_version} required by '{self.tfcps_tools_general.get_product_name(self.repository)}' (defined in .ScriptCollection/OCIImages/ImageDefinition.csv). Please update to a newer SCBuilder-image.")
105
-
106
- if self.is_pre_merge():
107
- GeneralUtilities.assert_condition(not self.__assert_no_new_changes,f"A pre-merge build can not be done with the assert-no-new-changes-option.")
108
-
109
- self.sc.git_set_local_configuration_value(self.repository, "core.autocrlf", "false")
110
-
111
- #ensure the artifacts-folder of the repository is git-ignored so build-results never show up as uncommitted changes
112
- self.sc.ensure_line_is_in_gitignore(self.repository, "Other/Resources/Artifacts")
113
-
114
- #ensure <repo>/.ScriptCollection/.gitignore is set up (ignores the cache-folder so cache-files never show up as uncommitted changes)
115
- self.sc.ensure_scriptcollection_gitignore_is_setup(self.repository)
116
-
117
- self.sc.log.log(f"Start building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(start_time,False)}. (Target environment-type: {self.target_environment_type})")
118
-
119
- self.tfcps_tools_general.ensure_required_environment_variables_are_set(self.repository)
120
-
121
- self.__run_custom_pre_codeunit_build_script()
122
-
123
- if self.__assert_no_new_changes:
124
- self.sc.assert_no_uncommitted_changes(self.repository,"Can not build codeunit: There are uncommitted changes in the repository.")
125
-
126
- try:
127
- xmlschema.validate(product_information_file, "https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/raw/main/Conventions/RepositoryStructure/CommonProjectStructure/productinformation.xsd")
128
- except Exception as exception:
129
- self.sc.log.log_exception(f"'{product_information_file}' could not be validated against the XSD:", exception, LogLevel.Warning)
130
-
131
- #run prepare-script
132
- self.run_prepare_script()
133
-
134
- hook.run_after_preparation(self.repository)
135
-
136
- #check if changelog exists
137
- changelog_file=os.path.join(self.repository,"Other","Resources","Changelog",f"v{self.tfcps_tools_general.get_version_of_project(self.repository)}.md")
138
- GeneralUtilities.assert_file_exists(changelog_file,f"Changelogfile \"{changelog_file}\" does not exist. Try to create it for example using \"sccreatechangelogentry -m ...\".")
139
-
140
- #mark current version as supported
141
- now = GeneralUtilities.get_now()
142
- project_version:str=self.tfcps_tools_general.get_version_of_project(self.repository)
143
- if not self.tfcps_tools_general.suport_information_exists(self.repository, project_version):
144
- amount_of_years_for_support:int=1
145
- support_time = timedelta(days=365*amount_of_years_for_support+30*3+1)
146
- until = now + support_time
147
- until_day = datetime(until.year, until.month, until.day, 0, 0, 0)
148
- from_day = datetime(now.year, now.month, now.day, 0, 0, 0)
149
- self.tfcps_tools_general.mark_current_version_as_supported(self.repository,project_version,from_day,until_day)
150
-
151
- codeunits:list[str]=self.tfcps_tools_general.get_codeunits(self.repository)
152
- GeneralUtilities.assert_condition(0<len(codeunits),f"No codeunits found in repository {self.repository}.")
153
- self.sc.log.log("Codeunits will be built in the following order:")
154
- for codeunit_name in codeunits:
155
- self.sc.log.log(f" - {codeunit_name}")
156
- for codeunit_name in codeunits:
157
- tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(os.path.join(self.repository,codeunit_name),self.sc.log.loglevel,self.target_environment_type,self.additionalargumentsfile,self.use_cache(),self.is_pre_merge())
158
- self.sc.log.log(GeneralUtilities.get_line())
159
- tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()
160
- hook.run_after_codeunit_was_built(tFCPS_CodeUnit_BuildCodeUnit)
161
-
162
- self.sc.log.log(GeneralUtilities.get_line())
163
-
164
- self.search_for_secrets()
165
- self.__normalize_line_endings_of_common_files()
166
- self.tfcps_tools_general.generate_svg_files_from_plantuml_files_for_repository(self.repository, self.use_cache())
167
-
168
- if self.is_pre_merge():
169
- self.__translate()
170
- self.__collect_metrics()
171
- self.__generate_loc_diagram()
172
-
173
- except Exception:
174
- error_occurred=True
175
- raise
176
- finally:
177
- if error_occurred:
178
- GeneralUtilities.ensure_file_does_not_exist(ready_to_merge_file)
179
- else:
180
- if self.is_pre_merge():
181
- GeneralUtilities.ensure_file_does_not_exist(ready_to_merge_file)#ensure it does not exist because the flag is not supposed to be on the main branch
182
- else:
183
- if self.is_working_branch():
184
- if self.add_ready_to_merge_flag():
185
- GeneralUtilities.ensure_file_exists(ready_to_merge_file)
186
-
187
-
188
- if self.__assert_no_new_changes:
189
- self.sc.assert_no_uncommitted_changes(self.repository,"There are new uncommitted changes in the repository.")
190
-
191
- self.__create_artifacts_archive(codeunits)
192
-
193
- end_time:datetime=GeneralUtilities.get_now()
194
- duration=end_time-start_time
195
- self.sc.log.log(f"Finished building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(end_time,False)}. (Duration: {GeneralUtilities.timedelta_to_simple_string(duration)})")
196
- self.sc.log.log(GeneralUtilities.get_line())
197
-
198
- @GeneralUtilities.check_arguments
199
- def __create_artifacts_archive(self, codeunits:list[str]) -> str:
200
- product_name:str=self.tfcps_tools_general.get_product_name(self.repository)
201
- product_version:str=self.tfcps_tools_general.get_version_of_project(self.repository)
202
- target_folder:str=os.path.join(self.repository,"Other","Resources","Artifacts")
203
- target_file_zip:str=os.path.join(target_folder,f"{product_name}_Artifacts_v{product_version}.zip")
204
- target_file_infotxt:str=os.path.join(target_folder,f"{product_name}_Artifacts_v{product_version}.information.txt")
205
-
206
- artifacts_folders:dict[str,str]={}
207
- for codeunit_name in codeunits:
208
- artifacts_folder = os.path.join(self.repository, codeunit_name, "Other", "Artifacts")
209
- GeneralUtilities.assert_folder_exists(artifacts_folder,f"The artifacts-folder '{artifacts_folder}' of codeunit '{codeunit_name}' does not exist.")
210
- artifacts_folders[codeunit_name]=artifacts_folder
211
-
212
- GeneralUtilities.ensure_directory_exists(target_folder)
213
- GeneralUtilities.ensure_file_does_not_exist(target_file_zip)#ensure the archive of a previous build gets replaced
214
- self.sc.create_zip_archive_of_folders(artifacts_folders,target_file_zip)
215
-
216
- hash_of_zip_file:str=GeneralUtilities.get_sha256_of_file(target_file_zip)
217
- creation_timestamp:datetime=GeneralUtilities.get_now()
218
- GeneralUtilities.ensure_file_does_not_exist(target_file_infotxt)
219
- GeneralUtilities.ensure_file_exists(target_file_infotxt)
220
- information_lines:list[str]=[
221
- f"Product: {product_name}",
222
- f"Version: {product_version}",
223
- f"Artifacts-archive-hash: {hash_of_zip_file}",
224
- f"Creation-timestamp: {GeneralUtilities.datetime_to_string_for_readable_entry(creation_timestamp,False)}",
225
- ]
226
- GeneralUtilities.write_lines_to_file(target_file_infotxt,information_lines)
227
- normalized_relative_path=GeneralUtilities.normalize_path(os.path.relpath(target_file_zip, self.repository).replace("\\","/"))
228
- self.sc.log.log(f"Created artifacts-archive \"{normalized_relative_path}\":")
229
- for information_line in information_lines:
230
- self.sc.log.log(f" {information_line}")
231
-
232
- @GeneralUtilities.check_arguments
233
- def __normalize_line_endings_of_common_files(self) -> None:
234
- #TODO add option to define exceptions (means: files which should not be normalized).
235
- self.sc.format_xml_file(os.path.join(self.repository, ".ScriptCollection", "ProductInformation.xml"))
236
- for codeunit in self.tfcps_tools_general.get_codeunits(self.repository):
237
- self.sc.format_xml_file(os.path.join(self.repository, codeunit, f"{codeunit}.codeunit.xml"))
238
- workspace_file = os.path.join(self.repository, f"{self.tfcps_tools_general.get_product_name(self.repository)}.code-workspace")
239
- if os.path.isfile(workspace_file):
240
- self.sc.format_json_file(workspace_file)
241
- self.sc.normalize_line_endings_of_files_in_folder(self.repository, ["txt", "md", "json", "xml", "csv", "yml", "yaml", "toml","gitignore", "gitattributes", "code-workspace"])
242
-
243
- @GeneralUtilities.check_arguments
244
- def is_working_branch(self)->bool:
245
- if self.sc.git_repository_has_uncommitted_changes(self.repository):
246
- return True
247
- if self.sc.git_get_current_branch_name(self.repository) != "main":
248
- return True
249
- return False
250
-
251
- @GeneralUtilities.check_arguments
252
- def __get_build_arguments(self, repository: str) -> list[str]:
253
- """Returns the arguments which describe the current build. They are passed to every script which is called by this build
254
- (the prepare-script of the repository as well as the optional user-specific custom scripts), so such a script can behave
255
- differently depending on the repository and the target-environment-type it is called for."""
256
- result: list[str] = ["--repository", repository, "--targetenvironmenttype", self.target_environment_type, "--verbosity", str(int(self.sc.log.loglevel))]
257
- if GeneralUtilities.string_has_content(self.additionalargumentsfile):
258
- result = result+["--additionalargumentsfile", self.additionalargumentsfile]
259
- if not self.__use_cache:
260
- if self.sc.git_repository_has_uncommitted_changes(repository):
261
- self.sc.log.log("No-cache-option can not be applied because there are uncommited changes in the repository.", LogLevel.Warning)
262
- else:
263
- result = result+["--nocache"]
264
- return result
265
-
266
- @GeneralUtilities.check_arguments
267
- def __run_custom_pre_codeunit_build_script(self) -> None:
268
- """Runs the optional user-specific script which prepares this machine for a codeunit-build. The script is located in
269
- '~/.ScriptCollection/TFCPS', so it is outside of any repository and is not part of a product's sourcecode.
270
- Which script is used depends on where the build runs, because both cases need different preparation-steps:
271
- - build directly on the host ('scbuildcodeunits'): 'CustomPreCodeUnitBuildScript.py'
272
- - build inside the container ('scbuildcodeunitsc' or a build-pipeline): 'CustomPreCodeUnitBuildScriptInContainer.py', located in
273
- the folder returned by TFCPS_Tools_General.get_custom_scripts_folder_for_container(). It is searched in the folder into which
274
- 'scbuildcodeunitsc' mounts that whole folder and - if it is not there - directly in the configuration-folder, because a
275
- build-runner usually gets the whole configuration-folder mounted instead (see the article 'Build-runner-configuration'). The
276
- script which the host itself runs before it starts the container is 'CustomPreCodeUnitBuildScriptForContainer.py' and is
277
- therefore run there and not here."""
278
- if self.sc.is_runnning_in_container():
279
- script_file: str = os.path.join(self.tfcps_tools_general.get_folder_of_custom_scripts_in_container(), "CustomPreCodeUnitBuildScriptInContainer.py")
280
- if not os.path.isfile(script_file):
281
- script_file = os.path.join(self.tfcps_tools_general.get_custom_scripts_folder_for_container(), "CustomPreCodeUnitBuildScriptInContainer.py")
282
- else:
283
- script_file: str = self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScript.py")
284
- self.tfcps_tools_general.run_custom_script_if_available(script_file, self.__get_build_arguments(self.repository))
285
-
286
- @GeneralUtilities.check_arguments
287
- def run_prepare_script(self):
288
- args = self.__get_build_arguments(self.repository)
289
- if os.path.isfile( os.path.join(self.repository,"Other","Scripts","PrepareBuildCodeunits.py")):
290
- self.sc.log.log("Prepare build codeunits...")
291
- self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["PrepareBuildCodeunits.py"]+args, os.path.join(self.repository,"Other","Scripts"),print_live_output=True)
292
-
293
- @GeneralUtilities.check_arguments
294
- def build_codeunits_in_container(self,base_mount_folder:str) -> tuple[bool, str]:
295
- #base_mount_folder is assumed to be an absolute path set correctly by the caller (see BuildCodeUnitsC in Executables.py, which defaults it to the repository itself).
296
- #it may be the repository itself or any parent-folder of it, which allows the caller to mount not only the repository but the whole surrounding folder-structure into the container.
297
- normalized_base_mount_folder = os.path.normpath(base_mount_folder)
298
- normalized_repository = os.path.normpath(self.repository)
299
- relative_repository_path = os.path.relpath(normalized_repository, normalized_base_mount_folder).replace(os.sep, "/")
300
- GeneralUtilities.assert_condition(relative_repository_path == "." or not relative_repository_path.startswith(".."), f"The repository '{self.repository}' is not located inside the base-mount-folder '{base_mount_folder}'.")
301
- container_base_mount_folder = f"/Workspace/Project/{os.path.basename(normalized_base_mount_folder)}"
302
- container_repository_folder = container_base_mount_folder if relative_repository_path == "." else f"{container_base_mount_folder}/{relative_repository_path}"
303
- image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "SCBuilder")
304
-
305
- #build the scbuildcodeunits-arguments based on the current state (analogous to the arguments accepted by the scbuildcodeunits-executable). each token must be a separate argument because run_program_argsasarray passes every list-element verbatim and does not split on spaces.
306
- scbuildcodeunits_arguments = ["scbuildcodeunits", "-r", container_repository_folder, "-v", "4"]
307
- if not self.__use_cache:
308
- scbuildcodeunits_arguments.append("-c")
309
- if self.__is_pre_merge:
310
- scbuildcodeunits_arguments.append("-p")
311
- if self.__assert_no_new_changes:
312
- scbuildcodeunits_arguments.append("-u")
313
- if self.__add_ready_to_merge_flag:
314
- scbuildcodeunits_arguments.append("-m")
315
- if GeneralUtilities.string_has_content(self.additionalargumentsfile):
316
- scbuildcodeunits_arguments += ["-a", self.__translate_path_into_container(self.additionalargumentsfile, container_repository_folder)]
317
-
318
- #if the repository is a git-submodule then "<repository>/.git" is a pointer-file referencing its gitdir via a path relative to the repository
319
- #(for example "gitdir: ../../.git/modules/Submodules/<product>"), and the gitdir's own "core.worktree" points back at the repository via a
320
- #path relative to the gitdir. Both references are relative, so they resolve correctly inside the container as long as the mounted folder-structure
321
- #preserves that relative path - which is what base_mount_folder is for: the caller has to mount a folder that also contains the repository's real
322
- #gitdir (typically the parent-repository the submodule belongs to). No file-rewriting is required; a base_mount_folder that does not cover the
323
- #real gitdir results in failing git-commands inside the container.
324
- test=True#TODO remove this
325
- if test:
326
- scbuildcodeunits_arguments=["bash","-c", "pip3 install scriptcollection --upgrade && scshowversion && "+" ".join(scbuildcodeunits_arguments)]
327
-
328
- #run the optional user-specific script which prepares this host for a container-build (for example to log in to the registry the image is pulled from).
329
- #it runs before the environment-variables are resolved, so it can also create the files their values are read from.
330
- self.tfcps_tools_general.run_custom_script_if_available(self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScriptForContainer.py"), self.__get_build_arguments(self.repository))
331
-
332
- #mount the optional user-specific scripts which prepare the container itself. the whole folder (not only the pre-codeunit-build-hook
333
- #itself) is mounted, so the hook can start a sibling script placed next to it (for example to keep the hook independent of a
334
- #package it only installs, while the actual preparation-logic which needs that package lives in the sibling script; see
335
- #get_custom_scripts_folder_for_container). The mount is writable (not read-only) because the scripts in that folder also use it as
336
- #their own download-cache-folder. The build inside the container runs the hook (see __run_custom_pre_codeunit_build_script); if the
337
- #folder does not exist on the host nothing is mounted and nothing is run.
338
- mount_arguments: list[str] = []
339
- custom_scripts_folder_for_inside_the_container: str = self.tfcps_tools_general.get_custom_scripts_folder_for_container()
340
- if os.path.isdir(custom_scripts_folder_for_inside_the_container):
341
- mount_arguments += ["-v", f"{custom_scripts_folder_for_inside_the_container}:{self.tfcps_tools_general.get_folder_of_custom_scripts_in_container()}"]
342
-
343
- #pass the environment-variables which are declared as required in <repository>/.ScriptCollection/ProductInformation.xml into the container
344
- #so they do not have to be specified explicitly on every scbuildcodeunits-call. Their values are resolved from the user-specific
345
- #configuration-file (see TFCPS_Tools_General.get_environment_variables_configuration_file), which only exists on the host.
346
- #only the names are passed as arguments; the values are given to the docker-client through its own environment, because arguments
347
- #are written to the log and are visible in the process-list of this host, which must not happen for a value which is typically a secret.
348
- required_environment_variables: dict[str, str] = self.tfcps_tools_general.get_required_environment_variables(self.repository)
349
- env_arguments: list[str] = []
350
- for env_variable_name in required_environment_variables:
351
- env_arguments += ["-e", env_variable_name]
352
-
353
- #run scbuildcodeunits inside the SCBuilder-image. base_mount_folder is mounted into the container (covering the repository and, for submodules, its real gitdir) and the docker-socket is forwarded because codeunit-builds often start containers (for example local test-services).
354
- docker_arguments = [
355
- "run", "--rm",
356
- "-v", f"{base_mount_folder}:{container_base_mount_folder}",
357
- "-v", "/var/run/docker.sock:/var/run/docker.sock",
358
- "-w", container_repository_folder,
359
- ] + mount_arguments + env_arguments + [
360
- image,
361
- ] + scbuildcodeunits_arguments
362
- self.sc.log.log(f"Build codeunits in container using image \"{image}\"...")
363
- # the exitcode is evaluated by the caller (returned as part of the result-tuple), so the program-runner must not raise on a non-zero exitcode here.
364
- result=self.sc.run_program_argsasarray("docker", docker_arguments, throw_exception_if_exitcode_is_not_zero=False, print_live_output=True, env_vars=required_environment_variables)
365
- exit_code:int=result[0]
366
- stdout:str=result[1] or GeneralUtilities.empty_string
367
- stderr:str=result[2] or GeneralUtilities.empty_string
368
- return (exit_code==0,f"{stdout}\n{stderr}")
369
-
370
- @GeneralUtilities.check_arguments
371
- def __translate_path_into_container(self, host_path: str, container_repository_folder: str) -> str:
372
- normalized_repository = os.path.normpath(self.repository)
373
- normalized_path = os.path.normpath(host_path)
374
- if normalized_path.startswith(normalized_repository):
375
- relative_path = os.path.relpath(normalized_path, normalized_repository).replace(os.sep, "/")
376
- return f"{container_repository_folder}/{relative_path}"
377
- return host_path
378
-
379
- @GeneralUtilities.check_arguments
380
- def __translate(self) -> None:
381
- for taskfile_name in ("Taskfile.yml", "Taskfile.yaml"):
382
- taskfile = os.path.join(self.repository, taskfile_name)
383
- if os.path.isfile(taskfile):
384
- with open(taskfile, "r", encoding="utf-8") as f:
385
- taskfile_content = yaml.safe_load(f)
386
- if isinstance(taskfile_content.get("tasks"), dict) and "Translate" in taskfile_content["tasks"]:
387
- self.sc.run_program("task", "Translate", self.repository, print_live_output=self.sc.log.loglevel == LogLevel.Debug)
388
- break
389
-
390
- @GeneralUtilities.check_arguments
391
- def __collect_metrics(self) -> None:
392
- project_version: str=self.tfcps_tools_general.get_version_of_project(self.repository)
393
- self.sc.log.log("Collect metrics...")
394
- loc = self.sc.get_lines_of_code_with_default_excluded_patterns(self.repository)
395
- loc_metric_folder = os.path.join(self.repository, "Other", "Metrics")
396
- GeneralUtilities.ensure_directory_exists(loc_metric_folder)
397
- loc_metric_file = os.path.join(loc_metric_folder, "RepositoryStatisticsPerCommit.csv")
398
- GeneralUtilities.ensure_file_exists(loc_metric_file)
399
-
400
- #remove legacy metrics-file. the following 2 lines should be removed after 2026-12-31
401
- legacy_metrics_file = os.path.join(loc_metric_folder, "LinesOfCode.csv")
402
- GeneralUtilities.ensure_file_does_not_exist(legacy_metrics_file)
403
-
404
- old_lines = GeneralUtilities.read_nonempty_lines_from_file(loc_metric_file)
405
- header_line="Version;Timestamp;LinesOfCode"
406
- new_lines = [header_line]
407
- current_version_string=f"v{project_version}"
408
- for old_line in old_lines:
409
- if not old_line.startswith(current_version_string+";") and old_line!=header_line:
410
- new_lines.append(old_line)
411
- c_date:datetime=GeneralUtilities.get_now().astimezone(timezone.utc)
412
- commit_date=GeneralUtilities.datetime_to_string_for_logfile_entry(c_date,False)
413
- new_lines.append(f"{current_version_string};{commit_date};{loc}")
414
- GeneralUtilities.write_lines_to_file(loc_metric_file, new_lines)
415
-
416
-
417
- @GeneralUtilities.check_arguments
418
- def __generate_loc_diagram(self):
419
- self.sc.log.log("Generate LoC-diagram...")
420
- loc_metric_folder = os.path.join(self.repository, "Other", "Metrics")
421
- GeneralUtilities.ensure_directory_exists(loc_metric_folder)
422
- loc_metric_file = os.path.join(loc_metric_folder, "RepositoryStatisticsPerCommit.csv")
423
- GeneralUtilities.ensure_file_exists(loc_metric_file)
424
-
425
- filenamebase="LoC-Diagram"
426
-
427
- diagram_definition_folder=os.path.join(self.repository, "Other", "Reference","Technical","Diagrams")
428
- GeneralUtilities.ensure_directory_exists(diagram_definition_folder)
429
-
430
- diagram_definition_file=os.path.join(diagram_definition_folder,f"{filenamebase}.json")
431
- GeneralUtilities.ensure_file_exists(diagram_definition_file)
432
- GeneralUtilities.write_text_to_file(diagram_definition_file,GeneralUtilities.empty_string)
433
-
434
- loc_data_file=os.path.join(diagram_definition_folder,f"{filenamebase}.csv")
435
- GeneralUtilities.ensure_file_exists(loc_data_file)
436
- csv_lines=[]
437
- for line in GeneralUtilities.read_lines_from_file(loc_metric_file):
438
- if GeneralUtilities.string_has_content(line):
439
- splitted=line.split(";")
440
- v=splitted[0]
441
- t=splitted[1]
442
- loc=splitted[2]
443
- csv_lines.append(f"{v},{t},{loc}")
444
- GeneralUtilities.write_lines_to_file(loc_data_file,csv_lines)
445
- self.sc.normalize_line_endings(loc_data_file) # ensure the generated LoC-diagram-csv always uses LF line-endings
446
- diagram_json = {
447
- "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
448
- "description": "Lines of Code over time",
449
- "width": 800,
450
- "height": 400,
451
- "data": {
452
- "url": f"./{filenamebase}.csv",
453
- "format": {
454
- "type": "csv"
455
- }
456
- },
457
- "mark": {
458
- "type": "line",
459
- "point": True
460
- },
461
- "encoding": {
462
- "x": {
463
- "field": "Timestamp",
464
- "type": "temporal",
465
- "title": "Date",
466
- "scale": {
467
- "type": "utc"#render the ticks in utc so the diagram looks the same independent of the timezone of the machine which builds it
468
- },
469
- "axis": {
470
- "format": "%Y-%m-%d %H:%M:%S (UTC)",#everything which is not a format-specifier starting with a percent-sign is taken literally
471
- "labelAngle": -45,#the timestamps are too long to be displayed horizontally next to each other
472
- "labelLimit": 250,#without this the labels would be truncated with an ellipsis because they are longer than the default-limit of 180 pixels
473
- "tickCount": 8
474
- }
475
- },
476
- "y": {
477
- "field": "LinesOfCode",
478
- "type": "quantitative",
479
- "title": "Lines of Code"
480
- },
481
- #the timestamp and the lines of code are not listed here although they are part of the tooltip: the fields which are encoded
482
- #as x respectively as y are always part of the generated tooltip anyway, so listing them again would show them twice.
483
- "tooltip": [
484
- {
485
- "field": "Version",
486
- "type": "ordinal"
487
- }
488
- ]
489
- }
490
- }
491
-
492
- with open(diagram_definition_file, "w", encoding="utf-8") as f:
493
- json.dump(
494
- diagram_json,
495
- f,
496
- indent=2,
497
- sort_keys=False,
498
- ensure_ascii=False
499
- )
500
- diagram_svg_file=os.path.join(self.repository,"Other","Reference","Technical","Diagrams",f"{filenamebase}.svg")
501
- GeneralUtilities.ensure_file_exists(diagram_svg_file)
502
- GeneralUtilities.assert_condition(not self.sc.file_is_git_ignored(f"Other/Reference/Technical/Diagrams/{filenamebase}.svg",self.repository),f"Other/Reference/Technical/Diagrams/{filenamebase}.svg must not be git-ignored")#because it should be referencable in markdown-files and viewable without building the codeunits.
503
- self.sc.generate_chart_diagram(diagram_definition_file,os.path.basename(diagram_svg_file))
504
- self.sc.add_tooltips_to_chart_diagram(diagram_svg_file)#so the exact values of a data-point are visible when the svg-file is opened directly in a browser
505
- self.sc.format_xml_file(diagram_svg_file)
506
-
507
- @GeneralUtilities.check_arguments
508
- def search_for_secrets(self) -> None:
509
- self.sc.log.log("Search for secrets...")
510
- self.__search_for_secrets_in_repository()
511
- self.__search_for_secrets_in_oci_images()
512
-
513
- @GeneralUtilities.check_arguments
514
- def __search_for_secrets_in_oci_images(self) -> None:
515
- #the repository-scan above can not find secrets which are only contained in a built image (build-arguments recorded in the
516
- #image-history, environment-variables of the image and files which are generated during the image-build), and betterleaks
517
- #does not look into the "*.tar"-files the image-builds produce. Since these images get published, they are scanned separately.
518
- scan: TFCPS_OCIImageSecretScan = TFCPS_OCIImageSecretScan(self.sc)
519
- findings: list[str] = []
520
- for image in self.__get_oci_images_of_repository():
521
- findings = findings+scan.search_for_secrets_in_image(image, self.repository)
522
- if 0 < len(findings):
523
- for finding in findings:
524
- self.sc.log.log(finding, LogLevel.Error)
525
- raise ValueError(f"Found {len(findings)} secret-finding(s) in the built OCI-image(s). A secret which is part of an image is readable by everybody who is allowed to pull that image. See {os.path.join(self.repository, '.betterleaks.toml')} to ignore known false positives.")
526
-
527
- @GeneralUtilities.check_arguments
528
- def __get_oci_images_of_repository(self) -> list[str]:
529
- """Returns the references of the images which were built by the codeunits of this repository. A codeunit builds an image
530
- exactly if it has an OCI-image-artifacts-folder; the built image is loaded into the local docker-instance by the
531
- codeunit-build, tagged with the lowercase codeunit-name and the codeunit-version."""
532
- result: list[str] = []
533
- for codeunit_name in self.tfcps_tools_general.get_codeunits(self.repository):
534
- codeunit_folder: str = os.path.join(self.repository, codeunit_name)
535
- if not os.path.isdir(os.path.join(codeunit_folder, "Other", "Artifacts", "BuildResult_OCIImage")):
536
- continue
537
- codeunit_version: str = self.tfcps_tools_general.get_version_of_codeunit(os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml"))
538
- result.append(f"{codeunit_name}:{codeunit_version}".lower())
539
- return result
540
-
541
- @GeneralUtilities.check_arguments
542
- def __search_for_secrets_in_repository(self) -> None:
543
- try:
544
- image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "Betterleaks")
545
- except Exception:
546
- image="ghcr.io/betterleaks/betterleaks:latest"
547
- config_file = os.path.join(self.repository, ".betterleaks.toml")
548
- #the filesystem-marker is checked in addition to the convention-based environment-variable (which the rest of this class uses as well),
549
- #because a wrong result here does not only change a message but makes the scan analyse the wrong folder.
550
- running_in_container = os.path.exists("/.dockerenv") or self.sc.is_runnning_in_container()
551
- if running_in_container:
552
- # We run inside the build-container with the docker-socket forwarded to the host-daemon.
553
- # A bind-mount of our in-container repository-path (e.g. "/__w/<repo>/<repo>" on a
554
- # GitHub-runner or "/Workspace/Repository" when the pipeline is run locally) would be
555
- # resolved by the host-daemon, where that path does not exist or points to unrelated
556
- # data (for example test-service-volumes written there by other sibling-containers).
557
- # The sibling betterleaks-container would then scan the wrong directory - without the
558
- # repository-content and without ".betterleaks.toml", which causes false positives.
559
- # Sharing our own volumes instead exposes the repository to betterleaks at the same path.
560
- mount_arguments = ["--volumes-from", self.__get_own_container_id()]
561
- repository_in_scan_container = self.repository
562
- else:
563
- # Running directly on the host: a normal bind-mount works because the path is resolved
564
- # on the same filesystem the docker-daemon uses.
565
- mount_arguments = ["-v", f"{self.repository}:/repo"]
566
- repository_in_scan_container = "/repo"
567
- scan_args = ["dir", repository_in_scan_container, "-v"]
568
- if os.path.isfile(config_file):
569
- # Pass the config explicitly instead of relying on auto-detection, because betterleaks
570
- # silently falls back to the default ruleset when it does not find the config at the
571
- # scan-root, which results in false positives.
572
- scan_args = scan_args + ["-c", f"{repository_in_scan_container}/.betterleaks.toml"]
573
- else:
574
- self.sc.log.log(f"No betterleaks-config found at '{config_file}'; scanning with default ruleset only.", LogLevel.Warning)
575
- # Verify that the docker-daemon is reachable before running the scan. The scan runs betterleaks via "docker run",
576
- # and "docker run" returns exit-code 1 both when the daemon is unreachable and when betterleaks actually finds
577
- # secrets. Without this pre-check an unreachable daemon (e.g. the socket is not forwarded into a build-container)
578
- # would be misreported below as "found secrets", which is a misleading infrastructure-error. "docker version
579
- # --format {{.Server.Version}}" queries the daemon (the server-part) and exits non-zero exactly when it can not
580
- # be reached, so it distinguishes "docker not available" from a real scan-result.
581
- daemon_check = self.sc.run_program_argsasarray("docker", ["version", "--format", "{{.Server.Version}}"], throw_exception_if_exitcode_is_not_zero=False, print_live_output=False)
582
- if daemon_check[0] != 0:
583
- raise ValueError(f"The secret-scan can not be performed because the docker-daemon is not reachable (exit-code {daemon_check[0]}: {daemon_check[2].strip()}). The scan runs betterleaks via 'docker run', which requires a running docker-daemon (inside a build-container its socket must be forwarded to the host-daemon). This is an infrastructure-problem, not a secret-finding in the repository.")
584
- args = ["run", "--rm"] + mount_arguments + [image] + scan_args
585
- result = self.sc.run_program_argsasarray("docker", args, throw_exception_if_exitcode_is_not_zero=False, print_live_output=self.sc.log.loglevel==LogLevel.Debug,print_errors_as_information=True)
586
- if result[0] != 0:
587
- for line in GeneralUtilities.string_to_lines(result[1]):
588
- self.sc.log.log(line, LogLevel.Information)
589
- for line in GeneralUtilities.string_to_lines(result[2]):
590
- self.sc.log.log(line, LogLevel.Error)
591
- raise ValueError(f"Found unignored secret findings (exit code {result[0]}). See {os.path.join(self.repository, '.betterleaks.toml')} to ignore known false positives.")
592
-
593
- @GeneralUtilities.check_arguments
594
- def __get_own_container_id(self) -> str:
595
- # Determine the id of the container this process runs in so its volumes can be shared with
596
- # sibling-containers via "docker run --volumes-from".
597
- # In mountinfo the own container-id only appears reliably in the source-path of the
598
- # "/etc/hostname"/"/etc/hosts"/"/etc/resolv.conf"-mounts (".../containers/<id>/..."). A plain
599
- # 64-hex-match there would also hit overlay-layer-hashes (which are not containers), so the
600
- # "containers/"-prefix must be matched explicitly.
601
- try:
602
- with open("/proc/self/mountinfo", "r", encoding="utf-8") as file_handle:
603
- match = re.search(r"/containers/([0-9a-f]{64})/", file_handle.read())
604
- if match is not None:
605
- return match.group(1)
606
- except Exception:
607
- pass
608
- # cgroup (v1): the container-id is part of the cgroup-path; here a plain 64-hex-match is safe.
609
- try:
610
- with open("/proc/self/cgroup", "r", encoding="utf-8") as file_handle:
611
- match = re.search(r"[0-9a-f]{64}", file_handle.read())
612
- if match is not None:
613
- return match.group(0)
614
- except Exception:
615
- pass
616
- # Fallback: the hostname equals the short container-id for containers started without an
617
- # explicit hostname.
618
- return socket.gethostname()
619
-
620
- @GeneralUtilities.check_arguments
621
- def use_cache(self) -> bool:
622
- return self.__use_cache
623
-
624
-
625
- @GeneralUtilities.check_arguments
626
- def is_pre_merge(self) -> bool:
627
- return self.__is_pre_merge
628
-
629
- @GeneralUtilities.check_arguments
630
- def add_ready_to_merge_flag(self) -> bool:
631
- if self.__add_ready_to_merge_flag:
632
- return True
633
- if self.sc.git_get_current_branch_name(self.repository)=="other/maintenance":
634
- return True
635
- return False
636
-
637
- @GeneralUtilities.check_arguments
638
- def update_dependencies(self) -> None:
639
- repository=self.repository
640
- self.sc.log.log("Update dependencies for product...")
641
- self.update_year_in_license_file()
642
- self.sc.assert_is_git_repository(repository)
643
- self.sc.assert_no_uncommitted_changes(repository)
644
- #the update is done while the codeunits are built regularly (see TFCPS_UpdateDependenciesHook): a codeunit is built before its
645
- #dependencies are updated (some programming-languages need that) and again afterwards if the update changed something. Using the
646
- #regular build means the update also gets all preparation-steps of a build - especially the package-sources which are required to
647
- #resolve dependencies which are not available on a public package-source.
648
- self.build_codeunits(TFCPS_UpdateDependenciesHook(self.sc, self.tfcps_tools_general))
649
- if self.sc.git_repository_has_uncommitted_changes(repository):
650
- changelog_folder = os.path.join(repository, "Other", "Resources", "Changelog")
651
- project_version:str=self.tfcps_tools_general.get_version_of_project(repository)
652
- changelog_file = os.path.join(changelog_folder, f"v{project_version}.md")
653
- if not os.path.isfile(changelog_file):
654
- self.__ensure_changelog_file_is_added(repository, project_version)
655
- self.build_codeunits()#check the codeunits are buildable at all with all updates together
656
- self.sc.git_commit(repository, "Updated dependencies", stage_all_changes=True)
657
-
658
- @GeneralUtilities.check_arguments
659
- def __ensure_changelog_file_is_added(self, repository_folder: str, version_of_project: str):
660
- changelog_file = os.path.join(repository_folder, "Other", "Resources", "Changelog", f"v{version_of_project}.md")
661
- if not os.path.isfile(changelog_file):
662
- GeneralUtilities.ensure_file_exists(changelog_file)
663
- GeneralUtilities.write_text_to_file(changelog_file, """# Release notes
664
-
665
- ## Changes
666
-
667
- - Updated dependencies.
668
- """)
669
-
670
- @GeneralUtilities.check_arguments
671
- def update_year_in_license_file(self) -> None:
672
- self.sc.update_year_in_first_line_of_file(os.path.join(self.repository, "License.txt"))
1
+ import os
2
+ import json
3
+ import re
4
+ import socket
5
+ from datetime import datetime, timedelta,timezone
6
+ import xmlschema
7
+ import yaml
8
+ from packaging.version import Version
9
+ from ..GeneralUtilities import GeneralUtilities
10
+ from ..ScriptCollectionCore import ScriptCollectionCore
11
+ from ..SCLog import LogLevel
12
+ from .TFCPS_BuildCodeUnitsHook import TFCPS_BuildCodeUnitsHook
13
+ from .TFCPS_CodeUnit_BuildCodeUnit import TFCPS_CodeUnit_BuildCodeUnit
14
+ from .TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
15
+ from .TFCPS_Tools_General import TFCPS_Tools_General
16
+
17
+
18
+ class TFCPS_UpdateDependenciesHook(TFCPS_BuildCodeUnitsHook):
19
+ """Updates the dependencies of the repository and of its codeunits while the codeunits are built.
20
+ The dependency-update is implemented as a hook of the regular codeunit-build (and not as an own reduced build-process, which is how
21
+ it was implemented before) so that it runs with exactly the same preparation as a normal build: the required environment-variables
22
+ are set, the custom pre-codeunit-build-script runs and therefore the package-sources which are needed to resolve the dependencies of
23
+ a codeunit are available. Without that preparation an update fails as soon as a codeunit has a dependency which is not available on a
24
+ public package-source - which was the case for an update inside a build-container."""
25
+
26
+ __sc: ScriptCollectionCore = None
27
+ __tfcps_tools_general: TFCPS_Tools_General = None
28
+
29
+ def __init__(self, sc: ScriptCollectionCore, tfcps_tools_general: TFCPS_Tools_General):
30
+ self.__sc = sc
31
+ self.__tfcps_tools_general = tfcps_tools_general
32
+
33
+ @GeneralUtilities.check_arguments
34
+ def run_after_preparation(self, repository: str) -> None:
35
+ update_dependencies_script_folder: str = os.path.join(repository, "Other", "Scripts")
36
+ if os.path.isfile(os.path.join(update_dependencies_script_folder, "UpdateDependencies.py")):
37
+ self.__sc.log.log("Update dependencies of the repository...")
38
+ self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", update_dependencies_script_folder)
39
+
40
+ @GeneralUtilities.check_arguments
41
+ def run_after_codeunit_was_built(self, codeunit_build: TFCPS_CodeUnit_BuildCodeUnit) -> None:
42
+ codeunit_name: str = codeunit_build.codeunit_name
43
+ codeunit_folder: str = codeunit_build.codeunit_folder
44
+ repository: str = codeunit_build.repository_folder
45
+ if not self.__tfcps_tools_general.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml")):
46
+ return
47
+ self.__sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
48
+ self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", os.path.join(codeunit_folder, "Other"))
49
+ if self.__sc.git_repository_has_uncommitted_changes(repository):
50
+ #the update changed something, so it has to be verified that the codeunit is still buildable with the updated dependencies.
51
+ self.__sc.log.log(f"Build codeunit {codeunit_name} again to verify it is still buildable with the updated dependencies...")
52
+ codeunit_build.build_codeunit()
53
+
54
+ class TFCPS_CodeUnit_BuildCodeUnits:
55
+ repository:str=None
56
+ tfcps_tools_general:TFCPS_Tools_General=None
57
+ sc:ScriptCollectionCore=None
58
+ target_environment_type:str=None
59
+ additionalargumentsfile:str=None
60
+ __use_cache:bool = None
61
+ __is_pre_merge:bool = None
62
+ __assert_no_new_changes:bool = None
63
+ __add_ready_to_merge_flag:bool = None
64
+ __fast_lane:bool = None
65
+
66
+ def __init__(self,repository:str,loglevel:LogLevel,target_environment_type:str,additionalargumentsfile:str,use_cache:bool,is_pre_merge:bool,assertnonewchanges:bool,add_ready_to_merge_flag:bool=False,fast_lane:bool=False):
67
+ self.sc=ScriptCollectionCore()
68
+ self.sc.log.loglevel=loglevel
69
+ self.__use_cache=use_cache
70
+ self.sc.assert_is_git_repository(repository)
71
+ self.repository=repository
72
+ self.tfcps_tools_general:TFCPS_Tools_General=TFCPS_Tools_General(self.sc)
73
+ allowed_target_environment_types=["Development","QualityCheck","Productive"]
74
+ GeneralUtilities.assert_condition(target_environment_type in allowed_target_environment_types,"Unknown target-environment-type. Allowed values are: "+", ".join(allowed_target_environment_types))
75
+ self.target_environment_type=target_environment_type
76
+ self.additionalargumentsfile=additionalargumentsfile
77
+ self.__is_pre_merge=is_pre_merge
78
+ self.__assert_no_new_changes=assertnonewchanges
79
+ self.__add_ready_to_merge_flag=add_ready_to_merge_flag
80
+ self.__fast_lane=fast_lane
81
+
82
+ @GeneralUtilities.check_arguments
83
+ def build_codeunits(self, hook: TFCPS_BuildCodeUnitsHook = None) -> None:
84
+ """Builds all codeunits of the repository. The optional hook allows a caller inside ScriptCollection to run additional actions
85
+ between the regular build-steps (see TFCPS_BuildCodeUnitsHook); a caller which only wants to build passes nothing."""
86
+ if hook is None:
87
+ hook = TFCPS_BuildCodeUnitsHook()
88
+ self.sc.log.log(GeneralUtilities.get_line())
89
+ start_time:datetime=GeneralUtilities.get_now()
90
+ ready_to_merge_file=os.path.join(self.repository,".ScriptCollection",".IsReadyToMerge")
91
+ error_occurred=False
92
+ try:
93
+ if self.__fast_lane:
94
+ current_branch_name = self.sc.git_get_current_branch_name(self.repository)
95
+ GeneralUtilities.assert_condition(self.sc.is_fix_branch(current_branch_name), f"Fastlane-builds are only allowed on branches whose name starts with 'fix/', but the current branch is '{current_branch_name}'.")
96
+
97
+ #assert that the product-information-file exists
98
+ product_information_file = os.path.join(self.repository, ".ScriptCollection", "ProductInformation.xml")
99
+ GeneralUtilities.assert_file_exists(product_information_file, f"The file '{product_information_file}' does not exist.")
100
+
101
+ #when the build runs inside a container, ensure the used SCBuilder-image is at least the version required by this repository (defined in .ScriptCollection/OCIImages/ImageDefinition.csv)
102
+ if self.sc.is_runnning_in_container():
103
+ scbuilder_version_environment_value = os.environ.get("SCBuilderVersion")
104
+ GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(scbuilder_version_environment_value), "The environment-variable 'SCBuilderVersion' is not set although the build runs inside a container (environment-variable 'ISRUNNINGINCONTAINER' is 'true').")
105
+ required_scbuilder_version = Version(self.tfcps_tools_general.oci_image_manager.get_tag_for_image(self.repository, "SCBuilder"))
106
+ version_match = re.search(r"\d+(\.\d+)+", scbuilder_version_environment_value)
107
+ GeneralUtilities.assert_condition(version_match is not None, f"The environment-variable 'SCBuilderVersion' (value: '{scbuilder_version_environment_value}') does not contain a version-string.")
108
+ actual_scbuilder_version = Version(version_match.group(0))
109
+ GeneralUtilities.assert_condition(actual_scbuilder_version >= required_scbuilder_version, f"The used SCBuilder-version {actual_scbuilder_version} is older than the version {required_scbuilder_version} required by '{self.tfcps_tools_general.get_product_name(self.repository)}' (defined in .ScriptCollection/OCIImages/ImageDefinition.csv). Please update to a newer SCBuilder-image.")
110
+
111
+ if self.is_pre_merge():
112
+ GeneralUtilities.assert_condition(not self.__assert_no_new_changes,f"A pre-merge build can not be done with the assert-no-new-changes-option.")
113
+
114
+ self.__update_openspec()
115
+
116
+ self.sc.git_set_local_configuration_value(self.repository, "core.autocrlf", "false")
117
+
118
+ #ensure the artifacts-folder of the repository is git-ignored so build-results never show up as uncommitted changes
119
+ self.sc.ensure_line_is_in_gitignore(self.repository, "Other/Resources/Artifacts")
120
+
121
+ #ensure <repo>/.ScriptCollection/.gitignore is set up (ignores the cache-folder so cache-files never show up as uncommitted changes)
122
+ self.sc.ensure_scriptcollection_gitignore_is_setup(self.repository)
123
+
124
+ self.sc.log.log(f"Start building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(start_time,False)}. (Target environment-type: {self.target_environment_type})")
125
+
126
+ self.tfcps_tools_general.ensure_required_environment_variables_are_set(self.repository)
127
+
128
+ self.__run_custom_pre_codeunit_build_script()
129
+
130
+ try:
131
+ xmlschema.validate(product_information_file, "https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/raw/main/Conventions/RepositoryStructure/CommonProjectStructure/productinformation.xsd")
132
+ except Exception as exception:
133
+ self.sc.log.log_exception(f"'{product_information_file}' could not be validated against the XSD:", exception, LogLevel.Warning)
134
+
135
+ if self.__assert_no_new_changes:
136
+ self.sc.assert_no_uncommitted_changes(self.repository,"Can not build codeunit: There are uncommitted changes in the repository.")
137
+
138
+ #run prepare-script
139
+ self.run_prepare_script()
140
+
141
+ hook.run_after_preparation(self.repository)
142
+
143
+ #check if changelog exists
144
+ changelog_file=os.path.join(self.repository,"Other","Resources","Changelog",f"v{self.tfcps_tools_general.get_version_of_project(self.repository)}.md")
145
+ GeneralUtilities.assert_file_exists(changelog_file,f"Changelogfile \"{changelog_file}\" does not exist. Try to create it for example using \"sccreatechangelogentry -m ...\".")
146
+
147
+ #mark current version as supported
148
+ now = GeneralUtilities.get_now()
149
+ project_version:str=self.tfcps_tools_general.get_version_of_project(self.repository)
150
+ if not self.tfcps_tools_general.suport_information_exists(self.repository, project_version):
151
+ amount_of_years_for_support:int=1
152
+ support_time = timedelta(days=365*amount_of_years_for_support+30*3+1)
153
+ until = now + support_time
154
+ until_day = datetime(until.year, until.month, until.day, 0, 0, 0)
155
+ from_day = datetime(now.year, now.month, now.day, 0, 0, 0)
156
+ self.tfcps_tools_general.mark_current_version_as_supported(self.repository,project_version,from_day,until_day)
157
+
158
+ codeunits:list[str]=self.tfcps_tools_general.get_codeunits(self.repository)
159
+ GeneralUtilities.assert_condition(0<len(codeunits),f"No codeunits found in repository {self.repository}.")
160
+ self.sc.log.log("Codeunits will be built in the following order:")
161
+ for codeunit_name in codeunits:
162
+ self.sc.log.log(f" - {codeunit_name}")
163
+ for codeunit_name in codeunits:
164
+ tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(os.path.join(self.repository,codeunit_name),self.sc.log.loglevel,self.target_environment_type,self.additionalargumentsfile,self.use_cache(),self.is_pre_merge(),self.__fast_lane)
165
+ self.sc.log.log(GeneralUtilities.get_line())
166
+ tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()
167
+ hook.run_after_codeunit_was_built(tFCPS_CodeUnit_BuildCodeUnit)
168
+ self.sc.log.log(GeneralUtilities.get_line())
169
+
170
+ self.search_for_secrets()
171
+ self.tfcps_tools_general.generate_svg_files_from_plantuml_files_for_repository(self.repository, self.use_cache())
172
+ self.__normalize_line_endings_of_common_files()
173
+
174
+ self.sc.log.log(GeneralUtilities.get_line())
175
+
176
+ if self.is_pre_merge():
177
+ self.__translate()
178
+ self.__collect_metrics()
179
+ self.__generate_loc_diagram()
180
+ self.sc.log.log(GeneralUtilities.get_line())
181
+
182
+ except Exception:
183
+ error_occurred=True
184
+ raise
185
+ finally:
186
+ if error_occurred:
187
+ GeneralUtilities.ensure_file_does_not_exist(ready_to_merge_file)
188
+ else:
189
+ if self.is_pre_merge():
190
+ GeneralUtilities.ensure_file_does_not_exist(ready_to_merge_file)#ensure it does not exist because the flag is not supposed to be on the main branch
191
+ else:
192
+ if self.is_working_branch():
193
+ if self.add_ready_to_merge_flag():
194
+ GeneralUtilities.ensure_file_exists(ready_to_merge_file)
195
+
196
+
197
+ if self.__assert_no_new_changes:
198
+ self.sc.assert_no_uncommitted_changes(self.repository,"There are new uncommitted changes in the repository.")
199
+
200
+ self.__create_artifacts_archive(codeunits)
201
+
202
+ end_time:datetime=GeneralUtilities.get_now()
203
+ duration=end_time-start_time
204
+ self.sc.log.log(f"Finished building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(end_time,False)}. (Duration: {GeneralUtilities.timedelta_to_simple_string(duration)})")
205
+ self.sc.log.log(GeneralUtilities.get_line())
206
+
207
+ @GeneralUtilities.check_arguments
208
+ def __create_artifacts_archive(self, codeunits:list[str]) -> str:
209
+ product_name:str=self.tfcps_tools_general.get_product_name(self.repository)
210
+ product_version:str=self.tfcps_tools_general.get_version_of_project(self.repository)
211
+ target_folder:str=os.path.join(self.repository,"Other","Resources","Artifacts")
212
+ target_file_zip:str=os.path.join(target_folder,f"{product_name}_Artifacts_v{product_version}.zip")
213
+ target_file_infotxt:str=os.path.join(target_folder,f"{product_name}_Artifacts_v{product_version}.information.txt")
214
+
215
+ artifacts_folders:dict[str,str]={}
216
+ for codeunit_name in codeunits:
217
+ artifacts_folder = os.path.join(self.repository, codeunit_name, "Other", "Artifacts")
218
+ GeneralUtilities.assert_folder_exists(artifacts_folder,f"The artifacts-folder '{artifacts_folder}' of codeunit '{codeunit_name}' does not exist.")
219
+ artifacts_folders[codeunit_name]=artifacts_folder
220
+
221
+ GeneralUtilities.ensure_directory_exists(target_folder)
222
+ GeneralUtilities.ensure_file_does_not_exist(target_file_zip)#ensure the archive of a previous build gets replaced
223
+ self.sc.create_zip_archive_of_folders(artifacts_folders,target_file_zip)
224
+
225
+ hash_of_zip_file:str=GeneralUtilities.get_sha256_of_file(target_file_zip)
226
+ creation_timestamp:datetime=GeneralUtilities.get_now()
227
+ GeneralUtilities.ensure_file_does_not_exist(target_file_infotxt)
228
+ GeneralUtilities.ensure_file_exists(target_file_infotxt)
229
+ information_lines:list[str]=[
230
+ f"Product: {product_name}",
231
+ f"Version: {product_version}",
232
+ f"Artifacts-archive-hash: {hash_of_zip_file}",
233
+ f"Creation-timestamp: {GeneralUtilities.datetime_to_string_for_readable_entry(creation_timestamp,False)}",
234
+ ]
235
+ GeneralUtilities.write_lines_to_file(target_file_infotxt,information_lines)
236
+ normalized_relative_path=GeneralUtilities.normalize_path(os.path.relpath(target_file_zip, self.repository).replace("\\","/"))
237
+ self.sc.log.log(f"Created artifacts-archive \"{normalized_relative_path}\":")
238
+ for information_line in information_lines:
239
+ self.sc.log.log(f" {information_line}")
240
+
241
+ @GeneralUtilities.check_arguments
242
+ def __normalize_line_endings_of_common_files(self) -> None:
243
+ self.sc.format_xml_file(os.path.join(self.repository, ".ScriptCollection", "ProductInformation.xml"))
244
+
245
+ for codeunit in self.tfcps_tools_general.get_codeunits(self.repository):
246
+ self.sc.format_xml_file(os.path.join(self.repository, codeunit, f"{codeunit}.codeunit.xml"))
247
+
248
+ workspace_file = os.path.join(self.repository, f"{self.tfcps_tools_general.get_product_name(self.repository)}.code-workspace")
249
+ if os.path.isfile(workspace_file):
250
+ self.sc.format_json_file(workspace_file)
251
+
252
+ #TODO add option to define exceptions (means: files which should not be normalized).
253
+ self.sc.normalize_line_endings_of_files_in_folder(self.repository, ["txt", "md", "py", "json", "xml", "svg", "csv", "yml", "yaml", "toml","gitignore", "gitattributes", "code-workspace"])
254
+
255
+ @GeneralUtilities.check_arguments
256
+ def is_working_branch(self)->bool:
257
+ if self.sc.git_repository_has_uncommitted_changes(self.repository):
258
+ return True
259
+ if self.sc.git_get_current_branch_name(self.repository) != "main":
260
+ return True
261
+ return False
262
+
263
+ @GeneralUtilities.check_arguments
264
+ def __get_build_arguments(self, repository: str) -> list[str]:
265
+ """Returns the arguments which describe the current build. They are passed to every script which is called by this build
266
+ (the prepare-script of the repository as well as the optional user-specific custom scripts), so such a script can behave
267
+ differently depending on the repository and the target-environment-type it is called for."""
268
+ result: list[str] = ["--repository", repository, "--targetenvironmenttype", self.target_environment_type, "--verbosity", str(int(self.sc.log.loglevel))]
269
+ if GeneralUtilities.string_has_content(self.additionalargumentsfile):
270
+ result = result+["--additionalargumentsfile", self.additionalargumentsfile]
271
+ if not self.__use_cache:
272
+ if self.sc.git_repository_has_uncommitted_changes(repository):
273
+ self.sc.log.log("No-cache-option can not be applied because there are uncommited changes in the repository.", LogLevel.Warning)
274
+ else:
275
+ result = result+["--nocache"]
276
+ return result
277
+
278
+ @GeneralUtilities.check_arguments
279
+ def __run_custom_pre_codeunit_build_script(self) -> None:
280
+ """Runs the optional user-specific script which prepares this machine for a codeunit-build. The script is located in
281
+ '~/.ScriptCollection/TFCPS', so it is outside of any repository and is not part of a product's sourcecode.
282
+ Which script is used depends on where the build runs, because both cases need different preparation-steps:
283
+ - build directly on the host ('scbuildcodeunits'): 'CustomPreCodeUnitBuildScript.py'
284
+ - build inside the container ('scbuildcodeunitsc' or a build-pipeline): 'CustomPreCodeUnitBuildScriptInContainer.py', located in
285
+ the folder returned by TFCPS_Tools_General.get_custom_scripts_folder_for_container(). It is searched in the folder into which
286
+ 'scbuildcodeunitsc' mounts that whole folder and - if it is not there - directly in the configuration-folder, because a
287
+ build-runner usually gets the whole configuration-folder mounted instead (see the article 'Build-runner-configuration'). The
288
+ script which the host itself runs before it starts the container is 'CustomPreCodeUnitBuildScriptForContainer.py' and is
289
+ therefore run there and not here."""
290
+ if self.sc.is_runnning_in_container():
291
+ script_file: str = os.path.join(self.tfcps_tools_general.get_folder_of_custom_scripts_in_container(), "CustomPreCodeUnitBuildScriptInContainer.py")
292
+ if not os.path.isfile(script_file):
293
+ script_file = os.path.join(self.tfcps_tools_general.get_custom_scripts_folder_for_container(), "CustomPreCodeUnitBuildScriptInContainer.py")
294
+ else:
295
+ script_file: str = self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScript.py")
296
+ self.tfcps_tools_general.run_custom_script_if_available(script_file, self.__get_build_arguments(self.repository))
297
+
298
+ @GeneralUtilities.check_arguments
299
+ def run_prepare_script(self):
300
+ args = self.__get_build_arguments(self.repository)
301
+ if os.path.isfile( os.path.join(self.repository,"Other","Scripts","PrepareBuildCodeunits.py")):
302
+ self.sc.log.log("Prepare build codeunits...")
303
+ self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["PrepareBuildCodeunits.py"]+args, os.path.join(self.repository,"Other","Scripts"),print_live_output=True)
304
+
305
+ @GeneralUtilities.check_arguments
306
+ def __update_openspec(self) -> None:
307
+ openspec_configuration_file: str = os.path.join(self.repository, "openspec", "config.yaml")
308
+ if os.path.isfile(openspec_configuration_file):
309
+ self.sc.log.log("Update openspec-instruction-files...")
310
+ self.sc.run_program_argsasarray("openspec", ["update", "--force"], self.repository)
311
+
312
+ @GeneralUtilities.check_arguments
313
+ def build_codeunits_in_container(self,base_mount_folder:str) -> tuple[bool, str]:
314
+ #base_mount_folder is assumed to be an absolute path set correctly by the caller (see BuildCodeUnitsC in Executables.py, which defaults it to the repository itself).
315
+ #it may be the repository itself or any parent-folder of it, which allows the caller to mount not only the repository but the whole surrounding folder-structure into the container.
316
+ normalized_base_mount_folder = os.path.normpath(base_mount_folder)
317
+ normalized_repository = os.path.normpath(self.repository)
318
+ relative_repository_path = os.path.relpath(normalized_repository, normalized_base_mount_folder).replace(os.sep, "/")
319
+ GeneralUtilities.assert_condition(relative_repository_path == "." or not relative_repository_path.startswith(".."), f"The repository '{self.repository}' is not located inside the base-mount-folder '{base_mount_folder}'.")
320
+ container_base_mount_folder = f"/Workspace/Project/{os.path.basename(normalized_base_mount_folder)}"
321
+ container_repository_folder = container_base_mount_folder if relative_repository_path == "." else f"{container_base_mount_folder}/{relative_repository_path}"
322
+ image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "SCBuilder")
323
+
324
+ #build the scbuildcodeunits-arguments based on the current state (analogous to the arguments accepted by the scbuildcodeunits-executable). each token must be a separate argument because run_program_argsasarray passes every list-element verbatim and does not split on spaces.
325
+ scbuildcodeunits_arguments = ["scbuildcodeunits", "-r", container_repository_folder, "-v", "4"]
326
+ if not self.__use_cache:
327
+ scbuildcodeunits_arguments.append("-c")
328
+ if self.__is_pre_merge:
329
+ scbuildcodeunits_arguments.append("-p")
330
+ if self.__assert_no_new_changes:
331
+ scbuildcodeunits_arguments.append("-u")
332
+ if self.__add_ready_to_merge_flag:
333
+ scbuildcodeunits_arguments.append("-m")
334
+ if self.__fast_lane:
335
+ scbuildcodeunits_arguments.append("-f")
336
+ if GeneralUtilities.string_has_content(self.additionalargumentsfile):
337
+ scbuildcodeunits_arguments += ["-a", self.__translate_path_into_container(self.additionalargumentsfile, container_repository_folder)]
338
+
339
+ update_scriptcollection=True
340
+ update_argument:str=""
341
+ if update_scriptcollection:
342
+ update_argument="pip3 install scriptcollection --upgrade && "
343
+ scbuildcodeunits_arguments=["bash","-c", f"{update_argument}scshowversion && "+" ".join(scbuildcodeunits_arguments)]
344
+
345
+ #run the optional user-specific script which prepares this host for a container-build (for example to log in to the registry the image is pulled from).
346
+ #it runs before the environment-variables are resolved, so it can also create the files their values are read from.
347
+ self.tfcps_tools_general.run_custom_script_if_available(self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScriptForContainer.py"), self.__get_build_arguments(self.repository))
348
+
349
+ #mount the optional user-specific scripts which prepare the container itself. the whole folder (not only the pre-codeunit-build-hook
350
+ #itself) is mounted, so the hook can start a sibling script placed next to it (for example to keep the hook independent of a
351
+ #package it only installs, while the actual preparation-logic which needs that package lives in the sibling script; see
352
+ #get_custom_scripts_folder_for_container). The mount is writable (not read-only) because the scripts in that folder also use it as
353
+ #their own download-cache-folder. The build inside the container runs the hook (see __run_custom_pre_codeunit_build_script); if the
354
+ #folder does not exist on the host nothing is mounted and nothing is run.
355
+ mount_arguments: list[str] = []
356
+ custom_scripts_folder_for_inside_the_container: str = self.tfcps_tools_general.get_custom_scripts_folder_for_container()
357
+ if os.path.isdir(custom_scripts_folder_for_inside_the_container):
358
+ mount_arguments += ["-v", f"{custom_scripts_folder_for_inside_the_container}:{self.tfcps_tools_general.get_folder_of_custom_scripts_in_container()}"]
359
+
360
+ #pass the environment-variables which are declared as required in <repository>/.ScriptCollection/ProductInformation.xml into the container
361
+ #so they do not have to be specified explicitly on every scbuildcodeunits-call. Their values are resolved from the user-specific
362
+ #configuration-file (see TFCPS_Tools_General.get_environment_variables_configuration_file), which only exists on the host.
363
+ #only the names are passed as arguments; the values are given to the docker-client through its own environment, because arguments
364
+ #are written to the log and are visible in the process-list of this host, which must not happen for a value which is typically a secret.
365
+ required_environment_variables: dict[str, str] = self.tfcps_tools_general.get_required_environment_variables(self.repository)
366
+ env_arguments: list[str] = []
367
+ for env_variable_name in required_environment_variables:
368
+ env_arguments += ["-e", env_variable_name]
369
+
370
+ #run scbuildcodeunits inside the SCBuilder-image. base_mount_folder is mounted into the container (covering the repository and, for submodules, its real gitdir) and the docker-socket is forwarded because codeunit-builds often start containers (for example local test-services).
371
+ docker_arguments = [
372
+ "run", "--rm",
373
+ "-v", f"{base_mount_folder}:{container_base_mount_folder}",
374
+ "-v", "/var/run/docker.sock:/var/run/docker.sock",
375
+ "-w", container_repository_folder,
376
+ ] + mount_arguments + env_arguments + [
377
+ image,
378
+ ] + scbuildcodeunits_arguments
379
+ self.sc.log.log(f"Build codeunits in container using image \"{image}\"...")
380
+ image_address, image_tag = ScriptCollectionCore.split_image_address_and_tag(image)
381
+ self.sc.docker_pull(image_address, image_tag)
382
+ # the exitcode is evaluated by the caller (returned as part of the result-tuple), so the program-runner must not raise on a non-zero exitcode here.
383
+ result=self.sc.run_program_argsasarray("docker", docker_arguments, throw_exception_if_exitcode_is_not_zero=False, print_live_output=True, env_vars=required_environment_variables)
384
+ exit_code:int=result[0]
385
+ stdout:str=result[1] or GeneralUtilities.empty_string
386
+ stderr:str=result[2] or GeneralUtilities.empty_string
387
+ return (exit_code==0,f"{stdout}\n{stderr}")
388
+
389
+ @GeneralUtilities.check_arguments
390
+ def __translate_path_into_container(self, host_path: str, container_repository_folder: str) -> str:
391
+ normalized_repository = os.path.normpath(self.repository)
392
+ normalized_path = os.path.normpath(host_path)
393
+ if normalized_path.startswith(normalized_repository):
394
+ relative_path = os.path.relpath(normalized_path, normalized_repository).replace(os.sep, "/")
395
+ return f"{container_repository_folder}/{relative_path}"
396
+ return host_path
397
+
398
+ @GeneralUtilities.check_arguments
399
+ def __translate(self) -> None:
400
+ for taskfile_name in ("Taskfile.yml", "Taskfile.yaml"):
401
+ taskfile = os.path.join(self.repository, taskfile_name)
402
+ if os.path.isfile(taskfile):
403
+ with open(taskfile, "r", encoding="utf-8") as f:
404
+ taskfile_content = yaml.safe_load(f)
405
+ if isinstance(taskfile_content.get("tasks"), dict) and "Translate" in taskfile_content["tasks"]:
406
+ self.sc.run_program("task", "Translate", self.repository, print_live_output=self.sc.log.loglevel == LogLevel.Debug)
407
+ break
408
+
409
+ @GeneralUtilities.check_arguments
410
+ def __collect_metrics(self) -> None:
411
+ project_version: str=self.tfcps_tools_general.get_version_of_project(self.repository)
412
+ self.sc.log.log("Collect metrics...")
413
+ loc = self.sc.get_lines_of_code_with_default_excluded_patterns(self.repository)
414
+ loc_metric_folder = os.path.join(self.repository, "Other", "Metrics")
415
+ GeneralUtilities.ensure_directory_exists(loc_metric_folder)
416
+ loc_metric_file = os.path.join(loc_metric_folder, "RepositoryStatisticsPerCommit.csv")
417
+ GeneralUtilities.ensure_file_exists(loc_metric_file)
418
+
419
+ #remove legacy metrics-file. the following 2 lines should be removed after 2026-12-31
420
+ legacy_metrics_file = os.path.join(loc_metric_folder, "LinesOfCode.csv")
421
+ GeneralUtilities.ensure_file_does_not_exist(legacy_metrics_file)
422
+
423
+ old_lines = GeneralUtilities.read_nonempty_lines_from_file(loc_metric_file)
424
+ header_line="Version;Timestamp;LinesOfCode"
425
+ new_lines = [header_line]
426
+ current_version_string=f"v{project_version}"
427
+ for old_line in old_lines:
428
+ if not old_line.startswith(current_version_string+";") and old_line!=header_line:
429
+ new_lines.append(old_line)
430
+ c_date:datetime=GeneralUtilities.get_now().astimezone(timezone.utc)
431
+ commit_date=GeneralUtilities.datetime_to_string_for_logfile_entry(c_date,False)
432
+ new_lines.append(f"{current_version_string};{commit_date};{loc}")
433
+ GeneralUtilities.write_lines_to_file(loc_metric_file, new_lines)
434
+
435
+
436
+ @GeneralUtilities.check_arguments
437
+ def __generate_loc_diagram(self):
438
+ self.sc.log.log("Generate LoC-diagram...")
439
+ loc_metric_folder = os.path.join(self.repository, "Other", "Metrics")
440
+ GeneralUtilities.ensure_directory_exists(loc_metric_folder)
441
+ loc_metric_file = os.path.join(loc_metric_folder, "RepositoryStatisticsPerCommit.csv")
442
+ GeneralUtilities.ensure_file_exists(loc_metric_file)
443
+
444
+ filenamebase="LoC-Diagram"
445
+
446
+ diagram_definition_folder=os.path.join(self.repository, "Other", "Reference","Technical","Diagrams")
447
+ GeneralUtilities.ensure_directory_exists(diagram_definition_folder)
448
+
449
+ diagram_definition_file=os.path.join(diagram_definition_folder,f"{filenamebase}.json")
450
+ GeneralUtilities.ensure_file_exists(diagram_definition_file)
451
+ GeneralUtilities.write_text_to_file(diagram_definition_file,GeneralUtilities.empty_string)
452
+
453
+ loc_data_file=os.path.join(diagram_definition_folder,f"{filenamebase}.csv")
454
+ GeneralUtilities.ensure_file_exists(loc_data_file)
455
+ csv_lines=[]
456
+ for line in GeneralUtilities.read_lines_from_file(loc_metric_file):
457
+ if GeneralUtilities.string_has_content(line):
458
+ splitted=line.split(";")
459
+ v=splitted[0]
460
+ t=splitted[1]
461
+ loc=splitted[2]
462
+ csv_lines.append(f"{v},{t},{loc}")
463
+ GeneralUtilities.write_lines_to_file(loc_data_file,csv_lines)
464
+ self.sc.normalize_line_endings(loc_data_file) # ensure the generated LoC-diagram-csv always uses LF line-endings
465
+ diagram_json = {
466
+ "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
467
+ "description": "Lines of Code over time",
468
+ "width": 800,
469
+ "height": 400,
470
+ "data": {
471
+ "url": f"./{filenamebase}.csv",
472
+ "format": {
473
+ "type": "csv"
474
+ }
475
+ },
476
+ "mark": {
477
+ "type": "line",
478
+ "point": True
479
+ },
480
+ "encoding": {
481
+ "x": {
482
+ "field": "Timestamp",
483
+ "type": "temporal",
484
+ "title": "Date",
485
+ "scale": {
486
+ "type": "utc"#render the ticks in utc so the diagram looks the same independent of the timezone of the machine which builds it
487
+ },
488
+ "axis": {
489
+ "format": "%Y-%m-%d %H:%M:%S (UTC)",#everything which is not a format-specifier starting with a percent-sign is taken literally
490
+ "labelAngle": -45,#the timestamps are too long to be displayed horizontally next to each other
491
+ "labelLimit": 250,#without this the labels would be truncated with an ellipsis because they are longer than the default-limit of 180 pixels
492
+ "tickCount": 8
493
+ }
494
+ },
495
+ "y": {
496
+ "field": "LinesOfCode",
497
+ "type": "quantitative",
498
+ "title": "Lines of Code"
499
+ },
500
+ #the timestamp and the lines of code are not listed here although they are part of the tooltip: the fields which are encoded
501
+ #as x respectively as y are always part of the generated tooltip anyway, so listing them again would show them twice.
502
+ "tooltip": [
503
+ {
504
+ "field": "Version",
505
+ "type": "ordinal"
506
+ }
507
+ ]
508
+ }
509
+ }
510
+
511
+ with open(diagram_definition_file, "w", encoding="utf-8") as f:
512
+ json.dump(
513
+ diagram_json,
514
+ f,
515
+ indent=2,
516
+ sort_keys=False,
517
+ ensure_ascii=False
518
+ )
519
+ diagram_svg_file=os.path.join(self.repository,"Other","Reference","Technical","Diagrams",f"{filenamebase}.svg")
520
+ GeneralUtilities.ensure_file_exists(diagram_svg_file)
521
+ GeneralUtilities.assert_condition(not self.sc.file_is_git_ignored(f"Other/Reference/Technical/Diagrams/{filenamebase}.svg",self.repository),f"Other/Reference/Technical/Diagrams/{filenamebase}.svg must not be git-ignored")#because it should be referencable in markdown-files and viewable without building the codeunits.
522
+ self.sc.generate_chart_diagram(diagram_definition_file,os.path.basename(diagram_svg_file))
523
+ self.sc.add_tooltips_to_chart_diagram(diagram_svg_file)#so the exact values of a data-point are visible when the svg-file is opened directly in a browser
524
+ self.sc.format_xml_file(diagram_svg_file)
525
+
526
+ @GeneralUtilities.check_arguments
527
+ def search_for_secrets(self) -> None:
528
+ self.sc.log.log("Search for secrets...")
529
+ self.__search_for_secrets_in_repository()
530
+ self.__search_for_secrets_in_oci_images()
531
+
532
+ @GeneralUtilities.check_arguments
533
+ def __search_for_secrets_in_oci_images(self) -> None:
534
+ #the repository-scan above can not find secrets which are only contained in a built image (build-arguments recorded in the
535
+ #image-history, environment-variables of the image and files which are generated during the image-build), and betterleaks
536
+ #does not look into the "*.tar"-files the image-builds produce. Since these images get published, they are scanned separately.
537
+ scan: TFCPS_OCIImageSecretScan = TFCPS_OCIImageSecretScan(self.sc)
538
+ findings: list[str] = []
539
+ for image in self.__get_oci_images_of_repository():
540
+ findings = findings+scan.search_for_secrets_in_image(image, self.repository)
541
+ if 0 < len(findings):
542
+ for finding in findings:
543
+ self.sc.log.log(finding, LogLevel.Error)
544
+ raise ValueError(f"Found {len(findings)} secret-finding(s) in the built OCI-image(s). A secret which is part of an image is readable by everybody who is allowed to pull that image. See {os.path.join(self.repository, '.betterleaks.toml')} to ignore known false positives.")
545
+
546
+ @GeneralUtilities.check_arguments
547
+ def __get_oci_images_of_repository(self) -> list[str]:
548
+ """Returns the references of the images which were built by the codeunits of this repository. A codeunit builds an image
549
+ exactly if it has an OCI-image-artifacts-folder; the built image is loaded into the local docker-instance by the
550
+ codeunit-build, tagged with the lowercase codeunit-name and the codeunit-version."""
551
+ result: list[str] = []
552
+ for codeunit_name in self.tfcps_tools_general.get_codeunits(self.repository):
553
+ codeunit_folder: str = os.path.join(self.repository, codeunit_name)
554
+ if not os.path.isdir(os.path.join(codeunit_folder, "Other", "Artifacts", "BuildResult_OCIImage")):
555
+ continue
556
+ codeunit_version: str = self.tfcps_tools_general.get_version_of_codeunit(os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml"))
557
+ result.append(f"{codeunit_name}:{codeunit_version}".lower())
558
+ return result
559
+
560
+ @GeneralUtilities.check_arguments
561
+ def __search_for_secrets_in_repository(self) -> None:
562
+ image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "Betterleaks")
563
+
564
+ config_file = os.path.join(self.repository, ".betterleaks.toml")
565
+ #the filesystem-marker is checked in addition to the convention-based environment-variable (which the rest of this class uses as well),
566
+ #because a wrong result here does not only change a message but makes the scan analyse the wrong folder.
567
+ running_in_container = os.path.exists("/.dockerenv") or self.sc.is_runnning_in_container()
568
+ if running_in_container:
569
+ # We run inside the build-container with the docker-socket forwarded to the host-daemon.
570
+ # A bind-mount of our in-container repository-path (e.g. "/__w/<repo>/<repo>" on a
571
+ # GitHub-runner or "/Workspace/Repository" when the pipeline is run locally) would be
572
+ # resolved by the host-daemon, where that path does not exist or points to unrelated
573
+ # data (for example test-service-volumes written there by other sibling-containers).
574
+ # The sibling betterleaks-container would then scan the wrong directory - without the
575
+ # repository-content and without ".betterleaks.toml", which causes false positives.
576
+ # Sharing our own volumes instead exposes the repository to betterleaks at the same path.
577
+ mount_arguments = ["--volumes-from", self.__get_own_container_id()]
578
+ repository_in_scan_container = self.repository
579
+ else:
580
+ # Running directly on the host: a normal bind-mount works because the path is resolved
581
+ # on the same filesystem the docker-daemon uses.
582
+ mount_arguments = ["-v", f"{self.repository}:/repo"]
583
+ repository_in_scan_container = "/repo"
584
+ scan_args = ["dir", repository_in_scan_container, "-v"]
585
+ if os.path.isfile(config_file):
586
+ # Pass the config explicitly instead of relying on auto-detection, because betterleaks
587
+ # silently falls back to the default ruleset when it does not find the config at the
588
+ # scan-root, which results in false positives.
589
+ scan_args = scan_args + ["-c", f"{repository_in_scan_container}/.betterleaks.toml"]
590
+ else:
591
+ self.sc.log.log(f"No betterleaks-config found at '{config_file}'; scanning with default ruleset only.", LogLevel.Warning)
592
+ # Verify that the docker-daemon is reachable before running the scan. The scan runs betterleaks via "docker run",
593
+ # and "docker run" returns exit-code 1 both when the daemon is unreachable and when betterleaks actually finds
594
+ # secrets. Without this pre-check an unreachable daemon (e.g. the socket is not forwarded into a build-container)
595
+ # would be misreported below as "found secrets", which is a misleading infrastructure-error. "docker version
596
+ # --format {{.Server.Version}}" queries the daemon (the server-part) and exits non-zero exactly when it can not
597
+ # be reached, so it distinguishes "docker not available" from a real scan-result.
598
+ daemon_check = self.sc.run_program_argsasarray("docker", ["version", "--format", "{{.Server.Version}}"], throw_exception_if_exitcode_is_not_zero=False, print_live_output=False)
599
+ if daemon_check[0] != 0:
600
+ raise ValueError(f"The secret-scan can not be performed because the docker-daemon is not reachable (exit-code {daemon_check[0]}: {daemon_check[2].strip()}). The scan runs betterleaks via 'docker run', which requires a running docker-daemon (inside a build-container its socket must be forwarded to the host-daemon). This is an infrastructure-problem, not a secret-finding in the repository.")
601
+ args = ["run", "--rm"] + mount_arguments + [image] + scan_args
602
+ image_address, image_tag = ScriptCollectionCore.split_image_address_and_tag(image)
603
+ self.sc.docker_pull(image_address, image_tag)
604
+ result = self.sc.run_program_argsasarray("docker", args, throw_exception_if_exitcode_is_not_zero=False, print_live_output=self.sc.log.loglevel==False, print_errors_as_information=True)
605
+ if result[0] != 0:
606
+ for line in GeneralUtilities.string_to_lines(result[1]):
607
+ self.sc.log.log(line, LogLevel.Information)
608
+ for line in GeneralUtilities.string_to_lines(result[2]):
609
+ self.sc.log.log(line, LogLevel.Error)
610
+ raise ValueError(f"Found unignored secret findings (exit code {result[0]}). See {os.path.join(self.repository, '.betterleaks.toml')} to ignore known false positives.")
611
+
612
+ @GeneralUtilities.check_arguments
613
+ def __get_own_container_id(self) -> str:
614
+ # Determine the id of the container this process runs in so its volumes can be shared with
615
+ # sibling-containers via "docker run --volumes-from".
616
+ # In mountinfo the own container-id only appears reliably in the source-path of the
617
+ # "/etc/hostname"/"/etc/hosts"/"/etc/resolv.conf"-mounts (".../containers/<id>/..."). A plain
618
+ # 64-hex-match there would also hit overlay-layer-hashes (which are not containers), so the
619
+ # "containers/"-prefix must be matched explicitly.
620
+ try:
621
+ with open("/proc/self/mountinfo", "r", encoding="utf-8") as file_handle:
622
+ match = re.search(r"/containers/([0-9a-f]{64})/", file_handle.read())
623
+ if match is not None:
624
+ return match.group(1)
625
+ except Exception:
626
+ pass
627
+ # cgroup (v1): the container-id is part of the cgroup-path; here a plain 64-hex-match is safe.
628
+ try:
629
+ with open("/proc/self/cgroup", "r", encoding="utf-8") as file_handle:
630
+ match = re.search(r"[0-9a-f]{64}", file_handle.read())
631
+ if match is not None:
632
+ return match.group(0)
633
+ except Exception:
634
+ pass
635
+ # Fallback: the hostname equals the short container-id for containers started without an
636
+ # explicit hostname.
637
+ return socket.gethostname()
638
+
639
+ @GeneralUtilities.check_arguments
640
+ def use_cache(self) -> bool:
641
+ return self.__use_cache
642
+
643
+
644
+ @GeneralUtilities.check_arguments
645
+ def is_pre_merge(self) -> bool:
646
+ return self.__is_pre_merge
647
+
648
+ @GeneralUtilities.check_arguments
649
+ def add_ready_to_merge_flag(self) -> bool:
650
+ if self.__add_ready_to_merge_flag:
651
+ return True
652
+ if self.sc.git_get_current_branch_name(self.repository)=="other/maintenance":
653
+ return True
654
+ return False
655
+
656
+ @GeneralUtilities.check_arguments
657
+ def update_dependencies(self) -> None:
658
+ repository=self.repository
659
+ self.sc.log.log("Update dependencies for product...")
660
+ self.update_year_in_license_file()
661
+ self.sc.assert_is_git_repository(repository)
662
+ self.sc.assert_no_uncommitted_changes(repository)
663
+ #the update is done while the codeunits are built regularly (see TFCPS_UpdateDependenciesHook): a codeunit is built before its
664
+ #dependencies are updated (some programming-languages need that) and again afterwards if the update changed something. Using the
665
+ #regular build means the update also gets all preparation-steps of a build - especially the package-sources which are required to
666
+ #resolve dependencies which are not available on a public package-source.
667
+ self.build_codeunits(TFCPS_UpdateDependenciesHook(self.sc, self.tfcps_tools_general))
668
+ if self.sc.git_repository_has_uncommitted_changes(repository):
669
+ changelog_folder = os.path.join(repository, "Other", "Resources", "Changelog")
670
+ project_version:str=self.tfcps_tools_general.get_version_of_project(repository)
671
+ changelog_file = os.path.join(changelog_folder, f"v{project_version}.md")
672
+ if not os.path.isfile(changelog_file):
673
+ self.__ensure_changelog_file_is_added(repository, project_version)
674
+ self.build_codeunits()#check the codeunits are buildable at all with all updates together
675
+ self.sc.git_commit(repository, "Updated dependencies", stage_all_changes=True)
676
+
677
+ @GeneralUtilities.check_arguments
678
+ def __ensure_changelog_file_is_added(self, repository_folder: str, version_of_project: str):
679
+ changelog_file = os.path.join(repository_folder, "Other", "Resources", "Changelog", f"v{version_of_project}.md")
680
+ if not os.path.isfile(changelog_file):
681
+ GeneralUtilities.ensure_file_exists(changelog_file)
682
+ GeneralUtilities.write_text_to_file(changelog_file, """# Release notes
683
+
684
+ ## Changes
685
+
686
+ - Updated dependencies.
687
+ """)
688
+
689
+ @GeneralUtilities.check_arguments
690
+ def update_year_in_license_file(self) -> None:
691
+ self.sc.update_year_in_first_line_of_file(os.path.join(self.repository, "License.txt"))