ScriptCollection 4.2.129__py3-none-any.whl → 4.2.131__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,352 +1,352 @@
1
- import os
2
- import json
3
- from datetime import datetime, timedelta,timezone
4
- import xmlschema
5
- import yaml
6
- from ..GeneralUtilities import GeneralUtilities
7
- from ..ScriptCollectionCore import ScriptCollectionCore
8
- from ..SCLog import LogLevel
9
- from .TFCPS_CodeUnit_BuildCodeUnit import TFCPS_CodeUnit_BuildCodeUnit
10
- from .TFCPS_Tools_General import TFCPS_Tools_General
11
-
12
- class TFCPS_CodeUnit_BuildCodeUnits:
13
- repository:str=None
14
- tfcps_tools_general:TFCPS_Tools_General=None
15
- sc:ScriptCollectionCore=None
16
- target_environment_type:str=None
17
- additionalargumentsfile:str=None
18
- __use_cache:bool
19
- __is_pre_merge:bool
20
- __assert_no_new_changes:bool
21
-
22
- def __init__(self,repository:str,loglevel:LogLevel,target_environment_type:str,additionalargumentsfile:str,use_cache:bool,is_pre_merge:bool,assertnonewchanges:bool):
23
- self.sc=ScriptCollectionCore()
24
- self.sc.log.loglevel=loglevel
25
- self.__use_cache=use_cache
26
- self.sc.assert_is_git_repository(repository)
27
- self.repository=repository
28
- self.tfcps_tools_general:TFCPS_Tools_General=TFCPS_Tools_General(self.sc)
29
- allowed_target_environment_types=["Development","QualityCheck","Productive"]
30
- GeneralUtilities.assert_condition(target_environment_type in allowed_target_environment_types,"Unknown target-environment-type. Allowed values are: "+", ".join(allowed_target_environment_types))
31
- self.target_environment_type=target_environment_type
32
- self.additionalargumentsfile=additionalargumentsfile
33
- self.__is_pre_merge=is_pre_merge
34
- self.__assert_no_new_changes=assertnonewchanges
35
-
36
- @GeneralUtilities.check_arguments
37
- def build_codeunits(self) -> None:
38
- self.sc.log.log(GeneralUtilities.get_line())
39
- start_time:datetime=GeneralUtilities.get_now()
40
- self.sc.log.log(f"Start building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(start_time,False)}. (Target environment-type: {self.target_environment_type})")
41
- if self.__assert_no_new_changes:
42
- self.sc.assert_no_uncommitted_changes(self.repository,"Can not build codeunit: There are uncommitted changes in the repository.")
43
-
44
- product_information_file = os.path.join(self.repository, ".ScriptCollection", "ProductInformation.xml")
45
- try:
46
- xmlschema.validate(product_information_file, "https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/raw/main/Conventions/RepositoryStructure/CommonProjectStructure/projectinformation.xsd")
47
- except Exception as exception:
48
- self.sc.log.log_exception(f"'{product_information_file}' could not be validated against the XSD:", exception, LogLevel.Warning)
49
-
50
- #check if changelog exists
51
- changelog_file=os.path.join(self.repository,"Other","Resources","Changelog",f"v{self.tfcps_tools_general.get_version_of_project(self.repository)}.md")
52
- GeneralUtilities.assert_file_exists(changelog_file,f"Changelogfile \"{changelog_file}\" does not exist. Try to create it for example using \"sccreatechangelogentry -m ...\".")
53
-
54
- #run prepare-script
55
- self.run_prepare_script()
56
-
57
- #mark current version as supported
58
- now = GeneralUtilities.get_now()
59
- project_version:str=self.tfcps_tools_general.get_version_of_project(self.repository)
60
- if not self.tfcps_tools_general.suport_information_exists(self.repository, project_version):
61
- amount_of_years_for_support:int=1
62
- support_time = timedelta(days=365*amount_of_years_for_support+30*3+1)
63
- until = now + support_time
64
- until_day = datetime(until.year, until.month, until.day, 0, 0, 0)
65
- from_day = datetime(now.year, now.month, now.day, 0, 0, 0)
66
- self.tfcps_tools_general.mark_current_version_as_supported(self.repository,project_version,from_day,until_day)
67
-
68
- codeunits:list[str]=self.tfcps_tools_general.get_codeunits(self.repository)
69
- GeneralUtilities.assert_condition(0<len(codeunits),f"No codeunits found in repository {self.repository}.")
70
- self.sc.log.log("Codeunits will be built in the following order:")
71
- for codeunit_name in codeunits:
72
- self.sc.log.log(f" - {codeunit_name}")
73
- for codeunit_name in codeunits:
74
- tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(os.path.join(self.repository,codeunit_name),self.sc.log.loglevel,self.target_environment_type,self.additionalargumentsfile,self.use_cache(),self.is_pre_merge())
75
- self.sc.log.log(GeneralUtilities.get_line())
76
- tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()
77
-
78
- self.sc.log.log(GeneralUtilities.get_line())
79
- self.__search_for_vulnerabilities()
80
- self.search_for_secrets()
81
- if self.is_pre_merge():
82
- self.__translate()
83
- self.__collect_metrics()
84
- self.__generate_loc_diagram()
85
-
86
- if self.__assert_no_new_changes:
87
- self.sc.assert_no_uncommitted_changes(self.repository,"There are new uncommitted changes in the repository.")
88
-
89
- end_time:datetime=GeneralUtilities.get_now()
90
- duration=end_time-start_time
91
- self.sc.log.log(f"Finished building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(end_time,False)}. (Duration: {GeneralUtilities.timedelta_to_simple_string(duration)})")
92
- self.sc.log.log(GeneralUtilities.get_line())
93
-
94
- @GeneralUtilities.check_arguments
95
- def run_prepare_script(self):
96
- args=["--repository",self.repository,"--targetenvironmenttype",self.target_environment_type,"--verbosity",str(int(self.sc.log.loglevel))]
97
- if GeneralUtilities.string_has_content(self.additionalargumentsfile):
98
- args=args+["--additionalargumentsfile", self.additionalargumentsfile]
99
- if not self.__use_cache:
100
- if self.sc.git_repository_has_uncommitted_changes(self.repository):
101
- self.sc.log.log("No-cache-option can not be applied because there are uncommited changes in the repository.",LogLevel.Warning)
102
- else:
103
- args=args+["--nocache"]
104
-
105
- if os.path.isfile( os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS","CustomPreCodeUnitBuildScript.py")):
106
- self.sc.log.log("Run custom pre-codeunitbuild script...")
107
- self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["CustomPreCodeUnitBuildScript.py"]+args, os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS"),print_live_output=True)
108
-
109
- if os.path.isfile( os.path.join(self.repository,"Other","Scripts","PrepareBuildCodeunits.py")):
110
- self.sc.log.log("Prepare build codeunits...")
111
- self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["PrepareBuildCodeunits.py"]+args, os.path.join(self.repository,"Other","Scripts"),print_live_output=True)
112
-
113
- @GeneralUtilities.check_arguments
114
- def build_codeunits_in_container(self) -> None:
115
- container_repository_folder = "/Workspace/Repository"
116
- image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "SCBuilder")
117
-
118
- #build the scbuildcodeunits-arguments based on the current state (analogous to the arguments accepted by the scbuildcodeunits-executable)
119
- scbuildcodeunits_arguments = ["-r", container_repository_folder, "-v", str(int(self.sc.log.loglevel)), "-e", self.target_environment_type]
120
- if not self.__use_cache:
121
- scbuildcodeunits_arguments.append("-c")
122
- if self.__is_pre_merge:
123
- scbuildcodeunits_arguments.append("-p")
124
- if self.__assert_no_new_changes:
125
- scbuildcodeunits_arguments.append("-u")
126
- if GeneralUtilities.string_has_content(self.additionalargumentsfile):
127
- scbuildcodeunits_arguments = scbuildcodeunits_arguments + ["-a", self.__translate_path_into_container(self.additionalargumentsfile, container_repository_folder)]
128
-
129
- #run scbuildcodeunits inside the SCBuilder-image. the repository is mounted into the container and the docker-socket is forwarded because codeunit-builds often start containers (for example local test-services).
130
- docker_arguments = [
131
- "run", "--rm",
132
- "-v", f"{self.repository}:{container_repository_folder}",
133
- "-v", "/var/run/docker.sock:/var/run/docker.sock",
134
- "-w", container_repository_folder,
135
- image,
136
- "scbuildcodeunits",
137
- ] + scbuildcodeunits_arguments
138
- self.sc.log.log(f"Build codeunits in container using image \"{image}\"...")
139
- self.sc.run_program_argsasarray("docker", docker_arguments, print_live_output=True)
140
-
141
- @GeneralUtilities.check_arguments
142
- def __translate_path_into_container(self, host_path: str, container_repository_folder: str) -> str:
143
- normalized_repository = os.path.normpath(self.repository)
144
- normalized_path = os.path.normpath(host_path)
145
- if normalized_path.startswith(normalized_repository):
146
- relative_path = os.path.relpath(normalized_path, normalized_repository).replace(os.sep, "/")
147
- return f"{container_repository_folder}/{relative_path}"
148
- return host_path
149
-
150
- @GeneralUtilities.check_arguments
151
- def __translate(self) -> None:
152
- for taskfile_name in ("Taskfile.yml", "Taskfile.yaml"):
153
- taskfile = os.path.join(self.repository, taskfile_name)
154
- if os.path.isfile(taskfile):
155
- with open(taskfile, "r", encoding="utf-8") as f:
156
- taskfile_content = yaml.safe_load(f)
157
- if isinstance(taskfile_content.get("tasks"), dict) and "Translate" in taskfile_content["tasks"]:
158
- self.sc.run_program("task", "Translate", self.repository, print_live_output=self.sc.log.loglevel == LogLevel.Debug)
159
- break
160
-
161
- @GeneralUtilities.check_arguments
162
- def __collect_metrics(self) -> None:
163
- project_version: str=self.tfcps_tools_general.get_version_of_project(self.repository)
164
- self.sc.log.log("Collect metrics...")
165
- loc = self.sc.get_lines_of_code_with_default_excluded_patterns(self.repository)
166
- loc_metric_folder = os.path.join(self.repository, "Other", "Metrics")
167
- GeneralUtilities.ensure_directory_exists(loc_metric_folder)
168
- loc_metric_file = os.path.join(loc_metric_folder, "RepositoryStatisticsPerCommit.csv")
169
- GeneralUtilities.ensure_file_exists(loc_metric_file)
170
-
171
- #remove legacy metrics-file. the following 2 lines should be removed after 2026-12-31
172
- legacy_metrics_file = os.path.join(loc_metric_folder, "LinesOfCode.csv")
173
- GeneralUtilities.ensure_file_does_not_exist(legacy_metrics_file)
174
-
175
- old_lines = GeneralUtilities.read_nonempty_lines_from_file(loc_metric_file)
176
- header_line="Version;Timestamp;LinesOfCode"
177
- new_lines = [header_line]
178
- current_version_string=f"v{project_version}"
179
- for old_line in old_lines:
180
- if not old_line.startswith(current_version_string+";") and old_line!=header_line:
181
- new_lines.append(old_line)
182
- c_date:datetime=GeneralUtilities.get_now().astimezone(timezone.utc)
183
- commit_date=GeneralUtilities.datetime_to_string_for_logfile_entry(c_date,False)
184
- new_lines.append(f"{current_version_string};{commit_date};{loc}")
185
- GeneralUtilities.write_lines_to_file(loc_metric_file, new_lines)
186
-
187
-
188
- @GeneralUtilities.check_arguments
189
- def __generate_loc_diagram(self):
190
- self.sc.log.log("Generate LoC-diagram...")
191
- loc_metric_folder = os.path.join(self.repository, "Other", "Metrics")
192
- GeneralUtilities.ensure_directory_exists(loc_metric_folder)
193
- loc_metric_file = os.path.join(loc_metric_folder, "RepositoryStatisticsPerCommit.csv")
194
- GeneralUtilities.ensure_file_exists(loc_metric_file)
195
-
196
- filenamebase="LoC-Diagram"
197
-
198
- diagram_definition_folder=os.path.join(self.repository, "Other", "Reference","Technical","Diagrams")
199
- GeneralUtilities.ensure_directory_exists(diagram_definition_folder)
200
-
201
- diagram_definition_file=os.path.join(diagram_definition_folder,f"{filenamebase}.json")
202
- GeneralUtilities.ensure_file_exists(diagram_definition_file)
203
- GeneralUtilities.write_text_to_file(diagram_definition_file,GeneralUtilities.empty_string)
204
-
205
- loc_data_file=os.path.join(diagram_definition_folder,f"{filenamebase}.csv")
206
- GeneralUtilities.ensure_file_exists(loc_data_file)
207
- csv_lines=[]
208
- for line in GeneralUtilities.read_lines_from_file(loc_metric_file):
209
- if GeneralUtilities.string_has_content(line):
210
- splitted=line.split(";")
211
- v=splitted[0]
212
- t=splitted[1]
213
- loc=splitted[2]
214
- csv_lines.append(f"{v},{t},{loc}")
215
- GeneralUtilities.write_lines_to_file(loc_data_file,csv_lines)
216
- diagram_json = {
217
- "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
218
- "description": "Lines of Code over time",
219
- "width": 800,
220
- "height": 400,
221
- "data": {
222
- "url": f"./{filenamebase}.csv",
223
- "format": {
224
- "type": "csv"
225
- }
226
- },
227
- "mark": {
228
- "type": "line",
229
- "point": True
230
- },
231
- "encoding": {
232
- "x": {
233
- "field": "Timestamp",
234
- "type": "temporal",
235
- "title": "Date"
236
- },
237
- "y": {
238
- "field": "LinesOfCode",
239
- "type": "quantitative",
240
- "title": "Lines of Code"
241
- },
242
- "tooltip": [
243
- {
244
- "field": "Version",
245
- "type": "ordinal"
246
- },
247
- {
248
- "field": "LinesOfCode",
249
- "type": "quantitative"
250
- },
251
- {
252
- "field": "Timestamp",
253
- "type": "temporal"
254
- }
255
- ]
256
- }
257
- }
258
-
259
- with open(diagram_definition_file, "w", encoding="utf-8") as f:
260
- json.dump(
261
- diagram_json,
262
- f,
263
- indent=2,
264
- sort_keys=False,
265
- ensure_ascii=False
266
- )
267
- diagram_svg_file=os.path.join(self.repository,"Other","Reference","Technical","Diagrams",f"{filenamebase}.svg")
268
- GeneralUtilities.ensure_file_exists(diagram_svg_file)
269
- GeneralUtilities.assert_condition(not self.sc.file_is_git_ignored(f"Other/Reference/Technical/Diagrams/{filenamebase}.svg",self.repository),f"Other/Reference/Technical/Diagrams/{filenamebase}.svg must not be git-ignored")#because it should be referencable in markdown-files and viewable without building the codeunits.
270
- self.sc.generate_chart_diagram(diagram_definition_file,os.path.basename(diagram_svg_file))
271
- self.sc.format_xml_file(diagram_svg_file)
272
-
273
- @GeneralUtilities.check_arguments
274
- def __search_for_vulnerabilities(self):
275
- pass#TODO
276
-
277
- @GeneralUtilities.check_arguments
278
- def search_for_secrets(self) -> None:
279
- self.sc.log.log("Search for secrets...")
280
- try:
281
- image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "Betterleaks")
282
- except Exception:
283
- image="ghcr.io/betterleaks/betterleaks:latest"
284
- if os.path.isdir(os.path.join(self.repository, ".git")):
285
- scan_args = ["git", "/repo"]
286
- if os.path.isfile(os.path.join(self.repository, ".git")):
287
- self.sc.log.log("Repository is a Git submodule (.git is a file); scanning working tree only (no history).", LogLevel.Warning)
288
- scan_args = ["dir", "/repo"] #usually the git-scan should be used here too but this does not work in a submodule yet, see https://github.com/betterleaks/betterleaks/issues/170
289
- else:
290
- scan_args = ["dir", "/repo"]
291
- args = ["run", "--rm", "-v", f"{self.repository}:/repo", image] + scan_args
292
- result = self.sc.run_program_argsasarray("docker", args, throw_exception_if_exitcode_is_not_zero=False, print_live_output=self.sc.log.loglevel==LogLevel.Debug)
293
- if result[0] != 0:
294
- for line in GeneralUtilities.string_to_lines(result[1]):
295
- self.sc.log.log(line, LogLevel.Information)
296
- for line in GeneralUtilities.string_to_lines(result[2]):
297
- self.sc.log.log(line, LogLevel.Error)
298
- raise ValueError(f"Found unignored secret findings (exit code {result[0]}). See {os.path.join(self.repository, '.betterleaks.toml')} to ignore known false positives.")
299
-
300
- @GeneralUtilities.check_arguments
301
- def use_cache(self) -> bool:
302
- return self.__use_cache
303
-
304
-
305
- @GeneralUtilities.check_arguments
306
- def is_pre_merge(self) -> bool:
307
- return self.__is_pre_merge
308
-
309
- @GeneralUtilities.check_arguments
310
- def update_dependencies(self) -> None:
311
- repository=self.repository
312
- self.sc.log.log("Update dependencies...")
313
- self.update_year_in_license_file()
314
- self.sc.assert_is_git_repository(repository)
315
- self.sc.assert_no_uncommitted_changes(repository)
316
- self.run_prepare_script()
317
- if os.path.isfile(os.path.join(repository,"Other","Scripts","UpdateDependencies.py")):
318
- self.sc.run_program(GeneralUtilities.get_python_executable(),"UpdateDependencies.py",os.path.join(repository,"Other","Scripts"))
319
- codeunits:list[str]=self.tfcps_tools_general.get_codeunits(repository)
320
- for codeunit_name in codeunits:
321
- self.sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
322
- codeunit_folder=os.path.join(repository,codeunit_name)
323
- tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(codeunit_folder,self.sc.log.loglevel,"QualityCheck",None,True,False)
324
- tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()#ensure requirements for updating are there (some programming-languages needs this)
325
- if self.tfcps_tools_general.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder,f"{codeunit_name}.codeunit.xml")):
326
- self.sc.run_program(GeneralUtilities.get_python_executable(),"UpdateDependencies.py",os.path.join(codeunit_folder,"Other"))
327
- tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()#check if codeunit is still buildable
328
- if self.sc.git_repository_has_uncommitted_changes(repository):
329
- changelog_folder = os.path.join(repository, "Other", "Resources", "Changelog")
330
- project_version:str=self.tfcps_tools_general.get_version_of_project(repository)
331
- changelog_file = os.path.join(changelog_folder, f"v{project_version}.md")
332
- if not os.path.isfile(changelog_file):
333
- self.__ensure_changelog_file_is_added(repository, project_version)
334
- t=TFCPS_CodeUnit_BuildCodeUnits(repository,self.sc.log.loglevel,"QualityCheck",None,True,False,False)
335
- t.build_codeunits()#check codeunits are buildable at all
336
- self.sc.git_commit(repository, "Updated dependencies", stage_all_changes=True)
337
-
338
- @GeneralUtilities.check_arguments
339
- def __ensure_changelog_file_is_added(self, repository_folder: str, version_of_project: str):
340
- changelog_file = os.path.join(repository_folder, "Other", "Resources", "Changelog", f"v{version_of_project}.md")
341
- if not os.path.isfile(changelog_file):
342
- GeneralUtilities.ensure_file_exists(changelog_file)
343
- GeneralUtilities.write_text_to_file(changelog_file, """# Release notes
344
-
345
- ## Changes
346
-
347
- - Updated dependencies.
348
- """)
349
-
350
- @GeneralUtilities.check_arguments
351
- def update_year_in_license_file(self) -> None:
352
- self.sc.update_year_in_first_line_of_file(os.path.join(self.repository, "License.txt"))
1
+ import os
2
+ import json
3
+ from datetime import datetime, timedelta,timezone
4
+ import xmlschema
5
+ import yaml
6
+ from ..GeneralUtilities import GeneralUtilities
7
+ from ..ScriptCollectionCore import ScriptCollectionCore
8
+ from ..SCLog import LogLevel
9
+ from .TFCPS_CodeUnit_BuildCodeUnit import TFCPS_CodeUnit_BuildCodeUnit
10
+ from .TFCPS_Tools_General import TFCPS_Tools_General
11
+
12
+ class TFCPS_CodeUnit_BuildCodeUnits:
13
+ repository:str=None
14
+ tfcps_tools_general:TFCPS_Tools_General=None
15
+ sc:ScriptCollectionCore=None
16
+ target_environment_type:str=None
17
+ additionalargumentsfile:str=None
18
+ __use_cache:bool = None
19
+ __is_pre_merge:bool = None
20
+ __assert_no_new_changes:bool = None
21
+
22
+ def __init__(self,repository:str,loglevel:LogLevel,target_environment_type:str,additionalargumentsfile:str,use_cache:bool,is_pre_merge:bool,assertnonewchanges:bool):
23
+ self.sc=ScriptCollectionCore()
24
+ self.sc.log.loglevel=loglevel
25
+ self.__use_cache=use_cache
26
+ self.sc.assert_is_git_repository(repository)
27
+ self.repository=repository
28
+ self.tfcps_tools_general:TFCPS_Tools_General=TFCPS_Tools_General(self.sc)
29
+ allowed_target_environment_types=["Development","QualityCheck","Productive"]
30
+ GeneralUtilities.assert_condition(target_environment_type in allowed_target_environment_types,"Unknown target-environment-type. Allowed values are: "+", ".join(allowed_target_environment_types))
31
+ self.target_environment_type=target_environment_type
32
+ self.additionalargumentsfile=additionalargumentsfile
33
+ self.__is_pre_merge=is_pre_merge
34
+ self.__assert_no_new_changes=assertnonewchanges
35
+
36
+ @GeneralUtilities.check_arguments
37
+ def build_codeunits(self) -> None:
38
+ self.sc.log.log(GeneralUtilities.get_line())
39
+ start_time:datetime=GeneralUtilities.get_now()
40
+ self.sc.log.log(f"Start building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(start_time,False)}. (Target environment-type: {self.target_environment_type})")
41
+ if self.__assert_no_new_changes:
42
+ self.sc.assert_no_uncommitted_changes(self.repository,"Can not build codeunit: There are uncommitted changes in the repository.")
43
+
44
+ product_information_file = os.path.join(self.repository, ".ScriptCollection", "ProductInformation.xml")
45
+ try:
46
+ xmlschema.validate(product_information_file, "https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/raw/main/Conventions/RepositoryStructure/CommonProjectStructure/projectinformation.xsd")
47
+ except Exception as exception:
48
+ self.sc.log.log_exception(f"'{product_information_file}' could not be validated against the XSD:", exception, LogLevel.Warning)
49
+
50
+ #check if changelog exists
51
+ changelog_file=os.path.join(self.repository,"Other","Resources","Changelog",f"v{self.tfcps_tools_general.get_version_of_project(self.repository)}.md")
52
+ GeneralUtilities.assert_file_exists(changelog_file,f"Changelogfile \"{changelog_file}\" does not exist. Try to create it for example using \"sccreatechangelogentry -m ...\".")
53
+
54
+ #run prepare-script
55
+ self.run_prepare_script()
56
+
57
+ #mark current version as supported
58
+ now = GeneralUtilities.get_now()
59
+ project_version:str=self.tfcps_tools_general.get_version_of_project(self.repository)
60
+ if not self.tfcps_tools_general.suport_information_exists(self.repository, project_version):
61
+ amount_of_years_for_support:int=1
62
+ support_time = timedelta(days=365*amount_of_years_for_support+30*3+1)
63
+ until = now + support_time
64
+ until_day = datetime(until.year, until.month, until.day, 0, 0, 0)
65
+ from_day = datetime(now.year, now.month, now.day, 0, 0, 0)
66
+ self.tfcps_tools_general.mark_current_version_as_supported(self.repository,project_version,from_day,until_day)
67
+
68
+ codeunits:list[str]=self.tfcps_tools_general.get_codeunits(self.repository)
69
+ GeneralUtilities.assert_condition(0<len(codeunits),f"No codeunits found in repository {self.repository}.")
70
+ self.sc.log.log("Codeunits will be built in the following order:")
71
+ for codeunit_name in codeunits:
72
+ self.sc.log.log(f" - {codeunit_name}")
73
+ for codeunit_name in codeunits:
74
+ tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(os.path.join(self.repository,codeunit_name),self.sc.log.loglevel,self.target_environment_type,self.additionalargumentsfile,self.use_cache(),self.is_pre_merge())
75
+ self.sc.log.log(GeneralUtilities.get_line())
76
+ tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()
77
+
78
+ self.sc.log.log(GeneralUtilities.get_line())
79
+ self.__search_for_vulnerabilities()
80
+ self.search_for_secrets()
81
+ if self.is_pre_merge():
82
+ self.__translate()
83
+ self.__collect_metrics()
84
+ self.__generate_loc_diagram()
85
+
86
+ if self.__assert_no_new_changes:
87
+ self.sc.assert_no_uncommitted_changes(self.repository,"There are new uncommitted changes in the repository.")
88
+
89
+ end_time:datetime=GeneralUtilities.get_now()
90
+ duration=end_time-start_time
91
+ self.sc.log.log(f"Finished building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(end_time,False)}. (Duration: {GeneralUtilities.timedelta_to_simple_string(duration)})")
92
+ self.sc.log.log(GeneralUtilities.get_line())
93
+
94
+ @GeneralUtilities.check_arguments
95
+ def run_prepare_script(self):
96
+ args=["--repository",self.repository,"--targetenvironmenttype",self.target_environment_type,"--verbosity",str(int(self.sc.log.loglevel))]
97
+ if GeneralUtilities.string_has_content(self.additionalargumentsfile):
98
+ args=args+["--additionalargumentsfile", self.additionalargumentsfile]
99
+ if not self.__use_cache:
100
+ if self.sc.git_repository_has_uncommitted_changes(self.repository):
101
+ self.sc.log.log("No-cache-option can not be applied because there are uncommited changes in the repository.",LogLevel.Warning)
102
+ else:
103
+ args=args+["--nocache"]
104
+
105
+ if os.path.isfile( os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS","CustomPreCodeUnitBuildScript.py")):
106
+ self.sc.log.log("Run custom pre-codeunitbuild script...")
107
+ self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["CustomPreCodeUnitBuildScript.py"]+args, os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS"),print_live_output=True)
108
+
109
+ if os.path.isfile( os.path.join(self.repository,"Other","Scripts","PrepareBuildCodeunits.py")):
110
+ self.sc.log.log("Prepare build codeunits...")
111
+ self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["PrepareBuildCodeunits.py"]+args, os.path.join(self.repository,"Other","Scripts"),print_live_output=True)
112
+
113
+ @GeneralUtilities.check_arguments
114
+ def build_codeunits_in_container(self) -> None:
115
+ container_repository_folder = "/Workspace/Repository"
116
+ image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "SCBuilder")
117
+
118
+ #build the scbuildcodeunits-arguments based on the current state (analogous to the arguments accepted by the scbuildcodeunits-executable)
119
+ scbuildcodeunits_arguments = ["-r", container_repository_folder, "-v", str(int(self.sc.log.loglevel)), "-e", self.target_environment_type]
120
+ if not self.__use_cache:
121
+ scbuildcodeunits_arguments.append("-c")
122
+ if self.__is_pre_merge:
123
+ scbuildcodeunits_arguments.append("-p")
124
+ if self.__assert_no_new_changes:
125
+ scbuildcodeunits_arguments.append("-u")
126
+ if GeneralUtilities.string_has_content(self.additionalargumentsfile):
127
+ scbuildcodeunits_arguments = scbuildcodeunits_arguments + ["-a", self.__translate_path_into_container(self.additionalargumentsfile, container_repository_folder)]
128
+
129
+ #run scbuildcodeunits inside the SCBuilder-image. the repository is mounted into the container and the docker-socket is forwarded because codeunit-builds often start containers (for example local test-services).
130
+ docker_arguments = [
131
+ "run", "--rm",
132
+ "-v", f"{self.repository}:{container_repository_folder}",
133
+ "-v", "/var/run/docker.sock:/var/run/docker.sock",
134
+ "-w", container_repository_folder,
135
+ image,
136
+ "scbuildcodeunits",
137
+ ] + scbuildcodeunits_arguments
138
+ self.sc.log.log(f"Build codeunits in container using image \"{image}\"...")
139
+ self.sc.run_program_argsasarray("docker", docker_arguments, print_live_output=True)
140
+
141
+ @GeneralUtilities.check_arguments
142
+ def __translate_path_into_container(self, host_path: str, container_repository_folder: str) -> str:
143
+ normalized_repository = os.path.normpath(self.repository)
144
+ normalized_path = os.path.normpath(host_path)
145
+ if normalized_path.startswith(normalized_repository):
146
+ relative_path = os.path.relpath(normalized_path, normalized_repository).replace(os.sep, "/")
147
+ return f"{container_repository_folder}/{relative_path}"
148
+ return host_path
149
+
150
+ @GeneralUtilities.check_arguments
151
+ def __translate(self) -> None:
152
+ for taskfile_name in ("Taskfile.yml", "Taskfile.yaml"):
153
+ taskfile = os.path.join(self.repository, taskfile_name)
154
+ if os.path.isfile(taskfile):
155
+ with open(taskfile, "r", encoding="utf-8") as f:
156
+ taskfile_content = yaml.safe_load(f)
157
+ if isinstance(taskfile_content.get("tasks"), dict) and "Translate" in taskfile_content["tasks"]:
158
+ self.sc.run_program("task", "Translate", self.repository, print_live_output=self.sc.log.loglevel == LogLevel.Debug)
159
+ break
160
+
161
+ @GeneralUtilities.check_arguments
162
+ def __collect_metrics(self) -> None:
163
+ project_version: str=self.tfcps_tools_general.get_version_of_project(self.repository)
164
+ self.sc.log.log("Collect metrics...")
165
+ loc = self.sc.get_lines_of_code_with_default_excluded_patterns(self.repository)
166
+ loc_metric_folder = os.path.join(self.repository, "Other", "Metrics")
167
+ GeneralUtilities.ensure_directory_exists(loc_metric_folder)
168
+ loc_metric_file = os.path.join(loc_metric_folder, "RepositoryStatisticsPerCommit.csv")
169
+ GeneralUtilities.ensure_file_exists(loc_metric_file)
170
+
171
+ #remove legacy metrics-file. the following 2 lines should be removed after 2026-12-31
172
+ legacy_metrics_file = os.path.join(loc_metric_folder, "LinesOfCode.csv")
173
+ GeneralUtilities.ensure_file_does_not_exist(legacy_metrics_file)
174
+
175
+ old_lines = GeneralUtilities.read_nonempty_lines_from_file(loc_metric_file)
176
+ header_line="Version;Timestamp;LinesOfCode"
177
+ new_lines = [header_line]
178
+ current_version_string=f"v{project_version}"
179
+ for old_line in old_lines:
180
+ if not old_line.startswith(current_version_string+";") and old_line!=header_line:
181
+ new_lines.append(old_line)
182
+ c_date:datetime=GeneralUtilities.get_now().astimezone(timezone.utc)
183
+ commit_date=GeneralUtilities.datetime_to_string_for_logfile_entry(c_date,False)
184
+ new_lines.append(f"{current_version_string};{commit_date};{loc}")
185
+ GeneralUtilities.write_lines_to_file(loc_metric_file, new_lines)
186
+
187
+
188
+ @GeneralUtilities.check_arguments
189
+ def __generate_loc_diagram(self):
190
+ self.sc.log.log("Generate LoC-diagram...")
191
+ loc_metric_folder = os.path.join(self.repository, "Other", "Metrics")
192
+ GeneralUtilities.ensure_directory_exists(loc_metric_folder)
193
+ loc_metric_file = os.path.join(loc_metric_folder, "RepositoryStatisticsPerCommit.csv")
194
+ GeneralUtilities.ensure_file_exists(loc_metric_file)
195
+
196
+ filenamebase="LoC-Diagram"
197
+
198
+ diagram_definition_folder=os.path.join(self.repository, "Other", "Reference","Technical","Diagrams")
199
+ GeneralUtilities.ensure_directory_exists(diagram_definition_folder)
200
+
201
+ diagram_definition_file=os.path.join(diagram_definition_folder,f"{filenamebase}.json")
202
+ GeneralUtilities.ensure_file_exists(diagram_definition_file)
203
+ GeneralUtilities.write_text_to_file(diagram_definition_file,GeneralUtilities.empty_string)
204
+
205
+ loc_data_file=os.path.join(diagram_definition_folder,f"{filenamebase}.csv")
206
+ GeneralUtilities.ensure_file_exists(loc_data_file)
207
+ csv_lines=[]
208
+ for line in GeneralUtilities.read_lines_from_file(loc_metric_file):
209
+ if GeneralUtilities.string_has_content(line):
210
+ splitted=line.split(";")
211
+ v=splitted[0]
212
+ t=splitted[1]
213
+ loc=splitted[2]
214
+ csv_lines.append(f"{v},{t},{loc}")
215
+ GeneralUtilities.write_lines_to_file(loc_data_file,csv_lines)
216
+ diagram_json = {
217
+ "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
218
+ "description": "Lines of Code over time",
219
+ "width": 800,
220
+ "height": 400,
221
+ "data": {
222
+ "url": f"./{filenamebase}.csv",
223
+ "format": {
224
+ "type": "csv"
225
+ }
226
+ },
227
+ "mark": {
228
+ "type": "line",
229
+ "point": True
230
+ },
231
+ "encoding": {
232
+ "x": {
233
+ "field": "Timestamp",
234
+ "type": "temporal",
235
+ "title": "Date"
236
+ },
237
+ "y": {
238
+ "field": "LinesOfCode",
239
+ "type": "quantitative",
240
+ "title": "Lines of Code"
241
+ },
242
+ "tooltip": [
243
+ {
244
+ "field": "Version",
245
+ "type": "ordinal"
246
+ },
247
+ {
248
+ "field": "LinesOfCode",
249
+ "type": "quantitative"
250
+ },
251
+ {
252
+ "field": "Timestamp",
253
+ "type": "temporal"
254
+ }
255
+ ]
256
+ }
257
+ }
258
+
259
+ with open(diagram_definition_file, "w", encoding="utf-8") as f:
260
+ json.dump(
261
+ diagram_json,
262
+ f,
263
+ indent=2,
264
+ sort_keys=False,
265
+ ensure_ascii=False
266
+ )
267
+ diagram_svg_file=os.path.join(self.repository,"Other","Reference","Technical","Diagrams",f"{filenamebase}.svg")
268
+ GeneralUtilities.ensure_file_exists(diagram_svg_file)
269
+ GeneralUtilities.assert_condition(not self.sc.file_is_git_ignored(f"Other/Reference/Technical/Diagrams/{filenamebase}.svg",self.repository),f"Other/Reference/Technical/Diagrams/{filenamebase}.svg must not be git-ignored")#because it should be referencable in markdown-files and viewable without building the codeunits.
270
+ self.sc.generate_chart_diagram(diagram_definition_file,os.path.basename(diagram_svg_file))
271
+ self.sc.format_xml_file(diagram_svg_file)
272
+
273
+ @GeneralUtilities.check_arguments
274
+ def __search_for_vulnerabilities(self):
275
+ pass#TODO
276
+
277
+ @GeneralUtilities.check_arguments
278
+ def search_for_secrets(self) -> None:
279
+ self.sc.log.log("Search for secrets...")
280
+ try:
281
+ image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "Betterleaks")
282
+ except Exception:
283
+ image="ghcr.io/betterleaks/betterleaks:latest"
284
+ if os.path.isdir(os.path.join(self.repository, ".git")):
285
+ scan_args = ["git", "/repo"]
286
+ if os.path.isfile(os.path.join(self.repository, ".git")):
287
+ self.sc.log.log("Repository is a Git submodule (.git is a file); scanning working tree only (no history).", LogLevel.Warning)
288
+ scan_args = ["dir", "/repo"] #usually the git-scan should be used here too but this does not work in a submodule yet, see https://github.com/betterleaks/betterleaks/issues/170
289
+ else:
290
+ scan_args = ["dir", "/repo"]
291
+ args = ["run", "--rm", "-v", f"{self.repository}:/repo", image] + scan_args
292
+ result = self.sc.run_program_argsasarray("docker", args, throw_exception_if_exitcode_is_not_zero=False, print_live_output=self.sc.log.loglevel==LogLevel.Debug)
293
+ if result[0] != 0:
294
+ for line in GeneralUtilities.string_to_lines(result[1]):
295
+ self.sc.log.log(line, LogLevel.Information)
296
+ for line in GeneralUtilities.string_to_lines(result[2]):
297
+ self.sc.log.log(line, LogLevel.Error)
298
+ raise ValueError(f"Found unignored secret findings (exit code {result[0]}). See {os.path.join(self.repository, '.betterleaks.toml')} to ignore known false positives.")
299
+
300
+ @GeneralUtilities.check_arguments
301
+ def use_cache(self) -> bool:
302
+ return self.__use_cache
303
+
304
+
305
+ @GeneralUtilities.check_arguments
306
+ def is_pre_merge(self) -> bool:
307
+ return self.__is_pre_merge
308
+
309
+ @GeneralUtilities.check_arguments
310
+ def update_dependencies(self) -> None:
311
+ repository=self.repository
312
+ self.sc.log.log("Update dependencies...")
313
+ self.update_year_in_license_file()
314
+ self.sc.assert_is_git_repository(repository)
315
+ self.sc.assert_no_uncommitted_changes(repository)
316
+ self.run_prepare_script()
317
+ if os.path.isfile(os.path.join(repository,"Other","Scripts","UpdateDependencies.py")):
318
+ self.sc.run_program(GeneralUtilities.get_python_executable(),"UpdateDependencies.py",os.path.join(repository,"Other","Scripts"))
319
+ codeunits:list[str]=self.tfcps_tools_general.get_codeunits(repository)
320
+ for codeunit_name in codeunits:
321
+ self.sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
322
+ codeunit_folder=os.path.join(repository,codeunit_name)
323
+ tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(codeunit_folder,self.sc.log.loglevel,"QualityCheck",None,True,False)
324
+ tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()#ensure requirements for updating are there (some programming-languages needs this)
325
+ if self.tfcps_tools_general.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder,f"{codeunit_name}.codeunit.xml")):
326
+ self.sc.run_program(GeneralUtilities.get_python_executable(),"UpdateDependencies.py",os.path.join(codeunit_folder,"Other"))
327
+ tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()#check if codeunit is still buildable
328
+ if self.sc.git_repository_has_uncommitted_changes(repository):
329
+ changelog_folder = os.path.join(repository, "Other", "Resources", "Changelog")
330
+ project_version:str=self.tfcps_tools_general.get_version_of_project(repository)
331
+ changelog_file = os.path.join(changelog_folder, f"v{project_version}.md")
332
+ if not os.path.isfile(changelog_file):
333
+ self.__ensure_changelog_file_is_added(repository, project_version)
334
+ t=TFCPS_CodeUnit_BuildCodeUnits(repository,self.sc.log.loglevel,"QualityCheck",None,True,False,False)
335
+ t.build_codeunits()#check codeunits are buildable at all
336
+ self.sc.git_commit(repository, "Updated dependencies", stage_all_changes=True)
337
+
338
+ @GeneralUtilities.check_arguments
339
+ def __ensure_changelog_file_is_added(self, repository_folder: str, version_of_project: str):
340
+ changelog_file = os.path.join(repository_folder, "Other", "Resources", "Changelog", f"v{version_of_project}.md")
341
+ if not os.path.isfile(changelog_file):
342
+ GeneralUtilities.ensure_file_exists(changelog_file)
343
+ GeneralUtilities.write_text_to_file(changelog_file, """# Release notes
344
+
345
+ ## Changes
346
+
347
+ - Updated dependencies.
348
+ """)
349
+
350
+ @GeneralUtilities.check_arguments
351
+ def update_year_in_license_file(self) -> None:
352
+ self.sc.update_year_in_first_line_of_file(os.path.join(self.repository, "License.txt"))