ScriptCollection 4.2.129__py3-none-any.whl → 4.2.131__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/AnionBuildPlatform.py +9 -9
- ScriptCollection/GeneralUtilities.py +4 -4
- ScriptCollection/OCIImages/OCIImageManager.py +216 -216
- ScriptCollection/ProcessesRunner.py +2 -2
- ScriptCollection/ProgramRunnerSudo.py +110 -110
- ScriptCollection/SCLog.py +6 -6
- ScriptCollection/ScriptCollectionCore.py +3706 -3706
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +2 -2
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +3 -3
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +8 -8
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +352 -352
- ScriptCollection/TFCPS/TFCPS_CreateRelease.py +18 -18
- ScriptCollection/TFCPS/TFCPS_Generic.py +44 -44
- ScriptCollection/TFCPS/TFCPS_MergeToMain.py +128 -128
- ScriptCollection/TFCPS/TFCPS_MergeToStable.py +355 -355
- ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +6 -6
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +51 -15
- ScriptCollection/UnassignedVariablesCheck.py +94 -0
- {scriptcollection-4.2.129.dist-info → scriptcollection-4.2.131.dist-info}/METADATA +1 -1
- {scriptcollection-4.2.129.dist-info → scriptcollection-4.2.131.dist-info}/RECORD +23 -22
- {scriptcollection-4.2.129.dist-info → scriptcollection-4.2.131.dist-info}/WHEEL +0 -0
- {scriptcollection-4.2.129.dist-info → scriptcollection-4.2.131.dist-info}/entry_points.txt +0 -0
- {scriptcollection-4.2.129.dist-info → scriptcollection-4.2.131.dist-info}/top_level.txt +0 -0
|
@@ -1,352 +1,352 @@
|
|
|
1
|
-
import os
|
|
2
|
-
import json
|
|
3
|
-
from datetime import datetime, timedelta,timezone
|
|
4
|
-
import xmlschema
|
|
5
|
-
import yaml
|
|
6
|
-
from ..GeneralUtilities import GeneralUtilities
|
|
7
|
-
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
8
|
-
from ..SCLog import LogLevel
|
|
9
|
-
from .TFCPS_CodeUnit_BuildCodeUnit import TFCPS_CodeUnit_BuildCodeUnit
|
|
10
|
-
from .TFCPS_Tools_General import TFCPS_Tools_General
|
|
11
|
-
|
|
12
|
-
class TFCPS_CodeUnit_BuildCodeUnits:
|
|
13
|
-
repository:str=None
|
|
14
|
-
tfcps_tools_general:TFCPS_Tools_General=None
|
|
15
|
-
sc:ScriptCollectionCore=None
|
|
16
|
-
target_environment_type:str=None
|
|
17
|
-
additionalargumentsfile:str=None
|
|
18
|
-
__use_cache:bool
|
|
19
|
-
__is_pre_merge:bool
|
|
20
|
-
__assert_no_new_changes:bool
|
|
21
|
-
|
|
22
|
-
def __init__(self,repository:str,loglevel:LogLevel,target_environment_type:str,additionalargumentsfile:str,use_cache:bool,is_pre_merge:bool,assertnonewchanges:bool):
|
|
23
|
-
self.sc=ScriptCollectionCore()
|
|
24
|
-
self.sc.log.loglevel=loglevel
|
|
25
|
-
self.__use_cache=use_cache
|
|
26
|
-
self.sc.assert_is_git_repository(repository)
|
|
27
|
-
self.repository=repository
|
|
28
|
-
self.tfcps_tools_general:TFCPS_Tools_General=TFCPS_Tools_General(self.sc)
|
|
29
|
-
allowed_target_environment_types=["Development","QualityCheck","Productive"]
|
|
30
|
-
GeneralUtilities.assert_condition(target_environment_type in allowed_target_environment_types,"Unknown target-environment-type. Allowed values are: "+", ".join(allowed_target_environment_types))
|
|
31
|
-
self.target_environment_type=target_environment_type
|
|
32
|
-
self.additionalargumentsfile=additionalargumentsfile
|
|
33
|
-
self.__is_pre_merge=is_pre_merge
|
|
34
|
-
self.__assert_no_new_changes=assertnonewchanges
|
|
35
|
-
|
|
36
|
-
@GeneralUtilities.check_arguments
|
|
37
|
-
def build_codeunits(self) -> None:
|
|
38
|
-
self.sc.log.log(GeneralUtilities.get_line())
|
|
39
|
-
start_time:datetime=GeneralUtilities.get_now()
|
|
40
|
-
self.sc.log.log(f"Start building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(start_time,False)}. (Target environment-type: {self.target_environment_type})")
|
|
41
|
-
if self.__assert_no_new_changes:
|
|
42
|
-
self.sc.assert_no_uncommitted_changes(self.repository,"Can not build codeunit: There are uncommitted changes in the repository.")
|
|
43
|
-
|
|
44
|
-
product_information_file = os.path.join(self.repository, ".ScriptCollection", "ProductInformation.xml")
|
|
45
|
-
try:
|
|
46
|
-
xmlschema.validate(product_information_file, "https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/raw/main/Conventions/RepositoryStructure/CommonProjectStructure/projectinformation.xsd")
|
|
47
|
-
except Exception as exception:
|
|
48
|
-
self.sc.log.log_exception(f"'{product_information_file}' could not be validated against the XSD:", exception, LogLevel.Warning)
|
|
49
|
-
|
|
50
|
-
#check if changelog exists
|
|
51
|
-
changelog_file=os.path.join(self.repository,"Other","Resources","Changelog",f"v{self.tfcps_tools_general.get_version_of_project(self.repository)}.md")
|
|
52
|
-
GeneralUtilities.assert_file_exists(changelog_file,f"Changelogfile \"{changelog_file}\" does not exist. Try to create it for example using \"sccreatechangelogentry -m ...\".")
|
|
53
|
-
|
|
54
|
-
#run prepare-script
|
|
55
|
-
self.run_prepare_script()
|
|
56
|
-
|
|
57
|
-
#mark current version as supported
|
|
58
|
-
now = GeneralUtilities.get_now()
|
|
59
|
-
project_version:str=self.tfcps_tools_general.get_version_of_project(self.repository)
|
|
60
|
-
if not self.tfcps_tools_general.suport_information_exists(self.repository, project_version):
|
|
61
|
-
amount_of_years_for_support:int=1
|
|
62
|
-
support_time = timedelta(days=365*amount_of_years_for_support+30*3+1)
|
|
63
|
-
until = now + support_time
|
|
64
|
-
until_day = datetime(until.year, until.month, until.day, 0, 0, 0)
|
|
65
|
-
from_day = datetime(now.year, now.month, now.day, 0, 0, 0)
|
|
66
|
-
self.tfcps_tools_general.mark_current_version_as_supported(self.repository,project_version,from_day,until_day)
|
|
67
|
-
|
|
68
|
-
codeunits:list[str]=self.tfcps_tools_general.get_codeunits(self.repository)
|
|
69
|
-
GeneralUtilities.assert_condition(0<len(codeunits),f"No codeunits found in repository {self.repository}.")
|
|
70
|
-
self.sc.log.log("Codeunits will be built in the following order:")
|
|
71
|
-
for codeunit_name in codeunits:
|
|
72
|
-
self.sc.log.log(f" - {codeunit_name}")
|
|
73
|
-
for codeunit_name in codeunits:
|
|
74
|
-
tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(os.path.join(self.repository,codeunit_name),self.sc.log.loglevel,self.target_environment_type,self.additionalargumentsfile,self.use_cache(),self.is_pre_merge())
|
|
75
|
-
self.sc.log.log(GeneralUtilities.get_line())
|
|
76
|
-
tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()
|
|
77
|
-
|
|
78
|
-
self.sc.log.log(GeneralUtilities.get_line())
|
|
79
|
-
self.__search_for_vulnerabilities()
|
|
80
|
-
self.search_for_secrets()
|
|
81
|
-
if self.is_pre_merge():
|
|
82
|
-
self.__translate()
|
|
83
|
-
self.__collect_metrics()
|
|
84
|
-
self.__generate_loc_diagram()
|
|
85
|
-
|
|
86
|
-
if self.__assert_no_new_changes:
|
|
87
|
-
self.sc.assert_no_uncommitted_changes(self.repository,"There are new uncommitted changes in the repository.")
|
|
88
|
-
|
|
89
|
-
end_time:datetime=GeneralUtilities.get_now()
|
|
90
|
-
duration=end_time-start_time
|
|
91
|
-
self.sc.log.log(f"Finished building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(end_time,False)}. (Duration: {GeneralUtilities.timedelta_to_simple_string(duration)})")
|
|
92
|
-
self.sc.log.log(GeneralUtilities.get_line())
|
|
93
|
-
|
|
94
|
-
@GeneralUtilities.check_arguments
|
|
95
|
-
def run_prepare_script(self):
|
|
96
|
-
args=["--repository",self.repository,"--targetenvironmenttype",self.target_environment_type,"--verbosity",str(int(self.sc.log.loglevel))]
|
|
97
|
-
if GeneralUtilities.string_has_content(self.additionalargumentsfile):
|
|
98
|
-
args=args+["--additionalargumentsfile", self.additionalargumentsfile]
|
|
99
|
-
if not self.__use_cache:
|
|
100
|
-
if self.sc.git_repository_has_uncommitted_changes(self.repository):
|
|
101
|
-
self.sc.log.log("No-cache-option can not be applied because there are uncommited changes in the repository.",LogLevel.Warning)
|
|
102
|
-
else:
|
|
103
|
-
args=args+["--nocache"]
|
|
104
|
-
|
|
105
|
-
if os.path.isfile( os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS","CustomPreCodeUnitBuildScript.py")):
|
|
106
|
-
self.sc.log.log("Run custom pre-codeunitbuild script...")
|
|
107
|
-
self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["CustomPreCodeUnitBuildScript.py"]+args, os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS"),print_live_output=True)
|
|
108
|
-
|
|
109
|
-
if os.path.isfile( os.path.join(self.repository,"Other","Scripts","PrepareBuildCodeunits.py")):
|
|
110
|
-
self.sc.log.log("Prepare build codeunits...")
|
|
111
|
-
self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["PrepareBuildCodeunits.py"]+args, os.path.join(self.repository,"Other","Scripts"),print_live_output=True)
|
|
112
|
-
|
|
113
|
-
@GeneralUtilities.check_arguments
|
|
114
|
-
def build_codeunits_in_container(self) -> None:
|
|
115
|
-
container_repository_folder = "/Workspace/Repository"
|
|
116
|
-
image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "SCBuilder")
|
|
117
|
-
|
|
118
|
-
#build the scbuildcodeunits-arguments based on the current state (analogous to the arguments accepted by the scbuildcodeunits-executable)
|
|
119
|
-
scbuildcodeunits_arguments = ["-r", container_repository_folder, "-v", str(int(self.sc.log.loglevel)), "-e", self.target_environment_type]
|
|
120
|
-
if not self.__use_cache:
|
|
121
|
-
scbuildcodeunits_arguments.append("-c")
|
|
122
|
-
if self.__is_pre_merge:
|
|
123
|
-
scbuildcodeunits_arguments.append("-p")
|
|
124
|
-
if self.__assert_no_new_changes:
|
|
125
|
-
scbuildcodeunits_arguments.append("-u")
|
|
126
|
-
if GeneralUtilities.string_has_content(self.additionalargumentsfile):
|
|
127
|
-
scbuildcodeunits_arguments = scbuildcodeunits_arguments + ["-a", self.__translate_path_into_container(self.additionalargumentsfile, container_repository_folder)]
|
|
128
|
-
|
|
129
|
-
#run scbuildcodeunits inside the SCBuilder-image. the repository is mounted into the container and the docker-socket is forwarded because codeunit-builds often start containers (for example local test-services).
|
|
130
|
-
docker_arguments = [
|
|
131
|
-
"run", "--rm",
|
|
132
|
-
"-v", f"{self.repository}:{container_repository_folder}",
|
|
133
|
-
"-v", "/var/run/docker.sock:/var/run/docker.sock",
|
|
134
|
-
"-w", container_repository_folder,
|
|
135
|
-
image,
|
|
136
|
-
"scbuildcodeunits",
|
|
137
|
-
] + scbuildcodeunits_arguments
|
|
138
|
-
self.sc.log.log(f"Build codeunits in container using image \"{image}\"...")
|
|
139
|
-
self.sc.run_program_argsasarray("docker", docker_arguments, print_live_output=True)
|
|
140
|
-
|
|
141
|
-
@GeneralUtilities.check_arguments
|
|
142
|
-
def __translate_path_into_container(self, host_path: str, container_repository_folder: str) -> str:
|
|
143
|
-
normalized_repository = os.path.normpath(self.repository)
|
|
144
|
-
normalized_path = os.path.normpath(host_path)
|
|
145
|
-
if normalized_path.startswith(normalized_repository):
|
|
146
|
-
relative_path = os.path.relpath(normalized_path, normalized_repository).replace(os.sep, "/")
|
|
147
|
-
return f"{container_repository_folder}/{relative_path}"
|
|
148
|
-
return host_path
|
|
149
|
-
|
|
150
|
-
@GeneralUtilities.check_arguments
|
|
151
|
-
def __translate(self) -> None:
|
|
152
|
-
for taskfile_name in ("Taskfile.yml", "Taskfile.yaml"):
|
|
153
|
-
taskfile = os.path.join(self.repository, taskfile_name)
|
|
154
|
-
if os.path.isfile(taskfile):
|
|
155
|
-
with open(taskfile, "r", encoding="utf-8") as f:
|
|
156
|
-
taskfile_content = yaml.safe_load(f)
|
|
157
|
-
if isinstance(taskfile_content.get("tasks"), dict) and "Translate" in taskfile_content["tasks"]:
|
|
158
|
-
self.sc.run_program("task", "Translate", self.repository, print_live_output=self.sc.log.loglevel == LogLevel.Debug)
|
|
159
|
-
break
|
|
160
|
-
|
|
161
|
-
@GeneralUtilities.check_arguments
|
|
162
|
-
def __collect_metrics(self) -> None:
|
|
163
|
-
project_version: str=self.tfcps_tools_general.get_version_of_project(self.repository)
|
|
164
|
-
self.sc.log.log("Collect metrics...")
|
|
165
|
-
loc = self.sc.get_lines_of_code_with_default_excluded_patterns(self.repository)
|
|
166
|
-
loc_metric_folder = os.path.join(self.repository, "Other", "Metrics")
|
|
167
|
-
GeneralUtilities.ensure_directory_exists(loc_metric_folder)
|
|
168
|
-
loc_metric_file = os.path.join(loc_metric_folder, "RepositoryStatisticsPerCommit.csv")
|
|
169
|
-
GeneralUtilities.ensure_file_exists(loc_metric_file)
|
|
170
|
-
|
|
171
|
-
#remove legacy metrics-file. the following 2 lines should be removed after 2026-12-31
|
|
172
|
-
legacy_metrics_file = os.path.join(loc_metric_folder, "LinesOfCode.csv")
|
|
173
|
-
GeneralUtilities.ensure_file_does_not_exist(legacy_metrics_file)
|
|
174
|
-
|
|
175
|
-
old_lines = GeneralUtilities.read_nonempty_lines_from_file(loc_metric_file)
|
|
176
|
-
header_line="Version;Timestamp;LinesOfCode"
|
|
177
|
-
new_lines = [header_line]
|
|
178
|
-
current_version_string=f"v{project_version}"
|
|
179
|
-
for old_line in old_lines:
|
|
180
|
-
if not old_line.startswith(current_version_string+";") and old_line!=header_line:
|
|
181
|
-
new_lines.append(old_line)
|
|
182
|
-
c_date:datetime=GeneralUtilities.get_now().astimezone(timezone.utc)
|
|
183
|
-
commit_date=GeneralUtilities.datetime_to_string_for_logfile_entry(c_date,False)
|
|
184
|
-
new_lines.append(f"{current_version_string};{commit_date};{loc}")
|
|
185
|
-
GeneralUtilities.write_lines_to_file(loc_metric_file, new_lines)
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
@GeneralUtilities.check_arguments
|
|
189
|
-
def __generate_loc_diagram(self):
|
|
190
|
-
self.sc.log.log("Generate LoC-diagram...")
|
|
191
|
-
loc_metric_folder = os.path.join(self.repository, "Other", "Metrics")
|
|
192
|
-
GeneralUtilities.ensure_directory_exists(loc_metric_folder)
|
|
193
|
-
loc_metric_file = os.path.join(loc_metric_folder, "RepositoryStatisticsPerCommit.csv")
|
|
194
|
-
GeneralUtilities.ensure_file_exists(loc_metric_file)
|
|
195
|
-
|
|
196
|
-
filenamebase="LoC-Diagram"
|
|
197
|
-
|
|
198
|
-
diagram_definition_folder=os.path.join(self.repository, "Other", "Reference","Technical","Diagrams")
|
|
199
|
-
GeneralUtilities.ensure_directory_exists(diagram_definition_folder)
|
|
200
|
-
|
|
201
|
-
diagram_definition_file=os.path.join(diagram_definition_folder,f"{filenamebase}.json")
|
|
202
|
-
GeneralUtilities.ensure_file_exists(diagram_definition_file)
|
|
203
|
-
GeneralUtilities.write_text_to_file(diagram_definition_file,GeneralUtilities.empty_string)
|
|
204
|
-
|
|
205
|
-
loc_data_file=os.path.join(diagram_definition_folder,f"{filenamebase}.csv")
|
|
206
|
-
GeneralUtilities.ensure_file_exists(loc_data_file)
|
|
207
|
-
csv_lines=[]
|
|
208
|
-
for line in GeneralUtilities.read_lines_from_file(loc_metric_file):
|
|
209
|
-
if GeneralUtilities.string_has_content(line):
|
|
210
|
-
splitted=line.split(";")
|
|
211
|
-
v=splitted[0]
|
|
212
|
-
t=splitted[1]
|
|
213
|
-
loc=splitted[2]
|
|
214
|
-
csv_lines.append(f"{v},{t},{loc}")
|
|
215
|
-
GeneralUtilities.write_lines_to_file(loc_data_file,csv_lines)
|
|
216
|
-
diagram_json = {
|
|
217
|
-
"$schema": "https://vega.github.io/schema/vega-lite/v5.json",
|
|
218
|
-
"description": "Lines of Code over time",
|
|
219
|
-
"width": 800,
|
|
220
|
-
"height": 400,
|
|
221
|
-
"data": {
|
|
222
|
-
"url": f"./{filenamebase}.csv",
|
|
223
|
-
"format": {
|
|
224
|
-
"type": "csv"
|
|
225
|
-
}
|
|
226
|
-
},
|
|
227
|
-
"mark": {
|
|
228
|
-
"type": "line",
|
|
229
|
-
"point": True
|
|
230
|
-
},
|
|
231
|
-
"encoding": {
|
|
232
|
-
"x": {
|
|
233
|
-
"field": "Timestamp",
|
|
234
|
-
"type": "temporal",
|
|
235
|
-
"title": "Date"
|
|
236
|
-
},
|
|
237
|
-
"y": {
|
|
238
|
-
"field": "LinesOfCode",
|
|
239
|
-
"type": "quantitative",
|
|
240
|
-
"title": "Lines of Code"
|
|
241
|
-
},
|
|
242
|
-
"tooltip": [
|
|
243
|
-
{
|
|
244
|
-
"field": "Version",
|
|
245
|
-
"type": "ordinal"
|
|
246
|
-
},
|
|
247
|
-
{
|
|
248
|
-
"field": "LinesOfCode",
|
|
249
|
-
"type": "quantitative"
|
|
250
|
-
},
|
|
251
|
-
{
|
|
252
|
-
"field": "Timestamp",
|
|
253
|
-
"type": "temporal"
|
|
254
|
-
}
|
|
255
|
-
]
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
with open(diagram_definition_file, "w", encoding="utf-8") as f:
|
|
260
|
-
json.dump(
|
|
261
|
-
diagram_json,
|
|
262
|
-
f,
|
|
263
|
-
indent=2,
|
|
264
|
-
sort_keys=False,
|
|
265
|
-
ensure_ascii=False
|
|
266
|
-
)
|
|
267
|
-
diagram_svg_file=os.path.join(self.repository,"Other","Reference","Technical","Diagrams",f"{filenamebase}.svg")
|
|
268
|
-
GeneralUtilities.ensure_file_exists(diagram_svg_file)
|
|
269
|
-
GeneralUtilities.assert_condition(not self.sc.file_is_git_ignored(f"Other/Reference/Technical/Diagrams/{filenamebase}.svg",self.repository),f"Other/Reference/Technical/Diagrams/{filenamebase}.svg must not be git-ignored")#because it should be referencable in markdown-files and viewable without building the codeunits.
|
|
270
|
-
self.sc.generate_chart_diagram(diagram_definition_file,os.path.basename(diagram_svg_file))
|
|
271
|
-
self.sc.format_xml_file(diagram_svg_file)
|
|
272
|
-
|
|
273
|
-
@GeneralUtilities.check_arguments
|
|
274
|
-
def __search_for_vulnerabilities(self):
|
|
275
|
-
pass#TODO
|
|
276
|
-
|
|
277
|
-
@GeneralUtilities.check_arguments
|
|
278
|
-
def search_for_secrets(self) -> None:
|
|
279
|
-
self.sc.log.log("Search for secrets...")
|
|
280
|
-
try:
|
|
281
|
-
image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "Betterleaks")
|
|
282
|
-
except Exception:
|
|
283
|
-
image="ghcr.io/betterleaks/betterleaks:latest"
|
|
284
|
-
if os.path.isdir(os.path.join(self.repository, ".git")):
|
|
285
|
-
scan_args = ["git", "/repo"]
|
|
286
|
-
if os.path.isfile(os.path.join(self.repository, ".git")):
|
|
287
|
-
self.sc.log.log("Repository is a Git submodule (.git is a file); scanning working tree only (no history).", LogLevel.Warning)
|
|
288
|
-
scan_args = ["dir", "/repo"] #usually the git-scan should be used here too but this does not work in a submodule yet, see https://github.com/betterleaks/betterleaks/issues/170
|
|
289
|
-
else:
|
|
290
|
-
scan_args = ["dir", "/repo"]
|
|
291
|
-
args = ["run", "--rm", "-v", f"{self.repository}:/repo", image] + scan_args
|
|
292
|
-
result = self.sc.run_program_argsasarray("docker", args, throw_exception_if_exitcode_is_not_zero=False, print_live_output=self.sc.log.loglevel==LogLevel.Debug)
|
|
293
|
-
if result[0] != 0:
|
|
294
|
-
for line in GeneralUtilities.string_to_lines(result[1]):
|
|
295
|
-
self.sc.log.log(line, LogLevel.Information)
|
|
296
|
-
for line in GeneralUtilities.string_to_lines(result[2]):
|
|
297
|
-
self.sc.log.log(line, LogLevel.Error)
|
|
298
|
-
raise ValueError(f"Found unignored secret findings (exit code {result[0]}). See {os.path.join(self.repository, '.betterleaks.toml')} to ignore known false positives.")
|
|
299
|
-
|
|
300
|
-
@GeneralUtilities.check_arguments
|
|
301
|
-
def use_cache(self) -> bool:
|
|
302
|
-
return self.__use_cache
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
@GeneralUtilities.check_arguments
|
|
306
|
-
def is_pre_merge(self) -> bool:
|
|
307
|
-
return self.__is_pre_merge
|
|
308
|
-
|
|
309
|
-
@GeneralUtilities.check_arguments
|
|
310
|
-
def update_dependencies(self) -> None:
|
|
311
|
-
repository=self.repository
|
|
312
|
-
self.sc.log.log("Update dependencies...")
|
|
313
|
-
self.update_year_in_license_file()
|
|
314
|
-
self.sc.assert_is_git_repository(repository)
|
|
315
|
-
self.sc.assert_no_uncommitted_changes(repository)
|
|
316
|
-
self.run_prepare_script()
|
|
317
|
-
if os.path.isfile(os.path.join(repository,"Other","Scripts","UpdateDependencies.py")):
|
|
318
|
-
self.sc.run_program(GeneralUtilities.get_python_executable(),"UpdateDependencies.py",os.path.join(repository,"Other","Scripts"))
|
|
319
|
-
codeunits:list[str]=self.tfcps_tools_general.get_codeunits(repository)
|
|
320
|
-
for codeunit_name in codeunits:
|
|
321
|
-
self.sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
|
|
322
|
-
codeunit_folder=os.path.join(repository,codeunit_name)
|
|
323
|
-
tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(codeunit_folder,self.sc.log.loglevel,"QualityCheck",None,True,False)
|
|
324
|
-
tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()#ensure requirements for updating are there (some programming-languages needs this)
|
|
325
|
-
if self.tfcps_tools_general.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder,f"{codeunit_name}.codeunit.xml")):
|
|
326
|
-
self.sc.run_program(GeneralUtilities.get_python_executable(),"UpdateDependencies.py",os.path.join(codeunit_folder,"Other"))
|
|
327
|
-
tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()#check if codeunit is still buildable
|
|
328
|
-
if self.sc.git_repository_has_uncommitted_changes(repository):
|
|
329
|
-
changelog_folder = os.path.join(repository, "Other", "Resources", "Changelog")
|
|
330
|
-
project_version:str=self.tfcps_tools_general.get_version_of_project(repository)
|
|
331
|
-
changelog_file = os.path.join(changelog_folder, f"v{project_version}.md")
|
|
332
|
-
if not os.path.isfile(changelog_file):
|
|
333
|
-
self.__ensure_changelog_file_is_added(repository, project_version)
|
|
334
|
-
t=TFCPS_CodeUnit_BuildCodeUnits(repository,self.sc.log.loglevel,"QualityCheck",None,True,False,False)
|
|
335
|
-
t.build_codeunits()#check codeunits are buildable at all
|
|
336
|
-
self.sc.git_commit(repository, "Updated dependencies", stage_all_changes=True)
|
|
337
|
-
|
|
338
|
-
@GeneralUtilities.check_arguments
|
|
339
|
-
def __ensure_changelog_file_is_added(self, repository_folder: str, version_of_project: str):
|
|
340
|
-
changelog_file = os.path.join(repository_folder, "Other", "Resources", "Changelog", f"v{version_of_project}.md")
|
|
341
|
-
if not os.path.isfile(changelog_file):
|
|
342
|
-
GeneralUtilities.ensure_file_exists(changelog_file)
|
|
343
|
-
GeneralUtilities.write_text_to_file(changelog_file, """# Release notes
|
|
344
|
-
|
|
345
|
-
## Changes
|
|
346
|
-
|
|
347
|
-
- Updated dependencies.
|
|
348
|
-
""")
|
|
349
|
-
|
|
350
|
-
@GeneralUtilities.check_arguments
|
|
351
|
-
def update_year_in_license_file(self) -> None:
|
|
352
|
-
self.sc.update_year_in_first_line_of_file(os.path.join(self.repository, "License.txt"))
|
|
1
|
+
import os
|
|
2
|
+
import json
|
|
3
|
+
from datetime import datetime, timedelta,timezone
|
|
4
|
+
import xmlschema
|
|
5
|
+
import yaml
|
|
6
|
+
from ..GeneralUtilities import GeneralUtilities
|
|
7
|
+
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
8
|
+
from ..SCLog import LogLevel
|
|
9
|
+
from .TFCPS_CodeUnit_BuildCodeUnit import TFCPS_CodeUnit_BuildCodeUnit
|
|
10
|
+
from .TFCPS_Tools_General import TFCPS_Tools_General
|
|
11
|
+
|
|
12
|
+
class TFCPS_CodeUnit_BuildCodeUnits:
|
|
13
|
+
repository:str=None
|
|
14
|
+
tfcps_tools_general:TFCPS_Tools_General=None
|
|
15
|
+
sc:ScriptCollectionCore=None
|
|
16
|
+
target_environment_type:str=None
|
|
17
|
+
additionalargumentsfile:str=None
|
|
18
|
+
__use_cache:bool = None
|
|
19
|
+
__is_pre_merge:bool = None
|
|
20
|
+
__assert_no_new_changes:bool = None
|
|
21
|
+
|
|
22
|
+
def __init__(self,repository:str,loglevel:LogLevel,target_environment_type:str,additionalargumentsfile:str,use_cache:bool,is_pre_merge:bool,assertnonewchanges:bool):
|
|
23
|
+
self.sc=ScriptCollectionCore()
|
|
24
|
+
self.sc.log.loglevel=loglevel
|
|
25
|
+
self.__use_cache=use_cache
|
|
26
|
+
self.sc.assert_is_git_repository(repository)
|
|
27
|
+
self.repository=repository
|
|
28
|
+
self.tfcps_tools_general:TFCPS_Tools_General=TFCPS_Tools_General(self.sc)
|
|
29
|
+
allowed_target_environment_types=["Development","QualityCheck","Productive"]
|
|
30
|
+
GeneralUtilities.assert_condition(target_environment_type in allowed_target_environment_types,"Unknown target-environment-type. Allowed values are: "+", ".join(allowed_target_environment_types))
|
|
31
|
+
self.target_environment_type=target_environment_type
|
|
32
|
+
self.additionalargumentsfile=additionalargumentsfile
|
|
33
|
+
self.__is_pre_merge=is_pre_merge
|
|
34
|
+
self.__assert_no_new_changes=assertnonewchanges
|
|
35
|
+
|
|
36
|
+
@GeneralUtilities.check_arguments
|
|
37
|
+
def build_codeunits(self) -> None:
|
|
38
|
+
self.sc.log.log(GeneralUtilities.get_line())
|
|
39
|
+
start_time:datetime=GeneralUtilities.get_now()
|
|
40
|
+
self.sc.log.log(f"Start building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(start_time,False)}. (Target environment-type: {self.target_environment_type})")
|
|
41
|
+
if self.__assert_no_new_changes:
|
|
42
|
+
self.sc.assert_no_uncommitted_changes(self.repository,"Can not build codeunit: There are uncommitted changes in the repository.")
|
|
43
|
+
|
|
44
|
+
product_information_file = os.path.join(self.repository, ".ScriptCollection", "ProductInformation.xml")
|
|
45
|
+
try:
|
|
46
|
+
xmlschema.validate(product_information_file, "https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/raw/main/Conventions/RepositoryStructure/CommonProjectStructure/projectinformation.xsd")
|
|
47
|
+
except Exception as exception:
|
|
48
|
+
self.sc.log.log_exception(f"'{product_information_file}' could not be validated against the XSD:", exception, LogLevel.Warning)
|
|
49
|
+
|
|
50
|
+
#check if changelog exists
|
|
51
|
+
changelog_file=os.path.join(self.repository,"Other","Resources","Changelog",f"v{self.tfcps_tools_general.get_version_of_project(self.repository)}.md")
|
|
52
|
+
GeneralUtilities.assert_file_exists(changelog_file,f"Changelogfile \"{changelog_file}\" does not exist. Try to create it for example using \"sccreatechangelogentry -m ...\".")
|
|
53
|
+
|
|
54
|
+
#run prepare-script
|
|
55
|
+
self.run_prepare_script()
|
|
56
|
+
|
|
57
|
+
#mark current version as supported
|
|
58
|
+
now = GeneralUtilities.get_now()
|
|
59
|
+
project_version:str=self.tfcps_tools_general.get_version_of_project(self.repository)
|
|
60
|
+
if not self.tfcps_tools_general.suport_information_exists(self.repository, project_version):
|
|
61
|
+
amount_of_years_for_support:int=1
|
|
62
|
+
support_time = timedelta(days=365*amount_of_years_for_support+30*3+1)
|
|
63
|
+
until = now + support_time
|
|
64
|
+
until_day = datetime(until.year, until.month, until.day, 0, 0, 0)
|
|
65
|
+
from_day = datetime(now.year, now.month, now.day, 0, 0, 0)
|
|
66
|
+
self.tfcps_tools_general.mark_current_version_as_supported(self.repository,project_version,from_day,until_day)
|
|
67
|
+
|
|
68
|
+
codeunits:list[str]=self.tfcps_tools_general.get_codeunits(self.repository)
|
|
69
|
+
GeneralUtilities.assert_condition(0<len(codeunits),f"No codeunits found in repository {self.repository}.")
|
|
70
|
+
self.sc.log.log("Codeunits will be built in the following order:")
|
|
71
|
+
for codeunit_name in codeunits:
|
|
72
|
+
self.sc.log.log(f" - {codeunit_name}")
|
|
73
|
+
for codeunit_name in codeunits:
|
|
74
|
+
tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(os.path.join(self.repository,codeunit_name),self.sc.log.loglevel,self.target_environment_type,self.additionalargumentsfile,self.use_cache(),self.is_pre_merge())
|
|
75
|
+
self.sc.log.log(GeneralUtilities.get_line())
|
|
76
|
+
tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()
|
|
77
|
+
|
|
78
|
+
self.sc.log.log(GeneralUtilities.get_line())
|
|
79
|
+
self.__search_for_vulnerabilities()
|
|
80
|
+
self.search_for_secrets()
|
|
81
|
+
if self.is_pre_merge():
|
|
82
|
+
self.__translate()
|
|
83
|
+
self.__collect_metrics()
|
|
84
|
+
self.__generate_loc_diagram()
|
|
85
|
+
|
|
86
|
+
if self.__assert_no_new_changes:
|
|
87
|
+
self.sc.assert_no_uncommitted_changes(self.repository,"There are new uncommitted changes in the repository.")
|
|
88
|
+
|
|
89
|
+
end_time:datetime=GeneralUtilities.get_now()
|
|
90
|
+
duration=end_time-start_time
|
|
91
|
+
self.sc.log.log(f"Finished building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(end_time,False)}. (Duration: {GeneralUtilities.timedelta_to_simple_string(duration)})")
|
|
92
|
+
self.sc.log.log(GeneralUtilities.get_line())
|
|
93
|
+
|
|
94
|
+
@GeneralUtilities.check_arguments
|
|
95
|
+
def run_prepare_script(self):
|
|
96
|
+
args=["--repository",self.repository,"--targetenvironmenttype",self.target_environment_type,"--verbosity",str(int(self.sc.log.loglevel))]
|
|
97
|
+
if GeneralUtilities.string_has_content(self.additionalargumentsfile):
|
|
98
|
+
args=args+["--additionalargumentsfile", self.additionalargumentsfile]
|
|
99
|
+
if not self.__use_cache:
|
|
100
|
+
if self.sc.git_repository_has_uncommitted_changes(self.repository):
|
|
101
|
+
self.sc.log.log("No-cache-option can not be applied because there are uncommited changes in the repository.",LogLevel.Warning)
|
|
102
|
+
else:
|
|
103
|
+
args=args+["--nocache"]
|
|
104
|
+
|
|
105
|
+
if os.path.isfile( os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS","CustomPreCodeUnitBuildScript.py")):
|
|
106
|
+
self.sc.log.log("Run custom pre-codeunitbuild script...")
|
|
107
|
+
self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["CustomPreCodeUnitBuildScript.py"]+args, os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS"),print_live_output=True)
|
|
108
|
+
|
|
109
|
+
if os.path.isfile( os.path.join(self.repository,"Other","Scripts","PrepareBuildCodeunits.py")):
|
|
110
|
+
self.sc.log.log("Prepare build codeunits...")
|
|
111
|
+
self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["PrepareBuildCodeunits.py"]+args, os.path.join(self.repository,"Other","Scripts"),print_live_output=True)
|
|
112
|
+
|
|
113
|
+
@GeneralUtilities.check_arguments
|
|
114
|
+
def build_codeunits_in_container(self) -> None:
|
|
115
|
+
container_repository_folder = "/Workspace/Repository"
|
|
116
|
+
image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "SCBuilder")
|
|
117
|
+
|
|
118
|
+
#build the scbuildcodeunits-arguments based on the current state (analogous to the arguments accepted by the scbuildcodeunits-executable)
|
|
119
|
+
scbuildcodeunits_arguments = ["-r", container_repository_folder, "-v", str(int(self.sc.log.loglevel)), "-e", self.target_environment_type]
|
|
120
|
+
if not self.__use_cache:
|
|
121
|
+
scbuildcodeunits_arguments.append("-c")
|
|
122
|
+
if self.__is_pre_merge:
|
|
123
|
+
scbuildcodeunits_arguments.append("-p")
|
|
124
|
+
if self.__assert_no_new_changes:
|
|
125
|
+
scbuildcodeunits_arguments.append("-u")
|
|
126
|
+
if GeneralUtilities.string_has_content(self.additionalargumentsfile):
|
|
127
|
+
scbuildcodeunits_arguments = scbuildcodeunits_arguments + ["-a", self.__translate_path_into_container(self.additionalargumentsfile, container_repository_folder)]
|
|
128
|
+
|
|
129
|
+
#run scbuildcodeunits inside the SCBuilder-image. the repository is mounted into the container and the docker-socket is forwarded because codeunit-builds often start containers (for example local test-services).
|
|
130
|
+
docker_arguments = [
|
|
131
|
+
"run", "--rm",
|
|
132
|
+
"-v", f"{self.repository}:{container_repository_folder}",
|
|
133
|
+
"-v", "/var/run/docker.sock:/var/run/docker.sock",
|
|
134
|
+
"-w", container_repository_folder,
|
|
135
|
+
image,
|
|
136
|
+
"scbuildcodeunits",
|
|
137
|
+
] + scbuildcodeunits_arguments
|
|
138
|
+
self.sc.log.log(f"Build codeunits in container using image \"{image}\"...")
|
|
139
|
+
self.sc.run_program_argsasarray("docker", docker_arguments, print_live_output=True)
|
|
140
|
+
|
|
141
|
+
@GeneralUtilities.check_arguments
|
|
142
|
+
def __translate_path_into_container(self, host_path: str, container_repository_folder: str) -> str:
|
|
143
|
+
normalized_repository = os.path.normpath(self.repository)
|
|
144
|
+
normalized_path = os.path.normpath(host_path)
|
|
145
|
+
if normalized_path.startswith(normalized_repository):
|
|
146
|
+
relative_path = os.path.relpath(normalized_path, normalized_repository).replace(os.sep, "/")
|
|
147
|
+
return f"{container_repository_folder}/{relative_path}"
|
|
148
|
+
return host_path
|
|
149
|
+
|
|
150
|
+
@GeneralUtilities.check_arguments
|
|
151
|
+
def __translate(self) -> None:
|
|
152
|
+
for taskfile_name in ("Taskfile.yml", "Taskfile.yaml"):
|
|
153
|
+
taskfile = os.path.join(self.repository, taskfile_name)
|
|
154
|
+
if os.path.isfile(taskfile):
|
|
155
|
+
with open(taskfile, "r", encoding="utf-8") as f:
|
|
156
|
+
taskfile_content = yaml.safe_load(f)
|
|
157
|
+
if isinstance(taskfile_content.get("tasks"), dict) and "Translate" in taskfile_content["tasks"]:
|
|
158
|
+
self.sc.run_program("task", "Translate", self.repository, print_live_output=self.sc.log.loglevel == LogLevel.Debug)
|
|
159
|
+
break
|
|
160
|
+
|
|
161
|
+
@GeneralUtilities.check_arguments
|
|
162
|
+
def __collect_metrics(self) -> None:
|
|
163
|
+
project_version: str=self.tfcps_tools_general.get_version_of_project(self.repository)
|
|
164
|
+
self.sc.log.log("Collect metrics...")
|
|
165
|
+
loc = self.sc.get_lines_of_code_with_default_excluded_patterns(self.repository)
|
|
166
|
+
loc_metric_folder = os.path.join(self.repository, "Other", "Metrics")
|
|
167
|
+
GeneralUtilities.ensure_directory_exists(loc_metric_folder)
|
|
168
|
+
loc_metric_file = os.path.join(loc_metric_folder, "RepositoryStatisticsPerCommit.csv")
|
|
169
|
+
GeneralUtilities.ensure_file_exists(loc_metric_file)
|
|
170
|
+
|
|
171
|
+
#remove legacy metrics-file. the following 2 lines should be removed after 2026-12-31
|
|
172
|
+
legacy_metrics_file = os.path.join(loc_metric_folder, "LinesOfCode.csv")
|
|
173
|
+
GeneralUtilities.ensure_file_does_not_exist(legacy_metrics_file)
|
|
174
|
+
|
|
175
|
+
old_lines = GeneralUtilities.read_nonempty_lines_from_file(loc_metric_file)
|
|
176
|
+
header_line="Version;Timestamp;LinesOfCode"
|
|
177
|
+
new_lines = [header_line]
|
|
178
|
+
current_version_string=f"v{project_version}"
|
|
179
|
+
for old_line in old_lines:
|
|
180
|
+
if not old_line.startswith(current_version_string+";") and old_line!=header_line:
|
|
181
|
+
new_lines.append(old_line)
|
|
182
|
+
c_date:datetime=GeneralUtilities.get_now().astimezone(timezone.utc)
|
|
183
|
+
commit_date=GeneralUtilities.datetime_to_string_for_logfile_entry(c_date,False)
|
|
184
|
+
new_lines.append(f"{current_version_string};{commit_date};{loc}")
|
|
185
|
+
GeneralUtilities.write_lines_to_file(loc_metric_file, new_lines)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@GeneralUtilities.check_arguments
|
|
189
|
+
def __generate_loc_diagram(self):
|
|
190
|
+
self.sc.log.log("Generate LoC-diagram...")
|
|
191
|
+
loc_metric_folder = os.path.join(self.repository, "Other", "Metrics")
|
|
192
|
+
GeneralUtilities.ensure_directory_exists(loc_metric_folder)
|
|
193
|
+
loc_metric_file = os.path.join(loc_metric_folder, "RepositoryStatisticsPerCommit.csv")
|
|
194
|
+
GeneralUtilities.ensure_file_exists(loc_metric_file)
|
|
195
|
+
|
|
196
|
+
filenamebase="LoC-Diagram"
|
|
197
|
+
|
|
198
|
+
diagram_definition_folder=os.path.join(self.repository, "Other", "Reference","Technical","Diagrams")
|
|
199
|
+
GeneralUtilities.ensure_directory_exists(diagram_definition_folder)
|
|
200
|
+
|
|
201
|
+
diagram_definition_file=os.path.join(diagram_definition_folder,f"{filenamebase}.json")
|
|
202
|
+
GeneralUtilities.ensure_file_exists(diagram_definition_file)
|
|
203
|
+
GeneralUtilities.write_text_to_file(diagram_definition_file,GeneralUtilities.empty_string)
|
|
204
|
+
|
|
205
|
+
loc_data_file=os.path.join(diagram_definition_folder,f"{filenamebase}.csv")
|
|
206
|
+
GeneralUtilities.ensure_file_exists(loc_data_file)
|
|
207
|
+
csv_lines=[]
|
|
208
|
+
for line in GeneralUtilities.read_lines_from_file(loc_metric_file):
|
|
209
|
+
if GeneralUtilities.string_has_content(line):
|
|
210
|
+
splitted=line.split(";")
|
|
211
|
+
v=splitted[0]
|
|
212
|
+
t=splitted[1]
|
|
213
|
+
loc=splitted[2]
|
|
214
|
+
csv_lines.append(f"{v},{t},{loc}")
|
|
215
|
+
GeneralUtilities.write_lines_to_file(loc_data_file,csv_lines)
|
|
216
|
+
diagram_json = {
|
|
217
|
+
"$schema": "https://vega.github.io/schema/vega-lite/v5.json",
|
|
218
|
+
"description": "Lines of Code over time",
|
|
219
|
+
"width": 800,
|
|
220
|
+
"height": 400,
|
|
221
|
+
"data": {
|
|
222
|
+
"url": f"./{filenamebase}.csv",
|
|
223
|
+
"format": {
|
|
224
|
+
"type": "csv"
|
|
225
|
+
}
|
|
226
|
+
},
|
|
227
|
+
"mark": {
|
|
228
|
+
"type": "line",
|
|
229
|
+
"point": True
|
|
230
|
+
},
|
|
231
|
+
"encoding": {
|
|
232
|
+
"x": {
|
|
233
|
+
"field": "Timestamp",
|
|
234
|
+
"type": "temporal",
|
|
235
|
+
"title": "Date"
|
|
236
|
+
},
|
|
237
|
+
"y": {
|
|
238
|
+
"field": "LinesOfCode",
|
|
239
|
+
"type": "quantitative",
|
|
240
|
+
"title": "Lines of Code"
|
|
241
|
+
},
|
|
242
|
+
"tooltip": [
|
|
243
|
+
{
|
|
244
|
+
"field": "Version",
|
|
245
|
+
"type": "ordinal"
|
|
246
|
+
},
|
|
247
|
+
{
|
|
248
|
+
"field": "LinesOfCode",
|
|
249
|
+
"type": "quantitative"
|
|
250
|
+
},
|
|
251
|
+
{
|
|
252
|
+
"field": "Timestamp",
|
|
253
|
+
"type": "temporal"
|
|
254
|
+
}
|
|
255
|
+
]
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
with open(diagram_definition_file, "w", encoding="utf-8") as f:
|
|
260
|
+
json.dump(
|
|
261
|
+
diagram_json,
|
|
262
|
+
f,
|
|
263
|
+
indent=2,
|
|
264
|
+
sort_keys=False,
|
|
265
|
+
ensure_ascii=False
|
|
266
|
+
)
|
|
267
|
+
diagram_svg_file=os.path.join(self.repository,"Other","Reference","Technical","Diagrams",f"{filenamebase}.svg")
|
|
268
|
+
GeneralUtilities.ensure_file_exists(diagram_svg_file)
|
|
269
|
+
GeneralUtilities.assert_condition(not self.sc.file_is_git_ignored(f"Other/Reference/Technical/Diagrams/{filenamebase}.svg",self.repository),f"Other/Reference/Technical/Diagrams/{filenamebase}.svg must not be git-ignored")#because it should be referencable in markdown-files and viewable without building the codeunits.
|
|
270
|
+
self.sc.generate_chart_diagram(diagram_definition_file,os.path.basename(diagram_svg_file))
|
|
271
|
+
self.sc.format_xml_file(diagram_svg_file)
|
|
272
|
+
|
|
273
|
+
@GeneralUtilities.check_arguments
|
|
274
|
+
def __search_for_vulnerabilities(self):
|
|
275
|
+
pass#TODO
|
|
276
|
+
|
|
277
|
+
@GeneralUtilities.check_arguments
|
|
278
|
+
def search_for_secrets(self) -> None:
|
|
279
|
+
self.sc.log.log("Search for secrets...")
|
|
280
|
+
try:
|
|
281
|
+
image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "Betterleaks")
|
|
282
|
+
except Exception:
|
|
283
|
+
image="ghcr.io/betterleaks/betterleaks:latest"
|
|
284
|
+
if os.path.isdir(os.path.join(self.repository, ".git")):
|
|
285
|
+
scan_args = ["git", "/repo"]
|
|
286
|
+
if os.path.isfile(os.path.join(self.repository, ".git")):
|
|
287
|
+
self.sc.log.log("Repository is a Git submodule (.git is a file); scanning working tree only (no history).", LogLevel.Warning)
|
|
288
|
+
scan_args = ["dir", "/repo"] #usually the git-scan should be used here too but this does not work in a submodule yet, see https://github.com/betterleaks/betterleaks/issues/170
|
|
289
|
+
else:
|
|
290
|
+
scan_args = ["dir", "/repo"]
|
|
291
|
+
args = ["run", "--rm", "-v", f"{self.repository}:/repo", image] + scan_args
|
|
292
|
+
result = self.sc.run_program_argsasarray("docker", args, throw_exception_if_exitcode_is_not_zero=False, print_live_output=self.sc.log.loglevel==LogLevel.Debug)
|
|
293
|
+
if result[0] != 0:
|
|
294
|
+
for line in GeneralUtilities.string_to_lines(result[1]):
|
|
295
|
+
self.sc.log.log(line, LogLevel.Information)
|
|
296
|
+
for line in GeneralUtilities.string_to_lines(result[2]):
|
|
297
|
+
self.sc.log.log(line, LogLevel.Error)
|
|
298
|
+
raise ValueError(f"Found unignored secret findings (exit code {result[0]}). See {os.path.join(self.repository, '.betterleaks.toml')} to ignore known false positives.")
|
|
299
|
+
|
|
300
|
+
@GeneralUtilities.check_arguments
|
|
301
|
+
def use_cache(self) -> bool:
|
|
302
|
+
return self.__use_cache
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
@GeneralUtilities.check_arguments
|
|
306
|
+
def is_pre_merge(self) -> bool:
|
|
307
|
+
return self.__is_pre_merge
|
|
308
|
+
|
|
309
|
+
@GeneralUtilities.check_arguments
|
|
310
|
+
def update_dependencies(self) -> None:
|
|
311
|
+
repository=self.repository
|
|
312
|
+
self.sc.log.log("Update dependencies...")
|
|
313
|
+
self.update_year_in_license_file()
|
|
314
|
+
self.sc.assert_is_git_repository(repository)
|
|
315
|
+
self.sc.assert_no_uncommitted_changes(repository)
|
|
316
|
+
self.run_prepare_script()
|
|
317
|
+
if os.path.isfile(os.path.join(repository,"Other","Scripts","UpdateDependencies.py")):
|
|
318
|
+
self.sc.run_program(GeneralUtilities.get_python_executable(),"UpdateDependencies.py",os.path.join(repository,"Other","Scripts"))
|
|
319
|
+
codeunits:list[str]=self.tfcps_tools_general.get_codeunits(repository)
|
|
320
|
+
for codeunit_name in codeunits:
|
|
321
|
+
self.sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
|
|
322
|
+
codeunit_folder=os.path.join(repository,codeunit_name)
|
|
323
|
+
tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(codeunit_folder,self.sc.log.loglevel,"QualityCheck",None,True,False)
|
|
324
|
+
tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()#ensure requirements for updating are there (some programming-languages needs this)
|
|
325
|
+
if self.tfcps_tools_general.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder,f"{codeunit_name}.codeunit.xml")):
|
|
326
|
+
self.sc.run_program(GeneralUtilities.get_python_executable(),"UpdateDependencies.py",os.path.join(codeunit_folder,"Other"))
|
|
327
|
+
tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()#check if codeunit is still buildable
|
|
328
|
+
if self.sc.git_repository_has_uncommitted_changes(repository):
|
|
329
|
+
changelog_folder = os.path.join(repository, "Other", "Resources", "Changelog")
|
|
330
|
+
project_version:str=self.tfcps_tools_general.get_version_of_project(repository)
|
|
331
|
+
changelog_file = os.path.join(changelog_folder, f"v{project_version}.md")
|
|
332
|
+
if not os.path.isfile(changelog_file):
|
|
333
|
+
self.__ensure_changelog_file_is_added(repository, project_version)
|
|
334
|
+
t=TFCPS_CodeUnit_BuildCodeUnits(repository,self.sc.log.loglevel,"QualityCheck",None,True,False,False)
|
|
335
|
+
t.build_codeunits()#check codeunits are buildable at all
|
|
336
|
+
self.sc.git_commit(repository, "Updated dependencies", stage_all_changes=True)
|
|
337
|
+
|
|
338
|
+
@GeneralUtilities.check_arguments
|
|
339
|
+
def __ensure_changelog_file_is_added(self, repository_folder: str, version_of_project: str):
|
|
340
|
+
changelog_file = os.path.join(repository_folder, "Other", "Resources", "Changelog", f"v{version_of_project}.md")
|
|
341
|
+
if not os.path.isfile(changelog_file):
|
|
342
|
+
GeneralUtilities.ensure_file_exists(changelog_file)
|
|
343
|
+
GeneralUtilities.write_text_to_file(changelog_file, """# Release notes
|
|
344
|
+
|
|
345
|
+
## Changes
|
|
346
|
+
|
|
347
|
+
- Updated dependencies.
|
|
348
|
+
""")
|
|
349
|
+
|
|
350
|
+
@GeneralUtilities.check_arguments
|
|
351
|
+
def update_year_in_license_file(self) -> None:
|
|
352
|
+
self.sc.update_year_in_first_line_of_file(os.path.join(self.repository, "License.txt"))
|