ScriptCollection 4.3.63__py3-none-any.whl → 4.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/AnionBuildPlatform.py +205 -205
- ScriptCollection/CertificateUpdater.py +157 -155
- ScriptCollection/Executables.py +4 -2
- ScriptCollection/GeneralUtilities.py +1696 -1696
- ScriptCollection/OCIImages/OCIImageManager.py +278 -278
- ScriptCollection/ProcessesRunner.py +43 -43
- ScriptCollection/ProgramRunnerSudo.py +110 -110
- ScriptCollection/Resources/CultureChooser/CultureChooser.js +0 -0
- ScriptCollection/Resources/CultureChooser/index.html +0 -0
- ScriptCollection/Resources/Dependencies/JRE/Version.txt +0 -0
- ScriptCollection/Resources/Dependencies/OpenAPIGenerator/Version.txt +0 -0
- ScriptCollection/Resources/Dependencies/PlantUML/Version.txt +0 -0
- ScriptCollection/Resources/Fonts/DejaVuSans-Bold.ttf +0 -0
- ScriptCollection/Resources/Fonts/DejaVuSans-BoldOblique.ttf +0 -0
- ScriptCollection/Resources/Fonts/DejaVuSans-Oblique.ttf +0 -0
- ScriptCollection/Resources/Fonts/DejaVuSans.ttf +0 -0
- ScriptCollection/Resources/Fonts/LICENSE.txt +0 -0
- ScriptCollection/Resources/MaintenanceSite/MaintenanceSite.html +0 -0
- ScriptCollection/SCLog.py +115 -115
- ScriptCollection/ScriptCollectionCore.py +4372 -4292
- ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +8 -2
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +716 -709
- ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +235 -235
- ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +258 -258
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +524 -524
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +157 -155
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +691 -672
- ScriptCollection/TFCPS/TFCPS_CreateRelease.py +98 -98
- ScriptCollection/TFCPS/TFCPS_Generic.py +44 -44
- ScriptCollection/TFCPS/TFCPS_MergeToMain.py +132 -131
- ScriptCollection/TFCPS/TFCPS_MergeToStable.py +358 -358
- ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +48 -48
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +38 -8
- {scriptcollection-4.3.63.dist-info → scriptcollection-4.4.0.dist-info}/METADATA +1 -1
- {scriptcollection-4.3.63.dist-info → scriptcollection-4.4.0.dist-info}/RECORD +27 -27
- {scriptcollection-4.3.63.dist-info → scriptcollection-4.4.0.dist-info}/WHEEL +0 -0
- {scriptcollection-4.3.63.dist-info → scriptcollection-4.4.0.dist-info}/entry_points.txt +0 -0
- {scriptcollection-4.3.63.dist-info → scriptcollection-4.4.0.dist-info}/top_level.txt +0 -0
|
@@ -1,524 +1,524 @@
|
|
|
1
|
-
import os
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
import shutil
|
|
4
|
-
import re
|
|
5
|
-
import json
|
|
6
|
-
import argparse
|
|
7
|
-
from abc import ABC, abstractmethod
|
|
8
|
-
import yaml
|
|
9
|
-
import xmlschema
|
|
10
|
-
from packaging.version import Version
|
|
11
|
-
from lxml import etree
|
|
12
|
-
from ..GeneralUtilities import GeneralUtilities, VersionEcholon
|
|
13
|
-
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
14
|
-
from ..SCLog import LogLevel
|
|
15
|
-
from .TFCPS_Tools_General import TFCPS_Tools_General
|
|
16
|
-
from .TFCPS_RemoteBuild import TFCPS_RemoteBuild, RunnerOperatingSystem
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
class TFCPS_CodeUnitSpecific_Base(ABC):
|
|
21
|
-
|
|
22
|
-
__current_file:str=None
|
|
23
|
-
__target_environment_type:str = None
|
|
24
|
-
__repository_folder:str=None
|
|
25
|
-
__codeunit_folder:str=None
|
|
26
|
-
__current_folder:str=None
|
|
27
|
-
__verbosity:LogLevel=None
|
|
28
|
-
__use_cache:bool=None
|
|
29
|
-
tfcps_Tools_General:TFCPS_Tools_General = None
|
|
30
|
-
_protected_sc:ScriptCollectionCore = None
|
|
31
|
-
__is_pre_merge:bool=False
|
|
32
|
-
__validate_developers_of_repository:bool=True#TODO must be setable to false
|
|
33
|
-
|
|
34
|
-
def __init__(self,current_file:str,verbosity:LogLevel,target_envionment_type:str,use_cache:bool,is_pre_merge:bool):
|
|
35
|
-
self.__verbosity=verbosity
|
|
36
|
-
self.__use_cache=use_cache
|
|
37
|
-
self.__target_environment_type=target_envionment_type
|
|
38
|
-
self.__current_file = str(Path(current_file).absolute())
|
|
39
|
-
self.__current_folder = os.path.dirname(self.__current_file)
|
|
40
|
-
self.__codeunit_folder=self.__search_codeunit_folder()
|
|
41
|
-
self.__is_pre_merge=is_pre_merge
|
|
42
|
-
self._protected_sc=ScriptCollectionCore()
|
|
43
|
-
self._protected_sc.log.loglevel=self.__verbosity
|
|
44
|
-
self.tfcps_Tools_General=TFCPS_Tools_General(self._protected_sc)
|
|
45
|
-
self.tfcps_Tools_General.assert_is_codeunit_folder(self.__codeunit_folder)
|
|
46
|
-
self.__repository_folder=GeneralUtilities.resolve_relative_path("..",self.__codeunit_folder)
|
|
47
|
-
self._protected_sc.assert_is_git_repository(self.__repository_folder)
|
|
48
|
-
#ensure the environment-variables which the product declares as required are also available when this codeunit-script is started
|
|
49
|
-
#directly (for example "<codeunit>/Other/Build/Build.py" or "task bb") and not by "scbuildcodeunits".
|
|
50
|
-
self.tfcps_Tools_General.ensure_required_environment_variables_are_set(self.__repository_folder)
|
|
51
|
-
|
|
52
|
-
def __search_codeunit_folder(self)->str:
|
|
53
|
-
current_path:str=os.path.dirname(self.__current_file)
|
|
54
|
-
enabled:bool=True
|
|
55
|
-
while enabled:
|
|
56
|
-
try:
|
|
57
|
-
current_path=GeneralUtilities.resolve_relative_path("..",current_path)
|
|
58
|
-
foldername=os.path.basename(current_path)
|
|
59
|
-
codeunit_file:str=os.path.join(current_path,f"{foldername}.codeunit.xml")
|
|
60
|
-
if os.path.isfile(codeunit_file):
|
|
61
|
-
return current_path
|
|
62
|
-
except:
|
|
63
|
-
enabled=False
|
|
64
|
-
raise ValueError(f"Can not find codeunit-folder for folder \"{self.__current_file}\".")
|
|
65
|
-
|
|
66
|
-
@abstractmethod
|
|
67
|
-
def get_dependencies(self)->dict[str,set[str]]:
|
|
68
|
-
raise ValueError(f"Operation is abstract.")
|
|
69
|
-
|
|
70
|
-
@abstractmethod
|
|
71
|
-
def get_available_versions(self,dependencyname:str)->list[str]:
|
|
72
|
-
raise ValueError(f"Operation is abstract.")
|
|
73
|
-
|
|
74
|
-
@abstractmethod
|
|
75
|
-
def set_dependency_version(self,name:str,new_version:str)->None:
|
|
76
|
-
raise ValueError(f"Operation is abstract.")
|
|
77
|
-
|
|
78
|
-
def update_dependencies(self):
|
|
79
|
-
self.update_dependencies_with_specific_echolon(VersionEcholon.LatestPatchOrLatestMinor)
|
|
80
|
-
|
|
81
|
-
def update_dependencies_with_specific_echolon(self, echolon: VersionEcholon):
|
|
82
|
-
ignored_dependencies=self.tfcps_Tools_General.get_dependencies_which_are_ignored_from_updates(self.get_codeunit_folder())
|
|
83
|
-
for ignored_dependency in ignored_dependencies:
|
|
84
|
-
self._protected_sc.log.log(f"Codeunit {self.get_codeunit_name()} ignores the dependency {ignored_dependency} in update-checks.", LogLevel.Warning)
|
|
85
|
-
|
|
86
|
-
dependencies_dict:dict[str,set[str]]=self.get_dependencies()
|
|
87
|
-
for dependencyname,dependency_versions in dependencies_dict.items():
|
|
88
|
-
GeneralUtilities.assert_condition(0<len(dependency_versions),f"Dependency {dependencyname} is not used.")
|
|
89
|
-
GeneralUtilities.assert_condition(len(dependency_versions)<2,f"Dependency {dependencyname} is used {len(dependency_versions)} times. Please consolidate it to one version before updating.")
|
|
90
|
-
dependency_version=next(iter(dependency_versions))
|
|
91
|
-
latest_currently_used_version=dependency_version
|
|
92
|
-
if dependencyname not in ignored_dependencies:
|
|
93
|
-
try:
|
|
94
|
-
available_versions:list[str]=self.get_available_versions(dependencyname)
|
|
95
|
-
for available_version in available_versions:
|
|
96
|
-
GeneralUtilities.assert_condition(re.match(r"^(\d+).(\d+).(\d+)$", available_version) is not None,f"Invalid-version-string: {available_version}")
|
|
97
|
-
desired_version=GeneralUtilities.choose_version(available_versions,latest_currently_used_version,echolon)
|
|
98
|
-
GeneralUtilities.assert_condition(Version(dependency_version)<=Version(desired_version),f"Desired version {desired_version} for dependency {dependencyname} is less than the actual used version {latest_currently_used_version}.")
|
|
99
|
-
update_dependency:bool=desired_version!=latest_currently_used_version
|
|
100
|
-
if update_dependency:
|
|
101
|
-
if len(dependency_versions)==1:
|
|
102
|
-
GeneralUtilities.write_message_to_stdout("Update dependency "+dependencyname+" (which is currently used in version "+dependency_version+") to version "+desired_version+".")
|
|
103
|
-
self.set_dependency_version(dependencyname,desired_version)
|
|
104
|
-
except Exception:
|
|
105
|
-
GeneralUtilities.write_message_to_stderr(f"Error while updating {dependencyname}.")
|
|
106
|
-
raise
|
|
107
|
-
|
|
108
|
-
def get_version_of_project(self)->str:
|
|
109
|
-
return self.tfcps_Tools_General.get_version_of_project(self.get_repository_folder())
|
|
110
|
-
|
|
111
|
-
@GeneralUtilities.check_arguments
|
|
112
|
-
def do_common_tasks_base(self,current_codeunit_version:str):
|
|
113
|
-
repository_folder: str =self.get_repository_folder()
|
|
114
|
-
self._protected_sc.assert_is_git_repository(repository_folder)
|
|
115
|
-
codeunit_name: str = self.get_codeunit_name()
|
|
116
|
-
project_version = self.tfcps_Tools_General.get_version_of_project(repository_folder)
|
|
117
|
-
if current_codeunit_version is None:
|
|
118
|
-
current_codeunit_version=project_version
|
|
119
|
-
codeunit_folder = os.path.join(repository_folder, codeunit_name)
|
|
120
|
-
|
|
121
|
-
# check codeunit-conformity
|
|
122
|
-
# The codeunit-folder must contain exactly one codeunit-file and its filename (without the ".codeunit.xml"-suffix)
|
|
123
|
-
# must be equal to the folder-name. (The equality of the folder-name and the name inside the file is checked further below.)
|
|
124
|
-
if not os.path.isdir(codeunit_folder):
|
|
125
|
-
raise ValueError(f'Codeunit-folder "{codeunit_folder}" does not exist.')
|
|
126
|
-
codeunit_filename_suffix = ".codeunit.xml"
|
|
127
|
-
codeunit_files_in_folder = [entry for entry in os.listdir(codeunit_folder) if entry.endswith(codeunit_filename_suffix) and os.path.isfile(os.path.join(codeunit_folder, entry))]
|
|
128
|
-
GeneralUtilities.assert_condition(len(codeunit_files_in_folder) == 1, f'The codeunit-folder "{codeunit_folder}" must contain exactly one codeunit-file (matching "*{codeunit_filename_suffix}") but contains {len(codeunit_files_in_folder)}.')
|
|
129
|
-
codeunit_name_in_codeunit_filename = codeunit_files_in_folder[0][:-len(codeunit_filename_suffix)]
|
|
130
|
-
GeneralUtilities.assert_condition(codeunit_name_in_codeunit_filename == codeunit_name, f'The codeunit-file "{codeunit_files_in_folder[0]}" must be named "{codeunit_name}{codeunit_filename_suffix}" to match its folder-name ("{codeunit_name}").')
|
|
131
|
-
supported_codeunitspecificationversion = "2.9.4" # should always be the latest version of the ProjectTemplates-repository
|
|
132
|
-
codeunit_file = os.path.join(codeunit_folder, f"{codeunit_name}{codeunit_filename_suffix}")
|
|
133
|
-
if not os.path.isfile(codeunit_file):
|
|
134
|
-
raise ValueError(f'Codeunitfile "{codeunit_file}" does not exist.')
|
|
135
|
-
# TODO implement usage of self.reference_latest_version_of_xsd_when_generating_xml
|
|
136
|
-
namespaces = {'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure', 'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
|
|
137
|
-
root: etree._ElementTree = etree.parse(codeunit_file)
|
|
138
|
-
|
|
139
|
-
# check codeunit-spcecification-version
|
|
140
|
-
try:
|
|
141
|
-
codeunit_file_version = root.xpath('//cps:codeunit/@codeunitspecificationversion', namespaces=namespaces)[0]
|
|
142
|
-
if codeunit_file_version != supported_codeunitspecificationversion:
|
|
143
|
-
raise ValueError(f"ScriptCollection only supports processing codeunits with codeunit-specification-version={supported_codeunitspecificationversion}.")
|
|
144
|
-
schemaLocation = root.xpath('//cps:codeunit/@xsi:schemaLocation', namespaces=namespaces)[0]
|
|
145
|
-
xmlschema.validate(codeunit_file, schemaLocation)
|
|
146
|
-
# TODO check if the properties codeunithastestablesourcecode, codeunithasupdatabledependencies, throwexceptionifcodeunitfilecannotbevalidated, developmentState and description exist and the values are valid
|
|
147
|
-
except Exception as exception:
|
|
148
|
-
self._protected_sc.log.log_exception(f'Codeunitfile "{codeunit_file}" can not be validated due to the following exception:', exception,LogLevel.Warning)
|
|
149
|
-
|
|
150
|
-
# check codeunit-name
|
|
151
|
-
codeunit_name_in_codeunit_file = root.xpath('//cps:codeunit/cps:name/text()', namespaces=namespaces)[0]
|
|
152
|
-
if codeunit_name != codeunit_name_in_codeunit_file:
|
|
153
|
-
raise ValueError(f"The folder-name ('{codeunit_name}') is not equal to the codeunit-name ('{codeunit_name_in_codeunit_file}').")
|
|
154
|
-
|
|
155
|
-
# check owner-name
|
|
156
|
-
codeunit_ownername_in_codeunit_file = self.tfcps_Tools_General. get_codeunit_owner_name(self.get_codeunit_file())
|
|
157
|
-
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(codeunit_ownername_in_codeunit_file), "No valid name for codeunitowner given.")
|
|
158
|
-
|
|
159
|
-
# check owner-emailaddress
|
|
160
|
-
codeunit_owneremailaddress_in_codeunit_file = self.tfcps_Tools_General.get_codeunit_owner_emailaddress(self.get_codeunit_file())
|
|
161
|
-
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(codeunit_owneremailaddress_in_codeunit_file), "No valid email-address for codeunitowner given.")
|
|
162
|
-
|
|
163
|
-
# check development-state
|
|
164
|
-
developmentstate = root.xpath('//cps:properties/@developmentstate', namespaces=namespaces)[0]
|
|
165
|
-
developmentstate_active = "Active development"
|
|
166
|
-
developmentstate_maintenance = "Maintenance-updates only"
|
|
167
|
-
developmentstate_inactive = "Inactive"
|
|
168
|
-
GeneralUtilities.assert_condition(developmentstate in (developmentstate_active, developmentstate_maintenance, developmentstate_inactive), f"Invalid development-state. Must be '{developmentstate_active}' or '{developmentstate_maintenance}' or '{developmentstate_inactive}' but was '{developmentstate}'.")
|
|
169
|
-
|
|
170
|
-
# check for mandatory files
|
|
171
|
-
files = ["Other/Build/Build.py", "Other/QualityCheck/Linting.py", "Other/Reference/GenerateReference.py"]
|
|
172
|
-
if self.tfcps_Tools_General.codeunit_has_testable_sourcecode(self.get_codeunit_file()):
|
|
173
|
-
# TODO check if the testsettings-section appears in the codeunit-file
|
|
174
|
-
files.append("Other/QualityCheck/RunTestcases.py")
|
|
175
|
-
if self.tfcps_Tools_General.codeunit_has_updatable_dependencies(self.get_codeunit_file()):
|
|
176
|
-
# TODO check if the updatesettings-section appears in the codeunit-file
|
|
177
|
-
files.append("Other/UpdateDependencies.py")
|
|
178
|
-
for file in files:
|
|
179
|
-
combined_file = os.path.join(codeunit_folder, file)
|
|
180
|
-
if not os.path.isfile(combined_file):
|
|
181
|
-
raise ValueError(f'The mandatory file "{file}" does not exist in the codeunit-folder.')
|
|
182
|
-
|
|
183
|
-
if os.path.isfile(os.path.join(codeunit_folder, "requirements.txt")):
|
|
184
|
-
self.install_requirementstxt_for_codeunit()
|
|
185
|
-
|
|
186
|
-
# check developer
|
|
187
|
-
if self.__validate_developers_of_repository:
|
|
188
|
-
expected_authors: list[tuple[str, str]] = []
|
|
189
|
-
expected_authors_in_xml = root.xpath('//cps:codeunit/cps:developerteam/cps:developer', namespaces=namespaces)
|
|
190
|
-
for expected_author in expected_authors_in_xml:
|
|
191
|
-
author_name = expected_author.xpath('./cps:developername/text()', namespaces=namespaces)[0]
|
|
192
|
-
author_emailaddress = expected_author.xpath('./cps:developeremailaddress/text()', namespaces=namespaces)[0]
|
|
193
|
-
expected_authors.append((author_name, author_emailaddress))
|
|
194
|
-
actual_authors: list[tuple[str, str]] = self.tfcps_Tools_General.get_all_authors_and_committers_of_repository(repository_folder, codeunit_name)
|
|
195
|
-
# TODO refactor this check to only check commits which are behind this but which are not already on main
|
|
196
|
-
# TODO verify also if the commit is signed by a valid key of the author
|
|
197
|
-
for actual_author in actual_authors:
|
|
198
|
-
if not (actual_author) in expected_authors:
|
|
199
|
-
actual_author_formatted = f"{actual_author[0]} <{actual_author[1]}>"
|
|
200
|
-
raise ValueError(f'Author/Comitter "{actual_author_formatted}" is not in the codeunit-developer-team. If {actual_author} is a authorized developer for this codeunit you should consider defining this in the codeunit-file or adapting the name using a .mailmap-file (see https://git-scm.com/docs/gitmailmap). The developer-team-check can also be disabled using the property validate_developers_of_repository.')
|
|
201
|
-
|
|
202
|
-
dependent_codeunits = self.tfcps_Tools_General.get_dependent_code_units(codeunit_file)
|
|
203
|
-
for dependent_codeunit in dependent_codeunits:
|
|
204
|
-
if not self.tfcps_Tools_General.dependent_codeunit_exists(repository_folder, dependent_codeunit):
|
|
205
|
-
raise ValueError(f"Codeunit {codeunit_name} does have dependent codeunit {dependent_codeunit} which does not exist.")
|
|
206
|
-
|
|
207
|
-
# TODO implement cycle-check for dependent codeunits
|
|
208
|
-
|
|
209
|
-
artifacts_folder = os.path.join(codeunit_folder, "Other", "Artifacts")
|
|
210
|
-
GeneralUtilities.ensure_directory_does_not_exist(artifacts_folder)
|
|
211
|
-
|
|
212
|
-
# get artifacts from dependent codeunits
|
|
213
|
-
self.tfcps_Tools_General.copy_artifacts_from_dependent_code_units(repository_folder, codeunit_name)
|
|
214
|
-
|
|
215
|
-
# update codeunit-version
|
|
216
|
-
self.tfcps_Tools_General.write_version_to_codeunit_file(self.get_codeunit_file(), current_codeunit_version)
|
|
217
|
-
|
|
218
|
-
# set project version
|
|
219
|
-
package_json_file = os.path.join(repository_folder, "package.json") # TDOO move this to a general project-specific (and codeunit-independent-script)
|
|
220
|
-
if os.path.isfile(package_json_file):
|
|
221
|
-
package_json_data: str = None
|
|
222
|
-
with open(package_json_file, "r", encoding="utf-8") as f1:
|
|
223
|
-
package_json_data = json.load(f1)
|
|
224
|
-
package_json_data["version"] = project_version
|
|
225
|
-
with open(package_json_file, "w", encoding="utf-8") as f2:
|
|
226
|
-
json.dump(package_json_data, f2, indent=2)
|
|
227
|
-
GeneralUtilities.write_text_to_file(package_json_file, GeneralUtilities.read_text_from_file(package_json_file).replace("\r", ""))
|
|
228
|
-
|
|
229
|
-
# set default constants
|
|
230
|
-
self.tfcps_Tools_General.set_default_constants(os.path.join(codeunit_folder))
|
|
231
|
-
|
|
232
|
-
# Hints-file
|
|
233
|
-
hints_file = os.path.join(codeunit_folder, "Other", "Reference", "ReferenceContent", "Hints.md")
|
|
234
|
-
if not os.path.isfile(hints_file):
|
|
235
|
-
raise ValueError(f"Hints-file '{hints_file}' does not exist.")
|
|
236
|
-
|
|
237
|
-
# Copy license-file
|
|
238
|
-
self.tfcps_Tools_General.copy_licence_file(self.get_codeunit_folder())
|
|
239
|
-
|
|
240
|
-
# Propagate the repository-level dependency-version-pins (e.g. PlantUML, OpenAPIGenerator, JRE) into the codeunit
|
|
241
|
-
# so each codeunit carries its own copy of the versions that are defined once at repository-level.
|
|
242
|
-
repository_dependencies_folder = os.path.join(repository_folder, "Other", "Resources", "Dependencies")
|
|
243
|
-
if os.path.isdir(repository_dependencies_folder):
|
|
244
|
-
for dependency_folder in GeneralUtilities.get_direct_folders_of_folder(repository_dependencies_folder):
|
|
245
|
-
dependency_name = os.path.basename(dependency_folder)
|
|
246
|
-
source_version_file = os.path.join(dependency_folder, "Version.txt")
|
|
247
|
-
target_version_file = os.path.join(codeunit_folder, "Other", "Resources", "Dependencies", dependency_name, "Version.txt")
|
|
248
|
-
if os.path.isfile(target_version_file):# only update dependencies, do not create new ones automatically
|
|
249
|
-
GeneralUtilities.ensure_directory_exists(os.path.dirname(target_version_file))
|
|
250
|
-
shutil.copyfile(source_version_file, target_version_file)
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
@GeneralUtilities.check_arguments
|
|
254
|
-
def generate_reference_using_docfx(self, generate_class_reference:bool=False):
|
|
255
|
-
reference_folder =os.path.join( self.get_codeunit_folder(),"Other","Reference")
|
|
256
|
-
class_reference_folder:str=os.path.join(reference_folder,"ReferenceContent","API")
|
|
257
|
-
toc_md_file:str=os.path.join(class_reference_folder,"toc.md")
|
|
258
|
-
GeneralUtilities.ensure_file_does_not_exist(toc_md_file)#should not be available when docfx is runninng because then toc.html will be generated from toc.md but toc.html will be generated by docfx in the target-folder
|
|
259
|
-
|
|
260
|
-
self.tfcps_Tools_General.generate_svg_files_from_plantuml_files_for_codeunit(self.get_codeunit_folder(), self.use_cache())
|
|
261
|
-
generated_reference_folder = GeneralUtilities.resolve_relative_path("../Artifacts/Reference", reference_folder)
|
|
262
|
-
GeneralUtilities.ensure_directory_does_not_exist(generated_reference_folder)
|
|
263
|
-
GeneralUtilities.ensure_directory_exists(generated_reference_folder)
|
|
264
|
-
obj_folder = os.path.join(reference_folder, "obj")
|
|
265
|
-
GeneralUtilities.ensure_folder_exists_and_is_empty(obj_folder)
|
|
266
|
-
self._protected_sc.run_program("docfx", "-t default,templates/darkfx docfx.json", reference_folder)
|
|
267
|
-
GeneralUtilities.ensure_directory_does_not_exist(obj_folder)
|
|
268
|
-
|
|
269
|
-
if generate_class_reference:# this codeunit has a class-reference, so a human-readable table-of-content can be generated for it
|
|
270
|
-
GeneralUtilities.ensure_file_exists(toc_md_file)
|
|
271
|
-
GeneralUtilities.write_text_to_file(toc_md_file, self.__generate_toc_md_file_content(os.path.join(class_reference_folder,"toc.yml")))
|
|
272
|
-
|
|
273
|
-
@staticmethod
|
|
274
|
-
@GeneralUtilities.check_arguments
|
|
275
|
-
def __generate_toc_md_file_content(toc_file:str)->str:
|
|
276
|
-
"""Generates the content of a markdown-table-of-content which contains a headline and links to the sub-files of the class-reference
|
|
277
|
-
based on the given toc.yml-file which was generated by docfx."""
|
|
278
|
-
toc_file_content = yaml.safe_load(GeneralUtilities.read_text_from_file(toc_file))
|
|
279
|
-
namespaces:list = list()
|
|
280
|
-
if isinstance(toc_file_content, dict):
|
|
281
|
-
namespaces = toc_file_content.get("items", list())
|
|
282
|
-
elif isinstance(toc_file_content, list):# older docfx-versions do not wrap the entries in an "items"-property
|
|
283
|
-
namespaces = toc_file_content
|
|
284
|
-
lines:list[str] = ["# Table of contents"]
|
|
285
|
-
for namespace in namespaces:
|
|
286
|
-
lines.append(GeneralUtilities.empty_string)
|
|
287
|
-
lines.append(f"## {TFCPS_CodeUnitSpecific_Base.__get_toc_entry_name(namespace)}")
|
|
288
|
-
lines.append(GeneralUtilities.empty_string)
|
|
289
|
-
entries:list = namespace.get("items", list())
|
|
290
|
-
if len(entries) == 0:
|
|
291
|
-
lines.append("(This namespace does not contain any documented type.)")
|
|
292
|
-
else:
|
|
293
|
-
for entry in entries:
|
|
294
|
-
lines.append(f"- [{TFCPS_CodeUnitSpecific_Base.__get_toc_entry_name(entry)}]({TFCPS_CodeUnitSpecific_Base.__get_toc_entry_link(entry)})")
|
|
295
|
-
return "\n".join(lines)+"\n"
|
|
296
|
-
|
|
297
|
-
@staticmethod
|
|
298
|
-
@GeneralUtilities.check_arguments
|
|
299
|
-
def __get_toc_entry_name(entry:dict)->str:
|
|
300
|
-
name:str = entry.get("name", entry.get("uid", GeneralUtilities.empty_string))
|
|
301
|
-
return name.replace("&", "&").replace("<", "<").replace(">", ">")# generic types like "List<T>" would be treated as html-tag otherwise
|
|
302
|
-
|
|
303
|
-
@staticmethod
|
|
304
|
-
@GeneralUtilities.check_arguments
|
|
305
|
-
def __get_toc_entry_link(entry:dict)->str:
|
|
306
|
-
uid:str = entry["uid"]
|
|
307
|
-
return "./"+uid.replace("`", "-")+".yml"# docfx replaces the backtick of generic types by a hyphen when generating the filenames
|
|
308
|
-
|
|
309
|
-
@GeneralUtilities.check_arguments
|
|
310
|
-
def use_cache(self)->bool:
|
|
311
|
-
return self.__use_cache
|
|
312
|
-
|
|
313
|
-
@GeneralUtilities.check_arguments
|
|
314
|
-
def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str) -> None:
|
|
315
|
-
"""Runs the given program for this codeunit on a remote task-runner that provides the given operating-system (see
|
|
316
|
-
TFCPS_RemoteBuild and the SCTaskRunnerWindows/SCTaskRunnerMacOS-codeunits). This is used for operating-system-bound
|
|
317
|
-
build-steps (e.g. flutter-windows- or flutter-ios-builds) which can not run on the current operating-system. After
|
|
318
|
-
the runner finished, only the codeunit-folder is mirrored back into the local repository, so it looks as if the
|
|
319
|
-
runner had built locally. working_directory must be a folder inside this codeunit."""
|
|
320
|
-
TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory)
|
|
321
|
-
|
|
322
|
-
@GeneralUtilities.check_arguments
|
|
323
|
-
def get_codeunit_folder(self)->str:
|
|
324
|
-
return self.__codeunit_folder
|
|
325
|
-
|
|
326
|
-
@GeneralUtilities.check_arguments
|
|
327
|
-
def get_codeunit_name(self)->str:
|
|
328
|
-
return os.path.basename(self.__codeunit_folder)
|
|
329
|
-
|
|
330
|
-
@GeneralUtilities.check_arguments
|
|
331
|
-
def get_repository_folder(self)->str:
|
|
332
|
-
return self.__repository_folder
|
|
333
|
-
|
|
334
|
-
@GeneralUtilities.check_arguments
|
|
335
|
-
def get_product_name(self)->str:
|
|
336
|
-
return self.__read_product_information_property("producttitle")
|
|
337
|
-
|
|
338
|
-
@GeneralUtilities.check_arguments
|
|
339
|
-
def get_remote_address(self)->str:
|
|
340
|
-
return self.__read_product_information_property("remoteaddress")
|
|
341
|
-
|
|
342
|
-
@GeneralUtilities.check_arguments
|
|
343
|
-
def __read_product_information_property(self, property_name: str) -> str:
|
|
344
|
-
product_information_file = os.path.join(self.get_repository_folder(), ".ScriptCollection", "ProductInformation.xml")
|
|
345
|
-
GeneralUtilities.assert_file_exists(product_information_file)
|
|
346
|
-
root: etree._ElementTree = etree.parse(product_information_file)
|
|
347
|
-
return str(root.xpath(f'/cps:productinformation/cps:{property_name}/text()', namespaces={'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'})[0])
|
|
348
|
-
|
|
349
|
-
@GeneralUtilities.check_arguments
|
|
350
|
-
def get_current_folder(self)->str:
|
|
351
|
-
return self.__current_folder
|
|
352
|
-
|
|
353
|
-
@GeneralUtilities.check_arguments
|
|
354
|
-
def get_verbosity(self)->LogLevel:
|
|
355
|
-
return self.__verbosity
|
|
356
|
-
|
|
357
|
-
@GeneralUtilities.check_arguments
|
|
358
|
-
def get_artifacts_folder(self) -> str:
|
|
359
|
-
return os.path.join(self.get_codeunit_folder(), "Other", "Artifacts")
|
|
360
|
-
|
|
361
|
-
@GeneralUtilities.check_arguments
|
|
362
|
-
def get_codeunit_file(self) -> str:
|
|
363
|
-
return os.path.join(self.get_codeunit_folder(), f"{self.get_codeunit_name()}.codeunit.xml")
|
|
364
|
-
|
|
365
|
-
def get_type_environment_type(self)->str:
|
|
366
|
-
return self.__target_environment_type
|
|
367
|
-
|
|
368
|
-
def get_target_environment_type(self)->str:
|
|
369
|
-
return self.__target_environment_type
|
|
370
|
-
|
|
371
|
-
@GeneralUtilities.check_arguments
|
|
372
|
-
def copy_source_files_to_output_directory(self) -> None:
|
|
373
|
-
self._protected_sc.log.log("Copy sourcecode...")
|
|
374
|
-
codeunit_folder =self.get_codeunit_folder()
|
|
375
|
-
result = self._protected_sc.run_program_argsasarray("git", ["ls-tree", "-r", "HEAD", "--name-only"], codeunit_folder)
|
|
376
|
-
files = [f for f in result[1].split('\n') if len(f) > 0]
|
|
377
|
-
for file in files:
|
|
378
|
-
full_source_file = os.path.join(codeunit_folder, file)
|
|
379
|
-
if os.path.isfile(full_source_file):
|
|
380
|
-
# Reson of isdir-check:
|
|
381
|
-
# Prevent trying to copy files which are not exist.
|
|
382
|
-
# Otherwise exceptions occurr because uncommitted deletions of files will result in an error here.
|
|
383
|
-
target_file = os.path.join(codeunit_folder, "Other", "Artifacts", "SourceCode", file)
|
|
384
|
-
target_folder = os.path.dirname(target_file)
|
|
385
|
-
GeneralUtilities.ensure_directory_exists(target_folder)
|
|
386
|
-
shutil.copyfile(full_source_file, target_file)
|
|
387
|
-
|
|
388
|
-
@GeneralUtilities.check_arguments
|
|
389
|
-
def run_testcases_common_post_task(self, repository_folder: str, codeunit_name: str, generate_badges: bool, targetenvironmenttype: str) -> None:
|
|
390
|
-
self._protected_sc.assert_is_git_repository(repository_folder)
|
|
391
|
-
coverage_file_folder = os.path.join(repository_folder, codeunit_name, "Other/Artifacts/TestCoverage")
|
|
392
|
-
coveragefiletarget = os.path.join(coverage_file_folder, "TestCoverage.xml")
|
|
393
|
-
self.__update_path_of_source_in_testcoverage_file(repository_folder, codeunit_name)
|
|
394
|
-
self.__standardized_tasks_generate_coverage_report(repository_folder, codeunit_name, generate_badges, targetenvironmenttype)
|
|
395
|
-
self.__check_testcoverage(coveragefiletarget, repository_folder, codeunit_name)
|
|
396
|
-
self.__format_xml_file(coveragefiletarget)
|
|
397
|
-
|
|
398
|
-
@GeneralUtilities.check_arguments
|
|
399
|
-
def __format_xml_file(self, xmlfile:str) -> None:
|
|
400
|
-
GeneralUtilities.write_text_to_file(xmlfile,self.__format_xml_content( GeneralUtilities.read_text_from_file(xmlfile)))
|
|
401
|
-
|
|
402
|
-
@GeneralUtilities.check_arguments
|
|
403
|
-
def __format_xml_content(self, xml:str) -> None:
|
|
404
|
-
root = etree.fromstring(xml)
|
|
405
|
-
return etree.tostring(root, pretty_print=True, encoding="unicode")
|
|
406
|
-
|
|
407
|
-
@GeneralUtilities.check_arguments
|
|
408
|
-
def __standardized_tasks_generate_coverage_report(self, repository_folder: str, codeunitname: str, generate_badges: bool, targetenvironmenttype: str, add_testcoverage_history_entry: bool = None) -> None:
|
|
409
|
-
"""This function expects that the file '<repositorybasefolder>/<codeunitname>/Other/Artifacts/TestCoverage/TestCoverage.xml'
|
|
410
|
-
which contains a test-coverage-report in the cobertura-format exists.
|
|
411
|
-
This script expectes that the testcoverage-reportfolder is '<repositorybasefolder>/<codeunitname>/Other/Artifacts/TestCoverageReport'.
|
|
412
|
-
This script expectes that a test-coverage-badges should be added to '<repositorybasefolder>/<codeunitname>/Other/Resources/Badges'."""
|
|
413
|
-
self._protected_sc.log.log("Generate testcoverage report..")
|
|
414
|
-
self._protected_sc.assert_is_git_repository(repository_folder)
|
|
415
|
-
codeunit_version = self.tfcps_Tools_General.get_version_of_codeunit(self.get_codeunit_file())
|
|
416
|
-
verbosity=0#TODO use loglevel-value here
|
|
417
|
-
if verbosity == 0:
|
|
418
|
-
verbose_argument_for_reportgenerator = "Off"
|
|
419
|
-
elif verbosity == 1:
|
|
420
|
-
verbose_argument_for_reportgenerator = "Error"
|
|
421
|
-
elif verbosity == 2:
|
|
422
|
-
verbose_argument_for_reportgenerator = "Info"
|
|
423
|
-
elif verbosity == 3:
|
|
424
|
-
verbose_argument_for_reportgenerator = "Verbose"
|
|
425
|
-
else:
|
|
426
|
-
raise ValueError(f"Unknown value for verbosity: {GeneralUtilities.str_none_safe(verbosity)}")
|
|
427
|
-
|
|
428
|
-
# Generating report
|
|
429
|
-
GeneralUtilities.ensure_directory_does_not_exist(os.path.join(repository_folder, codeunitname, f"{codeunitname}/Other/Artifacts/TestCoverageReport"))
|
|
430
|
-
GeneralUtilities.ensure_directory_exists(os.path.join(repository_folder, codeunitname, "Other/Artifacts/TestCoverageReport"))
|
|
431
|
-
|
|
432
|
-
if add_testcoverage_history_entry is None:
|
|
433
|
-
add_testcoverage_history_entry = self.__is_pre_merge
|
|
434
|
-
|
|
435
|
-
history_folder = f"{codeunitname}/Other/Resources/TestCoverageHistory"
|
|
436
|
-
history_folder_full = os.path.join(repository_folder, history_folder)
|
|
437
|
-
GeneralUtilities.ensure_directory_exists(history_folder_full)
|
|
438
|
-
files_before = set(GeneralUtilities.get_direct_files_of_folder(history_folder_full))
|
|
439
|
-
history_argument = f" -historydir:{history_folder}"
|
|
440
|
-
argument = f"-reports:{codeunitname}/Other/Artifacts/TestCoverage/TestCoverage.xml -targetdir:{codeunitname}/Other/Artifacts/TestCoverageReport --verbosity:{verbose_argument_for_reportgenerator}{history_argument} -title:{codeunitname} -tag:v{codeunit_version}"
|
|
441
|
-
self._protected_sc.run_program("reportgenerator", argument, repository_folder)
|
|
442
|
-
if not add_testcoverage_history_entry:
|
|
443
|
-
files_after = set(GeneralUtilities.get_direct_files_of_folder(history_folder_full))
|
|
444
|
-
for new_file in files_after - files_before:
|
|
445
|
-
GeneralUtilities.ensure_file_does_not_exist(new_file)
|
|
446
|
-
|
|
447
|
-
# Generating badges
|
|
448
|
-
if generate_badges:
|
|
449
|
-
testcoverageubfolger = "Other/Resources/TestCoverageBadges"
|
|
450
|
-
fulltestcoverageubfolger = os.path.join(repository_folder, codeunitname, testcoverageubfolger)
|
|
451
|
-
GeneralUtilities.ensure_directory_does_not_exist(fulltestcoverageubfolger)
|
|
452
|
-
GeneralUtilities.ensure_directory_exists(fulltestcoverageubfolger)
|
|
453
|
-
self._protected_sc.run_program("reportgenerator", f"-reports:Other/Artifacts/TestCoverage/TestCoverage.xml -targetdir:{testcoverageubfolger} -reporttypes:Badges --verbosity:{verbose_argument_for_reportgenerator}", os.path.join(repository_folder, codeunitname))
|
|
454
|
-
|
|
455
|
-
@GeneralUtilities.check_arguments
|
|
456
|
-
def __update_path_of_source_in_testcoverage_file(self, repository_folder: str, codeunitname: str) -> None:
|
|
457
|
-
self._protected_sc.assert_is_git_repository(repository_folder)
|
|
458
|
-
self._protected_sc.log.log("Update paths of source files in testcoverage files..")
|
|
459
|
-
folder = f"{repository_folder}/{codeunitname}/Other/Artifacts/TestCoverage"
|
|
460
|
-
filename = "TestCoverage.xml"
|
|
461
|
-
full_file = os.path.join(folder, filename)
|
|
462
|
-
GeneralUtilities.write_text_to_file(full_file, re.sub("<source>.+<\\/source>", f"<source><!--[repository]/-->./{codeunitname}/</source>", GeneralUtilities.read_text_from_file(full_file)))
|
|
463
|
-
self.__remove_not_existing_files_from_testcoverage_file(full_file, repository_folder, codeunitname)
|
|
464
|
-
|
|
465
|
-
@GeneralUtilities.check_arguments
|
|
466
|
-
def __remove_not_existing_files_from_testcoverage_file(self, testcoveragefile: str, repository_folder: str, codeunit_name: str) -> None:
|
|
467
|
-
self._protected_sc.assert_is_git_repository(repository_folder)
|
|
468
|
-
root: etree._ElementTree = etree.parse(testcoveragefile)
|
|
469
|
-
codeunit_folder = os.path.join(repository_folder, codeunit_name)
|
|
470
|
-
xpath = f"//coverage/packages/package[@name='{codeunit_name}']/classes/class"
|
|
471
|
-
coverage_report_classes = root.xpath(xpath)
|
|
472
|
-
found_existing_files = False
|
|
473
|
-
for coverage_report_class in coverage_report_classes:
|
|
474
|
-
filename = coverage_report_class.attrib['filename']
|
|
475
|
-
file = os.path.join(codeunit_folder, filename)
|
|
476
|
-
if os.path.isfile(file):
|
|
477
|
-
found_existing_files = True
|
|
478
|
-
else:
|
|
479
|
-
coverage_report_class.getparent().remove(coverage_report_class)
|
|
480
|
-
GeneralUtilities.assert_condition(found_existing_files, f"No existing files in testcoverage-report-file \"{testcoveragefile}\".")
|
|
481
|
-
result = etree.tostring(root).decode("utf-8")
|
|
482
|
-
GeneralUtilities.write_text_to_file(testcoveragefile, result)
|
|
483
|
-
|
|
484
|
-
@GeneralUtilities.check_arguments
|
|
485
|
-
def __check_testcoverage(self, testcoverage_file_in_cobertura_format: str, repository_folder: str, codeunitname: str) -> None:
|
|
486
|
-
self._protected_sc.assert_is_git_repository(repository_folder)
|
|
487
|
-
self._protected_sc.log.log("Check testcoverage..")
|
|
488
|
-
root: etree._ElementTree = etree.parse(testcoverage_file_in_cobertura_format)
|
|
489
|
-
if len(root.xpath('//coverage/packages/package')) != 1:
|
|
490
|
-
raise ValueError(f"'{testcoverage_file_in_cobertura_format}' must contain exactly 1 package.")
|
|
491
|
-
if root.xpath('//coverage/packages/package[1]/@name')[0] != codeunitname:
|
|
492
|
-
raise ValueError(f"The package name of the tested package in '{testcoverage_file_in_cobertura_format}' must be '{codeunitname}'.")
|
|
493
|
-
rates=root.xpath('//coverage/packages/package[1]/@line-rate')
|
|
494
|
-
coverage_in_percent = round(float(str(rates[0]))*100, 2)
|
|
495
|
-
technicalminimalrequiredtestcoverageinpercent = 0
|
|
496
|
-
if not technicalminimalrequiredtestcoverageinpercent < coverage_in_percent:
|
|
497
|
-
raise ValueError(f"The test-coverage of package '{codeunitname}' must be greater than {technicalminimalrequiredtestcoverageinpercent}%.")
|
|
498
|
-
minimalrequiredtestcoverageinpercent = self.get_testcoverage_threshold_from_codeunit_file()
|
|
499
|
-
if (coverage_in_percent < minimalrequiredtestcoverageinpercent):
|
|
500
|
-
raise ValueError(f"The testcoverage for codeunit {codeunitname} must be {minimalrequiredtestcoverageinpercent}% or more but is {coverage_in_percent}%.")
|
|
501
|
-
|
|
502
|
-
@GeneralUtilities.check_arguments
|
|
503
|
-
def get_testcoverage_threshold_from_codeunit_file(self):
|
|
504
|
-
root: etree._ElementTree = etree.parse(self.get_codeunit_file())
|
|
505
|
-
return float(str(root.xpath('//cps:properties/cps:testsettings/@minimalcodecoverageinpercent', namespaces={'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'})[0]))
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
@GeneralUtilities.check_arguments
|
|
509
|
-
def install_requirementstxt_for_codeunit(self):
|
|
510
|
-
self._protected_sc.install_requirementstxt_file(self.get_codeunit_folder()+"/requirements.txt")
|
|
511
|
-
|
|
512
|
-
class TFCPS_CodeUnitSpecific_Base_CLI():
|
|
513
|
-
|
|
514
|
-
@staticmethod
|
|
515
|
-
@GeneralUtilities.check_arguments
|
|
516
|
-
def get_base_parser()->argparse.ArgumentParser:
|
|
517
|
-
parser = argparse.ArgumentParser()
|
|
518
|
-
verbosity_values = ", ".join(f"{lvl.value}={lvl.name}" for lvl in LogLevel)
|
|
519
|
-
parser.add_argument('-e', '--targetenvironmenttype', required=False, default="QualityCheck")
|
|
520
|
-
parser.add_argument('-a', '--additionalargumentsfile', required=False, default=None)
|
|
521
|
-
parser.add_argument('-v', '--verbosity', required=False, default=3, help=f"Sets the loglevel. Possible values: {verbosity_values}")
|
|
522
|
-
parser.add_argument('-c', '--nocache', action='store_true', required=False, default=False)
|
|
523
|
-
parser.add_argument('-p', '--ispremerge', action='store_true', required=False, default=False)
|
|
524
|
-
return parser
|
|
1
|
+
import os
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
import shutil
|
|
4
|
+
import re
|
|
5
|
+
import json
|
|
6
|
+
import argparse
|
|
7
|
+
from abc import ABC, abstractmethod
|
|
8
|
+
import yaml
|
|
9
|
+
import xmlschema
|
|
10
|
+
from packaging.version import Version
|
|
11
|
+
from lxml import etree
|
|
12
|
+
from ..GeneralUtilities import GeneralUtilities, VersionEcholon
|
|
13
|
+
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
14
|
+
from ..SCLog import LogLevel
|
|
15
|
+
from .TFCPS_Tools_General import TFCPS_Tools_General
|
|
16
|
+
from .TFCPS_RemoteBuild import TFCPS_RemoteBuild, RunnerOperatingSystem
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TFCPS_CodeUnitSpecific_Base(ABC):
|
|
21
|
+
|
|
22
|
+
__current_file:str=None
|
|
23
|
+
__target_environment_type:str = None
|
|
24
|
+
__repository_folder:str=None
|
|
25
|
+
__codeunit_folder:str=None
|
|
26
|
+
__current_folder:str=None
|
|
27
|
+
__verbosity:LogLevel=None
|
|
28
|
+
__use_cache:bool=None
|
|
29
|
+
tfcps_Tools_General:TFCPS_Tools_General = None
|
|
30
|
+
_protected_sc:ScriptCollectionCore = None
|
|
31
|
+
__is_pre_merge:bool=False
|
|
32
|
+
__validate_developers_of_repository:bool=True#TODO must be setable to false
|
|
33
|
+
|
|
34
|
+
def __init__(self,current_file:str,verbosity:LogLevel,target_envionment_type:str,use_cache:bool,is_pre_merge:bool):
|
|
35
|
+
self.__verbosity=verbosity
|
|
36
|
+
self.__use_cache=use_cache
|
|
37
|
+
self.__target_environment_type=target_envionment_type
|
|
38
|
+
self.__current_file = str(Path(current_file).absolute())
|
|
39
|
+
self.__current_folder = os.path.dirname(self.__current_file)
|
|
40
|
+
self.__codeunit_folder=self.__search_codeunit_folder()
|
|
41
|
+
self.__is_pre_merge=is_pre_merge
|
|
42
|
+
self._protected_sc=ScriptCollectionCore()
|
|
43
|
+
self._protected_sc.log.loglevel=self.__verbosity
|
|
44
|
+
self.tfcps_Tools_General=TFCPS_Tools_General(self._protected_sc)
|
|
45
|
+
self.tfcps_Tools_General.assert_is_codeunit_folder(self.__codeunit_folder)
|
|
46
|
+
self.__repository_folder=GeneralUtilities.resolve_relative_path("..",self.__codeunit_folder)
|
|
47
|
+
self._protected_sc.assert_is_git_repository(self.__repository_folder)
|
|
48
|
+
#ensure the environment-variables which the product declares as required are also available when this codeunit-script is started
|
|
49
|
+
#directly (for example "<codeunit>/Other/Build/Build.py" or "task bb") and not by "scbuildcodeunits".
|
|
50
|
+
self.tfcps_Tools_General.ensure_required_environment_variables_are_set(self.__repository_folder)
|
|
51
|
+
|
|
52
|
+
def __search_codeunit_folder(self)->str:
|
|
53
|
+
current_path:str=os.path.dirname(self.__current_file)
|
|
54
|
+
enabled:bool=True
|
|
55
|
+
while enabled:
|
|
56
|
+
try:
|
|
57
|
+
current_path=GeneralUtilities.resolve_relative_path("..",current_path)
|
|
58
|
+
foldername=os.path.basename(current_path)
|
|
59
|
+
codeunit_file:str=os.path.join(current_path,f"{foldername}.codeunit.xml")
|
|
60
|
+
if os.path.isfile(codeunit_file):
|
|
61
|
+
return current_path
|
|
62
|
+
except:
|
|
63
|
+
enabled=False
|
|
64
|
+
raise ValueError(f"Can not find codeunit-folder for folder \"{self.__current_file}\".")
|
|
65
|
+
|
|
66
|
+
@abstractmethod
|
|
67
|
+
def get_dependencies(self)->dict[str,set[str]]:
|
|
68
|
+
raise ValueError(f"Operation is abstract.")
|
|
69
|
+
|
|
70
|
+
@abstractmethod
|
|
71
|
+
def get_available_versions(self,dependencyname:str)->list[str]:
|
|
72
|
+
raise ValueError(f"Operation is abstract.")
|
|
73
|
+
|
|
74
|
+
@abstractmethod
|
|
75
|
+
def set_dependency_version(self,name:str,new_version:str)->None:
|
|
76
|
+
raise ValueError(f"Operation is abstract.")
|
|
77
|
+
|
|
78
|
+
def update_dependencies(self):
|
|
79
|
+
self.update_dependencies_with_specific_echolon(VersionEcholon.LatestPatchOrLatestMinor)
|
|
80
|
+
|
|
81
|
+
def update_dependencies_with_specific_echolon(self, echolon: VersionEcholon):
|
|
82
|
+
ignored_dependencies=self.tfcps_Tools_General.get_dependencies_which_are_ignored_from_updates(self.get_codeunit_folder())
|
|
83
|
+
for ignored_dependency in ignored_dependencies:
|
|
84
|
+
self._protected_sc.log.log(f"Codeunit {self.get_codeunit_name()} ignores the dependency {ignored_dependency} in update-checks.", LogLevel.Warning)
|
|
85
|
+
|
|
86
|
+
dependencies_dict:dict[str,set[str]]=self.get_dependencies()
|
|
87
|
+
for dependencyname,dependency_versions in dependencies_dict.items():
|
|
88
|
+
GeneralUtilities.assert_condition(0<len(dependency_versions),f"Dependency {dependencyname} is not used.")
|
|
89
|
+
GeneralUtilities.assert_condition(len(dependency_versions)<2,f"Dependency {dependencyname} is used {len(dependency_versions)} times. Please consolidate it to one version before updating.")
|
|
90
|
+
dependency_version=next(iter(dependency_versions))
|
|
91
|
+
latest_currently_used_version=dependency_version
|
|
92
|
+
if dependencyname not in ignored_dependencies:
|
|
93
|
+
try:
|
|
94
|
+
available_versions:list[str]=self.get_available_versions(dependencyname)
|
|
95
|
+
for available_version in available_versions:
|
|
96
|
+
GeneralUtilities.assert_condition(re.match(r"^(\d+).(\d+).(\d+)$", available_version) is not None,f"Invalid-version-string: {available_version}")
|
|
97
|
+
desired_version=GeneralUtilities.choose_version(available_versions,latest_currently_used_version,echolon)
|
|
98
|
+
GeneralUtilities.assert_condition(Version(dependency_version)<=Version(desired_version),f"Desired version {desired_version} for dependency {dependencyname} is less than the actual used version {latest_currently_used_version}.")
|
|
99
|
+
update_dependency:bool=desired_version!=latest_currently_used_version
|
|
100
|
+
if update_dependency:
|
|
101
|
+
if len(dependency_versions)==1:
|
|
102
|
+
GeneralUtilities.write_message_to_stdout("Update dependency "+dependencyname+" (which is currently used in version "+dependency_version+") to version "+desired_version+".")
|
|
103
|
+
self.set_dependency_version(dependencyname,desired_version)
|
|
104
|
+
except Exception:
|
|
105
|
+
GeneralUtilities.write_message_to_stderr(f"Error while updating {dependencyname}.")
|
|
106
|
+
raise
|
|
107
|
+
|
|
108
|
+
def get_version_of_project(self)->str:
|
|
109
|
+
return self.tfcps_Tools_General.get_version_of_project(self.get_repository_folder())
|
|
110
|
+
|
|
111
|
+
@GeneralUtilities.check_arguments
|
|
112
|
+
def do_common_tasks_base(self,current_codeunit_version:str):
|
|
113
|
+
repository_folder: str =self.get_repository_folder()
|
|
114
|
+
self._protected_sc.assert_is_git_repository(repository_folder)
|
|
115
|
+
codeunit_name: str = self.get_codeunit_name()
|
|
116
|
+
project_version = self.tfcps_Tools_General.get_version_of_project(repository_folder)
|
|
117
|
+
if current_codeunit_version is None:
|
|
118
|
+
current_codeunit_version=project_version
|
|
119
|
+
codeunit_folder = os.path.join(repository_folder, codeunit_name)
|
|
120
|
+
|
|
121
|
+
# check codeunit-conformity
|
|
122
|
+
# The codeunit-folder must contain exactly one codeunit-file and its filename (without the ".codeunit.xml"-suffix)
|
|
123
|
+
# must be equal to the folder-name. (The equality of the folder-name and the name inside the file is checked further below.)
|
|
124
|
+
if not os.path.isdir(codeunit_folder):
|
|
125
|
+
raise ValueError(f'Codeunit-folder "{codeunit_folder}" does not exist.')
|
|
126
|
+
codeunit_filename_suffix = ".codeunit.xml"
|
|
127
|
+
codeunit_files_in_folder = [entry for entry in os.listdir(codeunit_folder) if entry.endswith(codeunit_filename_suffix) and os.path.isfile(os.path.join(codeunit_folder, entry))]
|
|
128
|
+
GeneralUtilities.assert_condition(len(codeunit_files_in_folder) == 1, f'The codeunit-folder "{codeunit_folder}" must contain exactly one codeunit-file (matching "*{codeunit_filename_suffix}") but contains {len(codeunit_files_in_folder)}.')
|
|
129
|
+
codeunit_name_in_codeunit_filename = codeunit_files_in_folder[0][:-len(codeunit_filename_suffix)]
|
|
130
|
+
GeneralUtilities.assert_condition(codeunit_name_in_codeunit_filename == codeunit_name, f'The codeunit-file "{codeunit_files_in_folder[0]}" must be named "{codeunit_name}{codeunit_filename_suffix}" to match its folder-name ("{codeunit_name}").')
|
|
131
|
+
supported_codeunitspecificationversion = "2.9.4" # should always be the latest version of the ProjectTemplates-repository
|
|
132
|
+
codeunit_file = os.path.join(codeunit_folder, f"{codeunit_name}{codeunit_filename_suffix}")
|
|
133
|
+
if not os.path.isfile(codeunit_file):
|
|
134
|
+
raise ValueError(f'Codeunitfile "{codeunit_file}" does not exist.')
|
|
135
|
+
# TODO implement usage of self.reference_latest_version_of_xsd_when_generating_xml
|
|
136
|
+
namespaces = {'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure', 'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
|
|
137
|
+
root: etree._ElementTree = etree.parse(codeunit_file)
|
|
138
|
+
|
|
139
|
+
# check codeunit-spcecification-version
|
|
140
|
+
try:
|
|
141
|
+
codeunit_file_version = root.xpath('//cps:codeunit/@codeunitspecificationversion', namespaces=namespaces)[0]
|
|
142
|
+
if codeunit_file_version != supported_codeunitspecificationversion:
|
|
143
|
+
raise ValueError(f"ScriptCollection only supports processing codeunits with codeunit-specification-version={supported_codeunitspecificationversion}.")
|
|
144
|
+
schemaLocation = root.xpath('//cps:codeunit/@xsi:schemaLocation', namespaces=namespaces)[0]
|
|
145
|
+
xmlschema.validate(codeunit_file, schemaLocation)
|
|
146
|
+
# TODO check if the properties codeunithastestablesourcecode, codeunithasupdatabledependencies, throwexceptionifcodeunitfilecannotbevalidated, developmentState and description exist and the values are valid
|
|
147
|
+
except Exception as exception:
|
|
148
|
+
self._protected_sc.log.log_exception(f'Codeunitfile "{codeunit_file}" can not be validated due to the following exception:', exception,LogLevel.Warning)
|
|
149
|
+
|
|
150
|
+
# check codeunit-name
|
|
151
|
+
codeunit_name_in_codeunit_file = root.xpath('//cps:codeunit/cps:name/text()', namespaces=namespaces)[0]
|
|
152
|
+
if codeunit_name != codeunit_name_in_codeunit_file:
|
|
153
|
+
raise ValueError(f"The folder-name ('{codeunit_name}') is not equal to the codeunit-name ('{codeunit_name_in_codeunit_file}').")
|
|
154
|
+
|
|
155
|
+
# check owner-name
|
|
156
|
+
codeunit_ownername_in_codeunit_file = self.tfcps_Tools_General. get_codeunit_owner_name(self.get_codeunit_file())
|
|
157
|
+
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(codeunit_ownername_in_codeunit_file), "No valid name for codeunitowner given.")
|
|
158
|
+
|
|
159
|
+
# check owner-emailaddress
|
|
160
|
+
codeunit_owneremailaddress_in_codeunit_file = self.tfcps_Tools_General.get_codeunit_owner_emailaddress(self.get_codeunit_file())
|
|
161
|
+
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(codeunit_owneremailaddress_in_codeunit_file), "No valid email-address for codeunitowner given.")
|
|
162
|
+
|
|
163
|
+
# check development-state
|
|
164
|
+
developmentstate = root.xpath('//cps:properties/@developmentstate', namespaces=namespaces)[0]
|
|
165
|
+
developmentstate_active = "Active development"
|
|
166
|
+
developmentstate_maintenance = "Maintenance-updates only"
|
|
167
|
+
developmentstate_inactive = "Inactive"
|
|
168
|
+
GeneralUtilities.assert_condition(developmentstate in (developmentstate_active, developmentstate_maintenance, developmentstate_inactive), f"Invalid development-state. Must be '{developmentstate_active}' or '{developmentstate_maintenance}' or '{developmentstate_inactive}' but was '{developmentstate}'.")
|
|
169
|
+
|
|
170
|
+
# check for mandatory files
|
|
171
|
+
files = ["Other/Build/Build.py", "Other/QualityCheck/Linting.py", "Other/Reference/GenerateReference.py"]
|
|
172
|
+
if self.tfcps_Tools_General.codeunit_has_testable_sourcecode(self.get_codeunit_file()):
|
|
173
|
+
# TODO check if the testsettings-section appears in the codeunit-file
|
|
174
|
+
files.append("Other/QualityCheck/RunTestcases.py")
|
|
175
|
+
if self.tfcps_Tools_General.codeunit_has_updatable_dependencies(self.get_codeunit_file()):
|
|
176
|
+
# TODO check if the updatesettings-section appears in the codeunit-file
|
|
177
|
+
files.append("Other/UpdateDependencies.py")
|
|
178
|
+
for file in files:
|
|
179
|
+
combined_file = os.path.join(codeunit_folder, file)
|
|
180
|
+
if not os.path.isfile(combined_file):
|
|
181
|
+
raise ValueError(f'The mandatory file "{file}" does not exist in the codeunit-folder.')
|
|
182
|
+
|
|
183
|
+
if os.path.isfile(os.path.join(codeunit_folder, "requirements.txt")):
|
|
184
|
+
self.install_requirementstxt_for_codeunit()
|
|
185
|
+
|
|
186
|
+
# check developer
|
|
187
|
+
if self.__validate_developers_of_repository:
|
|
188
|
+
expected_authors: list[tuple[str, str]] = []
|
|
189
|
+
expected_authors_in_xml = root.xpath('//cps:codeunit/cps:developerteam/cps:developer', namespaces=namespaces)
|
|
190
|
+
for expected_author in expected_authors_in_xml:
|
|
191
|
+
author_name = expected_author.xpath('./cps:developername/text()', namespaces=namespaces)[0]
|
|
192
|
+
author_emailaddress = expected_author.xpath('./cps:developeremailaddress/text()', namespaces=namespaces)[0]
|
|
193
|
+
expected_authors.append((author_name, author_emailaddress))
|
|
194
|
+
actual_authors: list[tuple[str, str]] = self.tfcps_Tools_General.get_all_authors_and_committers_of_repository(repository_folder, codeunit_name)
|
|
195
|
+
# TODO refactor this check to only check commits which are behind this but which are not already on main
|
|
196
|
+
# TODO verify also if the commit is signed by a valid key of the author
|
|
197
|
+
for actual_author in actual_authors:
|
|
198
|
+
if not (actual_author) in expected_authors:
|
|
199
|
+
actual_author_formatted = f"{actual_author[0]} <{actual_author[1]}>"
|
|
200
|
+
raise ValueError(f'Author/Comitter "{actual_author_formatted}" is not in the codeunit-developer-team. If {actual_author} is a authorized developer for this codeunit you should consider defining this in the codeunit-file or adapting the name using a .mailmap-file (see https://git-scm.com/docs/gitmailmap). The developer-team-check can also be disabled using the property validate_developers_of_repository.')
|
|
201
|
+
|
|
202
|
+
dependent_codeunits = self.tfcps_Tools_General.get_dependent_code_units(codeunit_file)
|
|
203
|
+
for dependent_codeunit in dependent_codeunits:
|
|
204
|
+
if not self.tfcps_Tools_General.dependent_codeunit_exists(repository_folder, dependent_codeunit):
|
|
205
|
+
raise ValueError(f"Codeunit {codeunit_name} does have dependent codeunit {dependent_codeunit} which does not exist.")
|
|
206
|
+
|
|
207
|
+
# TODO implement cycle-check for dependent codeunits
|
|
208
|
+
|
|
209
|
+
artifacts_folder = os.path.join(codeunit_folder, "Other", "Artifacts")
|
|
210
|
+
GeneralUtilities.ensure_directory_does_not_exist(artifacts_folder)
|
|
211
|
+
|
|
212
|
+
# get artifacts from dependent codeunits
|
|
213
|
+
self.tfcps_Tools_General.copy_artifacts_from_dependent_code_units(repository_folder, codeunit_name)
|
|
214
|
+
|
|
215
|
+
# update codeunit-version
|
|
216
|
+
self.tfcps_Tools_General.write_version_to_codeunit_file(self.get_codeunit_file(), current_codeunit_version)
|
|
217
|
+
|
|
218
|
+
# set project version
|
|
219
|
+
package_json_file = os.path.join(repository_folder, "package.json") # TDOO move this to a general project-specific (and codeunit-independent-script)
|
|
220
|
+
if os.path.isfile(package_json_file):
|
|
221
|
+
package_json_data: str = None
|
|
222
|
+
with open(package_json_file, "r", encoding="utf-8") as f1:
|
|
223
|
+
package_json_data = json.load(f1)
|
|
224
|
+
package_json_data["version"] = project_version
|
|
225
|
+
with open(package_json_file, "w", encoding="utf-8") as f2:
|
|
226
|
+
json.dump(package_json_data, f2, indent=2)
|
|
227
|
+
GeneralUtilities.write_text_to_file(package_json_file, GeneralUtilities.read_text_from_file(package_json_file).replace("\r", ""))
|
|
228
|
+
|
|
229
|
+
# set default constants
|
|
230
|
+
self.tfcps_Tools_General.set_default_constants(os.path.join(codeunit_folder))
|
|
231
|
+
|
|
232
|
+
# Hints-file
|
|
233
|
+
hints_file = os.path.join(codeunit_folder, "Other", "Reference", "ReferenceContent", "Hints.md")
|
|
234
|
+
if not os.path.isfile(hints_file):
|
|
235
|
+
raise ValueError(f"Hints-file '{hints_file}' does not exist.")
|
|
236
|
+
|
|
237
|
+
# Copy license-file
|
|
238
|
+
self.tfcps_Tools_General.copy_licence_file(self.get_codeunit_folder())
|
|
239
|
+
|
|
240
|
+
# Propagate the repository-level dependency-version-pins (e.g. PlantUML, OpenAPIGenerator, JRE) into the codeunit
|
|
241
|
+
# so each codeunit carries its own copy of the versions that are defined once at repository-level.
|
|
242
|
+
repository_dependencies_folder = os.path.join(repository_folder, "Other", "Resources", "Dependencies")
|
|
243
|
+
if os.path.isdir(repository_dependencies_folder):
|
|
244
|
+
for dependency_folder in GeneralUtilities.get_direct_folders_of_folder(repository_dependencies_folder):
|
|
245
|
+
dependency_name = os.path.basename(dependency_folder)
|
|
246
|
+
source_version_file = os.path.join(dependency_folder, "Version.txt")
|
|
247
|
+
target_version_file = os.path.join(codeunit_folder, "Other", "Resources", "Dependencies", dependency_name, "Version.txt")
|
|
248
|
+
if os.path.isfile(target_version_file):# only update dependencies, do not create new ones automatically
|
|
249
|
+
GeneralUtilities.ensure_directory_exists(os.path.dirname(target_version_file))
|
|
250
|
+
shutil.copyfile(source_version_file, target_version_file)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
@GeneralUtilities.check_arguments
|
|
254
|
+
def generate_reference_using_docfx(self, generate_class_reference:bool=False):
|
|
255
|
+
reference_folder =os.path.join( self.get_codeunit_folder(),"Other","Reference")
|
|
256
|
+
class_reference_folder:str=os.path.join(reference_folder,"ReferenceContent","API")
|
|
257
|
+
toc_md_file:str=os.path.join(class_reference_folder,"toc.md")
|
|
258
|
+
GeneralUtilities.ensure_file_does_not_exist(toc_md_file)#should not be available when docfx is runninng because then toc.html will be generated from toc.md but toc.html will be generated by docfx in the target-folder
|
|
259
|
+
|
|
260
|
+
self.tfcps_Tools_General.generate_svg_files_from_plantuml_files_for_codeunit(self.get_codeunit_folder(), self.use_cache())
|
|
261
|
+
generated_reference_folder = GeneralUtilities.resolve_relative_path("../Artifacts/Reference", reference_folder)
|
|
262
|
+
GeneralUtilities.ensure_directory_does_not_exist(generated_reference_folder)
|
|
263
|
+
GeneralUtilities.ensure_directory_exists(generated_reference_folder)
|
|
264
|
+
obj_folder = os.path.join(reference_folder, "obj")
|
|
265
|
+
GeneralUtilities.ensure_folder_exists_and_is_empty(obj_folder)
|
|
266
|
+
self._protected_sc.run_program("docfx", "-t default,templates/darkfx docfx.json", reference_folder)
|
|
267
|
+
GeneralUtilities.ensure_directory_does_not_exist(obj_folder)
|
|
268
|
+
|
|
269
|
+
if generate_class_reference:# this codeunit has a class-reference, so a human-readable table-of-content can be generated for it
|
|
270
|
+
GeneralUtilities.ensure_file_exists(toc_md_file)
|
|
271
|
+
GeneralUtilities.write_text_to_file(toc_md_file, self.__generate_toc_md_file_content(os.path.join(class_reference_folder,"toc.yml")))
|
|
272
|
+
|
|
273
|
+
@staticmethod
|
|
274
|
+
@GeneralUtilities.check_arguments
|
|
275
|
+
def __generate_toc_md_file_content(toc_file:str)->str:
|
|
276
|
+
"""Generates the content of a markdown-table-of-content which contains a headline and links to the sub-files of the class-reference
|
|
277
|
+
based on the given toc.yml-file which was generated by docfx."""
|
|
278
|
+
toc_file_content = yaml.safe_load(GeneralUtilities.read_text_from_file(toc_file))
|
|
279
|
+
namespaces:list = list()
|
|
280
|
+
if isinstance(toc_file_content, dict):
|
|
281
|
+
namespaces = toc_file_content.get("items", list())
|
|
282
|
+
elif isinstance(toc_file_content, list):# older docfx-versions do not wrap the entries in an "items"-property
|
|
283
|
+
namespaces = toc_file_content
|
|
284
|
+
lines:list[str] = ["# Table of contents"]
|
|
285
|
+
for namespace in namespaces:
|
|
286
|
+
lines.append(GeneralUtilities.empty_string)
|
|
287
|
+
lines.append(f"## {TFCPS_CodeUnitSpecific_Base.__get_toc_entry_name(namespace)}")
|
|
288
|
+
lines.append(GeneralUtilities.empty_string)
|
|
289
|
+
entries:list = namespace.get("items", list())
|
|
290
|
+
if len(entries) == 0:
|
|
291
|
+
lines.append("(This namespace does not contain any documented type.)")
|
|
292
|
+
else:
|
|
293
|
+
for entry in entries:
|
|
294
|
+
lines.append(f"- [{TFCPS_CodeUnitSpecific_Base.__get_toc_entry_name(entry)}]({TFCPS_CodeUnitSpecific_Base.__get_toc_entry_link(entry)})")
|
|
295
|
+
return "\n".join(lines)+"\n"
|
|
296
|
+
|
|
297
|
+
@staticmethod
|
|
298
|
+
@GeneralUtilities.check_arguments
|
|
299
|
+
def __get_toc_entry_name(entry:dict)->str:
|
|
300
|
+
name:str = entry.get("name", entry.get("uid", GeneralUtilities.empty_string))
|
|
301
|
+
return name.replace("&", "&").replace("<", "<").replace(">", ">")# generic types like "List<T>" would be treated as html-tag otherwise
|
|
302
|
+
|
|
303
|
+
@staticmethod
|
|
304
|
+
@GeneralUtilities.check_arguments
|
|
305
|
+
def __get_toc_entry_link(entry:dict)->str:
|
|
306
|
+
uid:str = entry["uid"]
|
|
307
|
+
return "./"+uid.replace("`", "-")+".yml"# docfx replaces the backtick of generic types by a hyphen when generating the filenames
|
|
308
|
+
|
|
309
|
+
@GeneralUtilities.check_arguments
|
|
310
|
+
def use_cache(self)->bool:
|
|
311
|
+
return self.__use_cache
|
|
312
|
+
|
|
313
|
+
@GeneralUtilities.check_arguments
|
|
314
|
+
def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str) -> None:
|
|
315
|
+
"""Runs the given program for this codeunit on a remote task-runner that provides the given operating-system (see
|
|
316
|
+
TFCPS_RemoteBuild and the SCTaskRunnerWindows/SCTaskRunnerMacOS-codeunits). This is used for operating-system-bound
|
|
317
|
+
build-steps (e.g. flutter-windows- or flutter-ios-builds) which can not run on the current operating-system. After
|
|
318
|
+
the runner finished, only the codeunit-folder is mirrored back into the local repository, so it looks as if the
|
|
319
|
+
runner had built locally. working_directory must be a folder inside this codeunit."""
|
|
320
|
+
TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory)
|
|
321
|
+
|
|
322
|
+
@GeneralUtilities.check_arguments
|
|
323
|
+
def get_codeunit_folder(self)->str:
|
|
324
|
+
return self.__codeunit_folder
|
|
325
|
+
|
|
326
|
+
@GeneralUtilities.check_arguments
|
|
327
|
+
def get_codeunit_name(self)->str:
|
|
328
|
+
return os.path.basename(self.__codeunit_folder)
|
|
329
|
+
|
|
330
|
+
@GeneralUtilities.check_arguments
|
|
331
|
+
def get_repository_folder(self)->str:
|
|
332
|
+
return self.__repository_folder
|
|
333
|
+
|
|
334
|
+
@GeneralUtilities.check_arguments
|
|
335
|
+
def get_product_name(self)->str:
|
|
336
|
+
return self.__read_product_information_property("producttitle")
|
|
337
|
+
|
|
338
|
+
@GeneralUtilities.check_arguments
|
|
339
|
+
def get_remote_address(self)->str:
|
|
340
|
+
return self.__read_product_information_property("remoteaddress")
|
|
341
|
+
|
|
342
|
+
@GeneralUtilities.check_arguments
|
|
343
|
+
def __read_product_information_property(self, property_name: str) -> str:
|
|
344
|
+
product_information_file = os.path.join(self.get_repository_folder(), ".ScriptCollection", "ProductInformation.xml")
|
|
345
|
+
GeneralUtilities.assert_file_exists(product_information_file)
|
|
346
|
+
root: etree._ElementTree = etree.parse(product_information_file)
|
|
347
|
+
return str(root.xpath(f'/cps:productinformation/cps:{property_name}/text()', namespaces={'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'})[0])
|
|
348
|
+
|
|
349
|
+
@GeneralUtilities.check_arguments
|
|
350
|
+
def get_current_folder(self)->str:
|
|
351
|
+
return self.__current_folder
|
|
352
|
+
|
|
353
|
+
@GeneralUtilities.check_arguments
|
|
354
|
+
def get_verbosity(self)->LogLevel:
|
|
355
|
+
return self.__verbosity
|
|
356
|
+
|
|
357
|
+
@GeneralUtilities.check_arguments
|
|
358
|
+
def get_artifacts_folder(self) -> str:
|
|
359
|
+
return os.path.join(self.get_codeunit_folder(), "Other", "Artifacts")
|
|
360
|
+
|
|
361
|
+
@GeneralUtilities.check_arguments
|
|
362
|
+
def get_codeunit_file(self) -> str:
|
|
363
|
+
return os.path.join(self.get_codeunit_folder(), f"{self.get_codeunit_name()}.codeunit.xml")
|
|
364
|
+
|
|
365
|
+
def get_type_environment_type(self)->str:
|
|
366
|
+
return self.__target_environment_type
|
|
367
|
+
|
|
368
|
+
def get_target_environment_type(self)->str:
|
|
369
|
+
return self.__target_environment_type
|
|
370
|
+
|
|
371
|
+
@GeneralUtilities.check_arguments
|
|
372
|
+
def copy_source_files_to_output_directory(self) -> None:
|
|
373
|
+
self._protected_sc.log.log("Copy sourcecode...")
|
|
374
|
+
codeunit_folder =self.get_codeunit_folder()
|
|
375
|
+
result = self._protected_sc.run_program_argsasarray("git", ["ls-tree", "-r", "HEAD", "--name-only"], codeunit_folder)
|
|
376
|
+
files = [f for f in result[1].split('\n') if len(f) > 0]
|
|
377
|
+
for file in files:
|
|
378
|
+
full_source_file = os.path.join(codeunit_folder, file)
|
|
379
|
+
if os.path.isfile(full_source_file):
|
|
380
|
+
# Reson of isdir-check:
|
|
381
|
+
# Prevent trying to copy files which are not exist.
|
|
382
|
+
# Otherwise exceptions occurr because uncommitted deletions of files will result in an error here.
|
|
383
|
+
target_file = os.path.join(codeunit_folder, "Other", "Artifacts", "SourceCode", file)
|
|
384
|
+
target_folder = os.path.dirname(target_file)
|
|
385
|
+
GeneralUtilities.ensure_directory_exists(target_folder)
|
|
386
|
+
shutil.copyfile(full_source_file, target_file)
|
|
387
|
+
|
|
388
|
+
@GeneralUtilities.check_arguments
|
|
389
|
+
def run_testcases_common_post_task(self, repository_folder: str, codeunit_name: str, generate_badges: bool, targetenvironmenttype: str) -> None:
|
|
390
|
+
self._protected_sc.assert_is_git_repository(repository_folder)
|
|
391
|
+
coverage_file_folder = os.path.join(repository_folder, codeunit_name, "Other/Artifacts/TestCoverage")
|
|
392
|
+
coveragefiletarget = os.path.join(coverage_file_folder, "TestCoverage.xml")
|
|
393
|
+
self.__update_path_of_source_in_testcoverage_file(repository_folder, codeunit_name)
|
|
394
|
+
self.__standardized_tasks_generate_coverage_report(repository_folder, codeunit_name, generate_badges, targetenvironmenttype)
|
|
395
|
+
self.__check_testcoverage(coveragefiletarget, repository_folder, codeunit_name)
|
|
396
|
+
self.__format_xml_file(coveragefiletarget)
|
|
397
|
+
|
|
398
|
+
@GeneralUtilities.check_arguments
|
|
399
|
+
def __format_xml_file(self, xmlfile:str) -> None:
|
|
400
|
+
GeneralUtilities.write_text_to_file(xmlfile,self.__format_xml_content( GeneralUtilities.read_text_from_file(xmlfile)))
|
|
401
|
+
|
|
402
|
+
@GeneralUtilities.check_arguments
|
|
403
|
+
def __format_xml_content(self, xml:str) -> None:
|
|
404
|
+
root = etree.fromstring(xml)
|
|
405
|
+
return etree.tostring(root, pretty_print=True, encoding="unicode")
|
|
406
|
+
|
|
407
|
+
@GeneralUtilities.check_arguments
|
|
408
|
+
def __standardized_tasks_generate_coverage_report(self, repository_folder: str, codeunitname: str, generate_badges: bool, targetenvironmenttype: str, add_testcoverage_history_entry: bool = None) -> None:
|
|
409
|
+
"""This function expects that the file '<repositorybasefolder>/<codeunitname>/Other/Artifacts/TestCoverage/TestCoverage.xml'
|
|
410
|
+
which contains a test-coverage-report in the cobertura-format exists.
|
|
411
|
+
This script expectes that the testcoverage-reportfolder is '<repositorybasefolder>/<codeunitname>/Other/Artifacts/TestCoverageReport'.
|
|
412
|
+
This script expectes that a test-coverage-badges should be added to '<repositorybasefolder>/<codeunitname>/Other/Resources/Badges'."""
|
|
413
|
+
self._protected_sc.log.log("Generate testcoverage report..")
|
|
414
|
+
self._protected_sc.assert_is_git_repository(repository_folder)
|
|
415
|
+
codeunit_version = self.tfcps_Tools_General.get_version_of_codeunit(self.get_codeunit_file())
|
|
416
|
+
verbosity=0#TODO use loglevel-value here
|
|
417
|
+
if verbosity == 0:
|
|
418
|
+
verbose_argument_for_reportgenerator = "Off"
|
|
419
|
+
elif verbosity == 1:
|
|
420
|
+
verbose_argument_for_reportgenerator = "Error"
|
|
421
|
+
elif verbosity == 2:
|
|
422
|
+
verbose_argument_for_reportgenerator = "Info"
|
|
423
|
+
elif verbosity == 3:
|
|
424
|
+
verbose_argument_for_reportgenerator = "Verbose"
|
|
425
|
+
else:
|
|
426
|
+
raise ValueError(f"Unknown value for verbosity: {GeneralUtilities.str_none_safe(verbosity)}")
|
|
427
|
+
|
|
428
|
+
# Generating report
|
|
429
|
+
GeneralUtilities.ensure_directory_does_not_exist(os.path.join(repository_folder, codeunitname, f"{codeunitname}/Other/Artifacts/TestCoverageReport"))
|
|
430
|
+
GeneralUtilities.ensure_directory_exists(os.path.join(repository_folder, codeunitname, "Other/Artifacts/TestCoverageReport"))
|
|
431
|
+
|
|
432
|
+
if add_testcoverage_history_entry is None:
|
|
433
|
+
add_testcoverage_history_entry = self.__is_pre_merge
|
|
434
|
+
|
|
435
|
+
history_folder = f"{codeunitname}/Other/Resources/TestCoverageHistory"
|
|
436
|
+
history_folder_full = os.path.join(repository_folder, history_folder)
|
|
437
|
+
GeneralUtilities.ensure_directory_exists(history_folder_full)
|
|
438
|
+
files_before = set(GeneralUtilities.get_direct_files_of_folder(history_folder_full))
|
|
439
|
+
history_argument = f" -historydir:{history_folder}"
|
|
440
|
+
argument = f"-reports:{codeunitname}/Other/Artifacts/TestCoverage/TestCoverage.xml -targetdir:{codeunitname}/Other/Artifacts/TestCoverageReport --verbosity:{verbose_argument_for_reportgenerator}{history_argument} -title:{codeunitname} -tag:v{codeunit_version}"
|
|
441
|
+
self._protected_sc.run_program("reportgenerator", argument, repository_folder)
|
|
442
|
+
if not add_testcoverage_history_entry:
|
|
443
|
+
files_after = set(GeneralUtilities.get_direct_files_of_folder(history_folder_full))
|
|
444
|
+
for new_file in files_after - files_before:
|
|
445
|
+
GeneralUtilities.ensure_file_does_not_exist(new_file)
|
|
446
|
+
|
|
447
|
+
# Generating badges
|
|
448
|
+
if generate_badges:
|
|
449
|
+
testcoverageubfolger = "Other/Resources/TestCoverageBadges"
|
|
450
|
+
fulltestcoverageubfolger = os.path.join(repository_folder, codeunitname, testcoverageubfolger)
|
|
451
|
+
GeneralUtilities.ensure_directory_does_not_exist(fulltestcoverageubfolger)
|
|
452
|
+
GeneralUtilities.ensure_directory_exists(fulltestcoverageubfolger)
|
|
453
|
+
self._protected_sc.run_program("reportgenerator", f"-reports:Other/Artifacts/TestCoverage/TestCoverage.xml -targetdir:{testcoverageubfolger} -reporttypes:Badges --verbosity:{verbose_argument_for_reportgenerator}", os.path.join(repository_folder, codeunitname))
|
|
454
|
+
|
|
455
|
+
@GeneralUtilities.check_arguments
|
|
456
|
+
def __update_path_of_source_in_testcoverage_file(self, repository_folder: str, codeunitname: str) -> None:
|
|
457
|
+
self._protected_sc.assert_is_git_repository(repository_folder)
|
|
458
|
+
self._protected_sc.log.log("Update paths of source files in testcoverage files..")
|
|
459
|
+
folder = f"{repository_folder}/{codeunitname}/Other/Artifacts/TestCoverage"
|
|
460
|
+
filename = "TestCoverage.xml"
|
|
461
|
+
full_file = os.path.join(folder, filename)
|
|
462
|
+
GeneralUtilities.write_text_to_file(full_file, re.sub("<source>.+<\\/source>", f"<source><!--[repository]/-->./{codeunitname}/</source>", GeneralUtilities.read_text_from_file(full_file)))
|
|
463
|
+
self.__remove_not_existing_files_from_testcoverage_file(full_file, repository_folder, codeunitname)
|
|
464
|
+
|
|
465
|
+
@GeneralUtilities.check_arguments
|
|
466
|
+
def __remove_not_existing_files_from_testcoverage_file(self, testcoveragefile: str, repository_folder: str, codeunit_name: str) -> None:
|
|
467
|
+
self._protected_sc.assert_is_git_repository(repository_folder)
|
|
468
|
+
root: etree._ElementTree = etree.parse(testcoveragefile)
|
|
469
|
+
codeunit_folder = os.path.join(repository_folder, codeunit_name)
|
|
470
|
+
xpath = f"//coverage/packages/package[@name='{codeunit_name}']/classes/class"
|
|
471
|
+
coverage_report_classes = root.xpath(xpath)
|
|
472
|
+
found_existing_files = False
|
|
473
|
+
for coverage_report_class in coverage_report_classes:
|
|
474
|
+
filename = coverage_report_class.attrib['filename']
|
|
475
|
+
file = os.path.join(codeunit_folder, filename)
|
|
476
|
+
if os.path.isfile(file):
|
|
477
|
+
found_existing_files = True
|
|
478
|
+
else:
|
|
479
|
+
coverage_report_class.getparent().remove(coverage_report_class)
|
|
480
|
+
GeneralUtilities.assert_condition(found_existing_files, f"No existing files in testcoverage-report-file \"{testcoveragefile}\".")
|
|
481
|
+
result = etree.tostring(root).decode("utf-8")
|
|
482
|
+
GeneralUtilities.write_text_to_file(testcoveragefile, result)
|
|
483
|
+
|
|
484
|
+
@GeneralUtilities.check_arguments
|
|
485
|
+
def __check_testcoverage(self, testcoverage_file_in_cobertura_format: str, repository_folder: str, codeunitname: str) -> None:
|
|
486
|
+
self._protected_sc.assert_is_git_repository(repository_folder)
|
|
487
|
+
self._protected_sc.log.log("Check testcoverage..")
|
|
488
|
+
root: etree._ElementTree = etree.parse(testcoverage_file_in_cobertura_format)
|
|
489
|
+
if len(root.xpath('//coverage/packages/package')) != 1:
|
|
490
|
+
raise ValueError(f"'{testcoverage_file_in_cobertura_format}' must contain exactly 1 package.")
|
|
491
|
+
if root.xpath('//coverage/packages/package[1]/@name')[0] != codeunitname:
|
|
492
|
+
raise ValueError(f"The package name of the tested package in '{testcoverage_file_in_cobertura_format}' must be '{codeunitname}'.")
|
|
493
|
+
rates=root.xpath('//coverage/packages/package[1]/@line-rate')
|
|
494
|
+
coverage_in_percent = round(float(str(rates[0]))*100, 2)
|
|
495
|
+
technicalminimalrequiredtestcoverageinpercent = 0
|
|
496
|
+
if not technicalminimalrequiredtestcoverageinpercent < coverage_in_percent:
|
|
497
|
+
raise ValueError(f"The test-coverage of package '{codeunitname}' must be greater than {technicalminimalrequiredtestcoverageinpercent}%.")
|
|
498
|
+
minimalrequiredtestcoverageinpercent = self.get_testcoverage_threshold_from_codeunit_file()
|
|
499
|
+
if (coverage_in_percent < minimalrequiredtestcoverageinpercent):
|
|
500
|
+
raise ValueError(f"The testcoverage for codeunit {codeunitname} must be {minimalrequiredtestcoverageinpercent}% or more but is {coverage_in_percent}%.")
|
|
501
|
+
|
|
502
|
+
@GeneralUtilities.check_arguments
|
|
503
|
+
def get_testcoverage_threshold_from_codeunit_file(self):
|
|
504
|
+
root: etree._ElementTree = etree.parse(self.get_codeunit_file())
|
|
505
|
+
return float(str(root.xpath('//cps:properties/cps:testsettings/@minimalcodecoverageinpercent', namespaces={'cps': 'https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/tree/main/Conventions/RepositoryStructure/CommonProjectStructure'})[0]))
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
@GeneralUtilities.check_arguments
|
|
509
|
+
def install_requirementstxt_for_codeunit(self):
|
|
510
|
+
self._protected_sc.install_requirementstxt_file(self.get_codeunit_folder()+"/requirements.txt")
|
|
511
|
+
|
|
512
|
+
class TFCPS_CodeUnitSpecific_Base_CLI():
|
|
513
|
+
|
|
514
|
+
@staticmethod
|
|
515
|
+
@GeneralUtilities.check_arguments
|
|
516
|
+
def get_base_parser()->argparse.ArgumentParser:
|
|
517
|
+
parser = argparse.ArgumentParser()
|
|
518
|
+
verbosity_values = ", ".join(f"{lvl.value}={lvl.name}" for lvl in LogLevel)
|
|
519
|
+
parser.add_argument('-e', '--targetenvironmenttype', required=False, default="QualityCheck")
|
|
520
|
+
parser.add_argument('-a', '--additionalargumentsfile', required=False, default=None)
|
|
521
|
+
parser.add_argument('-v', '--verbosity', required=False, default=3, help=f"Sets the loglevel. Possible values: {verbosity_values}")
|
|
522
|
+
parser.add_argument('-c', '--nocache', action='store_true', required=False, default=False)
|
|
523
|
+
parser.add_argument('-p', '--ispremerge', action='store_true', required=False, default=False)
|
|
524
|
+
return parser
|