ScriptCollection 4.4.2__py3-none-any.whl → 4.4.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/Executables.py +37 -5
- ScriptCollection/GeneralUtilities.py +13 -1
- ScriptCollection/SCLog.py +1 -1
- ScriptCollection/ScriptCollectionCore.py +117 -11
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +63 -2
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +1 -1
- ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py +128 -47
- ScriptCollection/TFCPS/Maven/TFCPS_CodeUnitSpecific_Maven.py +139 -7
- ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +8 -2
- ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +71 -0
- ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py +169 -7
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +62 -21
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +128 -11
- ScriptCollection/TFCPS/TFCPS_VisualRegressionTests.py +446 -0
- ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py +188 -0
- ScriptCollection/TFCPS/TypeScript/__init__.py +0 -0
- ScriptCollection/TestKind.py +41 -0
- {scriptcollection-4.4.2.dist-info → scriptcollection-4.4.6.dist-info}/METADATA +168 -168
- {scriptcollection-4.4.2.dist-info → scriptcollection-4.4.6.dist-info}/RECORD +22 -18
- {scriptcollection-4.4.2.dist-info → scriptcollection-4.4.6.dist-info}/entry_points.txt +1 -0
- {scriptcollection-4.4.2.dist-info → scriptcollection-4.4.6.dist-info}/WHEEL +0 -0
- {scriptcollection-4.4.2.dist-info → scriptcollection-4.4.6.dist-info}/top_level.txt +0 -0
|
@@ -1,34 +1,166 @@
|
|
|
1
|
-
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
import shutil
|
|
4
|
+
import requests
|
|
5
|
+
from lxml import etree
|
|
6
|
+
from ...GeneralUtilities import GeneralUtilities, Dependency
|
|
2
7
|
from ...SCLog import LogLevel
|
|
3
8
|
from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base, TFCPS_CodeUnitSpecific_Base_CLI
|
|
4
9
|
|
|
5
10
|
class TFCPS_CodeUnitSpecific_Maven_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
11
|
+
"""Implements the tasks of a codeunit which contains a maven-project.
|
|
12
|
+
|
|
13
|
+
The maven-project is expected in "<codeunit>/<codeunit>" and it contains its testcases in its own
|
|
14
|
+
"src/test"-folder, because that is the layout maven defines and the layout jacoco measures the
|
|
15
|
+
coverage of without further configuration."""
|
|
16
|
+
|
|
17
|
+
__pom_namespace: str = "http://maven.apache.org/POM/4.0.0"
|
|
6
18
|
|
|
7
19
|
def __init__(self, current_file: str, verbosity: LogLevel, targetenvironmenttype: str, use_cache: bool, is_pre_merge: bool):
|
|
8
20
|
super().__init__(current_file, verbosity, targetenvironmenttype, use_cache, is_pre_merge)
|
|
9
21
|
|
|
22
|
+
@GeneralUtilities.check_arguments
|
|
23
|
+
def get_maven_project_folder(self) -> str:
|
|
24
|
+
"""Returns the folder which contains the pom-file of this codeunit."""
|
|
25
|
+
return os.path.join(self.get_codeunit_folder(), self.get_codeunit_name())
|
|
26
|
+
|
|
27
|
+
@GeneralUtilities.check_arguments
|
|
28
|
+
def get_pom_file(self) -> str:
|
|
29
|
+
return os.path.join(self.get_maven_project_folder(), "pom.xml")
|
|
30
|
+
|
|
10
31
|
@GeneralUtilities.check_arguments
|
|
11
32
|
def build(self) -> None:
|
|
12
|
-
|
|
33
|
+
maven_project_folder = self.get_maven_project_folder()
|
|
34
|
+
target_folder = os.path.join(self.get_artifacts_folder(), "BuildResult_Maven")
|
|
35
|
+
GeneralUtilities.ensure_directory_exists(target_folder)
|
|
36
|
+
# The testcases are executed by RunTestcases.py and not by the build, so that a failing testcase is reported
|
|
37
|
+
# by the quality-check and not by the build-step.
|
|
38
|
+
self.__run_maven("-B -DskipTests clean package")
|
|
39
|
+
build_output_folder = os.path.join(maven_project_folder, "target")
|
|
40
|
+
artifacts = [file for file in GeneralUtilities.get_direct_files_of_folder(build_output_folder) if file.endswith(".jar")]
|
|
41
|
+
GeneralUtilities.assert_condition(0 < len(artifacts), f'The maven-build did not produce a jar-file in "{build_output_folder}".')
|
|
42
|
+
for artifact in artifacts:
|
|
43
|
+
shutil.copyfile(artifact, os.path.join(target_folder, os.path.basename(artifact)))
|
|
44
|
+
self.__generate_bom_for_maven_project()
|
|
45
|
+
self.copy_source_files_to_output_directory()
|
|
46
|
+
|
|
47
|
+
@GeneralUtilities.check_arguments
|
|
48
|
+
def __generate_bom_for_maven_project(self) -> None:
|
|
49
|
+
codeunit_version = self.tfcps_Tools_General.get_version_of_codeunit(self.get_codeunit_file())
|
|
50
|
+
bom_folder = os.path.join(self.get_artifacts_folder(), "BOM")
|
|
51
|
+
GeneralUtilities.ensure_directory_exists(bom_folder)
|
|
52
|
+
self.__run_maven("-B org.cyclonedx:cyclonedx-maven-plugin:makeAggregateBom")
|
|
53
|
+
generated_bom_file = os.path.join(self.get_maven_project_folder(), "target", "bom.xml")
|
|
54
|
+
GeneralUtilities.assert_file_exists(generated_bom_file)
|
|
55
|
+
target_bom_file = os.path.join(bom_folder, f"{self.get_codeunit_name()}.{codeunit_version}.bom.xml")
|
|
56
|
+
shutil.copyfile(generated_bom_file, target_bom_file)
|
|
57
|
+
self._protected_sc.format_xml_file(target_bom_file)
|
|
13
58
|
|
|
14
59
|
@GeneralUtilities.check_arguments
|
|
15
60
|
def linting(self) -> None:
|
|
16
|
-
|
|
61
|
+
# checkstyle is configured in the pom-file of the codeunit, so that the ruleset belongs to the codeunit
|
|
62
|
+
# and not to this implementation.
|
|
63
|
+
self.__run_maven("-B checkstyle:check")
|
|
64
|
+
|
|
65
|
+
@GeneralUtilities.check_arguments
|
|
66
|
+
def do_common_tasks(self, current_codeunit_version: str) -> None:
|
|
67
|
+
self.do_common_tasks_base(current_codeunit_version)
|
|
68
|
+
self.__set_version_in_pom_file(self.get_version_of_project())
|
|
69
|
+
|
|
70
|
+
@GeneralUtilities.check_arguments
|
|
71
|
+
def generate_reference(self) -> None:
|
|
72
|
+
self.generate_reference_using_docfx()
|
|
17
73
|
|
|
18
74
|
@GeneralUtilities.check_arguments
|
|
19
75
|
def run_testcases(self) -> None:
|
|
20
|
-
|
|
76
|
+
codeunit_name = self.get_codeunit_name()
|
|
77
|
+
# "verify" is used instead of "test" because the report of jacoco is bound to that phase.
|
|
78
|
+
self.__run_maven("-B verify")
|
|
79
|
+
jacoco_file = os.path.join(self.get_maven_project_folder(), "target", "site", "jacoco", "jacoco.xml")
|
|
80
|
+
coverage_file = os.path.join(self.get_artifacts_folder(), "TestCoverage", "TestCoverage.xml")
|
|
81
|
+
GeneralUtilities.ensure_file_does_not_exist(coverage_file)
|
|
82
|
+
self.tfcps_Tools_General.convert_jacoco_report_to_cobertura_report(jacoco_file, coverage_file, codeunit_name, self.get_codeunit_folder(), [f"{codeunit_name}/src/main/java", f"{codeunit_name}/src/test/java"])
|
|
83
|
+
self.run_testcases_common_post_task(self.get_repository_folder(), codeunit_name, True, self.get_target_environment_type())
|
|
21
84
|
|
|
85
|
+
@GeneralUtilities.check_arguments
|
|
86
|
+
def __run_maven(self, arguments: str) -> None:
|
|
87
|
+
self._protected_sc.run_program("mvn", arguments, self.get_maven_project_folder(), print_live_output=self.get_verbosity() == LogLevel.Debug)
|
|
88
|
+
|
|
89
|
+
@GeneralUtilities.check_arguments
|
|
22
90
|
def get_dependencies(self) -> dict[str, set[str]]:
|
|
23
|
-
return
|
|
91
|
+
return GeneralUtilities.merge_dependency_lists([self.__get_dependencies_from_pom_file()])
|
|
92
|
+
|
|
93
|
+
@GeneralUtilities.check_arguments
|
|
94
|
+
def __get_dependencies_from_pom_file(self) -> list[Dependency]:
|
|
95
|
+
root: etree._ElementTree = etree.parse(self.get_pom_file())
|
|
96
|
+
namespaces = {"pom": TFCPS_CodeUnitSpecific_Maven_Functions.__pom_namespace}
|
|
97
|
+
result: list[Dependency] = []
|
|
98
|
+
for dependency in root.xpath("//pom:dependencies/pom:dependency", namespaces=namespaces):
|
|
99
|
+
group_id = dependency.xpath("./pom:groupId/text()", namespaces=namespaces)
|
|
100
|
+
artifact_id = dependency.xpath("./pom:artifactId/text()", namespaces=namespaces)
|
|
101
|
+
dependency_version = dependency.xpath("./pom:version/text()", namespaces=namespaces)
|
|
102
|
+
if len(group_id) == 0 or len(artifact_id) == 0:
|
|
103
|
+
continue
|
|
104
|
+
if len(dependency_version) == 0:
|
|
105
|
+
# The version of such a dependency comes from a parent-pom or from a dependency-management-section,
|
|
106
|
+
# so it can not be updated here.
|
|
107
|
+
continue
|
|
108
|
+
result.append(Dependency(f"{group_id[0]}:{artifact_id[0]}", str(dependency_version[0])))
|
|
109
|
+
return result
|
|
24
110
|
|
|
25
111
|
@GeneralUtilities.check_arguments
|
|
26
112
|
def get_available_versions(self, dependencyname: str) -> list[str]:
|
|
27
|
-
|
|
113
|
+
group_id, artifact_id = TFCPS_CodeUnitSpecific_Maven_Functions.__split_dependency_name(dependencyname)
|
|
114
|
+
# The maven-central-search-api answers with the released versions of an artifact, newest first.
|
|
115
|
+
response = requests.get("https://search.maven.org/solrsearch/select", params={"q": f'g:"{group_id}" AND a:"{artifact_id}"', "core": "gav", "rows": "200", "wt": "json"}, timeout=60)
|
|
116
|
+
response.raise_for_status()
|
|
117
|
+
result: list[str] = []
|
|
118
|
+
for found_artifact in response.json()["response"]["docs"]:
|
|
119
|
+
found_version = str(found_artifact["v"])
|
|
120
|
+
if re.match(r"^(\d+)\.(\d+)\.(\d+)$", found_version) is not None:
|
|
121
|
+
result.append(found_version)
|
|
122
|
+
elif re.match(r"^(\d+)\.(\d+)$", found_version) is not None:
|
|
123
|
+
result.append(found_version+".0")
|
|
124
|
+
elif re.match(r"^(\d+)$", found_version) is not None:
|
|
125
|
+
result.append(found_version+".0.0")
|
|
126
|
+
return result
|
|
28
127
|
|
|
29
128
|
@GeneralUtilities.check_arguments
|
|
30
129
|
def set_dependency_version(self, name: str, new_version: str) -> None:
|
|
31
|
-
|
|
130
|
+
group_id, artifact_id = TFCPS_CodeUnitSpecific_Maven_Functions.__split_dependency_name(name)
|
|
131
|
+
pom_file = self.get_pom_file()
|
|
132
|
+
root: etree._ElementTree = etree.parse(pom_file)
|
|
133
|
+
namespaces = {"pom": TFCPS_CodeUnitSpecific_Maven_Functions.__pom_namespace}
|
|
134
|
+
for dependency in root.xpath("//pom:dependencies/pom:dependency", namespaces=namespaces):
|
|
135
|
+
group_id_of_dependency = dependency.xpath("./pom:groupId/text()", namespaces=namespaces)
|
|
136
|
+
artifact_id_of_dependency = dependency.xpath("./pom:artifactId/text()", namespaces=namespaces)
|
|
137
|
+
version_element = dependency.xpath("./pom:version", namespaces=namespaces)
|
|
138
|
+
if len(group_id_of_dependency) == 0 or len(artifact_id_of_dependency) == 0 or len(version_element) == 0:
|
|
139
|
+
continue
|
|
140
|
+
if str(group_id_of_dependency[0]) == group_id and str(artifact_id_of_dependency[0]) == artifact_id:
|
|
141
|
+
version_element[0].text = new_version
|
|
142
|
+
root.write(pom_file, pretty_print=True, xml_declaration=True, encoding="UTF-8")
|
|
143
|
+
|
|
144
|
+
@GeneralUtilities.check_arguments
|
|
145
|
+
def __set_version_in_pom_file(self, new_version: str) -> None:
|
|
146
|
+
pom_file = self.get_pom_file()
|
|
147
|
+
root: etree._ElementTree = etree.parse(pom_file)
|
|
148
|
+
namespaces = {"pom": TFCPS_CodeUnitSpecific_Maven_Functions.__pom_namespace}
|
|
149
|
+
# Only the version of the project itself is set here. A version-element inside a dependency is not touched,
|
|
150
|
+
# which is why the version-element is taken from the project-element directly and not searched anywhere.
|
|
151
|
+
version_elements = root.xpath("/pom:project/pom:version", namespaces=namespaces)
|
|
152
|
+
GeneralUtilities.assert_condition(len(version_elements) == 1, f'The pom-file "{pom_file}" must contain exactly one version-element directly in its project-element.')
|
|
153
|
+
version_elements[0].text = new_version
|
|
154
|
+
root.write(pom_file, pretty_print=True, xml_declaration=True, encoding="UTF-8")
|
|
155
|
+
|
|
156
|
+
@staticmethod
|
|
157
|
+
@GeneralUtilities.check_arguments
|
|
158
|
+
def __split_dependency_name(dependencyname: str) -> tuple[str, str]:
|
|
159
|
+
"""A maven-dependency is identified by its group-id and its artifact-id, so both of them together are used as
|
|
160
|
+
dependency-name in the format "<group-id>:<artifact-id>"."""
|
|
161
|
+
parts = dependencyname.split(":")
|
|
162
|
+
GeneralUtilities.assert_condition(len(parts) == 2, f'The name of a maven-dependency must have the format "<group-id>:<artifact-id>" but was "{dependencyname}".')
|
|
163
|
+
return (parts[0], parts[1])
|
|
32
164
|
|
|
33
165
|
class TFCPS_CodeUnitSpecific_Maven_CLI:
|
|
34
166
|
|
|
@@ -18,6 +18,12 @@ class TFCPS_CodeUnitSpecific_NodeJS_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
18
18
|
|
|
19
19
|
@GeneralUtilities.check_arguments
|
|
20
20
|
def build(self) -> None:
|
|
21
|
+
# The sourcecode-artifact of a previous run is removed before the build, because it is a copy of the
|
|
22
|
+
# codeunit inside the codeunit and the build-tools do not know that it is only an artifact. An
|
|
23
|
+
# angular-library for example is built by ng-packagr, which searches the whole project-folder for
|
|
24
|
+
# "ng-package.json"-files to find the entry-points of the package, finds the copied one and aborts. The
|
|
25
|
+
# copy is written again at the end of this method, so nothing is lost.
|
|
26
|
+
GeneralUtilities.ensure_directory_does_not_exist(os.path.join(self.get_codeunit_folder(), "Other", "Artifacts", "SourceCode"))
|
|
21
27
|
self._protected_sc.run_with_epew("npm", "run build", self.get_codeunit_folder(),print_live_output=self.get_verbosity()==LogLevel.Debug,encode_argument_in_base64=True)
|
|
22
28
|
self.standardized_tasks_build_bom_for_node_project()
|
|
23
29
|
self.copy_source_files_to_output_directory()
|
|
@@ -25,13 +31,13 @@ class TFCPS_CodeUnitSpecific_NodeJS_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
25
31
|
@GeneralUtilities.check_arguments
|
|
26
32
|
def linting(self) -> None:
|
|
27
33
|
codeunit_folder = self.get_codeunit_folder()
|
|
28
|
-
self._protected_sc.
|
|
34
|
+
self._protected_sc.normalize_invisible_characters_of_files_in_folder(codeunit_folder, ["ts", "js", "css", "scss", "sass"])
|
|
29
35
|
# The OpenAPI-generator writes its '.openapi-generator/FILES' metadata-file with the host's native
|
|
30
36
|
# line-endings (CRLF on Windows, LF on Linux). Normalize it to LF like the other generated text-files
|
|
31
37
|
# so the committed file does not drift between build-hosts.
|
|
32
38
|
for openapi_generator_files_file in self._protected_sc.get_not_git_ignored_files_of_folder(codeunit_folder):
|
|
33
39
|
if os.path.basename(openapi_generator_files_file) == "FILES" and os.path.basename(os.path.dirname(openapi_generator_files_file)) == ".openapi-generator":
|
|
34
|
-
self._protected_sc.
|
|
40
|
+
self._protected_sc.normalize_invisible_characters(openapi_generator_files_file)
|
|
35
41
|
for html_file in self._protected_sc.get_not_git_ignored_files_of_folder(codeunit_folder, ".html"):
|
|
36
42
|
self._protected_sc.format_html_file(html_file, os.path.basename(html_file) == "index.html")
|
|
37
43
|
self._protected_sc.run_with_epew("npm", "run lint", codeunit_folder,print_live_output=self.get_verbosity()==LogLevel.Debug,encode_argument_in_base64=True)
|
|
@@ -20,10 +20,81 @@ class TFCPS_CodeUnitSpecific_Python_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
20
20
|
GeneralUtilities.ensure_directory_does_not_exist(os.path.join(self.get_codeunit_folder(),"build"))
|
|
21
21
|
GeneralUtilities.ensure_directory_does_not_exist(os.path.join(self.get_codeunit_folder(),f"{self.get_codeunit_name()}.egg-info"))
|
|
22
22
|
GeneralUtilities.ensure_directory_exists(target_directory)
|
|
23
|
+
self.__check_compilable()
|
|
23
24
|
self._protected_sc.run_program(GeneralUtilities.get_python_executable(), f"-m build --wheel --outdir {target_directory}", codeunit_folder,print_live_output=self.get_verbosity()==LogLevel.Debug)
|
|
24
25
|
self.generate_bom_for_python_project()
|
|
25
26
|
self.copy_source_files_to_output_directory()
|
|
26
27
|
|
|
28
|
+
@GeneralUtilities.check_arguments
|
|
29
|
+
def __check_compilable(self) -> None:
|
|
30
|
+
self.__ensure_sourcecode_is_compilable_using_python_compile()
|
|
31
|
+
self.__ensure_sourcecode_has_no_errors_using_pylint()
|
|
32
|
+
|
|
33
|
+
@GeneralUtilities.check_arguments
|
|
34
|
+
def __ensure_sourcecode_has_no_errors_using_pylint(self) -> None:
|
|
35
|
+
"""Checks the sourcecode which goes into the wheel with pylint and raises when pylint reports a message
|
|
36
|
+
of the category error or fatal.
|
|
37
|
+
|
|
38
|
+
This finds the defects which the compile-check can not find, because they are not syntax-errors: a call
|
|
39
|
+
which does not pass an argument for a parameter which has no default-value (E1120), a name which is not
|
|
40
|
+
defined (E0602) and an attribute which the accessed object does not have (E1101), for example.
|
|
41
|
+
|
|
42
|
+
Only these two categories are checked: a convention-, refactor- or warning-message is a matter of style
|
|
43
|
+
and belongs to the linting-task, while an error- or a fatal-message means the sourcecode is broken,
|
|
44
|
+
which must not result in a wheel at all."""
|
|
45
|
+
codeunit_name = self.get_codeunit_name()
|
|
46
|
+
# The check runs in an isolated folder which only contains this codeunit, for the same reason as in
|
|
47
|
+
# "linting": otherwise a sibling-codeunit-source-folder of the repository would shadow the installed
|
|
48
|
+
# dependency-package of the same name while pylint resolves the imports, which would result in a
|
|
49
|
+
# false-positive import-error.
|
|
50
|
+
ignored_subfolders = ["Other", "__pycache__", "*.egg-info", "build", "dist", "venv", ".venv", ".pytest_cache", ".git"]
|
|
51
|
+
with tempfile.TemporaryDirectory(dir=GeneralUtilities.get_temp_folder()) as isolation_folder:
|
|
52
|
+
GeneralUtilities.copy_content_of_folder(self.get_codeunit_folder(), os.path.join(isolation_folder, codeunit_name), ignored_glob_patterms=ignored_subfolders)
|
|
53
|
+
pylint_configuration_file = os.path.join(self.get_repository_folder(), ".pylintrc")
|
|
54
|
+
if os.path.isfile(pylint_configuration_file):
|
|
55
|
+
GeneralUtilities.safe_copy(pylint_configuration_file, os.path.join(isolation_folder, ".pylintrc"))
|
|
56
|
+
# "--errors-only" suppresses every message whose category is not error or fatal, so the exitcode of
|
|
57
|
+
# pylint is only unequal to zero when such a message was found.
|
|
58
|
+
(exit_code, stdout, stderr, _) = self._protected_sc.run_program("pylint", f"--errors-only {codeunit_name}/{codeunit_name}", isolation_folder, throw_exception_if_exitcode_is_not_zero=False)
|
|
59
|
+
pylint_exitcode_for_a_usage_error: int = 32
|
|
60
|
+
if exit_code == pylint_exitcode_for_a_usage_error:
|
|
61
|
+
raise ValueError(f"Pylint could not be executed properly, so the sourcecode of the codeunit {codeunit_name} was not checked:\n{stdout}{stderr}")
|
|
62
|
+
if exit_code != 0:
|
|
63
|
+
raise ValueError(f"Pylint reported a message of the category error or fatal in the sourcecode of the codeunit {codeunit_name}, so no wheel was built:\n{stdout}{stderr}")
|
|
64
|
+
|
|
65
|
+
@GeneralUtilities.check_arguments
|
|
66
|
+
def __ensure_sourcecode_is_compilable_using_python_compile(self) -> None:
|
|
67
|
+
"""Checks every python-file which goes into the wheel and raises when one of them can not be compiled.
|
|
68
|
+
|
|
69
|
+
This check exists so that a codeunit whose sourcecode is broken does not result in a wheel at all. It is
|
|
70
|
+
done with the builtin "compile"-function instead of with a linter: "compile" parses the file and reports
|
|
71
|
+
a syntax-error without executing anything of it, so the check needs no further tool and can not have
|
|
72
|
+
side-effects. Importing the modules instead would find more, but it would run the code of every module.
|
|
73
|
+
Everything which is a matter of style belongs to the linting-task and is deliberately not checked here."""
|
|
74
|
+
codeunit_name = self.get_codeunit_name()
|
|
75
|
+
repository_folder = self.get_repository_folder()
|
|
76
|
+
source_folder = os.path.join(self.get_codeunit_folder(), codeunit_name)
|
|
77
|
+
errors: list[str] = []
|
|
78
|
+
for file in GeneralUtilities.get_all_files_of_folder(source_folder):
|
|
79
|
+
if not file.endswith(".py"):
|
|
80
|
+
continue
|
|
81
|
+
if self._protected_sc.file_is_git_ignored(os.path.relpath(file, repository_folder), repository_folder):
|
|
82
|
+
continue
|
|
83
|
+
try:
|
|
84
|
+
# The file is read as "utf-8-sig" so that a byte-order-mark is removed instead of being handed to
|
|
85
|
+
# "compile" as the first character, which would report a syntax-error although python itself
|
|
86
|
+
# accepts a sourcecode-file which starts with a byte-order-mark.
|
|
87
|
+
# The filename is passed so that it appears in the message of a syntax-error.
|
|
88
|
+
compile(GeneralUtilities.read_text_from_file(file, "utf-8-sig"), file, "exec")
|
|
89
|
+
except SyntaxError as exception:
|
|
90
|
+
errors.append(f'"{file}" (line {exception.lineno}): {exception.msg}')
|
|
91
|
+
except ValueError as exception:
|
|
92
|
+
# "compile" raises a ValueError instead of a SyntaxError when the sourcecode contains a
|
|
93
|
+
# null-byte, which is not a syntax-error but makes the file just as unusable.
|
|
94
|
+
errors.append(f'"{file}": {exception}')
|
|
95
|
+
if 0 < len(errors):
|
|
96
|
+
raise ValueError(f"The sourcecode of the codeunit {codeunit_name} can not be compiled, so no wheel was built:\n" + "\n".join(errors))
|
|
97
|
+
|
|
27
98
|
@GeneralUtilities.check_arguments
|
|
28
99
|
def generate_bom_for_python_project(self) -> None:
|
|
29
100
|
codeunit_folder: str=self.get_codeunit_folder()
|
|
@@ -1,34 +1,196 @@
|
|
|
1
|
-
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
import shutil
|
|
4
|
+
import requests
|
|
5
|
+
from lxml import etree
|
|
6
|
+
from ...GeneralUtilities import GeneralUtilities, Dependency
|
|
2
7
|
from ...SCLog import LogLevel
|
|
3
8
|
from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base, TFCPS_CodeUnitSpecific_Base_CLI
|
|
4
9
|
|
|
5
10
|
class TFCPS_CodeUnitSpecific_Rust_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
11
|
+
"""Implements the tasks of a codeunit which contains a cargo-project.
|
|
12
|
+
|
|
13
|
+
The cargo-project is expected in "<codeunit>/<codeunit>" and it contains its testcases in the same
|
|
14
|
+
project, because that is the layout cargo defines: unit-tests live next to the code they test and
|
|
15
|
+
integration-tests live in the "tests"-folder of the project."""
|
|
6
16
|
|
|
7
17
|
def __init__(self, current_file: str, verbosity: LogLevel, targetenvironmenttype: str, use_cache: bool, is_pre_merge: bool):
|
|
8
18
|
super().__init__(current_file, verbosity, targetenvironmenttype, use_cache, is_pre_merge)
|
|
9
19
|
|
|
20
|
+
@GeneralUtilities.check_arguments
|
|
21
|
+
def get_cargo_project_folder(self) -> str:
|
|
22
|
+
"""Returns the folder which contains the Cargo.toml-file of this codeunit."""
|
|
23
|
+
return os.path.join(self.get_codeunit_folder(), self.get_codeunit_name())
|
|
24
|
+
|
|
25
|
+
@GeneralUtilities.check_arguments
|
|
26
|
+
def get_cargo_file(self) -> str:
|
|
27
|
+
return os.path.join(self.get_cargo_project_folder(), "Cargo.toml")
|
|
28
|
+
|
|
10
29
|
@GeneralUtilities.check_arguments
|
|
11
30
|
def build(self) -> None:
|
|
12
|
-
|
|
31
|
+
target_folder = os.path.join(self.get_artifacts_folder(), "BuildResult_Rust")
|
|
32
|
+
GeneralUtilities.ensure_directory_exists(target_folder)
|
|
33
|
+
self.__run_cargo("build --release")
|
|
34
|
+
# cargo puts the executable and the library of the release-profile directly into "target/release"; everything
|
|
35
|
+
# below that folder is intermediate build-output and is therefore not part of the artifact.
|
|
36
|
+
build_output_folder = os.path.join(self.get_cargo_project_folder(), "target", "release")
|
|
37
|
+
artifacts = [file for file in GeneralUtilities.get_direct_files_of_folder(build_output_folder) if TFCPS_CodeUnitSpecific_Rust_Functions.__file_is_build_artifact(file)]
|
|
38
|
+
GeneralUtilities.assert_condition(0 < len(artifacts), f'The cargo-build did not produce an artifact in "{build_output_folder}".')
|
|
39
|
+
for artifact in artifacts:
|
|
40
|
+
shutil.copyfile(artifact, os.path.join(target_folder, os.path.basename(artifact)))
|
|
41
|
+
self.__generate_bom_for_cargo_project()
|
|
42
|
+
self.copy_source_files_to_output_directory()
|
|
43
|
+
|
|
44
|
+
@staticmethod
|
|
45
|
+
@GeneralUtilities.check_arguments
|
|
46
|
+
def __file_is_build_artifact(file: str) -> bool:
|
|
47
|
+
# cargo writes further files (for example ".d"-dependency-files and the "*.pdb"-symbols on windows) next to the
|
|
48
|
+
# executable and the library, so the artifact is selected by its extension instead of taking everything.
|
|
49
|
+
extensions_of_artifacts = ("", ".exe", ".dll", ".so", ".dylib", ".a", ".lib", ".rlib")
|
|
50
|
+
return os.path.splitext(file)[1] in extensions_of_artifacts
|
|
51
|
+
|
|
52
|
+
@GeneralUtilities.check_arguments
|
|
53
|
+
def __generate_bom_for_cargo_project(self) -> None:
|
|
54
|
+
codeunit_version = self.tfcps_Tools_General.get_version_of_codeunit(self.get_codeunit_file())
|
|
55
|
+
bom_folder = os.path.join(self.get_artifacts_folder(), "BOM")
|
|
56
|
+
GeneralUtilities.ensure_directory_exists(bom_folder)
|
|
57
|
+
target_bom_file = os.path.join(bom_folder, f"{self.get_codeunit_name()}.{codeunit_version}.bom.xml")
|
|
58
|
+
self.__run_cargo("cyclonedx --format xml")
|
|
59
|
+
# cargo-cyclonedx writes its result into the folder of the cargo-project and derives the filename from the name
|
|
60
|
+
# of the crate, so the generated file is searched instead of being predicted from the name of the codeunit.
|
|
61
|
+
generated_bom_files = [file for file in GeneralUtilities.get_direct_files_of_folder(self.get_cargo_project_folder()) if file.endswith(".xml")]
|
|
62
|
+
GeneralUtilities.assert_condition(len(generated_bom_files) == 1, f'Expected exactly one generated bill-of-materials in "{self.get_cargo_project_folder()}" but found {len(generated_bom_files)}.')
|
|
63
|
+
shutil.move(generated_bom_files[0], target_bom_file)
|
|
64
|
+
self._protected_sc.format_xml_file(target_bom_file)
|
|
13
65
|
|
|
14
66
|
@GeneralUtilities.check_arguments
|
|
15
67
|
def linting(self) -> None:
|
|
16
|
-
|
|
68
|
+
# "--all-targets" includes the testcases in the check, and "-D warnings" makes clippy exit with a non-zero
|
|
69
|
+
# exit-code for a warning as well, because a warning is a linting-issue too.
|
|
70
|
+
self.__run_cargo("clippy --all-targets --all-features -- -D warnings")
|
|
71
|
+
|
|
72
|
+
@GeneralUtilities.check_arguments
|
|
73
|
+
def do_common_tasks(self, current_codeunit_version: str) -> None:
|
|
74
|
+
self.do_common_tasks_base(current_codeunit_version)
|
|
75
|
+
self.__set_version_in_cargo_file(self.get_version_of_project())
|
|
76
|
+
|
|
77
|
+
@GeneralUtilities.check_arguments
|
|
78
|
+
def generate_reference(self) -> None:
|
|
79
|
+
self.generate_reference_using_docfx()
|
|
17
80
|
|
|
18
81
|
@GeneralUtilities.check_arguments
|
|
19
82
|
def run_testcases(self) -> None:
|
|
20
|
-
|
|
83
|
+
codeunit_name = self.get_codeunit_name()
|
|
84
|
+
coverage_file = os.path.join(self.get_artifacts_folder(), "TestCoverage", "TestCoverage.xml")
|
|
85
|
+
GeneralUtilities.ensure_directory_exists(os.path.dirname(coverage_file))
|
|
86
|
+
GeneralUtilities.ensure_file_does_not_exist(coverage_file)
|
|
87
|
+
# cargo-llvm-cov runs the testcases and writes the coverage-report in the cobertura-format directly, so no
|
|
88
|
+
# conversion of the report-format is required here.
|
|
89
|
+
self.__run_cargo(f"llvm-cov --all-features --cobertura --output-path {coverage_file}")
|
|
90
|
+
GeneralUtilities.assert_file_exists(coverage_file)
|
|
91
|
+
self.__normalize_coverage_file(coverage_file)
|
|
92
|
+
self.run_testcases_common_post_task(self.get_repository_folder(), codeunit_name, True, self.get_target_environment_type())
|
|
21
93
|
|
|
94
|
+
@GeneralUtilities.check_arguments
|
|
95
|
+
def __normalize_coverage_file(self, coverage_file: str) -> None:
|
|
96
|
+
"""Makes the coverage-report which cargo-llvm-cov wrote match the expectations of the further processing:
|
|
97
|
+
it must contain exactly one package which is named like the codeunit, and the filename of a class must be
|
|
98
|
+
relative to the codeunit-folder (cargo-llvm-cov writes it relative to the cargo-project-folder)."""
|
|
99
|
+
codeunit_name = self.get_codeunit_name()
|
|
100
|
+
tree = etree.parse(coverage_file)
|
|
101
|
+
root = tree.getroot()
|
|
102
|
+
for package in root.findall(".//package"):
|
|
103
|
+
package.set("name", codeunit_name)
|
|
104
|
+
for class_element in root.findall(".//class"):
|
|
105
|
+
filename = class_element.get("filename")
|
|
106
|
+
if filename is not None:
|
|
107
|
+
filename_with_slashes = filename.replace("\\", "/")
|
|
108
|
+
class_element.set("filename", f"{codeunit_name}/{filename_with_slashes}")
|
|
109
|
+
class_element.set("name", os.path.basename(filename_with_slashes))
|
|
110
|
+
tree.write(coverage_file, pretty_print=True, xml_declaration=True, encoding="UTF-8")
|
|
111
|
+
self.tfcps_Tools_General.merge_packages(coverage_file, codeunit_name)
|
|
112
|
+
|
|
113
|
+
@GeneralUtilities.check_arguments
|
|
114
|
+
def __run_cargo(self, arguments: str) -> None:
|
|
115
|
+
self._protected_sc.run_program("cargo", arguments, self.get_cargo_project_folder(), print_live_output=self.get_verbosity() == LogLevel.Debug)
|
|
116
|
+
|
|
117
|
+
@GeneralUtilities.check_arguments
|
|
22
118
|
def get_dependencies(self) -> dict[str, set[str]]:
|
|
23
|
-
return
|
|
119
|
+
return GeneralUtilities.merge_dependency_lists([self.__get_dependencies_from_cargo_file()])
|
|
120
|
+
|
|
121
|
+
@GeneralUtilities.check_arguments
|
|
122
|
+
def __get_dependencies_from_cargo_file(self) -> list[Dependency]:
|
|
123
|
+
result: list[Dependency] = []
|
|
124
|
+
for section, name, dependency_version in self.__enumerate_dependency_entries():
|
|
125
|
+
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(section), "Internal error while reading the dependencies.")
|
|
126
|
+
result.append(Dependency(name, dependency_version))
|
|
127
|
+
return result
|
|
128
|
+
|
|
129
|
+
@GeneralUtilities.check_arguments
|
|
130
|
+
def __enumerate_dependency_entries(self) -> list[tuple[str, str, str]]:
|
|
131
|
+
"""Returns the dependencies of the cargo-file as tuples of the section they are declared in, their name and
|
|
132
|
+
their version. Only a dependency which declares a plain version-string is returned, because a dependency which
|
|
133
|
+
is declared as a table (for example a path- or git-dependency) has no version which could be updated."""
|
|
134
|
+
dependency_sections = ("dependencies", "dev-dependencies", "build-dependencies")
|
|
135
|
+
current_section: str = None
|
|
136
|
+
result: list[tuple[str, str, str]] = []
|
|
137
|
+
for line in GeneralUtilities.read_lines_from_file(self.get_cargo_file()):
|
|
138
|
+
stripped_line = line.strip()
|
|
139
|
+
if stripped_line.startswith("[") and stripped_line.endswith("]"):
|
|
140
|
+
current_section = stripped_line[1:-1].strip()
|
|
141
|
+
continue
|
|
142
|
+
if current_section not in dependency_sections or not GeneralUtilities.string_has_content(stripped_line) or stripped_line.startswith("#"):
|
|
143
|
+
continue
|
|
144
|
+
match = re.match(r'^([A-Za-z0-9_\-]+)\s*=\s*"([^"]+)"$', stripped_line)
|
|
145
|
+
if match is not None:
|
|
146
|
+
result.append((current_section, match.group(1), match.group(2)))
|
|
147
|
+
return result
|
|
24
148
|
|
|
25
149
|
@GeneralUtilities.check_arguments
|
|
26
150
|
def get_available_versions(self, dependencyname: str) -> list[str]:
|
|
27
|
-
|
|
151
|
+
# The crates.io-api answers with all published versions of a crate.
|
|
152
|
+
response = requests.get(f"https://crates.io/api/v1/crates/{dependencyname}/versions", headers={"User-Agent": "ScriptCollection"}, timeout=60)
|
|
153
|
+
response.raise_for_status()
|
|
154
|
+
result: list[str] = []
|
|
155
|
+
for found_version in response.json()["versions"]:
|
|
156
|
+
if found_version.get("yanked", False):
|
|
157
|
+
continue
|
|
158
|
+
version_string = str(found_version["num"])
|
|
159
|
+
if re.match(r"^(\d+)\.(\d+)\.(\d+)$", version_string) is not None:
|
|
160
|
+
result.append(version_string)
|
|
161
|
+
return result
|
|
28
162
|
|
|
29
163
|
@GeneralUtilities.check_arguments
|
|
30
164
|
def set_dependency_version(self, name: str, new_version: str) -> None:
|
|
31
|
-
|
|
165
|
+
cargo_file = self.get_cargo_file()
|
|
166
|
+
new_lines: list[str] = []
|
|
167
|
+
for line in GeneralUtilities.read_lines_from_file(cargo_file):
|
|
168
|
+
match = re.match(r'^(\s*)('+re.escape(name)+r')(\s*=\s*)"([^"]+)"(\s*)$', line)
|
|
169
|
+
if match is None:
|
|
170
|
+
new_lines.append(line)
|
|
171
|
+
else:
|
|
172
|
+
new_lines.append(f'{match.group(1)}{match.group(2)}{match.group(3)}"{new_version}"')
|
|
173
|
+
GeneralUtilities.write_lines_to_file(cargo_file, new_lines)
|
|
174
|
+
|
|
175
|
+
@GeneralUtilities.check_arguments
|
|
176
|
+
def __set_version_in_cargo_file(self, new_version: str) -> None:
|
|
177
|
+
cargo_file = self.get_cargo_file()
|
|
178
|
+
new_lines: list[str] = []
|
|
179
|
+
is_in_package_section: bool = False
|
|
180
|
+
version_was_set: bool = False
|
|
181
|
+
for line in GeneralUtilities.read_lines_from_file(cargo_file):
|
|
182
|
+
stripped_line = line.strip()
|
|
183
|
+
if stripped_line.startswith("[") and stripped_line.endswith("]"):
|
|
184
|
+
is_in_package_section = stripped_line[1:-1].strip() == "package"
|
|
185
|
+
# Only the version in the package-section is the version of the project itself; a version in a
|
|
186
|
+
# dependency-section belongs to a dependency and must not be changed here.
|
|
187
|
+
if is_in_package_section and re.match(r'^version\s*=\s*"[^"]+"$', stripped_line) is not None:
|
|
188
|
+
new_lines.append(f'version = "{new_version}"')
|
|
189
|
+
version_was_set = True
|
|
190
|
+
else:
|
|
191
|
+
new_lines.append(line)
|
|
192
|
+
GeneralUtilities.assert_condition(version_was_set, f'The cargo-file "{cargo_file}" does not contain a version in its package-section.')
|
|
193
|
+
GeneralUtilities.write_lines_to_file(cargo_file, new_lines)
|
|
32
194
|
|
|
33
195
|
class TFCPS_CodeUnitSpecific_Rust_CLI:
|
|
34
196
|
|