ScriptCollection 4.4.71__py3-none-any.whl → 4.4.73__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/Executables.py +22 -3
- ScriptCollection/GeneralUtilities.py +42 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/AccountChecks.py +71 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/ContainerChecks.py +34 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/FileSystemChecks.py +141 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/KernelChecks.py +49 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/LogChecks.py +39 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/NetworkChecks.py +67 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/PackageIntegrityChecks.py +62 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/PersistenceChecks.py +99 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/ProcessChecks.py +88 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/SSHChecks.py +56 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/__init__.py +0 -0
- ScriptCollection/LinuxServerSecurityCheck/LinuxServerSecurityChecker.py +90 -0
- ScriptCollection/LinuxServerSecurityCheck/LinuxSystemAccess.py +85 -0
- ScriptCollection/LinuxServerSecurityCheck/PackageOwnershipLookup.py +42 -0
- ScriptCollection/LinuxServerSecurityCheck/SecurityCheckBase.py +19 -0
- ScriptCollection/LinuxServerSecurityCheck/SecurityFinding.py +19 -0
- ScriptCollection/LinuxServerSecurityCheck/SuspiciousCommandPatterns.py +46 -0
- ScriptCollection/LinuxServerSecurityCheck/SystemAccessBase.py +244 -0
- ScriptCollection/LinuxServerSecurityCheck/SystemEntities.py +45 -0
- ScriptCollection/LinuxServerSecurityCheck/__init__.py +0 -0
- ScriptCollection/OCIImages/AbstractImageHandler.py +14 -26
- ScriptCollection/OCIImages/AbstractTagPatternImageHandler.py +46 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py +5 -14
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py +5 -14
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGeneric.py +6 -15
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGenericV.py +6 -15
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabCE.py +7 -8
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabEE.py +7 -8
- ScriptCollection/OCIImages/OCIImageManager.py +73 -68
- ScriptCollection/OCIImages/OCIRegistryClient.py +117 -0
- ScriptCollection/ScriptCollectionCore.py +9 -4
- ScriptCollection/TFCPS/Android/TFCPS_CodeUnitSpecific_Android.py +479 -0
- ScriptCollection/TFCPS/Android/__init__.py +0 -0
- ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py +6 -5
- ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +38 -4
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +4 -4
- ScriptCollection/TFCPS/Maven/TFCPS_CodeUnitSpecific_Maven.py +1 -0
- ScriptCollection/TFCPS/Rust/CargoLockFile.py +151 -0
- ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py +5 -0
- ScriptCollection/TFCPS/SCTaskRunnerServer.py +55 -9
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +6 -2
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +14 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +19 -0
- ScriptCollection/TFCPS/TFCPS_RemoteBuild.py +39 -11
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +34 -2
- {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.73.dist-info}/METADATA +3 -3
- {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.73.dist-info}/RECORD +52 -27
- {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.73.dist-info}/entry_points.txt +1 -0
- {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.73.dist-info}/WHEEL +0 -0
- {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.73.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,479 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
import base64
|
|
4
|
+
import binascii
|
|
5
|
+
import zipfile
|
|
6
|
+
import shlex
|
|
7
|
+
import shutil
|
|
8
|
+
import requests
|
|
9
|
+
from lxml import etree
|
|
10
|
+
from ...GeneralUtilities import GeneralUtilities, Dependency
|
|
11
|
+
from ...SCLog import LogLevel
|
|
12
|
+
from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base, TFCPS_CodeUnitSpecific_Base_CLI
|
|
13
|
+
from ..TFCPS_RemoteBuild import RunnerOperatingSystem
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class TFCPS_CodeUnitSpecific_Android_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
17
|
+
"""Implements the tasks of a codeunit which contains a native android-app (java or kotlin) built with gradle.
|
|
18
|
+
|
|
19
|
+
The gradle-project is expected in "<codeunit>/<codeunit>" as a single-module-project: its "build.gradle" applies the
|
|
20
|
+
android-application-plugin itself and its testcases are located in its own "src/test"-folder, because that is the
|
|
21
|
+
layout gradle defines and the layout the coverage-report of the android-gradle-plugin measures without further
|
|
22
|
+
configuration. The project has to contain the gradle-wrapper ("gradlew" and "gradle/wrapper"), so the gradle-version
|
|
23
|
+
is pinned by the repository and not by the machine which builds it.
|
|
24
|
+
|
|
25
|
+
Every gradle-task of the project needs the android-sdk. It is therefore always run on an android-task-runner (see
|
|
26
|
+
SCTaskRunnerAndroid and the remote-build-article in the reference), analogous to the android-target of a
|
|
27
|
+
flutter-codeunit: the android-sdk is not part of SCBuilder.
|
|
28
|
+
|
|
29
|
+
Requirements for the "build.gradle" of the project:
|
|
30
|
+
- It declares exactly one "versionCode" and exactly one "versionName" (they are set by do_common_tasks).
|
|
31
|
+
- The debug-build-type sets "enableUnitTestCoverage = true", so the task "createDebugUnitTestCoverageReport" exists.
|
|
32
|
+
- It applies the plugin "org.cyclonedx.bom" and lets the task "cyclonedxBom" write its xml-output to
|
|
33
|
+
"build/outputs/bom/bom.xml" (see get_bom_file_relative_to_build_folder), because only the "build/outputs"-folder is
|
|
34
|
+
transferred back from the runner by the build.
|
|
35
|
+
- Only if the codeunit has visual-regression-tests (see run_visual_regression_tests): it applies the plugin
|
|
36
|
+
"io.github.takahirom.roborazzi" and configures "roborazzi { outputDir = file('../Other/Resources/VisualRegressionBaselines');
|
|
37
|
+
compare { outputDir = layout.buildDirectory.dir('outputs/roborazzi-comparison') } }" (see
|
|
38
|
+
get_visual_regression_baselines_folder and get_visual_regression_comparison_folder_relative_to_build_folder).
|
|
39
|
+
- The release-build-type is signed with the upload-key if the environment-variables "SC_ANDROID_SIGNING_STORE_FILE",
|
|
40
|
+
"SC_ANDROID_SIGNING_STORE_PASSWORD", "SC_ANDROID_SIGNING_KEY_ALIAS" and "SC_ANDROID_SIGNING_KEY_PASSWORD" are set,
|
|
41
|
+
and with the debug-key otherwise (see build), for example:
|
|
42
|
+
"signingConfigs { release { if (System.getenv('SC_ANDROID_SIGNING_STORE_FILE') != null) { storeFile =
|
|
43
|
+
file(System.getenv('SC_ANDROID_SIGNING_STORE_FILE')); storePassword = System.getenv('SC_ANDROID_SIGNING_STORE_PASSWORD');
|
|
44
|
+
keyAlias = System.getenv('SC_ANDROID_SIGNING_KEY_ALIAS'); keyPassword = System.getenv('SC_ANDROID_SIGNING_KEY_PASSWORD') } } }"
|
|
45
|
+
and "release { signingConfig = System.getenv('SC_ANDROID_SIGNING_STORE_FILE') != null ? signingConfigs.release :
|
|
46
|
+
signingConfigs.debug }"."""
|
|
47
|
+
|
|
48
|
+
# The environment-variables (of the machine which runs the build, for example the build-pipeline) which contain the
|
|
49
|
+
# upload-key of a productive build, mapped to the environment-variables of the gradle-run on the runner which
|
|
50
|
+
# receive them (see build). The keystore is given as the base64-encoding of the keystore-file, because an
|
|
51
|
+
# environment-variable can not contain a file.
|
|
52
|
+
__signing_environment_variables: dict[str, str] = {
|
|
53
|
+
"Android_Signing_Keystore": "SC_ANDROID_SIGNING_KEYSTORE_BASE64",
|
|
54
|
+
"Android_Signing_StorePassword": "SC_ANDROID_SIGNING_STORE_PASSWORD",
|
|
55
|
+
"Android_Signing_KeyAlias": "SC_ANDROID_SIGNING_KEY_ALIAS",
|
|
56
|
+
"Android_Signing_KeyPassword": "SC_ANDROID_SIGNING_KEY_PASSWORD",
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
# The name the subject of the debug-certificate of the android-sdk contains ("CN=Android Debug,O=Android,C=US").
|
|
60
|
+
__subject_of_the_debug_certificate: bytes = b"Android Debug"
|
|
61
|
+
|
|
62
|
+
# The repositories in which a dependency or a gradle-plugin of an android-app is published. Gradle itself searches
|
|
63
|
+
# all of them (they are the repositories a project declares by "google()", "mavenCentral()" and
|
|
64
|
+
# "gradlePluginPortal()"), so a dependency can be located in any of them.
|
|
65
|
+
__maven_repositories: list[str] = ["https://dl.google.com/dl/android/maven2", "https://repo1.maven.org/maven2", "https://plugins.gradle.org/m2"]
|
|
66
|
+
|
|
67
|
+
# A dependency in the format "<group-id>:<artifact-id>:<version>" inside a string-literal of the build-file.
|
|
68
|
+
__dependency_pattern = re.compile(r"""(?P<quote>['"])(?P<group>[A-Za-z0-9_.\-]+):(?P<artifact>[A-Za-z0-9_.\-]+):(?P<version>[A-Za-z0-9_.\-+]+)(?P=quote)""")
|
|
69
|
+
|
|
70
|
+
# A plugin which is declared with its version in the plugins-block of the build-file, for example
|
|
71
|
+
# "id 'com.android.application' version '9.1.0'".
|
|
72
|
+
__plugin_pattern = re.compile(r"""id\s*\(?\s*(?P<quote1>['"])(?P<id>[A-Za-z0-9_.\-]+)(?P=quote1)\s*\)?\s*version\s*\(?\s*(?P<quote2>['"])(?P<version>[A-Za-z0-9_.\-+]+)(?P=quote2)""")
|
|
73
|
+
|
|
74
|
+
# The folder of the codeunit which contains the baseline-images of the visual-regression-tests. It is the same
|
|
75
|
+
# folder the visual-regression-tests of the other codeunit-types use (see TFCPS_VisualRegressionTests).
|
|
76
|
+
__visual_regression_baselines_folder_relative_to_codeunit: str = "Other/Resources/VisualRegressionBaselines"
|
|
77
|
+
|
|
78
|
+
# The folder the screenshots of failed visual-regression-tests (the actual image and the comparison-image) are
|
|
79
|
+
# preserved in. It is the same folder the visual-regression-tests of the other codeunit-types use.
|
|
80
|
+
__visual_regression_failures_folder_relative_to_codeunit: str = "Other/Artifacts/VisualRegressionTestFailures"
|
|
81
|
+
|
|
82
|
+
# The file (inside the comparison-folder) the exit-code of the verifying gradle-run is written to on the runner.
|
|
83
|
+
__exit_code_file_name: str = "GradleExitCode.txt"
|
|
84
|
+
|
|
85
|
+
__version_code_pattern = re.compile(r"(?P<prefix>\bversionCode\s*=?\s*)(?P<value>\d+)")
|
|
86
|
+
__version_name_pattern = re.compile(r"""(?P<prefix>\bversionName\s*=?\s*)(?P<quote>['"])(?P<value>[^'"]*)(?P=quote)""")
|
|
87
|
+
|
|
88
|
+
def __init__(self, current_file: str, verbosity: LogLevel, targetenvironmenttype: str, use_cache: bool, is_pre_merge: bool):
|
|
89
|
+
super().__init__(current_file, verbosity, targetenvironmenttype, use_cache, is_pre_merge)
|
|
90
|
+
|
|
91
|
+
@GeneralUtilities.check_arguments
|
|
92
|
+
def get_gradle_project_folder(self) -> str:
|
|
93
|
+
"""Returns the folder which contains the build-file of this codeunit."""
|
|
94
|
+
return os.path.join(self.get_codeunit_folder(), self.get_codeunit_name())
|
|
95
|
+
|
|
96
|
+
@GeneralUtilities.check_arguments
|
|
97
|
+
def get_build_file(self) -> str:
|
|
98
|
+
return os.path.join(self.get_gradle_project_folder(), "build.gradle")
|
|
99
|
+
|
|
100
|
+
@GeneralUtilities.check_arguments
|
|
101
|
+
def get_build_folder(self) -> str:
|
|
102
|
+
return os.path.join(self.get_gradle_project_folder(), "build")
|
|
103
|
+
|
|
104
|
+
@staticmethod
|
|
105
|
+
@GeneralUtilities.check_arguments
|
|
106
|
+
def get_bom_file_relative_to_build_folder() -> str:
|
|
107
|
+
return "outputs/bom/bom.xml"
|
|
108
|
+
|
|
109
|
+
@GeneralUtilities.check_arguments
|
|
110
|
+
def get_visual_regression_baselines_folder(self) -> str:
|
|
111
|
+
return os.path.join(self.get_codeunit_folder(), TFCPS_CodeUnitSpecific_Android_Functions.__visual_regression_baselines_folder_relative_to_codeunit)
|
|
112
|
+
|
|
113
|
+
@staticmethod
|
|
114
|
+
@GeneralUtilities.check_arguments
|
|
115
|
+
def get_visual_regression_comparison_folder_relative_to_build_folder() -> str:
|
|
116
|
+
return "outputs/roborazzi-comparison"
|
|
117
|
+
|
|
118
|
+
@GeneralUtilities.check_arguments
|
|
119
|
+
def build(self) -> None:
|
|
120
|
+
"""Builds the app-bundle (which is what is uploaded to a store) and the apk (which can be installed directly) of the
|
|
121
|
+
release-build-type and generates the bill-of-materials. All of them are generated by one gradle-run, so the runner
|
|
122
|
+
has to be used only once for them.
|
|
123
|
+
|
|
124
|
+
A productive build (target-environment-type "Productive") is signed with the upload-key, so its app-bundle can be
|
|
125
|
+
uploaded to Google Play. The upload-key is taken from the environment-variables "Android_Signing_Keystore" (the
|
|
126
|
+
base64-encoding of the keystore-file), "Android_Signing_StorePassword", "Android_Signing_KeyAlias" and
|
|
127
|
+
"Android_Signing_KeyPassword", which are required for a productive build. They are given to the gradle-run on the
|
|
128
|
+
runner as environment-variables only (see TFCPS_RemoteBuild.run_program_on_runner); on the runner the keystore
|
|
129
|
+
is written into a temporary file outside of the workspace which is removed when gradle ends. Afterwards it is
|
|
130
|
+
verified that the app-bundle is signed, but not with the debug-key.
|
|
131
|
+
Every other build is signed with the debug-key of the runner, so a developer can build and install the app
|
|
132
|
+
without the upload-key."""
|
|
133
|
+
codeunit_name = self.get_codeunit_name()
|
|
134
|
+
codeunit_version: str = self.tfcps_Tools_General.get_version_of_codeunit(self.get_codeunit_file())
|
|
135
|
+
outputs_folder = os.path.join(self.get_build_folder(), "outputs")
|
|
136
|
+
GeneralUtilities.ensure_directory_does_not_exist(outputs_folder)
|
|
137
|
+
gradle_tasks = ["bundleRelease", "assembleRelease", "cyclonedxBom"]
|
|
138
|
+
is_signed_with_upload_key = self.get_target_environment_type() == "Productive"
|
|
139
|
+
if is_signed_with_upload_key:
|
|
140
|
+
self._protected_sc.log.log("Build the release-build signed with the upload-key...")
|
|
141
|
+
signing_environment_variables = TFCPS_CodeUnitSpecific_Android_Functions.get_signing_environment_variables_for_runner(dict(os.environ))
|
|
142
|
+
script = TFCPS_CodeUnitSpecific_Android_Functions.get_script_which_signs_with_upload_key(self.__get_gradle_arguments(gradle_tasks))
|
|
143
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", ["-c", script], self.get_gradle_project_folder(), outputs_folder, self.__get_folders_which_are_not_transferred(), signing_environment_variables)
|
|
144
|
+
else:
|
|
145
|
+
self._protected_sc.log.log(f'Build the release-build signed with the debug-key of the runner (only a build with target-environment-type "Productive" is signed with the upload-key, this one has "{self.get_target_environment_type()}")...')
|
|
146
|
+
self.__run_gradle_on_runner(gradle_tasks, outputs_folder)
|
|
147
|
+
|
|
148
|
+
aab_folder = os.path.join(self.get_artifacts_folder(), "BuildResult_AAB")
|
|
149
|
+
GeneralUtilities.ensure_folder_exists_and_is_empty(aab_folder)
|
|
150
|
+
aab_file_original = self._protected_sc.find_file_by_extension(os.path.join(outputs_folder, "bundle", "release"), "aab")
|
|
151
|
+
if is_signed_with_upload_key:
|
|
152
|
+
GeneralUtilities.assert_condition(TFCPS_CodeUnitSpecific_Android_Functions.is_signed(aab_file_original), f'The app-bundle "{aab_file_original}" is not signed. The release-build-type of "{self.get_build_file()}" has to use the upload-key which is given by the environment-variables "SC_ANDROID_SIGNING_*" (see the docstring of TFCPS_CodeUnitSpecific_Android_Functions).')
|
|
153
|
+
GeneralUtilities.assert_condition(not TFCPS_CodeUnitSpecific_Android_Functions.is_signed_with_debug_key(aab_file_original), f'The app-bundle "{aab_file_original}" is signed with the debug-key, so Google Play would reject it. The release-build-type of "{self.get_build_file()}" has to use the upload-key which is given by the environment-variables "SC_ANDROID_SIGNING_*" (see the docstring of TFCPS_CodeUnitSpecific_Android_Functions).')
|
|
154
|
+
shutil.copyfile(aab_file_original, os.path.join(aab_folder, f"{codeunit_name}.aab"))
|
|
155
|
+
|
|
156
|
+
apk_folder = os.path.join(self.get_artifacts_folder(), "BuildResult_APK")
|
|
157
|
+
GeneralUtilities.ensure_folder_exists_and_is_empty(apk_folder)
|
|
158
|
+
apk_file_original = self._protected_sc.find_file_by_extension(os.path.join(outputs_folder, "apk", "release"), "apk")
|
|
159
|
+
shutil.copyfile(apk_file_original, os.path.join(apk_folder, f"{codeunit_name}_v{codeunit_version}.apk"))
|
|
160
|
+
|
|
161
|
+
bom_folder = os.path.join(self.get_artifacts_folder(), "BOM")
|
|
162
|
+
GeneralUtilities.ensure_directory_exists(bom_folder)
|
|
163
|
+
generated_bom_file = os.path.join(self.get_build_folder(), TFCPS_CodeUnitSpecific_Android_Functions.get_bom_file_relative_to_build_folder())
|
|
164
|
+
GeneralUtilities.assert_file_exists(generated_bom_file)
|
|
165
|
+
target_bom_file = os.path.join(bom_folder, f"{codeunit_name}.{codeunit_version}.bom.xml")
|
|
166
|
+
shutil.copyfile(generated_bom_file, target_bom_file)
|
|
167
|
+
self._protected_sc.format_xml_file(target_bom_file)
|
|
168
|
+
|
|
169
|
+
self.copy_source_files_to_output_directory()
|
|
170
|
+
|
|
171
|
+
@GeneralUtilities.check_arguments
|
|
172
|
+
def linting(self) -> None:
|
|
173
|
+
self._protected_sc.normalize_invisible_characters_of_files_in_folder(self.get_codeunit_folder(), ["java", "kt"])
|
|
174
|
+
# The rules of android-lint (for example whether a warning lets the linting fail) are configured in the "lint"-block
|
|
175
|
+
# of the build-file, so that the ruleset belongs to the codeunit and not to this implementation.
|
|
176
|
+
reports_folder = os.path.join(self.get_build_folder(), "reports")
|
|
177
|
+
GeneralUtilities.ensure_directory_does_not_exist(reports_folder)
|
|
178
|
+
self.__run_gradle_on_runner(["lintDebug"], reports_folder)
|
|
179
|
+
|
|
180
|
+
@GeneralUtilities.check_arguments
|
|
181
|
+
def do_common_tasks(self, current_codeunit_version: str) -> None:
|
|
182
|
+
self.do_common_tasks_base(current_codeunit_version)
|
|
183
|
+
self.__set_version_in_build_file(self.get_version_of_project())
|
|
184
|
+
|
|
185
|
+
@GeneralUtilities.check_arguments
|
|
186
|
+
def generate_reference(self) -> None:
|
|
187
|
+
self.generate_reference_using_docfx()
|
|
188
|
+
|
|
189
|
+
@GeneralUtilities.check_arguments
|
|
190
|
+
def run_testcases(self) -> None:
|
|
191
|
+
codeunit_name = self.get_codeunit_name()
|
|
192
|
+
reports_folder = os.path.join(self.get_build_folder(), "reports")
|
|
193
|
+
GeneralUtilities.ensure_directory_does_not_exist(reports_folder)
|
|
194
|
+
# "createDebugUnitTestCoverageReport" depends on "testDebugUnitTest", so this runs the testcases and writes the
|
|
195
|
+
# jacoco-report of their coverage.
|
|
196
|
+
self.__run_gradle_on_runner(["createDebugUnitTestCoverageReport"], reports_folder)
|
|
197
|
+
jacoco_file = os.path.join(reports_folder, "coverage", "test", "debug", "report.xml")
|
|
198
|
+
coverage_file = os.path.join(self.get_artifacts_folder(), "TestCoverage", "TestCoverage.xml")
|
|
199
|
+
GeneralUtilities.ensure_file_does_not_exist(coverage_file)
|
|
200
|
+
self.tfcps_Tools_General.convert_jacoco_report_to_cobertura_report(jacoco_file, coverage_file, codeunit_name, self.get_codeunit_folder(), [f"{codeunit_name}/src/main/java", f"{codeunit_name}/src/main/kotlin"])
|
|
201
|
+
self.run_testcases_common_post_task(self.get_repository_folder(), codeunit_name, True, self.get_target_environment_type())
|
|
202
|
+
|
|
203
|
+
@GeneralUtilities.check_arguments
|
|
204
|
+
def run_visual_regression_tests(self, update_baselines: bool) -> None:
|
|
205
|
+
"""Runs the visual-regression-tests of the app, which are the unit-tests that capture screenshots with roborazzi
|
|
206
|
+
(rendered by robolectric). They run on the android-task-runner like every other gradle-task of the project, so
|
|
207
|
+
the baseline-images are always generated and compared in the same, fixed environment, regardless of whether the
|
|
208
|
+
build runs on a developer-machine or in the build-pipeline: a screenshot depends on the environment which
|
|
209
|
+
renders it.
|
|
210
|
+
|
|
211
|
+
If 'update_baselines' is true, the baseline-images in "Other/Resources/VisualRegressionBaselines" are
|
|
212
|
+
(re-)generated ("recordRoborazziDebug"); the folder is emptied before, so the baseline-image of a removed
|
|
213
|
+
screenshot does not remain. Otherwise the screenshots are compared against them ("verifyRoborazziDebug"); if a
|
|
214
|
+
comparison fails, the actual images and the comparison-images are preserved in
|
|
215
|
+
"Other/Artifacts/VisualRegressionTestFailures" and an exception is raised.
|
|
216
|
+
|
|
217
|
+
In a normal run of the unit-tests (see run_testcases) the screenshot-testcases run as well, but roborazzi does
|
|
218
|
+
neither record nor compare anything there."""
|
|
219
|
+
baselines_folder = self.get_visual_regression_baselines_folder()
|
|
220
|
+
if update_baselines:
|
|
221
|
+
GeneralUtilities.ensure_folder_exists_and_is_empty(baselines_folder)
|
|
222
|
+
self.__run_gradle_on_runner(["recordRoborazziDebug"], baselines_folder)
|
|
223
|
+
GeneralUtilities.assert_condition(0 < len(TFCPS_CodeUnitSpecific_Android_Functions.__get_png_files(baselines_folder)), f'Recording the visual-regression-baselines did not produce any image in "{baselines_folder}".')
|
|
224
|
+
else:
|
|
225
|
+
GeneralUtilities.assert_condition(0 < len(TFCPS_CodeUnitSpecific_Android_Functions.__get_png_files(baselines_folder)), f'There are no visual-regression-baselines in "{baselines_folder}". Generate them using "Other/QualityCheck/UpdateVisualRegressionBaselines.py".')
|
|
226
|
+
comparison_folder = os.path.join(self.get_build_folder(), TFCPS_CodeUnitSpecific_Android_Functions.get_visual_regression_comparison_folder_relative_to_build_folder())
|
|
227
|
+
GeneralUtilities.ensure_directory_does_not_exist(comparison_folder)
|
|
228
|
+
exit_code = self.__run_gradle_on_runner_and_get_exit_code(["verifyRoborazziDebug"], comparison_folder)
|
|
229
|
+
if exit_code != 0:
|
|
230
|
+
failures_folder = os.path.join(self.get_codeunit_folder(), TFCPS_CodeUnitSpecific_Android_Functions.__visual_regression_failures_folder_relative_to_codeunit)
|
|
231
|
+
GeneralUtilities.ensure_folder_exists_and_is_empty(failures_folder)
|
|
232
|
+
for image in TFCPS_CodeUnitSpecific_Android_Functions.__get_png_files(comparison_folder):
|
|
233
|
+
shutil.copyfile(image, os.path.join(failures_folder, os.path.basename(image)))
|
|
234
|
+
raise ValueError(f'The visual-regression-tests of the codeunit "{self.get_codeunit_name()}" failed (gradle exit-code {exit_code}). See the runner-log above for details. The screenshots of the failed comparisons are located in "{failures_folder}". If the change of the user-interface is intended, update the baselines using "Other/QualityCheck/UpdateVisualRegressionBaselines.py".')
|
|
235
|
+
|
|
236
|
+
@staticmethod
|
|
237
|
+
@GeneralUtilities.check_arguments
|
|
238
|
+
def __get_png_files(folder: str) -> list[str]:
|
|
239
|
+
if not os.path.isdir(folder):
|
|
240
|
+
return []
|
|
241
|
+
return [file for file in GeneralUtilities.get_direct_files_of_folder(folder) if file.lower().endswith(".png")]
|
|
242
|
+
|
|
243
|
+
@GeneralUtilities.check_arguments
|
|
244
|
+
def __get_gradle_arguments(self, tasks: list[str]) -> list[str]:
|
|
245
|
+
# "bash gradlew" instead of "./gradlew": the executable-bit of the wrapper-script does not survive a checkout on
|
|
246
|
+
# windows, from where the repository is transferred to the runner.
|
|
247
|
+
result: list[str] = ["gradlew", "--console=plain", "--no-daemon"]+tasks
|
|
248
|
+
if self.get_verbosity() == LogLevel.Debug:
|
|
249
|
+
result.append("--info")
|
|
250
|
+
return result
|
|
251
|
+
|
|
252
|
+
@GeneralUtilities.check_arguments
|
|
253
|
+
def __get_folders_which_are_not_transferred(self) -> list[str]:
|
|
254
|
+
# These are generated for the machine they were generated on: "build" and ".gradle" contain absolute paths and
|
|
255
|
+
# caches of that machine, and "local.properties" states the folder of the android-sdk of that machine (it is
|
|
256
|
+
# written by android-studio). On the runner they would describe folders which do not exist there.
|
|
257
|
+
gradle_project_folder = self.get_gradle_project_folder()
|
|
258
|
+
return [os.path.join(gradle_project_folder, "build"), os.path.join(gradle_project_folder, ".gradle"), os.path.join(gradle_project_folder, "local.properties")]
|
|
259
|
+
|
|
260
|
+
@GeneralUtilities.check_arguments
|
|
261
|
+
def __run_gradle_on_runner(self, tasks: list[str], result_folder: str) -> None:
|
|
262
|
+
"""Runs the given gradle-tasks on an android-task-runner and writes the content of result_folder (which must be a
|
|
263
|
+
folder inside this codeunit) back into this codeunit. A failure of gradle raises an exception."""
|
|
264
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", self.__get_gradle_arguments(tasks), self.get_gradle_project_folder(), result_folder, self.__get_folders_which_are_not_transferred(), {})
|
|
265
|
+
|
|
266
|
+
@GeneralUtilities.check_arguments
|
|
267
|
+
def __run_gradle_on_runner_and_get_exit_code(self, tasks: list[str], result_folder: str) -> int:
|
|
268
|
+
"""Like __run_gradle_on_runner, but a failure of gradle does not raise an exception: its exit-code is returned
|
|
269
|
+
instead. This is required when the result-folder is needed exactly when gradle fails (for example the
|
|
270
|
+
screenshots of failed visual-regression-tests), because the runner returns the result-folder only for a job
|
|
271
|
+
which succeeded. The exit-code is therefore written into the result-folder and the job itself always
|
|
272
|
+
succeeds."""
|
|
273
|
+
result_folder_relative_to_project = os.path.relpath(result_folder, self.get_gradle_project_folder()).replace("\\", "/")
|
|
274
|
+
script = TFCPS_CodeUnitSpecific_Android_Functions.get_script_which_preserves_exit_code(self.__get_gradle_arguments(tasks), result_folder_relative_to_project, TFCPS_CodeUnitSpecific_Android_Functions.__exit_code_file_name)
|
|
275
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", ["-c", script], self.get_gradle_project_folder(), result_folder, self.__get_folders_which_are_not_transferred(), {})
|
|
276
|
+
local_exit_code_file = os.path.join(result_folder, TFCPS_CodeUnitSpecific_Android_Functions.__exit_code_file_name)
|
|
277
|
+
GeneralUtilities.assert_file_exists(local_exit_code_file)
|
|
278
|
+
return int(GeneralUtilities.read_text_from_file(local_exit_code_file).strip())
|
|
279
|
+
|
|
280
|
+
@staticmethod
|
|
281
|
+
@GeneralUtilities.check_arguments
|
|
282
|
+
def get_signing_environment_variables_for_runner(environment: dict[str, str]) -> dict[str, str]:
|
|
283
|
+
"""Returns the environment-variables which give the upload-key to the gradle-run on the runner, taken from the
|
|
284
|
+
given environment-variables of this machine (see build). The names are compared case-insensitively, because
|
|
285
|
+
Windows exposes the names of environment-variables upper-cased via os.environ."""
|
|
286
|
+
environment_upper = {name.upper(): value for name, value in environment.items()}
|
|
287
|
+
missing_variables = [name for name in TFCPS_CodeUnitSpecific_Android_Functions.__signing_environment_variables if len((environment_upper.get(name.upper()) or GeneralUtilities.empty_string).strip()) == 0]
|
|
288
|
+
GeneralUtilities.assert_condition(len(missing_variables) == 0, f'A productive build of an android-app is signed with the upload-key, which requires the environment-variable(s) {", ".join(missing_variables)}. "Android_Signing_Keystore" contains the base64-encoding of the keystore-file.')
|
|
289
|
+
result: dict[str, str] = {}
|
|
290
|
+
for name, name_on_runner in TFCPS_CodeUnitSpecific_Android_Functions.__signing_environment_variables.items():
|
|
291
|
+
result[name_on_runner] = environment_upper[name.upper()]
|
|
292
|
+
# A base64-encoding is often wrapped into several lines (for example by the base64-tool), which is not part of it.
|
|
293
|
+
keystore_base64 = "".join(result["SC_ANDROID_SIGNING_KEYSTORE_BASE64"].split())
|
|
294
|
+
try:
|
|
295
|
+
base64.b64decode(keystore_base64, validate=True)
|
|
296
|
+
except binascii.Error as exception:
|
|
297
|
+
raise ValueError('The environment-variable "Android_Signing_Keystore" does not contain a valid base64-encoding of the keystore-file.') from exception
|
|
298
|
+
result["SC_ANDROID_SIGNING_KEYSTORE_BASE64"] = keystore_base64
|
|
299
|
+
return result
|
|
300
|
+
|
|
301
|
+
@staticmethod
|
|
302
|
+
@GeneralUtilities.check_arguments
|
|
303
|
+
def get_script_which_signs_with_upload_key(gradle_arguments: list[str]) -> str:
|
|
304
|
+
"""Returns the bash-script which runs gradle on the runner with the upload-key: it writes the keystore (which it
|
|
305
|
+
gets base64-encoded in an environment-variable) into a temporary file outside of the workspace of the job, gives
|
|
306
|
+
that file to gradle via "SC_ANDROID_SIGNING_STORE_FILE" and removes it when the script ends, regardless of how it
|
|
307
|
+
ends."""
|
|
308
|
+
return "set -e; " \
|
|
309
|
+
+ 'keystore_file="$(mktemp)"; ' \
|
|
310
|
+
+ """trap 'rm -f "$keystore_file"' EXIT; """ \
|
|
311
|
+
+ """printf '%s' "$SC_ANDROID_SIGNING_KEYSTORE_BASE64" | base64 -d > "$keystore_file"; """ \
|
|
312
|
+
+ "unset SC_ANDROID_SIGNING_KEYSTORE_BASE64; " \
|
|
313
|
+
+ 'export SC_ANDROID_SIGNING_STORE_FILE="$keystore_file"; ' \
|
|
314
|
+
+ "bash " + shlex.join(gradle_arguments)
|
|
315
|
+
|
|
316
|
+
@staticmethod
|
|
317
|
+
@GeneralUtilities.check_arguments
|
|
318
|
+
def __get_signature_blocks(signed_file: str) -> list[bytes]:
|
|
319
|
+
"""Returns the signature-blocks of a jar-signed file (an app-bundle is jar-signed): the files "META-INF/*.RSA",
|
|
320
|
+
"META-INF/*.DSA" and "META-INF/*.EC", which contain the certificate of the signer."""
|
|
321
|
+
with zipfile.ZipFile(signed_file) as archive:
|
|
322
|
+
return [archive.read(name) for name in archive.namelist() if re.fullmatch(r"META-INF/[^/]+\.(RSA|DSA|EC)", name, re.IGNORECASE)]
|
|
323
|
+
|
|
324
|
+
@staticmethod
|
|
325
|
+
@GeneralUtilities.check_arguments
|
|
326
|
+
def is_signed(signed_file: str) -> bool:
|
|
327
|
+
"""Whether the given app-bundle is signed."""
|
|
328
|
+
return len(TFCPS_CodeUnitSpecific_Android_Functions.__get_signature_blocks(signed_file)) > 0
|
|
329
|
+
|
|
330
|
+
@staticmethod
|
|
331
|
+
@GeneralUtilities.check_arguments
|
|
332
|
+
def is_signed_with_debug_key(signed_file: str) -> bool:
|
|
333
|
+
"""Whether the given app-bundle is signed with the debug-key of an android-sdk. The certificate of that key is
|
|
334
|
+
always issued to "CN=Android Debug,O=Android,C=US", whose name is contained in the signature-block as text."""
|
|
335
|
+
return any(TFCPS_CodeUnitSpecific_Android_Functions.__subject_of_the_debug_certificate in block for block in TFCPS_CodeUnitSpecific_Android_Functions.__get_signature_blocks(signed_file))
|
|
336
|
+
|
|
337
|
+
@staticmethod
|
|
338
|
+
@GeneralUtilities.check_arguments
|
|
339
|
+
def get_script_which_preserves_exit_code(gradle_arguments: list[str], result_folder_relative_to_project: str, exit_code_file_name: str) -> str:
|
|
340
|
+
"""Returns the bash-script which runs gradle with the given arguments and writes its exit-code into the given
|
|
341
|
+
file inside the given folder instead of failing."""
|
|
342
|
+
exit_code_file = f"{result_folder_relative_to_project}/{exit_code_file_name}"
|
|
343
|
+
return f"mkdir -p {shlex.quote(result_folder_relative_to_project)}; bash {shlex.join(gradle_arguments)}; echo $? > {shlex.quote(exit_code_file)}"
|
|
344
|
+
|
|
345
|
+
@GeneralUtilities.check_arguments
|
|
346
|
+
def __set_version_in_build_file(self, new_version: str) -> None:
|
|
347
|
+
build_file = self.get_build_file()
|
|
348
|
+
content = GeneralUtilities.read_text_from_file(build_file)
|
|
349
|
+
content = TFCPS_CodeUnitSpecific_Android_Functions.set_version_in_build_file_content(content, new_version)
|
|
350
|
+
GeneralUtilities.write_text_to_file(build_file, content)
|
|
351
|
+
|
|
352
|
+
@staticmethod
|
|
353
|
+
@GeneralUtilities.check_arguments
|
|
354
|
+
def get_version_code(version: str) -> int:
|
|
355
|
+
"""Returns the versionCode for the given version. Android requires that the versionCode of an update is greater
|
|
356
|
+
than the one of the installed app, so it is derived from the version: every part of the version gets 3 digits."""
|
|
357
|
+
match = re.match(r"^(\d+)\.(\d+)\.(\d+)$", version)
|
|
358
|
+
GeneralUtilities.assert_condition(match is not None, f'"{version}" is not a version in the format "<major>.<minor>.<patch>".')
|
|
359
|
+
major, minor, patch = int(match.group(1)), int(match.group(2)), int(match.group(3))
|
|
360
|
+
GeneralUtilities.assert_condition(minor < 1000 and patch < 1000, f'The minor- and the patch-part of the version "{version}" must be less than 1000 to be representable as versionCode.')
|
|
361
|
+
result = major*1000000+minor*1000+patch
|
|
362
|
+
# 2100000000 is the greatest versionCode google-play accepts.
|
|
363
|
+
GeneralUtilities.assert_condition(result <= 2100000000, f'The version "{version}" is too large to be representable as versionCode.')
|
|
364
|
+
return result
|
|
365
|
+
|
|
366
|
+
@staticmethod
|
|
367
|
+
@GeneralUtilities.check_arguments
|
|
368
|
+
def set_version_in_build_file_content(content: str, new_version: str) -> str:
|
|
369
|
+
GeneralUtilities.assert_condition(len(TFCPS_CodeUnitSpecific_Android_Functions.__version_code_pattern.findall(content)) == 1, "The build-file must contain exactly one versionCode.")
|
|
370
|
+
GeneralUtilities.assert_condition(len(TFCPS_CodeUnitSpecific_Android_Functions.__version_name_pattern.findall(content)) == 1, "The build-file must contain exactly one versionName.")
|
|
371
|
+
version_code = TFCPS_CodeUnitSpecific_Android_Functions.get_version_code(new_version)
|
|
372
|
+
content = TFCPS_CodeUnitSpecific_Android_Functions.__version_code_pattern.sub(lambda match: f"{match.group('prefix')}{version_code}", content)
|
|
373
|
+
content = TFCPS_CodeUnitSpecific_Android_Functions.__version_name_pattern.sub(lambda match: f"{match.group('prefix')}{match.group('quote')}{new_version}{match.group('quote')}", content)
|
|
374
|
+
return content
|
|
375
|
+
|
|
376
|
+
@GeneralUtilities.check_arguments
|
|
377
|
+
def get_dependencies(self) -> dict[str, set[str]]:
|
|
378
|
+
return GeneralUtilities.merge_dependency_lists([TFCPS_CodeUnitSpecific_Android_Functions.get_dependencies_from_build_file_content(GeneralUtilities.read_text_from_file(self.get_build_file()))])
|
|
379
|
+
|
|
380
|
+
@staticmethod
|
|
381
|
+
@GeneralUtilities.check_arguments
|
|
382
|
+
def get_dependencies_from_build_file_content(content: str) -> list[Dependency]:
|
|
383
|
+
"""Returns the dependencies ("<group-id>:<artifact-id>") and the plugins ("<plugin-id>") which are declared with an
|
|
384
|
+
explicit version in the given content of a build-file."""
|
|
385
|
+
result: list[Dependency] = []
|
|
386
|
+
for match in TFCPS_CodeUnitSpecific_Android_Functions.__dependency_pattern.finditer(content):
|
|
387
|
+
result.append(Dependency(f"{match.group('group')}:{match.group('artifact')}", match.group("version")))
|
|
388
|
+
for match in TFCPS_CodeUnitSpecific_Android_Functions.__plugin_pattern.finditer(content):
|
|
389
|
+
result.append(Dependency(match.group("id"), match.group("version")))
|
|
390
|
+
return result
|
|
391
|
+
|
|
392
|
+
@GeneralUtilities.check_arguments
|
|
393
|
+
def get_available_versions(self, dependencyname: str) -> list[str]:
|
|
394
|
+
return list(self.__get_released_versions(dependencyname).keys())
|
|
395
|
+
|
|
396
|
+
@GeneralUtilities.check_arguments
|
|
397
|
+
def __get_released_versions(self, dependencyname: str) -> dict[str, str]:
|
|
398
|
+
"""Returns the released versions of the given dependency as dictionary from the version in the format
|
|
399
|
+
"<major>.<minor>.<patch>" (which the update-mechanism requires) to the version as it is published (for example
|
|
400
|
+
"4.18" for a dependency which omits the patch-part)."""
|
|
401
|
+
group_id, artifact_id = TFCPS_CodeUnitSpecific_Android_Functions.get_group_and_artifact_of_dependency(dependencyname)
|
|
402
|
+
found_in_any_repository: bool = False
|
|
403
|
+
result: dict[str, str] = {}
|
|
404
|
+
for repository in TFCPS_CodeUnitSpecific_Android_Functions.__maven_repositories:
|
|
405
|
+
metadata_address = f"{repository}/{group_id.replace('.', '/')}/{artifact_id}/maven-metadata.xml"
|
|
406
|
+
response = requests.get(metadata_address, timeout=60)
|
|
407
|
+
if response.status_code == 404:
|
|
408
|
+
continue
|
|
409
|
+
response.raise_for_status()
|
|
410
|
+
found_in_any_repository = True
|
|
411
|
+
for found_version in etree.fromstring(response.content).xpath("/metadata/versioning/versions/version/text()"):
|
|
412
|
+
normalized_version = TFCPS_CodeUnitSpecific_Android_Functions.normalize_released_version(str(found_version))
|
|
413
|
+
if normalized_version is not None:
|
|
414
|
+
result[normalized_version] = str(found_version)
|
|
415
|
+
GeneralUtilities.assert_condition(found_in_any_repository, f'The dependency "{dependencyname}" was not found in any of the repositories {", ".join(TFCPS_CodeUnitSpecific_Android_Functions.__maven_repositories)}.')
|
|
416
|
+
return result
|
|
417
|
+
|
|
418
|
+
@staticmethod
|
|
419
|
+
@GeneralUtilities.check_arguments
|
|
420
|
+
def normalize_released_version(version: str) -> str:
|
|
421
|
+
"""Returns the given version in the format "<major>.<minor>.<patch>", or None if it is not the version of a release
|
|
422
|
+
(for example "1.2.0-alpha01")."""
|
|
423
|
+
if re.match(r"^(\d+)\.(\d+)\.(\d+)$", version) is not None:
|
|
424
|
+
return version
|
|
425
|
+
if re.match(r"^(\d+)\.(\d+)$", version) is not None:
|
|
426
|
+
return version+".0"
|
|
427
|
+
return None
|
|
428
|
+
|
|
429
|
+
@staticmethod
|
|
430
|
+
@GeneralUtilities.check_arguments
|
|
431
|
+
def get_group_and_artifact_of_dependency(dependencyname: str) -> tuple[str, str]:
|
|
432
|
+
"""A dependency is identified by "<group-id>:<artifact-id>". A plugin is identified by its plugin-id, and gradle
|
|
433
|
+
resolves it through its marker-artifact "<plugin-id>:<plugin-id>.gradle.plugin"."""
|
|
434
|
+
parts = dependencyname.split(":")
|
|
435
|
+
if len(parts) == 1:
|
|
436
|
+
return (dependencyname, f"{dependencyname}.gradle.plugin")
|
|
437
|
+
GeneralUtilities.assert_condition(len(parts) == 2, f'The name of a dependency must have the format "<group-id>:<artifact-id>" or "<plugin-id>" but was "{dependencyname}".')
|
|
438
|
+
return (parts[0], parts[1])
|
|
439
|
+
|
|
440
|
+
@GeneralUtilities.check_arguments
|
|
441
|
+
def set_dependency_version(self, name: str, new_version: str) -> None:
|
|
442
|
+
build_file = self.get_build_file()
|
|
443
|
+
content = GeneralUtilities.read_text_from_file(build_file)
|
|
444
|
+
# The version is written in the format in which it is published, because that is the only one gradle can resolve.
|
|
445
|
+
published_version = self.__get_released_versions(name)[new_version]
|
|
446
|
+
content = TFCPS_CodeUnitSpecific_Android_Functions.set_dependency_version_in_build_file_content(content, name, published_version)
|
|
447
|
+
GeneralUtilities.write_text_to_file(build_file, content)
|
|
448
|
+
|
|
449
|
+
@staticmethod
|
|
450
|
+
@GeneralUtilities.check_arguments
|
|
451
|
+
def set_dependency_version_in_build_file_content(content: str, name: str, new_version: str) -> str:
|
|
452
|
+
if ":" in name:
|
|
453
|
+
group_id, artifact_id = TFCPS_CodeUnitSpecific_Android_Functions.get_group_and_artifact_of_dependency(name)
|
|
454
|
+
|
|
455
|
+
def replace_dependency(match: re.Match) -> str:
|
|
456
|
+
if match.group("group") == group_id and match.group("artifact") == artifact_id:
|
|
457
|
+
return f"{match.group('quote')}{group_id}:{artifact_id}:{new_version}{match.group('quote')}"
|
|
458
|
+
return match.group(0)
|
|
459
|
+
return TFCPS_CodeUnitSpecific_Android_Functions.__dependency_pattern.sub(replace_dependency, content)
|
|
460
|
+
else:
|
|
461
|
+
def replace_plugin(match: re.Match) -> str:
|
|
462
|
+
if match.group("id") == name:
|
|
463
|
+
version_start = match.start("version")-match.start(0)
|
|
464
|
+
version_end = match.end("version")-match.start(0)
|
|
465
|
+
return match.group(0)[:version_start]+new_version+match.group(0)[version_end:]
|
|
466
|
+
return match.group(0)
|
|
467
|
+
return TFCPS_CodeUnitSpecific_Android_Functions.__plugin_pattern.sub(replace_plugin, content)
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
class TFCPS_CodeUnitSpecific_Android_CLI:
|
|
471
|
+
|
|
472
|
+
@staticmethod
|
|
473
|
+
@GeneralUtilities.check_arguments
|
|
474
|
+
def parse(file: str) -> TFCPS_CodeUnitSpecific_Android_Functions:
|
|
475
|
+
parser = TFCPS_CodeUnitSpecific_Base_CLI.get_base_parser()
|
|
476
|
+
#add custom parameter if desired
|
|
477
|
+
args = parser.parse_args()
|
|
478
|
+
result: TFCPS_CodeUnitSpecific_Android_Functions = TFCPS_CodeUnitSpecific_Android_Functions(file, LogLevel(int(args.verbosity)), args.targetenvironmenttype, not args.nocache, args.ispremerge)
|
|
479
|
+
return result
|
|
File without changes
|
|
@@ -169,7 +169,7 @@ class TFCPS_CodeUnitSpecific_CPP_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
169
169
|
#run in - which is also the folder __collect_output_files searches afterwards.
|
|
170
170
|
#Nothing of a C/C++-codeunit is left out of the archive the runner gets: where a build-command puts its
|
|
171
171
|
#output is defined by that command, so there is no folder which is known to contain state of this machine.
|
|
172
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "/bin/sh", ["-c", " && ".join(self.macos_build_commands)], source_folder, source_folder, [])
|
|
172
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "/bin/sh", ["-c", " && ".join(self.macos_build_commands)], source_folder, source_folder, [], {})
|
|
173
173
|
self.__collect_output_files(source_folder, self.macos_output_file_patterns, "BuildResult_MacOS")
|
|
174
174
|
|
|
175
175
|
@GeneralUtilities.check_arguments
|
|
@@ -207,11 +207,12 @@ class TFCPS_CodeUnitSpecific_CPP_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
207
207
|
|
|
208
208
|
@GeneralUtilities.check_arguments
|
|
209
209
|
def linting(self) -> None:
|
|
210
|
-
# There is no single, universal C/C++-linter (unlike e.g. clippy for Rust)
|
|
211
|
-
#
|
|
212
|
-
# this kind owns regardless of that is its own scripts, so only those are normalized by default.
|
|
210
|
+
# There is no single, universal C/C++-linter (unlike e.g. clippy for Rust). The invisible characters of the
|
|
211
|
+
# C/C++-sourcecode are normalized in the whole codeunit, which also affects a vendored foreign upstream.
|
|
213
212
|
# TODO add real C/C++-linting (for example clang-format/clang-tidy) once a codeunit needs it for its own sourcecode.
|
|
214
|
-
self.
|
|
213
|
+
codeunit_folder = self.get_codeunit_folder()
|
|
214
|
+
self._protected_sc.normalize_invisible_characters_of_files_in_folder(codeunit_folder, ["c", "cpp"])
|
|
215
|
+
self._protected_sc.normalize_invisible_characters_of_files_in_folder(os.path.join(codeunit_folder, "Other"), ["py", "md", "xml"])
|
|
215
216
|
|
|
216
217
|
@GeneralUtilities.check_arguments
|
|
217
218
|
def do_common_tasks(self, current_codeunit_version: str) -> None:
|
|
@@ -4,6 +4,7 @@ from urllib import request
|
|
|
4
4
|
import time
|
|
5
5
|
import ssl
|
|
6
6
|
from datetime import timedelta,datetime
|
|
7
|
+
from typing import Callable
|
|
7
8
|
from ...GeneralUtilities import GeneralUtilities, Platform
|
|
8
9
|
from ...SCLog import LogLevel
|
|
9
10
|
from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base,TFCPS_CodeUnitSpecific_Base_CLI
|
|
@@ -11,8 +12,13 @@ from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base,TFCPS_Code
|
|
|
11
12
|
|
|
12
13
|
class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
13
14
|
|
|
15
|
+
# Maps the name of a dependency (the name of its folder in "Other/Resources/Dependencies") to the function which returns its
|
|
16
|
+
# available versions. The function gets the currently used version as argument.
|
|
17
|
+
__available_versions_functions:dict[str,Callable[[str],list[str]]]=None
|
|
18
|
+
|
|
14
19
|
def __init__(self,current_file:str,verbosity:LogLevel,targetenvironmenttype:str,use_cache:bool,is_pre_merge:bool):
|
|
15
20
|
super().__init__(current_file, verbosity,targetenvironmenttype,use_cache,is_pre_merge)
|
|
21
|
+
self.__available_versions_functions=dict[str,Callable[[str],list[str]]]()
|
|
16
22
|
|
|
17
23
|
@GeneralUtilities.check_arguments
|
|
18
24
|
def build(self,platforms:list[Platform],custom_arguments:dict[str,str]) -> None:
|
|
@@ -135,17 +141,45 @@ class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
135
141
|
def run_testcases(self) -> None:
|
|
136
142
|
pass#TODO
|
|
137
143
|
|
|
144
|
+
@GeneralUtilities.check_arguments
|
|
145
|
+
def register_available_versions_function(self,dependency_name:str,available_versions_function:Callable[[str],list[str]])->None:
|
|
146
|
+
"""Registers the function which returns the available versions of the given dependency.
|
|
147
|
+
The dependencies of a docker-codeunit are the folders in "Other/Resources/Dependencies", and there is no general way to find out
|
|
148
|
+
where the available versions of such a dependency come from (npm, nuget, pypi, the tags of a git-repository, ...). So the
|
|
149
|
+
"Other/UpdateDependencies.py"-script of the codeunit registers one function per dependency before it calls update_dependencies.
|
|
150
|
+
The function gets the currently used version as argument and must return only versions in the format "<major>.<minor>.<patch>"."""
|
|
151
|
+
GeneralUtilities.assert_condition(dependency_name not in self.__available_versions_functions,f"A function for the available versions of dependency {dependency_name} is already registered.")
|
|
152
|
+
self.__available_versions_functions[dependency_name]=available_versions_function
|
|
153
|
+
|
|
154
|
+
@GeneralUtilities.check_arguments
|
|
155
|
+
def __get_dependencies_folder(self)->str:
|
|
156
|
+
return os.path.join(self.get_codeunit_folder(),"Other","Resources","Dependencies")
|
|
157
|
+
|
|
138
158
|
@GeneralUtilities.check_arguments
|
|
139
159
|
def get_dependencies(self)->dict[str,set[str]]:
|
|
140
|
-
|
|
141
|
-
|
|
160
|
+
"""Returns the dependencies which are pinned in "Other/Resources/Dependencies/<dependency>/Version.txt"."""
|
|
161
|
+
result:dict[str,set[str]]=dict[str,set[str]]()
|
|
162
|
+
dependencies_folder:str=self.__get_dependencies_folder()
|
|
163
|
+
if not os.path.isdir(dependencies_folder):
|
|
164
|
+
return result
|
|
165
|
+
resources_folder:str=os.path.dirname(dependencies_folder)
|
|
166
|
+
for dependency_name in sorted(os.listdir(dependencies_folder)):
|
|
167
|
+
if os.path.isfile(os.path.join(dependencies_folder,dependency_name,"Version.txt")):
|
|
168
|
+
result[dependency_name]={self.tfcps_Tools_General.get_dependency_version_in_resources_folder(resources_folder,dependency_name)}
|
|
169
|
+
return result
|
|
170
|
+
|
|
142
171
|
@GeneralUtilities.check_arguments
|
|
143
172
|
def get_available_versions(self,dependencyname:str)->list[str]:
|
|
144
|
-
|
|
173
|
+
if dependencyname not in self.__available_versions_functions:
|
|
174
|
+
raise ValueError(f"No function for the available versions of dependency {dependencyname} is registered. Register one using register_available_versions_function or add the dependency to the ignored dependencies in the codeunit-file.")
|
|
175
|
+
current_version:str=next(iter(self.get_dependencies()[dependencyname]))
|
|
176
|
+
return self.__available_versions_functions[dependencyname](current_version)
|
|
145
177
|
|
|
146
178
|
@GeneralUtilities.check_arguments
|
|
147
179
|
def set_dependency_version(self,name:str,new_version:str)->None:
|
|
148
|
-
|
|
180
|
+
version_file:str=os.path.join(self.__get_dependencies_folder(),name,"Version.txt")
|
|
181
|
+
GeneralUtilities.assert_file_exists(version_file)
|
|
182
|
+
GeneralUtilities.write_text_to_file(version_file,new_version)
|
|
149
183
|
|
|
150
184
|
@GeneralUtilities.check_arguments
|
|
151
185
|
@GeneralUtilities.deprecated("Use image_is_working_via_network instead, which tests the image over a user-defined docker-network without publishing a host-port.")
|
|
@@ -112,7 +112,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
112
112
|
"environment as a runner-built one.", LogLevel.Warning)
|
|
113
113
|
self._protected_sc.run_with_epew("flutter", "build windows", src_folder)
|
|
114
114
|
else:
|
|
115
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder, windows_release_folder, folders_which_are_not_transferred)
|
|
115
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder, windows_release_folder, folders_which_are_not_transferred, {})
|
|
116
116
|
windows_folder = os.path.join(artifacts_folder, "BuildResult_Windows")
|
|
117
117
|
GeneralUtilities.ensure_directory_does_not_exist(windows_folder)
|
|
118
118
|
GeneralUtilities.ensure_directory_exists(windows_folder)
|
|
@@ -134,7 +134,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
134
134
|
# builds), analogous to how ios-builds always run on the iOS-task-runner. See SCTaskRunnerMacOS and
|
|
135
135
|
# the remote-build-article in the reference.
|
|
136
136
|
macos_release_folder = os.path.join(src_folder, "build/macos/Build/Products/Release")
|
|
137
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder, macos_release_folder, folders_which_are_not_transferred)
|
|
137
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder, macos_release_folder, folders_which_are_not_transferred, {})
|
|
138
138
|
macos_folder = os.path.join(artifacts_folder, "BuildResult_MacOS")
|
|
139
139
|
GeneralUtilities.ensure_directory_does_not_exist(macos_folder)
|
|
140
140
|
GeneralUtilities.ensure_directory_exists(macos_folder)
|
|
@@ -143,7 +143,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
143
143
|
# iOS-builds must run on macOS and therefore always run on the dedicated iOS-task-runner (uniform
|
|
144
144
|
# builds). See SCTaskRunnerIOS and the remote-build-article in the reference.
|
|
145
145
|
ios_release_folder = os.path.join(src_folder, "build/ios/iphoneos")
|
|
146
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder, ios_release_folder, folders_which_are_not_transferred)
|
|
146
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder, ios_release_folder, folders_which_are_not_transferred, {})
|
|
147
147
|
ios_folder = os.path.join(artifacts_folder, "BuildResult_IOS")
|
|
148
148
|
GeneralUtilities.ensure_directory_does_not_exist(ios_folder)
|
|
149
149
|
GeneralUtilities.ensure_directory_exists(ios_folder)
|
|
@@ -153,7 +153,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
153
153
|
# remote-build-article in the reference), analogous to how ios-builds always run on a macOS-task-runner:
|
|
154
154
|
# the Android-SDK/NDK-toolchain no longer lives in SCBuilder itself, it moved into SCTaskRunnerAndroid.
|
|
155
155
|
aab_release_folder = os.path.join(src_folder, "build/app/outputs/bundle/release")
|
|
156
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder, aab_release_folder, folders_which_are_not_transferred)
|
|
156
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder, aab_release_folder, folders_which_are_not_transferred, {})
|
|
157
157
|
# The app-bundle which came back from the runner is published as the build-result of this codeunit,
|
|
158
158
|
# analogous to the targets above: it is the artifact of an android-build, and every codeunit has to have
|
|
159
159
|
# an artifact whose name matches "BuildResult_.+" (see TFCPS_CodeUnit_BuildCodeUnit.build_codeunit).
|
|
@@ -58,6 +58,7 @@ class TFCPS_CodeUnitSpecific_Maven_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
58
58
|
|
|
59
59
|
@GeneralUtilities.check_arguments
|
|
60
60
|
def linting(self) -> None:
|
|
61
|
+
self._protected_sc.normalize_invisible_characters_of_files_in_folder(self.get_codeunit_folder(), ["java"])
|
|
61
62
|
# checkstyle is configured in the pom-file of the codeunit, so that the ruleset belongs to the codeunit
|
|
62
63
|
# and not to this implementation.
|
|
63
64
|
self.__run_maven("-B checkstyle:check")
|