ScriptCollection 4.4.71__py3-none-any.whl → 4.4.72__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. ScriptCollection/Executables.py +3 -3
  2. ScriptCollection/GeneralUtilities.py +42 -0
  3. ScriptCollection/OCIImages/AbstractImageHandler.py +14 -26
  4. ScriptCollection/OCIImages/AbstractTagPatternImageHandler.py +46 -0
  5. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py +5 -14
  6. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py +5 -14
  7. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGeneric.py +6 -15
  8. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGenericV.py +6 -15
  9. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabCE.py +7 -8
  10. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabEE.py +7 -8
  11. ScriptCollection/OCIImages/OCIImageManager.py +73 -68
  12. ScriptCollection/OCIImages/OCIRegistryClient.py +117 -0
  13. ScriptCollection/ScriptCollectionCore.py +9 -4
  14. ScriptCollection/TFCPS/Android/TFCPS_CodeUnitSpecific_Android.py +375 -0
  15. ScriptCollection/TFCPS/Android/__init__.py +0 -0
  16. ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py +5 -4
  17. ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +38 -4
  18. ScriptCollection/TFCPS/Maven/TFCPS_CodeUnitSpecific_Maven.py +1 -0
  19. ScriptCollection/TFCPS/Rust/CargoLockFile.py +151 -0
  20. ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py +5 -0
  21. ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +1 -0
  22. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +14 -0
  23. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +19 -0
  24. ScriptCollection/TFCPS/TFCPS_Tools_General.py +15 -2
  25. {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.72.dist-info}/METADATA +177 -177
  26. {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.72.dist-info}/RECORD +29 -24
  27. {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.72.dist-info}/WHEEL +0 -0
  28. {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.72.dist-info}/entry_points.txt +0 -0
  29. {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.72.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,117 @@
1
+ import base64
2
+ import re
3
+ from urllib.parse import urlencode
4
+ import requests
5
+ from ..GeneralUtilities import GeneralUtilities
6
+ from ..ScriptCollectionCore import ScriptCollectionCore
7
+
8
+
9
+ class OCIRegistryClient:
10
+ """Reads information about images from OCI-registries using the OCI-distribution-api (see
11
+ https://github.com/opencontainers/distribution-spec/blob/main/spec.md ), which is implemented by docker-hub, ghcr.io,
12
+ quay.io, mcr.microsoft.com and by a custom deployed docker-registry as well. So there is one implementation for all registries
13
+ and no registry-specific api is used."""
14
+
15
+ __sc: ScriptCollectionCore = None
16
+ __docker_hub_address: str = "docker.io"
17
+ __docker_hub_api_host: str = "registry-1.docker.io"
18
+ __page_size: int = 1000
19
+ __timeout_in_seconds: int = 30
20
+
21
+ def __init__(self, sc: ScriptCollectionCore):
22
+ self.__sc = sc
23
+
24
+ @GeneralUtilities.check_arguments
25
+ def split_registry_address(self, registry_address: str) -> tuple[str, str]:
26
+ """Splits an image-address without tag into the host of the registry and the repository inside the registry, following the
27
+ rules of docker: the first segment is a host if it contains a "." or a ":" or if it is "localhost", otherwise the image is
28
+ located on docker-hub. On docker-hub an image without namespace belongs to the namespace "library".
29
+ Examples: "docker.io/debian" -> ("docker.io", "library/debian"), "aniondev/scbuilder" -> ("docker.io", "aniondev/scbuilder"),
30
+ "myregistry.example.com:5000/debian" -> ("myregistry.example.com:5000", "debian")."""
31
+ segments = registry_address.split("/")
32
+ first_segment = segments[0]
33
+ if len(segments) > 1 and ("." in first_segment or ":" in first_segment or first_segment == "localhost"):
34
+ host = first_segment
35
+ repository = "/".join(segments[1:])
36
+ else:
37
+ host = OCIRegistryClient.__docker_hub_address
38
+ repository = registry_address
39
+ if host == OCIRegistryClient.__docker_hub_address and "/" not in repository:
40
+ repository = f"library/{repository}"
41
+ return (host, repository)
42
+
43
+ @GeneralUtilities.check_arguments
44
+ def get_tags(self, registry_address: str) -> list[str]:
45
+ """Returns all tags of the image which is located at the given image-address (without tag), for example
46
+ "docker.io/library/debian" or "ghcr.io/betterleaks/betterleaks".
47
+ If credentials for the registry are defined on this machine (see
48
+ ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) then they are used, otherwise the registry is
49
+ accessed anonymously."""
50
+ host, repository = self.split_registry_address(registry_address)
51
+ api_host = OCIRegistryClient.__docker_hub_api_host if host == OCIRegistryClient.__docker_hub_address else host
52
+ credentials = self.__get_credentials(host)
53
+ result: list[str] = []
54
+ url = f"https://{api_host}/v2/{repository}/tags/list?n={OCIRegistryClient.__page_size}"
55
+ authorization_header: str = None
56
+ while url is not None:
57
+ response = self.send_get_request(url, authorization_header)
58
+ if response.status_code == 401 and authorization_header is None:
59
+ authorization_header = self.__get_authorization_header(response.headers.get("WWW-Authenticate"), credentials)
60
+ response = self.send_get_request(url, authorization_header)
61
+ response.raise_for_status()
62
+ result.extend(response.json().get("tags") or [])
63
+ url = self.__get_url_of_next_page(api_host, response)
64
+ return result
65
+
66
+ @GeneralUtilities.check_arguments
67
+ def send_get_request(self, url: str, authorization_header: str) -> requests.Response:
68
+ """Sends a get-request. This is the only function of this class which communicates with a server."""
69
+ headers = {}
70
+ if authorization_header is not None:
71
+ headers["Authorization"] = authorization_header
72
+ return requests.get(url, headers=headers, timeout=OCIRegistryClient.__timeout_in_seconds)
73
+
74
+ @GeneralUtilities.check_arguments
75
+ def __get_credentials(self, host: str) -> tuple[str, str]:
76
+ """Returns (username, password) of the registry with the given host or None if no credentials are defined for it."""
77
+ for address, username, password in self.__sc.get_docker_registry_credentials_from_environment_variables():
78
+ if address.rstrip("/") == host:
79
+ return (username, password)
80
+ return None
81
+
82
+ @GeneralUtilities.check_arguments
83
+ def __get_authorization_header(self, authentication_challenge: str, credentials: tuple[str, str]) -> str:
84
+ """Returns the value of the authorization-header which answers the given challenge (the WWW-Authenticate-header of a 401-response)."""
85
+ GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(authentication_challenge), "The registry requires authentication but does not tell how.")
86
+ scheme = authentication_challenge.split(" ", 1)[0].lower()
87
+ if scheme == "basic":
88
+ GeneralUtilities.assert_condition(credentials is not None, "The registry requires basic-authentication but no credentials are defined for it.")
89
+ return self.__get_basic_authorization_header(credentials)
90
+ GeneralUtilities.assert_condition(scheme == "bearer", f"Unsupported authentication-scheme in challenge \"{authentication_challenge}\".")
91
+ parameters = dict(re.findall(r'(\w+)="([^"]*)"', authentication_challenge))
92
+ GeneralUtilities.assert_condition("realm" in parameters, f"No realm defined in challenge \"{authentication_challenge}\".")
93
+ token_parameters = {key: value for key, value in parameters.items() if key in ("service", "scope")}
94
+ token_url = f"{parameters['realm']}?{urlencode(token_parameters)}"
95
+ token_authorization_header = None if credentials is None else self.__get_basic_authorization_header(credentials)
96
+ response = self.send_get_request(token_url, token_authorization_header)
97
+ response.raise_for_status()
98
+ token_data = response.json()
99
+ token = token_data.get("token") or token_data.get("access_token")
100
+ GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(token), f"The token-service \"{parameters['realm']}\" did not return a token.")
101
+ return f"Bearer {token}"
102
+
103
+ @GeneralUtilities.check_arguments
104
+ def __get_basic_authorization_header(self, credentials: tuple[str, str]) -> str:
105
+ encoded_credentials = base64.b64encode(f"{credentials[0]}:{credentials[1]}".encode("utf-8")).decode("ascii")
106
+ return f"Basic {encoded_credentials}"
107
+
108
+ @GeneralUtilities.check_arguments
109
+ def __get_url_of_next_page(self, api_host: str, response: requests.Response) -> str:
110
+ """Returns the url of the next page of a paginated tag-list or None if the response contains the last page."""
111
+ next_link = response.links.get("next")
112
+ if next_link is None:
113
+ return None
114
+ next_url: str = next_link["url"]
115
+ if next_url.startswith("/"):
116
+ next_url = f"https://{api_host}{next_url}"
117
+ return next_url
@@ -41,7 +41,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
41
41
  from .ProgramRunnerPopen import ProgramRunnerPopen
42
42
  from .SCLog import SCLog, LogLevel
43
43
 
44
- version = "4.4.71"
44
+ version = "4.4.72"
45
45
  __version__ = version
46
46
 
47
47
  class VSCodeWorkspaceShellTask:
@@ -4396,14 +4396,19 @@ OCR-content:
4396
4396
  return False
4397
4397
  return True
4398
4398
 
4399
- def reclaim_space_from_docker(self,remove_containers:bool,remove_volumes:bool,remove_images:bool, amount_of_attempts: int = 5):
4399
+ def reclaim_space_from_docker(self,remove_containers:bool,remove_named_volumes:bool,remove_images:bool, amount_of_attempts: int = 5):
4400
4400
  self.log.log("Reclaim disk space from docker...",LogLevel.Debug)
4401
4401
  if remove_containers:
4402
4402
  self.run_program_with_retry("docker","container prune -f",amount_of_attempts=amount_of_attempts)
4403
- if remove_volumes:
4404
- self.run_program_with_retry("docker","volume prune -f",amount_of_attempts=amount_of_attempts)
4403
+ self.run_program_with_retry("docker","network prune -f",amount_of_attempts=amount_of_attempts)# networks are only unused when the containers which used them are removed.
4404
+ if remove_named_volumes:
4405
+ self.run_program_with_retry("docker","volume prune -a -f",amount_of_attempts=amount_of_attempts)# since docker 23 "volume prune" without "-a" only removes anonymous volumes.
4406
+ else:
4407
+ self.run_program_with_retry("docker","volume prune -f",amount_of_attempts=amount_of_attempts)# removes only anonymous volumes.
4405
4408
  if remove_images:
4406
4409
  self.run_program_with_retry("docker","image prune -a -f",amount_of_attempts=amount_of_attempts)
4410
+ else:
4411
+ self.run_program_with_retry("docker","image prune -f",amount_of_attempts=amount_of_attempts)# removes only dangling images (untagged leftovers of builds).
4407
4412
  self.run_program_with_retry("docker","builder prune -a -f",amount_of_attempts=amount_of_attempts)
4408
4413
  self.run_program_with_retry("docker","buildx prune -a -f",amount_of_attempts=amount_of_attempts,throw_exception_if_exitcode_is_not_zero=False) # buildx prune is not available on every machine.
4409
4414
  self.run_program_with_retry("docker","system df",print_live_output=self.log.loglevel==LogLevel.Debug,amount_of_attempts=amount_of_attempts)
@@ -0,0 +1,375 @@
1
+ import os
2
+ import re
3
+ import shlex
4
+ import shutil
5
+ import requests
6
+ from lxml import etree
7
+ from ...GeneralUtilities import GeneralUtilities, Dependency
8
+ from ...SCLog import LogLevel
9
+ from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base, TFCPS_CodeUnitSpecific_Base_CLI
10
+ from ..TFCPS_RemoteBuild import RunnerOperatingSystem
11
+
12
+
13
+ class TFCPS_CodeUnitSpecific_Android_Functions(TFCPS_CodeUnitSpecific_Base):
14
+ """Implements the tasks of a codeunit which contains a native android-app (java or kotlin) built with gradle.
15
+
16
+ The gradle-project is expected in "<codeunit>/<codeunit>" as a single-module-project: its "build.gradle" applies the
17
+ android-application-plugin itself and its testcases are located in its own "src/test"-folder, because that is the
18
+ layout gradle defines and the layout the coverage-report of the android-gradle-plugin measures without further
19
+ configuration. The project has to contain the gradle-wrapper ("gradlew" and "gradle/wrapper"), so the gradle-version
20
+ is pinned by the repository and not by the machine which builds it.
21
+
22
+ Every gradle-task of the project needs the android-sdk. It is therefore always run on an android-task-runner (see
23
+ SCTaskRunnerAndroid and the remote-build-article in the reference), analogous to the android-target of a
24
+ flutter-codeunit: the android-sdk is not part of SCBuilder.
25
+
26
+ Requirements for the "build.gradle" of the project:
27
+ - It declares exactly one "versionCode" and exactly one "versionName" (they are set by do_common_tasks).
28
+ - The debug-build-type sets "enableUnitTestCoverage = true", so the task "createDebugUnitTestCoverageReport" exists.
29
+ - It applies the plugin "org.cyclonedx.bom" and lets the task "cyclonedxBom" write its xml-output to
30
+ "build/outputs/bom/bom.xml" (see get_bom_file_relative_to_build_folder), because only the "build/outputs"-folder is
31
+ transferred back from the runner by the build.
32
+ - Only if the codeunit has visual-regression-tests (see run_visual_regression_tests): it applies the plugin
33
+ "io.github.takahirom.roborazzi" and configures "roborazzi { outputDir = file('../Other/Resources/VisualRegressionBaselines');
34
+ compare { outputDir = layout.buildDirectory.dir('outputs/roborazzi-comparison') } }" (see
35
+ get_visual_regression_baselines_folder and get_visual_regression_comparison_folder_relative_to_build_folder)."""
36
+
37
+ # The repositories in which a dependency or a gradle-plugin of an android-app is published. Gradle itself searches
38
+ # all of them (they are the repositories a project declares by "google()", "mavenCentral()" and
39
+ # "gradlePluginPortal()"), so a dependency can be located in any of them.
40
+ __maven_repositories: list[str] = ["https://dl.google.com/dl/android/maven2", "https://repo1.maven.org/maven2", "https://plugins.gradle.org/m2"]
41
+
42
+ # A dependency in the format "<group-id>:<artifact-id>:<version>" inside a string-literal of the build-file.
43
+ __dependency_pattern = re.compile(r"""(?P<quote>['"])(?P<group>[A-Za-z0-9_.\-]+):(?P<artifact>[A-Za-z0-9_.\-]+):(?P<version>[A-Za-z0-9_.\-+]+)(?P=quote)""")
44
+
45
+ # A plugin which is declared with its version in the plugins-block of the build-file, for example
46
+ # "id 'com.android.application' version '9.1.0'".
47
+ __plugin_pattern = re.compile(r"""id\s*\(?\s*(?P<quote1>['"])(?P<id>[A-Za-z0-9_.\-]+)(?P=quote1)\s*\)?\s*version\s*\(?\s*(?P<quote2>['"])(?P<version>[A-Za-z0-9_.\-+]+)(?P=quote2)""")
48
+
49
+ # The folder of the codeunit which contains the baseline-images of the visual-regression-tests. It is the same
50
+ # folder the visual-regression-tests of the other codeunit-types use (see TFCPS_VisualRegressionTests).
51
+ __visual_regression_baselines_folder_relative_to_codeunit: str = "Other/Resources/VisualRegressionBaselines"
52
+
53
+ # The folder the screenshots of failed visual-regression-tests (the actual image and the comparison-image) are
54
+ # preserved in. It is the same folder the visual-regression-tests of the other codeunit-types use.
55
+ __visual_regression_failures_folder_relative_to_codeunit: str = "Other/Artifacts/VisualRegressionTestFailures"
56
+
57
+ # The file (inside the comparison-folder) the exit-code of the verifying gradle-run is written to on the runner.
58
+ __exit_code_file_name: str = "GradleExitCode.txt"
59
+
60
+ __version_code_pattern = re.compile(r"(?P<prefix>\bversionCode\s*=?\s*)(?P<value>\d+)")
61
+ __version_name_pattern = re.compile(r"""(?P<prefix>\bversionName\s*=?\s*)(?P<quote>['"])(?P<value>[^'"]*)(?P=quote)""")
62
+
63
+ def __init__(self, current_file: str, verbosity: LogLevel, targetenvironmenttype: str, use_cache: bool, is_pre_merge: bool):
64
+ super().__init__(current_file, verbosity, targetenvironmenttype, use_cache, is_pre_merge)
65
+
66
+ @GeneralUtilities.check_arguments
67
+ def get_gradle_project_folder(self) -> str:
68
+ """Returns the folder which contains the build-file of this codeunit."""
69
+ return os.path.join(self.get_codeunit_folder(), self.get_codeunit_name())
70
+
71
+ @GeneralUtilities.check_arguments
72
+ def get_build_file(self) -> str:
73
+ return os.path.join(self.get_gradle_project_folder(), "build.gradle")
74
+
75
+ @GeneralUtilities.check_arguments
76
+ def get_build_folder(self) -> str:
77
+ return os.path.join(self.get_gradle_project_folder(), "build")
78
+
79
+ @staticmethod
80
+ @GeneralUtilities.check_arguments
81
+ def get_bom_file_relative_to_build_folder() -> str:
82
+ return "outputs/bom/bom.xml"
83
+
84
+ @GeneralUtilities.check_arguments
85
+ def get_visual_regression_baselines_folder(self) -> str:
86
+ return os.path.join(self.get_codeunit_folder(), TFCPS_CodeUnitSpecific_Android_Functions.__visual_regression_baselines_folder_relative_to_codeunit)
87
+
88
+ @staticmethod
89
+ @GeneralUtilities.check_arguments
90
+ def get_visual_regression_comparison_folder_relative_to_build_folder() -> str:
91
+ return "outputs/roborazzi-comparison"
92
+
93
+ @GeneralUtilities.check_arguments
94
+ def build(self) -> None:
95
+ """Builds the app-bundle (which is what is uploaded to a store) and the apk (which can be installed directly) of the
96
+ release-build-type and generates the bill-of-materials. All of them are generated by one gradle-run, so the runner
97
+ has to be used only once for them."""
98
+ codeunit_name = self.get_codeunit_name()
99
+ codeunit_version: str = self.tfcps_Tools_General.get_version_of_codeunit(self.get_codeunit_file())
100
+ outputs_folder = os.path.join(self.get_build_folder(), "outputs")
101
+ GeneralUtilities.ensure_directory_does_not_exist(outputs_folder)
102
+ self.__run_gradle_on_runner(["bundleRelease", "assembleRelease", "cyclonedxBom"], outputs_folder)
103
+
104
+ aab_folder = os.path.join(self.get_artifacts_folder(), "BuildResult_AAB")
105
+ GeneralUtilities.ensure_folder_exists_and_is_empty(aab_folder)
106
+ aab_file_original = self._protected_sc.find_file_by_extension(os.path.join(outputs_folder, "bundle", "release"), "aab")
107
+ shutil.copyfile(aab_file_original, os.path.join(aab_folder, f"{codeunit_name}.aab"))
108
+
109
+ apk_folder = os.path.join(self.get_artifacts_folder(), "BuildResult_APK")
110
+ GeneralUtilities.ensure_folder_exists_and_is_empty(apk_folder)
111
+ apk_file_original = self._protected_sc.find_file_by_extension(os.path.join(outputs_folder, "apk", "release"), "apk")
112
+ shutil.copyfile(apk_file_original, os.path.join(apk_folder, f"{codeunit_name}_v{codeunit_version}.apk"))
113
+
114
+ bom_folder = os.path.join(self.get_artifacts_folder(), "BOM")
115
+ GeneralUtilities.ensure_directory_exists(bom_folder)
116
+ generated_bom_file = os.path.join(self.get_build_folder(), TFCPS_CodeUnitSpecific_Android_Functions.get_bom_file_relative_to_build_folder())
117
+ GeneralUtilities.assert_file_exists(generated_bom_file)
118
+ target_bom_file = os.path.join(bom_folder, f"{codeunit_name}.{codeunit_version}.bom.xml")
119
+ shutil.copyfile(generated_bom_file, target_bom_file)
120
+ self._protected_sc.format_xml_file(target_bom_file)
121
+
122
+ self.copy_source_files_to_output_directory()
123
+
124
+ @GeneralUtilities.check_arguments
125
+ def linting(self) -> None:
126
+ self._protected_sc.normalize_invisible_characters_of_files_in_folder(self.get_codeunit_folder(), ["java", "kt"])
127
+ # The rules of android-lint (for example whether a warning lets the linting fail) are configured in the "lint"-block
128
+ # of the build-file, so that the ruleset belongs to the codeunit and not to this implementation.
129
+ reports_folder = os.path.join(self.get_build_folder(), "reports")
130
+ GeneralUtilities.ensure_directory_does_not_exist(reports_folder)
131
+ self.__run_gradle_on_runner(["lintDebug"], reports_folder)
132
+
133
+ @GeneralUtilities.check_arguments
134
+ def do_common_tasks(self, current_codeunit_version: str) -> None:
135
+ self.do_common_tasks_base(current_codeunit_version)
136
+ self.__set_version_in_build_file(self.get_version_of_project())
137
+
138
+ @GeneralUtilities.check_arguments
139
+ def generate_reference(self) -> None:
140
+ self.generate_reference_using_docfx()
141
+
142
+ @GeneralUtilities.check_arguments
143
+ def run_testcases(self) -> None:
144
+ codeunit_name = self.get_codeunit_name()
145
+ reports_folder = os.path.join(self.get_build_folder(), "reports")
146
+ GeneralUtilities.ensure_directory_does_not_exist(reports_folder)
147
+ # "createDebugUnitTestCoverageReport" depends on "testDebugUnitTest", so this runs the testcases and writes the
148
+ # jacoco-report of their coverage.
149
+ self.__run_gradle_on_runner(["createDebugUnitTestCoverageReport"], reports_folder)
150
+ jacoco_file = os.path.join(reports_folder, "coverage", "test", "debug", "report.xml")
151
+ coverage_file = os.path.join(self.get_artifacts_folder(), "TestCoverage", "TestCoverage.xml")
152
+ GeneralUtilities.ensure_file_does_not_exist(coverage_file)
153
+ self.tfcps_Tools_General.convert_jacoco_report_to_cobertura_report(jacoco_file, coverage_file, codeunit_name, self.get_codeunit_folder(), [f"{codeunit_name}/src/main/java", f"{codeunit_name}/src/main/kotlin"])
154
+ self.run_testcases_common_post_task(self.get_repository_folder(), codeunit_name, True, self.get_target_environment_type())
155
+
156
+ @GeneralUtilities.check_arguments
157
+ def run_visual_regression_tests(self, update_baselines: bool) -> None:
158
+ """Runs the visual-regression-tests of the app, which are the unit-tests that capture screenshots with roborazzi
159
+ (rendered by robolectric). They run on the android-task-runner like every other gradle-task of the project, so
160
+ the baseline-images are always generated and compared in the same, fixed environment, regardless of whether the
161
+ build runs on a developer-machine or in the build-pipeline: a screenshot depends on the environment which
162
+ renders it.
163
+
164
+ If 'update_baselines' is true, the baseline-images in "Other/Resources/VisualRegressionBaselines" are
165
+ (re-)generated ("recordRoborazziDebug"); the folder is emptied before, so the baseline-image of a removed
166
+ screenshot does not remain. Otherwise the screenshots are compared against them ("verifyRoborazziDebug"); if a
167
+ comparison fails, the actual images and the comparison-images are preserved in
168
+ "Other/Artifacts/VisualRegressionTestFailures" and an exception is raised.
169
+
170
+ In a normal run of the unit-tests (see run_testcases) the screenshot-testcases run as well, but roborazzi does
171
+ neither record nor compare anything there."""
172
+ baselines_folder = self.get_visual_regression_baselines_folder()
173
+ if update_baselines:
174
+ GeneralUtilities.ensure_folder_exists_and_is_empty(baselines_folder)
175
+ self.__run_gradle_on_runner(["recordRoborazziDebug"], baselines_folder)
176
+ GeneralUtilities.assert_condition(0 < len(TFCPS_CodeUnitSpecific_Android_Functions.__get_png_files(baselines_folder)), f'Recording the visual-regression-baselines did not produce any image in "{baselines_folder}".')
177
+ else:
178
+ GeneralUtilities.assert_condition(0 < len(TFCPS_CodeUnitSpecific_Android_Functions.__get_png_files(baselines_folder)), f'There are no visual-regression-baselines in "{baselines_folder}". Generate them using "Other/QualityCheck/UpdateVisualRegressionBaselines.py".')
179
+ comparison_folder = os.path.join(self.get_build_folder(), TFCPS_CodeUnitSpecific_Android_Functions.get_visual_regression_comparison_folder_relative_to_build_folder())
180
+ GeneralUtilities.ensure_directory_does_not_exist(comparison_folder)
181
+ exit_code = self.__run_gradle_on_runner_and_get_exit_code(["verifyRoborazziDebug"], comparison_folder)
182
+ if exit_code != 0:
183
+ failures_folder = os.path.join(self.get_codeunit_folder(), TFCPS_CodeUnitSpecific_Android_Functions.__visual_regression_failures_folder_relative_to_codeunit)
184
+ GeneralUtilities.ensure_folder_exists_and_is_empty(failures_folder)
185
+ for image in TFCPS_CodeUnitSpecific_Android_Functions.__get_png_files(comparison_folder):
186
+ shutil.copyfile(image, os.path.join(failures_folder, os.path.basename(image)))
187
+ raise ValueError(f'The visual-regression-tests of the codeunit "{self.get_codeunit_name()}" failed (gradle exit-code {exit_code}). See the runner-log above for details. The screenshots of the failed comparisons are located in "{failures_folder}". If the change of the user-interface is intended, update the baselines using "Other/QualityCheck/UpdateVisualRegressionBaselines.py".')
188
+
189
+ @staticmethod
190
+ @GeneralUtilities.check_arguments
191
+ def __get_png_files(folder: str) -> list[str]:
192
+ if not os.path.isdir(folder):
193
+ return []
194
+ return [file for file in GeneralUtilities.get_direct_files_of_folder(folder) if file.lower().endswith(".png")]
195
+
196
+ @GeneralUtilities.check_arguments
197
+ def __get_gradle_arguments(self, tasks: list[str]) -> list[str]:
198
+ # "bash gradlew" instead of "./gradlew": the executable-bit of the wrapper-script does not survive a checkout on
199
+ # windows, from where the repository is transferred to the runner.
200
+ result: list[str] = ["gradlew", "--console=plain", "--no-daemon"]+tasks
201
+ if self.get_verbosity() == LogLevel.Debug:
202
+ result.append("--info")
203
+ return result
204
+
205
+ @GeneralUtilities.check_arguments
206
+ def __get_folders_which_are_not_transferred(self) -> list[str]:
207
+ # These are generated for the machine they were generated on: "build" and ".gradle" contain absolute paths and
208
+ # caches of that machine, and "local.properties" states the folder of the android-sdk of that machine (it is
209
+ # written by android-studio). On the runner they would describe folders which do not exist there.
210
+ gradle_project_folder = self.get_gradle_project_folder()
211
+ return [os.path.join(gradle_project_folder, "build"), os.path.join(gradle_project_folder, ".gradle"), os.path.join(gradle_project_folder, "local.properties")]
212
+
213
+ @GeneralUtilities.check_arguments
214
+ def __run_gradle_on_runner(self, tasks: list[str], result_folder: str) -> None:
215
+ """Runs the given gradle-tasks on an android-task-runner and writes the content of result_folder (which must be a
216
+ folder inside this codeunit) back into this codeunit. A failure of gradle raises an exception."""
217
+ self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", self.__get_gradle_arguments(tasks), self.get_gradle_project_folder(), result_folder, self.__get_folders_which_are_not_transferred())
218
+
219
+ @GeneralUtilities.check_arguments
220
+ def __run_gradle_on_runner_and_get_exit_code(self, tasks: list[str], result_folder: str) -> int:
221
+ """Like __run_gradle_on_runner, but a failure of gradle does not raise an exception: its exit-code is returned
222
+ instead. This is required when the result-folder is needed exactly when gradle fails (for example the
223
+ screenshots of failed visual-regression-tests), because the runner returns the result-folder only for a job
224
+ which succeeded. The exit-code is therefore written into the result-folder and the job itself always
225
+ succeeds."""
226
+ result_folder_relative_to_project = os.path.relpath(result_folder, self.get_gradle_project_folder()).replace("\\", "/")
227
+ script = TFCPS_CodeUnitSpecific_Android_Functions.get_script_which_preserves_exit_code(self.__get_gradle_arguments(tasks), result_folder_relative_to_project, TFCPS_CodeUnitSpecific_Android_Functions.__exit_code_file_name)
228
+ self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", ["-c", script], self.get_gradle_project_folder(), result_folder, self.__get_folders_which_are_not_transferred())
229
+ local_exit_code_file = os.path.join(result_folder, TFCPS_CodeUnitSpecific_Android_Functions.__exit_code_file_name)
230
+ GeneralUtilities.assert_file_exists(local_exit_code_file)
231
+ return int(GeneralUtilities.read_text_from_file(local_exit_code_file).strip())
232
+
233
+ @staticmethod
234
+ @GeneralUtilities.check_arguments
235
+ def get_script_which_preserves_exit_code(gradle_arguments: list[str], result_folder_relative_to_project: str, exit_code_file_name: str) -> str:
236
+ """Returns the bash-script which runs gradle with the given arguments and writes its exit-code into the given
237
+ file inside the given folder instead of failing."""
238
+ exit_code_file = f"{result_folder_relative_to_project}/{exit_code_file_name}"
239
+ return f"mkdir -p {shlex.quote(result_folder_relative_to_project)}; bash {shlex.join(gradle_arguments)}; echo $? > {shlex.quote(exit_code_file)}"
240
+
241
+ @GeneralUtilities.check_arguments
242
+ def __set_version_in_build_file(self, new_version: str) -> None:
243
+ build_file = self.get_build_file()
244
+ content = GeneralUtilities.read_text_from_file(build_file)
245
+ content = TFCPS_CodeUnitSpecific_Android_Functions.set_version_in_build_file_content(content, new_version)
246
+ GeneralUtilities.write_text_to_file(build_file, content)
247
+
248
+ @staticmethod
249
+ @GeneralUtilities.check_arguments
250
+ def get_version_code(version: str) -> int:
251
+ """Returns the versionCode for the given version. Android requires that the versionCode of an update is greater
252
+ than the one of the installed app, so it is derived from the version: every part of the version gets 3 digits."""
253
+ match = re.match(r"^(\d+)\.(\d+)\.(\d+)$", version)
254
+ GeneralUtilities.assert_condition(match is not None, f'"{version}" is not a version in the format "<major>.<minor>.<patch>".')
255
+ major, minor, patch = int(match.group(1)), int(match.group(2)), int(match.group(3))
256
+ GeneralUtilities.assert_condition(minor < 1000 and patch < 1000, f'The minor- and the patch-part of the version "{version}" must be less than 1000 to be representable as versionCode.')
257
+ result = major*1000000+minor*1000+patch
258
+ # 2100000000 is the greatest versionCode google-play accepts.
259
+ GeneralUtilities.assert_condition(result <= 2100000000, f'The version "{version}" is too large to be representable as versionCode.')
260
+ return result
261
+
262
+ @staticmethod
263
+ @GeneralUtilities.check_arguments
264
+ def set_version_in_build_file_content(content: str, new_version: str) -> str:
265
+ GeneralUtilities.assert_condition(len(TFCPS_CodeUnitSpecific_Android_Functions.__version_code_pattern.findall(content)) == 1, "The build-file must contain exactly one versionCode.")
266
+ GeneralUtilities.assert_condition(len(TFCPS_CodeUnitSpecific_Android_Functions.__version_name_pattern.findall(content)) == 1, "The build-file must contain exactly one versionName.")
267
+ version_code = TFCPS_CodeUnitSpecific_Android_Functions.get_version_code(new_version)
268
+ content = TFCPS_CodeUnitSpecific_Android_Functions.__version_code_pattern.sub(lambda match: f"{match.group('prefix')}{version_code}", content)
269
+ content = TFCPS_CodeUnitSpecific_Android_Functions.__version_name_pattern.sub(lambda match: f"{match.group('prefix')}{match.group('quote')}{new_version}{match.group('quote')}", content)
270
+ return content
271
+
272
+ @GeneralUtilities.check_arguments
273
+ def get_dependencies(self) -> dict[str, set[str]]:
274
+ return GeneralUtilities.merge_dependency_lists([TFCPS_CodeUnitSpecific_Android_Functions.get_dependencies_from_build_file_content(GeneralUtilities.read_text_from_file(self.get_build_file()))])
275
+
276
+ @staticmethod
277
+ @GeneralUtilities.check_arguments
278
+ def get_dependencies_from_build_file_content(content: str) -> list[Dependency]:
279
+ """Returns the dependencies ("<group-id>:<artifact-id>") and the plugins ("<plugin-id>") which are declared with an
280
+ explicit version in the given content of a build-file."""
281
+ result: list[Dependency] = []
282
+ for match in TFCPS_CodeUnitSpecific_Android_Functions.__dependency_pattern.finditer(content):
283
+ result.append(Dependency(f"{match.group('group')}:{match.group('artifact')}", match.group("version")))
284
+ for match in TFCPS_CodeUnitSpecific_Android_Functions.__plugin_pattern.finditer(content):
285
+ result.append(Dependency(match.group("id"), match.group("version")))
286
+ return result
287
+
288
+ @GeneralUtilities.check_arguments
289
+ def get_available_versions(self, dependencyname: str) -> list[str]:
290
+ return list(self.__get_released_versions(dependencyname).keys())
291
+
292
+ @GeneralUtilities.check_arguments
293
+ def __get_released_versions(self, dependencyname: str) -> dict[str, str]:
294
+ """Returns the released versions of the given dependency as dictionary from the version in the format
295
+ "<major>.<minor>.<patch>" (which the update-mechanism requires) to the version as it is published (for example
296
+ "4.18" for a dependency which omits the patch-part)."""
297
+ group_id, artifact_id = TFCPS_CodeUnitSpecific_Android_Functions.get_group_and_artifact_of_dependency(dependencyname)
298
+ found_in_any_repository: bool = False
299
+ result: dict[str, str] = {}
300
+ for repository in TFCPS_CodeUnitSpecific_Android_Functions.__maven_repositories:
301
+ metadata_address = f"{repository}/{group_id.replace('.', '/')}/{artifact_id}/maven-metadata.xml"
302
+ response = requests.get(metadata_address, timeout=60)
303
+ if response.status_code == 404:
304
+ continue
305
+ response.raise_for_status()
306
+ found_in_any_repository = True
307
+ for found_version in etree.fromstring(response.content).xpath("/metadata/versioning/versions/version/text()"):
308
+ normalized_version = TFCPS_CodeUnitSpecific_Android_Functions.normalize_released_version(str(found_version))
309
+ if normalized_version is not None:
310
+ result[normalized_version] = str(found_version)
311
+ GeneralUtilities.assert_condition(found_in_any_repository, f'The dependency "{dependencyname}" was not found in any of the repositories {", ".join(TFCPS_CodeUnitSpecific_Android_Functions.__maven_repositories)}.')
312
+ return result
313
+
314
+ @staticmethod
315
+ @GeneralUtilities.check_arguments
316
+ def normalize_released_version(version: str) -> str:
317
+ """Returns the given version in the format "<major>.<minor>.<patch>", or None if it is not the version of a release
318
+ (for example "1.2.0-alpha01")."""
319
+ if re.match(r"^(\d+)\.(\d+)\.(\d+)$", version) is not None:
320
+ return version
321
+ if re.match(r"^(\d+)\.(\d+)$", version) is not None:
322
+ return version+".0"
323
+ return None
324
+
325
+ @staticmethod
326
+ @GeneralUtilities.check_arguments
327
+ def get_group_and_artifact_of_dependency(dependencyname: str) -> tuple[str, str]:
328
+ """A dependency is identified by "<group-id>:<artifact-id>". A plugin is identified by its plugin-id, and gradle
329
+ resolves it through its marker-artifact "<plugin-id>:<plugin-id>.gradle.plugin"."""
330
+ parts = dependencyname.split(":")
331
+ if len(parts) == 1:
332
+ return (dependencyname, f"{dependencyname}.gradle.plugin")
333
+ GeneralUtilities.assert_condition(len(parts) == 2, f'The name of a dependency must have the format "<group-id>:<artifact-id>" or "<plugin-id>" but was "{dependencyname}".')
334
+ return (parts[0], parts[1])
335
+
336
+ @GeneralUtilities.check_arguments
337
+ def set_dependency_version(self, name: str, new_version: str) -> None:
338
+ build_file = self.get_build_file()
339
+ content = GeneralUtilities.read_text_from_file(build_file)
340
+ # The version is written in the format in which it is published, because that is the only one gradle can resolve.
341
+ published_version = self.__get_released_versions(name)[new_version]
342
+ content = TFCPS_CodeUnitSpecific_Android_Functions.set_dependency_version_in_build_file_content(content, name, published_version)
343
+ GeneralUtilities.write_text_to_file(build_file, content)
344
+
345
+ @staticmethod
346
+ @GeneralUtilities.check_arguments
347
+ def set_dependency_version_in_build_file_content(content: str, name: str, new_version: str) -> str:
348
+ if ":" in name:
349
+ group_id, artifact_id = TFCPS_CodeUnitSpecific_Android_Functions.get_group_and_artifact_of_dependency(name)
350
+
351
+ def replace_dependency(match: re.Match) -> str:
352
+ if match.group("group") == group_id and match.group("artifact") == artifact_id:
353
+ return f"{match.group('quote')}{group_id}:{artifact_id}:{new_version}{match.group('quote')}"
354
+ return match.group(0)
355
+ return TFCPS_CodeUnitSpecific_Android_Functions.__dependency_pattern.sub(replace_dependency, content)
356
+ else:
357
+ def replace_plugin(match: re.Match) -> str:
358
+ if match.group("id") == name:
359
+ version_start = match.start("version")-match.start(0)
360
+ version_end = match.end("version")-match.start(0)
361
+ return match.group(0)[:version_start]+new_version+match.group(0)[version_end:]
362
+ return match.group(0)
363
+ return TFCPS_CodeUnitSpecific_Android_Functions.__plugin_pattern.sub(replace_plugin, content)
364
+
365
+
366
+ class TFCPS_CodeUnitSpecific_Android_CLI:
367
+
368
+ @staticmethod
369
+ @GeneralUtilities.check_arguments
370
+ def parse(file: str) -> TFCPS_CodeUnitSpecific_Android_Functions:
371
+ parser = TFCPS_CodeUnitSpecific_Base_CLI.get_base_parser()
372
+ #add custom parameter if desired
373
+ args = parser.parse_args()
374
+ result: TFCPS_CodeUnitSpecific_Android_Functions = TFCPS_CodeUnitSpecific_Android_Functions(file, LogLevel(int(args.verbosity)), args.targetenvironmenttype, not args.nocache, args.ispremerge)
375
+ return result
File without changes
@@ -207,11 +207,12 @@ class TFCPS_CodeUnitSpecific_CPP_Functions(TFCPS_CodeUnitSpecific_Base):
207
207
 
208
208
  @GeneralUtilities.check_arguments
209
209
  def linting(self) -> None:
210
- # There is no single, universal C/C++-linter (unlike e.g. clippy for Rust), and a codeunit of this kind can
211
- # even vendor a foreign upstream whose sourcecode must not be changed here at all. What every codeunit of
212
- # this kind owns regardless of that is its own scripts, so only those are normalized by default.
210
+ # There is no single, universal C/C++-linter (unlike e.g. clippy for Rust). The invisible characters of the
211
+ # C/C++-sourcecode are normalized in the whole codeunit, which also affects a vendored foreign upstream.
213
212
  # TODO add real C/C++-linting (for example clang-format/clang-tidy) once a codeunit needs it for its own sourcecode.
214
- self._protected_sc.normalize_invisible_characters_of_files_in_folder(os.path.join(self.get_codeunit_folder(), "Other"), ["py", "md", "xml"])
213
+ codeunit_folder = self.get_codeunit_folder()
214
+ self._protected_sc.normalize_invisible_characters_of_files_in_folder(codeunit_folder, ["c", "cpp"])
215
+ self._protected_sc.normalize_invisible_characters_of_files_in_folder(os.path.join(codeunit_folder, "Other"), ["py", "md", "xml"])
215
216
 
216
217
  @GeneralUtilities.check_arguments
217
218
  def do_common_tasks(self, current_codeunit_version: str) -> None:
@@ -4,6 +4,7 @@ from urllib import request
4
4
  import time
5
5
  import ssl
6
6
  from datetime import timedelta,datetime
7
+ from typing import Callable
7
8
  from ...GeneralUtilities import GeneralUtilities, Platform
8
9
  from ...SCLog import LogLevel
9
10
  from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base,TFCPS_CodeUnitSpecific_Base_CLI
@@ -11,8 +12,13 @@ from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base,TFCPS_Code
11
12
 
12
13
  class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
13
14
 
15
+ # Maps the name of a dependency (the name of its folder in "Other/Resources/Dependencies") to the function which returns its
16
+ # available versions. The function gets the currently used version as argument.
17
+ __available_versions_functions:dict[str,Callable[[str],list[str]]]=None
18
+
14
19
  def __init__(self,current_file:str,verbosity:LogLevel,targetenvironmenttype:str,use_cache:bool,is_pre_merge:bool):
15
20
  super().__init__(current_file, verbosity,targetenvironmenttype,use_cache,is_pre_merge)
21
+ self.__available_versions_functions=dict[str,Callable[[str],list[str]]]()
16
22
 
17
23
  @GeneralUtilities.check_arguments
18
24
  def build(self,platforms:list[Platform],custom_arguments:dict[str,str]) -> None:
@@ -135,17 +141,45 @@ class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
135
141
  def run_testcases(self) -> None:
136
142
  pass#TODO
137
143
 
144
+ @GeneralUtilities.check_arguments
145
+ def register_available_versions_function(self,dependency_name:str,available_versions_function:Callable[[str],list[str]])->None:
146
+ """Registers the function which returns the available versions of the given dependency.
147
+ The dependencies of a docker-codeunit are the folders in "Other/Resources/Dependencies", and there is no general way to find out
148
+ where the available versions of such a dependency come from (npm, nuget, pypi, the tags of a git-repository, ...). So the
149
+ "Other/UpdateDependencies.py"-script of the codeunit registers one function per dependency before it calls update_dependencies.
150
+ The function gets the currently used version as argument and must return only versions in the format "<major>.<minor>.<patch>"."""
151
+ GeneralUtilities.assert_condition(dependency_name not in self.__available_versions_functions,f"A function for the available versions of dependency {dependency_name} is already registered.")
152
+ self.__available_versions_functions[dependency_name]=available_versions_function
153
+
154
+ @GeneralUtilities.check_arguments
155
+ def __get_dependencies_folder(self)->str:
156
+ return os.path.join(self.get_codeunit_folder(),"Other","Resources","Dependencies")
157
+
138
158
  @GeneralUtilities.check_arguments
139
159
  def get_dependencies(self)->dict[str,set[str]]:
140
- return dict[str,set[str]]()#TODO
141
-
160
+ """Returns the dependencies which are pinned in "Other/Resources/Dependencies/<dependency>/Version.txt"."""
161
+ result:dict[str,set[str]]=dict[str,set[str]]()
162
+ dependencies_folder:str=self.__get_dependencies_folder()
163
+ if not os.path.isdir(dependencies_folder):
164
+ return result
165
+ resources_folder:str=os.path.dirname(dependencies_folder)
166
+ for dependency_name in sorted(os.listdir(dependencies_folder)):
167
+ if os.path.isfile(os.path.join(dependencies_folder,dependency_name,"Version.txt")):
168
+ result[dependency_name]={self.tfcps_Tools_General.get_dependency_version_in_resources_folder(resources_folder,dependency_name)}
169
+ return result
170
+
142
171
  @GeneralUtilities.check_arguments
143
172
  def get_available_versions(self,dependencyname:str)->list[str]:
144
- return []#TODO
173
+ if dependencyname not in self.__available_versions_functions:
174
+ raise ValueError(f"No function for the available versions of dependency {dependencyname} is registered. Register one using register_available_versions_function or add the dependency to the ignored dependencies in the codeunit-file.")
175
+ current_version:str=next(iter(self.get_dependencies()[dependencyname]))
176
+ return self.__available_versions_functions[dependencyname](current_version)
145
177
 
146
178
  @GeneralUtilities.check_arguments
147
179
  def set_dependency_version(self,name:str,new_version:str)->None:
148
- raise ValueError(f"Operation is not implemented.")
180
+ version_file:str=os.path.join(self.__get_dependencies_folder(),name,"Version.txt")
181
+ GeneralUtilities.assert_file_exists(version_file)
182
+ GeneralUtilities.write_text_to_file(version_file,new_version)
149
183
 
150
184
  @GeneralUtilities.check_arguments
151
185
  @GeneralUtilities.deprecated("Use image_is_working_via_network instead, which tests the image over a user-defined docker-network without publishing a host-port.")
@@ -58,6 +58,7 @@ class TFCPS_CodeUnitSpecific_Maven_Functions(TFCPS_CodeUnitSpecific_Base):
58
58
 
59
59
  @GeneralUtilities.check_arguments
60
60
  def linting(self) -> None:
61
+ self._protected_sc.normalize_invisible_characters_of_files_in_folder(self.get_codeunit_folder(), ["java"])
61
62
  # checkstyle is configured in the pom-file of the codeunit, so that the ruleset belongs to the codeunit
62
63
  # and not to this implementation.
63
64
  self.__run_maven("-B checkstyle:check")