ScriptCollection 4.4.28__py3-none-any.whl → 4.4.30__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/ScriptCollectionCore.py +38 -13
- ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py +3 -1
- ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +28 -0
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +14 -5
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +7 -4
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +12 -2
- ScriptCollection/TFCPS/TFCPS_RemoteBuild.py +35 -5
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +8 -4
- {scriptcollection-4.4.28.dist-info → scriptcollection-4.4.30.dist-info}/METADATA +1 -1
- {scriptcollection-4.4.28.dist-info → scriptcollection-4.4.30.dist-info}/RECORD +13 -13
- {scriptcollection-4.4.28.dist-info → scriptcollection-4.4.30.dist-info}/WHEEL +0 -0
- {scriptcollection-4.4.28.dist-info → scriptcollection-4.4.30.dist-info}/entry_points.txt +0 -0
- {scriptcollection-4.4.28.dist-info → scriptcollection-4.4.30.dist-info}/top_level.txt +0 -0
|
@@ -40,7 +40,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
|
|
|
40
40
|
from .ProgramRunnerPopen import ProgramRunnerPopen
|
|
41
41
|
from .SCLog import SCLog, LogLevel
|
|
42
42
|
|
|
43
|
-
version = "4.4.
|
|
43
|
+
version = "4.4.30"
|
|
44
44
|
__version__ = version
|
|
45
45
|
|
|
46
46
|
class VSCodeWorkspaceShellTask:
|
|
@@ -242,13 +242,35 @@ class ScriptCollectionCore:
|
|
|
242
242
|
GeneralUtilities.ensure_directory_exists(result)
|
|
243
243
|
return result
|
|
244
244
|
|
|
245
|
-
|
|
245
|
+
@GeneralUtilities.check_arguments
|
|
246
|
+
def get_registry_credentials_file_in_container(self)->str:
|
|
247
|
+
"""Returns the file inside a build-container to which the host mounts its registry-credentials-file (see
|
|
248
|
+
TFCPS_CodeUnit_BuildCodeUnits.__run_scriptcollection_executable_in_container). It is an own path (and not the
|
|
249
|
+
configuration-folder of the container-user) because the home-folder inside the container depends on the user the
|
|
250
|
+
image runs as, while this path is defined by the mount and is therefore identical for both sides."""
|
|
251
|
+
return "/Workspace/ScriptCollectionConfiguration/RegistryCredentials.csv"
|
|
252
|
+
|
|
253
|
+
@GeneralUtilities.check_arguments
|
|
254
|
+
def get_registry_credentials_file_in_configuration_folder(self)->str:
|
|
255
|
+
"""Returns the registry-credentials-file in the configuration-folder of the current user, and creates it if it does not exist yet.
|
|
256
|
+
This is the file of the machine on which a command was started, so it is also the file which is mounted into a build-container
|
|
257
|
+
(in contrast to __get_docker_registry_credentials_file, which inside a container resolves to that mount instead)."""
|
|
246
258
|
result=os.path.join(self.get_global_cache_folder(),"RegistryCredentials.csv")
|
|
247
259
|
if not os.path.isfile(result):
|
|
248
260
|
GeneralUtilities.ensure_file_exists(result)
|
|
249
261
|
GeneralUtilities.write_lines_to_file(result,["RegistryName;Username;Password"])
|
|
250
262
|
return result
|
|
251
263
|
|
|
264
|
+
def __get_docker_registry_credentials_file(self)->str:
|
|
265
|
+
"""Returns the file which maps a registry to the credentials which are used for it. The file which the host mounted into a
|
|
266
|
+
build-container has precedence over the file in the configuration-folder of the current user: in a container that
|
|
267
|
+
configuration-folder belongs to the container-user and therefore never contains the configuration of the machine on which the
|
|
268
|
+
build was started."""
|
|
269
|
+
mounted_file=self.get_registry_credentials_file_in_container()
|
|
270
|
+
if os.path.isfile(mounted_file):
|
|
271
|
+
return mounted_file
|
|
272
|
+
return self.get_registry_credentials_file_in_configuration_folder()
|
|
273
|
+
|
|
252
274
|
def __load_credentials_if_required_and_available(self,registry_url:str,registry_username:str,registry_password:str)->tuple[str,str]:
|
|
253
275
|
"""Returns the credentials for the given registry: the ones which were passed by the caller, or - if the caller passed none - the
|
|
254
276
|
ones which are available for this registry on this machine (see __get_docker_registry_credentials). The last matching entry is
|
|
@@ -455,6 +477,10 @@ class ScriptCollectionCore:
|
|
|
455
477
|
if not force and self.local_docker_image_exists(image, tag):
|
|
456
478
|
self.log.log(f"Image \"{image_with_tag}\" is already available locally and therefore does not have to be pulled.", LogLevel.Debug)
|
|
457
479
|
return
|
|
480
|
+
#the image can be located in a registry which is not publicly readable, so the login to the registries which are defined for this
|
|
481
|
+
#machine is done before the pull. Without it the pull of such an image fails with "no basic auth credentials" respectively "401
|
|
482
|
+
#Unauthorized", even though the credentials are configured.
|
|
483
|
+
self.login_to_defined_docker_registries()
|
|
458
484
|
self.log.log(f"Pull image \"{image_with_tag}\"...")
|
|
459
485
|
self.run_program_with_retry("docker", f"pull {image_with_tag}", print_errors_as_information=True, print_live_output=self.log.loglevel == LogLevel.Debug)
|
|
460
486
|
|
|
@@ -4750,24 +4776,23 @@ OCR-content:
|
|
|
4750
4776
|
|
|
4751
4777
|
def prepare_build_pipeline_for_gitlab(self):
|
|
4752
4778
|
repository_folder:str=os.getcwd()
|
|
4753
|
-
self.__prepare_build_pipeline(repository_folder)
|
|
4779
|
+
self.__prepare_build_pipeline(repository_folder,"CI_COMMIT_REF_NAME")
|
|
4754
4780
|
|
|
4755
4781
|
def prepare_build_pipeline_for_github(self):
|
|
4756
4782
|
repository_folder:str=os.getcwd()
|
|
4757
4783
|
self.assert_scbuilder_image_in_github_workflow_matches_image_definition(repository_folder)
|
|
4758
|
-
self.__prepare_build_pipeline(repository_folder)
|
|
4784
|
+
self.__prepare_build_pipeline(repository_folder,"GITHUB_REF_NAME")
|
|
4759
4785
|
|
|
4760
|
-
def __ensure_branch_is_checked_out(self,repository_folder:str) -> None:
|
|
4761
|
-
|
|
4762
|
-
|
|
4763
|
-
|
|
4764
|
-
|
|
4765
|
-
|
|
4766
|
-
self.run_program_argsasarray("git",["checkout","-b",branch_name], repository_folder)
|
|
4786
|
+
def __ensure_branch_is_checked_out(self,repository_folder:str,branchname_environment_variable_name:str) -> None:
|
|
4787
|
+
"""Checks out the branch the pipeline was triggered for, because the runner checks out the commit in a detached-head-state while the versioning requires the real branch-name.
|
|
4788
|
+
"checkout -B" is used and not "checkout -b", because the branch already exists when the runner reuses the build-directory of a previous pipeline-run of the same branch."""
|
|
4789
|
+
branch_name:str=os.environ.get(branchname_environment_variable_name)
|
|
4790
|
+
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(branch_name), f"The environment-variable \"{branchname_environment_variable_name}\", which contains the name of the branch the pipeline runs for, is not set.")
|
|
4791
|
+
self.run_program_argsasarray("git",["checkout","-B",branch_name], repository_folder)
|
|
4767
4792
|
|
|
4768
|
-
def __prepare_build_pipeline(self,repository_folder:str) -> None:
|
|
4793
|
+
def __prepare_build_pipeline(self,repository_folder:str,branchname_environment_variable_name:str) -> None:
|
|
4769
4794
|
GeneralUtilities.assert_condition(self.is_running_in_build_container(), "This function should only be run in the build container.")
|
|
4770
|
-
self.__ensure_branch_is_checked_out(repository_folder)
|
|
4795
|
+
self.__ensure_branch_is_checked_out(repository_folder,branchname_environment_variable_name)
|
|
4771
4796
|
|
|
4772
4797
|
expected_image = self.__get_scbuilder_image_from_image_definition_file(repository_folder)
|
|
4773
4798
|
GeneralUtilities.assert_condition("scbuilder:" in expected_image, f"The SCBuilder-image '{expected_image}' defined in the image-definition-file of the repository is not a valid SCBuilder-image. It must contain 'scbuilder:'.")
|
|
@@ -167,7 +167,9 @@ class TFCPS_CodeUnitSpecific_CPP_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
167
167
|
#The result-folder is the sourcecode-folder itself: where a build-command of a C/C++-codeunit puts its
|
|
168
168
|
#output is defined by that command, so the only folder which is known to contain it is the one the commands
|
|
169
169
|
#run in - which is also the folder __collect_output_files searches afterwards.
|
|
170
|
-
|
|
170
|
+
#Nothing of a C/C++-codeunit is left out of the archive the runner gets: where a build-command puts its
|
|
171
|
+
#output is defined by that command, so there is no folder which is known to contain state of this machine.
|
|
172
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "/bin/sh", ["-c", " && ".join(self.macos_build_commands)], source_folder, source_folder, [])
|
|
171
173
|
self.__collect_output_files(source_folder, self.macos_output_file_patterns, "BuildResult_MacOS")
|
|
172
174
|
|
|
173
175
|
@GeneralUtilities.check_arguments
|
|
@@ -25,9 +25,16 @@ class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
25
25
|
artifacts_folder = GeneralUtilities.resolve_relative_path("Other/Artifacts", codeunit_folder)
|
|
26
26
|
app_artifacts_folder = os.path.join(artifacts_folder, "BuildResult_OCIImage")
|
|
27
27
|
GeneralUtilities.ensure_folder_exists_and_is_empty(app_artifacts_folder)
|
|
28
|
+
image_build_arguments = self.__get_build_arguments_for_images_of_repository()
|
|
29
|
+
#the base-image of the Dockerfile is resolved by buildkit and not by this process, so the login to the registries which are
|
|
30
|
+
#defined for this machine has to be done before the build. Without it buildkit requests the manifest of the base-image
|
|
31
|
+
#anonymously and the build fails with "401 Unauthorized" as soon as the base-image comes from a registry which is not
|
|
32
|
+
#publicly readable.
|
|
33
|
+
self._protected_sc.login_to_defined_docker_registries()
|
|
28
34
|
for platform in platforms:
|
|
29
35
|
#builder must be created once before with "docker buildx create --use"
|
|
30
36
|
args = ["buildx","build", "--platform",GeneralUtilities.platform_to_docker_platform_str(platform), "--pull", "--force-rm", "--progress=plain", "--build-arg", f"TargetEnvironmentType={self.get_target_environment_type()}", "--build-arg", f"CodeUnitName={codeunitname}", "--build-arg", f"CodeUnitVersion={codeunitversion}", "--build-arg", f"CodeUnitOwnerName={self.tfcps_Tools_General.get_codeunit_owner_name(self.get_codeunit_file())}", "--build-arg", f"CodeUnitOwnerEMailAddress={self.tfcps_Tools_General.get_codeunit_owner_emailaddress(self.get_codeunit_file())}", "--build-arg", f"Platform={GeneralUtilities.platform_to_dash_str(platform)}", "--build-arg", f"DotNetRuntime={GeneralUtilities.platform_to_dotnet_runtime_identifier(platform)}", "--build-arg", f"PlatformForGoVersion={GeneralUtilities.platform_to_go_runtime_identifier(platform)}"]
|
|
37
|
+
args = args+image_build_arguments
|
|
31
38
|
for custom_argument_key, custom_argument_value in custom_arguments.items():
|
|
32
39
|
args.append("--build-arg")
|
|
33
40
|
args.append(f"{custom_argument_key}={custom_argument_value}")
|
|
@@ -48,6 +55,27 @@ class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
48
55
|
self.copy_source_files_to_output_directory()
|
|
49
56
|
|
|
50
57
|
|
|
58
|
+
@GeneralUtilities.check_arguments
|
|
59
|
+
def __get_build_arguments_for_images_of_repository(self) -> list[str]:
|
|
60
|
+
"""Returns a '--build-arg'-pair for every image which the repository declares in
|
|
61
|
+
'.ScriptCollection/OCIImages/ImageDefinition.csv': the argument 'image_<imagename in lowercase>' contains the address with tag
|
|
62
|
+
from which that image has to be taken on this machine.
|
|
63
|
+
A Dockerfile has to declare its base-image through such an argument ('ARG image_debian' followed by 'FROM ${image_debian}')
|
|
64
|
+
instead of writing a registry-address into its 'FROM'-line: the address inside the Dockerfile is resolved by buildkit, so
|
|
65
|
+
neither the custom registry which is preferred on this machine nor the fallback-registry of the repository would apply to it
|
|
66
|
+
(see OCIImageManager.get_registry_address_for_image).
|
|
67
|
+
The arguments are named like the environment-variables which the local test-services get for the same purpose (see
|
|
68
|
+
TFCPS_Tools_General.pull_images_of_test_services), so an image is addressed by the same name everywhere.
|
|
69
|
+
Every declared image is passed and not only the one a Dockerfile uses, because which of them it needs is only known to the
|
|
70
|
+
Dockerfile itself - a multi-stage-build uses several of them."""
|
|
71
|
+
repository_folder = self.get_repository_folder()
|
|
72
|
+
oci_image_manager = self.tfcps_Tools_General.oci_image_manager
|
|
73
|
+
result: list[str] = []
|
|
74
|
+
for image_name in oci_image_manager.get_used_images_in_repository(repository_folder):
|
|
75
|
+
image_address_with_tag = oci_image_manager.get_registry_address_for_image_with_default_tag(repository_folder, image_name)
|
|
76
|
+
result = result+["--build-arg", f"image_{image_name.lower()}={image_address_with_tag}"]
|
|
77
|
+
return result
|
|
78
|
+
|
|
51
79
|
@GeneralUtilities.check_arguments
|
|
52
80
|
def __generate_sbom_for_docker_image(self) -> None:
|
|
53
81
|
codeunitname=self.get_codeunit_name()
|
|
@@ -68,6 +68,14 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
68
68
|
else:
|
|
69
69
|
src_folder = GeneralUtilities.resolve_relative_path(package_name, codeunit_folder) # TODO replace packagename
|
|
70
70
|
artifacts_folder = os.path.join(codeunit_folder, "Other", "Artifacts")
|
|
71
|
+
# The targets below which are built on a remote task-runner send the whole working-tree of the repository to
|
|
72
|
+
# it, and this folder is the one part of it which only applies to the machine the build was started on: its
|
|
73
|
+
# "package_config.json" states the folder of every package of the app as an absolute path of this machine,
|
|
74
|
+
# and the flutter-tool of the runner resolves the packages again only when that file is older than
|
|
75
|
+
# "pubspec.yaml". A transferred one therefore makes the runner compile against folders which do not exist
|
|
76
|
+
# there ("Error when reading '/C:/tools/flutter/packages/flutter/lib/material.dart'"), so it is left out of
|
|
77
|
+
# the archive; the runner generates the one of its own machine while it builds.
|
|
78
|
+
folders_which_are_not_transferred: list[str] = [os.path.join(src_folder, ".dart_tool")]
|
|
71
79
|
|
|
72
80
|
target_names: dict[str, str] = {
|
|
73
81
|
"web": "WebApplication",
|
|
@@ -98,7 +106,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
98
106
|
"environment as a runner-built one.", LogLevel.Warning)
|
|
99
107
|
self._protected_sc.run_with_epew("flutter", "build windows", src_folder)
|
|
100
108
|
else:
|
|
101
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder, windows_release_folder)
|
|
109
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder, windows_release_folder, folders_which_are_not_transferred)
|
|
102
110
|
windows_folder = os.path.join(artifacts_folder, "BuildResult_Windows")
|
|
103
111
|
GeneralUtilities.ensure_directory_does_not_exist(windows_folder)
|
|
104
112
|
GeneralUtilities.ensure_directory_exists(windows_folder)
|
|
@@ -120,7 +128,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
120
128
|
# builds), analogous to how ios-builds always run on the iOS-task-runner. See SCTaskRunnerMacOS and
|
|
121
129
|
# the remote-build-article in the reference.
|
|
122
130
|
macos_release_folder = os.path.join(src_folder, "build/macos/Build/Products/Release")
|
|
123
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder, macos_release_folder)
|
|
131
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder, macos_release_folder, folders_which_are_not_transferred)
|
|
124
132
|
macos_folder = os.path.join(artifacts_folder, "BuildResult_MacOS")
|
|
125
133
|
GeneralUtilities.ensure_directory_does_not_exist(macos_folder)
|
|
126
134
|
GeneralUtilities.ensure_directory_exists(macos_folder)
|
|
@@ -129,7 +137,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
129
137
|
# iOS-builds must run on macOS and therefore always run on the dedicated iOS-task-runner (uniform
|
|
130
138
|
# builds). See SCTaskRunnerIOS and the remote-build-article in the reference.
|
|
131
139
|
ios_release_folder = os.path.join(src_folder, "build/ios/iphoneos")
|
|
132
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder, ios_release_folder)
|
|
140
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder, ios_release_folder, folders_which_are_not_transferred)
|
|
133
141
|
ios_folder = os.path.join(artifacts_folder, "BuildResult_IOS")
|
|
134
142
|
GeneralUtilities.ensure_directory_does_not_exist(ios_folder)
|
|
135
143
|
GeneralUtilities.ensure_directory_exists(ios_folder)
|
|
@@ -139,7 +147,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
139
147
|
# remote-build-article in the reference), analogous to how ios-builds always run on a macOS-task-runner:
|
|
140
148
|
# the Android-SDK/NDK-toolchain no longer lives in SCBuilder itself, it moved into SCTaskRunnerAndroid.
|
|
141
149
|
aab_release_folder = os.path.join(src_folder, "build/app/outputs/bundle/release")
|
|
142
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder, aab_release_folder)
|
|
150
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder, aab_release_folder, folders_which_are_not_transferred)
|
|
143
151
|
# The app-bundle which came back from the runner is published as the build-result of this codeunit,
|
|
144
152
|
# analogous to the targets above: it is the artifact of an android-build, and every codeunit has to have
|
|
145
153
|
# an artifact whose name matches "BuildResult_.+" (see TFCPS_CodeUnit_BuildCodeUnit.build_codeunit).
|
|
@@ -168,7 +176,8 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
168
176
|
with zipfile.ZipFile(apks_file, "r") as apks_archive:
|
|
169
177
|
apks_archive.extract("universal.apk", apk_folder)
|
|
170
178
|
GeneralUtilities.ensure_file_does_not_exist(apks_file)
|
|
171
|
-
|
|
179
|
+
codeunit_version: str = self.tfcps_Tools_General.get_version_of_codeunit(self.get_codeunit_file())
|
|
180
|
+
os.rename(os.path.join(apk_folder, "universal.apk"), os.path.join(apk_folder, f"{codeunit_name}_v{codeunit_version}.apk"))
|
|
172
181
|
else:
|
|
173
182
|
raise ValueError(f"Not supported target: {target}")
|
|
174
183
|
self.__generate_bom_for_flutter_package(package_name)
|
|
@@ -188,7 +188,7 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
|
|
|
188
188
|
GeneralUtilities.assert_condition(developmentstate in (developmentstate_active, developmentstate_maintenance, developmentstate_inactive), f"Invalid development-state. Must be '{developmentstate_active}' or '{developmentstate_maintenance}' or '{developmentstate_inactive}' but was '{developmentstate}'.")
|
|
189
189
|
|
|
190
190
|
# check for mandatory files
|
|
191
|
-
files = ["Other/Build/Build.py", "Other/QualityCheck/Linting.py", "Other/Reference/GenerateReference.py"]
|
|
191
|
+
files = ["ReadMe.md", "Other/Build/Build.py", "Other/QualityCheck/Linting.py", "Other/Reference/GenerateReference.py"]
|
|
192
192
|
if self.tfcps_Tools_General.codeunit_has_testable_sourcecode(self.get_codeunit_file()):
|
|
193
193
|
# TODO check if the testsettings-section appears in the codeunit-file
|
|
194
194
|
files.append("Other/QualityCheck/RunTestcases.py")
|
|
@@ -364,14 +364,17 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
|
|
|
364
364
|
return self.__use_cache
|
|
365
365
|
|
|
366
366
|
@GeneralUtilities.check_arguments
|
|
367
|
-
def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str, result_folder: str) -> None:
|
|
367
|
+
def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str]) -> None:
|
|
368
368
|
"""Runs the given program for this codeunit on a remote task-runner that provides the given operating-system (see
|
|
369
369
|
TFCPS_RemoteBuild and the SCTaskRunnerWindows/SCTaskRunnerMacOS-codeunits). This is used for operating-system-bound
|
|
370
370
|
build-steps (e.g. flutter-windows- or flutter-ios-builds) which can not run on the current operating-system. After
|
|
371
371
|
the runner finished, the content of result_folder is written back into the local repository, so the build-step can
|
|
372
372
|
continue with the result as if it had been produced here; nothing else of the local repository is touched.
|
|
373
|
-
working_directory and result_folder must be folders inside this codeunit.
|
|
374
|
-
|
|
373
|
+
working_directory and result_folder must be folders inside this codeunit.
|
|
374
|
+
folders_which_are_not_transferred are the folders of this codeunit whose content only applies to this machine
|
|
375
|
+
and which are therefore left out of the repository-archive the runner gets (see
|
|
376
|
+
TFCPS_RemoteBuild.run_program_on_runner)."""
|
|
377
|
+
TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory, result_folder, folders_which_are_not_transferred)
|
|
375
378
|
|
|
376
379
|
@GeneralUtilities.check_arguments
|
|
377
380
|
def get_codeunit_folder(self)->str:
|
|
@@ -406,13 +406,23 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
406
406
|
#custom registries as a build on the host. Without the mount the build in the container would find no mapping at all - the
|
|
407
407
|
#configuration-folder inside the container belongs to the container-user and is empty - and would therefore fall back to the
|
|
408
408
|
#upstream-registries defined in the repository, which are rate-limited. The mount is read-only because a build never changes the file.
|
|
409
|
-
#Only this file is mounted and not the whole configuration-folder, so nothing else of it (in particular not EnvironmentVariables.csv or
|
|
410
|
-
#RegistryCredentials.csv, which contain credentials) is exposed to the container.
|
|
411
409
|
oci_image_manager = self.tfcps_tools_general.oci_image_manager
|
|
412
410
|
image_registries_file: str = oci_image_manager.get_image_registries_file_in_configuration_folder()
|
|
413
411
|
if os.path.isfile(image_registries_file):
|
|
414
412
|
mount_arguments += ["-v", f"{image_registries_file}:{oci_image_manager.get_image_registries_file_in_container()}:ro"]
|
|
415
413
|
|
|
414
|
+
#mount the machine-wide registry-credentials-file of this host (if it exists), so the build inside the container can log in to the
|
|
415
|
+
#registries which the images come from. Without the mount a custom registry which is not publicly readable can not be used inside the
|
|
416
|
+
#container at all, because the configuration-folder of the container-user does not contain the credentials of this host. The mount is
|
|
417
|
+
#read-only because a build never changes the file. The alternative way to provide the credentials are the environment-variables
|
|
418
|
+
#'OCIRegistry_<registryname>_*' (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables), which is the
|
|
419
|
+
#way a build-pipeline provides them from its own secret-store; both sources are used together.
|
|
420
|
+
#Only the two image-related files are mounted and not the whole configuration-folder, so nothing else of it (in particular not
|
|
421
|
+
#EnvironmentVariables.csv) is exposed to the container.
|
|
422
|
+
registry_credentials_file: str = self.sc.get_registry_credentials_file_in_configuration_folder()
|
|
423
|
+
if os.path.isfile(registry_credentials_file):
|
|
424
|
+
mount_arguments += ["-v", f"{registry_credentials_file}:{self.sc.get_registry_credentials_file_in_container()}:ro"]
|
|
425
|
+
|
|
416
426
|
#pass the environment-variables which are declared as required in <repository>/.ScriptCollection/ProductInformation.xml into the container
|
|
417
427
|
#so they do not have to be specified explicitly on every call. Their values are resolved from the user-specific
|
|
418
428
|
#configuration-file (see TFCPS_Tools_General.get_environment_variables_configuration_file), which only exists on the host.
|
|
@@ -76,7 +76,11 @@ class TFCPS_RemoteBuild:
|
|
|
76
76
|
over HTTPS to the runner, the runner runs the given program on its (correct) operating-system, and afterwards only the
|
|
77
77
|
folder in which that build-step produces its result is written back into the local repository. Everything else which
|
|
78
78
|
changed in the workspace of the runner stays there: it is state of that machine (its toolchain-paths, its caches) and
|
|
79
|
-
not a result, and a local repository is something a developer works in and which therefore must not be replaced.
|
|
79
|
+
not a result, and a local repository is something a developer works in and which therefore must not be replaced.
|
|
80
|
+
|
|
81
|
+
The only part of the working-tree which is not sent are the folders the caller states as
|
|
82
|
+
'folders_which_are_not_transferred' (see run_program_on_runner): a folder which describes the machine it was
|
|
83
|
+
generated on is state of this machine and not state of the repository, so on the runner it describes nothing."""
|
|
80
84
|
|
|
81
85
|
__sc: ScriptCollectionCore = None
|
|
82
86
|
|
|
@@ -89,7 +93,14 @@ class TFCPS_RemoteBuild:
|
|
|
89
93
|
return len(self.__load_runner_endpoints()) > 0
|
|
90
94
|
|
|
91
95
|
@GeneralUtilities.check_arguments
|
|
92
|
-
def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
|
|
96
|
+
def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str], poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
|
|
97
|
+
"""Runs the given program on a runner which provides the given operating-system, see the docstring of this class.
|
|
98
|
+
|
|
99
|
+
'folders_which_are_not_transferred' are folders (as paths inside the repository) which are left out of the
|
|
100
|
+
archive which is sent to the runner, because their content only applies to this machine (for example the
|
|
101
|
+
'.dart_tool'-folder of a flutter-package, whose 'package_config.json' states the folder of every package of
|
|
102
|
+
the app as an absolute path of this machine). Such a folder is generated again by the toolchain of the
|
|
103
|
+
runner while it builds; transferring it makes the runner use the paths of this machine instead."""
|
|
93
104
|
endpoints = self.__load_runner_endpoints()
|
|
94
105
|
if len(endpoints) == 0:
|
|
95
106
|
raise ValueError("No remote-build-runner is configured. Define runners either in "
|
|
@@ -100,7 +111,8 @@ class TFCPS_RemoteBuild:
|
|
|
100
111
|
working_directory_relative = os.path.relpath(working_directory, repository_folder).replace("\\", "/")
|
|
101
112
|
result_folder_relative = os.path.relpath(result_folder, repository_folder).replace("\\", "/")
|
|
102
113
|
self.__sc.log.log(f"Delegate '{program} {' '.join(arguments)}' (codeunit '{codeunit_name}', folder '{working_directory_relative}', result-folder '{result_folder_relative}') to the {required_os.value}-runner at {endpoint.url}...")
|
|
103
|
-
|
|
114
|
+
folders_which_are_not_transferred_relative = [os.path.relpath(folder, repository_folder).replace("\\", "/") for folder in folders_which_are_not_transferred]
|
|
115
|
+
archive_file = self.__create_repository_archive(repository_folder, folders_which_are_not_transferred_relative)
|
|
104
116
|
job_id: str = None
|
|
105
117
|
try:
|
|
106
118
|
metadata_headers = {
|
|
@@ -189,15 +201,33 @@ class TFCPS_RemoteBuild:
|
|
|
189
201
|
time.sleep(retry_interval_in_seconds)
|
|
190
202
|
|
|
191
203
|
@GeneralUtilities.check_arguments
|
|
192
|
-
def __create_repository_archive(self, repository_folder: str) -> str:
|
|
204
|
+
def __create_repository_archive(self, repository_folder: str, folders_which_are_not_transferred_relative: list[str]) -> str:
|
|
193
205
|
# Pack the entire repository-working-tree (including .git, uncommitted changes and git-ignored files) so the runner
|
|
194
206
|
# has the exact same state - including secrets that are required e.g. for signing windows-builds. Uses tarfile so
|
|
195
207
|
# it works identically on Windows and Linux (scbuildcodeunits runs on both) and preserves symlinks/permissions.
|
|
208
|
+
# The folders which describe this machine (see run_program_on_runner) are left out while the archive is packed
|
|
209
|
+
# and not removed from the working-tree before: the working-tree belongs to the developer, and a tool of theirs
|
|
210
|
+
# (for example the dart-extension of an ide, which resolves the packages of a flutter-package whenever its
|
|
211
|
+
# "pubspec.yaml" changes - which a build does) can generate such a folder again at any moment, so removing it
|
|
212
|
+
# would only be a race against that tool.
|
|
196
213
|
archive_file = os.path.join(GeneralUtilities.get_temp_folder(), f"sc-remotebuild-payload-{uuid.uuid4()}.tar.gz")
|
|
197
214
|
with tarfile.open(archive_file, "w:gz") as tar:
|
|
198
|
-
tar.add(repository_folder, arcname=".")
|
|
215
|
+
tar.add(repository_folder, arcname=".", filter=lambda entry: None if self.__entry_belongs_to_a_folder_which_is_not_transferred(entry.name, folders_which_are_not_transferred_relative) else entry)
|
|
199
216
|
return archive_file
|
|
200
217
|
|
|
218
|
+
@staticmethod
|
|
219
|
+
@GeneralUtilities.check_arguments
|
|
220
|
+
def __entry_belongs_to_a_folder_which_is_not_transferred(entry_name: str, folders_which_are_not_transferred_relative: list[str]) -> bool:
|
|
221
|
+
"""Whether the entry with the given name inside the archive (which is named relative to the repository-folder,
|
|
222
|
+
prefixed with './' because the repository is added under that name) is one of the given folders or lies inside
|
|
223
|
+
one of them."""
|
|
224
|
+
entry_path = entry_name.replace("\\", "/").removeprefix("./")
|
|
225
|
+
for folder in folders_which_are_not_transferred_relative:
|
|
226
|
+
folder_path = folder.replace("\\", "/").strip("/")
|
|
227
|
+
if entry_path == folder_path or entry_path.startswith(f"{folder_path}/"):
|
|
228
|
+
return True
|
|
229
|
+
return False
|
|
230
|
+
|
|
201
231
|
@GeneralUtilities.check_arguments
|
|
202
232
|
def __write_result_into_the_repository(self, result_bytes: bytes, result_folder: str) -> None:
|
|
203
233
|
"""Writes what the runner produced into the folder of the local repository in which the build-step produces its
|
|
@@ -389,10 +389,14 @@ class TFCPS_Tools_General:
|
|
|
389
389
|
return
|
|
390
390
|
version_element.text = current_version
|
|
391
391
|
file_ended_with_newline: bool = GeneralUtilities.read_binary_from_file(codeunit_file).endswith(b"\n")
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
392
|
+
#the file is written through a binary file-object instead of by its name because the serializer opens a file which is given by its name in text-mode, which
|
|
393
|
+
#translates every line-separator it writes to the one of the current operating-system: on windows that converted the codeunit-files to crlf on every build,
|
|
394
|
+
#while the trailing newline below (which was always written in binary-mode) stayed lf, so the files ended up with mixed line-separators. The parser normalizes
|
|
395
|
+
#the line-separators of the file it reads to lf anyway (this is defined by the xml-specification), so a codeunit-file is written with lf on every operating-system.
|
|
396
|
+
with open(codeunit_file, "wb") as file_object:
|
|
397
|
+
tree.write(file_object, xml_declaration=True, encoding="utf-8")
|
|
398
|
+
if file_ended_with_newline:
|
|
399
|
+
#the serializer does not terminate the last line, so without this the trailing newline of the file would get lost on every write.
|
|
396
400
|
file_object.write(b"\n")
|
|
397
401
|
|
|
398
402
|
@GeneralUtilities.check_arguments
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scriptcollection
|
|
3
|
-
Version: 4.4.
|
|
3
|
+
Version: 4.4.30
|
|
4
4
|
Summary: The ScriptCollection is the place for reusable scripts.
|
|
5
5
|
Author-email: Marius Göcke <marius.goecke@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/anionDev/ScriptCollection
|
|
@@ -12,7 +12,7 @@ ScriptCollection/ProgramRunnerSudo.py,sha256=ot9Ib-plQ8wejR9M2XXfnx6GR-va9k8SH1g
|
|
|
12
12
|
ScriptCollection/RepositoryDiagramGenerator.py,sha256=K_u2NW_p4ozdWD50irJEkkcDVa70TcUJpZN63TeXSLw,7092
|
|
13
13
|
ScriptCollection/RepositoryHistoryAnalyzer.py,sha256=rD8eC5lMUe5MHmCBHfYft_G-N9IbvtsFGkU7EMwPjrY,5779
|
|
14
14
|
ScriptCollection/SCLog.py,sha256=Fjf8Pdi63cXHGMxL_aI6A0dwGm7QIfrv3czjvpSUlaM,4705
|
|
15
|
-
ScriptCollection/ScriptCollectionCore.py,sha256=
|
|
15
|
+
ScriptCollection/ScriptCollectionCore.py,sha256=BGPQ7NXBPAO2jWEhu9lj-nL4wRMCmDyXSa-61rpa5Y8,288002
|
|
16
16
|
ScriptCollection/TestKind.py,sha256=EBTLR8yBz5VmWA2NZIpDTbQNg6tkfv_kAsIzEJ--mCI,1756
|
|
17
17
|
ScriptCollection/UnassignedVariablesCheck.py,sha256=8xNcp1D5HeOLO1a1NEerq2TsnpFhw7yzYPkaay-oisY,3804
|
|
18
18
|
ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -35,29 +35,29 @@ ScriptCollection/TFCPS/PackageSource.py,sha256=tQoMbxrIbV3mYHKH2wgOoGzIHY1TvBH8E
|
|
|
35
35
|
ScriptCollection/TFCPS/PlantUMLRenderer.py,sha256=upnafiGKV-SdRI8iRBkbbWHN2u6v9KxoswdKi12AwoY,11060
|
|
36
36
|
ScriptCollection/TFCPS/SCTaskRunnerServer.py,sha256=tub8_1QBu0nWAaGnBWp2nErOW-SOHf_UL6qjOSGbnxU,31026
|
|
37
37
|
ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py,sha256=67_DQwoH5bhxxSmwtuAeoW1NOCMiLMzOy48OqlJX-KY,1744
|
|
38
|
-
ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=
|
|
38
|
+
ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=umUfQDErp7xFLkDnSbQMxzUWnKtcGlrZnIvFcTpfGJs,40499
|
|
39
39
|
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=Mj9t1SyRpkYZE74cLsGWYv4xB89gXL2ld-RX3xvvGdw,9835
|
|
40
|
-
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=
|
|
40
|
+
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=6FmsF2M3GI8-Xjay0oUaLPTQ8yrHVSUiF_6mc01YoRM,49601
|
|
41
41
|
ScriptCollection/TFCPS/TFCPS_CreateRelease.py,sha256=kpEmSnfzfgB4OjcCuS_X6jgW_ftcphxEnjjoLoAHTLY,6582
|
|
42
42
|
ScriptCollection/TFCPS/TFCPS_Generic.py,sha256=mRBcf7XuNWOSXbZYc3NuJow18AMZ30D6Z-BtyNYfNcM,2072
|
|
43
43
|
ScriptCollection/TFCPS/TFCPS_MergeToMain.py,sha256=Krt5oyGNcRAN3szsgU7LoTh8yVTOdHW-Kq7R9MDDV-Q,7673
|
|
44
44
|
ScriptCollection/TFCPS/TFCPS_MergeToStable.py,sha256=oYEhz5ZWMnbu97_CcYEqdDxakUFVppyOVQiJzADq59c,22434
|
|
45
45
|
ScriptCollection/TFCPS/TFCPS_OCIImageSecretScan.py,sha256=rhw0glr4gZ5enG9faOD51xE4MMK1cIdNjUdaswiMsrY,14622
|
|
46
46
|
ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py,sha256=XLVsFTA1jvfDbZ7wYJgIdEByZhF_bYVbA5jkisuvK4k,2322
|
|
47
|
-
ScriptCollection/TFCPS/TFCPS_RemoteBuild.py,sha256=
|
|
48
|
-
ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=
|
|
47
|
+
ScriptCollection/TFCPS/TFCPS_RemoteBuild.py,sha256=2Tx5HLk1ZftBWDCTiGZdP2QiiGNyWsEuTCSt33dNQjc,23891
|
|
48
|
+
ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=eGtOfeJEiHV-JoxkbBGw0EP9vDJJAlQWKpJ0hUASwUY,160284
|
|
49
49
|
ScriptCollection/TFCPS/TFCPS_VisualRegressionTests.py,sha256=6_63udxHnPlu_TMk_ftLZTnzriJO8MCxVCVAsweZqHs,37073
|
|
50
50
|
ScriptCollection/TFCPS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
51
|
-
ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py,sha256=
|
|
51
|
+
ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py,sha256=BNZPMjGiWLMBH7QDtP06WEVLMnI38NLX4Dpd13bSbi4,16614
|
|
52
52
|
ScriptCollection/TFCPS/CPP/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
53
|
-
ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=
|
|
53
|
+
ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=zoy1uwgK_9Fxji8mwHoa6X-VQRc9AMKcT1tfJGZMQeI,22343
|
|
54
54
|
ScriptCollection/TFCPS/Docker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
55
55
|
ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py,sha256=bT6Gd5pQpZCw4OQz6HWkPCSn5z__eUUEisABLDSxd0o,200
|
|
56
56
|
ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py,sha256=QyjOfMY22JWCvKjMelHiDWbJiWqotOfebpJpgDUaoO4,237
|
|
57
57
|
ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py,sha256=i0zEGehj0sttxjjZtoq2KFSKp_ulxVyWp_ZgAhIY_So,241
|
|
58
58
|
ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=GCFzDNAnmm40EeqOYgD_EeuEwDY58ujwJ1-Zn_naamc,61797
|
|
59
59
|
ScriptCollection/TFCPS/DotNet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
60
|
-
ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=
|
|
60
|
+
ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=dW7KoCvpIbjGaN-SRoaKPmF02GVVyT7zJAVR0yIjtag,52842
|
|
61
61
|
ScriptCollection/TFCPS/Flutter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
62
62
|
ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py,sha256=m4Pax-U6o5XuspLnLfnWjyQMtoC7zBDDNJW9EhLWjeE,9012
|
|
63
63
|
ScriptCollection/TFCPS/Go/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -71,8 +71,8 @@ ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=yyjWPMHQjPLkML
|
|
|
71
71
|
ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
72
72
|
ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py,sha256=5z990qyO5durX07xRWRwju1hl6wpjQAy2f-yRwJ-7s0,11856
|
|
73
73
|
ScriptCollection/TFCPS/TypeScript/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
74
|
-
scriptcollection-4.4.
|
|
75
|
-
scriptcollection-4.4.
|
|
76
|
-
scriptcollection-4.4.
|
|
77
|
-
scriptcollection-4.4.
|
|
78
|
-
scriptcollection-4.4.
|
|
74
|
+
scriptcollection-4.4.30.dist-info/METADATA,sha256=afxz3vYiDErPCZNzqM6zrv2xVN4yce3S1-MvBo6ebcg,7690
|
|
75
|
+
scriptcollection-4.4.30.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
76
|
+
scriptcollection-4.4.30.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
|
|
77
|
+
scriptcollection-4.4.30.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
|
|
78
|
+
scriptcollection-4.4.30.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|