ScriptCollection 4.4.28__py3-none-any.whl → 4.4.30__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -40,7 +40,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
40
40
  from .ProgramRunnerPopen import ProgramRunnerPopen
41
41
  from .SCLog import SCLog, LogLevel
42
42
 
43
- version = "4.4.28"
43
+ version = "4.4.30"
44
44
  __version__ = version
45
45
 
46
46
  class VSCodeWorkspaceShellTask:
@@ -242,13 +242,35 @@ class ScriptCollectionCore:
242
242
  GeneralUtilities.ensure_directory_exists(result)
243
243
  return result
244
244
 
245
- def __get_docker_registry_credentials_file(self)->str:
245
+ @GeneralUtilities.check_arguments
246
+ def get_registry_credentials_file_in_container(self)->str:
247
+ """Returns the file inside a build-container to which the host mounts its registry-credentials-file (see
248
+ TFCPS_CodeUnit_BuildCodeUnits.__run_scriptcollection_executable_in_container). It is an own path (and not the
249
+ configuration-folder of the container-user) because the home-folder inside the container depends on the user the
250
+ image runs as, while this path is defined by the mount and is therefore identical for both sides."""
251
+ return "/Workspace/ScriptCollectionConfiguration/RegistryCredentials.csv"
252
+
253
+ @GeneralUtilities.check_arguments
254
+ def get_registry_credentials_file_in_configuration_folder(self)->str:
255
+ """Returns the registry-credentials-file in the configuration-folder of the current user, and creates it if it does not exist yet.
256
+ This is the file of the machine on which a command was started, so it is also the file which is mounted into a build-container
257
+ (in contrast to __get_docker_registry_credentials_file, which inside a container resolves to that mount instead)."""
246
258
  result=os.path.join(self.get_global_cache_folder(),"RegistryCredentials.csv")
247
259
  if not os.path.isfile(result):
248
260
  GeneralUtilities.ensure_file_exists(result)
249
261
  GeneralUtilities.write_lines_to_file(result,["RegistryName;Username;Password"])
250
262
  return result
251
263
 
264
+ def __get_docker_registry_credentials_file(self)->str:
265
+ """Returns the file which maps a registry to the credentials which are used for it. The file which the host mounted into a
266
+ build-container has precedence over the file in the configuration-folder of the current user: in a container that
267
+ configuration-folder belongs to the container-user and therefore never contains the configuration of the machine on which the
268
+ build was started."""
269
+ mounted_file=self.get_registry_credentials_file_in_container()
270
+ if os.path.isfile(mounted_file):
271
+ return mounted_file
272
+ return self.get_registry_credentials_file_in_configuration_folder()
273
+
252
274
  def __load_credentials_if_required_and_available(self,registry_url:str,registry_username:str,registry_password:str)->tuple[str,str]:
253
275
  """Returns the credentials for the given registry: the ones which were passed by the caller, or - if the caller passed none - the
254
276
  ones which are available for this registry on this machine (see __get_docker_registry_credentials). The last matching entry is
@@ -455,6 +477,10 @@ class ScriptCollectionCore:
455
477
  if not force and self.local_docker_image_exists(image, tag):
456
478
  self.log.log(f"Image \"{image_with_tag}\" is already available locally and therefore does not have to be pulled.", LogLevel.Debug)
457
479
  return
480
+ #the image can be located in a registry which is not publicly readable, so the login to the registries which are defined for this
481
+ #machine is done before the pull. Without it the pull of such an image fails with "no basic auth credentials" respectively "401
482
+ #Unauthorized", even though the credentials are configured.
483
+ self.login_to_defined_docker_registries()
458
484
  self.log.log(f"Pull image \"{image_with_tag}\"...")
459
485
  self.run_program_with_retry("docker", f"pull {image_with_tag}", print_errors_as_information=True, print_live_output=self.log.loglevel == LogLevel.Debug)
460
486
 
@@ -4750,24 +4776,23 @@ OCR-content:
4750
4776
 
4751
4777
  def prepare_build_pipeline_for_gitlab(self):
4752
4778
  repository_folder:str=os.getcwd()
4753
- self.__prepare_build_pipeline(repository_folder)
4779
+ self.__prepare_build_pipeline(repository_folder,"CI_COMMIT_REF_NAME")
4754
4780
 
4755
4781
  def prepare_build_pipeline_for_github(self):
4756
4782
  repository_folder:str=os.getcwd()
4757
4783
  self.assert_scbuilder_image_in_github_workflow_matches_image_definition(repository_folder)
4758
- self.__prepare_build_pipeline(repository_folder)
4784
+ self.__prepare_build_pipeline(repository_folder,"GITHUB_REF_NAME")
4759
4785
 
4760
- def __ensure_branch_is_checked_out(self,repository_folder:str) -> None:
4761
- current_branch_result = self.run_program_argsasarray("git", ["symbolic-ref", "--short", "HEAD"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
4762
- repository_is_on_any_branch:bool = current_branch_result[0] == 0
4763
- if not repository_is_on_any_branch:
4764
- timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
4765
- branch_name:str=f"pipeline_{timestamp}"
4766
- self.run_program_argsasarray("git",["checkout","-b",branch_name], repository_folder)
4786
+ def __ensure_branch_is_checked_out(self,repository_folder:str,branchname_environment_variable_name:str) -> None:
4787
+ """Checks out the branch the pipeline was triggered for, because the runner checks out the commit in a detached-head-state while the versioning requires the real branch-name.
4788
+ "checkout -B" is used and not "checkout -b", because the branch already exists when the runner reuses the build-directory of a previous pipeline-run of the same branch."""
4789
+ branch_name:str=os.environ.get(branchname_environment_variable_name)
4790
+ GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(branch_name), f"The environment-variable \"{branchname_environment_variable_name}\", which contains the name of the branch the pipeline runs for, is not set.")
4791
+ self.run_program_argsasarray("git",["checkout","-B",branch_name], repository_folder)
4767
4792
 
4768
- def __prepare_build_pipeline(self,repository_folder:str) -> None:
4793
+ def __prepare_build_pipeline(self,repository_folder:str,branchname_environment_variable_name:str) -> None:
4769
4794
  GeneralUtilities.assert_condition(self.is_running_in_build_container(), "This function should only be run in the build container.")
4770
- self.__ensure_branch_is_checked_out(repository_folder)
4795
+ self.__ensure_branch_is_checked_out(repository_folder,branchname_environment_variable_name)
4771
4796
 
4772
4797
  expected_image = self.__get_scbuilder_image_from_image_definition_file(repository_folder)
4773
4798
  GeneralUtilities.assert_condition("scbuilder:" in expected_image, f"The SCBuilder-image '{expected_image}' defined in the image-definition-file of the repository is not a valid SCBuilder-image. It must contain 'scbuilder:'.")
@@ -167,7 +167,9 @@ class TFCPS_CodeUnitSpecific_CPP_Functions(TFCPS_CodeUnitSpecific_Base):
167
167
  #The result-folder is the sourcecode-folder itself: where a build-command of a C/C++-codeunit puts its
168
168
  #output is defined by that command, so the only folder which is known to contain it is the one the commands
169
169
  #run in - which is also the folder __collect_output_files searches afterwards.
170
- self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "/bin/sh", ["-c", " && ".join(self.macos_build_commands)], source_folder, source_folder)
170
+ #Nothing of a C/C++-codeunit is left out of the archive the runner gets: where a build-command puts its
171
+ #output is defined by that command, so there is no folder which is known to contain state of this machine.
172
+ self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "/bin/sh", ["-c", " && ".join(self.macos_build_commands)], source_folder, source_folder, [])
171
173
  self.__collect_output_files(source_folder, self.macos_output_file_patterns, "BuildResult_MacOS")
172
174
 
173
175
  @GeneralUtilities.check_arguments
@@ -25,9 +25,16 @@ class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
25
25
  artifacts_folder = GeneralUtilities.resolve_relative_path("Other/Artifacts", codeunit_folder)
26
26
  app_artifacts_folder = os.path.join(artifacts_folder, "BuildResult_OCIImage")
27
27
  GeneralUtilities.ensure_folder_exists_and_is_empty(app_artifacts_folder)
28
+ image_build_arguments = self.__get_build_arguments_for_images_of_repository()
29
+ #the base-image of the Dockerfile is resolved by buildkit and not by this process, so the login to the registries which are
30
+ #defined for this machine has to be done before the build. Without it buildkit requests the manifest of the base-image
31
+ #anonymously and the build fails with "401 Unauthorized" as soon as the base-image comes from a registry which is not
32
+ #publicly readable.
33
+ self._protected_sc.login_to_defined_docker_registries()
28
34
  for platform in platforms:
29
35
  #builder must be created once before with "docker buildx create --use"
30
36
  args = ["buildx","build", "--platform",GeneralUtilities.platform_to_docker_platform_str(platform), "--pull", "--force-rm", "--progress=plain", "--build-arg", f"TargetEnvironmentType={self.get_target_environment_type()}", "--build-arg", f"CodeUnitName={codeunitname}", "--build-arg", f"CodeUnitVersion={codeunitversion}", "--build-arg", f"CodeUnitOwnerName={self.tfcps_Tools_General.get_codeunit_owner_name(self.get_codeunit_file())}", "--build-arg", f"CodeUnitOwnerEMailAddress={self.tfcps_Tools_General.get_codeunit_owner_emailaddress(self.get_codeunit_file())}", "--build-arg", f"Platform={GeneralUtilities.platform_to_dash_str(platform)}", "--build-arg", f"DotNetRuntime={GeneralUtilities.platform_to_dotnet_runtime_identifier(platform)}", "--build-arg", f"PlatformForGoVersion={GeneralUtilities.platform_to_go_runtime_identifier(platform)}"]
37
+ args = args+image_build_arguments
31
38
  for custom_argument_key, custom_argument_value in custom_arguments.items():
32
39
  args.append("--build-arg")
33
40
  args.append(f"{custom_argument_key}={custom_argument_value}")
@@ -48,6 +55,27 @@ class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
48
55
  self.copy_source_files_to_output_directory()
49
56
 
50
57
 
58
+ @GeneralUtilities.check_arguments
59
+ def __get_build_arguments_for_images_of_repository(self) -> list[str]:
60
+ """Returns a '--build-arg'-pair for every image which the repository declares in
61
+ '.ScriptCollection/OCIImages/ImageDefinition.csv': the argument 'image_<imagename in lowercase>' contains the address with tag
62
+ from which that image has to be taken on this machine.
63
+ A Dockerfile has to declare its base-image through such an argument ('ARG image_debian' followed by 'FROM ${image_debian}')
64
+ instead of writing a registry-address into its 'FROM'-line: the address inside the Dockerfile is resolved by buildkit, so
65
+ neither the custom registry which is preferred on this machine nor the fallback-registry of the repository would apply to it
66
+ (see OCIImageManager.get_registry_address_for_image).
67
+ The arguments are named like the environment-variables which the local test-services get for the same purpose (see
68
+ TFCPS_Tools_General.pull_images_of_test_services), so an image is addressed by the same name everywhere.
69
+ Every declared image is passed and not only the one a Dockerfile uses, because which of them it needs is only known to the
70
+ Dockerfile itself - a multi-stage-build uses several of them."""
71
+ repository_folder = self.get_repository_folder()
72
+ oci_image_manager = self.tfcps_Tools_General.oci_image_manager
73
+ result: list[str] = []
74
+ for image_name in oci_image_manager.get_used_images_in_repository(repository_folder):
75
+ image_address_with_tag = oci_image_manager.get_registry_address_for_image_with_default_tag(repository_folder, image_name)
76
+ result = result+["--build-arg", f"image_{image_name.lower()}={image_address_with_tag}"]
77
+ return result
78
+
51
79
  @GeneralUtilities.check_arguments
52
80
  def __generate_sbom_for_docker_image(self) -> None:
53
81
  codeunitname=self.get_codeunit_name()
@@ -68,6 +68,14 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
68
68
  else:
69
69
  src_folder = GeneralUtilities.resolve_relative_path(package_name, codeunit_folder) # TODO replace packagename
70
70
  artifacts_folder = os.path.join(codeunit_folder, "Other", "Artifacts")
71
+ # The targets below which are built on a remote task-runner send the whole working-tree of the repository to
72
+ # it, and this folder is the one part of it which only applies to the machine the build was started on: its
73
+ # "package_config.json" states the folder of every package of the app as an absolute path of this machine,
74
+ # and the flutter-tool of the runner resolves the packages again only when that file is older than
75
+ # "pubspec.yaml". A transferred one therefore makes the runner compile against folders which do not exist
76
+ # there ("Error when reading '/C:/tools/flutter/packages/flutter/lib/material.dart'"), so it is left out of
77
+ # the archive; the runner generates the one of its own machine while it builds.
78
+ folders_which_are_not_transferred: list[str] = [os.path.join(src_folder, ".dart_tool")]
71
79
 
72
80
  target_names: dict[str, str] = {
73
81
  "web": "WebApplication",
@@ -98,7 +106,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
98
106
  "environment as a runner-built one.", LogLevel.Warning)
99
107
  self._protected_sc.run_with_epew("flutter", "build windows", src_folder)
100
108
  else:
101
- self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder, windows_release_folder)
109
+ self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder, windows_release_folder, folders_which_are_not_transferred)
102
110
  windows_folder = os.path.join(artifacts_folder, "BuildResult_Windows")
103
111
  GeneralUtilities.ensure_directory_does_not_exist(windows_folder)
104
112
  GeneralUtilities.ensure_directory_exists(windows_folder)
@@ -120,7 +128,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
120
128
  # builds), analogous to how ios-builds always run on the iOS-task-runner. See SCTaskRunnerMacOS and
121
129
  # the remote-build-article in the reference.
122
130
  macos_release_folder = os.path.join(src_folder, "build/macos/Build/Products/Release")
123
- self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder, macos_release_folder)
131
+ self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder, macos_release_folder, folders_which_are_not_transferred)
124
132
  macos_folder = os.path.join(artifacts_folder, "BuildResult_MacOS")
125
133
  GeneralUtilities.ensure_directory_does_not_exist(macos_folder)
126
134
  GeneralUtilities.ensure_directory_exists(macos_folder)
@@ -129,7 +137,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
129
137
  # iOS-builds must run on macOS and therefore always run on the dedicated iOS-task-runner (uniform
130
138
  # builds). See SCTaskRunnerIOS and the remote-build-article in the reference.
131
139
  ios_release_folder = os.path.join(src_folder, "build/ios/iphoneos")
132
- self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder, ios_release_folder)
140
+ self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder, ios_release_folder, folders_which_are_not_transferred)
133
141
  ios_folder = os.path.join(artifacts_folder, "BuildResult_IOS")
134
142
  GeneralUtilities.ensure_directory_does_not_exist(ios_folder)
135
143
  GeneralUtilities.ensure_directory_exists(ios_folder)
@@ -139,7 +147,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
139
147
  # remote-build-article in the reference), analogous to how ios-builds always run on a macOS-task-runner:
140
148
  # the Android-SDK/NDK-toolchain no longer lives in SCBuilder itself, it moved into SCTaskRunnerAndroid.
141
149
  aab_release_folder = os.path.join(src_folder, "build/app/outputs/bundle/release")
142
- self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder, aab_release_folder)
150
+ self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder, aab_release_folder, folders_which_are_not_transferred)
143
151
  # The app-bundle which came back from the runner is published as the build-result of this codeunit,
144
152
  # analogous to the targets above: it is the artifact of an android-build, and every codeunit has to have
145
153
  # an artifact whose name matches "BuildResult_.+" (see TFCPS_CodeUnit_BuildCodeUnit.build_codeunit).
@@ -168,7 +176,8 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
168
176
  with zipfile.ZipFile(apks_file, "r") as apks_archive:
169
177
  apks_archive.extract("universal.apk", apk_folder)
170
178
  GeneralUtilities.ensure_file_does_not_exist(apks_file)
171
- os.rename(os.path.join(apk_folder, "universal.apk"), os.path.join(apk_folder, f"{codeunit_name}.apk"))
179
+ codeunit_version: str = self.tfcps_Tools_General.get_version_of_codeunit(self.get_codeunit_file())
180
+ os.rename(os.path.join(apk_folder, "universal.apk"), os.path.join(apk_folder, f"{codeunit_name}_v{codeunit_version}.apk"))
172
181
  else:
173
182
  raise ValueError(f"Not supported target: {target}")
174
183
  self.__generate_bom_for_flutter_package(package_name)
@@ -188,7 +188,7 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
188
188
  GeneralUtilities.assert_condition(developmentstate in (developmentstate_active, developmentstate_maintenance, developmentstate_inactive), f"Invalid development-state. Must be '{developmentstate_active}' or '{developmentstate_maintenance}' or '{developmentstate_inactive}' but was '{developmentstate}'.")
189
189
 
190
190
  # check for mandatory files
191
- files = ["Other/Build/Build.py", "Other/QualityCheck/Linting.py", "Other/Reference/GenerateReference.py"]
191
+ files = ["ReadMe.md", "Other/Build/Build.py", "Other/QualityCheck/Linting.py", "Other/Reference/GenerateReference.py"]
192
192
  if self.tfcps_Tools_General.codeunit_has_testable_sourcecode(self.get_codeunit_file()):
193
193
  # TODO check if the testsettings-section appears in the codeunit-file
194
194
  files.append("Other/QualityCheck/RunTestcases.py")
@@ -364,14 +364,17 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
364
364
  return self.__use_cache
365
365
 
366
366
  @GeneralUtilities.check_arguments
367
- def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str, result_folder: str) -> None:
367
+ def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str]) -> None:
368
368
  """Runs the given program for this codeunit on a remote task-runner that provides the given operating-system (see
369
369
  TFCPS_RemoteBuild and the SCTaskRunnerWindows/SCTaskRunnerMacOS-codeunits). This is used for operating-system-bound
370
370
  build-steps (e.g. flutter-windows- or flutter-ios-builds) which can not run on the current operating-system. After
371
371
  the runner finished, the content of result_folder is written back into the local repository, so the build-step can
372
372
  continue with the result as if it had been produced here; nothing else of the local repository is touched.
373
- working_directory and result_folder must be folders inside this codeunit."""
374
- TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory, result_folder)
373
+ working_directory and result_folder must be folders inside this codeunit.
374
+ folders_which_are_not_transferred are the folders of this codeunit whose content only applies to this machine
375
+ and which are therefore left out of the repository-archive the runner gets (see
376
+ TFCPS_RemoteBuild.run_program_on_runner)."""
377
+ TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory, result_folder, folders_which_are_not_transferred)
375
378
 
376
379
  @GeneralUtilities.check_arguments
377
380
  def get_codeunit_folder(self)->str:
@@ -406,13 +406,23 @@ class TFCPS_CodeUnit_BuildCodeUnits:
406
406
  #custom registries as a build on the host. Without the mount the build in the container would find no mapping at all - the
407
407
  #configuration-folder inside the container belongs to the container-user and is empty - and would therefore fall back to the
408
408
  #upstream-registries defined in the repository, which are rate-limited. The mount is read-only because a build never changes the file.
409
- #Only this file is mounted and not the whole configuration-folder, so nothing else of it (in particular not EnvironmentVariables.csv or
410
- #RegistryCredentials.csv, which contain credentials) is exposed to the container.
411
409
  oci_image_manager = self.tfcps_tools_general.oci_image_manager
412
410
  image_registries_file: str = oci_image_manager.get_image_registries_file_in_configuration_folder()
413
411
  if os.path.isfile(image_registries_file):
414
412
  mount_arguments += ["-v", f"{image_registries_file}:{oci_image_manager.get_image_registries_file_in_container()}:ro"]
415
413
 
414
+ #mount the machine-wide registry-credentials-file of this host (if it exists), so the build inside the container can log in to the
415
+ #registries which the images come from. Without the mount a custom registry which is not publicly readable can not be used inside the
416
+ #container at all, because the configuration-folder of the container-user does not contain the credentials of this host. The mount is
417
+ #read-only because a build never changes the file. The alternative way to provide the credentials are the environment-variables
418
+ #'OCIRegistry_<registryname>_*' (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables), which is the
419
+ #way a build-pipeline provides them from its own secret-store; both sources are used together.
420
+ #Only the two image-related files are mounted and not the whole configuration-folder, so nothing else of it (in particular not
421
+ #EnvironmentVariables.csv) is exposed to the container.
422
+ registry_credentials_file: str = self.sc.get_registry_credentials_file_in_configuration_folder()
423
+ if os.path.isfile(registry_credentials_file):
424
+ mount_arguments += ["-v", f"{registry_credentials_file}:{self.sc.get_registry_credentials_file_in_container()}:ro"]
425
+
416
426
  #pass the environment-variables which are declared as required in <repository>/.ScriptCollection/ProductInformation.xml into the container
417
427
  #so they do not have to be specified explicitly on every call. Their values are resolved from the user-specific
418
428
  #configuration-file (see TFCPS_Tools_General.get_environment_variables_configuration_file), which only exists on the host.
@@ -76,7 +76,11 @@ class TFCPS_RemoteBuild:
76
76
  over HTTPS to the runner, the runner runs the given program on its (correct) operating-system, and afterwards only the
77
77
  folder in which that build-step produces its result is written back into the local repository. Everything else which
78
78
  changed in the workspace of the runner stays there: it is state of that machine (its toolchain-paths, its caches) and
79
- not a result, and a local repository is something a developer works in and which therefore must not be replaced."""
79
+ not a result, and a local repository is something a developer works in and which therefore must not be replaced.
80
+
81
+ The only part of the working-tree which is not sent are the folders the caller states as
82
+ 'folders_which_are_not_transferred' (see run_program_on_runner): a folder which describes the machine it was
83
+ generated on is state of this machine and not state of the repository, so on the runner it describes nothing."""
80
84
 
81
85
  __sc: ScriptCollectionCore = None
82
86
 
@@ -89,7 +93,14 @@ class TFCPS_RemoteBuild:
89
93
  return len(self.__load_runner_endpoints()) > 0
90
94
 
91
95
  @GeneralUtilities.check_arguments
92
- def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
96
+ def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str], poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
97
+ """Runs the given program on a runner which provides the given operating-system, see the docstring of this class.
98
+
99
+ 'folders_which_are_not_transferred' are folders (as paths inside the repository) which are left out of the
100
+ archive which is sent to the runner, because their content only applies to this machine (for example the
101
+ '.dart_tool'-folder of a flutter-package, whose 'package_config.json' states the folder of every package of
102
+ the app as an absolute path of this machine). Such a folder is generated again by the toolchain of the
103
+ runner while it builds; transferring it makes the runner use the paths of this machine instead."""
93
104
  endpoints = self.__load_runner_endpoints()
94
105
  if len(endpoints) == 0:
95
106
  raise ValueError("No remote-build-runner is configured. Define runners either in "
@@ -100,7 +111,8 @@ class TFCPS_RemoteBuild:
100
111
  working_directory_relative = os.path.relpath(working_directory, repository_folder).replace("\\", "/")
101
112
  result_folder_relative = os.path.relpath(result_folder, repository_folder).replace("\\", "/")
102
113
  self.__sc.log.log(f"Delegate '{program} {' '.join(arguments)}' (codeunit '{codeunit_name}', folder '{working_directory_relative}', result-folder '{result_folder_relative}') to the {required_os.value}-runner at {endpoint.url}...")
103
- archive_file = self.__create_repository_archive(repository_folder)
114
+ folders_which_are_not_transferred_relative = [os.path.relpath(folder, repository_folder).replace("\\", "/") for folder in folders_which_are_not_transferred]
115
+ archive_file = self.__create_repository_archive(repository_folder, folders_which_are_not_transferred_relative)
104
116
  job_id: str = None
105
117
  try:
106
118
  metadata_headers = {
@@ -189,15 +201,33 @@ class TFCPS_RemoteBuild:
189
201
  time.sleep(retry_interval_in_seconds)
190
202
 
191
203
  @GeneralUtilities.check_arguments
192
- def __create_repository_archive(self, repository_folder: str) -> str:
204
+ def __create_repository_archive(self, repository_folder: str, folders_which_are_not_transferred_relative: list[str]) -> str:
193
205
  # Pack the entire repository-working-tree (including .git, uncommitted changes and git-ignored files) so the runner
194
206
  # has the exact same state - including secrets that are required e.g. for signing windows-builds. Uses tarfile so
195
207
  # it works identically on Windows and Linux (scbuildcodeunits runs on both) and preserves symlinks/permissions.
208
+ # The folders which describe this machine (see run_program_on_runner) are left out while the archive is packed
209
+ # and not removed from the working-tree before: the working-tree belongs to the developer, and a tool of theirs
210
+ # (for example the dart-extension of an ide, which resolves the packages of a flutter-package whenever its
211
+ # "pubspec.yaml" changes - which a build does) can generate such a folder again at any moment, so removing it
212
+ # would only be a race against that tool.
196
213
  archive_file = os.path.join(GeneralUtilities.get_temp_folder(), f"sc-remotebuild-payload-{uuid.uuid4()}.tar.gz")
197
214
  with tarfile.open(archive_file, "w:gz") as tar:
198
- tar.add(repository_folder, arcname=".")
215
+ tar.add(repository_folder, arcname=".", filter=lambda entry: None if self.__entry_belongs_to_a_folder_which_is_not_transferred(entry.name, folders_which_are_not_transferred_relative) else entry)
199
216
  return archive_file
200
217
 
218
+ @staticmethod
219
+ @GeneralUtilities.check_arguments
220
+ def __entry_belongs_to_a_folder_which_is_not_transferred(entry_name: str, folders_which_are_not_transferred_relative: list[str]) -> bool:
221
+ """Whether the entry with the given name inside the archive (which is named relative to the repository-folder,
222
+ prefixed with './' because the repository is added under that name) is one of the given folders or lies inside
223
+ one of them."""
224
+ entry_path = entry_name.replace("\\", "/").removeprefix("./")
225
+ for folder in folders_which_are_not_transferred_relative:
226
+ folder_path = folder.replace("\\", "/").strip("/")
227
+ if entry_path == folder_path or entry_path.startswith(f"{folder_path}/"):
228
+ return True
229
+ return False
230
+
201
231
  @GeneralUtilities.check_arguments
202
232
  def __write_result_into_the_repository(self, result_bytes: bytes, result_folder: str) -> None:
203
233
  """Writes what the runner produced into the folder of the local repository in which the build-step produces its
@@ -389,10 +389,14 @@ class TFCPS_Tools_General:
389
389
  return
390
390
  version_element.text = current_version
391
391
  file_ended_with_newline: bool = GeneralUtilities.read_binary_from_file(codeunit_file).endswith(b"\n")
392
- tree.write(codeunit_file, xml_declaration=True, encoding="utf-8")
393
- if file_ended_with_newline:
394
- #the serializer does not terminate the last line, so without this the trailing newline of the file would get lost on every write.
395
- with open(codeunit_file, "ab") as file_object:
392
+ #the file is written through a binary file-object instead of by its name because the serializer opens a file which is given by its name in text-mode, which
393
+ #translates every line-separator it writes to the one of the current operating-system: on windows that converted the codeunit-files to crlf on every build,
394
+ #while the trailing newline below (which was always written in binary-mode) stayed lf, so the files ended up with mixed line-separators. The parser normalizes
395
+ #the line-separators of the file it reads to lf anyway (this is defined by the xml-specification), so a codeunit-file is written with lf on every operating-system.
396
+ with open(codeunit_file, "wb") as file_object:
397
+ tree.write(file_object, xml_declaration=True, encoding="utf-8")
398
+ if file_ended_with_newline:
399
+ #the serializer does not terminate the last line, so without this the trailing newline of the file would get lost on every write.
396
400
  file_object.write(b"\n")
397
401
 
398
402
  @GeneralUtilities.check_arguments
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptcollection
3
- Version: 4.4.28
3
+ Version: 4.4.30
4
4
  Summary: The ScriptCollection is the place for reusable scripts.
5
5
  Author-email: Marius Göcke <marius.goecke@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/anionDev/ScriptCollection
@@ -12,7 +12,7 @@ ScriptCollection/ProgramRunnerSudo.py,sha256=ot9Ib-plQ8wejR9M2XXfnx6GR-va9k8SH1g
12
12
  ScriptCollection/RepositoryDiagramGenerator.py,sha256=K_u2NW_p4ozdWD50irJEkkcDVa70TcUJpZN63TeXSLw,7092
13
13
  ScriptCollection/RepositoryHistoryAnalyzer.py,sha256=rD8eC5lMUe5MHmCBHfYft_G-N9IbvtsFGkU7EMwPjrY,5779
14
14
  ScriptCollection/SCLog.py,sha256=Fjf8Pdi63cXHGMxL_aI6A0dwGm7QIfrv3czjvpSUlaM,4705
15
- ScriptCollection/ScriptCollectionCore.py,sha256=nwxFn7tXBV95ixMPgTMQgZ9apII1j_OBdWxZJlXXukA,285393
15
+ ScriptCollection/ScriptCollectionCore.py,sha256=BGPQ7NXBPAO2jWEhu9lj-nL4wRMCmDyXSa-61rpa5Y8,288002
16
16
  ScriptCollection/TestKind.py,sha256=EBTLR8yBz5VmWA2NZIpDTbQNg6tkfv_kAsIzEJ--mCI,1756
17
17
  ScriptCollection/UnassignedVariablesCheck.py,sha256=8xNcp1D5HeOLO1a1NEerq2TsnpFhw7yzYPkaay-oisY,3804
18
18
  ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -35,29 +35,29 @@ ScriptCollection/TFCPS/PackageSource.py,sha256=tQoMbxrIbV3mYHKH2wgOoGzIHY1TvBH8E
35
35
  ScriptCollection/TFCPS/PlantUMLRenderer.py,sha256=upnafiGKV-SdRI8iRBkbbWHN2u6v9KxoswdKi12AwoY,11060
36
36
  ScriptCollection/TFCPS/SCTaskRunnerServer.py,sha256=tub8_1QBu0nWAaGnBWp2nErOW-SOHf_UL6qjOSGbnxU,31026
37
37
  ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py,sha256=67_DQwoH5bhxxSmwtuAeoW1NOCMiLMzOy48OqlJX-KY,1744
38
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=aCfGCbevCOV1IJ444dhvwJkGDKQNH3mqIucPAuSUq-o,40149
38
+ ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=umUfQDErp7xFLkDnSbQMxzUWnKtcGlrZnIvFcTpfGJs,40499
39
39
  ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=Mj9t1SyRpkYZE74cLsGWYv4xB89gXL2ld-RX3xvvGdw,9835
40
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=l8xfp7n-gDF_cwngox6gJkv5ShHP0oGSBzVXENj-3bM,48546
40
+ ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=6FmsF2M3GI8-Xjay0oUaLPTQ8yrHVSUiF_6mc01YoRM,49601
41
41
  ScriptCollection/TFCPS/TFCPS_CreateRelease.py,sha256=kpEmSnfzfgB4OjcCuS_X6jgW_ftcphxEnjjoLoAHTLY,6582
42
42
  ScriptCollection/TFCPS/TFCPS_Generic.py,sha256=mRBcf7XuNWOSXbZYc3NuJow18AMZ30D6Z-BtyNYfNcM,2072
43
43
  ScriptCollection/TFCPS/TFCPS_MergeToMain.py,sha256=Krt5oyGNcRAN3szsgU7LoTh8yVTOdHW-Kq7R9MDDV-Q,7673
44
44
  ScriptCollection/TFCPS/TFCPS_MergeToStable.py,sha256=oYEhz5ZWMnbu97_CcYEqdDxakUFVppyOVQiJzADq59c,22434
45
45
  ScriptCollection/TFCPS/TFCPS_OCIImageSecretScan.py,sha256=rhw0glr4gZ5enG9faOD51xE4MMK1cIdNjUdaswiMsrY,14622
46
46
  ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py,sha256=XLVsFTA1jvfDbZ7wYJgIdEByZhF_bYVbA5jkisuvK4k,2322
47
- ScriptCollection/TFCPS/TFCPS_RemoteBuild.py,sha256=gQQbk7Va7c_0VL2pu_UYM-GPkbQxe0vvs2D_gJdZulI,21103
48
- ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=49gtLQO_V3T9SYiuIp9sqDxSZ4BSWhl8GD7BEV-Hils,159611
47
+ ScriptCollection/TFCPS/TFCPS_RemoteBuild.py,sha256=2Tx5HLk1ZftBWDCTiGZdP2QiiGNyWsEuTCSt33dNQjc,23891
48
+ ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=eGtOfeJEiHV-JoxkbBGw0EP9vDJJAlQWKpJ0hUASwUY,160284
49
49
  ScriptCollection/TFCPS/TFCPS_VisualRegressionTests.py,sha256=6_63udxHnPlu_TMk_ftLZTnzriJO8MCxVCVAsweZqHs,37073
50
50
  ScriptCollection/TFCPS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
51
- ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py,sha256=4skj2I73VOo93HMXcBuWmdyuoE9zjqr2H_DtsYrcvNo,16375
51
+ ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py,sha256=BNZPMjGiWLMBH7QDtP06WEVLMnI38NLX4Dpd13bSbi4,16614
52
52
  ScriptCollection/TFCPS/CPP/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
53
- ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=919mNfzVRurTJhWi8WV0oEAFoH-yd4a6m2Rbxcf2oZ4,19927
53
+ ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=zoy1uwgK_9Fxji8mwHoa6X-VQRc9AMKcT1tfJGZMQeI,22343
54
54
  ScriptCollection/TFCPS/Docker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
55
55
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py,sha256=bT6Gd5pQpZCw4OQz6HWkPCSn5z__eUUEisABLDSxd0o,200
56
56
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py,sha256=QyjOfMY22JWCvKjMelHiDWbJiWqotOfebpJpgDUaoO4,237
57
57
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py,sha256=i0zEGehj0sttxjjZtoq2KFSKp_ulxVyWp_ZgAhIY_So,241
58
58
  ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=GCFzDNAnmm40EeqOYgD_EeuEwDY58ujwJ1-Zn_naamc,61797
59
59
  ScriptCollection/TFCPS/DotNet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
60
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=KR_Kvxp5yeWrCQXqUwmiv8IPGKCbHMUmdrsNovpowVs,51698
60
+ ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=dW7KoCvpIbjGaN-SRoaKPmF02GVVyT7zJAVR0yIjtag,52842
61
61
  ScriptCollection/TFCPS/Flutter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
62
62
  ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py,sha256=m4Pax-U6o5XuspLnLfnWjyQMtoC7zBDDNJW9EhLWjeE,9012
63
63
  ScriptCollection/TFCPS/Go/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -71,8 +71,8 @@ ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=yyjWPMHQjPLkML
71
71
  ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
72
72
  ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py,sha256=5z990qyO5durX07xRWRwju1hl6wpjQAy2f-yRwJ-7s0,11856
73
73
  ScriptCollection/TFCPS/TypeScript/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
74
- scriptcollection-4.4.28.dist-info/METADATA,sha256=9gAzv0f4JSOokYZOEtxExgHHa9II1AgHMOOe3T0quvo,7690
75
- scriptcollection-4.4.28.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
76
- scriptcollection-4.4.28.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
77
- scriptcollection-4.4.28.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
78
- scriptcollection-4.4.28.dist-info/RECORD,,
74
+ scriptcollection-4.4.30.dist-info/METADATA,sha256=afxz3vYiDErPCZNzqM6zrv2xVN4yce3S1-MvBo6ebcg,7690
75
+ scriptcollection-4.4.30.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
76
+ scriptcollection-4.4.30.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
77
+ scriptcollection-4.4.30.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
78
+ scriptcollection-4.4.30.dist-info/RECORD,,