ScriptCollection 4.4.29__py3-none-any.whl → 4.4.30__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -40,7 +40,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
40
40
  from .ProgramRunnerPopen import ProgramRunnerPopen
41
41
  from .SCLog import SCLog, LogLevel
42
42
 
43
- version = "4.4.29"
43
+ version = "4.4.30"
44
44
  __version__ = version
45
45
 
46
46
  class VSCodeWorkspaceShellTask:
@@ -242,13 +242,35 @@ class ScriptCollectionCore:
242
242
  GeneralUtilities.ensure_directory_exists(result)
243
243
  return result
244
244
 
245
- def __get_docker_registry_credentials_file(self)->str:
245
+ @GeneralUtilities.check_arguments
246
+ def get_registry_credentials_file_in_container(self)->str:
247
+ """Returns the file inside a build-container to which the host mounts its registry-credentials-file (see
248
+ TFCPS_CodeUnit_BuildCodeUnits.__run_scriptcollection_executable_in_container). It is an own path (and not the
249
+ configuration-folder of the container-user) because the home-folder inside the container depends on the user the
250
+ image runs as, while this path is defined by the mount and is therefore identical for both sides."""
251
+ return "/Workspace/ScriptCollectionConfiguration/RegistryCredentials.csv"
252
+
253
+ @GeneralUtilities.check_arguments
254
+ def get_registry_credentials_file_in_configuration_folder(self)->str:
255
+ """Returns the registry-credentials-file in the configuration-folder of the current user, and creates it if it does not exist yet.
256
+ This is the file of the machine on which a command was started, so it is also the file which is mounted into a build-container
257
+ (in contrast to __get_docker_registry_credentials_file, which inside a container resolves to that mount instead)."""
246
258
  result=os.path.join(self.get_global_cache_folder(),"RegistryCredentials.csv")
247
259
  if not os.path.isfile(result):
248
260
  GeneralUtilities.ensure_file_exists(result)
249
261
  GeneralUtilities.write_lines_to_file(result,["RegistryName;Username;Password"])
250
262
  return result
251
263
 
264
+ def __get_docker_registry_credentials_file(self)->str:
265
+ """Returns the file which maps a registry to the credentials which are used for it. The file which the host mounted into a
266
+ build-container has precedence over the file in the configuration-folder of the current user: in a container that
267
+ configuration-folder belongs to the container-user and therefore never contains the configuration of the machine on which the
268
+ build was started."""
269
+ mounted_file=self.get_registry_credentials_file_in_container()
270
+ if os.path.isfile(mounted_file):
271
+ return mounted_file
272
+ return self.get_registry_credentials_file_in_configuration_folder()
273
+
252
274
  def __load_credentials_if_required_and_available(self,registry_url:str,registry_username:str,registry_password:str)->tuple[str,str]:
253
275
  """Returns the credentials for the given registry: the ones which were passed by the caller, or - if the caller passed none - the
254
276
  ones which are available for this registry on this machine (see __get_docker_registry_credentials). The last matching entry is
@@ -455,6 +477,10 @@ class ScriptCollectionCore:
455
477
  if not force and self.local_docker_image_exists(image, tag):
456
478
  self.log.log(f"Image \"{image_with_tag}\" is already available locally and therefore does not have to be pulled.", LogLevel.Debug)
457
479
  return
480
+ #the image can be located in a registry which is not publicly readable, so the login to the registries which are defined for this
481
+ #machine is done before the pull. Without it the pull of such an image fails with "no basic auth credentials" respectively "401
482
+ #Unauthorized", even though the credentials are configured.
483
+ self.login_to_defined_docker_registries()
458
484
  self.log.log(f"Pull image \"{image_with_tag}\"...")
459
485
  self.run_program_with_retry("docker", f"pull {image_with_tag}", print_errors_as_information=True, print_live_output=self.log.loglevel == LogLevel.Debug)
460
486
 
@@ -4750,24 +4776,23 @@ OCR-content:
4750
4776
 
4751
4777
  def prepare_build_pipeline_for_gitlab(self):
4752
4778
  repository_folder:str=os.getcwd()
4753
- self.__prepare_build_pipeline(repository_folder)
4779
+ self.__prepare_build_pipeline(repository_folder,"CI_COMMIT_REF_NAME")
4754
4780
 
4755
4781
  def prepare_build_pipeline_for_github(self):
4756
4782
  repository_folder:str=os.getcwd()
4757
4783
  self.assert_scbuilder_image_in_github_workflow_matches_image_definition(repository_folder)
4758
- self.__prepare_build_pipeline(repository_folder)
4784
+ self.__prepare_build_pipeline(repository_folder,"GITHUB_REF_NAME")
4759
4785
 
4760
- def __ensure_branch_is_checked_out(self,repository_folder:str) -> None:
4761
- current_branch_result = self.run_program_argsasarray("git", ["symbolic-ref", "--short", "HEAD"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
4762
- repository_is_on_any_branch:bool = current_branch_result[0] == 0
4763
- if not repository_is_on_any_branch:
4764
- timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
4765
- branch_name:str=f"pipeline_{timestamp}"
4766
- self.run_program_argsasarray("git",["checkout","-b",branch_name], repository_folder)
4786
+ def __ensure_branch_is_checked_out(self,repository_folder:str,branchname_environment_variable_name:str) -> None:
4787
+ """Checks out the branch the pipeline was triggered for, because the runner checks out the commit in a detached-head-state while the versioning requires the real branch-name.
4788
+ "checkout -B" is used and not "checkout -b", because the branch already exists when the runner reuses the build-directory of a previous pipeline-run of the same branch."""
4789
+ branch_name:str=os.environ.get(branchname_environment_variable_name)
4790
+ GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(branch_name), f"The environment-variable \"{branchname_environment_variable_name}\", which contains the name of the branch the pipeline runs for, is not set.")
4791
+ self.run_program_argsasarray("git",["checkout","-B",branch_name], repository_folder)
4767
4792
 
4768
- def __prepare_build_pipeline(self,repository_folder:str) -> None:
4793
+ def __prepare_build_pipeline(self,repository_folder:str,branchname_environment_variable_name:str) -> None:
4769
4794
  GeneralUtilities.assert_condition(self.is_running_in_build_container(), "This function should only be run in the build container.")
4770
- self.__ensure_branch_is_checked_out(repository_folder)
4795
+ self.__ensure_branch_is_checked_out(repository_folder,branchname_environment_variable_name)
4771
4796
 
4772
4797
  expected_image = self.__get_scbuilder_image_from_image_definition_file(repository_folder)
4773
4798
  GeneralUtilities.assert_condition("scbuilder:" in expected_image, f"The SCBuilder-image '{expected_image}' defined in the image-definition-file of the repository is not a valid SCBuilder-image. It must contain 'scbuilder:'.")
@@ -25,9 +25,16 @@ class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
25
25
  artifacts_folder = GeneralUtilities.resolve_relative_path("Other/Artifacts", codeunit_folder)
26
26
  app_artifacts_folder = os.path.join(artifacts_folder, "BuildResult_OCIImage")
27
27
  GeneralUtilities.ensure_folder_exists_and_is_empty(app_artifacts_folder)
28
+ image_build_arguments = self.__get_build_arguments_for_images_of_repository()
29
+ #the base-image of the Dockerfile is resolved by buildkit and not by this process, so the login to the registries which are
30
+ #defined for this machine has to be done before the build. Without it buildkit requests the manifest of the base-image
31
+ #anonymously and the build fails with "401 Unauthorized" as soon as the base-image comes from a registry which is not
32
+ #publicly readable.
33
+ self._protected_sc.login_to_defined_docker_registries()
28
34
  for platform in platforms:
29
35
  #builder must be created once before with "docker buildx create --use"
30
36
  args = ["buildx","build", "--platform",GeneralUtilities.platform_to_docker_platform_str(platform), "--pull", "--force-rm", "--progress=plain", "--build-arg", f"TargetEnvironmentType={self.get_target_environment_type()}", "--build-arg", f"CodeUnitName={codeunitname}", "--build-arg", f"CodeUnitVersion={codeunitversion}", "--build-arg", f"CodeUnitOwnerName={self.tfcps_Tools_General.get_codeunit_owner_name(self.get_codeunit_file())}", "--build-arg", f"CodeUnitOwnerEMailAddress={self.tfcps_Tools_General.get_codeunit_owner_emailaddress(self.get_codeunit_file())}", "--build-arg", f"Platform={GeneralUtilities.platform_to_dash_str(platform)}", "--build-arg", f"DotNetRuntime={GeneralUtilities.platform_to_dotnet_runtime_identifier(platform)}", "--build-arg", f"PlatformForGoVersion={GeneralUtilities.platform_to_go_runtime_identifier(platform)}"]
37
+ args = args+image_build_arguments
31
38
  for custom_argument_key, custom_argument_value in custom_arguments.items():
32
39
  args.append("--build-arg")
33
40
  args.append(f"{custom_argument_key}={custom_argument_value}")
@@ -48,6 +55,27 @@ class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
48
55
  self.copy_source_files_to_output_directory()
49
56
 
50
57
 
58
+ @GeneralUtilities.check_arguments
59
+ def __get_build_arguments_for_images_of_repository(self) -> list[str]:
60
+ """Returns a '--build-arg'-pair for every image which the repository declares in
61
+ '.ScriptCollection/OCIImages/ImageDefinition.csv': the argument 'image_<imagename in lowercase>' contains the address with tag
62
+ from which that image has to be taken on this machine.
63
+ A Dockerfile has to declare its base-image through such an argument ('ARG image_debian' followed by 'FROM ${image_debian}')
64
+ instead of writing a registry-address into its 'FROM'-line: the address inside the Dockerfile is resolved by buildkit, so
65
+ neither the custom registry which is preferred on this machine nor the fallback-registry of the repository would apply to it
66
+ (see OCIImageManager.get_registry_address_for_image).
67
+ The arguments are named like the environment-variables which the local test-services get for the same purpose (see
68
+ TFCPS_Tools_General.pull_images_of_test_services), so an image is addressed by the same name everywhere.
69
+ Every declared image is passed and not only the one a Dockerfile uses, because which of them it needs is only known to the
70
+ Dockerfile itself - a multi-stage-build uses several of them."""
71
+ repository_folder = self.get_repository_folder()
72
+ oci_image_manager = self.tfcps_Tools_General.oci_image_manager
73
+ result: list[str] = []
74
+ for image_name in oci_image_manager.get_used_images_in_repository(repository_folder):
75
+ image_address_with_tag = oci_image_manager.get_registry_address_for_image_with_default_tag(repository_folder, image_name)
76
+ result = result+["--build-arg", f"image_{image_name.lower()}={image_address_with_tag}"]
77
+ return result
78
+
51
79
  @GeneralUtilities.check_arguments
52
80
  def __generate_sbom_for_docker_image(self) -> None:
53
81
  codeunitname=self.get_codeunit_name()
@@ -406,13 +406,23 @@ class TFCPS_CodeUnit_BuildCodeUnits:
406
406
  #custom registries as a build on the host. Without the mount the build in the container would find no mapping at all - the
407
407
  #configuration-folder inside the container belongs to the container-user and is empty - and would therefore fall back to the
408
408
  #upstream-registries defined in the repository, which are rate-limited. The mount is read-only because a build never changes the file.
409
- #Only this file is mounted and not the whole configuration-folder, so nothing else of it (in particular not EnvironmentVariables.csv or
410
- #RegistryCredentials.csv, which contain credentials) is exposed to the container.
411
409
  oci_image_manager = self.tfcps_tools_general.oci_image_manager
412
410
  image_registries_file: str = oci_image_manager.get_image_registries_file_in_configuration_folder()
413
411
  if os.path.isfile(image_registries_file):
414
412
  mount_arguments += ["-v", f"{image_registries_file}:{oci_image_manager.get_image_registries_file_in_container()}:ro"]
415
413
 
414
+ #mount the machine-wide registry-credentials-file of this host (if it exists), so the build inside the container can log in to the
415
+ #registries which the images come from. Without the mount a custom registry which is not publicly readable can not be used inside the
416
+ #container at all, because the configuration-folder of the container-user does not contain the credentials of this host. The mount is
417
+ #read-only because a build never changes the file. The alternative way to provide the credentials are the environment-variables
418
+ #'OCIRegistry_<registryname>_*' (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables), which is the
419
+ #way a build-pipeline provides them from its own secret-store; both sources are used together.
420
+ #Only the two image-related files are mounted and not the whole configuration-folder, so nothing else of it (in particular not
421
+ #EnvironmentVariables.csv) is exposed to the container.
422
+ registry_credentials_file: str = self.sc.get_registry_credentials_file_in_configuration_folder()
423
+ if os.path.isfile(registry_credentials_file):
424
+ mount_arguments += ["-v", f"{registry_credentials_file}:{self.sc.get_registry_credentials_file_in_container()}:ro"]
425
+
416
426
  #pass the environment-variables which are declared as required in <repository>/.ScriptCollection/ProductInformation.xml into the container
417
427
  #so they do not have to be specified explicitly on every call. Their values are resolved from the user-specific
418
428
  #configuration-file (see TFCPS_Tools_General.get_environment_variables_configuration_file), which only exists on the host.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptcollection
3
- Version: 4.4.29
3
+ Version: 4.4.30
4
4
  Summary: The ScriptCollection is the place for reusable scripts.
5
5
  Author-email: Marius Göcke <marius.goecke@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/anionDev/ScriptCollection
@@ -12,7 +12,7 @@ ScriptCollection/ProgramRunnerSudo.py,sha256=ot9Ib-plQ8wejR9M2XXfnx6GR-va9k8SH1g
12
12
  ScriptCollection/RepositoryDiagramGenerator.py,sha256=K_u2NW_p4ozdWD50irJEkkcDVa70TcUJpZN63TeXSLw,7092
13
13
  ScriptCollection/RepositoryHistoryAnalyzer.py,sha256=rD8eC5lMUe5MHmCBHfYft_G-N9IbvtsFGkU7EMwPjrY,5779
14
14
  ScriptCollection/SCLog.py,sha256=Fjf8Pdi63cXHGMxL_aI6A0dwGm7QIfrv3czjvpSUlaM,4705
15
- ScriptCollection/ScriptCollectionCore.py,sha256=PJysQBH2oGfznko_coqHB-LcQdfV1PshFmYZwHGupNk,285393
15
+ ScriptCollection/ScriptCollectionCore.py,sha256=BGPQ7NXBPAO2jWEhu9lj-nL4wRMCmDyXSa-61rpa5Y8,288002
16
16
  ScriptCollection/TestKind.py,sha256=EBTLR8yBz5VmWA2NZIpDTbQNg6tkfv_kAsIzEJ--mCI,1756
17
17
  ScriptCollection/UnassignedVariablesCheck.py,sha256=8xNcp1D5HeOLO1a1NEerq2TsnpFhw7yzYPkaay-oisY,3804
18
18
  ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -37,7 +37,7 @@ ScriptCollection/TFCPS/SCTaskRunnerServer.py,sha256=tub8_1QBu0nWAaGnBWp2nErOW-SO
37
37
  ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py,sha256=67_DQwoH5bhxxSmwtuAeoW1NOCMiLMzOy48OqlJX-KY,1744
38
38
  ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=umUfQDErp7xFLkDnSbQMxzUWnKtcGlrZnIvFcTpfGJs,40499
39
39
  ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=Mj9t1SyRpkYZE74cLsGWYv4xB89gXL2ld-RX3xvvGdw,9835
40
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=l8xfp7n-gDF_cwngox6gJkv5ShHP0oGSBzVXENj-3bM,48546
40
+ ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=6FmsF2M3GI8-Xjay0oUaLPTQ8yrHVSUiF_6mc01YoRM,49601
41
41
  ScriptCollection/TFCPS/TFCPS_CreateRelease.py,sha256=kpEmSnfzfgB4OjcCuS_X6jgW_ftcphxEnjjoLoAHTLY,6582
42
42
  ScriptCollection/TFCPS/TFCPS_Generic.py,sha256=mRBcf7XuNWOSXbZYc3NuJow18AMZ30D6Z-BtyNYfNcM,2072
43
43
  ScriptCollection/TFCPS/TFCPS_MergeToMain.py,sha256=Krt5oyGNcRAN3szsgU7LoTh8yVTOdHW-Kq7R9MDDV-Q,7673
@@ -50,7 +50,7 @@ ScriptCollection/TFCPS/TFCPS_VisualRegressionTests.py,sha256=6_63udxHnPlu_TMk_ft
50
50
  ScriptCollection/TFCPS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
51
51
  ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py,sha256=BNZPMjGiWLMBH7QDtP06WEVLMnI38NLX4Dpd13bSbi4,16614
52
52
  ScriptCollection/TFCPS/CPP/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
53
- ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=919mNfzVRurTJhWi8WV0oEAFoH-yd4a6m2Rbxcf2oZ4,19927
53
+ ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=zoy1uwgK_9Fxji8mwHoa6X-VQRc9AMKcT1tfJGZMQeI,22343
54
54
  ScriptCollection/TFCPS/Docker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
55
55
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py,sha256=bT6Gd5pQpZCw4OQz6HWkPCSn5z__eUUEisABLDSxd0o,200
56
56
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py,sha256=QyjOfMY22JWCvKjMelHiDWbJiWqotOfebpJpgDUaoO4,237
@@ -71,8 +71,8 @@ ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=yyjWPMHQjPLkML
71
71
  ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
72
72
  ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py,sha256=5z990qyO5durX07xRWRwju1hl6wpjQAy2f-yRwJ-7s0,11856
73
73
  ScriptCollection/TFCPS/TypeScript/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
74
- scriptcollection-4.4.29.dist-info/METADATA,sha256=x84CpGE0zK9m7D6Dq6XKd0Sp5zaNmAoINt2n99IV4Pc,7690
75
- scriptcollection-4.4.29.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
76
- scriptcollection-4.4.29.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
77
- scriptcollection-4.4.29.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
78
- scriptcollection-4.4.29.dist-info/RECORD,,
74
+ scriptcollection-4.4.30.dist-info/METADATA,sha256=afxz3vYiDErPCZNzqM6zrv2xVN4yce3S1-MvBo6ebcg,7690
75
+ scriptcollection-4.4.30.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
76
+ scriptcollection-4.4.30.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
77
+ scriptcollection-4.4.30.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
78
+ scriptcollection-4.4.30.dist-info/RECORD,,