ScriptCollection 4.4.29__py3-none-any.whl → 4.4.30__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/ScriptCollectionCore.py +38 -13
- ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +28 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +12 -2
- {scriptcollection-4.4.29.dist-info → scriptcollection-4.4.30.dist-info}/METADATA +1 -1
- {scriptcollection-4.4.29.dist-info → scriptcollection-4.4.30.dist-info}/RECORD +8 -8
- {scriptcollection-4.4.29.dist-info → scriptcollection-4.4.30.dist-info}/WHEEL +0 -0
- {scriptcollection-4.4.29.dist-info → scriptcollection-4.4.30.dist-info}/entry_points.txt +0 -0
- {scriptcollection-4.4.29.dist-info → scriptcollection-4.4.30.dist-info}/top_level.txt +0 -0
|
@@ -40,7 +40,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
|
|
|
40
40
|
from .ProgramRunnerPopen import ProgramRunnerPopen
|
|
41
41
|
from .SCLog import SCLog, LogLevel
|
|
42
42
|
|
|
43
|
-
version = "4.4.
|
|
43
|
+
version = "4.4.30"
|
|
44
44
|
__version__ = version
|
|
45
45
|
|
|
46
46
|
class VSCodeWorkspaceShellTask:
|
|
@@ -242,13 +242,35 @@ class ScriptCollectionCore:
|
|
|
242
242
|
GeneralUtilities.ensure_directory_exists(result)
|
|
243
243
|
return result
|
|
244
244
|
|
|
245
|
-
|
|
245
|
+
@GeneralUtilities.check_arguments
|
|
246
|
+
def get_registry_credentials_file_in_container(self)->str:
|
|
247
|
+
"""Returns the file inside a build-container to which the host mounts its registry-credentials-file (see
|
|
248
|
+
TFCPS_CodeUnit_BuildCodeUnits.__run_scriptcollection_executable_in_container). It is an own path (and not the
|
|
249
|
+
configuration-folder of the container-user) because the home-folder inside the container depends on the user the
|
|
250
|
+
image runs as, while this path is defined by the mount and is therefore identical for both sides."""
|
|
251
|
+
return "/Workspace/ScriptCollectionConfiguration/RegistryCredentials.csv"
|
|
252
|
+
|
|
253
|
+
@GeneralUtilities.check_arguments
|
|
254
|
+
def get_registry_credentials_file_in_configuration_folder(self)->str:
|
|
255
|
+
"""Returns the registry-credentials-file in the configuration-folder of the current user, and creates it if it does not exist yet.
|
|
256
|
+
This is the file of the machine on which a command was started, so it is also the file which is mounted into a build-container
|
|
257
|
+
(in contrast to __get_docker_registry_credentials_file, which inside a container resolves to that mount instead)."""
|
|
246
258
|
result=os.path.join(self.get_global_cache_folder(),"RegistryCredentials.csv")
|
|
247
259
|
if not os.path.isfile(result):
|
|
248
260
|
GeneralUtilities.ensure_file_exists(result)
|
|
249
261
|
GeneralUtilities.write_lines_to_file(result,["RegistryName;Username;Password"])
|
|
250
262
|
return result
|
|
251
263
|
|
|
264
|
+
def __get_docker_registry_credentials_file(self)->str:
|
|
265
|
+
"""Returns the file which maps a registry to the credentials which are used for it. The file which the host mounted into a
|
|
266
|
+
build-container has precedence over the file in the configuration-folder of the current user: in a container that
|
|
267
|
+
configuration-folder belongs to the container-user and therefore never contains the configuration of the machine on which the
|
|
268
|
+
build was started."""
|
|
269
|
+
mounted_file=self.get_registry_credentials_file_in_container()
|
|
270
|
+
if os.path.isfile(mounted_file):
|
|
271
|
+
return mounted_file
|
|
272
|
+
return self.get_registry_credentials_file_in_configuration_folder()
|
|
273
|
+
|
|
252
274
|
def __load_credentials_if_required_and_available(self,registry_url:str,registry_username:str,registry_password:str)->tuple[str,str]:
|
|
253
275
|
"""Returns the credentials for the given registry: the ones which were passed by the caller, or - if the caller passed none - the
|
|
254
276
|
ones which are available for this registry on this machine (see __get_docker_registry_credentials). The last matching entry is
|
|
@@ -455,6 +477,10 @@ class ScriptCollectionCore:
|
|
|
455
477
|
if not force and self.local_docker_image_exists(image, tag):
|
|
456
478
|
self.log.log(f"Image \"{image_with_tag}\" is already available locally and therefore does not have to be pulled.", LogLevel.Debug)
|
|
457
479
|
return
|
|
480
|
+
#the image can be located in a registry which is not publicly readable, so the login to the registries which are defined for this
|
|
481
|
+
#machine is done before the pull. Without it the pull of such an image fails with "no basic auth credentials" respectively "401
|
|
482
|
+
#Unauthorized", even though the credentials are configured.
|
|
483
|
+
self.login_to_defined_docker_registries()
|
|
458
484
|
self.log.log(f"Pull image \"{image_with_tag}\"...")
|
|
459
485
|
self.run_program_with_retry("docker", f"pull {image_with_tag}", print_errors_as_information=True, print_live_output=self.log.loglevel == LogLevel.Debug)
|
|
460
486
|
|
|
@@ -4750,24 +4776,23 @@ OCR-content:
|
|
|
4750
4776
|
|
|
4751
4777
|
def prepare_build_pipeline_for_gitlab(self):
|
|
4752
4778
|
repository_folder:str=os.getcwd()
|
|
4753
|
-
self.__prepare_build_pipeline(repository_folder)
|
|
4779
|
+
self.__prepare_build_pipeline(repository_folder,"CI_COMMIT_REF_NAME")
|
|
4754
4780
|
|
|
4755
4781
|
def prepare_build_pipeline_for_github(self):
|
|
4756
4782
|
repository_folder:str=os.getcwd()
|
|
4757
4783
|
self.assert_scbuilder_image_in_github_workflow_matches_image_definition(repository_folder)
|
|
4758
|
-
self.__prepare_build_pipeline(repository_folder)
|
|
4784
|
+
self.__prepare_build_pipeline(repository_folder,"GITHUB_REF_NAME")
|
|
4759
4785
|
|
|
4760
|
-
def __ensure_branch_is_checked_out(self,repository_folder:str) -> None:
|
|
4761
|
-
|
|
4762
|
-
|
|
4763
|
-
|
|
4764
|
-
|
|
4765
|
-
|
|
4766
|
-
self.run_program_argsasarray("git",["checkout","-b",branch_name], repository_folder)
|
|
4786
|
+
def __ensure_branch_is_checked_out(self,repository_folder:str,branchname_environment_variable_name:str) -> None:
|
|
4787
|
+
"""Checks out the branch the pipeline was triggered for, because the runner checks out the commit in a detached-head-state while the versioning requires the real branch-name.
|
|
4788
|
+
"checkout -B" is used and not "checkout -b", because the branch already exists when the runner reuses the build-directory of a previous pipeline-run of the same branch."""
|
|
4789
|
+
branch_name:str=os.environ.get(branchname_environment_variable_name)
|
|
4790
|
+
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(branch_name), f"The environment-variable \"{branchname_environment_variable_name}\", which contains the name of the branch the pipeline runs for, is not set.")
|
|
4791
|
+
self.run_program_argsasarray("git",["checkout","-B",branch_name], repository_folder)
|
|
4767
4792
|
|
|
4768
|
-
def __prepare_build_pipeline(self,repository_folder:str) -> None:
|
|
4793
|
+
def __prepare_build_pipeline(self,repository_folder:str,branchname_environment_variable_name:str) -> None:
|
|
4769
4794
|
GeneralUtilities.assert_condition(self.is_running_in_build_container(), "This function should only be run in the build container.")
|
|
4770
|
-
self.__ensure_branch_is_checked_out(repository_folder)
|
|
4795
|
+
self.__ensure_branch_is_checked_out(repository_folder,branchname_environment_variable_name)
|
|
4771
4796
|
|
|
4772
4797
|
expected_image = self.__get_scbuilder_image_from_image_definition_file(repository_folder)
|
|
4773
4798
|
GeneralUtilities.assert_condition("scbuilder:" in expected_image, f"The SCBuilder-image '{expected_image}' defined in the image-definition-file of the repository is not a valid SCBuilder-image. It must contain 'scbuilder:'.")
|
|
@@ -25,9 +25,16 @@ class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
25
25
|
artifacts_folder = GeneralUtilities.resolve_relative_path("Other/Artifacts", codeunit_folder)
|
|
26
26
|
app_artifacts_folder = os.path.join(artifacts_folder, "BuildResult_OCIImage")
|
|
27
27
|
GeneralUtilities.ensure_folder_exists_and_is_empty(app_artifacts_folder)
|
|
28
|
+
image_build_arguments = self.__get_build_arguments_for_images_of_repository()
|
|
29
|
+
#the base-image of the Dockerfile is resolved by buildkit and not by this process, so the login to the registries which are
|
|
30
|
+
#defined for this machine has to be done before the build. Without it buildkit requests the manifest of the base-image
|
|
31
|
+
#anonymously and the build fails with "401 Unauthorized" as soon as the base-image comes from a registry which is not
|
|
32
|
+
#publicly readable.
|
|
33
|
+
self._protected_sc.login_to_defined_docker_registries()
|
|
28
34
|
for platform in platforms:
|
|
29
35
|
#builder must be created once before with "docker buildx create --use"
|
|
30
36
|
args = ["buildx","build", "--platform",GeneralUtilities.platform_to_docker_platform_str(platform), "--pull", "--force-rm", "--progress=plain", "--build-arg", f"TargetEnvironmentType={self.get_target_environment_type()}", "--build-arg", f"CodeUnitName={codeunitname}", "--build-arg", f"CodeUnitVersion={codeunitversion}", "--build-arg", f"CodeUnitOwnerName={self.tfcps_Tools_General.get_codeunit_owner_name(self.get_codeunit_file())}", "--build-arg", f"CodeUnitOwnerEMailAddress={self.tfcps_Tools_General.get_codeunit_owner_emailaddress(self.get_codeunit_file())}", "--build-arg", f"Platform={GeneralUtilities.platform_to_dash_str(platform)}", "--build-arg", f"DotNetRuntime={GeneralUtilities.platform_to_dotnet_runtime_identifier(platform)}", "--build-arg", f"PlatformForGoVersion={GeneralUtilities.platform_to_go_runtime_identifier(platform)}"]
|
|
37
|
+
args = args+image_build_arguments
|
|
31
38
|
for custom_argument_key, custom_argument_value in custom_arguments.items():
|
|
32
39
|
args.append("--build-arg")
|
|
33
40
|
args.append(f"{custom_argument_key}={custom_argument_value}")
|
|
@@ -48,6 +55,27 @@ class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
48
55
|
self.copy_source_files_to_output_directory()
|
|
49
56
|
|
|
50
57
|
|
|
58
|
+
@GeneralUtilities.check_arguments
|
|
59
|
+
def __get_build_arguments_for_images_of_repository(self) -> list[str]:
|
|
60
|
+
"""Returns a '--build-arg'-pair for every image which the repository declares in
|
|
61
|
+
'.ScriptCollection/OCIImages/ImageDefinition.csv': the argument 'image_<imagename in lowercase>' contains the address with tag
|
|
62
|
+
from which that image has to be taken on this machine.
|
|
63
|
+
A Dockerfile has to declare its base-image through such an argument ('ARG image_debian' followed by 'FROM ${image_debian}')
|
|
64
|
+
instead of writing a registry-address into its 'FROM'-line: the address inside the Dockerfile is resolved by buildkit, so
|
|
65
|
+
neither the custom registry which is preferred on this machine nor the fallback-registry of the repository would apply to it
|
|
66
|
+
(see OCIImageManager.get_registry_address_for_image).
|
|
67
|
+
The arguments are named like the environment-variables which the local test-services get for the same purpose (see
|
|
68
|
+
TFCPS_Tools_General.pull_images_of_test_services), so an image is addressed by the same name everywhere.
|
|
69
|
+
Every declared image is passed and not only the one a Dockerfile uses, because which of them it needs is only known to the
|
|
70
|
+
Dockerfile itself - a multi-stage-build uses several of them."""
|
|
71
|
+
repository_folder = self.get_repository_folder()
|
|
72
|
+
oci_image_manager = self.tfcps_Tools_General.oci_image_manager
|
|
73
|
+
result: list[str] = []
|
|
74
|
+
for image_name in oci_image_manager.get_used_images_in_repository(repository_folder):
|
|
75
|
+
image_address_with_tag = oci_image_manager.get_registry_address_for_image_with_default_tag(repository_folder, image_name)
|
|
76
|
+
result = result+["--build-arg", f"image_{image_name.lower()}={image_address_with_tag}"]
|
|
77
|
+
return result
|
|
78
|
+
|
|
51
79
|
@GeneralUtilities.check_arguments
|
|
52
80
|
def __generate_sbom_for_docker_image(self) -> None:
|
|
53
81
|
codeunitname=self.get_codeunit_name()
|
|
@@ -406,13 +406,23 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
406
406
|
#custom registries as a build on the host. Without the mount the build in the container would find no mapping at all - the
|
|
407
407
|
#configuration-folder inside the container belongs to the container-user and is empty - and would therefore fall back to the
|
|
408
408
|
#upstream-registries defined in the repository, which are rate-limited. The mount is read-only because a build never changes the file.
|
|
409
|
-
#Only this file is mounted and not the whole configuration-folder, so nothing else of it (in particular not EnvironmentVariables.csv or
|
|
410
|
-
#RegistryCredentials.csv, which contain credentials) is exposed to the container.
|
|
411
409
|
oci_image_manager = self.tfcps_tools_general.oci_image_manager
|
|
412
410
|
image_registries_file: str = oci_image_manager.get_image_registries_file_in_configuration_folder()
|
|
413
411
|
if os.path.isfile(image_registries_file):
|
|
414
412
|
mount_arguments += ["-v", f"{image_registries_file}:{oci_image_manager.get_image_registries_file_in_container()}:ro"]
|
|
415
413
|
|
|
414
|
+
#mount the machine-wide registry-credentials-file of this host (if it exists), so the build inside the container can log in to the
|
|
415
|
+
#registries which the images come from. Without the mount a custom registry which is not publicly readable can not be used inside the
|
|
416
|
+
#container at all, because the configuration-folder of the container-user does not contain the credentials of this host. The mount is
|
|
417
|
+
#read-only because a build never changes the file. The alternative way to provide the credentials are the environment-variables
|
|
418
|
+
#'OCIRegistry_<registryname>_*' (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables), which is the
|
|
419
|
+
#way a build-pipeline provides them from its own secret-store; both sources are used together.
|
|
420
|
+
#Only the two image-related files are mounted and not the whole configuration-folder, so nothing else of it (in particular not
|
|
421
|
+
#EnvironmentVariables.csv) is exposed to the container.
|
|
422
|
+
registry_credentials_file: str = self.sc.get_registry_credentials_file_in_configuration_folder()
|
|
423
|
+
if os.path.isfile(registry_credentials_file):
|
|
424
|
+
mount_arguments += ["-v", f"{registry_credentials_file}:{self.sc.get_registry_credentials_file_in_container()}:ro"]
|
|
425
|
+
|
|
416
426
|
#pass the environment-variables which are declared as required in <repository>/.ScriptCollection/ProductInformation.xml into the container
|
|
417
427
|
#so they do not have to be specified explicitly on every call. Their values are resolved from the user-specific
|
|
418
428
|
#configuration-file (see TFCPS_Tools_General.get_environment_variables_configuration_file), which only exists on the host.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scriptcollection
|
|
3
|
-
Version: 4.4.
|
|
3
|
+
Version: 4.4.30
|
|
4
4
|
Summary: The ScriptCollection is the place for reusable scripts.
|
|
5
5
|
Author-email: Marius Göcke <marius.goecke@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/anionDev/ScriptCollection
|
|
@@ -12,7 +12,7 @@ ScriptCollection/ProgramRunnerSudo.py,sha256=ot9Ib-plQ8wejR9M2XXfnx6GR-va9k8SH1g
|
|
|
12
12
|
ScriptCollection/RepositoryDiagramGenerator.py,sha256=K_u2NW_p4ozdWD50irJEkkcDVa70TcUJpZN63TeXSLw,7092
|
|
13
13
|
ScriptCollection/RepositoryHistoryAnalyzer.py,sha256=rD8eC5lMUe5MHmCBHfYft_G-N9IbvtsFGkU7EMwPjrY,5779
|
|
14
14
|
ScriptCollection/SCLog.py,sha256=Fjf8Pdi63cXHGMxL_aI6A0dwGm7QIfrv3czjvpSUlaM,4705
|
|
15
|
-
ScriptCollection/ScriptCollectionCore.py,sha256=
|
|
15
|
+
ScriptCollection/ScriptCollectionCore.py,sha256=BGPQ7NXBPAO2jWEhu9lj-nL4wRMCmDyXSa-61rpa5Y8,288002
|
|
16
16
|
ScriptCollection/TestKind.py,sha256=EBTLR8yBz5VmWA2NZIpDTbQNg6tkfv_kAsIzEJ--mCI,1756
|
|
17
17
|
ScriptCollection/UnassignedVariablesCheck.py,sha256=8xNcp1D5HeOLO1a1NEerq2TsnpFhw7yzYPkaay-oisY,3804
|
|
18
18
|
ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -37,7 +37,7 @@ ScriptCollection/TFCPS/SCTaskRunnerServer.py,sha256=tub8_1QBu0nWAaGnBWp2nErOW-SO
|
|
|
37
37
|
ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py,sha256=67_DQwoH5bhxxSmwtuAeoW1NOCMiLMzOy48OqlJX-KY,1744
|
|
38
38
|
ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=umUfQDErp7xFLkDnSbQMxzUWnKtcGlrZnIvFcTpfGJs,40499
|
|
39
39
|
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=Mj9t1SyRpkYZE74cLsGWYv4xB89gXL2ld-RX3xvvGdw,9835
|
|
40
|
-
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=
|
|
40
|
+
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=6FmsF2M3GI8-Xjay0oUaLPTQ8yrHVSUiF_6mc01YoRM,49601
|
|
41
41
|
ScriptCollection/TFCPS/TFCPS_CreateRelease.py,sha256=kpEmSnfzfgB4OjcCuS_X6jgW_ftcphxEnjjoLoAHTLY,6582
|
|
42
42
|
ScriptCollection/TFCPS/TFCPS_Generic.py,sha256=mRBcf7XuNWOSXbZYc3NuJow18AMZ30D6Z-BtyNYfNcM,2072
|
|
43
43
|
ScriptCollection/TFCPS/TFCPS_MergeToMain.py,sha256=Krt5oyGNcRAN3szsgU7LoTh8yVTOdHW-Kq7R9MDDV-Q,7673
|
|
@@ -50,7 +50,7 @@ ScriptCollection/TFCPS/TFCPS_VisualRegressionTests.py,sha256=6_63udxHnPlu_TMk_ft
|
|
|
50
50
|
ScriptCollection/TFCPS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
51
51
|
ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py,sha256=BNZPMjGiWLMBH7QDtP06WEVLMnI38NLX4Dpd13bSbi4,16614
|
|
52
52
|
ScriptCollection/TFCPS/CPP/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
53
|
-
ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=
|
|
53
|
+
ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=zoy1uwgK_9Fxji8mwHoa6X-VQRc9AMKcT1tfJGZMQeI,22343
|
|
54
54
|
ScriptCollection/TFCPS/Docker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
55
55
|
ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py,sha256=bT6Gd5pQpZCw4OQz6HWkPCSn5z__eUUEisABLDSxd0o,200
|
|
56
56
|
ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py,sha256=QyjOfMY22JWCvKjMelHiDWbJiWqotOfebpJpgDUaoO4,237
|
|
@@ -71,8 +71,8 @@ ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=yyjWPMHQjPLkML
|
|
|
71
71
|
ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
72
72
|
ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py,sha256=5z990qyO5durX07xRWRwju1hl6wpjQAy2f-yRwJ-7s0,11856
|
|
73
73
|
ScriptCollection/TFCPS/TypeScript/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
74
|
-
scriptcollection-4.4.
|
|
75
|
-
scriptcollection-4.4.
|
|
76
|
-
scriptcollection-4.4.
|
|
77
|
-
scriptcollection-4.4.
|
|
78
|
-
scriptcollection-4.4.
|
|
74
|
+
scriptcollection-4.4.30.dist-info/METADATA,sha256=afxz3vYiDErPCZNzqM6zrv2xVN4yce3S1-MvBo6ebcg,7690
|
|
75
|
+
scriptcollection-4.4.30.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
76
|
+
scriptcollection-4.4.30.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
|
|
77
|
+
scriptcollection-4.4.30.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
|
|
78
|
+
scriptcollection-4.4.30.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|