ScriptCollection 4.4.66__py3-none-any.whl → 4.4.68__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/AgentRunner.py +91 -6
- ScriptCollection/AgentStatusReporting/AgentStatus.py +16 -0
- ScriptCollection/AgentStatusReporting/AgentStatusEventChannel.py +36 -0
- ScriptCollection/AgentStatusReporting/AgentStatusRelay.py +87 -0
- ScriptCollection/AgentStatusReporting/AgentStatusReporter.py +14 -0
- ScriptCollection/AgentStatusReporting/AgentStatusReportingSession.py +76 -0
- ScriptCollection/AgentStatusReporting/HerdrAgentStatusReporter.py +72 -0
- ScriptCollection/AgentStatusReporting/__init__.py +0 -0
- ScriptCollection/Executables.py +14 -0
- ScriptCollection/ScriptCollectionCore.py +1 -1
- ScriptCollection/TFCPS/ContainerBuildEnvironment.py +165 -0
- ScriptCollection/TFCPS/DotNet/NuGetPackageSourceRegistration.py +91 -0
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +2 -75
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +31 -141
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +1 -1
- {scriptcollection-4.4.66.dist-info → scriptcollection-4.4.68.dist-info}/METADATA +1 -1
- {scriptcollection-4.4.66.dist-info → scriptcollection-4.4.68.dist-info}/RECORD +20 -11
- {scriptcollection-4.4.66.dist-info → scriptcollection-4.4.68.dist-info}/entry_points.txt +1 -0
- {scriptcollection-4.4.66.dist-info → scriptcollection-4.4.68.dist-info}/WHEEL +0 -0
- {scriptcollection-4.4.66.dist-info → scriptcollection-4.4.68.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
from ..GeneralUtilities import GeneralUtilities
|
|
4
|
+
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
5
|
+
from .TFCPS_Tools_General import TFCPS_Tools_General
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ContainerMount:
|
|
9
|
+
"""A file or folder of the host which is made available in a container."""
|
|
10
|
+
source: str = None
|
|
11
|
+
target: str = None
|
|
12
|
+
read_only: bool = None
|
|
13
|
+
|
|
14
|
+
def __init__(self, source: str, target: str, read_only: bool):
|
|
15
|
+
self.source = source
|
|
16
|
+
self.target = target
|
|
17
|
+
self.read_only = read_only
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ContainerBuildEnvironment:
|
|
21
|
+
"""The build-environment which a container gets from the host on which it is started: the mounts, the environment-variables and the
|
|
22
|
+
host-preparation-hook. It is the one place which defines it, so that every container which builds a repository (the build-container of
|
|
23
|
+
'scbuildcodeunits -c' and the agent-sandbox of 'scrais') gets the same one. How the mounts are applied (bind-mount or copy) is decided by
|
|
24
|
+
the caller, because that depends on where the caller itself runs. See the article "Configuration-folder" of the reference for what is
|
|
25
|
+
mounted and why."""
|
|
26
|
+
|
|
27
|
+
__sc: ScriptCollectionCore = None
|
|
28
|
+
__tfcps_tools_general: TFCPS_Tools_General = None
|
|
29
|
+
__repository: str = None
|
|
30
|
+
|
|
31
|
+
def __init__(self, sc: ScriptCollectionCore, tfcps_tools_general: TFCPS_Tools_General, repository: str):
|
|
32
|
+
self.__sc = sc
|
|
33
|
+
self.__tfcps_tools_general = tfcps_tools_general
|
|
34
|
+
self.__repository = repository
|
|
35
|
+
|
|
36
|
+
@GeneralUtilities.check_arguments
|
|
37
|
+
def run_host_preparation_hook(self, build_arguments: list[str]) -> None:
|
|
38
|
+
"""Runs the optional user-specific script which prepares this host for a container-start (for example to log in to the registry the
|
|
39
|
+
image is pulled from). It has to run before get_environment_variables, so it can also create the files their values are read from."""
|
|
40
|
+
self.__tfcps_tools_general.run_custom_script_if_available(self.__tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScriptForContainer.py"), build_arguments)
|
|
41
|
+
|
|
42
|
+
@GeneralUtilities.check_arguments
|
|
43
|
+
def get_mounts(self) -> list[ContainerMount]:
|
|
44
|
+
"""Returns the files and folders of this host which a container gets. A part which does not exist on this host is not returned."""
|
|
45
|
+
result: list[ContainerMount] = []
|
|
46
|
+
|
|
47
|
+
#the optional user-specific scripts which prepare the container itself. the whole folder (not only the pre-codeunit-build-hook
|
|
48
|
+
#itself) is mounted, so the hook can start a sibling script placed next to it (for example to keep the hook independent of a
|
|
49
|
+
#package it only installs, while the actual preparation-logic which needs that package lives in the sibling script; see
|
|
50
|
+
#TFCPS_Tools_General.get_custom_scripts_folder_for_container). The mount is writable (not read-only) because the scripts in that
|
|
51
|
+
#folder also use it as their own download-cache-folder. A build inside the container runs the hook (see
|
|
52
|
+
#TFCPS_CodeUnit_BuildCodeUnits.__run_custom_pre_codeunit_build_script).
|
|
53
|
+
custom_scripts_folder: str = self.__tfcps_tools_general.get_custom_scripts_folder_for_container()
|
|
54
|
+
if os.path.isdir(custom_scripts_folder):
|
|
55
|
+
result.append(ContainerMount(custom_scripts_folder, self.__tfcps_tools_general.get_folder_of_custom_scripts_in_container(), False))
|
|
56
|
+
|
|
57
|
+
#the machine-wide image-registries-file of this host, so the container takes the images from the same custom registries as this
|
|
58
|
+
#host. Without it the container would find no mapping at all - the configuration-folder inside the container belongs to the
|
|
59
|
+
#container-user and is empty - and would therefore fall back to the upstream-registries defined in the repository, which are
|
|
60
|
+
#rate-limited. It is read-only because a build never changes the file.
|
|
61
|
+
oci_image_manager = self.__tfcps_tools_general.oci_image_manager
|
|
62
|
+
image_registries_file: str = oci_image_manager.get_image_registries_file_in_configuration_folder()
|
|
63
|
+
if os.path.isfile(image_registries_file):
|
|
64
|
+
result.append(ContainerMount(image_registries_file, oci_image_manager.get_image_registries_file_in_container(), True))
|
|
65
|
+
|
|
66
|
+
#the 'TFCPS'-folder of the configuration-folder of this host, so the container resolves the required environment-variables of the
|
|
67
|
+
#product (see TFCPS_Tools_General.get_required_environment_variables) and the OCI-registry-credentials (see
|
|
68
|
+
#ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) exactly like this host does - including a registry
|
|
69
|
+
#which is not publicly readable. The whole folder is mounted and not only EnvironmentVariables.csv in it, because a value of that file
|
|
70
|
+
#can point to a secret-file next to it ('file'-kind with a relative path, which is the recommended form) and such a value is resolved
|
|
71
|
+
#relative to the file itself. Only this folder (and not the whole configuration-folder) is mounted, so nothing else of it is exposed.
|
|
72
|
+
#It is read-only because a build never changes it.
|
|
73
|
+
tfcps_configuration_folder: str = self.__sc.get_tfcps_configuration_folder()
|
|
74
|
+
if os.path.isdir(tfcps_configuration_folder):
|
|
75
|
+
result.append(ContainerMount(tfcps_configuration_folder, self.__sc.get_tfcps_configuration_folder_in_container(), True))
|
|
76
|
+
|
|
77
|
+
#the files of the root-CA-certificates which this host declares via TrustedRootCA_<name>_Path. They are installed into the
|
|
78
|
+
#trust-store of the container by the entrypoint of the SCBuilder-image (see __get_trusted_root_cas).
|
|
79
|
+
file_cas, _ = self.__get_trusted_root_cas()
|
|
80
|
+
for ca_name, host_path in file_cas.items():
|
|
81
|
+
result.append(ContainerMount(host_path, f"/Workspace/RootCAs/{ca_name}.crt", True))
|
|
82
|
+
return result
|
|
83
|
+
|
|
84
|
+
@GeneralUtilities.check_arguments
|
|
85
|
+
def get_environment_variables(self) -> dict[str, str]:
|
|
86
|
+
"""Returns the environment-variables (name and value) which a container gets: the ones which are declared as required in
|
|
87
|
+
<repository>/.ScriptCollection/ProductInformation.xml (including the ones which are declared for this machine, see
|
|
88
|
+
TFCPS_Tools_General.get_required_environment_variable_names) and the TrustedRootCA_<name>_ContentBase64-variables. The values are
|
|
89
|
+
typically secrets: a caller must give them to docker through the environment of the docker-client and pass only their names as
|
|
90
|
+
arguments, because arguments are written to the log and are visible in the process-list of this host."""
|
|
91
|
+
result: dict[str, str] = self.__tfcps_tools_general.get_required_environment_variables(self.__repository)
|
|
92
|
+
_, inline_cas = self.__get_trusted_root_cas()
|
|
93
|
+
result.update(inline_cas)
|
|
94
|
+
return result
|
|
95
|
+
|
|
96
|
+
@GeneralUtilities.check_arguments
|
|
97
|
+
def __get_trusted_root_cas(self) -> tuple[dict[str, str], dict[str, str]]:
|
|
98
|
+
"""Resolves the TrustedRootCA_<name>_Path and TrustedRootCA_<name>_ContentBase64 environment-variables of this host. Returns
|
|
99
|
+
(file_cas, inline_cas):
|
|
100
|
+
- file_cas: <name> -> host-path of the certificate for each _Path-variable.
|
|
101
|
+
- inline_cas: variable-name -> value for each _ContentBase64-variable.
|
|
102
|
+
Candidate-names are collected from two sources (identical to the OCI-registry-credentials, see
|
|
103
|
+
ScriptCollectionCore.__get_defined_oci_registry_names): the machine-wide environment-variables-configuration-file and the
|
|
104
|
+
environment of the current process. The value is then resolved through ScriptCollectionCore.resolve_environment_variable, so a
|
|
105
|
+
CSV-entry takes precedence over an environment-variable of the same name - consistent with every other environment-variable
|
|
106
|
+
this script-collection resolves. If a name is declared in the file but its value can not be determined there (for example a
|
|
107
|
+
'file'-entry pointing to a missing file), the resolver falls back to the environment and fails only if neither source yields a
|
|
108
|
+
value. The actual trust-store update happens in SCBuilder's EntryPoint.sh, which scans /Workspace/RootCAs and the
|
|
109
|
+
TrustedRootCA_*-environment-variables and runs update-ca-certificates once before any build-step. The <name>-part becomes the
|
|
110
|
+
filename inside /usr/local/share/ca-certificates/ and must therefore be unique across the two env-var-forms; a duplicate is
|
|
111
|
+
rejected here with a clear error, before the container is started."""
|
|
112
|
+
path_pattern = re.compile(r"^TrustedRootCA_(.+)_Path$", re.IGNORECASE)
|
|
113
|
+
content_b64_pattern = re.compile(r"^TrustedRootCA_(.+)_ContentBase64$", re.IGNORECASE)
|
|
114
|
+
#collect every candidate-variable-name from both sources before any value is resolved; this way a secret-value is read only for
|
|
115
|
+
#a name that is actually declared and the discovery is identical to the OCI-registry-credential-discovery. The configuration-
|
|
116
|
+
#file is scanned before the environment and a name found in both is kept with the casing of the file: on Windows, os.environ
|
|
117
|
+
#normalizes a variable-name to uppercase as soon as it is set, so without this preference the same declaration in both places
|
|
118
|
+
#would be treated as two different names and resolution would silently pick the environment-value instead of the file-value.
|
|
119
|
+
candidate_names: dict[str, str] = {}#lower-case variable-name -> canonical variable-name (file-casing preferred)
|
|
120
|
+
configuration_file = self.__sc.get_environment_variables_configuration_file()
|
|
121
|
+
if os.path.isfile(configuration_file):
|
|
122
|
+
for entry in GeneralUtilities.read_csv_file(configuration_file, True):
|
|
123
|
+
if 0 == len(entry):
|
|
124
|
+
continue
|
|
125
|
+
env_name = entry[0]
|
|
126
|
+
if path_pattern.match(env_name) is not None or content_b64_pattern.match(env_name) is not None:
|
|
127
|
+
candidate_names.setdefault(env_name.lower(), env_name)
|
|
128
|
+
for env_name in os.environ.keys():
|
|
129
|
+
if path_pattern.match(env_name) is not None or content_b64_pattern.match(env_name) is not None:
|
|
130
|
+
candidate_names.setdefault(env_name.lower(), env_name)
|
|
131
|
+
seen_names: dict[str, str] = {}#lower-case <name>-part -> originating variable-name, used only to produce an informative error on duplicate <name>.
|
|
132
|
+
file_cas: dict[str, str] = {}
|
|
133
|
+
inline_cas: dict[str, str] = {}
|
|
134
|
+
#sorted so the log-output (and therefore the order of the mounts and environment-variables) stays deterministic across runs.
|
|
135
|
+
for env_name in sorted(candidate_names.values(), key=str.lower):
|
|
136
|
+
env_value = self.__sc.resolve_environment_variable(env_name, f"the TrustedRootCA-mechanism (declared via \"{env_name}\")")
|
|
137
|
+
if not GeneralUtilities.string_has_content(env_value):
|
|
138
|
+
continue
|
|
139
|
+
path_match = path_pattern.match(env_name)
|
|
140
|
+
if path_match is not None:
|
|
141
|
+
ca_name = path_match.group(1)
|
|
142
|
+
self.__check_trusted_root_ca_name_is_unique(seen_names, ca_name, env_name)
|
|
143
|
+
host_path = env_value.strip()
|
|
144
|
+
if not os.path.isfile(host_path):
|
|
145
|
+
raise ValueError(f"Environment-variable \"{env_name}\" points to a file which does not exist on this host: \"{host_path}\".")
|
|
146
|
+
with open(host_path, "r", encoding="utf-8", errors="replace") as pem_file:
|
|
147
|
+
head = pem_file.read(64)
|
|
148
|
+
if "-----BEGIN CERTIFICATE-----" not in head:
|
|
149
|
+
raise ValueError(f"Environment-variable \"{env_name}\" (\"{host_path}\") does not look like a PEM-encoded certificate (missing \"-----BEGIN CERTIFICATE-----\"-header).")
|
|
150
|
+
file_cas[ca_name] = host_path
|
|
151
|
+
continue
|
|
152
|
+
content_b64_match = content_b64_pattern.match(env_name)
|
|
153
|
+
if content_b64_match is not None:
|
|
154
|
+
ca_name = content_b64_match.group(1)
|
|
155
|
+
self.__check_trusted_root_ca_name_is_unique(seen_names, ca_name, env_name)
|
|
156
|
+
inline_cas[env_name] = env_value
|
|
157
|
+
continue
|
|
158
|
+
return file_cas, inline_cas
|
|
159
|
+
|
|
160
|
+
@GeneralUtilities.check_arguments
|
|
161
|
+
def __check_trusted_root_ca_name_is_unique(self, seen_names: dict[str, str], ca_name: str, env_name: str) -> None:
|
|
162
|
+
key = ca_name.lower()
|
|
163
|
+
if key in seen_names:
|
|
164
|
+
raise ValueError(f"Multiple TrustedRootCA-definitions for name \"{ca_name}\" found (\"{seen_names[key]}\" and \"{env_name}\"). The <name>-part becomes the filename in the trust-store and must therefore be unique across _Path and _ContentBase64.")
|
|
165
|
+
seen_names[key] = env_name
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from ...GeneralUtilities import GeneralUtilities
|
|
3
|
+
from ...ScriptCollectionCore import ScriptCollectionCore
|
|
4
|
+
from ...SCLog import LogLevel
|
|
5
|
+
from ..PackageSource import PackageSource
|
|
6
|
+
from ..TFCPS_Tools_General import TFCPS_Tools_General
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class NuGetPackageSourceRegistration:
|
|
10
|
+
"""Registers the NuGet-sources which are declared for the current machine (see TFCPS_Tools_General.get_declared_package_sources). It is
|
|
11
|
+
used by every DotNet-codeunit-build and by the command-line-tool 'scregisterpackagesources' (which the agent-sandbox of scrais runs before
|
|
12
|
+
the agent starts), so a source is registered on the same way everywhere."""
|
|
13
|
+
|
|
14
|
+
#the output of the dotnet-cli is localized. The language is set explicitly because the output of "dotnet nuget list source" gets parsed, which would otherwise only work on machines which are configured to use english.
|
|
15
|
+
__dotnet_cli_environment_variables: dict = {"DOTNET_CLI_UI_LANGUAGE": "en-US"}
|
|
16
|
+
__sc: ScriptCollectionCore = None
|
|
17
|
+
__tfcps_tools_general: TFCPS_Tools_General = None
|
|
18
|
+
|
|
19
|
+
def __init__(self, sc: ScriptCollectionCore, tfcps_tools_general: TFCPS_Tools_General):
|
|
20
|
+
self.__sc = sc
|
|
21
|
+
self.__tfcps_tools_general = tfcps_tools_general
|
|
22
|
+
|
|
23
|
+
@GeneralUtilities.check_arguments
|
|
24
|
+
def register_declared_package_sources(self, folder: str) -> None:
|
|
25
|
+
"""Ensures that every NuGet-source which is declared for this machine is registered with its current credentials, so a codeunit can
|
|
26
|
+
also use dependencies which are not available on nuget.org. The dotnet-cli is run in the given folder.
|
|
27
|
+
A source which is already registered under the declared name is removed and registered again, so an outdated registration (for
|
|
28
|
+
example one which points to another url) can not let the registration fail. Sources which are not declared are not touched at
|
|
29
|
+
all: the registration is stored in the NuGet-configuration of the current user, which - when the build runs directly on the
|
|
30
|
+
host - is the configuration the user also uses for their own work and which must not be destroyed by a build."""
|
|
31
|
+
package_sources: list[PackageSource] = self.__tfcps_tools_general.get_declared_package_sources("CSharp")
|
|
32
|
+
if len(package_sources) == 0:
|
|
33
|
+
return
|
|
34
|
+
for package_source in package_sources:
|
|
35
|
+
registered_sources: dict[str, str] = self.__get_registered_nuget_sources(folder)
|
|
36
|
+
name_of_source_with_same_name: str = None
|
|
37
|
+
name_of_source_with_same_url: str = None
|
|
38
|
+
for registered_name, registered_url in registered_sources.items():
|
|
39
|
+
if registered_name.lower() == package_source.name.lower():
|
|
40
|
+
name_of_source_with_same_name = registered_name
|
|
41
|
+
elif self.__nuget_sources_are_equal(registered_url, package_source.url):
|
|
42
|
+
#the same feed can already be registered under another name (which is the usual case on a machine on which the user
|
|
43
|
+
#configured the feed manually). Such a source is updated instead of being added a second time, because a second
|
|
44
|
+
#registration of the same url would let every restore query the feed twice.
|
|
45
|
+
name_of_source_with_same_url = registered_name
|
|
46
|
+
if name_of_source_with_same_name is not None:
|
|
47
|
+
self.__remove_nuget_source(folder, name_of_source_with_same_name)
|
|
48
|
+
if name_of_source_with_same_url is None:
|
|
49
|
+
arguments = ["nuget", "add", "source", package_source.url, "--name", package_source.name]
|
|
50
|
+
else:
|
|
51
|
+
#the credentials are set again even when the source already exists, because a token can have been rotated since it was registered.
|
|
52
|
+
arguments = ["nuget", "update", "source", name_of_source_with_same_url, "--source", package_source.url]
|
|
53
|
+
if package_source.has_credentials():
|
|
54
|
+
arguments = arguments+["--username", package_source.username, "--password", package_source.password]
|
|
55
|
+
if not GeneralUtilities.current_system_is_windows():
|
|
56
|
+
#without this the password can not be encrypted, which lets the command fail on the operating-systems which do not support the encryption.
|
|
57
|
+
arguments.append("--store-password-in-clear-text")
|
|
58
|
+
arguments_for_log = list(arguments)
|
|
59
|
+
if package_source.has_credentials():
|
|
60
|
+
arguments_for_log[arguments_for_log.index("--password")+1] = "***"#the password must not be written to the build-log
|
|
61
|
+
self.__sc.log.log(f"Add NuGet-source \"{package_source.name}\" ({package_source.url})..." if name_of_source_with_same_url is None else f"Update the already registered NuGet-source \"{name_of_source_with_same_url}\" ({package_source.url})...", LogLevel.Debug)
|
|
62
|
+
self.__sc.run_program_argsasarray("dotnet", arguments, folder, arguments_for_log=arguments_for_log, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
|
|
63
|
+
|
|
64
|
+
@GeneralUtilities.check_arguments
|
|
65
|
+
def __remove_nuget_source(self, folder: str, name: str) -> None:
|
|
66
|
+
"""Removes the registration of the NuGet-source with the given name, so it can be registered again with its currently declared url and credentials."""
|
|
67
|
+
self.__sc.log.log(f"Remove the outdated registration of the NuGet-source \"{name}\"...", LogLevel.Debug)
|
|
68
|
+
self.__sc.run_program_argsasarray("dotnet", ["nuget", "remove", "source", name], folder, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
|
|
69
|
+
|
|
70
|
+
@GeneralUtilities.check_arguments
|
|
71
|
+
def __get_registered_nuget_sources(self, folder: str) -> dict[str, str]:
|
|
72
|
+
"""Returns the NuGet-sources which are registered for the given folder as a mapping of their name to their url. The sources are
|
|
73
|
+
read from the output of "dotnet nuget list source", which prints the name (with its state) and the url of a source in two
|
|
74
|
+
consecutive lines."""
|
|
75
|
+
result: dict[str, str] = {}
|
|
76
|
+
list_result = self.__sc.run_program_argsasarray("dotnet", ["nuget", "list", "source"], folder, throw_exception_if_exitcode_is_not_zero=False, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
|
|
77
|
+
name_pattern = re.compile(r"^\s*\d+\.\s+(.+?)\s+\[[^\[\]]+\]\s*$")
|
|
78
|
+
current_name: str = None
|
|
79
|
+
for line in list_result[1].splitlines():
|
|
80
|
+
match = name_pattern.match(line)
|
|
81
|
+
if match is not None:
|
|
82
|
+
current_name = match.group(1).strip()
|
|
83
|
+
elif current_name is not None and GeneralUtilities.string_has_content(line):
|
|
84
|
+
result[current_name] = line.strip()
|
|
85
|
+
current_name = None
|
|
86
|
+
return result
|
|
87
|
+
|
|
88
|
+
@GeneralUtilities.check_arguments
|
|
89
|
+
def __nuget_sources_are_equal(self, url: str, other_url: str) -> bool:
|
|
90
|
+
"""Compares two urls of package-sources. A trailing slash and the case are ignored because they do not address a different feed."""
|
|
91
|
+
return url.strip().rstrip("/").lower() == other_url.strip().rstrip("/").lower()
|
|
@@ -11,15 +11,13 @@ from packaging.version import Version
|
|
|
11
11
|
from .CertificateGeneratorInformationBase import CertificateGeneratorInformationBase
|
|
12
12
|
from ...GeneralUtilities import GeneralUtilities, VersionEcholon
|
|
13
13
|
from ...SCLog import LogLevel
|
|
14
|
-
from
|
|
14
|
+
from .NuGetPackageSourceRegistration import NuGetPackageSourceRegistration
|
|
15
15
|
from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base,TFCPS_CodeUnitSpecific_Base_CLI
|
|
16
16
|
|
|
17
17
|
class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
18
18
|
|
|
19
19
|
is_library:bool = None
|
|
20
20
|
csproj_file:bool = None
|
|
21
|
-
#the output of the dotnet-cli is localized. The language is set explicitly for the commands whose output gets parsed, because otherwise the parsing would only work on machines which are configured to use english.
|
|
22
|
-
__dotnet_cli_environment_variables:dict = {"DOTNET_CLI_UI_LANGUAGE": "en-US"}
|
|
23
21
|
#the only accepted form of a version of a package-reference: an exact version in square brackets. Everything else is a version-range.
|
|
24
22
|
__pinned_version_regex = re.compile(r"^\[[^,\[\]]+\]$")
|
|
25
23
|
|
|
@@ -29,78 +27,7 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
29
27
|
self.is_library="<OutputType>Library</OutputType>" in GeneralUtilities.read_text_from_file(self.csproj_file)#TODO do a real check by checking this property using xpath
|
|
30
28
|
#every dotnet-operation of a codeunit (build, test, linting, ...) restores the dependencies of the codeunit, so the sources
|
|
31
29
|
#they are downloaded from have to be available for all of them and not only for the build.
|
|
32
|
-
self.
|
|
33
|
-
|
|
34
|
-
@GeneralUtilities.check_arguments
|
|
35
|
-
def __ensure_declared_package_sources_are_available(self) -> None:
|
|
36
|
-
"""Ensures that every NuGet-source which is declared for this machine (see TFCPS_Tools_General.get_declared_package_sources) is
|
|
37
|
-
registered with its current credentials, so a codeunit can also use dependencies which are not available on nuget.org.
|
|
38
|
-
A source which is already registered under the declared name is removed and registered again, so an outdated registration (for
|
|
39
|
-
example one which points to another url) can not let the registration fail. Sources which are not declared are not touched at
|
|
40
|
-
all: the registration is stored in the NuGet-configuration of the current user, which - when the build runs directly on the
|
|
41
|
-
host - is the configuration the user also uses for their own work and which must not be destroyed by a build."""
|
|
42
|
-
package_sources: list[PackageSource] = self.tfcps_Tools_General.get_declared_package_sources("CSharp")
|
|
43
|
-
if len(package_sources) == 0:
|
|
44
|
-
return
|
|
45
|
-
codeunit_folder: str = self.get_codeunit_folder()
|
|
46
|
-
for package_source in package_sources:
|
|
47
|
-
registered_sources: dict[str, str] = self.__get_registered_nuget_sources(codeunit_folder)
|
|
48
|
-
name_of_source_with_same_name: str = None
|
|
49
|
-
name_of_source_with_same_url: str = None
|
|
50
|
-
for registered_name, registered_url in registered_sources.items():
|
|
51
|
-
if registered_name.lower() == package_source.name.lower():
|
|
52
|
-
name_of_source_with_same_name = registered_name
|
|
53
|
-
elif self.__nuget_sources_are_equal(registered_url, package_source.url):
|
|
54
|
-
#the same feed can already be registered under another name (which is the usual case on a machine on which the user
|
|
55
|
-
#configured the feed manually). Such a source is updated instead of being added a second time, because a second
|
|
56
|
-
#registration of the same url would let every restore query the feed twice.
|
|
57
|
-
name_of_source_with_same_url = registered_name
|
|
58
|
-
if name_of_source_with_same_name is not None:
|
|
59
|
-
self.__remove_nuget_source(codeunit_folder, name_of_source_with_same_name)
|
|
60
|
-
if name_of_source_with_same_url is None:
|
|
61
|
-
arguments = ["nuget", "add", "source", package_source.url, "--name", package_source.name]
|
|
62
|
-
else:
|
|
63
|
-
#the credentials are set again even when the source already exists, because a token can have been rotated since it was registered.
|
|
64
|
-
arguments = ["nuget", "update", "source", name_of_source_with_same_url, "--source", package_source.url]
|
|
65
|
-
if package_source.has_credentials():
|
|
66
|
-
arguments = arguments+["--username", package_source.username, "--password", package_source.password]
|
|
67
|
-
if not GeneralUtilities.current_system_is_windows():
|
|
68
|
-
#without this the password can not be encrypted, which lets the command fail on the operating-systems which do not support the encryption.
|
|
69
|
-
arguments.append("--store-password-in-clear-text")
|
|
70
|
-
arguments_for_log = list(arguments)
|
|
71
|
-
if package_source.has_credentials():
|
|
72
|
-
arguments_for_log[arguments_for_log.index("--password")+1] = "***"#the password must not be written to the build-log
|
|
73
|
-
self._protected_sc.log.log(f"Add NuGet-source \"{package_source.name}\" ({package_source.url})..." if name_of_source_with_same_url is None else f"Update the already registered NuGet-source \"{name_of_source_with_same_url}\" ({package_source.url})...", LogLevel.Debug)
|
|
74
|
-
self._protected_sc.run_program_argsasarray("dotnet", arguments, codeunit_folder, arguments_for_log=arguments_for_log, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
|
|
75
|
-
|
|
76
|
-
@GeneralUtilities.check_arguments
|
|
77
|
-
def __remove_nuget_source(self, folder: str, name: str) -> None:
|
|
78
|
-
"""Removes the registration of the NuGet-source with the given name, so it can be registered again with its currently declared url and credentials."""
|
|
79
|
-
self._protected_sc.log.log(f"Remove the outdated registration of the NuGet-source \"{name}\"...", LogLevel.Debug)
|
|
80
|
-
self._protected_sc.run_program_argsasarray("dotnet", ["nuget", "remove", "source", name], folder, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
|
|
81
|
-
|
|
82
|
-
@GeneralUtilities.check_arguments
|
|
83
|
-
def __get_registered_nuget_sources(self, folder: str) -> dict[str, str]:
|
|
84
|
-
"""Returns the NuGet-sources which are registered for the given folder as a mapping of their name to their url. The sources are
|
|
85
|
-
read from the output of "dotnet nuget list source", which prints the name (with its state) and the url of a source in two
|
|
86
|
-
consecutive lines."""
|
|
87
|
-
result: dict[str, str] = {}
|
|
88
|
-
list_result = self._protected_sc.run_program_argsasarray("dotnet", ["nuget", "list", "source"], folder, throw_exception_if_exitcode_is_not_zero=False, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
|
|
89
|
-
name_pattern = re.compile(r"^\s*\d+\.\s+(.+?)\s+\[[^\[\]]+\]\s*$")
|
|
90
|
-
current_name: str = None
|
|
91
|
-
for line in list_result[1].splitlines():
|
|
92
|
-
match = name_pattern.match(line)
|
|
93
|
-
if match is not None:
|
|
94
|
-
current_name = match.group(1).strip()
|
|
95
|
-
elif current_name is not None and GeneralUtilities.string_has_content(line):
|
|
96
|
-
result[current_name] = line.strip()
|
|
97
|
-
current_name = None
|
|
98
|
-
return result
|
|
99
|
-
|
|
100
|
-
@GeneralUtilities.check_arguments
|
|
101
|
-
def __nuget_sources_are_equal(self, url: str, other_url: str) -> bool:
|
|
102
|
-
"""Compares two urls of package-sources. A trailing slash and the case are ignored because they do not address a different feed."""
|
|
103
|
-
return url.strip().rstrip("/").lower() == other_url.strip().rstrip("/").lower()
|
|
30
|
+
NuGetPackageSourceRegistration(self._protected_sc, self.tfcps_Tools_General).register_declared_package_sources(self.get_codeunit_folder())
|
|
104
31
|
|
|
105
32
|
@GeneralUtilities.check_arguments
|
|
106
33
|
def get_runtimes(self) -> list[str]:
|
|
@@ -15,6 +15,7 @@ from .TFCPS_CodeSpellCheck import TFCPS_CodeSpellCheck
|
|
|
15
15
|
from .TFCPS_FindingsSorter import TFCPS_FindingsSorter
|
|
16
16
|
from .TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
|
|
17
17
|
from .TFCPS_Tools_General import TFCPS_Tools_General, CommonProjectStructureVersion
|
|
18
|
+
from .ContainerBuildEnvironment import ContainerBuildEnvironment
|
|
18
19
|
|
|
19
20
|
|
|
20
21
|
class TFCPS_UpdateDependenciesHook(TFCPS_BuildCodeUnitsHook):
|
|
@@ -139,9 +140,6 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
139
140
|
#ensure <repo>/.gitignore is set up (ignores the cache-folder so cache-files never show up as uncommitted changes)
|
|
140
141
|
self.__ensure_global_gitignore_is_setup(self.repository)
|
|
141
142
|
|
|
142
|
-
#validate the openspec-specifications if openspec is set up in the repository
|
|
143
|
-
self.__validate_openspec()
|
|
144
|
-
|
|
145
143
|
self.sc.log.log(f"Start building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(start_time,False)}. (Target environment-type: {self.target_environment_type})")
|
|
146
144
|
|
|
147
145
|
#ensure common cache-files are available
|
|
@@ -200,21 +198,23 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
200
198
|
hook.run_after_codeunit_was_built(tFCPS_CodeUnit_BuildCodeUnit)
|
|
201
199
|
self.sc.log.log(GeneralUtilities.get_line())
|
|
202
200
|
|
|
203
|
-
self.
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
201
|
+
self.sc.log.log("Do project post-build-tasks...")
|
|
202
|
+
|
|
203
|
+
if not self.__fast_lane:
|
|
204
|
+
self.__do_project_linting(self.repository)
|
|
205
|
+
self.__update_agent_sandbox_image(self.repository)
|
|
206
|
+
self.search_for_secrets()
|
|
208
207
|
self.__normalize_line_endings_of_common_files()
|
|
209
|
-
self.search_for_secrets()
|
|
210
|
-
|
|
211
|
-
self.sc.log.log(GeneralUtilities.get_line())
|
|
212
208
|
|
|
213
209
|
if self.is_pre_merge():
|
|
214
210
|
self.__collect_metrics()
|
|
215
211
|
self.__generate_loc_diagram()
|
|
216
|
-
self.sc.log.log(GeneralUtilities.get_line())
|
|
217
212
|
|
|
213
|
+
self.tfcps_tools_general.generate_svg_files_from_plantuml_files_for_repository(self.repository, self.use_cache())
|
|
214
|
+
self.tfcps_tools_general.generate_svg_files_from_vega_files_for_repository(self.repository, self.use_cache())
|
|
215
|
+
self.tfcps_tools_general.generate_svg_files_from_vegalite_files_for_repository(self.repository, self.use_cache())
|
|
216
|
+
|
|
217
|
+
self.sc.log.log(GeneralUtilities.get_line())
|
|
218
218
|
except Exception:
|
|
219
219
|
error_occurred=True
|
|
220
220
|
raise
|
|
@@ -286,6 +286,7 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
286
286
|
self.sc.log.log("Do project-linting...")
|
|
287
287
|
self.__do_project_linting_using_codespell(repository_folder)
|
|
288
288
|
self.__sort_findings(repository_folder)
|
|
289
|
+
self.__validate_openspec(repository_folder)
|
|
289
290
|
|
|
290
291
|
@GeneralUtilities.check_arguments
|
|
291
292
|
def __sort_findings(self, repository_folder: str) -> None:
|
|
@@ -436,11 +437,14 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
436
437
|
self.sc.run_with_epew("openspec", "update --force", self.repository)
|
|
437
438
|
|
|
438
439
|
@GeneralUtilities.check_arguments
|
|
439
|
-
def __validate_openspec(self) -> None:
|
|
440
|
-
openspec_configuration_file: str = os.path.join(
|
|
440
|
+
def __validate_openspec(self,repository:str) -> None:
|
|
441
|
+
openspec_configuration_file: str = os.path.join(repository, "openspec", "config.yaml")
|
|
441
442
|
if os.path.isfile(openspec_configuration_file):
|
|
442
443
|
self.sc.log.log("Validate openspec-specifications...")
|
|
443
|
-
|
|
444
|
+
try:
|
|
445
|
+
self.sc.run_with_epew("openspec", "validate --all --strict", repository)
|
|
446
|
+
except Exception as e:
|
|
447
|
+
self.sc.log.log_exception(f"Openspec validation failed.",e,LogLevel.Warning)
|
|
444
448
|
|
|
445
449
|
@GeneralUtilities.check_arguments
|
|
446
450
|
def get_default_base_mount_folder(self) -> str:
|
|
@@ -565,61 +569,19 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
565
569
|
update_argument="pip3 install scriptcollection --upgrade && "
|
|
566
570
|
executable_arguments=["bash","-c", f"{update_argument}scshowversion && "+" ".join(executable_arguments)]
|
|
567
571
|
|
|
568
|
-
#
|
|
569
|
-
#
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
#itself) is mounted, so the hook can start a sibling script placed next to it (for example to keep the hook independent of a
|
|
574
|
-
#package it only installs, while the actual preparation-logic which needs that package lives in the sibling script; see
|
|
575
|
-
#get_custom_scripts_folder_for_container). The mount is writable (not read-only) because the scripts in that folder also use it as
|
|
576
|
-
#their own download-cache-folder. The build inside the container runs the hook (see __run_custom_pre_codeunit_build_script); if the
|
|
577
|
-
#folder does not exist on the host nothing is mounted and nothing is run.
|
|
572
|
+
#the build-environment (host-hook, mounts and environment-variables) is defined by ContainerBuildEnvironment, which is also used by
|
|
573
|
+
#the agent-sandbox of scrais, so that both containers get the same one. The hook runs before the environment-variables are resolved,
|
|
574
|
+
#so it can also create the files their values are read from.
|
|
575
|
+
container_build_environment: ContainerBuildEnvironment = ContainerBuildEnvironment(self.sc, self.tfcps_tools_general, self.repository)
|
|
576
|
+
container_build_environment.run_host_preparation_hook(self.__get_build_arguments(self.repository))
|
|
578
577
|
mount_arguments: list[str] = []
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
#
|
|
584
|
-
#
|
|
585
|
-
|
|
586
|
-
#upstream-registries defined in the repository, which are rate-limited. The mount is read-only because a build never changes the file.
|
|
587
|
-
oci_image_manager = self.tfcps_tools_general.oci_image_manager
|
|
588
|
-
image_registries_file: str = oci_image_manager.get_image_registries_file_in_configuration_folder()
|
|
589
|
-
if os.path.isfile(image_registries_file):
|
|
590
|
-
mount_arguments += ["-v", f"{image_registries_file}:{oci_image_manager.get_image_registries_file_in_container()}:ro"]
|
|
591
|
-
|
|
592
|
-
#mount the 'TFCPS'-folder of the configuration-folder of this host (if it exists), so the build inside the container resolves the
|
|
593
|
-
#required environment-variables of the product (see TFCPS_Tools_General.get_required_environment_variables) and the
|
|
594
|
-
#OCI-registry-credentials (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) exactly like a
|
|
595
|
-
#build on the host does - including a registry which is not publicly readable, which without this mount could not be used inside
|
|
596
|
-
#the container at all, because the configuration-folder of the container-user does not contain the credentials of this host.
|
|
597
|
-
#The whole folder is mounted and not only EnvironmentVariables.csv in it, because a value of that file can point to a secret-file
|
|
598
|
-
#next to it ('file'-kind with a relative path, which is the recommended form) and such a value is resolved relative to the file
|
|
599
|
-
#itself - so with only the file mounted, every entry of that kind would fail inside the container. Only this folder (and not the
|
|
600
|
-
#whole configuration-folder) is mounted, so nothing else of it is exposed to the container. The mount is read-only because a build
|
|
601
|
-
#never changes it. The alternative way to provide a value is an environment-variable of the build-pipeline itself (the pipeline's
|
|
602
|
-
#own secret-store); both sources are used together, the file taking precedence.
|
|
603
|
-
tfcps_configuration_folder: str = self.sc.get_tfcps_configuration_folder()
|
|
604
|
-
if os.path.isdir(tfcps_configuration_folder):
|
|
605
|
-
mount_arguments += ["-v", f"{tfcps_configuration_folder}:{self.sc.get_tfcps_configuration_folder_in_container()}:ro"]
|
|
606
|
-
|
|
607
|
-
#install the host-declared trusted root-CA-certificates into the container (needed when the build pulls packages from a privately
|
|
608
|
-
#hosted registry whose certificate-chain is not publicly rooted - for example an internally hosted gitlab-nuget-source). The
|
|
609
|
-
#two env-var forms are detected automatically from both the machine-wide environment-variables-configuration-file and the
|
|
610
|
-
#environment of the current process; see __collect_trusted_root_ca_arguments. _Path-variables add file-mounts, the inline
|
|
611
|
-
#_ContentBase64-form is returned as a name->value dict that gets merged into required_environment_variables below so its
|
|
612
|
-
#value flows through the docker-client's own environment (not through argv) just like a required secret does.
|
|
613
|
-
trusted_root_ca_mount_arguments, trusted_root_ca_inline_env_vars = self.__collect_trusted_root_ca_arguments()
|
|
614
|
-
mount_arguments += trusted_root_ca_mount_arguments
|
|
615
|
-
|
|
616
|
-
#pass the environment-variables which are declared as required in <repository>/.ScriptCollection/ProductInformation.xml into the container
|
|
617
|
-
#so they do not have to be specified explicitly on every call. Their values are resolved from the user-specific
|
|
618
|
-
#configuration-file (see TFCPS_Tools_General.get_environment_variables_configuration_file), which only exists on the host.
|
|
619
|
-
#only the names are passed as arguments; the values are given to the docker-client through its own environment, because arguments
|
|
620
|
-
#are written to the log and are visible in the process-list of this host, which must not happen for a value which is typically a secret.
|
|
621
|
-
required_environment_variables: dict[str, str] = self.tfcps_tools_general.get_required_environment_variables(self.repository)
|
|
622
|
-
required_environment_variables.update(trusted_root_ca_inline_env_vars)
|
|
578
|
+
for mount in container_build_environment.get_mounts():
|
|
579
|
+
mount_arguments += ["-v", f"{mount.source}:{mount.target}{':ro' if mount.read_only else ''}"]
|
|
580
|
+
|
|
581
|
+
#only the names of the environment-variables are passed as arguments; the values are given to the docker-client through its own
|
|
582
|
+
#environment, because arguments are written to the log and are visible in the process-list of this host, which must not happen for
|
|
583
|
+
#a value which is typically a secret.
|
|
584
|
+
required_environment_variables: dict[str, str] = container_build_environment.get_environment_variables()
|
|
623
585
|
env_arguments: list[str] = []
|
|
624
586
|
for env_variable_name in required_environment_variables:
|
|
625
587
|
env_arguments += ["-e", env_variable_name]
|
|
@@ -652,78 +614,6 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
652
614
|
stderr:str=result[2] or GeneralUtilities.empty_string
|
|
653
615
|
return (exit_code==0,f"{stdout}\n{stderr}")
|
|
654
616
|
|
|
655
|
-
@GeneralUtilities.check_arguments
|
|
656
|
-
def __collect_trusted_root_ca_arguments(self) -> tuple[list[str], dict[str, str]]:
|
|
657
|
-
"""Resolves the TrustedRootCA_<name>_Path and TrustedRootCA_<name>_ContentBase64 environment-variables of the current host into
|
|
658
|
-
docker-arguments which install those certificates into the trust-store of the build-container. Returns (mount_arguments, inline_env_vars):
|
|
659
|
-
- mount_arguments: '-v <host-path>:/Workspace/RootCAs/<name>.crt:ro' entries for each _Path-variable.
|
|
660
|
-
- inline_env_vars: a name->value dict for each _ContentBase64-variable; the caller merges it into required_environment_variables
|
|
661
|
-
so the values flow through the docker-client's own environment, not through argv.
|
|
662
|
-
Candidate-names are collected from two sources (identical to the OCI-registry-credentials, see
|
|
663
|
-
ScriptCollectionCore.__get_defined_oci_registry_names): the machine-wide environment-variables-configuration-file and the
|
|
664
|
-
environment of the current process. The value is then resolved through ScriptCollectionCore.resolve_environment_variable, so a
|
|
665
|
-
CSV-entry takes precedence over an environment-variable of the same name - consistent with every other environment-variable
|
|
666
|
-
this script-collection resolves. If a name is declared in the file but its value can not be determined there (for example a
|
|
667
|
-
'file'-entry pointing to a missing file), the resolver falls back to the environment and fails only if neither source yields a
|
|
668
|
-
value. The actual trust-store update happens in SCBuilder's EntryPoint.sh, which scans /Workspace/RootCAs and the
|
|
669
|
-
TrustedRootCA_*-environment-variables and runs update-ca-certificates once before any build-step. The <name>-part becomes the
|
|
670
|
-
filename inside /usr/local/share/ca-certificates/ and must therefore be unique across the two env-var-forms; a duplicate is
|
|
671
|
-
rejected here with a clear error, before the container is started."""
|
|
672
|
-
path_pattern = re.compile(r"^TrustedRootCA_(.+)_Path$", re.IGNORECASE)
|
|
673
|
-
content_b64_pattern = re.compile(r"^TrustedRootCA_(.+)_ContentBase64$", re.IGNORECASE)
|
|
674
|
-
#collect every candidate-variable-name from both sources before any value is resolved; this way a secret-value is read only for
|
|
675
|
-
#a name that is actually declared and the discovery is identical to the OCI-registry-credential-discovery. The configuration-
|
|
676
|
-
#file is scanned before the environment and a name found in both is kept with the casing of the file: on Windows, os.environ
|
|
677
|
-
#normalizes a variable-name to uppercase as soon as it is set, so without this preference the same declaration in both places
|
|
678
|
-
#would be treated as two different names and resolution would silently pick the environment-value instead of the file-value.
|
|
679
|
-
candidate_names: dict[str, str] = {}#lower-case variable-name -> canonical variable-name (file-casing preferred)
|
|
680
|
-
configuration_file = self.sc.get_environment_variables_configuration_file()
|
|
681
|
-
if os.path.isfile(configuration_file):
|
|
682
|
-
for entry in GeneralUtilities.read_csv_file(configuration_file, True):
|
|
683
|
-
if 0 == len(entry):
|
|
684
|
-
continue
|
|
685
|
-
env_name = entry[0]
|
|
686
|
-
if path_pattern.match(env_name) is not None or content_b64_pattern.match(env_name) is not None:
|
|
687
|
-
candidate_names.setdefault(env_name.lower(), env_name)
|
|
688
|
-
for env_name in os.environ.keys():
|
|
689
|
-
if path_pattern.match(env_name) is not None or content_b64_pattern.match(env_name) is not None:
|
|
690
|
-
candidate_names.setdefault(env_name.lower(), env_name)
|
|
691
|
-
seen_names: dict[str, str] = {}#lower-case <name>-part -> originating variable-name, used only to produce an informative error on duplicate <name>.
|
|
692
|
-
mount_arguments: list[str] = []
|
|
693
|
-
inline_env_vars: dict[str, str] = {}
|
|
694
|
-
#sorted so the log-output (and therefore the order of "-v"-/"-e"-arguments) stays deterministic across runs.
|
|
695
|
-
for env_name in sorted(candidate_names.values(), key=str.lower):
|
|
696
|
-
env_value = self.sc.resolve_environment_variable(env_name, f"the TrustedRootCA-mechanism (declared via \"{env_name}\")")
|
|
697
|
-
if not GeneralUtilities.string_has_content(env_value):
|
|
698
|
-
continue
|
|
699
|
-
path_match = path_pattern.match(env_name)
|
|
700
|
-
if path_match is not None:
|
|
701
|
-
ca_name = path_match.group(1)
|
|
702
|
-
self.__check_trusted_root_ca_name_is_unique(seen_names, ca_name, env_name)
|
|
703
|
-
host_path = env_value.strip()
|
|
704
|
-
if not os.path.isfile(host_path):
|
|
705
|
-
raise ValueError(f"Environment-variable \"{env_name}\" points to a file which does not exist on this host: \"{host_path}\".")
|
|
706
|
-
with open(host_path, "r", encoding="utf-8", errors="replace") as pem_file:
|
|
707
|
-
head = pem_file.read(64)
|
|
708
|
-
if "-----BEGIN CERTIFICATE-----" not in head:
|
|
709
|
-
raise ValueError(f"Environment-variable \"{env_name}\" (\"{host_path}\") does not look like a PEM-encoded certificate (missing \"-----BEGIN CERTIFICATE-----\"-header).")
|
|
710
|
-
mount_arguments += ["-v", f"{host_path}:/Workspace/RootCAs/{ca_name}.crt:ro"]
|
|
711
|
-
continue
|
|
712
|
-
content_b64_match = content_b64_pattern.match(env_name)
|
|
713
|
-
if content_b64_match is not None:
|
|
714
|
-
ca_name = content_b64_match.group(1)
|
|
715
|
-
self.__check_trusted_root_ca_name_is_unique(seen_names, ca_name, env_name)
|
|
716
|
-
inline_env_vars[env_name] = env_value
|
|
717
|
-
continue
|
|
718
|
-
return mount_arguments, inline_env_vars
|
|
719
|
-
|
|
720
|
-
@GeneralUtilities.check_arguments
|
|
721
|
-
def __check_trusted_root_ca_name_is_unique(self, seen_names: dict[str, str], ca_name: str, env_name: str) -> None:
|
|
722
|
-
key = ca_name.lower()
|
|
723
|
-
if key in seen_names:
|
|
724
|
-
raise ValueError(f"Multiple TrustedRootCA-definitions for name \"{ca_name}\" found (\"{seen_names[key]}\" and \"{env_name}\"). The <name>-part becomes the filename in the trust-store and must therefore be unique across _Path and _ContentBase64.")
|
|
725
|
-
seen_names[key] = env_name
|
|
726
|
-
|
|
727
617
|
@GeneralUtilities.check_arguments
|
|
728
618
|
def __translate_path_into_container(self, host_path: str, container_repository_folder: str) -> str:
|
|
729
619
|
normalized_repository = os.path.normpath(self.repository)
|
|
@@ -1958,7 +1958,7 @@ class TFCPS_Tools_General:
|
|
|
1958
1958
|
arch_tags.append(arch_tag)
|
|
1959
1959
|
# Load tar → local image
|
|
1960
1960
|
GeneralUtilities.write_message_to_stdout(f"Loading {tar_path}...")
|
|
1961
|
-
result = self.__sc.
|
|
1961
|
+
result = self.__sc.run_program_argsasarray_with_retry("docker",[ "load", "-i", tar_path],amount_of_attempts=3,delay_in_seconds=5)
|
|
1962
1962
|
# docker load outputs: "Loaded image: sha256:abc123..." or "Loaded image ID: ..."
|
|
1963
1963
|
# we need the loaded image ID
|
|
1964
1964
|
loaded_id = None
|