ScriptCollection 4.4.66__py3-none-any.whl → 4.4.68__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,165 @@
1
+ import os
2
+ import re
3
+ from ..GeneralUtilities import GeneralUtilities
4
+ from ..ScriptCollectionCore import ScriptCollectionCore
5
+ from .TFCPS_Tools_General import TFCPS_Tools_General
6
+
7
+
8
+ class ContainerMount:
9
+ """A file or folder of the host which is made available in a container."""
10
+ source: str = None
11
+ target: str = None
12
+ read_only: bool = None
13
+
14
+ def __init__(self, source: str, target: str, read_only: bool):
15
+ self.source = source
16
+ self.target = target
17
+ self.read_only = read_only
18
+
19
+
20
+ class ContainerBuildEnvironment:
21
+ """The build-environment which a container gets from the host on which it is started: the mounts, the environment-variables and the
22
+ host-preparation-hook. It is the one place which defines it, so that every container which builds a repository (the build-container of
23
+ 'scbuildcodeunits -c' and the agent-sandbox of 'scrais') gets the same one. How the mounts are applied (bind-mount or copy) is decided by
24
+ the caller, because that depends on where the caller itself runs. See the article "Configuration-folder" of the reference for what is
25
+ mounted and why."""
26
+
27
+ __sc: ScriptCollectionCore = None
28
+ __tfcps_tools_general: TFCPS_Tools_General = None
29
+ __repository: str = None
30
+
31
+ def __init__(self, sc: ScriptCollectionCore, tfcps_tools_general: TFCPS_Tools_General, repository: str):
32
+ self.__sc = sc
33
+ self.__tfcps_tools_general = tfcps_tools_general
34
+ self.__repository = repository
35
+
36
+ @GeneralUtilities.check_arguments
37
+ def run_host_preparation_hook(self, build_arguments: list[str]) -> None:
38
+ """Runs the optional user-specific script which prepares this host for a container-start (for example to log in to the registry the
39
+ image is pulled from). It has to run before get_environment_variables, so it can also create the files their values are read from."""
40
+ self.__tfcps_tools_general.run_custom_script_if_available(self.__tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScriptForContainer.py"), build_arguments)
41
+
42
+ @GeneralUtilities.check_arguments
43
+ def get_mounts(self) -> list[ContainerMount]:
44
+ """Returns the files and folders of this host which a container gets. A part which does not exist on this host is not returned."""
45
+ result: list[ContainerMount] = []
46
+
47
+ #the optional user-specific scripts which prepare the container itself. the whole folder (not only the pre-codeunit-build-hook
48
+ #itself) is mounted, so the hook can start a sibling script placed next to it (for example to keep the hook independent of a
49
+ #package it only installs, while the actual preparation-logic which needs that package lives in the sibling script; see
50
+ #TFCPS_Tools_General.get_custom_scripts_folder_for_container). The mount is writable (not read-only) because the scripts in that
51
+ #folder also use it as their own download-cache-folder. A build inside the container runs the hook (see
52
+ #TFCPS_CodeUnit_BuildCodeUnits.__run_custom_pre_codeunit_build_script).
53
+ custom_scripts_folder: str = self.__tfcps_tools_general.get_custom_scripts_folder_for_container()
54
+ if os.path.isdir(custom_scripts_folder):
55
+ result.append(ContainerMount(custom_scripts_folder, self.__tfcps_tools_general.get_folder_of_custom_scripts_in_container(), False))
56
+
57
+ #the machine-wide image-registries-file of this host, so the container takes the images from the same custom registries as this
58
+ #host. Without it the container would find no mapping at all - the configuration-folder inside the container belongs to the
59
+ #container-user and is empty - and would therefore fall back to the upstream-registries defined in the repository, which are
60
+ #rate-limited. It is read-only because a build never changes the file.
61
+ oci_image_manager = self.__tfcps_tools_general.oci_image_manager
62
+ image_registries_file: str = oci_image_manager.get_image_registries_file_in_configuration_folder()
63
+ if os.path.isfile(image_registries_file):
64
+ result.append(ContainerMount(image_registries_file, oci_image_manager.get_image_registries_file_in_container(), True))
65
+
66
+ #the 'TFCPS'-folder of the configuration-folder of this host, so the container resolves the required environment-variables of the
67
+ #product (see TFCPS_Tools_General.get_required_environment_variables) and the OCI-registry-credentials (see
68
+ #ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) exactly like this host does - including a registry
69
+ #which is not publicly readable. The whole folder is mounted and not only EnvironmentVariables.csv in it, because a value of that file
70
+ #can point to a secret-file next to it ('file'-kind with a relative path, which is the recommended form) and such a value is resolved
71
+ #relative to the file itself. Only this folder (and not the whole configuration-folder) is mounted, so nothing else of it is exposed.
72
+ #It is read-only because a build never changes it.
73
+ tfcps_configuration_folder: str = self.__sc.get_tfcps_configuration_folder()
74
+ if os.path.isdir(tfcps_configuration_folder):
75
+ result.append(ContainerMount(tfcps_configuration_folder, self.__sc.get_tfcps_configuration_folder_in_container(), True))
76
+
77
+ #the files of the root-CA-certificates which this host declares via TrustedRootCA_<name>_Path. They are installed into the
78
+ #trust-store of the container by the entrypoint of the SCBuilder-image (see __get_trusted_root_cas).
79
+ file_cas, _ = self.__get_trusted_root_cas()
80
+ for ca_name, host_path in file_cas.items():
81
+ result.append(ContainerMount(host_path, f"/Workspace/RootCAs/{ca_name}.crt", True))
82
+ return result
83
+
84
+ @GeneralUtilities.check_arguments
85
+ def get_environment_variables(self) -> dict[str, str]:
86
+ """Returns the environment-variables (name and value) which a container gets: the ones which are declared as required in
87
+ <repository>/.ScriptCollection/ProductInformation.xml (including the ones which are declared for this machine, see
88
+ TFCPS_Tools_General.get_required_environment_variable_names) and the TrustedRootCA_<name>_ContentBase64-variables. The values are
89
+ typically secrets: a caller must give them to docker through the environment of the docker-client and pass only their names as
90
+ arguments, because arguments are written to the log and are visible in the process-list of this host."""
91
+ result: dict[str, str] = self.__tfcps_tools_general.get_required_environment_variables(self.__repository)
92
+ _, inline_cas = self.__get_trusted_root_cas()
93
+ result.update(inline_cas)
94
+ return result
95
+
96
+ @GeneralUtilities.check_arguments
97
+ def __get_trusted_root_cas(self) -> tuple[dict[str, str], dict[str, str]]:
98
+ """Resolves the TrustedRootCA_<name>_Path and TrustedRootCA_<name>_ContentBase64 environment-variables of this host. Returns
99
+ (file_cas, inline_cas):
100
+ - file_cas: <name> -> host-path of the certificate for each _Path-variable.
101
+ - inline_cas: variable-name -> value for each _ContentBase64-variable.
102
+ Candidate-names are collected from two sources (identical to the OCI-registry-credentials, see
103
+ ScriptCollectionCore.__get_defined_oci_registry_names): the machine-wide environment-variables-configuration-file and the
104
+ environment of the current process. The value is then resolved through ScriptCollectionCore.resolve_environment_variable, so a
105
+ CSV-entry takes precedence over an environment-variable of the same name - consistent with every other environment-variable
106
+ this script-collection resolves. If a name is declared in the file but its value can not be determined there (for example a
107
+ 'file'-entry pointing to a missing file), the resolver falls back to the environment and fails only if neither source yields a
108
+ value. The actual trust-store update happens in SCBuilder's EntryPoint.sh, which scans /Workspace/RootCAs and the
109
+ TrustedRootCA_*-environment-variables and runs update-ca-certificates once before any build-step. The <name>-part becomes the
110
+ filename inside /usr/local/share/ca-certificates/ and must therefore be unique across the two env-var-forms; a duplicate is
111
+ rejected here with a clear error, before the container is started."""
112
+ path_pattern = re.compile(r"^TrustedRootCA_(.+)_Path$", re.IGNORECASE)
113
+ content_b64_pattern = re.compile(r"^TrustedRootCA_(.+)_ContentBase64$", re.IGNORECASE)
114
+ #collect every candidate-variable-name from both sources before any value is resolved; this way a secret-value is read only for
115
+ #a name that is actually declared and the discovery is identical to the OCI-registry-credential-discovery. The configuration-
116
+ #file is scanned before the environment and a name found in both is kept with the casing of the file: on Windows, os.environ
117
+ #normalizes a variable-name to uppercase as soon as it is set, so without this preference the same declaration in both places
118
+ #would be treated as two different names and resolution would silently pick the environment-value instead of the file-value.
119
+ candidate_names: dict[str, str] = {}#lower-case variable-name -> canonical variable-name (file-casing preferred)
120
+ configuration_file = self.__sc.get_environment_variables_configuration_file()
121
+ if os.path.isfile(configuration_file):
122
+ for entry in GeneralUtilities.read_csv_file(configuration_file, True):
123
+ if 0 == len(entry):
124
+ continue
125
+ env_name = entry[0]
126
+ if path_pattern.match(env_name) is not None or content_b64_pattern.match(env_name) is not None:
127
+ candidate_names.setdefault(env_name.lower(), env_name)
128
+ for env_name in os.environ.keys():
129
+ if path_pattern.match(env_name) is not None or content_b64_pattern.match(env_name) is not None:
130
+ candidate_names.setdefault(env_name.lower(), env_name)
131
+ seen_names: dict[str, str] = {}#lower-case <name>-part -> originating variable-name, used only to produce an informative error on duplicate <name>.
132
+ file_cas: dict[str, str] = {}
133
+ inline_cas: dict[str, str] = {}
134
+ #sorted so the log-output (and therefore the order of the mounts and environment-variables) stays deterministic across runs.
135
+ for env_name in sorted(candidate_names.values(), key=str.lower):
136
+ env_value = self.__sc.resolve_environment_variable(env_name, f"the TrustedRootCA-mechanism (declared via \"{env_name}\")")
137
+ if not GeneralUtilities.string_has_content(env_value):
138
+ continue
139
+ path_match = path_pattern.match(env_name)
140
+ if path_match is not None:
141
+ ca_name = path_match.group(1)
142
+ self.__check_trusted_root_ca_name_is_unique(seen_names, ca_name, env_name)
143
+ host_path = env_value.strip()
144
+ if not os.path.isfile(host_path):
145
+ raise ValueError(f"Environment-variable \"{env_name}\" points to a file which does not exist on this host: \"{host_path}\".")
146
+ with open(host_path, "r", encoding="utf-8", errors="replace") as pem_file:
147
+ head = pem_file.read(64)
148
+ if "-----BEGIN CERTIFICATE-----" not in head:
149
+ raise ValueError(f"Environment-variable \"{env_name}\" (\"{host_path}\") does not look like a PEM-encoded certificate (missing \"-----BEGIN CERTIFICATE-----\"-header).")
150
+ file_cas[ca_name] = host_path
151
+ continue
152
+ content_b64_match = content_b64_pattern.match(env_name)
153
+ if content_b64_match is not None:
154
+ ca_name = content_b64_match.group(1)
155
+ self.__check_trusted_root_ca_name_is_unique(seen_names, ca_name, env_name)
156
+ inline_cas[env_name] = env_value
157
+ continue
158
+ return file_cas, inline_cas
159
+
160
+ @GeneralUtilities.check_arguments
161
+ def __check_trusted_root_ca_name_is_unique(self, seen_names: dict[str, str], ca_name: str, env_name: str) -> None:
162
+ key = ca_name.lower()
163
+ if key in seen_names:
164
+ raise ValueError(f"Multiple TrustedRootCA-definitions for name \"{ca_name}\" found (\"{seen_names[key]}\" and \"{env_name}\"). The <name>-part becomes the filename in the trust-store and must therefore be unique across _Path and _ContentBase64.")
165
+ seen_names[key] = env_name
@@ -0,0 +1,91 @@
1
+ import re
2
+ from ...GeneralUtilities import GeneralUtilities
3
+ from ...ScriptCollectionCore import ScriptCollectionCore
4
+ from ...SCLog import LogLevel
5
+ from ..PackageSource import PackageSource
6
+ from ..TFCPS_Tools_General import TFCPS_Tools_General
7
+
8
+
9
+ class NuGetPackageSourceRegistration:
10
+ """Registers the NuGet-sources which are declared for the current machine (see TFCPS_Tools_General.get_declared_package_sources). It is
11
+ used by every DotNet-codeunit-build and by the command-line-tool 'scregisterpackagesources' (which the agent-sandbox of scrais runs before
12
+ the agent starts), so a source is registered on the same way everywhere."""
13
+
14
+ #the output of the dotnet-cli is localized. The language is set explicitly because the output of "dotnet nuget list source" gets parsed, which would otherwise only work on machines which are configured to use english.
15
+ __dotnet_cli_environment_variables: dict = {"DOTNET_CLI_UI_LANGUAGE": "en-US"}
16
+ __sc: ScriptCollectionCore = None
17
+ __tfcps_tools_general: TFCPS_Tools_General = None
18
+
19
+ def __init__(self, sc: ScriptCollectionCore, tfcps_tools_general: TFCPS_Tools_General):
20
+ self.__sc = sc
21
+ self.__tfcps_tools_general = tfcps_tools_general
22
+
23
+ @GeneralUtilities.check_arguments
24
+ def register_declared_package_sources(self, folder: str) -> None:
25
+ """Ensures that every NuGet-source which is declared for this machine is registered with its current credentials, so a codeunit can
26
+ also use dependencies which are not available on nuget.org. The dotnet-cli is run in the given folder.
27
+ A source which is already registered under the declared name is removed and registered again, so an outdated registration (for
28
+ example one which points to another url) can not let the registration fail. Sources which are not declared are not touched at
29
+ all: the registration is stored in the NuGet-configuration of the current user, which - when the build runs directly on the
30
+ host - is the configuration the user also uses for their own work and which must not be destroyed by a build."""
31
+ package_sources: list[PackageSource] = self.__tfcps_tools_general.get_declared_package_sources("CSharp")
32
+ if len(package_sources) == 0:
33
+ return
34
+ for package_source in package_sources:
35
+ registered_sources: dict[str, str] = self.__get_registered_nuget_sources(folder)
36
+ name_of_source_with_same_name: str = None
37
+ name_of_source_with_same_url: str = None
38
+ for registered_name, registered_url in registered_sources.items():
39
+ if registered_name.lower() == package_source.name.lower():
40
+ name_of_source_with_same_name = registered_name
41
+ elif self.__nuget_sources_are_equal(registered_url, package_source.url):
42
+ #the same feed can already be registered under another name (which is the usual case on a machine on which the user
43
+ #configured the feed manually). Such a source is updated instead of being added a second time, because a second
44
+ #registration of the same url would let every restore query the feed twice.
45
+ name_of_source_with_same_url = registered_name
46
+ if name_of_source_with_same_name is not None:
47
+ self.__remove_nuget_source(folder, name_of_source_with_same_name)
48
+ if name_of_source_with_same_url is None:
49
+ arguments = ["nuget", "add", "source", package_source.url, "--name", package_source.name]
50
+ else:
51
+ #the credentials are set again even when the source already exists, because a token can have been rotated since it was registered.
52
+ arguments = ["nuget", "update", "source", name_of_source_with_same_url, "--source", package_source.url]
53
+ if package_source.has_credentials():
54
+ arguments = arguments+["--username", package_source.username, "--password", package_source.password]
55
+ if not GeneralUtilities.current_system_is_windows():
56
+ #without this the password can not be encrypted, which lets the command fail on the operating-systems which do not support the encryption.
57
+ arguments.append("--store-password-in-clear-text")
58
+ arguments_for_log = list(arguments)
59
+ if package_source.has_credentials():
60
+ arguments_for_log[arguments_for_log.index("--password")+1] = "***"#the password must not be written to the build-log
61
+ self.__sc.log.log(f"Add NuGet-source \"{package_source.name}\" ({package_source.url})..." if name_of_source_with_same_url is None else f"Update the already registered NuGet-source \"{name_of_source_with_same_url}\" ({package_source.url})...", LogLevel.Debug)
62
+ self.__sc.run_program_argsasarray("dotnet", arguments, folder, arguments_for_log=arguments_for_log, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
63
+
64
+ @GeneralUtilities.check_arguments
65
+ def __remove_nuget_source(self, folder: str, name: str) -> None:
66
+ """Removes the registration of the NuGet-source with the given name, so it can be registered again with its currently declared url and credentials."""
67
+ self.__sc.log.log(f"Remove the outdated registration of the NuGet-source \"{name}\"...", LogLevel.Debug)
68
+ self.__sc.run_program_argsasarray("dotnet", ["nuget", "remove", "source", name], folder, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
69
+
70
+ @GeneralUtilities.check_arguments
71
+ def __get_registered_nuget_sources(self, folder: str) -> dict[str, str]:
72
+ """Returns the NuGet-sources which are registered for the given folder as a mapping of their name to their url. The sources are
73
+ read from the output of "dotnet nuget list source", which prints the name (with its state) and the url of a source in two
74
+ consecutive lines."""
75
+ result: dict[str, str] = {}
76
+ list_result = self.__sc.run_program_argsasarray("dotnet", ["nuget", "list", "source"], folder, throw_exception_if_exitcode_is_not_zero=False, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
77
+ name_pattern = re.compile(r"^\s*\d+\.\s+(.+?)\s+\[[^\[\]]+\]\s*$")
78
+ current_name: str = None
79
+ for line in list_result[1].splitlines():
80
+ match = name_pattern.match(line)
81
+ if match is not None:
82
+ current_name = match.group(1).strip()
83
+ elif current_name is not None and GeneralUtilities.string_has_content(line):
84
+ result[current_name] = line.strip()
85
+ current_name = None
86
+ return result
87
+
88
+ @GeneralUtilities.check_arguments
89
+ def __nuget_sources_are_equal(self, url: str, other_url: str) -> bool:
90
+ """Compares two urls of package-sources. A trailing slash and the case are ignored because they do not address a different feed."""
91
+ return url.strip().rstrip("/").lower() == other_url.strip().rstrip("/").lower()
@@ -11,15 +11,13 @@ from packaging.version import Version
11
11
  from .CertificateGeneratorInformationBase import CertificateGeneratorInformationBase
12
12
  from ...GeneralUtilities import GeneralUtilities, VersionEcholon
13
13
  from ...SCLog import LogLevel
14
- from ..PackageSource import PackageSource
14
+ from .NuGetPackageSourceRegistration import NuGetPackageSourceRegistration
15
15
  from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base,TFCPS_CodeUnitSpecific_Base_CLI
16
16
 
17
17
  class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
18
18
 
19
19
  is_library:bool = None
20
20
  csproj_file:bool = None
21
- #the output of the dotnet-cli is localized. The language is set explicitly for the commands whose output gets parsed, because otherwise the parsing would only work on machines which are configured to use english.
22
- __dotnet_cli_environment_variables:dict = {"DOTNET_CLI_UI_LANGUAGE": "en-US"}
23
21
  #the only accepted form of a version of a package-reference: an exact version in square brackets. Everything else is a version-range.
24
22
  __pinned_version_regex = re.compile(r"^\[[^,\[\]]+\]$")
25
23
 
@@ -29,78 +27,7 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
29
27
  self.is_library="<OutputType>Library</OutputType>" in GeneralUtilities.read_text_from_file(self.csproj_file)#TODO do a real check by checking this property using xpath
30
28
  #every dotnet-operation of a codeunit (build, test, linting, ...) restores the dependencies of the codeunit, so the sources
31
29
  #they are downloaded from have to be available for all of them and not only for the build.
32
- self.__ensure_declared_package_sources_are_available()
33
-
34
- @GeneralUtilities.check_arguments
35
- def __ensure_declared_package_sources_are_available(self) -> None:
36
- """Ensures that every NuGet-source which is declared for this machine (see TFCPS_Tools_General.get_declared_package_sources) is
37
- registered with its current credentials, so a codeunit can also use dependencies which are not available on nuget.org.
38
- A source which is already registered under the declared name is removed and registered again, so an outdated registration (for
39
- example one which points to another url) can not let the registration fail. Sources which are not declared are not touched at
40
- all: the registration is stored in the NuGet-configuration of the current user, which - when the build runs directly on the
41
- host - is the configuration the user also uses for their own work and which must not be destroyed by a build."""
42
- package_sources: list[PackageSource] = self.tfcps_Tools_General.get_declared_package_sources("CSharp")
43
- if len(package_sources) == 0:
44
- return
45
- codeunit_folder: str = self.get_codeunit_folder()
46
- for package_source in package_sources:
47
- registered_sources: dict[str, str] = self.__get_registered_nuget_sources(codeunit_folder)
48
- name_of_source_with_same_name: str = None
49
- name_of_source_with_same_url: str = None
50
- for registered_name, registered_url in registered_sources.items():
51
- if registered_name.lower() == package_source.name.lower():
52
- name_of_source_with_same_name = registered_name
53
- elif self.__nuget_sources_are_equal(registered_url, package_source.url):
54
- #the same feed can already be registered under another name (which is the usual case on a machine on which the user
55
- #configured the feed manually). Such a source is updated instead of being added a second time, because a second
56
- #registration of the same url would let every restore query the feed twice.
57
- name_of_source_with_same_url = registered_name
58
- if name_of_source_with_same_name is not None:
59
- self.__remove_nuget_source(codeunit_folder, name_of_source_with_same_name)
60
- if name_of_source_with_same_url is None:
61
- arguments = ["nuget", "add", "source", package_source.url, "--name", package_source.name]
62
- else:
63
- #the credentials are set again even when the source already exists, because a token can have been rotated since it was registered.
64
- arguments = ["nuget", "update", "source", name_of_source_with_same_url, "--source", package_source.url]
65
- if package_source.has_credentials():
66
- arguments = arguments+["--username", package_source.username, "--password", package_source.password]
67
- if not GeneralUtilities.current_system_is_windows():
68
- #without this the password can not be encrypted, which lets the command fail on the operating-systems which do not support the encryption.
69
- arguments.append("--store-password-in-clear-text")
70
- arguments_for_log = list(arguments)
71
- if package_source.has_credentials():
72
- arguments_for_log[arguments_for_log.index("--password")+1] = "***"#the password must not be written to the build-log
73
- self._protected_sc.log.log(f"Add NuGet-source \"{package_source.name}\" ({package_source.url})..." if name_of_source_with_same_url is None else f"Update the already registered NuGet-source \"{name_of_source_with_same_url}\" ({package_source.url})...", LogLevel.Debug)
74
- self._protected_sc.run_program_argsasarray("dotnet", arguments, codeunit_folder, arguments_for_log=arguments_for_log, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
75
-
76
- @GeneralUtilities.check_arguments
77
- def __remove_nuget_source(self, folder: str, name: str) -> None:
78
- """Removes the registration of the NuGet-source with the given name, so it can be registered again with its currently declared url and credentials."""
79
- self._protected_sc.log.log(f"Remove the outdated registration of the NuGet-source \"{name}\"...", LogLevel.Debug)
80
- self._protected_sc.run_program_argsasarray("dotnet", ["nuget", "remove", "source", name], folder, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
81
-
82
- @GeneralUtilities.check_arguments
83
- def __get_registered_nuget_sources(self, folder: str) -> dict[str, str]:
84
- """Returns the NuGet-sources which are registered for the given folder as a mapping of their name to their url. The sources are
85
- read from the output of "dotnet nuget list source", which prints the name (with its state) and the url of a source in two
86
- consecutive lines."""
87
- result: dict[str, str] = {}
88
- list_result = self._protected_sc.run_program_argsasarray("dotnet", ["nuget", "list", "source"], folder, throw_exception_if_exitcode_is_not_zero=False, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
89
- name_pattern = re.compile(r"^\s*\d+\.\s+(.+?)\s+\[[^\[\]]+\]\s*$")
90
- current_name: str = None
91
- for line in list_result[1].splitlines():
92
- match = name_pattern.match(line)
93
- if match is not None:
94
- current_name = match.group(1).strip()
95
- elif current_name is not None and GeneralUtilities.string_has_content(line):
96
- result[current_name] = line.strip()
97
- current_name = None
98
- return result
99
-
100
- @GeneralUtilities.check_arguments
101
- def __nuget_sources_are_equal(self, url: str, other_url: str) -> bool:
102
- """Compares two urls of package-sources. A trailing slash and the case are ignored because they do not address a different feed."""
103
- return url.strip().rstrip("/").lower() == other_url.strip().rstrip("/").lower()
30
+ NuGetPackageSourceRegistration(self._protected_sc, self.tfcps_Tools_General).register_declared_package_sources(self.get_codeunit_folder())
104
31
 
105
32
  @GeneralUtilities.check_arguments
106
33
  def get_runtimes(self) -> list[str]:
@@ -15,6 +15,7 @@ from .TFCPS_CodeSpellCheck import TFCPS_CodeSpellCheck
15
15
  from .TFCPS_FindingsSorter import TFCPS_FindingsSorter
16
16
  from .TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
17
17
  from .TFCPS_Tools_General import TFCPS_Tools_General, CommonProjectStructureVersion
18
+ from .ContainerBuildEnvironment import ContainerBuildEnvironment
18
19
 
19
20
 
20
21
  class TFCPS_UpdateDependenciesHook(TFCPS_BuildCodeUnitsHook):
@@ -139,9 +140,6 @@ class TFCPS_CodeUnit_BuildCodeUnits:
139
140
  #ensure <repo>/.gitignore is set up (ignores the cache-folder so cache-files never show up as uncommitted changes)
140
141
  self.__ensure_global_gitignore_is_setup(self.repository)
141
142
 
142
- #validate the openspec-specifications if openspec is set up in the repository
143
- self.__validate_openspec()
144
-
145
143
  self.sc.log.log(f"Start building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(start_time,False)}. (Target environment-type: {self.target_environment_type})")
146
144
 
147
145
  #ensure common cache-files are available
@@ -200,21 +198,23 @@ class TFCPS_CodeUnit_BuildCodeUnits:
200
198
  hook.run_after_codeunit_was_built(tFCPS_CodeUnit_BuildCodeUnit)
201
199
  self.sc.log.log(GeneralUtilities.get_line())
202
200
 
203
- self.tfcps_tools_general.generate_svg_files_from_plantuml_files_for_repository(self.repository, self.use_cache())
204
- self.tfcps_tools_general.generate_svg_files_from_vega_files_for_repository(self.repository, self.use_cache())
205
- self.tfcps_tools_general.generate_svg_files_from_vegalite_files_for_repository(self.repository, self.use_cache())
206
- self.__do_project_linting(self.repository)
207
- self.__update_agent_sandbox_image(self.repository)
201
+ self.sc.log.log("Do project post-build-tasks...")
202
+
203
+ if not self.__fast_lane:
204
+ self.__do_project_linting(self.repository)
205
+ self.__update_agent_sandbox_image(self.repository)
206
+ self.search_for_secrets()
208
207
  self.__normalize_line_endings_of_common_files()
209
- self.search_for_secrets()
210
-
211
- self.sc.log.log(GeneralUtilities.get_line())
212
208
 
213
209
  if self.is_pre_merge():
214
210
  self.__collect_metrics()
215
211
  self.__generate_loc_diagram()
216
- self.sc.log.log(GeneralUtilities.get_line())
217
212
 
213
+ self.tfcps_tools_general.generate_svg_files_from_plantuml_files_for_repository(self.repository, self.use_cache())
214
+ self.tfcps_tools_general.generate_svg_files_from_vega_files_for_repository(self.repository, self.use_cache())
215
+ self.tfcps_tools_general.generate_svg_files_from_vegalite_files_for_repository(self.repository, self.use_cache())
216
+
217
+ self.sc.log.log(GeneralUtilities.get_line())
218
218
  except Exception:
219
219
  error_occurred=True
220
220
  raise
@@ -286,6 +286,7 @@ class TFCPS_CodeUnit_BuildCodeUnits:
286
286
  self.sc.log.log("Do project-linting...")
287
287
  self.__do_project_linting_using_codespell(repository_folder)
288
288
  self.__sort_findings(repository_folder)
289
+ self.__validate_openspec(repository_folder)
289
290
 
290
291
  @GeneralUtilities.check_arguments
291
292
  def __sort_findings(self, repository_folder: str) -> None:
@@ -436,11 +437,14 @@ class TFCPS_CodeUnit_BuildCodeUnits:
436
437
  self.sc.run_with_epew("openspec", "update --force", self.repository)
437
438
 
438
439
  @GeneralUtilities.check_arguments
439
- def __validate_openspec(self) -> None:
440
- openspec_configuration_file: str = os.path.join(self.repository, "openspec", "config.yaml")
440
+ def __validate_openspec(self,repository:str) -> None:
441
+ openspec_configuration_file: str = os.path.join(repository, "openspec", "config.yaml")
441
442
  if os.path.isfile(openspec_configuration_file):
442
443
  self.sc.log.log("Validate openspec-specifications...")
443
- self.sc.run_with_epew("openspec", "validate --all --strict", self.repository)
444
+ try:
445
+ self.sc.run_with_epew("openspec", "validate --all --strict", repository)
446
+ except Exception as e:
447
+ self.sc.log.log_exception(f"Openspec validation failed.",e,LogLevel.Warning)
444
448
 
445
449
  @GeneralUtilities.check_arguments
446
450
  def get_default_base_mount_folder(self) -> str:
@@ -565,61 +569,19 @@ class TFCPS_CodeUnit_BuildCodeUnits:
565
569
  update_argument="pip3 install scriptcollection --upgrade && "
566
570
  executable_arguments=["bash","-c", f"{update_argument}scshowversion && "+" ".join(executable_arguments)]
567
571
 
568
- #run the optional user-specific script which prepares this host for a container-build (for example to log in to the registry the image is pulled from).
569
- #it runs before the environment-variables are resolved, so it can also create the files their values are read from.
570
- self.tfcps_tools_general.run_custom_script_if_available(self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScriptForContainer.py"), self.__get_build_arguments(self.repository))
571
-
572
- #mount the optional user-specific scripts which prepare the container itself. the whole folder (not only the pre-codeunit-build-hook
573
- #itself) is mounted, so the hook can start a sibling script placed next to it (for example to keep the hook independent of a
574
- #package it only installs, while the actual preparation-logic which needs that package lives in the sibling script; see
575
- #get_custom_scripts_folder_for_container). The mount is writable (not read-only) because the scripts in that folder also use it as
576
- #their own download-cache-folder. The build inside the container runs the hook (see __run_custom_pre_codeunit_build_script); if the
577
- #folder does not exist on the host nothing is mounted and nothing is run.
572
+ #the build-environment (host-hook, mounts and environment-variables) is defined by ContainerBuildEnvironment, which is also used by
573
+ #the agent-sandbox of scrais, so that both containers get the same one. The hook runs before the environment-variables are resolved,
574
+ #so it can also create the files their values are read from.
575
+ container_build_environment: ContainerBuildEnvironment = ContainerBuildEnvironment(self.sc, self.tfcps_tools_general, self.repository)
576
+ container_build_environment.run_host_preparation_hook(self.__get_build_arguments(self.repository))
578
577
  mount_arguments: list[str] = []
579
- custom_scripts_folder_for_inside_the_container: str = self.tfcps_tools_general.get_custom_scripts_folder_for_container()
580
- if os.path.isdir(custom_scripts_folder_for_inside_the_container):
581
- mount_arguments += ["-v", f"{custom_scripts_folder_for_inside_the_container}:{self.tfcps_tools_general.get_folder_of_custom_scripts_in_container()}"]
582
-
583
- #mount the machine-wide image-registries-file of this host (if it exists), so the build inside the container takes the images from the same
584
- #custom registries as a build on the host. Without the mount the build in the container would find no mapping at all - the
585
- #configuration-folder inside the container belongs to the container-user and is empty - and would therefore fall back to the
586
- #upstream-registries defined in the repository, which are rate-limited. The mount is read-only because a build never changes the file.
587
- oci_image_manager = self.tfcps_tools_general.oci_image_manager
588
- image_registries_file: str = oci_image_manager.get_image_registries_file_in_configuration_folder()
589
- if os.path.isfile(image_registries_file):
590
- mount_arguments += ["-v", f"{image_registries_file}:{oci_image_manager.get_image_registries_file_in_container()}:ro"]
591
-
592
- #mount the 'TFCPS'-folder of the configuration-folder of this host (if it exists), so the build inside the container resolves the
593
- #required environment-variables of the product (see TFCPS_Tools_General.get_required_environment_variables) and the
594
- #OCI-registry-credentials (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) exactly like a
595
- #build on the host does - including a registry which is not publicly readable, which without this mount could not be used inside
596
- #the container at all, because the configuration-folder of the container-user does not contain the credentials of this host.
597
- #The whole folder is mounted and not only EnvironmentVariables.csv in it, because a value of that file can point to a secret-file
598
- #next to it ('file'-kind with a relative path, which is the recommended form) and such a value is resolved relative to the file
599
- #itself - so with only the file mounted, every entry of that kind would fail inside the container. Only this folder (and not the
600
- #whole configuration-folder) is mounted, so nothing else of it is exposed to the container. The mount is read-only because a build
601
- #never changes it. The alternative way to provide a value is an environment-variable of the build-pipeline itself (the pipeline's
602
- #own secret-store); both sources are used together, the file taking precedence.
603
- tfcps_configuration_folder: str = self.sc.get_tfcps_configuration_folder()
604
- if os.path.isdir(tfcps_configuration_folder):
605
- mount_arguments += ["-v", f"{tfcps_configuration_folder}:{self.sc.get_tfcps_configuration_folder_in_container()}:ro"]
606
-
607
- #install the host-declared trusted root-CA-certificates into the container (needed when the build pulls packages from a privately
608
- #hosted registry whose certificate-chain is not publicly rooted - for example an internally hosted gitlab-nuget-source). The
609
- #two env-var forms are detected automatically from both the machine-wide environment-variables-configuration-file and the
610
- #environment of the current process; see __collect_trusted_root_ca_arguments. _Path-variables add file-mounts, the inline
611
- #_ContentBase64-form is returned as a name->value dict that gets merged into required_environment_variables below so its
612
- #value flows through the docker-client's own environment (not through argv) just like a required secret does.
613
- trusted_root_ca_mount_arguments, trusted_root_ca_inline_env_vars = self.__collect_trusted_root_ca_arguments()
614
- mount_arguments += trusted_root_ca_mount_arguments
615
-
616
- #pass the environment-variables which are declared as required in <repository>/.ScriptCollection/ProductInformation.xml into the container
617
- #so they do not have to be specified explicitly on every call. Their values are resolved from the user-specific
618
- #configuration-file (see TFCPS_Tools_General.get_environment_variables_configuration_file), which only exists on the host.
619
- #only the names are passed as arguments; the values are given to the docker-client through its own environment, because arguments
620
- #are written to the log and are visible in the process-list of this host, which must not happen for a value which is typically a secret.
621
- required_environment_variables: dict[str, str] = self.tfcps_tools_general.get_required_environment_variables(self.repository)
622
- required_environment_variables.update(trusted_root_ca_inline_env_vars)
578
+ for mount in container_build_environment.get_mounts():
579
+ mount_arguments += ["-v", f"{mount.source}:{mount.target}{':ro' if mount.read_only else ''}"]
580
+
581
+ #only the names of the environment-variables are passed as arguments; the values are given to the docker-client through its own
582
+ #environment, because arguments are written to the log and are visible in the process-list of this host, which must not happen for
583
+ #a value which is typically a secret.
584
+ required_environment_variables: dict[str, str] = container_build_environment.get_environment_variables()
623
585
  env_arguments: list[str] = []
624
586
  for env_variable_name in required_environment_variables:
625
587
  env_arguments += ["-e", env_variable_name]
@@ -652,78 +614,6 @@ class TFCPS_CodeUnit_BuildCodeUnits:
652
614
  stderr:str=result[2] or GeneralUtilities.empty_string
653
615
  return (exit_code==0,f"{stdout}\n{stderr}")
654
616
 
655
- @GeneralUtilities.check_arguments
656
- def __collect_trusted_root_ca_arguments(self) -> tuple[list[str], dict[str, str]]:
657
- """Resolves the TrustedRootCA_<name>_Path and TrustedRootCA_<name>_ContentBase64 environment-variables of the current host into
658
- docker-arguments which install those certificates into the trust-store of the build-container. Returns (mount_arguments, inline_env_vars):
659
- - mount_arguments: '-v <host-path>:/Workspace/RootCAs/<name>.crt:ro' entries for each _Path-variable.
660
- - inline_env_vars: a name->value dict for each _ContentBase64-variable; the caller merges it into required_environment_variables
661
- so the values flow through the docker-client's own environment, not through argv.
662
- Candidate-names are collected from two sources (identical to the OCI-registry-credentials, see
663
- ScriptCollectionCore.__get_defined_oci_registry_names): the machine-wide environment-variables-configuration-file and the
664
- environment of the current process. The value is then resolved through ScriptCollectionCore.resolve_environment_variable, so a
665
- CSV-entry takes precedence over an environment-variable of the same name - consistent with every other environment-variable
666
- this script-collection resolves. If a name is declared in the file but its value can not be determined there (for example a
667
- 'file'-entry pointing to a missing file), the resolver falls back to the environment and fails only if neither source yields a
668
- value. The actual trust-store update happens in SCBuilder's EntryPoint.sh, which scans /Workspace/RootCAs and the
669
- TrustedRootCA_*-environment-variables and runs update-ca-certificates once before any build-step. The <name>-part becomes the
670
- filename inside /usr/local/share/ca-certificates/ and must therefore be unique across the two env-var-forms; a duplicate is
671
- rejected here with a clear error, before the container is started."""
672
- path_pattern = re.compile(r"^TrustedRootCA_(.+)_Path$", re.IGNORECASE)
673
- content_b64_pattern = re.compile(r"^TrustedRootCA_(.+)_ContentBase64$", re.IGNORECASE)
674
- #collect every candidate-variable-name from both sources before any value is resolved; this way a secret-value is read only for
675
- #a name that is actually declared and the discovery is identical to the OCI-registry-credential-discovery. The configuration-
676
- #file is scanned before the environment and a name found in both is kept with the casing of the file: on Windows, os.environ
677
- #normalizes a variable-name to uppercase as soon as it is set, so without this preference the same declaration in both places
678
- #would be treated as two different names and resolution would silently pick the environment-value instead of the file-value.
679
- candidate_names: dict[str, str] = {}#lower-case variable-name -> canonical variable-name (file-casing preferred)
680
- configuration_file = self.sc.get_environment_variables_configuration_file()
681
- if os.path.isfile(configuration_file):
682
- for entry in GeneralUtilities.read_csv_file(configuration_file, True):
683
- if 0 == len(entry):
684
- continue
685
- env_name = entry[0]
686
- if path_pattern.match(env_name) is not None or content_b64_pattern.match(env_name) is not None:
687
- candidate_names.setdefault(env_name.lower(), env_name)
688
- for env_name in os.environ.keys():
689
- if path_pattern.match(env_name) is not None or content_b64_pattern.match(env_name) is not None:
690
- candidate_names.setdefault(env_name.lower(), env_name)
691
- seen_names: dict[str, str] = {}#lower-case <name>-part -> originating variable-name, used only to produce an informative error on duplicate <name>.
692
- mount_arguments: list[str] = []
693
- inline_env_vars: dict[str, str] = {}
694
- #sorted so the log-output (and therefore the order of "-v"-/"-e"-arguments) stays deterministic across runs.
695
- for env_name in sorted(candidate_names.values(), key=str.lower):
696
- env_value = self.sc.resolve_environment_variable(env_name, f"the TrustedRootCA-mechanism (declared via \"{env_name}\")")
697
- if not GeneralUtilities.string_has_content(env_value):
698
- continue
699
- path_match = path_pattern.match(env_name)
700
- if path_match is not None:
701
- ca_name = path_match.group(1)
702
- self.__check_trusted_root_ca_name_is_unique(seen_names, ca_name, env_name)
703
- host_path = env_value.strip()
704
- if not os.path.isfile(host_path):
705
- raise ValueError(f"Environment-variable \"{env_name}\" points to a file which does not exist on this host: \"{host_path}\".")
706
- with open(host_path, "r", encoding="utf-8", errors="replace") as pem_file:
707
- head = pem_file.read(64)
708
- if "-----BEGIN CERTIFICATE-----" not in head:
709
- raise ValueError(f"Environment-variable \"{env_name}\" (\"{host_path}\") does not look like a PEM-encoded certificate (missing \"-----BEGIN CERTIFICATE-----\"-header).")
710
- mount_arguments += ["-v", f"{host_path}:/Workspace/RootCAs/{ca_name}.crt:ro"]
711
- continue
712
- content_b64_match = content_b64_pattern.match(env_name)
713
- if content_b64_match is not None:
714
- ca_name = content_b64_match.group(1)
715
- self.__check_trusted_root_ca_name_is_unique(seen_names, ca_name, env_name)
716
- inline_env_vars[env_name] = env_value
717
- continue
718
- return mount_arguments, inline_env_vars
719
-
720
- @GeneralUtilities.check_arguments
721
- def __check_trusted_root_ca_name_is_unique(self, seen_names: dict[str, str], ca_name: str, env_name: str) -> None:
722
- key = ca_name.lower()
723
- if key in seen_names:
724
- raise ValueError(f"Multiple TrustedRootCA-definitions for name \"{ca_name}\" found (\"{seen_names[key]}\" and \"{env_name}\"). The <name>-part becomes the filename in the trust-store and must therefore be unique across _Path and _ContentBase64.")
725
- seen_names[key] = env_name
726
-
727
617
  @GeneralUtilities.check_arguments
728
618
  def __translate_path_into_container(self, host_path: str, container_repository_folder: str) -> str:
729
619
  normalized_repository = os.path.normpath(self.repository)
@@ -1958,7 +1958,7 @@ class TFCPS_Tools_General:
1958
1958
  arch_tags.append(arch_tag)
1959
1959
  # Load tar → local image
1960
1960
  GeneralUtilities.write_message_to_stdout(f"Loading {tar_path}...")
1961
- result = self.__sc.run_program_argsasarray("docker",[ "load", "-i", tar_path])
1961
+ result = self.__sc.run_program_argsasarray_with_retry("docker",[ "load", "-i", tar_path],amount_of_attempts=3,delay_in_seconds=5)
1962
1962
  # docker load outputs: "Loaded image: sha256:abc123..." or "Loaded image ID: ..."
1963
1963
  # we need the loaded image ID
1964
1964
  loaded_id = None
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptcollection
3
- Version: 4.4.66
3
+ Version: 4.4.68
4
4
  Summary: The ScriptCollection is the place for reusable scripts.
5
5
  Author-email: Marius Göcke <marius.goecke@gmail.com>
6
6
  License-Expression: GPL-3.0-only