ScriptCollection 4.4.67__py3-none-any.whl → 4.4.68__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,7 +6,10 @@ import tempfile
6
6
  from enum import Enum
7
7
  from .GeneralUtilities import GeneralUtilities
8
8
  from .ScriptCollectionCore import ScriptCollectionCore
9
+ from .SCLog import LogLevel
9
10
  from .OCIImages.OCIImageManager import OCIImageManager
11
+ from .TFCPS.ContainerBuildEnvironment import ContainerBuildEnvironment
12
+ from .TFCPS.TFCPS_Tools_General import TFCPS_Tools_General
10
13
  from .AgentStatusReporting.AgentStatusReportingSession import AgentStatusReportingSession
11
14
 
12
15
 
@@ -22,12 +25,26 @@ class AgentRunnerConfiguration:
22
25
  codingagent: CodingAgent = None
23
26
  image: str = None
24
27
  drop_capabilities: bool = None
28
+ provide_build_environment: bool = None
25
29
 
26
30
  @GeneralUtilities.check_arguments
27
31
  def __init__(self):
28
32
  self.codingagent = None
29
33
  self.image = None
30
34
  self.drop_capabilities = None
35
+ self.provide_build_environment = None
36
+
37
+
38
+ class SandboxBuildEnvironment:
39
+ """The build-environment (see ContainerBuildEnvironment) prepared for one start of the sandbox: the docker-arguments of its mounts
40
+ and its environment-variables. The values of the environment-variables are typically secrets, so only their names may become
41
+ docker-arguments; the values are given to the docker-client through its own environment."""
42
+ volume_arguments: list[str] = None
43
+ environment_variables: dict[str, str] = None
44
+
45
+ def __init__(self, volume_arguments: list[str], environment_variables: dict[str, str]):
46
+ self.volume_arguments = volume_arguments
47
+ self.environment_variables = environment_variables
31
48
 
32
49
 
33
50
  class AgentRunner:
@@ -53,12 +70,35 @@ class AgentRunner:
53
70
  if mount_docker_socket:
54
71
  self.__print_docker_socket_warning()
55
72
  additional_folder_volume_arguments = additional_folder_volume_arguments + ["--volume", "/var/run/docker.sock:/var/run/docker.sock"]
56
- return self.__run_sandbox(repository, agentRunnerConfiguration, prompt, interactive, initialization_script, additional_folder_volume_arguments)
73
+ build_environment: SandboxBuildEnvironment = None
74
+ if agentRunnerConfiguration.provide_build_environment:
75
+ build_environment = self.__prepare_build_environment(repository)
76
+ return self.__run_sandbox(repository, agentRunnerConfiguration, prompt, interactive, initialization_script, additional_folder_volume_arguments, build_environment)
57
77
 
58
78
  @GeneralUtilities.check_arguments
59
79
  def __print_docker_socket_warning(self) -> None:
60
80
  GeneralUtilities.write_message_to_stderr("Warning: Mounting the docker-socket of the host into the sandbox is dangerous because it makes the sandbox ineffective.")
61
81
 
82
+ @GeneralUtilities.check_arguments
83
+ def __prepare_build_environment(self, repository: str) -> SandboxBuildEnvironment:
84
+ # The sandbox gets the same build-environment as a build-container of "scbuildcodeunits -c" (see ContainerBuildEnvironment), so the agent can
85
+ # build the repository and restore its dependencies (also from private package-sources). The host-hook runs first, like for "scbuildcodeunits -c",
86
+ # because it can create the files the values of the environment-variables are read from. A sandbox has no target-environment of its own;
87
+ # "Development" is passed because the hook expects a valid one and an agent develops the repository.
88
+ sc = ScriptCollectionCore()
89
+ container_build_environment = ContainerBuildEnvironment(sc, TFCPS_Tools_General(sc), repository)
90
+ self.__print_build_environment_warning(sc.get_tfcps_configuration_folder())
91
+ verbosity = LogLevel.Debug if self.verbose else LogLevel.Information
92
+ container_build_environment.run_host_preparation_hook(["--repository", repository, "--targetenvironmenttype", "Development", "--verbosity", str(int(verbosity))])
93
+ volume_arguments: list[str] = []
94
+ for mount in container_build_environment.get_mounts():
95
+ volume_arguments = volume_arguments + ["--volume", f"{mount.source.replace(os.sep, '/')}:{mount.target}{':ro' if mount.read_only else ''}"]
96
+ return SandboxBuildEnvironment(volume_arguments, container_build_environment.get_environment_variables())
97
+
98
+ @GeneralUtilities.check_arguments
99
+ def __print_build_environment_warning(self, tfcps_configuration_folder: str) -> None:
100
+ GeneralUtilities.write_message_to_stderr(f"Warning: The build-environment is provided to the sandbox. The agent can read every secret of \"{tfcps_configuration_folder}\" and the values of the required environment-variables of the repository.")
101
+
62
102
  @GeneralUtilities.check_arguments
63
103
  def __get_additional_folder_volume_arguments(self, additional_folders: list[str]) -> list[str]:
64
104
  # Every additional folder is mounted under "/Workspace/Other/AdditionalResources/<folder-name>" (preserving the folder-name).
@@ -127,10 +167,20 @@ class AgentRunner:
127
167
 
128
168
  for name, value in [("codingagent", agentRunnerConfiguration.codingagent), ("image", agentRunnerConfiguration.image), ("drop_capabilities", agentRunnerConfiguration.drop_capabilities)]:
129
169
  GeneralUtilities.assert_condition(value is not None, f"The value '{name}' is neither given as argument nor defined in '{agent_sandbox_configuration_file}'.")
170
+
171
+ # Unlike the values above, provide_build_environment is not required: the build-environment is opt-in, so it is only provided if it is enabled
172
+ # explicitly. Requiring it would break every existing configuration-file.
173
+ if configuration_overrides.provide_build_environment is not None:
174
+ agentRunnerConfiguration.provide_build_environment = configuration_overrides.provide_build_environment
175
+ elif config_parser.has_option(section, "provide_build_environment"):
176
+ agentRunnerConfiguration.provide_build_environment = config_parser.getboolean(section, "provide_build_environment")
177
+ else:
178
+ agentRunnerConfiguration.provide_build_environment = False
130
179
  return agentRunnerConfiguration
131
180
 
132
181
  @GeneralUtilities.check_arguments
133
- def __run_sandbox(self, repository: str, agentRunnerConfiguration: AgentRunnerConfiguration, prompt: str, interactive: bool, initialization_script: str, additional_folder_volume_arguments: list[str]) -> int:
182
+ def __run_sandbox(self, repository: str, agentRunnerConfiguration: AgentRunnerConfiguration, prompt: str, interactive: bool, initialization_script: str, additional_folder_volume_arguments: list[str], build_environment: SandboxBuildEnvironment) -> int:
183
+ # build_environment is None if the build-environment is not provided.
134
184
  # The container always runs Linux.
135
185
  # The repository is the folder given by the caller (which was asserted to be the git-root-directory before).
136
186
  # It is mounted into the container under "/Workspace/<repository-name>" (preserving the folder-name) and used as working-directory of the agent.
@@ -142,7 +192,7 @@ class AgentRunner:
142
192
  status_session = AgentStatusReportingSession.try_create(dict(os.environ), f"scrais-{agentRunnerConfiguration.codingagent.value}", self.verbose)
143
193
  status_start_line = None if status_session is None else status_session.get_start_report_line(prompt is not None)
144
194
  # The entrypoint is always used: if an initialization-script is present it runs it first, and then always starts the agent.
145
- entrypoint_script_content = self.__get_entrypoint_script_content(agent_command, initialization_script, status_start_line)
195
+ entrypoint_script_content = self.__get_entrypoint_script_content(agent_command, initialization_script, status_start_line, build_environment is not None, container_workspace_folder)
146
196
  with tempfile.NamedTemporaryFile("w", suffix=".sh", delete=False, newline="\n", encoding="utf-8") as entrypoint_file_handle:
147
197
  host_entrypoint_file = entrypoint_file_handle.name
148
198
  entrypoint_file_handle.write(entrypoint_script_content)
@@ -166,14 +216,30 @@ class AgentRunner:
166
216
  docker_command = docker_command + ["--volume", f"{initialization_script_absolute_path.replace(os.sep, '/')}:/Workspace/Temp/InitializationScript.sh:ro"]
167
217
  if status_session is not None:
168
218
  docker_command = docker_command + status_session.get_volume_arguments()
219
+ # The environment of the docker-client; it differs from the one of this process only if the build-environment is provided.
220
+ docker_client_environment: dict[str, str] = None
221
+ if build_environment is not None:
222
+ docker_command = docker_command + build_environment.volume_arguments
223
+ # Only the names are passed as arguments ("--env <name>" takes the value from the environment of the docker-client), because
224
+ # arguments are visible in the process-list of the host and the values are typically secrets.
225
+ for environment_variable_name in build_environment.environment_variables:
226
+ docker_command = docker_command + ["--env", environment_variable_name]
227
+ docker_client_environment = {**os.environ, **build_environment.environment_variables}
169
228
  image = self.__get_image_to_use(agentRunnerConfiguration.image, host_repository_folder)
170
- docker_command = docker_command + ["--workdir", container_workspace_folder, "--entrypoint", "sh", image, "/Workspace/Entrypoint.sh"]
229
+ docker_command = docker_command + ["--workdir", container_workspace_folder]
230
+ if build_environment is None:
231
+ docker_command = docker_command + ["--entrypoint", "sh", image, "/Workspace/Entrypoint.sh"]
232
+ else:
233
+ # With the build-environment the entrypoint of the image is kept and runs the generated entrypoint as its command: the entrypoint of the
234
+ # SCBuilder-image installs the trusted root-CA-certificates of the build-environment (and then runs "exec \"$@\""), exactly like in a
235
+ # build-container of "scbuildcodeunits -c". Without the build-environment the entrypoint of the image is replaced as before.
236
+ docker_command = docker_command + [image, "sh", "/Workspace/Entrypoint.sh"]
171
237
  # Nothing is installed in the container; the coding-agent-CLI is assumed to be available in the image.
172
238
  # subprocess.run blocks until the container has exited, which happens when the agent is finished (or was terminated by the user in interactive mode).
173
239
  # The exit-code of the container is returned (and not checked here), so that the caller can pass it through.
174
240
  if status_session is not None:
175
241
  status_session.start()
176
- return subprocess.run(docker_command, check=False).returncode
242
+ return subprocess.run(docker_command, check=False, env=docker_client_environment).returncode
177
243
  finally:
178
244
  if status_session is not None:
179
245
  status_session.stop()
@@ -193,7 +259,7 @@ class AgentRunner:
193
259
  return configured_image
194
260
 
195
261
  @GeneralUtilities.check_arguments
196
- def __get_entrypoint_script_content(self, agent_command: list[str], initialization_script: str, status_start_line: str) -> str:
262
+ def __get_entrypoint_script_content(self, agent_command: list[str], initialization_script: str, status_start_line: str, provide_build_environment: bool, container_workspace_folder: str) -> str:
197
263
  # The entrypoint-script is always used as the container's entrypoint (located at "/Workspace/Entrypoint.sh").
198
264
  # It is regenerated before every container-start, so the decision whether the initialization-script is called is made here in python
199
265
  # (no runtime-check in the script): if an initialization-script is present its call is included (and the script is removed afterwards),
@@ -201,6 +267,12 @@ class AgentRunner:
201
267
  # status_start_line is None if the status is not reported; it is the only runtime-check, because only the container knows whether its image supports the status-report.
202
268
  agent_command_line = shlex.join(agent_command)
203
269
  lines = ["#!/bin/sh", "set -e"]
270
+ if provide_build_environment:
271
+ # The declared package-sources are registered before the initialization-script, so that script and the agent can already use them directly
272
+ # (for example with "dotnet restore") and not only through "scbuildcodeunits". The command comes from the ScriptCollection of the image; an
273
+ # image without it is rejected instead of starting the agent without the package-sources.
274
+ lines.append('if ! command -v scregisterpackagesources >/dev/null 2>&1; then echo "The build-environment requires the command \\"scregisterpackagesources\\" of ScriptCollection, which is not available in this image. Use a newer image." >&2; exit 1; fi')
275
+ lines.append(f"scregisterpackagesources -r {shlex.quote(container_workspace_folder)}")
204
276
  if initialization_script is not None:
205
277
  # A line is written to stdout before the initialization-script runs, so it is unambiguous in the container-output that an initialization-script was configured and executed (without it one cannot tell whether the agent started unconfigured because no script was declared or because a declared script did nothing).
206
278
  lines.append('echo "Running initialization-script for the agent-sandbox..."')
@@ -14,6 +14,7 @@ from .SCLog import LogLevel, SCLog
14
14
  from .TFCPS.TFCPS_CodeUnit_BuildCodeUnits import TFCPS_CodeUnit_BuildCodeUnits
15
15
  from .TFCPS.TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
16
16
  from .TFCPS.TFCPS_Tools_General import TFCPS_Tools_General
17
+ from .TFCPS.DotNet.NuGetPackageSourceRegistration import NuGetPackageSourceRegistration
17
18
  from .OCIImages.OCIImageManager import OCIImageManager
18
19
  from .AgentRunner import AgentRunner, AgentRunnerConfiguration, CodingAgent
19
20
 
@@ -28,6 +29,7 @@ def RunAgentInSandbox() -> int:
28
29
  parser.add_argument('--image', default=None, required=False, help='OCI-image which should be used for the sandbox: either an image-name defined in ".ScriptCollection/OCIImages/ImageDefinition.csv" or an image-address including tag. Overrides the value from ".ScriptCollection/AgentSandboxConfiguration.ini".')
29
30
  parser.add_argument('-s','--dropcapabilities', default=None, required=False, choices=["true", "false"], help='Defines whether all capabilities should be dropped in the sandbox. Overrides the value from ".ScriptCollection/AgentSandboxConfiguration.ini".')
30
31
  parser.add_argument('-a', '--additionalfolder', action='append', default=None, required=False, help='Folder which should additionally be mounted (read-write) into the sandbox under "/Workspace/Other/AdditionalResources/<folder-name>". Can be given multiple times.')
32
+ parser.add_argument('-b','--providebuildenvironment', default=None, required=False, choices=["true", "false"], help='Defines whether the sandbox gets the same build-environment as a build-container of "scbuildcodeunits -c" (required environment-variables, configuration-folder, custom scripts, image-registries, trusted root-CAs and registered package-sources). The agent can then read every secret of the TFCPS-configuration-folder. Overrides the value from ".ScriptCollection/AgentSandboxConfiguration.ini".')
31
33
  parser.add_argument('-d','--mountdockersocket', action='store_true', required=False, default=False, help='Mounts the docker-socket of the host (/var/run/docker.sock) into the sandbox so that the agent can use docker. DANGEROUS: this effectively gives the agent administrator-access to the docker-daemon of the host and makes the restrictions of the sandbox ineffective. Can only be set via this argument (not in the configuration-file); a warning is printed to stderr.')
32
34
  args = parser.parse_args()
33
35
  repo=os.getcwd()
@@ -40,6 +42,8 @@ def RunAgentInSandbox() -> int:
40
42
  configuration_overrides.image = args.image
41
43
  if args.dropcapabilities is not None:
42
44
  configuration_overrides.drop_capabilities = args.dropcapabilities == "true"
45
+ if args.providebuildenvironment is not None:
46
+ configuration_overrides.provide_build_environment = args.providebuildenvironment == "true"
43
47
  additional_folders: list[str] = []
44
48
  if args.additionalfolder is not None:
45
49
  additional_folders = [GeneralUtilities.resolve_relative_path(additional_folder, os.getcwd()) for additional_folder in args.additionalfolder]
@@ -1352,6 +1356,16 @@ def ShowProjectVersion() -> int:
1352
1356
  return 0
1353
1357
 
1354
1358
 
1359
+ def RegisterPackageSources() -> int:
1360
+ parser = argparse.ArgumentParser(description="Registers every package-source which is declared in the environment of this process by the environment-variables \"Dependency_<technology>_<name>_URL\" (and optionally \"_Username\" and \"_Password\") with its credentials on this machine, exactly like a codeunit-build does it. Package-sources which are not declared are not changed.")
1361
+ parser.add_argument('-r', '--folder', required=False, default=None, help="Folder in which the tools of the technologies (for example the dotnet-cli) are run. Defaults to the current working directory.")
1362
+ args = parser.parse_args()
1363
+ folder = GeneralUtilities.resolve_relative_path(args.folder, os.getcwd()) if args.folder is not None else os.getcwd()
1364
+ sc: ScriptCollectionCore = ScriptCollectionCore()
1365
+ NuGetPackageSourceRegistration(sc, TFCPS_Tools_General(sc)).register_declared_package_sources(folder)
1366
+ return 0
1367
+
1368
+
1355
1369
  def RunCommandInFolder() -> int:
1356
1370
  parser = argparse.ArgumentParser(description="Runs a command in the base-folder if (and only if) the actual folder is located inside the base-folder and not inside an excluded folder. The command is always executed in the base-folder; the actual folder is only passed to the command by using the placeholder \"{actual_folder}\" in the arguments (for example: -c git -a \"-C {actual_folder} status\").")
1357
1371
  parser.add_argument('-b', '--basefolder', required=True, help="Folder in which the command is executed.")
@@ -40,7 +40,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
40
40
  from .ProgramRunnerPopen import ProgramRunnerPopen
41
41
  from .SCLog import SCLog, LogLevel
42
42
 
43
- version = "4.4.67"
43
+ version = "4.4.68"
44
44
  __version__ = version
45
45
 
46
46
  class VSCodeWorkspaceShellTask:
@@ -0,0 +1,165 @@
1
+ import os
2
+ import re
3
+ from ..GeneralUtilities import GeneralUtilities
4
+ from ..ScriptCollectionCore import ScriptCollectionCore
5
+ from .TFCPS_Tools_General import TFCPS_Tools_General
6
+
7
+
8
+ class ContainerMount:
9
+ """A file or folder of the host which is made available in a container."""
10
+ source: str = None
11
+ target: str = None
12
+ read_only: bool = None
13
+
14
+ def __init__(self, source: str, target: str, read_only: bool):
15
+ self.source = source
16
+ self.target = target
17
+ self.read_only = read_only
18
+
19
+
20
+ class ContainerBuildEnvironment:
21
+ """The build-environment which a container gets from the host on which it is started: the mounts, the environment-variables and the
22
+ host-preparation-hook. It is the one place which defines it, so that every container which builds a repository (the build-container of
23
+ 'scbuildcodeunits -c' and the agent-sandbox of 'scrais') gets the same one. How the mounts are applied (bind-mount or copy) is decided by
24
+ the caller, because that depends on where the caller itself runs. See the article "Configuration-folder" of the reference for what is
25
+ mounted and why."""
26
+
27
+ __sc: ScriptCollectionCore = None
28
+ __tfcps_tools_general: TFCPS_Tools_General = None
29
+ __repository: str = None
30
+
31
+ def __init__(self, sc: ScriptCollectionCore, tfcps_tools_general: TFCPS_Tools_General, repository: str):
32
+ self.__sc = sc
33
+ self.__tfcps_tools_general = tfcps_tools_general
34
+ self.__repository = repository
35
+
36
+ @GeneralUtilities.check_arguments
37
+ def run_host_preparation_hook(self, build_arguments: list[str]) -> None:
38
+ """Runs the optional user-specific script which prepares this host for a container-start (for example to log in to the registry the
39
+ image is pulled from). It has to run before get_environment_variables, so it can also create the files their values are read from."""
40
+ self.__tfcps_tools_general.run_custom_script_if_available(self.__tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScriptForContainer.py"), build_arguments)
41
+
42
+ @GeneralUtilities.check_arguments
43
+ def get_mounts(self) -> list[ContainerMount]:
44
+ """Returns the files and folders of this host which a container gets. A part which does not exist on this host is not returned."""
45
+ result: list[ContainerMount] = []
46
+
47
+ #the optional user-specific scripts which prepare the container itself. the whole folder (not only the pre-codeunit-build-hook
48
+ #itself) is mounted, so the hook can start a sibling script placed next to it (for example to keep the hook independent of a
49
+ #package it only installs, while the actual preparation-logic which needs that package lives in the sibling script; see
50
+ #TFCPS_Tools_General.get_custom_scripts_folder_for_container). The mount is writable (not read-only) because the scripts in that
51
+ #folder also use it as their own download-cache-folder. A build inside the container runs the hook (see
52
+ #TFCPS_CodeUnit_BuildCodeUnits.__run_custom_pre_codeunit_build_script).
53
+ custom_scripts_folder: str = self.__tfcps_tools_general.get_custom_scripts_folder_for_container()
54
+ if os.path.isdir(custom_scripts_folder):
55
+ result.append(ContainerMount(custom_scripts_folder, self.__tfcps_tools_general.get_folder_of_custom_scripts_in_container(), False))
56
+
57
+ #the machine-wide image-registries-file of this host, so the container takes the images from the same custom registries as this
58
+ #host. Without it the container would find no mapping at all - the configuration-folder inside the container belongs to the
59
+ #container-user and is empty - and would therefore fall back to the upstream-registries defined in the repository, which are
60
+ #rate-limited. It is read-only because a build never changes the file.
61
+ oci_image_manager = self.__tfcps_tools_general.oci_image_manager
62
+ image_registries_file: str = oci_image_manager.get_image_registries_file_in_configuration_folder()
63
+ if os.path.isfile(image_registries_file):
64
+ result.append(ContainerMount(image_registries_file, oci_image_manager.get_image_registries_file_in_container(), True))
65
+
66
+ #the 'TFCPS'-folder of the configuration-folder of this host, so the container resolves the required environment-variables of the
67
+ #product (see TFCPS_Tools_General.get_required_environment_variables) and the OCI-registry-credentials (see
68
+ #ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) exactly like this host does - including a registry
69
+ #which is not publicly readable. The whole folder is mounted and not only EnvironmentVariables.csv in it, because a value of that file
70
+ #can point to a secret-file next to it ('file'-kind with a relative path, which is the recommended form) and such a value is resolved
71
+ #relative to the file itself. Only this folder (and not the whole configuration-folder) is mounted, so nothing else of it is exposed.
72
+ #It is read-only because a build never changes it.
73
+ tfcps_configuration_folder: str = self.__sc.get_tfcps_configuration_folder()
74
+ if os.path.isdir(tfcps_configuration_folder):
75
+ result.append(ContainerMount(tfcps_configuration_folder, self.__sc.get_tfcps_configuration_folder_in_container(), True))
76
+
77
+ #the files of the root-CA-certificates which this host declares via TrustedRootCA_<name>_Path. They are installed into the
78
+ #trust-store of the container by the entrypoint of the SCBuilder-image (see __get_trusted_root_cas).
79
+ file_cas, _ = self.__get_trusted_root_cas()
80
+ for ca_name, host_path in file_cas.items():
81
+ result.append(ContainerMount(host_path, f"/Workspace/RootCAs/{ca_name}.crt", True))
82
+ return result
83
+
84
+ @GeneralUtilities.check_arguments
85
+ def get_environment_variables(self) -> dict[str, str]:
86
+ """Returns the environment-variables (name and value) which a container gets: the ones which are declared as required in
87
+ <repository>/.ScriptCollection/ProductInformation.xml (including the ones which are declared for this machine, see
88
+ TFCPS_Tools_General.get_required_environment_variable_names) and the TrustedRootCA_<name>_ContentBase64-variables. The values are
89
+ typically secrets: a caller must give them to docker through the environment of the docker-client and pass only their names as
90
+ arguments, because arguments are written to the log and are visible in the process-list of this host."""
91
+ result: dict[str, str] = self.__tfcps_tools_general.get_required_environment_variables(self.__repository)
92
+ _, inline_cas = self.__get_trusted_root_cas()
93
+ result.update(inline_cas)
94
+ return result
95
+
96
+ @GeneralUtilities.check_arguments
97
+ def __get_trusted_root_cas(self) -> tuple[dict[str, str], dict[str, str]]:
98
+ """Resolves the TrustedRootCA_<name>_Path and TrustedRootCA_<name>_ContentBase64 environment-variables of this host. Returns
99
+ (file_cas, inline_cas):
100
+ - file_cas: <name> -> host-path of the certificate for each _Path-variable.
101
+ - inline_cas: variable-name -> value for each _ContentBase64-variable.
102
+ Candidate-names are collected from two sources (identical to the OCI-registry-credentials, see
103
+ ScriptCollectionCore.__get_defined_oci_registry_names): the machine-wide environment-variables-configuration-file and the
104
+ environment of the current process. The value is then resolved through ScriptCollectionCore.resolve_environment_variable, so a
105
+ CSV-entry takes precedence over an environment-variable of the same name - consistent with every other environment-variable
106
+ this script-collection resolves. If a name is declared in the file but its value can not be determined there (for example a
107
+ 'file'-entry pointing to a missing file), the resolver falls back to the environment and fails only if neither source yields a
108
+ value. The actual trust-store update happens in SCBuilder's EntryPoint.sh, which scans /Workspace/RootCAs and the
109
+ TrustedRootCA_*-environment-variables and runs update-ca-certificates once before any build-step. The <name>-part becomes the
110
+ filename inside /usr/local/share/ca-certificates/ and must therefore be unique across the two env-var-forms; a duplicate is
111
+ rejected here with a clear error, before the container is started."""
112
+ path_pattern = re.compile(r"^TrustedRootCA_(.+)_Path$", re.IGNORECASE)
113
+ content_b64_pattern = re.compile(r"^TrustedRootCA_(.+)_ContentBase64$", re.IGNORECASE)
114
+ #collect every candidate-variable-name from both sources before any value is resolved; this way a secret-value is read only for
115
+ #a name that is actually declared and the discovery is identical to the OCI-registry-credential-discovery. The configuration-
116
+ #file is scanned before the environment and a name found in both is kept with the casing of the file: on Windows, os.environ
117
+ #normalizes a variable-name to uppercase as soon as it is set, so without this preference the same declaration in both places
118
+ #would be treated as two different names and resolution would silently pick the environment-value instead of the file-value.
119
+ candidate_names: dict[str, str] = {}#lower-case variable-name -> canonical variable-name (file-casing preferred)
120
+ configuration_file = self.__sc.get_environment_variables_configuration_file()
121
+ if os.path.isfile(configuration_file):
122
+ for entry in GeneralUtilities.read_csv_file(configuration_file, True):
123
+ if 0 == len(entry):
124
+ continue
125
+ env_name = entry[0]
126
+ if path_pattern.match(env_name) is not None or content_b64_pattern.match(env_name) is not None:
127
+ candidate_names.setdefault(env_name.lower(), env_name)
128
+ for env_name in os.environ.keys():
129
+ if path_pattern.match(env_name) is not None or content_b64_pattern.match(env_name) is not None:
130
+ candidate_names.setdefault(env_name.lower(), env_name)
131
+ seen_names: dict[str, str] = {}#lower-case <name>-part -> originating variable-name, used only to produce an informative error on duplicate <name>.
132
+ file_cas: dict[str, str] = {}
133
+ inline_cas: dict[str, str] = {}
134
+ #sorted so the log-output (and therefore the order of the mounts and environment-variables) stays deterministic across runs.
135
+ for env_name in sorted(candidate_names.values(), key=str.lower):
136
+ env_value = self.__sc.resolve_environment_variable(env_name, f"the TrustedRootCA-mechanism (declared via \"{env_name}\")")
137
+ if not GeneralUtilities.string_has_content(env_value):
138
+ continue
139
+ path_match = path_pattern.match(env_name)
140
+ if path_match is not None:
141
+ ca_name = path_match.group(1)
142
+ self.__check_trusted_root_ca_name_is_unique(seen_names, ca_name, env_name)
143
+ host_path = env_value.strip()
144
+ if not os.path.isfile(host_path):
145
+ raise ValueError(f"Environment-variable \"{env_name}\" points to a file which does not exist on this host: \"{host_path}\".")
146
+ with open(host_path, "r", encoding="utf-8", errors="replace") as pem_file:
147
+ head = pem_file.read(64)
148
+ if "-----BEGIN CERTIFICATE-----" not in head:
149
+ raise ValueError(f"Environment-variable \"{env_name}\" (\"{host_path}\") does not look like a PEM-encoded certificate (missing \"-----BEGIN CERTIFICATE-----\"-header).")
150
+ file_cas[ca_name] = host_path
151
+ continue
152
+ content_b64_match = content_b64_pattern.match(env_name)
153
+ if content_b64_match is not None:
154
+ ca_name = content_b64_match.group(1)
155
+ self.__check_trusted_root_ca_name_is_unique(seen_names, ca_name, env_name)
156
+ inline_cas[env_name] = env_value
157
+ continue
158
+ return file_cas, inline_cas
159
+
160
+ @GeneralUtilities.check_arguments
161
+ def __check_trusted_root_ca_name_is_unique(self, seen_names: dict[str, str], ca_name: str, env_name: str) -> None:
162
+ key = ca_name.lower()
163
+ if key in seen_names:
164
+ raise ValueError(f"Multiple TrustedRootCA-definitions for name \"{ca_name}\" found (\"{seen_names[key]}\" and \"{env_name}\"). The <name>-part becomes the filename in the trust-store and must therefore be unique across _Path and _ContentBase64.")
165
+ seen_names[key] = env_name
@@ -0,0 +1,91 @@
1
+ import re
2
+ from ...GeneralUtilities import GeneralUtilities
3
+ from ...ScriptCollectionCore import ScriptCollectionCore
4
+ from ...SCLog import LogLevel
5
+ from ..PackageSource import PackageSource
6
+ from ..TFCPS_Tools_General import TFCPS_Tools_General
7
+
8
+
9
+ class NuGetPackageSourceRegistration:
10
+ """Registers the NuGet-sources which are declared for the current machine (see TFCPS_Tools_General.get_declared_package_sources). It is
11
+ used by every DotNet-codeunit-build and by the command-line-tool 'scregisterpackagesources' (which the agent-sandbox of scrais runs before
12
+ the agent starts), so a source is registered on the same way everywhere."""
13
+
14
+ #the output of the dotnet-cli is localized. The language is set explicitly because the output of "dotnet nuget list source" gets parsed, which would otherwise only work on machines which are configured to use english.
15
+ __dotnet_cli_environment_variables: dict = {"DOTNET_CLI_UI_LANGUAGE": "en-US"}
16
+ __sc: ScriptCollectionCore = None
17
+ __tfcps_tools_general: TFCPS_Tools_General = None
18
+
19
+ def __init__(self, sc: ScriptCollectionCore, tfcps_tools_general: TFCPS_Tools_General):
20
+ self.__sc = sc
21
+ self.__tfcps_tools_general = tfcps_tools_general
22
+
23
+ @GeneralUtilities.check_arguments
24
+ def register_declared_package_sources(self, folder: str) -> None:
25
+ """Ensures that every NuGet-source which is declared for this machine is registered with its current credentials, so a codeunit can
26
+ also use dependencies which are not available on nuget.org. The dotnet-cli is run in the given folder.
27
+ A source which is already registered under the declared name is removed and registered again, so an outdated registration (for
28
+ example one which points to another url) can not let the registration fail. Sources which are not declared are not touched at
29
+ all: the registration is stored in the NuGet-configuration of the current user, which - when the build runs directly on the
30
+ host - is the configuration the user also uses for their own work and which must not be destroyed by a build."""
31
+ package_sources: list[PackageSource] = self.__tfcps_tools_general.get_declared_package_sources("CSharp")
32
+ if len(package_sources) == 0:
33
+ return
34
+ for package_source in package_sources:
35
+ registered_sources: dict[str, str] = self.__get_registered_nuget_sources(folder)
36
+ name_of_source_with_same_name: str = None
37
+ name_of_source_with_same_url: str = None
38
+ for registered_name, registered_url in registered_sources.items():
39
+ if registered_name.lower() == package_source.name.lower():
40
+ name_of_source_with_same_name = registered_name
41
+ elif self.__nuget_sources_are_equal(registered_url, package_source.url):
42
+ #the same feed can already be registered under another name (which is the usual case on a machine on which the user
43
+ #configured the feed manually). Such a source is updated instead of being added a second time, because a second
44
+ #registration of the same url would let every restore query the feed twice.
45
+ name_of_source_with_same_url = registered_name
46
+ if name_of_source_with_same_name is not None:
47
+ self.__remove_nuget_source(folder, name_of_source_with_same_name)
48
+ if name_of_source_with_same_url is None:
49
+ arguments = ["nuget", "add", "source", package_source.url, "--name", package_source.name]
50
+ else:
51
+ #the credentials are set again even when the source already exists, because a token can have been rotated since it was registered.
52
+ arguments = ["nuget", "update", "source", name_of_source_with_same_url, "--source", package_source.url]
53
+ if package_source.has_credentials():
54
+ arguments = arguments+["--username", package_source.username, "--password", package_source.password]
55
+ if not GeneralUtilities.current_system_is_windows():
56
+ #without this the password can not be encrypted, which lets the command fail on the operating-systems which do not support the encryption.
57
+ arguments.append("--store-password-in-clear-text")
58
+ arguments_for_log = list(arguments)
59
+ if package_source.has_credentials():
60
+ arguments_for_log[arguments_for_log.index("--password")+1] = "***"#the password must not be written to the build-log
61
+ self.__sc.log.log(f"Add NuGet-source \"{package_source.name}\" ({package_source.url})..." if name_of_source_with_same_url is None else f"Update the already registered NuGet-source \"{name_of_source_with_same_url}\" ({package_source.url})...", LogLevel.Debug)
62
+ self.__sc.run_program_argsasarray("dotnet", arguments, folder, arguments_for_log=arguments_for_log, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
63
+
64
+ @GeneralUtilities.check_arguments
65
+ def __remove_nuget_source(self, folder: str, name: str) -> None:
66
+ """Removes the registration of the NuGet-source with the given name, so it can be registered again with its currently declared url and credentials."""
67
+ self.__sc.log.log(f"Remove the outdated registration of the NuGet-source \"{name}\"...", LogLevel.Debug)
68
+ self.__sc.run_program_argsasarray("dotnet", ["nuget", "remove", "source", name], folder, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
69
+
70
+ @GeneralUtilities.check_arguments
71
+ def __get_registered_nuget_sources(self, folder: str) -> dict[str, str]:
72
+ """Returns the NuGet-sources which are registered for the given folder as a mapping of their name to their url. The sources are
73
+ read from the output of "dotnet nuget list source", which prints the name (with its state) and the url of a source in two
74
+ consecutive lines."""
75
+ result: dict[str, str] = {}
76
+ list_result = self.__sc.run_program_argsasarray("dotnet", ["nuget", "list", "source"], folder, throw_exception_if_exitcode_is_not_zero=False, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
77
+ name_pattern = re.compile(r"^\s*\d+\.\s+(.+?)\s+\[[^\[\]]+\]\s*$")
78
+ current_name: str = None
79
+ for line in list_result[1].splitlines():
80
+ match = name_pattern.match(line)
81
+ if match is not None:
82
+ current_name = match.group(1).strip()
83
+ elif current_name is not None and GeneralUtilities.string_has_content(line):
84
+ result[current_name] = line.strip()
85
+ current_name = None
86
+ return result
87
+
88
+ @GeneralUtilities.check_arguments
89
+ def __nuget_sources_are_equal(self, url: str, other_url: str) -> bool:
90
+ """Compares two urls of package-sources. A trailing slash and the case are ignored because they do not address a different feed."""
91
+ return url.strip().rstrip("/").lower() == other_url.strip().rstrip("/").lower()
@@ -11,15 +11,13 @@ from packaging.version import Version
11
11
  from .CertificateGeneratorInformationBase import CertificateGeneratorInformationBase
12
12
  from ...GeneralUtilities import GeneralUtilities, VersionEcholon
13
13
  from ...SCLog import LogLevel
14
- from ..PackageSource import PackageSource
14
+ from .NuGetPackageSourceRegistration import NuGetPackageSourceRegistration
15
15
  from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base,TFCPS_CodeUnitSpecific_Base_CLI
16
16
 
17
17
  class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
18
18
 
19
19
  is_library:bool = None
20
20
  csproj_file:bool = None
21
- #the output of the dotnet-cli is localized. The language is set explicitly for the commands whose output gets parsed, because otherwise the parsing would only work on machines which are configured to use english.
22
- __dotnet_cli_environment_variables:dict = {"DOTNET_CLI_UI_LANGUAGE": "en-US"}
23
21
  #the only accepted form of a version of a package-reference: an exact version in square brackets. Everything else is a version-range.
24
22
  __pinned_version_regex = re.compile(r"^\[[^,\[\]]+\]$")
25
23
 
@@ -29,78 +27,7 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
29
27
  self.is_library="<OutputType>Library</OutputType>" in GeneralUtilities.read_text_from_file(self.csproj_file)#TODO do a real check by checking this property using xpath
30
28
  #every dotnet-operation of a codeunit (build, test, linting, ...) restores the dependencies of the codeunit, so the sources
31
29
  #they are downloaded from have to be available for all of them and not only for the build.
32
- self.__ensure_declared_package_sources_are_available()
33
-
34
- @GeneralUtilities.check_arguments
35
- def __ensure_declared_package_sources_are_available(self) -> None:
36
- """Ensures that every NuGet-source which is declared for this machine (see TFCPS_Tools_General.get_declared_package_sources) is
37
- registered with its current credentials, so a codeunit can also use dependencies which are not available on nuget.org.
38
- A source which is already registered under the declared name is removed and registered again, so an outdated registration (for
39
- example one which points to another url) can not let the registration fail. Sources which are not declared are not touched at
40
- all: the registration is stored in the NuGet-configuration of the current user, which - when the build runs directly on the
41
- host - is the configuration the user also uses for their own work and which must not be destroyed by a build."""
42
- package_sources: list[PackageSource] = self.tfcps_Tools_General.get_declared_package_sources("CSharp")
43
- if len(package_sources) == 0:
44
- return
45
- codeunit_folder: str = self.get_codeunit_folder()
46
- for package_source in package_sources:
47
- registered_sources: dict[str, str] = self.__get_registered_nuget_sources(codeunit_folder)
48
- name_of_source_with_same_name: str = None
49
- name_of_source_with_same_url: str = None
50
- for registered_name, registered_url in registered_sources.items():
51
- if registered_name.lower() == package_source.name.lower():
52
- name_of_source_with_same_name = registered_name
53
- elif self.__nuget_sources_are_equal(registered_url, package_source.url):
54
- #the same feed can already be registered under another name (which is the usual case on a machine on which the user
55
- #configured the feed manually). Such a source is updated instead of being added a second time, because a second
56
- #registration of the same url would let every restore query the feed twice.
57
- name_of_source_with_same_url = registered_name
58
- if name_of_source_with_same_name is not None:
59
- self.__remove_nuget_source(codeunit_folder, name_of_source_with_same_name)
60
- if name_of_source_with_same_url is None:
61
- arguments = ["nuget", "add", "source", package_source.url, "--name", package_source.name]
62
- else:
63
- #the credentials are set again even when the source already exists, because a token can have been rotated since it was registered.
64
- arguments = ["nuget", "update", "source", name_of_source_with_same_url, "--source", package_source.url]
65
- if package_source.has_credentials():
66
- arguments = arguments+["--username", package_source.username, "--password", package_source.password]
67
- if not GeneralUtilities.current_system_is_windows():
68
- #without this the password can not be encrypted, which lets the command fail on the operating-systems which do not support the encryption.
69
- arguments.append("--store-password-in-clear-text")
70
- arguments_for_log = list(arguments)
71
- if package_source.has_credentials():
72
- arguments_for_log[arguments_for_log.index("--password")+1] = "***"#the password must not be written to the build-log
73
- self._protected_sc.log.log(f"Add NuGet-source \"{package_source.name}\" ({package_source.url})..." if name_of_source_with_same_url is None else f"Update the already registered NuGet-source \"{name_of_source_with_same_url}\" ({package_source.url})...", LogLevel.Debug)
74
- self._protected_sc.run_program_argsasarray("dotnet", arguments, codeunit_folder, arguments_for_log=arguments_for_log, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
75
-
76
- @GeneralUtilities.check_arguments
77
- def __remove_nuget_source(self, folder: str, name: str) -> None:
78
- """Removes the registration of the NuGet-source with the given name, so it can be registered again with its currently declared url and credentials."""
79
- self._protected_sc.log.log(f"Remove the outdated registration of the NuGet-source \"{name}\"...", LogLevel.Debug)
80
- self._protected_sc.run_program_argsasarray("dotnet", ["nuget", "remove", "source", name], folder, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
81
-
82
- @GeneralUtilities.check_arguments
83
- def __get_registered_nuget_sources(self, folder: str) -> dict[str, str]:
84
- """Returns the NuGet-sources which are registered for the given folder as a mapping of their name to their url. The sources are
85
- read from the output of "dotnet nuget list source", which prints the name (with its state) and the url of a source in two
86
- consecutive lines."""
87
- result: dict[str, str] = {}
88
- list_result = self._protected_sc.run_program_argsasarray("dotnet", ["nuget", "list", "source"], folder, throw_exception_if_exitcode_is_not_zero=False, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
89
- name_pattern = re.compile(r"^\s*\d+\.\s+(.+?)\s+\[[^\[\]]+\]\s*$")
90
- current_name: str = None
91
- for line in list_result[1].splitlines():
92
- match = name_pattern.match(line)
93
- if match is not None:
94
- current_name = match.group(1).strip()
95
- elif current_name is not None and GeneralUtilities.string_has_content(line):
96
- result[current_name] = line.strip()
97
- current_name = None
98
- return result
99
-
100
- @GeneralUtilities.check_arguments
101
- def __nuget_sources_are_equal(self, url: str, other_url: str) -> bool:
102
- """Compares two urls of package-sources. A trailing slash and the case are ignored because they do not address a different feed."""
103
- return url.strip().rstrip("/").lower() == other_url.strip().rstrip("/").lower()
30
+ NuGetPackageSourceRegistration(self._protected_sc, self.tfcps_Tools_General).register_declared_package_sources(self.get_codeunit_folder())
104
31
 
105
32
  @GeneralUtilities.check_arguments
106
33
  def get_runtimes(self) -> list[str]:
@@ -15,6 +15,7 @@ from .TFCPS_CodeSpellCheck import TFCPS_CodeSpellCheck
15
15
  from .TFCPS_FindingsSorter import TFCPS_FindingsSorter
16
16
  from .TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
17
17
  from .TFCPS_Tools_General import TFCPS_Tools_General, CommonProjectStructureVersion
18
+ from .ContainerBuildEnvironment import ContainerBuildEnvironment
18
19
 
19
20
 
20
21
  class TFCPS_UpdateDependenciesHook(TFCPS_BuildCodeUnitsHook):
@@ -197,21 +198,23 @@ class TFCPS_CodeUnit_BuildCodeUnits:
197
198
  hook.run_after_codeunit_was_built(tFCPS_CodeUnit_BuildCodeUnit)
198
199
  self.sc.log.log(GeneralUtilities.get_line())
199
200
 
200
- self.tfcps_tools_general.generate_svg_files_from_plantuml_files_for_repository(self.repository, self.use_cache())
201
- self.tfcps_tools_general.generate_svg_files_from_vega_files_for_repository(self.repository, self.use_cache())
202
- self.tfcps_tools_general.generate_svg_files_from_vegalite_files_for_repository(self.repository, self.use_cache())
203
- self.__do_project_linting(self.repository)
204
- self.__update_agent_sandbox_image(self.repository)
201
+ self.sc.log.log("Do project post-build-tasks...")
202
+
203
+ if not self.__fast_lane:
204
+ self.__do_project_linting(self.repository)
205
+ self.__update_agent_sandbox_image(self.repository)
206
+ self.search_for_secrets()
205
207
  self.__normalize_line_endings_of_common_files()
206
- self.search_for_secrets()
207
-
208
- self.sc.log.log(GeneralUtilities.get_line())
209
208
 
210
209
  if self.is_pre_merge():
211
210
  self.__collect_metrics()
212
211
  self.__generate_loc_diagram()
213
- self.sc.log.log(GeneralUtilities.get_line())
214
212
 
213
+ self.tfcps_tools_general.generate_svg_files_from_plantuml_files_for_repository(self.repository, self.use_cache())
214
+ self.tfcps_tools_general.generate_svg_files_from_vega_files_for_repository(self.repository, self.use_cache())
215
+ self.tfcps_tools_general.generate_svg_files_from_vegalite_files_for_repository(self.repository, self.use_cache())
216
+
217
+ self.sc.log.log(GeneralUtilities.get_line())
215
218
  except Exception:
216
219
  error_occurred=True
217
220
  raise
@@ -566,61 +569,19 @@ class TFCPS_CodeUnit_BuildCodeUnits:
566
569
  update_argument="pip3 install scriptcollection --upgrade && "
567
570
  executable_arguments=["bash","-c", f"{update_argument}scshowversion && "+" ".join(executable_arguments)]
568
571
 
569
- #run the optional user-specific script which prepares this host for a container-build (for example to log in to the registry the image is pulled from).
570
- #it runs before the environment-variables are resolved, so it can also create the files their values are read from.
571
- self.tfcps_tools_general.run_custom_script_if_available(self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScriptForContainer.py"), self.__get_build_arguments(self.repository))
572
-
573
- #mount the optional user-specific scripts which prepare the container itself. the whole folder (not only the pre-codeunit-build-hook
574
- #itself) is mounted, so the hook can start a sibling script placed next to it (for example to keep the hook independent of a
575
- #package it only installs, while the actual preparation-logic which needs that package lives in the sibling script; see
576
- #get_custom_scripts_folder_for_container). The mount is writable (not read-only) because the scripts in that folder also use it as
577
- #their own download-cache-folder. The build inside the container runs the hook (see __run_custom_pre_codeunit_build_script); if the
578
- #folder does not exist on the host nothing is mounted and nothing is run.
572
+ #the build-environment (host-hook, mounts and environment-variables) is defined by ContainerBuildEnvironment, which is also used by
573
+ #the agent-sandbox of scrais, so that both containers get the same one. The hook runs before the environment-variables are resolved,
574
+ #so it can also create the files their values are read from.
575
+ container_build_environment: ContainerBuildEnvironment = ContainerBuildEnvironment(self.sc, self.tfcps_tools_general, self.repository)
576
+ container_build_environment.run_host_preparation_hook(self.__get_build_arguments(self.repository))
579
577
  mount_arguments: list[str] = []
580
- custom_scripts_folder_for_inside_the_container: str = self.tfcps_tools_general.get_custom_scripts_folder_for_container()
581
- if os.path.isdir(custom_scripts_folder_for_inside_the_container):
582
- mount_arguments += ["-v", f"{custom_scripts_folder_for_inside_the_container}:{self.tfcps_tools_general.get_folder_of_custom_scripts_in_container()}"]
583
-
584
- #mount the machine-wide image-registries-file of this host (if it exists), so the build inside the container takes the images from the same
585
- #custom registries as a build on the host. Without the mount the build in the container would find no mapping at all - the
586
- #configuration-folder inside the container belongs to the container-user and is empty - and would therefore fall back to the
587
- #upstream-registries defined in the repository, which are rate-limited. The mount is read-only because a build never changes the file.
588
- oci_image_manager = self.tfcps_tools_general.oci_image_manager
589
- image_registries_file: str = oci_image_manager.get_image_registries_file_in_configuration_folder()
590
- if os.path.isfile(image_registries_file):
591
- mount_arguments += ["-v", f"{image_registries_file}:{oci_image_manager.get_image_registries_file_in_container()}:ro"]
592
-
593
- #mount the 'TFCPS'-folder of the configuration-folder of this host (if it exists), so the build inside the container resolves the
594
- #required environment-variables of the product (see TFCPS_Tools_General.get_required_environment_variables) and the
595
- #OCI-registry-credentials (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) exactly like a
596
- #build on the host does - including a registry which is not publicly readable, which without this mount could not be used inside
597
- #the container at all, because the configuration-folder of the container-user does not contain the credentials of this host.
598
- #The whole folder is mounted and not only EnvironmentVariables.csv in it, because a value of that file can point to a secret-file
599
- #next to it ('file'-kind with a relative path, which is the recommended form) and such a value is resolved relative to the file
600
- #itself - so with only the file mounted, every entry of that kind would fail inside the container. Only this folder (and not the
601
- #whole configuration-folder) is mounted, so nothing else of it is exposed to the container. The mount is read-only because a build
602
- #never changes it. The alternative way to provide a value is an environment-variable of the build-pipeline itself (the pipeline's
603
- #own secret-store); both sources are used together, the file taking precedence.
604
- tfcps_configuration_folder: str = self.sc.get_tfcps_configuration_folder()
605
- if os.path.isdir(tfcps_configuration_folder):
606
- mount_arguments += ["-v", f"{tfcps_configuration_folder}:{self.sc.get_tfcps_configuration_folder_in_container()}:ro"]
607
-
608
- #install the host-declared trusted root-CA-certificates into the container (needed when the build pulls packages from a privately
609
- #hosted registry whose certificate-chain is not publicly rooted - for example an internally hosted gitlab-nuget-source). The
610
- #two env-var forms are detected automatically from both the machine-wide environment-variables-configuration-file and the
611
- #environment of the current process; see __collect_trusted_root_ca_arguments. _Path-variables add file-mounts, the inline
612
- #_ContentBase64-form is returned as a name->value dict that gets merged into required_environment_variables below so its
613
- #value flows through the docker-client's own environment (not through argv) just like a required secret does.
614
- trusted_root_ca_mount_arguments, trusted_root_ca_inline_env_vars = self.__collect_trusted_root_ca_arguments()
615
- mount_arguments += trusted_root_ca_mount_arguments
616
-
617
- #pass the environment-variables which are declared as required in <repository>/.ScriptCollection/ProductInformation.xml into the container
618
- #so they do not have to be specified explicitly on every call. Their values are resolved from the user-specific
619
- #configuration-file (see TFCPS_Tools_General.get_environment_variables_configuration_file), which only exists on the host.
620
- #only the names are passed as arguments; the values are given to the docker-client through its own environment, because arguments
621
- #are written to the log and are visible in the process-list of this host, which must not happen for a value which is typically a secret.
622
- required_environment_variables: dict[str, str] = self.tfcps_tools_general.get_required_environment_variables(self.repository)
623
- required_environment_variables.update(trusted_root_ca_inline_env_vars)
578
+ for mount in container_build_environment.get_mounts():
579
+ mount_arguments += ["-v", f"{mount.source}:{mount.target}{':ro' if mount.read_only else ''}"]
580
+
581
+ #only the names of the environment-variables are passed as arguments; the values are given to the docker-client through its own
582
+ #environment, because arguments are written to the log and are visible in the process-list of this host, which must not happen for
583
+ #a value which is typically a secret.
584
+ required_environment_variables: dict[str, str] = container_build_environment.get_environment_variables()
624
585
  env_arguments: list[str] = []
625
586
  for env_variable_name in required_environment_variables:
626
587
  env_arguments += ["-e", env_variable_name]
@@ -653,78 +614,6 @@ class TFCPS_CodeUnit_BuildCodeUnits:
653
614
  stderr:str=result[2] or GeneralUtilities.empty_string
654
615
  return (exit_code==0,f"{stdout}\n{stderr}")
655
616
 
656
- @GeneralUtilities.check_arguments
657
- def __collect_trusted_root_ca_arguments(self) -> tuple[list[str], dict[str, str]]:
658
- """Resolves the TrustedRootCA_<name>_Path and TrustedRootCA_<name>_ContentBase64 environment-variables of the current host into
659
- docker-arguments which install those certificates into the trust-store of the build-container. Returns (mount_arguments, inline_env_vars):
660
- - mount_arguments: '-v <host-path>:/Workspace/RootCAs/<name>.crt:ro' entries for each _Path-variable.
661
- - inline_env_vars: a name->value dict for each _ContentBase64-variable; the caller merges it into required_environment_variables
662
- so the values flow through the docker-client's own environment, not through argv.
663
- Candidate-names are collected from two sources (identical to the OCI-registry-credentials, see
664
- ScriptCollectionCore.__get_defined_oci_registry_names): the machine-wide environment-variables-configuration-file and the
665
- environment of the current process. The value is then resolved through ScriptCollectionCore.resolve_environment_variable, so a
666
- CSV-entry takes precedence over an environment-variable of the same name - consistent with every other environment-variable
667
- this script-collection resolves. If a name is declared in the file but its value can not be determined there (for example a
668
- 'file'-entry pointing to a missing file), the resolver falls back to the environment and fails only if neither source yields a
669
- value. The actual trust-store update happens in SCBuilder's EntryPoint.sh, which scans /Workspace/RootCAs and the
670
- TrustedRootCA_*-environment-variables and runs update-ca-certificates once before any build-step. The <name>-part becomes the
671
- filename inside /usr/local/share/ca-certificates/ and must therefore be unique across the two env-var-forms; a duplicate is
672
- rejected here with a clear error, before the container is started."""
673
- path_pattern = re.compile(r"^TrustedRootCA_(.+)_Path$", re.IGNORECASE)
674
- content_b64_pattern = re.compile(r"^TrustedRootCA_(.+)_ContentBase64$", re.IGNORECASE)
675
- #collect every candidate-variable-name from both sources before any value is resolved; this way a secret-value is read only for
676
- #a name that is actually declared and the discovery is identical to the OCI-registry-credential-discovery. The configuration-
677
- #file is scanned before the environment and a name found in both is kept with the casing of the file: on Windows, os.environ
678
- #normalizes a variable-name to uppercase as soon as it is set, so without this preference the same declaration in both places
679
- #would be treated as two different names and resolution would silently pick the environment-value instead of the file-value.
680
- candidate_names: dict[str, str] = {}#lower-case variable-name -> canonical variable-name (file-casing preferred)
681
- configuration_file = self.sc.get_environment_variables_configuration_file()
682
- if os.path.isfile(configuration_file):
683
- for entry in GeneralUtilities.read_csv_file(configuration_file, True):
684
- if 0 == len(entry):
685
- continue
686
- env_name = entry[0]
687
- if path_pattern.match(env_name) is not None or content_b64_pattern.match(env_name) is not None:
688
- candidate_names.setdefault(env_name.lower(), env_name)
689
- for env_name in os.environ.keys():
690
- if path_pattern.match(env_name) is not None or content_b64_pattern.match(env_name) is not None:
691
- candidate_names.setdefault(env_name.lower(), env_name)
692
- seen_names: dict[str, str] = {}#lower-case <name>-part -> originating variable-name, used only to produce an informative error on duplicate <name>.
693
- mount_arguments: list[str] = []
694
- inline_env_vars: dict[str, str] = {}
695
- #sorted so the log-output (and therefore the order of "-v"-/"-e"-arguments) stays deterministic across runs.
696
- for env_name in sorted(candidate_names.values(), key=str.lower):
697
- env_value = self.sc.resolve_environment_variable(env_name, f"the TrustedRootCA-mechanism (declared via \"{env_name}\")")
698
- if not GeneralUtilities.string_has_content(env_value):
699
- continue
700
- path_match = path_pattern.match(env_name)
701
- if path_match is not None:
702
- ca_name = path_match.group(1)
703
- self.__check_trusted_root_ca_name_is_unique(seen_names, ca_name, env_name)
704
- host_path = env_value.strip()
705
- if not os.path.isfile(host_path):
706
- raise ValueError(f"Environment-variable \"{env_name}\" points to a file which does not exist on this host: \"{host_path}\".")
707
- with open(host_path, "r", encoding="utf-8", errors="replace") as pem_file:
708
- head = pem_file.read(64)
709
- if "-----BEGIN CERTIFICATE-----" not in head:
710
- raise ValueError(f"Environment-variable \"{env_name}\" (\"{host_path}\") does not look like a PEM-encoded certificate (missing \"-----BEGIN CERTIFICATE-----\"-header).")
711
- mount_arguments += ["-v", f"{host_path}:/Workspace/RootCAs/{ca_name}.crt:ro"]
712
- continue
713
- content_b64_match = content_b64_pattern.match(env_name)
714
- if content_b64_match is not None:
715
- ca_name = content_b64_match.group(1)
716
- self.__check_trusted_root_ca_name_is_unique(seen_names, ca_name, env_name)
717
- inline_env_vars[env_name] = env_value
718
- continue
719
- return mount_arguments, inline_env_vars
720
-
721
- @GeneralUtilities.check_arguments
722
- def __check_trusted_root_ca_name_is_unique(self, seen_names: dict[str, str], ca_name: str, env_name: str) -> None:
723
- key = ca_name.lower()
724
- if key in seen_names:
725
- raise ValueError(f"Multiple TrustedRootCA-definitions for name \"{ca_name}\" found (\"{seen_names[key]}\" and \"{env_name}\"). The <name>-part becomes the filename in the trust-store and must therefore be unique across _Path and _ContentBase64.")
726
- seen_names[key] = env_name
727
-
728
617
  @GeneralUtilities.check_arguments
729
618
  def __translate_path_into_container(self, host_path: str, container_repository_folder: str) -> str:
730
619
  normalized_repository = os.path.normpath(self.repository)
@@ -1958,7 +1958,7 @@ class TFCPS_Tools_General:
1958
1958
  arch_tags.append(arch_tag)
1959
1959
  # Load tar → local image
1960
1960
  GeneralUtilities.write_message_to_stdout(f"Loading {tar_path}...")
1961
- result = self.__sc.run_program_argsasarray("docker",[ "load", "-i", tar_path])
1961
+ result = self.__sc.run_program_argsasarray_with_retry("docker",[ "load", "-i", tar_path],amount_of_attempts=3,delay_in_seconds=5)
1962
1962
  # docker load outputs: "Loaded image: sha256:abc123..." or "Loaded image ID: ..."
1963
1963
  # we need the loaded image ID
1964
1964
  loaded_id = None
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptcollection
3
- Version: 4.4.67
3
+ Version: 4.4.68
4
4
  Summary: The ScriptCollection is the place for reusable scripts.
5
5
  Author-email: Marius Göcke <marius.goecke@gmail.com>
6
6
  License-Expression: GPL-3.0-only
@@ -1,7 +1,7 @@
1
- ScriptCollection/AgentRunner.py,sha256=8UiY6mAZoSSb1LlKGJ29qhLUW_0DtryrC2Kdn5cDPgg,19820
1
+ ScriptCollection/AgentRunner.py,sha256=m6uYei6dCqrIGPN6YQvtrqFWFxWDqSQi3DtH82f79Zg,26181
2
2
  ScriptCollection/ArbTranslationsOrganizer.py,sha256=3TO4fjkYADcHGyUXdiQk-RtfCxtf9ojuBS3CbZtgXBE,12218
3
3
  ScriptCollection/CertificateUpdater.py,sha256=yZc7KESc1F_Mc6HP2UfLmNHRc2MvtVqp0t60J_CqY3s,9755
4
- ScriptCollection/Executables.py,sha256=Z-YRlSgD4GN7XjZuarXHZTPOuWtNr2rF5OzlwjJ3Yuo,82192
4
+ ScriptCollection/Executables.py,sha256=JWxzWKpLqZM0oJBhq_LP8cnLxNN9l4APqAROrTOeS_w,83902
5
5
  ScriptCollection/GeneralUtilities.py,sha256=kW6y52rdD-dJrjCtIgsdwYinz6FtclGj-xc_4tL1MUg,80022
6
6
  ScriptCollection/HTTPMaintenanceOverheadHelper.py,sha256=TToNtyO1XzsMbBsTBf3o0xgOK0v4Jf03qw2Z0xb2nCk,2007
7
7
  ScriptCollection/ProcessesRunner.py,sha256=wtNFdmj9klETTIDUSE90vaR5ClEX4V7dgJNVbp9_wi0,1381
@@ -12,7 +12,7 @@ ScriptCollection/ProgramRunnerSudo.py,sha256=vwKq-MOcqsgiKhSf-MBUDBC_UyaU8evP3Ro
12
12
  ScriptCollection/RepositoryDiagramGenerator.py,sha256=K_u2NW_p4ozdWD50irJEkkcDVa70TcUJpZN63TeXSLw,7092
13
13
  ScriptCollection/RepositoryHistoryAnalyzer.py,sha256=u9141B9D-Kef-9uo3yR7BXKUMzIKLVsesnYk7H-cWm4,6157
14
14
  ScriptCollection/SCLog.py,sha256=Fjf8Pdi63cXHGMxL_aI6A0dwGm7QIfrv3czjvpSUlaM,4705
15
- ScriptCollection/ScriptCollectionCore.py,sha256=TIqhsMEv5XvRpZlih-5_U-wqGiwr3CbpOYShk1sMHrU,310633
15
+ ScriptCollection/ScriptCollectionCore.py,sha256=jIBrMaE0DI8HU6kk8LYg1mY4jPMUPtoFw-SLUYQ3lvg,310633
16
16
  ScriptCollection/TestKind.py,sha256=jqLJwNvQeyea7SZf-UFzMC9pM0znv4do7FIC1xq8u9M,1672
17
17
  ScriptCollection/UnassignedVariablesCheck.py,sha256=8xNcp1D5HeOLO1a1NEerq2TsnpFhw7yzYPkaay-oisY,3804
18
18
  ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -38,6 +38,7 @@ ScriptCollection/Resources/CultureChooser/index.html,sha256=gxQzbrSp4WU52TqN-gcB
38
38
  ScriptCollection/Resources/Dependencies/JRE/Version.txt,sha256=WlFzNJMH4lfVzW5QGHnuE52jLujSGpQ48zVB-Cln-7g,9
39
39
  ScriptCollection/Resources/Dependencies/OpenAPIGenerator/Version.txt,sha256=hdAgOhOHx4s13fJ-j0A06oYoM9rdEfDe0RzHoeAKXxo,7
40
40
  ScriptCollection/Resources/MaintenanceSite/MaintenanceSite.html,sha256=CX9S1bdOz6xU2aGfr03tJRGczQrlpn-IeODc15d5kc0,232
41
+ ScriptCollection/TFCPS/ContainerBuildEnvironment.py,sha256=LIlWZnxqKGqTyAOxrdUOh2yTLmaTlA8skqIII9-SElk,12620
41
42
  ScriptCollection/TFCPS/NpmDependencies.py,sha256=IfUh9EyrkHxovGRWYagNMZjBUXyKjZ4AG-Z8leehw24,6304
42
43
  ScriptCollection/TFCPS/PackageSource.py,sha256=tQoMbxrIbV3mYHKH2wgOoGzIHY1TvBH8EAFpWGhryrQ,1311
43
44
  ScriptCollection/TFCPS/PlantUMLRenderer.py,sha256=He5woqx9IR96vMXtHo12BmeXmcjLoifGD8Sup0bpxuo,11060
@@ -47,7 +48,7 @@ ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py,sha256=67_DQwoH5bhxxSmwtuAeoW
47
48
  ScriptCollection/TFCPS/TFCPS_CodeSpellCheck.py,sha256=zV62O7UiGjewx1yQUIWmUJ9D-5ipkV_f28uNhp-JWUs,5306
48
49
  ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=Rh7S84B8QbeWn5qJxRItoI7jquJYvFnNRi_WiX3FjWo,40414
49
50
  ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=CB0K8OZNtQESdS2pqX9RQqN-lI5M-cbmXYOR0rbuRlk,9741
50
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=5H8Artt40lrl3vvAHaZwiBPiL4ycHCZdk68s9mRIcI8,70924
51
+ ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=ECnHGnhqdYpOj-2mmGrSrqMQe1JZw2lr0M_Ui-ACBMA,59956
51
52
  ScriptCollection/TFCPS/TFCPS_CreateRelease.py,sha256=BVK3fpEyjcXYVHIb6YYLH7bWe6iKRaCtPrJ3tXAemYw,6582
52
53
  ScriptCollection/TFCPS/TFCPS_FindingsSorter.py,sha256=3ChuKf-Myr3sBtQhYGw6dHlk93HO6PAH-Jq1oPYdW1s,4804
53
54
  ScriptCollection/TFCPS/TFCPS_Generic.py,sha256=mRBcf7XuNWOSXbZYc3NuJow18AMZ30D6Z-BtyNYfNcM,2072
@@ -56,7 +57,7 @@ ScriptCollection/TFCPS/TFCPS_MergeToStable.py,sha256=GdpFcgIFRvdc1Q9b7uXlPlN99s7
56
57
  ScriptCollection/TFCPS/TFCPS_OCIImageSecretScan.py,sha256=Jb7AaMcJDW-gYEYBUVZAG3QLTTNWmXlvVEW-lZsykws,14821
57
58
  ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py,sha256=XLVsFTA1jvfDbZ7wYJgIdEByZhF_bYVbA5jkisuvK4k,2322
58
59
  ScriptCollection/TFCPS/TFCPS_RemoteBuild.py,sha256=2Tx5HLk1ZftBWDCTiGZdP2QiiGNyWsEuTCSt33dNQjc,23891
59
- ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=iiMy4LDY-bQk6PEi0onVhhKPsoWJHhgMaS5xDQVSlM8,160357
60
+ ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=J9dJrDXwTFLdCpnHV90890pkM5PARy_6U6hf8zSAjiE,160408
60
61
  ScriptCollection/TFCPS/TFCPS_VisualRegressionTests.py,sha256=F90CF5kLCgV_M3yN696xWzF1cglWLXecau9My2ZyGxw,42230
61
62
  ScriptCollection/TFCPS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
62
63
  ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py,sha256=BNZPMjGiWLMBH7QDtP06WEVLMnI38NLX4Dpd13bSbi4,16614
@@ -66,7 +67,8 @@ ScriptCollection/TFCPS/Docker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5N
66
67
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py,sha256=bT6Gd5pQpZCw4OQz6HWkPCSn5z__eUUEisABLDSxd0o,200
67
68
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py,sha256=QyjOfMY22JWCvKjMelHiDWbJiWqotOfebpJpgDUaoO4,237
68
69
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py,sha256=i0zEGehj0sttxjjZtoq2KFSKp_ulxVyWp_ZgAhIY_So,241
69
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=-vdQfhPcf69R80Lfsos3k04JMe2hUsClW9XUNu2KHv0,71170
70
+ ScriptCollection/TFCPS/DotNet/NuGetPackageSourceRegistration.py,sha256=E2TiFbwyPZvhozoWqLksc4uH50LbFAaVWOGfUo6for4,7050
71
+ ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=4oRPob7-8mC-8HqkGgyLn2s4t1n4iLOLp5tU6IOHFB4,65013
70
72
  ScriptCollection/TFCPS/DotNet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
71
73
  ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=9PMxJFB4nyKGzl916UOCvHB5toIyLcWk-ZAM8vPdLxw,64571
72
74
  ScriptCollection/TFCPS/Flutter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -82,8 +84,8 @@ ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=yyjWPMHQjPLkML
82
84
  ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
83
85
  ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py,sha256=ILVhJHvwG5TvPBBqSvyACYb7W9QnWrJpxKgHu8IQu4g,9249
84
86
  ScriptCollection/TFCPS/TypeScript/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
85
- scriptcollection-4.4.67.dist-info/METADATA,sha256=ItwhmyrDdan16Rtin5x-2mX33PfHWM2SNJr28T51hes,8032
86
- scriptcollection-4.4.67.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
87
- scriptcollection-4.4.67.dist-info/entry_points.txt,sha256=kI-HqAydo8a0FGjGeXh6i1pGU73Pit0xACdjxBxNkis,6254
88
- scriptcollection-4.4.67.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
89
- scriptcollection-4.4.67.dist-info/RECORD,,
87
+ scriptcollection-4.4.68.dist-info/METADATA,sha256=y_9VqqKqcCh3owvuuFyOoOxE9hi84h3bKbipoZKoyC8,8032
88
+ scriptcollection-4.4.68.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
89
+ scriptcollection-4.4.68.dist-info/entry_points.txt,sha256=ukXPbwM_VSzeSUOLRwWwq4aVAsfFKSzOjQnPkcJH59w,6333
90
+ scriptcollection-4.4.68.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
91
+ scriptcollection-4.4.68.dist-info/RECORD,,
@@ -60,6 +60,7 @@ scprintosname = ScriptCollection.Executables:PrintOSName
60
60
  scrais = ScriptCollection.Executables:RunAgentInSandbox
61
61
  screclaimspacefromdocker = ScriptCollection.Executables:ReclaimSpaceFromDocker
62
62
  scregexreplaceinfile = ScriptCollection.Executables:RegexReplaceInFile
63
+ scregisterpackagesources = ScriptCollection.Executables:RegisterPackageSources
63
64
  scremovefile = ScriptCollection.Executables:RemoveFile
64
65
  scremovefolder = ScriptCollection.Executables:RemoveFolder
65
66
  scremoveinvisiblecharactersfromfile = ScriptCollection.Executables:RemoveInvisibleCharactersFromFile