ScriptCollection 4.4.67__py3-none-any.whl → 4.4.69__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,8 +6,13 @@ import tempfile
6
6
  from enum import Enum
7
7
  from .GeneralUtilities import GeneralUtilities
8
8
  from .ScriptCollectionCore import ScriptCollectionCore
9
+ from .SCLog import LogLevel
9
10
  from .OCIImages.OCIImageManager import OCIImageManager
11
+ from .TFCPS.ContainerBuildEnvironment import ContainerBuildEnvironment
12
+ from .TFCPS.TFCPS_Tools_General import TFCPS_Tools_General
10
13
  from .AgentStatusReporting.AgentStatusReportingSession import AgentStatusReportingSession
14
+ from .AgentSandboxNetwork.SandboxNetworkConfiguration import SandboxNetworkConfiguration, SandboxNetworkMode
15
+ from .AgentSandboxNetwork.SandboxEgressProxy import SandboxEgressProxy
11
16
 
12
17
 
13
18
  class CodingAgent(Enum):
@@ -22,12 +27,31 @@ class AgentRunnerConfiguration:
22
27
  codingagent: CodingAgent = None
23
28
  image: str = None
24
29
  drop_capabilities: bool = None
30
+ provide_build_environment: bool = None
31
+ allowed_domains: list[str] = None
32
+ egress_proxy_image: str = None
25
33
 
26
34
  @GeneralUtilities.check_arguments
27
35
  def __init__(self):
28
36
  self.codingagent = None
29
37
  self.image = None
30
38
  self.drop_capabilities = None
39
+ self.provide_build_environment = None
40
+ # None means that no allowlist is defined (unrestricted network-access); an empty list is an allowlist without domain (no network-access).
41
+ self.allowed_domains = None
42
+ self.egress_proxy_image = None
43
+
44
+
45
+ class SandboxBuildEnvironment:
46
+ """The build-environment (see ContainerBuildEnvironment) prepared for one start of the sandbox: the docker-arguments of its mounts
47
+ and its environment-variables. The values of the environment-variables are typically secrets, so only their names may become
48
+ docker-arguments; the values are given to the docker-client through its own environment."""
49
+ volume_arguments: list[str] = None
50
+ environment_variables: dict[str, str] = None
51
+
52
+ def __init__(self, volume_arguments: list[str], environment_variables: dict[str, str]):
53
+ self.volume_arguments = volume_arguments
54
+ self.environment_variables = environment_variables
31
55
 
32
56
 
33
57
  class AgentRunner:
@@ -48,17 +72,42 @@ class AgentRunner:
48
72
  GeneralUtilities.assert_condition(interactive or prompt is not None, "A prompt is required when the agent is not run in interactive mode.")
49
73
  self.__assert_is_git_root_dir(repository)
50
74
  agentRunnerConfiguration: AgentRunnerConfiguration = self.__load_configuration(repository, configuration_overrides)
75
+ # The network-configuration is validated before anything is prepared or started, so an invalid allowlist is rejected without side-effects.
76
+ network_configuration = SandboxNetworkConfiguration(agentRunnerConfiguration.allowed_domains, agentRunnerConfiguration.egress_proxy_image)
51
77
  initialization_script = self.__get_initialization_script(repository)
52
78
  additional_folder_volume_arguments = self.__get_additional_folder_volume_arguments(additional_folders)
53
79
  if mount_docker_socket:
54
80
  self.__print_docker_socket_warning()
55
81
  additional_folder_volume_arguments = additional_folder_volume_arguments + ["--volume", "/var/run/docker.sock:/var/run/docker.sock"]
56
- return self.__run_sandbox(repository, agentRunnerConfiguration, prompt, interactive, initialization_script, additional_folder_volume_arguments)
82
+ build_environment: SandboxBuildEnvironment = None
83
+ if agentRunnerConfiguration.provide_build_environment:
84
+ build_environment = self.__prepare_build_environment(repository)
85
+ return self.__run_sandbox(repository, agentRunnerConfiguration, prompt, interactive, initialization_script, additional_folder_volume_arguments, build_environment, network_configuration)
57
86
 
58
87
  @GeneralUtilities.check_arguments
59
88
  def __print_docker_socket_warning(self) -> None:
60
89
  GeneralUtilities.write_message_to_stderr("Warning: Mounting the docker-socket of the host into the sandbox is dangerous because it makes the sandbox ineffective.")
61
90
 
91
+ @GeneralUtilities.check_arguments
92
+ def __prepare_build_environment(self, repository: str) -> SandboxBuildEnvironment:
93
+ # The sandbox gets the same build-environment as a build-container of "scbuildcodeunits -c" (see ContainerBuildEnvironment), so the agent can
94
+ # build the repository and restore its dependencies (also from private package-sources). The host-hook runs first, like for "scbuildcodeunits -c",
95
+ # because it can create the files the values of the environment-variables are read from. A sandbox has no target-environment of its own;
96
+ # "Development" is passed because the hook expects a valid one and an agent develops the repository.
97
+ sc = ScriptCollectionCore()
98
+ container_build_environment = ContainerBuildEnvironment(sc, TFCPS_Tools_General(sc), repository)
99
+ self.__print_build_environment_warning(sc.get_tfcps_configuration_folder())
100
+ verbosity = LogLevel.Debug if self.verbose else LogLevel.Information
101
+ container_build_environment.run_host_preparation_hook(["--repository", repository, "--targetenvironmenttype", "Development", "--verbosity", str(int(verbosity))])
102
+ volume_arguments: list[str] = []
103
+ for mount in container_build_environment.get_mounts():
104
+ volume_arguments = volume_arguments + ["--volume", f"{mount.source.replace(os.sep, '/')}:{mount.target}{':ro' if mount.read_only else ''}"]
105
+ return SandboxBuildEnvironment(volume_arguments, container_build_environment.get_environment_variables())
106
+
107
+ @GeneralUtilities.check_arguments
108
+ def __print_build_environment_warning(self, tfcps_configuration_folder: str) -> None:
109
+ GeneralUtilities.write_message_to_stderr(f"Warning: The build-environment is provided to the sandbox. The agent can read every secret of \"{tfcps_configuration_folder}\" and the values of the required environment-variables of the repository.")
110
+
62
111
  @GeneralUtilities.check_arguments
63
112
  def __get_additional_folder_volume_arguments(self, additional_folders: list[str]) -> list[str]:
64
113
  # Every additional folder is mounted under "/Workspace/Other/AdditionalResources/<folder-name>" (preserving the folder-name).
@@ -127,10 +176,35 @@ class AgentRunner:
127
176
 
128
177
  for name, value in [("codingagent", agentRunnerConfiguration.codingagent), ("image", agentRunnerConfiguration.image), ("drop_capabilities", agentRunnerConfiguration.drop_capabilities)]:
129
178
  GeneralUtilities.assert_condition(value is not None, f"The value '{name}' is neither given as argument nor defined in '{agent_sandbox_configuration_file}'.")
179
+
180
+ # Unlike the values above, provide_build_environment is not required: the build-environment is opt-in, so it is only provided if it is enabled
181
+ # explicitly. Requiring it would break every existing configuration-file.
182
+ if configuration_overrides.provide_build_environment is not None:
183
+ agentRunnerConfiguration.provide_build_environment = configuration_overrides.provide_build_environment
184
+ elif config_parser.has_option(section, "provide_build_environment"):
185
+ agentRunnerConfiguration.provide_build_environment = config_parser.getboolean(section, "provide_build_environment")
186
+ else:
187
+ agentRunnerConfiguration.provide_build_environment = False
188
+
189
+ # The allowlist and egress_proxy_image are not required either: without allowlist the network-access is unrestricted as before. The allowlist is
190
+ # the own section "AllowedDomains" with one entry per domain, whose value is a comment for the reader of the file (for example why the domain is
191
+ # needed); the comment does not influence the behavior. A missing section (no allowlist) and an empty section (allowlist without domain, which
192
+ # blocks the network-access) are distinguished deliberately. A domain which is listed twice is rejected by the parser.
193
+ allowed_domains_section = "AllowedDomains"
194
+ if configuration_overrides.allowed_domains is not None:
195
+ agentRunnerConfiguration.allowed_domains = configuration_overrides.allowed_domains
196
+ elif config_parser.has_section(allowed_domains_section):
197
+ agentRunnerConfiguration.allowed_domains = list(config_parser[allowed_domains_section].keys())
198
+
199
+ if configuration_overrides.egress_proxy_image is not None:
200
+ agentRunnerConfiguration.egress_proxy_image = configuration_overrides.egress_proxy_image
201
+ elif config_parser.has_option(section, "egress_proxy_image"):
202
+ agentRunnerConfiguration.egress_proxy_image = config_parser.get(section, "egress_proxy_image")
130
203
  return agentRunnerConfiguration
131
204
 
132
205
  @GeneralUtilities.check_arguments
133
- def __run_sandbox(self, repository: str, agentRunnerConfiguration: AgentRunnerConfiguration, prompt: str, interactive: bool, initialization_script: str, additional_folder_volume_arguments: list[str]) -> int:
206
+ def __run_sandbox(self, repository: str, agentRunnerConfiguration: AgentRunnerConfiguration, prompt: str, interactive: bool, initialization_script: str, additional_folder_volume_arguments: list[str], build_environment: SandboxBuildEnvironment, network_configuration: SandboxNetworkConfiguration) -> int:
207
+ # build_environment is None if the build-environment is not provided.
134
208
  # The container always runs Linux.
135
209
  # The repository is the folder given by the caller (which was asserted to be the git-root-directory before).
136
210
  # It is mounted into the container under "/Workspace/<repository-name>" (preserving the folder-name) and used as working-directory of the agent.
@@ -142,10 +216,14 @@ class AgentRunner:
142
216
  status_session = AgentStatusReportingSession.try_create(dict(os.environ), f"scrais-{agentRunnerConfiguration.codingagent.value}", self.verbose)
143
217
  status_start_line = None if status_session is None else status_session.get_start_report_line(prompt is not None)
144
218
  # The entrypoint is always used: if an initialization-script is present it runs it first, and then always starts the agent.
145
- entrypoint_script_content = self.__get_entrypoint_script_content(agent_command, initialization_script, status_start_line)
219
+ entrypoint_script_content = self.__get_entrypoint_script_content(agent_command, initialization_script, status_start_line, build_environment is not None, container_workspace_folder)
146
220
  with tempfile.NamedTemporaryFile("w", suffix=".sh", delete=False, newline="\n", encoding="utf-8") as entrypoint_file_handle:
147
221
  host_entrypoint_file = entrypoint_file_handle.name
148
222
  entrypoint_file_handle.write(entrypoint_script_content)
223
+ network_description = network_configuration.get_description()
224
+ if network_description is not None:
225
+ GeneralUtilities.write_message_to_stderr(network_description)
226
+ egress_proxy: SandboxEgressProxy = None
149
227
  try:
150
228
  # Make the entrypoint-script executable (chmod +x). On Linux-/macOS-hosts the execute-bit propagates into the container; on Windows-hosts this has no effect, which is why the script is additionally invoked via "sh" below.
151
229
  os.chmod(host_entrypoint_file, 0o755)
@@ -166,19 +244,53 @@ class AgentRunner:
166
244
  docker_command = docker_command + ["--volume", f"{initialization_script_absolute_path.replace(os.sep, '/')}:/Workspace/Temp/InitializationScript.sh:ro"]
167
245
  if status_session is not None:
168
246
  docker_command = docker_command + status_session.get_volume_arguments()
247
+ if network_configuration.get_mode() == SandboxNetworkMode.Allowlist:
248
+ # The egress-proxy is assigned before it is started, so the finally-block below also removes what a failing start created.
249
+ egress_proxy = SandboxEgressProxy(network_configuration.allowed_domains, self.__get_image_to_use(network_configuration.egress_proxy_image, host_repository_folder), self.verbose)
250
+ egress_proxy.start()
251
+ docker_command = docker_command + self.__get_network_arguments(network_configuration, egress_proxy)
252
+ # The environment of the docker-client; it differs from the one of this process only if the build-environment is provided.
253
+ docker_client_environment: dict[str, str] = None
254
+ if build_environment is not None:
255
+ docker_command = docker_command + build_environment.volume_arguments
256
+ # Only the names are passed as arguments ("--env <name>" takes the value from the environment of the docker-client), because
257
+ # arguments are visible in the process-list of the host and the values are typically secrets.
258
+ for environment_variable_name in build_environment.environment_variables:
259
+ docker_command = docker_command + ["--env", environment_variable_name]
260
+ docker_client_environment = {**os.environ, **build_environment.environment_variables}
169
261
  image = self.__get_image_to_use(agentRunnerConfiguration.image, host_repository_folder)
170
- docker_command = docker_command + ["--workdir", container_workspace_folder, "--entrypoint", "sh", image, "/Workspace/Entrypoint.sh"]
262
+ docker_command = docker_command + ["--workdir", container_workspace_folder]
263
+ if build_environment is None:
264
+ docker_command = docker_command + ["--entrypoint", "sh", image, "/Workspace/Entrypoint.sh"]
265
+ else:
266
+ # With the build-environment the entrypoint of the image is kept and runs the generated entrypoint as its command: the entrypoint of the
267
+ # SCBuilder-image installs the trusted root-CA-certificates of the build-environment (and then runs "exec \"$@\""), exactly like in a
268
+ # build-container of "scbuildcodeunits -c". Without the build-environment the entrypoint of the image is replaced as before.
269
+ docker_command = docker_command + [image, "sh", "/Workspace/Entrypoint.sh"]
171
270
  # Nothing is installed in the container; the coding-agent-CLI is assumed to be available in the image.
172
271
  # subprocess.run blocks until the container has exited, which happens when the agent is finished (or was terminated by the user in interactive mode).
173
272
  # The exit-code of the container is returned (and not checked here), so that the caller can pass it through.
174
273
  if status_session is not None:
175
274
  status_session.start()
176
- return subprocess.run(docker_command, check=False).returncode
275
+ return subprocess.run(docker_command, check=False, env=docker_client_environment).returncode
177
276
  finally:
178
277
  if status_session is not None:
179
278
  status_session.stop()
279
+ if egress_proxy is not None:
280
+ egress_proxy.stop()
180
281
  GeneralUtilities.ensure_file_does_not_exist(host_entrypoint_file)
181
282
 
283
+ @GeneralUtilities.check_arguments
284
+ def __get_network_arguments(self, network_configuration: SandboxNetworkConfiguration, egress_proxy: SandboxEgressProxy) -> list[str]:
285
+ # egress_proxy is the started egress-proxy in mode allowlist and None otherwise.
286
+ # Without allowlist no argument is added, so the sandbox is attached to the default bridge-network as before.
287
+ mode = network_configuration.get_mode()
288
+ if mode == SandboxNetworkMode.NoNetwork:
289
+ return ["--network", "none"]
290
+ if mode == SandboxNetworkMode.Allowlist:
291
+ return egress_proxy.get_sandbox_docker_arguments()
292
+ return []
293
+
182
294
  @GeneralUtilities.check_arguments
183
295
  def __get_image_to_use(self, configured_image: str, repository: str) -> str:
184
296
  # If the configured image is an image-name defined in "<repository>/.ScriptCollection/OCIImages/ImageDefinition.csv" then the image (address including tag) is resolved
@@ -193,7 +305,7 @@ class AgentRunner:
193
305
  return configured_image
194
306
 
195
307
  @GeneralUtilities.check_arguments
196
- def __get_entrypoint_script_content(self, agent_command: list[str], initialization_script: str, status_start_line: str) -> str:
308
+ def __get_entrypoint_script_content(self, agent_command: list[str], initialization_script: str, status_start_line: str, provide_build_environment: bool, container_workspace_folder: str) -> str:
197
309
  # The entrypoint-script is always used as the container's entrypoint (located at "/Workspace/Entrypoint.sh").
198
310
  # It is regenerated before every container-start, so the decision whether the initialization-script is called is made here in python
199
311
  # (no runtime-check in the script): if an initialization-script is present its call is included (and the script is removed afterwards),
@@ -201,6 +313,12 @@ class AgentRunner:
201
313
  # status_start_line is None if the status is not reported; it is the only runtime-check, because only the container knows whether its image supports the status-report.
202
314
  agent_command_line = shlex.join(agent_command)
203
315
  lines = ["#!/bin/sh", "set -e"]
316
+ if provide_build_environment:
317
+ # The declared package-sources are registered before the initialization-script, so that script and the agent can already use them directly
318
+ # (for example with "dotnet restore") and not only through "scbuildcodeunits". The command comes from the ScriptCollection of the image; an
319
+ # image without it is rejected instead of starting the agent without the package-sources.
320
+ lines.append('if ! command -v scregisterpackagesources >/dev/null 2>&1; then echo "The build-environment requires the command \\"scregisterpackagesources\\" of ScriptCollection, which is not available in this image. Use a newer image." >&2; exit 1; fi')
321
+ lines.append(f"scregisterpackagesources -r {shlex.quote(container_workspace_folder)}")
204
322
  if initialization_script is not None:
205
323
  # A line is written to stdout before the initialization-script runs, so it is unambiguous in the container-output that an initialization-script was configured and executed (without it one cannot tell whether the agent started unconfigured because no script was declared or because a declared script did nothing).
206
324
  lines.append('echo "Running initialization-script for the agent-sandbox..."')
@@ -0,0 +1,126 @@
1
+ import os
2
+ import subprocess
3
+ import tempfile
4
+ import uuid
5
+ from ..GeneralUtilities import GeneralUtilities
6
+
7
+
8
+ class SandboxEgressProxy:
9
+ """Enforces an allowlist of domains for the agent-sandbox. For one run it creates an internal docker-network (which has no route to the outside)
10
+ and a Squid-container which is attached to this network and to the default bridge-network, so only the proxy can reach the outside.
11
+ The proxy only allows CONNECT to port 443 of the allowed domains (TLS is tunneled, not intercepted).
12
+ The sandbox is attached only to the internal network, gets the proxy announced by the usual proxy-environment-variables and has no DNS-server
13
+ for external names, so the restriction is enforced outside of the sandbox and can not be lifted from inside of it."""
14
+ proxy_port: int = 3128
15
+ squid_configuration_file_in_container: str = "/etc/squid/squid.conf"
16
+ access_log_file_in_container: str = "/var/log/squid/access.log"
17
+ __allowed_domains: list[str] = None
18
+ __image: str = None
19
+ __verbose: bool = None
20
+ __network_name: str = None
21
+ __proxy_container_name: str = None
22
+ __configuration_file: str = None
23
+ __network_was_created: bool = None
24
+ __proxy_container_was_created: bool = None
25
+
26
+ @GeneralUtilities.check_arguments
27
+ def __init__(self, allowed_domains: list[str], image: str, verbose: bool):
28
+ # allowed_domains must already be validated (see SandboxNetworkConfiguration). image is the image-address including tag.
29
+ GeneralUtilities.assert_condition(len(allowed_domains) > 0, "The egress-proxy requires at least one allowed domain.")
30
+ self.__allowed_domains = allowed_domains
31
+ self.__image = image
32
+ self.__verbose = verbose
33
+ # The run-id makes the names unique, so parallel runs do not collide. The prefix "scrais-" makes leftovers of a killed run recognizable.
34
+ run_id = uuid.uuid4().hex[:12]
35
+ self.__network_name = f"scrais-{run_id}"
36
+ self.__proxy_container_name = f"scrais-proxy-{run_id}"
37
+ self.__network_was_created = False
38
+ self.__proxy_container_was_created = False
39
+
40
+ @staticmethod
41
+ @GeneralUtilities.check_arguments
42
+ def get_squid_configuration(allowed_domains: list[str]) -> str:
43
+ # "dstdomain" uses the same syntax as the allowlist (a leading dot includes the subdomains). "-n" disables reverse-DNS-lookups, so a CONNECT to an
44
+ # IP-address can not match because of its reverse name. Plain HTTP and every port except 443 are denied. Nothing is cached. The access-log is
45
+ # written unbuffered to a file in the container, so the denied requests can be read with "docker exec" (the output of Squid does not reach
46
+ # "docker logs" when the image runs Squid under a service-manager like pebble). The pinger is disabled because it needs the capability NET_RAW,
47
+ # which the proxy-container does not get.
48
+ lines = [
49
+ f"http_port {SandboxEgressProxy.proxy_port}",
50
+ f"acl allowed_domains dstdomain -n {' '.join(allowed_domains)}",
51
+ "acl https_port port 443",
52
+ "http_access deny !CONNECT",
53
+ "http_access deny CONNECT !https_port",
54
+ "http_access allow CONNECT allowed_domains",
55
+ "http_access deny all",
56
+ "cache deny all",
57
+ f"access_log stdio:{SandboxEgressProxy.access_log_file_in_container}",
58
+ "buffered_logs off",
59
+ "pinger_enable off",
60
+ ]
61
+ return "\n".join(lines) + "\n"
62
+
63
+ @GeneralUtilities.check_arguments
64
+ def start(self) -> None:
65
+ # stop() must be called afterwards in any case (also if this function fails), because it removes everything which was created until then.
66
+ with tempfile.NamedTemporaryFile("w", suffix=".conf", delete=False, newline="\n", encoding="utf-8") as configuration_file_handle:
67
+ self.__configuration_file = configuration_file_handle.name
68
+ configuration_file_handle.write(SandboxEgressProxy.get_squid_configuration(self.__allowed_domains))
69
+ # The configuration-file must be readable by the (unprivileged) user of squid in the container.
70
+ os.chmod(self.__configuration_file, 0o644)
71
+ self.__run_docker(["network", "create", "--internal", self.__network_name])
72
+ self.__network_was_created = True
73
+ # The container is marked as created before "docker run", because a failing start can leave a created container behind.
74
+ self.__proxy_container_was_created = True
75
+ # Squid runs as an unprivileged user in the image and needs no capability, so the proxy-container gets none.
76
+ self.__run_docker(["run", "--detach", "--name", self.__proxy_container_name, "--network", self.__network_name, "--cap-drop", "all", "--security-opt", "no-new-privileges:true", "--volume", f"{self.__configuration_file.replace(os.sep, '/')}:{SandboxEgressProxy.squid_configuration_file_in_container}:ro", self.__image])
77
+ self.__run_docker(["network", "connect", "bridge", self.__proxy_container_name])
78
+
79
+ @GeneralUtilities.check_arguments
80
+ def get_sandbox_docker_arguments(self) -> list[str]:
81
+ # The proxy is announced in both spellings because tools differ in which one they read. The address is no secret, so the values are passed as arguments.
82
+ # "--dns 127.0.0.1" lets the resolution of external names fail (nothing listens there); the name of the proxy-container is still resolved by the
83
+ # embedded DNS of docker.
84
+ proxy_address = f"http://{self.__proxy_container_name}:{SandboxEgressProxy.proxy_port}"
85
+ result = ["--network", self.__network_name, "--dns", "127.0.0.1"]
86
+ for environment_variable_name in ["HTTPS_PROXY", "HTTP_PROXY", "https_proxy", "http_proxy"]:
87
+ result = result + ["--env", f"{environment_variable_name}={proxy_address}"]
88
+ return result
89
+
90
+ @GeneralUtilities.check_arguments
91
+ def stop(self) -> None:
92
+ # Removes everything which start() created. A failure of the removal does not hide the result of the run; it is reported with the name of the
93
+ # resource, so it can be removed manually.
94
+ if self.__proxy_container_was_created:
95
+ if self.__verbose:
96
+ self.__print_denied_requests()
97
+ self.__remove(["rm", "--force", self.__proxy_container_name], f"proxy-container '{self.__proxy_container_name}'")
98
+ self.__proxy_container_was_created = False
99
+ if self.__network_was_created:
100
+ self.__remove(["network", "rm", self.__network_name], f"network '{self.__network_name}'")
101
+ self.__network_was_created = False
102
+ if self.__configuration_file is not None:
103
+ GeneralUtilities.ensure_file_does_not_exist(self.__configuration_file)
104
+ self.__configuration_file = None
105
+
106
+ @GeneralUtilities.check_arguments
107
+ def __print_denied_requests(self) -> None:
108
+ access_log = subprocess.run(["docker", "exec", self.__proxy_container_name, "cat", SandboxEgressProxy.access_log_file_in_container], capture_output=True, text=True, check=False)
109
+ denied_requests = [line for line in access_log.stdout.splitlines() if "TCP_DENIED" in line]
110
+ if len(denied_requests) == 0:
111
+ GeneralUtilities.write_message_to_stderr("The egress-proxy denied no request.")
112
+ else:
113
+ GeneralUtilities.write_message_to_stderr("The egress-proxy denied the following requests:")
114
+ for denied_request in denied_requests:
115
+ GeneralUtilities.write_message_to_stderr(f" {denied_request}")
116
+
117
+ @GeneralUtilities.check_arguments
118
+ def __remove(self, docker_arguments: list[str], resource_description: str) -> None:
119
+ result = subprocess.run(["docker"] + docker_arguments, capture_output=True, text=True, check=False)
120
+ if result.returncode != 0:
121
+ GeneralUtilities.write_message_to_stderr(f"Warning: The {resource_description} of the sandbox could not be removed and must be removed manually: {result.stderr.strip()}")
122
+
123
+ @GeneralUtilities.check_arguments
124
+ def __run_docker(self, docker_arguments: list[str]) -> None:
125
+ result = subprocess.run(["docker"] + docker_arguments, capture_output=True, text=True, check=False)
126
+ GeneralUtilities.assert_condition(result.returncode == 0, f"\"docker {' '.join(docker_arguments)}\" failed with exit-code {result.returncode}: {result.stderr.strip()}")
@@ -0,0 +1,95 @@
1
+ import ipaddress
2
+ import re
3
+ from enum import Enum
4
+ from ..GeneralUtilities import GeneralUtilities
5
+
6
+
7
+ class SandboxNetworkMode(Enum):
8
+ Unrestricted = "unrestricted"
9
+ Allowlist = "allowlist"
10
+ NoNetwork = "none"
11
+
12
+
13
+ class SandboxNetworkConfiguration:
14
+ """The network-access of the agent-sandbox. Without allowlist (allowed_domains is None) the network-access is unrestricted. With an allowlist
15
+ everything except the listed domains is blocked: an allowlist with domains allows only these domains via HTTPS (enforced by an egress-proxy),
16
+ an allowlist without domain blocks the network-access completely.
17
+ An entry without leading dot allows exactly this host, an entry with leading dot allows the domain and all of its subdomains."""
18
+ __label_pattern = re.compile(r"^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$")
19
+ allowed_domains: list[str] = None
20
+ egress_proxy_image: str = None
21
+
22
+ @GeneralUtilities.check_arguments
23
+ def __init__(self, allowed_domains: list[str], egress_proxy_image: str):
24
+ # allowed_domains is None if no allowlist is defined. egress_proxy_image is only required if the allowlist contains at least one domain.
25
+ # Invalid configurations are rejected here, so nothing is started for them.
26
+ if allowed_domains is not None:
27
+ allowed_domains = [allowed_domain.lower() for allowed_domain in allowed_domains]
28
+ for allowed_domain in allowed_domains:
29
+ SandboxNetworkConfiguration.__assert_entry_is_valid(allowed_domain)
30
+ SandboxNetworkConfiguration.__assert_no_entry_is_redundant(allowed_domains)
31
+ self.allowed_domains = allowed_domains
32
+ self.egress_proxy_image = egress_proxy_image
33
+ if self.get_mode() == SandboxNetworkMode.Allowlist:
34
+ GeneralUtilities.assert_condition(egress_proxy_image is not None, "The value 'egress_proxy_image' is required because the allowlist (section 'AllowedDomains' or argument '--alloweddomain') contains at least one domain, but it is neither given as argument nor defined in the configuration-file.")
35
+
36
+ @GeneralUtilities.check_arguments
37
+ def get_mode(self) -> SandboxNetworkMode:
38
+ if self.allowed_domains is None:
39
+ return SandboxNetworkMode.Unrestricted
40
+ if len(self.allowed_domains) == 0:
41
+ return SandboxNetworkMode.NoNetwork
42
+ return SandboxNetworkMode.Allowlist
43
+
44
+ @GeneralUtilities.check_arguments
45
+ def get_description(self) -> str:
46
+ # The description of the restriction which is shown at start. It is None for the unrestricted network-access, so the output stays as before.
47
+ mode = self.get_mode()
48
+ if mode == SandboxNetworkMode.NoNetwork:
49
+ return "The sandbox has no network access."
50
+ if mode == SandboxNetworkMode.Allowlist:
51
+ return f"Network access of the sandbox is restricted to {', '.join(self.allowed_domains)} via HTTPS."
52
+ return None
53
+
54
+ @staticmethod
55
+ @GeneralUtilities.check_arguments
56
+ def __assert_entry_is_valid(entry: str) -> None:
57
+ host = entry.removeprefix(".")
58
+ invalid_entry_message = f"The allowlist-entry '{entry}' is invalid. Expected a hostname like 'example.com' (exactly this host) or '.example.com' (the domain and all of its subdomains) without scheme, port, path or wildcard."
59
+ GeneralUtilities.assert_condition(not SandboxNetworkConfiguration.__is_ip_address(host), f"The allowlist-entry '{entry}' is an IP-address, but only hostnames are allowed.")
60
+ labels = host.split(".")
61
+ GeneralUtilities.assert_condition(len(host) <= 253 and len(labels) >= 2, invalid_entry_message)
62
+ for label in labels:
63
+ GeneralUtilities.assert_condition(SandboxNetworkConfiguration.__label_pattern.match(label) is not None, invalid_entry_message)
64
+
65
+ @staticmethod
66
+ @GeneralUtilities.check_arguments
67
+ def __is_ip_address(host: str) -> bool:
68
+ try:
69
+ ipaddress.ip_address(host.strip("[]"))
70
+ return True
71
+ except ValueError:
72
+ return False
73
+
74
+ @staticmethod
75
+ @GeneralUtilities.check_arguments
76
+ def __assert_no_entry_is_redundant(allowed_domains: list[str]) -> None:
77
+ # A redundant entry is rejected because it shows a misunderstanding of the matching (and the egress-proxy would complain about overlapping entries).
78
+ for index, entry in enumerate(allowed_domains):
79
+ for other_index, other_entry in enumerate(allowed_domains):
80
+ if index == other_index:
81
+ continue
82
+ if entry == other_entry and other_index > index:
83
+ # A duplicate is reported only once (at its second occurrence).
84
+ continue
85
+ if SandboxNetworkConfiguration.__covers(other_entry, entry):
86
+ raise ValueError(f"The allowlist-entry '{entry}' is already covered by the allowlist-entry '{other_entry}'.")
87
+
88
+ @staticmethod
89
+ @GeneralUtilities.check_arguments
90
+ def __covers(covering_entry: str, covered_entry: str) -> bool:
91
+ # Returns whether everything which is allowed by covered_entry is also allowed by covering_entry.
92
+ if covering_entry.startswith("."):
93
+ covered_host = covered_entry.removeprefix(".")
94
+ return covered_host == covering_entry[1:] or covered_host.endswith(covering_entry)
95
+ return covering_entry == covered_entry
File without changes
@@ -14,6 +14,7 @@ from .SCLog import LogLevel, SCLog
14
14
  from .TFCPS.TFCPS_CodeUnit_BuildCodeUnits import TFCPS_CodeUnit_BuildCodeUnits
15
15
  from .TFCPS.TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
16
16
  from .TFCPS.TFCPS_Tools_General import TFCPS_Tools_General
17
+ from .TFCPS.DotNet.NuGetPackageSourceRegistration import NuGetPackageSourceRegistration
17
18
  from .OCIImages.OCIImageManager import OCIImageManager
18
19
  from .AgentRunner import AgentRunner, AgentRunnerConfiguration, CodingAgent
19
20
 
@@ -28,6 +29,9 @@ def RunAgentInSandbox() -> int:
28
29
  parser.add_argument('--image', default=None, required=False, help='OCI-image which should be used for the sandbox: either an image-name defined in ".ScriptCollection/OCIImages/ImageDefinition.csv" or an image-address including tag. Overrides the value from ".ScriptCollection/AgentSandboxConfiguration.ini".')
29
30
  parser.add_argument('-s','--dropcapabilities', default=None, required=False, choices=["true", "false"], help='Defines whether all capabilities should be dropped in the sandbox. Overrides the value from ".ScriptCollection/AgentSandboxConfiguration.ini".')
30
31
  parser.add_argument('-a', '--additionalfolder', action='append', default=None, required=False, help='Folder which should additionally be mounted (read-write) into the sandbox under "/Workspace/Other/AdditionalResources/<folder-name>". Can be given multiple times.')
32
+ parser.add_argument('-b','--providebuildenvironment', default=None, required=False, choices=["true", "false"], help='Defines whether the sandbox gets the same build-environment as a build-container of "scbuildcodeunits -c" (required environment-variables, configuration-folder, custom scripts, image-registries, trusted root-CAs and registered package-sources). The agent can then read every secret of the TFCPS-configuration-folder. Overrides the value from ".ScriptCollection/AgentSandboxConfiguration.ini".')
33
+ parser.add_argument('--alloweddomain', action='append', default=None, required=False, help='Domain which the sandbox may reach via HTTPS: "example.com" allows exactly this host, ".example.com" the domain and all of its subdomains. If an allowlist is defined, everything except the allowlist is blocked. Can be given multiple times. If given at least once, it replaces the section "AllowedDomains" of ".ScriptCollection/AgentSandboxConfiguration.ini" completely.')
34
+ parser.add_argument('--egressproxyimage', default=None, required=False, help='Squid-image of the egress-proxy which enforces the allowlist: either an image-name defined in ".ScriptCollection/OCIImages/ImageDefinition.csv" or an image-address including tag. Required if the allowlist contains at least one domain. Overrides the value from ".ScriptCollection/AgentSandboxConfiguration.ini".')
31
35
  parser.add_argument('-d','--mountdockersocket', action='store_true', required=False, default=False, help='Mounts the docker-socket of the host (/var/run/docker.sock) into the sandbox so that the agent can use docker. DANGEROUS: this effectively gives the agent administrator-access to the docker-daemon of the host and makes the restrictions of the sandbox ineffective. Can only be set via this argument (not in the configuration-file); a warning is printed to stderr.')
32
36
  args = parser.parse_args()
33
37
  repo=os.getcwd()
@@ -40,6 +44,10 @@ def RunAgentInSandbox() -> int:
40
44
  configuration_overrides.image = args.image
41
45
  if args.dropcapabilities is not None:
42
46
  configuration_overrides.drop_capabilities = args.dropcapabilities == "true"
47
+ if args.providebuildenvironment is not None:
48
+ configuration_overrides.provide_build_environment = args.providebuildenvironment == "true"
49
+ configuration_overrides.allowed_domains = args.alloweddomain
50
+ configuration_overrides.egress_proxy_image = args.egressproxyimage
43
51
  additional_folders: list[str] = []
44
52
  if args.additionalfolder is not None:
45
53
  additional_folders = [GeneralUtilities.resolve_relative_path(additional_folder, os.getcwd()) for additional_folder in args.additionalfolder]
@@ -1352,6 +1360,16 @@ def ShowProjectVersion() -> int:
1352
1360
  return 0
1353
1361
 
1354
1362
 
1363
+ def RegisterPackageSources() -> int:
1364
+ parser = argparse.ArgumentParser(description="Registers every package-source which is declared in the environment of this process by the environment-variables \"Dependency_<technology>_<name>_URL\" (and optionally \"_Username\" and \"_Password\") with its credentials on this machine, exactly like a codeunit-build does it. Package-sources which are not declared are not changed.")
1365
+ parser.add_argument('-r', '--folder', required=False, default=None, help="Folder in which the tools of the technologies (for example the dotnet-cli) are run. Defaults to the current working directory.")
1366
+ args = parser.parse_args()
1367
+ folder = GeneralUtilities.resolve_relative_path(args.folder, os.getcwd()) if args.folder is not None else os.getcwd()
1368
+ sc: ScriptCollectionCore = ScriptCollectionCore()
1369
+ NuGetPackageSourceRegistration(sc, TFCPS_Tools_General(sc)).register_declared_package_sources(folder)
1370
+ return 0
1371
+
1372
+
1355
1373
  def RunCommandInFolder() -> int:
1356
1374
  parser = argparse.ArgumentParser(description="Runs a command in the base-folder if (and only if) the actual folder is located inside the base-folder and not inside an excluded folder. The command is always executed in the base-folder; the actual folder is only passed to the command by using the placeholder \"{actual_folder}\" in the arguments (for example: -c git -a \"-C {actual_folder} status\").")
1357
1375
  parser.add_argument('-b', '--basefolder', required=True, help="Folder in which the command is executed.")
@@ -40,7 +40,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
40
40
  from .ProgramRunnerPopen import ProgramRunnerPopen
41
41
  from .SCLog import SCLog, LogLevel
42
42
 
43
- version = "4.4.67"
43
+ version = "4.4.69"
44
44
  __version__ = version
45
45
 
46
46
  class VSCodeWorkspaceShellTask:
@@ -0,0 +1,165 @@
1
+ import os
2
+ import re
3
+ from ..GeneralUtilities import GeneralUtilities
4
+ from ..ScriptCollectionCore import ScriptCollectionCore
5
+ from .TFCPS_Tools_General import TFCPS_Tools_General
6
+
7
+
8
+ class ContainerMount:
9
+ """A file or folder of the host which is made available in a container."""
10
+ source: str = None
11
+ target: str = None
12
+ read_only: bool = None
13
+
14
+ def __init__(self, source: str, target: str, read_only: bool):
15
+ self.source = source
16
+ self.target = target
17
+ self.read_only = read_only
18
+
19
+
20
+ class ContainerBuildEnvironment:
21
+ """The build-environment which a container gets from the host on which it is started: the mounts, the environment-variables and the
22
+ host-preparation-hook. It is the one place which defines it, so that every container which builds a repository (the build-container of
23
+ 'scbuildcodeunits -c' and the agent-sandbox of 'scrais') gets the same one. How the mounts are applied (bind-mount or copy) is decided by
24
+ the caller, because that depends on where the caller itself runs. See the article "Configuration-folder" of the reference for what is
25
+ mounted and why."""
26
+
27
+ __sc: ScriptCollectionCore = None
28
+ __tfcps_tools_general: TFCPS_Tools_General = None
29
+ __repository: str = None
30
+
31
+ def __init__(self, sc: ScriptCollectionCore, tfcps_tools_general: TFCPS_Tools_General, repository: str):
32
+ self.__sc = sc
33
+ self.__tfcps_tools_general = tfcps_tools_general
34
+ self.__repository = repository
35
+
36
+ @GeneralUtilities.check_arguments
37
+ def run_host_preparation_hook(self, build_arguments: list[str]) -> None:
38
+ """Runs the optional user-specific script which prepares this host for a container-start (for example to log in to the registry the
39
+ image is pulled from). It has to run before get_environment_variables, so it can also create the files their values are read from."""
40
+ self.__tfcps_tools_general.run_custom_script_if_available(self.__tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScriptForContainer.py"), build_arguments)
41
+
42
+ @GeneralUtilities.check_arguments
43
+ def get_mounts(self) -> list[ContainerMount]:
44
+ """Returns the files and folders of this host which a container gets. A part which does not exist on this host is not returned."""
45
+ result: list[ContainerMount] = []
46
+
47
+ #the optional user-specific scripts which prepare the container itself. the whole folder (not only the pre-codeunit-build-hook
48
+ #itself) is mounted, so the hook can start a sibling script placed next to it (for example to keep the hook independent of a
49
+ #package it only installs, while the actual preparation-logic which needs that package lives in the sibling script; see
50
+ #TFCPS_Tools_General.get_custom_scripts_folder_for_container). The mount is writable (not read-only) because the scripts in that
51
+ #folder also use it as their own download-cache-folder. A build inside the container runs the hook (see
52
+ #TFCPS_CodeUnit_BuildCodeUnits.__run_custom_pre_codeunit_build_script).
53
+ custom_scripts_folder: str = self.__tfcps_tools_general.get_custom_scripts_folder_for_container()
54
+ if os.path.isdir(custom_scripts_folder):
55
+ result.append(ContainerMount(custom_scripts_folder, self.__tfcps_tools_general.get_folder_of_custom_scripts_in_container(), False))
56
+
57
+ #the machine-wide image-registries-file of this host, so the container takes the images from the same custom registries as this
58
+ #host. Without it the container would find no mapping at all - the configuration-folder inside the container belongs to the
59
+ #container-user and is empty - and would therefore fall back to the upstream-registries defined in the repository, which are
60
+ #rate-limited. It is read-only because a build never changes the file.
61
+ oci_image_manager = self.__tfcps_tools_general.oci_image_manager
62
+ image_registries_file: str = oci_image_manager.get_image_registries_file_in_configuration_folder()
63
+ if os.path.isfile(image_registries_file):
64
+ result.append(ContainerMount(image_registries_file, oci_image_manager.get_image_registries_file_in_container(), True))
65
+
66
+ #the 'TFCPS'-folder of the configuration-folder of this host, so the container resolves the required environment-variables of the
67
+ #product (see TFCPS_Tools_General.get_required_environment_variables) and the OCI-registry-credentials (see
68
+ #ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) exactly like this host does - including a registry
69
+ #which is not publicly readable. The whole folder is mounted and not only EnvironmentVariables.csv in it, because a value of that file
70
+ #can point to a secret-file next to it ('file'-kind with a relative path, which is the recommended form) and such a value is resolved
71
+ #relative to the file itself. Only this folder (and not the whole configuration-folder) is mounted, so nothing else of it is exposed.
72
+ #It is read-only because a build never changes it.
73
+ tfcps_configuration_folder: str = self.__sc.get_tfcps_configuration_folder()
74
+ if os.path.isdir(tfcps_configuration_folder):
75
+ result.append(ContainerMount(tfcps_configuration_folder, self.__sc.get_tfcps_configuration_folder_in_container(), True))
76
+
77
+ #the files of the root-CA-certificates which this host declares via TrustedRootCA_<name>_Path. They are installed into the
78
+ #trust-store of the container by the entrypoint of the SCBuilder-image (see __get_trusted_root_cas).
79
+ file_cas, _ = self.__get_trusted_root_cas()
80
+ for ca_name, host_path in file_cas.items():
81
+ result.append(ContainerMount(host_path, f"/Workspace/RootCAs/{ca_name}.crt", True))
82
+ return result
83
+
84
+ @GeneralUtilities.check_arguments
85
+ def get_environment_variables(self) -> dict[str, str]:
86
+ """Returns the environment-variables (name and value) which a container gets: the ones which are declared as required in
87
+ <repository>/.ScriptCollection/ProductInformation.xml (including the ones which are declared for this machine, see
88
+ TFCPS_Tools_General.get_required_environment_variable_names) and the TrustedRootCA_<name>_ContentBase64-variables. The values are
89
+ typically secrets: a caller must give them to docker through the environment of the docker-client and pass only their names as
90
+ arguments, because arguments are written to the log and are visible in the process-list of this host."""
91
+ result: dict[str, str] = self.__tfcps_tools_general.get_required_environment_variables(self.__repository)
92
+ _, inline_cas = self.__get_trusted_root_cas()
93
+ result.update(inline_cas)
94
+ return result
95
+
96
+ @GeneralUtilities.check_arguments
97
+ def __get_trusted_root_cas(self) -> tuple[dict[str, str], dict[str, str]]:
98
+ """Resolves the TrustedRootCA_<name>_Path and TrustedRootCA_<name>_ContentBase64 environment-variables of this host. Returns
99
+ (file_cas, inline_cas):
100
+ - file_cas: <name> -> host-path of the certificate for each _Path-variable.
101
+ - inline_cas: variable-name -> value for each _ContentBase64-variable.
102
+ Candidate-names are collected from two sources (identical to the OCI-registry-credentials, see
103
+ ScriptCollectionCore.__get_defined_oci_registry_names): the machine-wide environment-variables-configuration-file and the
104
+ environment of the current process. The value is then resolved through ScriptCollectionCore.resolve_environment_variable, so a
105
+ CSV-entry takes precedence over an environment-variable of the same name - consistent with every other environment-variable
106
+ this script-collection resolves. If a name is declared in the file but its value can not be determined there (for example a
107
+ 'file'-entry pointing to a missing file), the resolver falls back to the environment and fails only if neither source yields a
108
+ value. The actual trust-store update happens in SCBuilder's EntryPoint.sh, which scans /Workspace/RootCAs and the
109
+ TrustedRootCA_*-environment-variables and runs update-ca-certificates once before any build-step. The <name>-part becomes the
110
+ filename inside /usr/local/share/ca-certificates/ and must therefore be unique across the two env-var-forms; a duplicate is
111
+ rejected here with a clear error, before the container is started."""
112
+ path_pattern = re.compile(r"^TrustedRootCA_(.+)_Path$", re.IGNORECASE)
113
+ content_b64_pattern = re.compile(r"^TrustedRootCA_(.+)_ContentBase64$", re.IGNORECASE)
114
+ #collect every candidate-variable-name from both sources before any value is resolved; this way a secret-value is read only for
115
+ #a name that is actually declared and the discovery is identical to the OCI-registry-credential-discovery. The configuration-
116
+ #file is scanned before the environment and a name found in both is kept with the casing of the file: on Windows, os.environ
117
+ #normalizes a variable-name to uppercase as soon as it is set, so without this preference the same declaration in both places
118
+ #would be treated as two different names and resolution would silently pick the environment-value instead of the file-value.
119
+ candidate_names: dict[str, str] = {}#lower-case variable-name -> canonical variable-name (file-casing preferred)
120
+ configuration_file = self.__sc.get_environment_variables_configuration_file()
121
+ if os.path.isfile(configuration_file):
122
+ for entry in GeneralUtilities.read_csv_file(configuration_file, True):
123
+ if 0 == len(entry):
124
+ continue
125
+ env_name = entry[0]
126
+ if path_pattern.match(env_name) is not None or content_b64_pattern.match(env_name) is not None:
127
+ candidate_names.setdefault(env_name.lower(), env_name)
128
+ for env_name in os.environ.keys():
129
+ if path_pattern.match(env_name) is not None or content_b64_pattern.match(env_name) is not None:
130
+ candidate_names.setdefault(env_name.lower(), env_name)
131
+ seen_names: dict[str, str] = {}#lower-case <name>-part -> originating variable-name, used only to produce an informative error on duplicate <name>.
132
+ file_cas: dict[str, str] = {}
133
+ inline_cas: dict[str, str] = {}
134
+ #sorted so the log-output (and therefore the order of the mounts and environment-variables) stays deterministic across runs.
135
+ for env_name in sorted(candidate_names.values(), key=str.lower):
136
+ env_value = self.__sc.resolve_environment_variable(env_name, f"the TrustedRootCA-mechanism (declared via \"{env_name}\")")
137
+ if not GeneralUtilities.string_has_content(env_value):
138
+ continue
139
+ path_match = path_pattern.match(env_name)
140
+ if path_match is not None:
141
+ ca_name = path_match.group(1)
142
+ self.__check_trusted_root_ca_name_is_unique(seen_names, ca_name, env_name)
143
+ host_path = env_value.strip()
144
+ if not os.path.isfile(host_path):
145
+ raise ValueError(f"Environment-variable \"{env_name}\" points to a file which does not exist on this host: \"{host_path}\".")
146
+ with open(host_path, "r", encoding="utf-8", errors="replace") as pem_file:
147
+ head = pem_file.read(64)
148
+ if "-----BEGIN CERTIFICATE-----" not in head:
149
+ raise ValueError(f"Environment-variable \"{env_name}\" (\"{host_path}\") does not look like a PEM-encoded certificate (missing \"-----BEGIN CERTIFICATE-----\"-header).")
150
+ file_cas[ca_name] = host_path
151
+ continue
152
+ content_b64_match = content_b64_pattern.match(env_name)
153
+ if content_b64_match is not None:
154
+ ca_name = content_b64_match.group(1)
155
+ self.__check_trusted_root_ca_name_is_unique(seen_names, ca_name, env_name)
156
+ inline_cas[env_name] = env_value
157
+ continue
158
+ return file_cas, inline_cas
159
+
160
+ @GeneralUtilities.check_arguments
161
+ def __check_trusted_root_ca_name_is_unique(self, seen_names: dict[str, str], ca_name: str, env_name: str) -> None:
162
+ key = ca_name.lower()
163
+ if key in seen_names:
164
+ raise ValueError(f"Multiple TrustedRootCA-definitions for name \"{ca_name}\" found (\"{seen_names[key]}\" and \"{env_name}\"). The <name>-part becomes the filename in the trust-store and must therefore be unique across _Path and _ContentBase64.")
165
+ seen_names[key] = env_name
@@ -0,0 +1,91 @@
1
+ import re
2
+ from ...GeneralUtilities import GeneralUtilities
3
+ from ...ScriptCollectionCore import ScriptCollectionCore
4
+ from ...SCLog import LogLevel
5
+ from ..PackageSource import PackageSource
6
+ from ..TFCPS_Tools_General import TFCPS_Tools_General
7
+
8
+
9
+ class NuGetPackageSourceRegistration:
10
+ """Registers the NuGet-sources which are declared for the current machine (see TFCPS_Tools_General.get_declared_package_sources). It is
11
+ used by every DotNet-codeunit-build and by the command-line-tool 'scregisterpackagesources' (which the agent-sandbox of scrais runs before
12
+ the agent starts), so a source is registered on the same way everywhere."""
13
+
14
+ #the output of the dotnet-cli is localized. The language is set explicitly because the output of "dotnet nuget list source" gets parsed, which would otherwise only work on machines which are configured to use english.
15
+ __dotnet_cli_environment_variables: dict = {"DOTNET_CLI_UI_LANGUAGE": "en-US"}
16
+ __sc: ScriptCollectionCore = None
17
+ __tfcps_tools_general: TFCPS_Tools_General = None
18
+
19
+ def __init__(self, sc: ScriptCollectionCore, tfcps_tools_general: TFCPS_Tools_General):
20
+ self.__sc = sc
21
+ self.__tfcps_tools_general = tfcps_tools_general
22
+
23
+ @GeneralUtilities.check_arguments
24
+ def register_declared_package_sources(self, folder: str) -> None:
25
+ """Ensures that every NuGet-source which is declared for this machine is registered with its current credentials, so a codeunit can
26
+ also use dependencies which are not available on nuget.org. The dotnet-cli is run in the given folder.
27
+ A source which is already registered under the declared name is removed and registered again, so an outdated registration (for
28
+ example one which points to another url) can not let the registration fail. Sources which are not declared are not touched at
29
+ all: the registration is stored in the NuGet-configuration of the current user, which - when the build runs directly on the
30
+ host - is the configuration the user also uses for their own work and which must not be destroyed by a build."""
31
+ package_sources: list[PackageSource] = self.__tfcps_tools_general.get_declared_package_sources("CSharp")
32
+ if len(package_sources) == 0:
33
+ return
34
+ for package_source in package_sources:
35
+ registered_sources: dict[str, str] = self.__get_registered_nuget_sources(folder)
36
+ name_of_source_with_same_name: str = None
37
+ name_of_source_with_same_url: str = None
38
+ for registered_name, registered_url in registered_sources.items():
39
+ if registered_name.lower() == package_source.name.lower():
40
+ name_of_source_with_same_name = registered_name
41
+ elif self.__nuget_sources_are_equal(registered_url, package_source.url):
42
+ #the same feed can already be registered under another name (which is the usual case on a machine on which the user
43
+ #configured the feed manually). Such a source is updated instead of being added a second time, because a second
44
+ #registration of the same url would let every restore query the feed twice.
45
+ name_of_source_with_same_url = registered_name
46
+ if name_of_source_with_same_name is not None:
47
+ self.__remove_nuget_source(folder, name_of_source_with_same_name)
48
+ if name_of_source_with_same_url is None:
49
+ arguments = ["nuget", "add", "source", package_source.url, "--name", package_source.name]
50
+ else:
51
+ #the credentials are set again even when the source already exists, because a token can have been rotated since it was registered.
52
+ arguments = ["nuget", "update", "source", name_of_source_with_same_url, "--source", package_source.url]
53
+ if package_source.has_credentials():
54
+ arguments = arguments+["--username", package_source.username, "--password", package_source.password]
55
+ if not GeneralUtilities.current_system_is_windows():
56
+ #without this the password can not be encrypted, which lets the command fail on the operating-systems which do not support the encryption.
57
+ arguments.append("--store-password-in-clear-text")
58
+ arguments_for_log = list(arguments)
59
+ if package_source.has_credentials():
60
+ arguments_for_log[arguments_for_log.index("--password")+1] = "***"#the password must not be written to the build-log
61
+ self.__sc.log.log(f"Add NuGet-source \"{package_source.name}\" ({package_source.url})..." if name_of_source_with_same_url is None else f"Update the already registered NuGet-source \"{name_of_source_with_same_url}\" ({package_source.url})...", LogLevel.Debug)
62
+ self.__sc.run_program_argsasarray("dotnet", arguments, folder, arguments_for_log=arguments_for_log, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
63
+
64
+ @GeneralUtilities.check_arguments
65
+ def __remove_nuget_source(self, folder: str, name: str) -> None:
66
+ """Removes the registration of the NuGet-source with the given name, so it can be registered again with its currently declared url and credentials."""
67
+ self.__sc.log.log(f"Remove the outdated registration of the NuGet-source \"{name}\"...", LogLevel.Debug)
68
+ self.__sc.run_program_argsasarray("dotnet", ["nuget", "remove", "source", name], folder, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
69
+
70
+ @GeneralUtilities.check_arguments
71
+ def __get_registered_nuget_sources(self, folder: str) -> dict[str, str]:
72
+ """Returns the NuGet-sources which are registered for the given folder as a mapping of their name to their url. The sources are
73
+ read from the output of "dotnet nuget list source", which prints the name (with its state) and the url of a source in two
74
+ consecutive lines."""
75
+ result: dict[str, str] = {}
76
+ list_result = self.__sc.run_program_argsasarray("dotnet", ["nuget", "list", "source"], folder, throw_exception_if_exitcode_is_not_zero=False, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
77
+ name_pattern = re.compile(r"^\s*\d+\.\s+(.+?)\s+\[[^\[\]]+\]\s*$")
78
+ current_name: str = None
79
+ for line in list_result[1].splitlines():
80
+ match = name_pattern.match(line)
81
+ if match is not None:
82
+ current_name = match.group(1).strip()
83
+ elif current_name is not None and GeneralUtilities.string_has_content(line):
84
+ result[current_name] = line.strip()
85
+ current_name = None
86
+ return result
87
+
88
+ @GeneralUtilities.check_arguments
89
+ def __nuget_sources_are_equal(self, url: str, other_url: str) -> bool:
90
+ """Compares two urls of package-sources. A trailing slash and the case are ignored because they do not address a different feed."""
91
+ return url.strip().rstrip("/").lower() == other_url.strip().rstrip("/").lower()
@@ -11,15 +11,13 @@ from packaging.version import Version
11
11
  from .CertificateGeneratorInformationBase import CertificateGeneratorInformationBase
12
12
  from ...GeneralUtilities import GeneralUtilities, VersionEcholon
13
13
  from ...SCLog import LogLevel
14
- from ..PackageSource import PackageSource
14
+ from .NuGetPackageSourceRegistration import NuGetPackageSourceRegistration
15
15
  from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base,TFCPS_CodeUnitSpecific_Base_CLI
16
16
 
17
17
  class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
18
18
 
19
19
  is_library:bool = None
20
20
  csproj_file:bool = None
21
- #the output of the dotnet-cli is localized. The language is set explicitly for the commands whose output gets parsed, because otherwise the parsing would only work on machines which are configured to use english.
22
- __dotnet_cli_environment_variables:dict = {"DOTNET_CLI_UI_LANGUAGE": "en-US"}
23
21
  #the only accepted form of a version of a package-reference: an exact version in square brackets. Everything else is a version-range.
24
22
  __pinned_version_regex = re.compile(r"^\[[^,\[\]]+\]$")
25
23
 
@@ -29,78 +27,7 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
29
27
  self.is_library="<OutputType>Library</OutputType>" in GeneralUtilities.read_text_from_file(self.csproj_file)#TODO do a real check by checking this property using xpath
30
28
  #every dotnet-operation of a codeunit (build, test, linting, ...) restores the dependencies of the codeunit, so the sources
31
29
  #they are downloaded from have to be available for all of them and not only for the build.
32
- self.__ensure_declared_package_sources_are_available()
33
-
34
- @GeneralUtilities.check_arguments
35
- def __ensure_declared_package_sources_are_available(self) -> None:
36
- """Ensures that every NuGet-source which is declared for this machine (see TFCPS_Tools_General.get_declared_package_sources) is
37
- registered with its current credentials, so a codeunit can also use dependencies which are not available on nuget.org.
38
- A source which is already registered under the declared name is removed and registered again, so an outdated registration (for
39
- example one which points to another url) can not let the registration fail. Sources which are not declared are not touched at
40
- all: the registration is stored in the NuGet-configuration of the current user, which - when the build runs directly on the
41
- host - is the configuration the user also uses for their own work and which must not be destroyed by a build."""
42
- package_sources: list[PackageSource] = self.tfcps_Tools_General.get_declared_package_sources("CSharp")
43
- if len(package_sources) == 0:
44
- return
45
- codeunit_folder: str = self.get_codeunit_folder()
46
- for package_source in package_sources:
47
- registered_sources: dict[str, str] = self.__get_registered_nuget_sources(codeunit_folder)
48
- name_of_source_with_same_name: str = None
49
- name_of_source_with_same_url: str = None
50
- for registered_name, registered_url in registered_sources.items():
51
- if registered_name.lower() == package_source.name.lower():
52
- name_of_source_with_same_name = registered_name
53
- elif self.__nuget_sources_are_equal(registered_url, package_source.url):
54
- #the same feed can already be registered under another name (which is the usual case on a machine on which the user
55
- #configured the feed manually). Such a source is updated instead of being added a second time, because a second
56
- #registration of the same url would let every restore query the feed twice.
57
- name_of_source_with_same_url = registered_name
58
- if name_of_source_with_same_name is not None:
59
- self.__remove_nuget_source(codeunit_folder, name_of_source_with_same_name)
60
- if name_of_source_with_same_url is None:
61
- arguments = ["nuget", "add", "source", package_source.url, "--name", package_source.name]
62
- else:
63
- #the credentials are set again even when the source already exists, because a token can have been rotated since it was registered.
64
- arguments = ["nuget", "update", "source", name_of_source_with_same_url, "--source", package_source.url]
65
- if package_source.has_credentials():
66
- arguments = arguments+["--username", package_source.username, "--password", package_source.password]
67
- if not GeneralUtilities.current_system_is_windows():
68
- #without this the password can not be encrypted, which lets the command fail on the operating-systems which do not support the encryption.
69
- arguments.append("--store-password-in-clear-text")
70
- arguments_for_log = list(arguments)
71
- if package_source.has_credentials():
72
- arguments_for_log[arguments_for_log.index("--password")+1] = "***"#the password must not be written to the build-log
73
- self._protected_sc.log.log(f"Add NuGet-source \"{package_source.name}\" ({package_source.url})..." if name_of_source_with_same_url is None else f"Update the already registered NuGet-source \"{name_of_source_with_same_url}\" ({package_source.url})...", LogLevel.Debug)
74
- self._protected_sc.run_program_argsasarray("dotnet", arguments, codeunit_folder, arguments_for_log=arguments_for_log, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
75
-
76
- @GeneralUtilities.check_arguments
77
- def __remove_nuget_source(self, folder: str, name: str) -> None:
78
- """Removes the registration of the NuGet-source with the given name, so it can be registered again with its currently declared url and credentials."""
79
- self._protected_sc.log.log(f"Remove the outdated registration of the NuGet-source \"{name}\"...", LogLevel.Debug)
80
- self._protected_sc.run_program_argsasarray("dotnet", ["nuget", "remove", "source", name], folder, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
81
-
82
- @GeneralUtilities.check_arguments
83
- def __get_registered_nuget_sources(self, folder: str) -> dict[str, str]:
84
- """Returns the NuGet-sources which are registered for the given folder as a mapping of their name to their url. The sources are
85
- read from the output of "dotnet nuget list source", which prints the name (with its state) and the url of a source in two
86
- consecutive lines."""
87
- result: dict[str, str] = {}
88
- list_result = self._protected_sc.run_program_argsasarray("dotnet", ["nuget", "list", "source"], folder, throw_exception_if_exitcode_is_not_zero=False, print_live_output=False, env_vars=self.__dotnet_cli_environment_variables)
89
- name_pattern = re.compile(r"^\s*\d+\.\s+(.+?)\s+\[[^\[\]]+\]\s*$")
90
- current_name: str = None
91
- for line in list_result[1].splitlines():
92
- match = name_pattern.match(line)
93
- if match is not None:
94
- current_name = match.group(1).strip()
95
- elif current_name is not None and GeneralUtilities.string_has_content(line):
96
- result[current_name] = line.strip()
97
- current_name = None
98
- return result
99
-
100
- @GeneralUtilities.check_arguments
101
- def __nuget_sources_are_equal(self, url: str, other_url: str) -> bool:
102
- """Compares two urls of package-sources. A trailing slash and the case are ignored because they do not address a different feed."""
103
- return url.strip().rstrip("/").lower() == other_url.strip().rstrip("/").lower()
30
+ NuGetPackageSourceRegistration(self._protected_sc, self.tfcps_Tools_General).register_declared_package_sources(self.get_codeunit_folder())
104
31
 
105
32
  @GeneralUtilities.check_arguments
106
33
  def get_runtimes(self) -> list[str]:
@@ -15,6 +15,7 @@ from .TFCPS_CodeSpellCheck import TFCPS_CodeSpellCheck
15
15
  from .TFCPS_FindingsSorter import TFCPS_FindingsSorter
16
16
  from .TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
17
17
  from .TFCPS_Tools_General import TFCPS_Tools_General, CommonProjectStructureVersion
18
+ from .ContainerBuildEnvironment import ContainerBuildEnvironment
18
19
 
19
20
 
20
21
  class TFCPS_UpdateDependenciesHook(TFCPS_BuildCodeUnitsHook):
@@ -197,21 +198,23 @@ class TFCPS_CodeUnit_BuildCodeUnits:
197
198
  hook.run_after_codeunit_was_built(tFCPS_CodeUnit_BuildCodeUnit)
198
199
  self.sc.log.log(GeneralUtilities.get_line())
199
200
 
200
- self.tfcps_tools_general.generate_svg_files_from_plantuml_files_for_repository(self.repository, self.use_cache())
201
- self.tfcps_tools_general.generate_svg_files_from_vega_files_for_repository(self.repository, self.use_cache())
202
- self.tfcps_tools_general.generate_svg_files_from_vegalite_files_for_repository(self.repository, self.use_cache())
203
- self.__do_project_linting(self.repository)
204
- self.__update_agent_sandbox_image(self.repository)
201
+ self.sc.log.log("Do project post-build-tasks...")
202
+
203
+ if not self.__fast_lane:
204
+ self.__do_project_linting(self.repository)
205
+ self.__update_agent_images(self.repository)
206
+ self.search_for_secrets()
205
207
  self.__normalize_line_endings_of_common_files()
206
- self.search_for_secrets()
207
-
208
- self.sc.log.log(GeneralUtilities.get_line())
209
208
 
210
209
  if self.is_pre_merge():
211
210
  self.__collect_metrics()
212
211
  self.__generate_loc_diagram()
213
- self.sc.log.log(GeneralUtilities.get_line())
214
212
 
213
+ self.tfcps_tools_general.generate_svg_files_from_plantuml_files_for_repository(self.repository, self.use_cache())
214
+ self.tfcps_tools_general.generate_svg_files_from_vega_files_for_repository(self.repository, self.use_cache())
215
+ self.tfcps_tools_general.generate_svg_files_from_vegalite_files_for_repository(self.repository, self.use_cache())
216
+
217
+ self.sc.log.log(GeneralUtilities.get_line())
215
218
  except Exception:
216
219
  error_occurred=True
217
220
  raise
@@ -239,25 +242,31 @@ class TFCPS_CodeUnit_BuildCodeUnits:
239
242
 
240
243
 
241
244
  @GeneralUtilities.check_arguments
242
- def __update_agent_sandbox_image(self, repository_folder: str) -> None:
243
- """Sets the value "image" in "<repository>/.ScriptCollection/AgentSandboxConfiguration.ini" to the SCBuilder-image (address including tag)
244
- defined in "<repository>/.ScriptCollection/OCIImages/ImageDefinition.csv".
245
- Nothing is done if one of these files does not exist or if the image-definition-file does not contain an entry for SCBuilder."""
245
+ def __update_agent_images(self, repository_folder: str) -> None:
246
+ # The image of the sandbox and the image of its egress-proxy (which enforces the allowlist of the sandbox) follow the image-definition-file.
247
+ self.__update_agent_sandbox_image(repository_folder, "SCBuilder", "image")
248
+ self.__update_agent_sandbox_image(repository_folder, "Squid", "egress_proxy_image")
249
+
250
+ @GeneralUtilities.check_arguments
251
+ def __update_agent_sandbox_image(self, repository_folder: str, image: str, key: str) -> None:
252
+ """Sets the value of the given key in "<repository>/.ScriptCollection/AgentSandboxConfiguration.ini" to the given image (address including tag)
253
+ as defined in "<repository>/.ScriptCollection/OCIImages/ImageDefinition.csv".
254
+ Nothing is done if one of these files does not exist or if the image-definition-file does not contain an entry for the image."""
246
255
  image_definition_file = os.path.join(repository_folder, ".ScriptCollection", "OCIImages", "ImageDefinition.csv")
247
256
  agent_sandbox_configuration_file = os.path.join(repository_folder, ".ScriptCollection", "AgentSandboxConfiguration.ini")
248
257
  if not os.path.isfile(image_definition_file) or not os.path.isfile(agent_sandbox_configuration_file):
249
258
  return
250
- scbuilder_image: str = None
259
+ image_value: str = None
251
260
  for line in GeneralUtilities.read_nonempty_lines_from_file(image_definition_file):
252
- if line.startswith("SCBuilder;"):
261
+ if line.startswith(f"{image};"):
253
262
  columns = line.split(";")
254
- scbuilder_image = f"{columns[1]}:{columns[2]}"
263
+ image_value = f"{columns[1]}:{columns[2]}"
255
264
  break
256
- if scbuilder_image is None:
265
+ if image_value is None:
257
266
  return
258
- self.sc.log.log(f"Set image of agent-sandbox to '{scbuilder_image}'...", LogLevel.Debug)
267
+ self.sc.log.log(f"Set '{key}' of agent-sandbox to '{image_value}'...", LogLevel.Debug)
259
268
  section_header = "[AgentSandboxConfiguration]"
260
- image_line = f"image = {scbuilder_image}"
269
+ image_line = f"{key} = {image_value}"
261
270
  lines: list[str] = GeneralUtilities.read_lines_from_file(agent_sandbox_configuration_file)
262
271
  updated_lines: list[str] = []
263
272
  current_section: str = None
@@ -266,7 +275,7 @@ class TFCPS_CodeUnit_BuildCodeUnits:
266
275
  stripped_line = line.strip()
267
276
  if stripped_line.startswith("[") and stripped_line.endswith("]"):
268
277
  current_section = stripped_line
269
- elif current_section == section_header and re.match(r"^image\s*=", stripped_line):
278
+ elif current_section == section_header and re.match(rf"^{re.escape(key)}\s*=", stripped_line):
270
279
  line = image_line
271
280
  image_was_set = True
272
281
  updated_lines.append(line)
@@ -566,61 +575,19 @@ class TFCPS_CodeUnit_BuildCodeUnits:
566
575
  update_argument="pip3 install scriptcollection --upgrade && "
567
576
  executable_arguments=["bash","-c", f"{update_argument}scshowversion && "+" ".join(executable_arguments)]
568
577
 
569
- #run the optional user-specific script which prepares this host for a container-build (for example to log in to the registry the image is pulled from).
570
- #it runs before the environment-variables are resolved, so it can also create the files their values are read from.
571
- self.tfcps_tools_general.run_custom_script_if_available(self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScriptForContainer.py"), self.__get_build_arguments(self.repository))
572
-
573
- #mount the optional user-specific scripts which prepare the container itself. the whole folder (not only the pre-codeunit-build-hook
574
- #itself) is mounted, so the hook can start a sibling script placed next to it (for example to keep the hook independent of a
575
- #package it only installs, while the actual preparation-logic which needs that package lives in the sibling script; see
576
- #get_custom_scripts_folder_for_container). The mount is writable (not read-only) because the scripts in that folder also use it as
577
- #their own download-cache-folder. The build inside the container runs the hook (see __run_custom_pre_codeunit_build_script); if the
578
- #folder does not exist on the host nothing is mounted and nothing is run.
578
+ #the build-environment (host-hook, mounts and environment-variables) is defined by ContainerBuildEnvironment, which is also used by
579
+ #the agent-sandbox of scrais, so that both containers get the same one. The hook runs before the environment-variables are resolved,
580
+ #so it can also create the files their values are read from.
581
+ container_build_environment: ContainerBuildEnvironment = ContainerBuildEnvironment(self.sc, self.tfcps_tools_general, self.repository)
582
+ container_build_environment.run_host_preparation_hook(self.__get_build_arguments(self.repository))
579
583
  mount_arguments: list[str] = []
580
- custom_scripts_folder_for_inside_the_container: str = self.tfcps_tools_general.get_custom_scripts_folder_for_container()
581
- if os.path.isdir(custom_scripts_folder_for_inside_the_container):
582
- mount_arguments += ["-v", f"{custom_scripts_folder_for_inside_the_container}:{self.tfcps_tools_general.get_folder_of_custom_scripts_in_container()}"]
583
-
584
- #mount the machine-wide image-registries-file of this host (if it exists), so the build inside the container takes the images from the same
585
- #custom registries as a build on the host. Without the mount the build in the container would find no mapping at all - the
586
- #configuration-folder inside the container belongs to the container-user and is empty - and would therefore fall back to the
587
- #upstream-registries defined in the repository, which are rate-limited. The mount is read-only because a build never changes the file.
588
- oci_image_manager = self.tfcps_tools_general.oci_image_manager
589
- image_registries_file: str = oci_image_manager.get_image_registries_file_in_configuration_folder()
590
- if os.path.isfile(image_registries_file):
591
- mount_arguments += ["-v", f"{image_registries_file}:{oci_image_manager.get_image_registries_file_in_container()}:ro"]
592
-
593
- #mount the 'TFCPS'-folder of the configuration-folder of this host (if it exists), so the build inside the container resolves the
594
- #required environment-variables of the product (see TFCPS_Tools_General.get_required_environment_variables) and the
595
- #OCI-registry-credentials (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) exactly like a
596
- #build on the host does - including a registry which is not publicly readable, which without this mount could not be used inside
597
- #the container at all, because the configuration-folder of the container-user does not contain the credentials of this host.
598
- #The whole folder is mounted and not only EnvironmentVariables.csv in it, because a value of that file can point to a secret-file
599
- #next to it ('file'-kind with a relative path, which is the recommended form) and such a value is resolved relative to the file
600
- #itself - so with only the file mounted, every entry of that kind would fail inside the container. Only this folder (and not the
601
- #whole configuration-folder) is mounted, so nothing else of it is exposed to the container. The mount is read-only because a build
602
- #never changes it. The alternative way to provide a value is an environment-variable of the build-pipeline itself (the pipeline's
603
- #own secret-store); both sources are used together, the file taking precedence.
604
- tfcps_configuration_folder: str = self.sc.get_tfcps_configuration_folder()
605
- if os.path.isdir(tfcps_configuration_folder):
606
- mount_arguments += ["-v", f"{tfcps_configuration_folder}:{self.sc.get_tfcps_configuration_folder_in_container()}:ro"]
607
-
608
- #install the host-declared trusted root-CA-certificates into the container (needed when the build pulls packages from a privately
609
- #hosted registry whose certificate-chain is not publicly rooted - for example an internally hosted gitlab-nuget-source). The
610
- #two env-var forms are detected automatically from both the machine-wide environment-variables-configuration-file and the
611
- #environment of the current process; see __collect_trusted_root_ca_arguments. _Path-variables add file-mounts, the inline
612
- #_ContentBase64-form is returned as a name->value dict that gets merged into required_environment_variables below so its
613
- #value flows through the docker-client's own environment (not through argv) just like a required secret does.
614
- trusted_root_ca_mount_arguments, trusted_root_ca_inline_env_vars = self.__collect_trusted_root_ca_arguments()
615
- mount_arguments += trusted_root_ca_mount_arguments
616
-
617
- #pass the environment-variables which are declared as required in <repository>/.ScriptCollection/ProductInformation.xml into the container
618
- #so they do not have to be specified explicitly on every call. Their values are resolved from the user-specific
619
- #configuration-file (see TFCPS_Tools_General.get_environment_variables_configuration_file), which only exists on the host.
620
- #only the names are passed as arguments; the values are given to the docker-client through its own environment, because arguments
621
- #are written to the log and are visible in the process-list of this host, which must not happen for a value which is typically a secret.
622
- required_environment_variables: dict[str, str] = self.tfcps_tools_general.get_required_environment_variables(self.repository)
623
- required_environment_variables.update(trusted_root_ca_inline_env_vars)
584
+ for mount in container_build_environment.get_mounts():
585
+ mount_arguments += ["-v", f"{mount.source}:{mount.target}{':ro' if mount.read_only else ''}"]
586
+
587
+ #only the names of the environment-variables are passed as arguments; the values are given to the docker-client through its own
588
+ #environment, because arguments are written to the log and are visible in the process-list of this host, which must not happen for
589
+ #a value which is typically a secret.
590
+ required_environment_variables: dict[str, str] = container_build_environment.get_environment_variables()
624
591
  env_arguments: list[str] = []
625
592
  for env_variable_name in required_environment_variables:
626
593
  env_arguments += ["-e", env_variable_name]
@@ -653,78 +620,6 @@ class TFCPS_CodeUnit_BuildCodeUnits:
653
620
  stderr:str=result[2] or GeneralUtilities.empty_string
654
621
  return (exit_code==0,f"{stdout}\n{stderr}")
655
622
 
656
- @GeneralUtilities.check_arguments
657
- def __collect_trusted_root_ca_arguments(self) -> tuple[list[str], dict[str, str]]:
658
- """Resolves the TrustedRootCA_<name>_Path and TrustedRootCA_<name>_ContentBase64 environment-variables of the current host into
659
- docker-arguments which install those certificates into the trust-store of the build-container. Returns (mount_arguments, inline_env_vars):
660
- - mount_arguments: '-v <host-path>:/Workspace/RootCAs/<name>.crt:ro' entries for each _Path-variable.
661
- - inline_env_vars: a name->value dict for each _ContentBase64-variable; the caller merges it into required_environment_variables
662
- so the values flow through the docker-client's own environment, not through argv.
663
- Candidate-names are collected from two sources (identical to the OCI-registry-credentials, see
664
- ScriptCollectionCore.__get_defined_oci_registry_names): the machine-wide environment-variables-configuration-file and the
665
- environment of the current process. The value is then resolved through ScriptCollectionCore.resolve_environment_variable, so a
666
- CSV-entry takes precedence over an environment-variable of the same name - consistent with every other environment-variable
667
- this script-collection resolves. If a name is declared in the file but its value can not be determined there (for example a
668
- 'file'-entry pointing to a missing file), the resolver falls back to the environment and fails only if neither source yields a
669
- value. The actual trust-store update happens in SCBuilder's EntryPoint.sh, which scans /Workspace/RootCAs and the
670
- TrustedRootCA_*-environment-variables and runs update-ca-certificates once before any build-step. The <name>-part becomes the
671
- filename inside /usr/local/share/ca-certificates/ and must therefore be unique across the two env-var-forms; a duplicate is
672
- rejected here with a clear error, before the container is started."""
673
- path_pattern = re.compile(r"^TrustedRootCA_(.+)_Path$", re.IGNORECASE)
674
- content_b64_pattern = re.compile(r"^TrustedRootCA_(.+)_ContentBase64$", re.IGNORECASE)
675
- #collect every candidate-variable-name from both sources before any value is resolved; this way a secret-value is read only for
676
- #a name that is actually declared and the discovery is identical to the OCI-registry-credential-discovery. The configuration-
677
- #file is scanned before the environment and a name found in both is kept with the casing of the file: on Windows, os.environ
678
- #normalizes a variable-name to uppercase as soon as it is set, so without this preference the same declaration in both places
679
- #would be treated as two different names and resolution would silently pick the environment-value instead of the file-value.
680
- candidate_names: dict[str, str] = {}#lower-case variable-name -> canonical variable-name (file-casing preferred)
681
- configuration_file = self.sc.get_environment_variables_configuration_file()
682
- if os.path.isfile(configuration_file):
683
- for entry in GeneralUtilities.read_csv_file(configuration_file, True):
684
- if 0 == len(entry):
685
- continue
686
- env_name = entry[0]
687
- if path_pattern.match(env_name) is not None or content_b64_pattern.match(env_name) is not None:
688
- candidate_names.setdefault(env_name.lower(), env_name)
689
- for env_name in os.environ.keys():
690
- if path_pattern.match(env_name) is not None or content_b64_pattern.match(env_name) is not None:
691
- candidate_names.setdefault(env_name.lower(), env_name)
692
- seen_names: dict[str, str] = {}#lower-case <name>-part -> originating variable-name, used only to produce an informative error on duplicate <name>.
693
- mount_arguments: list[str] = []
694
- inline_env_vars: dict[str, str] = {}
695
- #sorted so the log-output (and therefore the order of "-v"-/"-e"-arguments) stays deterministic across runs.
696
- for env_name in sorted(candidate_names.values(), key=str.lower):
697
- env_value = self.sc.resolve_environment_variable(env_name, f"the TrustedRootCA-mechanism (declared via \"{env_name}\")")
698
- if not GeneralUtilities.string_has_content(env_value):
699
- continue
700
- path_match = path_pattern.match(env_name)
701
- if path_match is not None:
702
- ca_name = path_match.group(1)
703
- self.__check_trusted_root_ca_name_is_unique(seen_names, ca_name, env_name)
704
- host_path = env_value.strip()
705
- if not os.path.isfile(host_path):
706
- raise ValueError(f"Environment-variable \"{env_name}\" points to a file which does not exist on this host: \"{host_path}\".")
707
- with open(host_path, "r", encoding="utf-8", errors="replace") as pem_file:
708
- head = pem_file.read(64)
709
- if "-----BEGIN CERTIFICATE-----" not in head:
710
- raise ValueError(f"Environment-variable \"{env_name}\" (\"{host_path}\") does not look like a PEM-encoded certificate (missing \"-----BEGIN CERTIFICATE-----\"-header).")
711
- mount_arguments += ["-v", f"{host_path}:/Workspace/RootCAs/{ca_name}.crt:ro"]
712
- continue
713
- content_b64_match = content_b64_pattern.match(env_name)
714
- if content_b64_match is not None:
715
- ca_name = content_b64_match.group(1)
716
- self.__check_trusted_root_ca_name_is_unique(seen_names, ca_name, env_name)
717
- inline_env_vars[env_name] = env_value
718
- continue
719
- return mount_arguments, inline_env_vars
720
-
721
- @GeneralUtilities.check_arguments
722
- def __check_trusted_root_ca_name_is_unique(self, seen_names: dict[str, str], ca_name: str, env_name: str) -> None:
723
- key = ca_name.lower()
724
- if key in seen_names:
725
- raise ValueError(f"Multiple TrustedRootCA-definitions for name \"{ca_name}\" found (\"{seen_names[key]}\" and \"{env_name}\"). The <name>-part becomes the filename in the trust-store and must therefore be unique across _Path and _ContentBase64.")
726
- seen_names[key] = env_name
727
-
728
623
  @GeneralUtilities.check_arguments
729
624
  def __translate_path_into_container(self, host_path: str, container_repository_folder: str) -> str:
730
625
  normalized_repository = os.path.normpath(self.repository)
@@ -1958,7 +1958,7 @@ class TFCPS_Tools_General:
1958
1958
  arch_tags.append(arch_tag)
1959
1959
  # Load tar → local image
1960
1960
  GeneralUtilities.write_message_to_stdout(f"Loading {tar_path}...")
1961
- result = self.__sc.run_program_argsasarray("docker",[ "load", "-i", tar_path])
1961
+ result = self.__sc.run_program_argsasarray_with_retry("docker",[ "load", "-i", tar_path],amount_of_attempts=3,delay_in_seconds=5)
1962
1962
  # docker load outputs: "Loaded image: sha256:abc123..." or "Loaded image ID: ..."
1963
1963
  # we need the loaded image ID
1964
1964
  loaded_id = None
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptcollection
3
- Version: 4.4.67
3
+ Version: 4.4.69
4
4
  Summary: The ScriptCollection is the place for reusable scripts.
5
5
  Author-email: Marius Göcke <marius.goecke@gmail.com>
6
6
  License-Expression: GPL-3.0-only
@@ -1,7 +1,7 @@
1
- ScriptCollection/AgentRunner.py,sha256=8UiY6mAZoSSb1LlKGJ29qhLUW_0DtryrC2Kdn5cDPgg,19820
1
+ ScriptCollection/AgentRunner.py,sha256=v3wRllPSjeIuIK6uhCfQLRdAm0Kpq-DOs7VVWUEIU2s,29866
2
2
  ScriptCollection/ArbTranslationsOrganizer.py,sha256=3TO4fjkYADcHGyUXdiQk-RtfCxtf9ojuBS3CbZtgXBE,12218
3
3
  ScriptCollection/CertificateUpdater.py,sha256=yZc7KESc1F_Mc6HP2UfLmNHRc2MvtVqp0t60J_CqY3s,9755
4
- ScriptCollection/Executables.py,sha256=Z-YRlSgD4GN7XjZuarXHZTPOuWtNr2rF5OzlwjJ3Yuo,82192
4
+ ScriptCollection/Executables.py,sha256=N_tAJcDRS2rniRXYFZPWQs7ar_g8HkaR0EddAJ7CZiw,84901
5
5
  ScriptCollection/GeneralUtilities.py,sha256=kW6y52rdD-dJrjCtIgsdwYinz6FtclGj-xc_4tL1MUg,80022
6
6
  ScriptCollection/HTTPMaintenanceOverheadHelper.py,sha256=TToNtyO1XzsMbBsTBf3o0xgOK0v4Jf03qw2Z0xb2nCk,2007
7
7
  ScriptCollection/ProcessesRunner.py,sha256=wtNFdmj9klETTIDUSE90vaR5ClEX4V7dgJNVbp9_wi0,1381
@@ -12,10 +12,13 @@ ScriptCollection/ProgramRunnerSudo.py,sha256=vwKq-MOcqsgiKhSf-MBUDBC_UyaU8evP3Ro
12
12
  ScriptCollection/RepositoryDiagramGenerator.py,sha256=K_u2NW_p4ozdWD50irJEkkcDVa70TcUJpZN63TeXSLw,7092
13
13
  ScriptCollection/RepositoryHistoryAnalyzer.py,sha256=u9141B9D-Kef-9uo3yR7BXKUMzIKLVsesnYk7H-cWm4,6157
14
14
  ScriptCollection/SCLog.py,sha256=Fjf8Pdi63cXHGMxL_aI6A0dwGm7QIfrv3czjvpSUlaM,4705
15
- ScriptCollection/ScriptCollectionCore.py,sha256=TIqhsMEv5XvRpZlih-5_U-wqGiwr3CbpOYShk1sMHrU,310633
15
+ ScriptCollection/ScriptCollectionCore.py,sha256=C4S0Kk1T3asMF4zlKPNTnGm5QZYBO1LdiO9QalUzOmE,310633
16
16
  ScriptCollection/TestKind.py,sha256=jqLJwNvQeyea7SZf-UFzMC9pM0znv4do7FIC1xq8u9M,1672
17
17
  ScriptCollection/UnassignedVariablesCheck.py,sha256=8xNcp1D5HeOLO1a1NEerq2TsnpFhw7yzYPkaay-oisY,3804
18
18
  ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
19
+ ScriptCollection/AgentSandboxNetwork/SandboxEgressProxy.py,sha256=taHYkvKCjUteJisZOYNY283m_DAs0odl-WmewlobwGQ,8385
20
+ ScriptCollection/AgentSandboxNetwork/SandboxNetworkConfiguration.py,sha256=OQFUbalueSt0XGq5sUjJLmmxb7VYi4JMk9vzl0GPB4A,5377
21
+ ScriptCollection/AgentSandboxNetwork/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
19
22
  ScriptCollection/AgentStatusReporting/AgentStatus.py,sha256=eKz2RN1XEhPjaLxALmvtV-k2094RUTMEXT7HFKsSImg,563
20
23
  ScriptCollection/AgentStatusReporting/AgentStatusEventChannel.py,sha256=LnAi5qA2NjRwcDEg8ALsOeHspO_CPcEs8A4GiaFd0bY,1732
21
24
  ScriptCollection/AgentStatusReporting/AgentStatusRelay.py,sha256=JtfM5OQVnojUdGdiRipSZKePIk-A5L1SPxIMET1XYWM,3639
@@ -38,6 +41,7 @@ ScriptCollection/Resources/CultureChooser/index.html,sha256=gxQzbrSp4WU52TqN-gcB
38
41
  ScriptCollection/Resources/Dependencies/JRE/Version.txt,sha256=WlFzNJMH4lfVzW5QGHnuE52jLujSGpQ48zVB-Cln-7g,9
39
42
  ScriptCollection/Resources/Dependencies/OpenAPIGenerator/Version.txt,sha256=hdAgOhOHx4s13fJ-j0A06oYoM9rdEfDe0RzHoeAKXxo,7
40
43
  ScriptCollection/Resources/MaintenanceSite/MaintenanceSite.html,sha256=CX9S1bdOz6xU2aGfr03tJRGczQrlpn-IeODc15d5kc0,232
44
+ ScriptCollection/TFCPS/ContainerBuildEnvironment.py,sha256=LIlWZnxqKGqTyAOxrdUOh2yTLmaTlA8skqIII9-SElk,12620
41
45
  ScriptCollection/TFCPS/NpmDependencies.py,sha256=IfUh9EyrkHxovGRWYagNMZjBUXyKjZ4AG-Z8leehw24,6304
42
46
  ScriptCollection/TFCPS/PackageSource.py,sha256=tQoMbxrIbV3mYHKH2wgOoGzIHY1TvBH8EAFpWGhryrQ,1311
43
47
  ScriptCollection/TFCPS/PlantUMLRenderer.py,sha256=He5woqx9IR96vMXtHo12BmeXmcjLoifGD8Sup0bpxuo,11060
@@ -47,7 +51,7 @@ ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py,sha256=67_DQwoH5bhxxSmwtuAeoW
47
51
  ScriptCollection/TFCPS/TFCPS_CodeSpellCheck.py,sha256=zV62O7UiGjewx1yQUIWmUJ9D-5ipkV_f28uNhp-JWUs,5306
48
52
  ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=Rh7S84B8QbeWn5qJxRItoI7jquJYvFnNRi_WiX3FjWo,40414
49
53
  ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=CB0K8OZNtQESdS2pqX9RQqN-lI5M-cbmXYOR0rbuRlk,9741
50
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=5H8Artt40lrl3vvAHaZwiBPiL4ycHCZdk68s9mRIcI8,70924
54
+ ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=mMBCIUxocQjpGLwEs_Jdzr_zpxSqtzs_Gh3GcinaF0s,60412
51
55
  ScriptCollection/TFCPS/TFCPS_CreateRelease.py,sha256=BVK3fpEyjcXYVHIb6YYLH7bWe6iKRaCtPrJ3tXAemYw,6582
52
56
  ScriptCollection/TFCPS/TFCPS_FindingsSorter.py,sha256=3ChuKf-Myr3sBtQhYGw6dHlk93HO6PAH-Jq1oPYdW1s,4804
53
57
  ScriptCollection/TFCPS/TFCPS_Generic.py,sha256=mRBcf7XuNWOSXbZYc3NuJow18AMZ30D6Z-BtyNYfNcM,2072
@@ -56,7 +60,7 @@ ScriptCollection/TFCPS/TFCPS_MergeToStable.py,sha256=GdpFcgIFRvdc1Q9b7uXlPlN99s7
56
60
  ScriptCollection/TFCPS/TFCPS_OCIImageSecretScan.py,sha256=Jb7AaMcJDW-gYEYBUVZAG3QLTTNWmXlvVEW-lZsykws,14821
57
61
  ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py,sha256=XLVsFTA1jvfDbZ7wYJgIdEByZhF_bYVbA5jkisuvK4k,2322
58
62
  ScriptCollection/TFCPS/TFCPS_RemoteBuild.py,sha256=2Tx5HLk1ZftBWDCTiGZdP2QiiGNyWsEuTCSt33dNQjc,23891
59
- ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=iiMy4LDY-bQk6PEi0onVhhKPsoWJHhgMaS5xDQVSlM8,160357
63
+ ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=J9dJrDXwTFLdCpnHV90890pkM5PARy_6U6hf8zSAjiE,160408
60
64
  ScriptCollection/TFCPS/TFCPS_VisualRegressionTests.py,sha256=F90CF5kLCgV_M3yN696xWzF1cglWLXecau9My2ZyGxw,42230
61
65
  ScriptCollection/TFCPS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
62
66
  ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py,sha256=BNZPMjGiWLMBH7QDtP06WEVLMnI38NLX4Dpd13bSbi4,16614
@@ -66,7 +70,8 @@ ScriptCollection/TFCPS/Docker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5N
66
70
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py,sha256=bT6Gd5pQpZCw4OQz6HWkPCSn5z__eUUEisABLDSxd0o,200
67
71
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py,sha256=QyjOfMY22JWCvKjMelHiDWbJiWqotOfebpJpgDUaoO4,237
68
72
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py,sha256=i0zEGehj0sttxjjZtoq2KFSKp_ulxVyWp_ZgAhIY_So,241
69
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=-vdQfhPcf69R80Lfsos3k04JMe2hUsClW9XUNu2KHv0,71170
73
+ ScriptCollection/TFCPS/DotNet/NuGetPackageSourceRegistration.py,sha256=E2TiFbwyPZvhozoWqLksc4uH50LbFAaVWOGfUo6for4,7050
74
+ ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=4oRPob7-8mC-8HqkGgyLn2s4t1n4iLOLp5tU6IOHFB4,65013
70
75
  ScriptCollection/TFCPS/DotNet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
71
76
  ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=9PMxJFB4nyKGzl916UOCvHB5toIyLcWk-ZAM8vPdLxw,64571
72
77
  ScriptCollection/TFCPS/Flutter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -82,8 +87,8 @@ ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=yyjWPMHQjPLkML
82
87
  ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
83
88
  ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py,sha256=ILVhJHvwG5TvPBBqSvyACYb7W9QnWrJpxKgHu8IQu4g,9249
84
89
  ScriptCollection/TFCPS/TypeScript/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
85
- scriptcollection-4.4.67.dist-info/METADATA,sha256=ItwhmyrDdan16Rtin5x-2mX33PfHWM2SNJr28T51hes,8032
86
- scriptcollection-4.4.67.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
87
- scriptcollection-4.4.67.dist-info/entry_points.txt,sha256=kI-HqAydo8a0FGjGeXh6i1pGU73Pit0xACdjxBxNkis,6254
88
- scriptcollection-4.4.67.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
89
- scriptcollection-4.4.67.dist-info/RECORD,,
90
+ scriptcollection-4.4.69.dist-info/METADATA,sha256=fqKQHSsZZgSjWeFgvRP3DTWtmqpCvRr-hHmCjdaRhQo,8032
91
+ scriptcollection-4.4.69.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
92
+ scriptcollection-4.4.69.dist-info/entry_points.txt,sha256=ukXPbwM_VSzeSUOLRwWwq4aVAsfFKSzOjQnPkcJH59w,6333
93
+ scriptcollection-4.4.69.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
94
+ scriptcollection-4.4.69.dist-info/RECORD,,
@@ -60,6 +60,7 @@ scprintosname = ScriptCollection.Executables:PrintOSName
60
60
  scrais = ScriptCollection.Executables:RunAgentInSandbox
61
61
  screclaimspacefromdocker = ScriptCollection.Executables:ReclaimSpaceFromDocker
62
62
  scregexreplaceinfile = ScriptCollection.Executables:RegexReplaceInFile
63
+ scregisterpackagesources = ScriptCollection.Executables:RegisterPackageSources
63
64
  scremovefile = ScriptCollection.Executables:RemoveFile
64
65
  scremovefolder = ScriptCollection.Executables:RemoveFolder
65
66
  scremoveinvisiblecharactersfromfile = ScriptCollection.Executables:RemoveInvisibleCharactersFromFile