ScriptCollection 4.4.66__py3-none-any.whl → 4.4.68__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/AgentRunner.py +91 -6
- ScriptCollection/AgentStatusReporting/AgentStatus.py +16 -0
- ScriptCollection/AgentStatusReporting/AgentStatusEventChannel.py +36 -0
- ScriptCollection/AgentStatusReporting/AgentStatusRelay.py +87 -0
- ScriptCollection/AgentStatusReporting/AgentStatusReporter.py +14 -0
- ScriptCollection/AgentStatusReporting/AgentStatusReportingSession.py +76 -0
- ScriptCollection/AgentStatusReporting/HerdrAgentStatusReporter.py +72 -0
- ScriptCollection/AgentStatusReporting/__init__.py +0 -0
- ScriptCollection/Executables.py +14 -0
- ScriptCollection/ScriptCollectionCore.py +1 -1
- ScriptCollection/TFCPS/ContainerBuildEnvironment.py +165 -0
- ScriptCollection/TFCPS/DotNet/NuGetPackageSourceRegistration.py +91 -0
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +2 -75
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +31 -141
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +1 -1
- {scriptcollection-4.4.66.dist-info → scriptcollection-4.4.68.dist-info}/METADATA +1 -1
- {scriptcollection-4.4.66.dist-info → scriptcollection-4.4.68.dist-info}/RECORD +20 -11
- {scriptcollection-4.4.66.dist-info → scriptcollection-4.4.68.dist-info}/entry_points.txt +1 -0
- {scriptcollection-4.4.66.dist-info → scriptcollection-4.4.68.dist-info}/WHEEL +0 -0
- {scriptcollection-4.4.66.dist-info → scriptcollection-4.4.68.dist-info}/top_level.txt +0 -0
ScriptCollection/AgentRunner.py
CHANGED
|
@@ -6,7 +6,11 @@ import tempfile
|
|
|
6
6
|
from enum import Enum
|
|
7
7
|
from .GeneralUtilities import GeneralUtilities
|
|
8
8
|
from .ScriptCollectionCore import ScriptCollectionCore
|
|
9
|
+
from .SCLog import LogLevel
|
|
9
10
|
from .OCIImages.OCIImageManager import OCIImageManager
|
|
11
|
+
from .TFCPS.ContainerBuildEnvironment import ContainerBuildEnvironment
|
|
12
|
+
from .TFCPS.TFCPS_Tools_General import TFCPS_Tools_General
|
|
13
|
+
from .AgentStatusReporting.AgentStatusReportingSession import AgentStatusReportingSession
|
|
10
14
|
|
|
11
15
|
|
|
12
16
|
class CodingAgent(Enum):
|
|
@@ -21,12 +25,26 @@ class AgentRunnerConfiguration:
|
|
|
21
25
|
codingagent: CodingAgent = None
|
|
22
26
|
image: str = None
|
|
23
27
|
drop_capabilities: bool = None
|
|
28
|
+
provide_build_environment: bool = None
|
|
24
29
|
|
|
25
30
|
@GeneralUtilities.check_arguments
|
|
26
31
|
def __init__(self):
|
|
27
32
|
self.codingagent = None
|
|
28
33
|
self.image = None
|
|
29
34
|
self.drop_capabilities = None
|
|
35
|
+
self.provide_build_environment = None
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class SandboxBuildEnvironment:
|
|
39
|
+
"""The build-environment (see ContainerBuildEnvironment) prepared for one start of the sandbox: the docker-arguments of its mounts
|
|
40
|
+
and its environment-variables. The values of the environment-variables are typically secrets, so only their names may become
|
|
41
|
+
docker-arguments; the values are given to the docker-client through its own environment."""
|
|
42
|
+
volume_arguments: list[str] = None
|
|
43
|
+
environment_variables: dict[str, str] = None
|
|
44
|
+
|
|
45
|
+
def __init__(self, volume_arguments: list[str], environment_variables: dict[str, str]):
|
|
46
|
+
self.volume_arguments = volume_arguments
|
|
47
|
+
self.environment_variables = environment_variables
|
|
30
48
|
|
|
31
49
|
|
|
32
50
|
class AgentRunner:
|
|
@@ -52,12 +70,35 @@ class AgentRunner:
|
|
|
52
70
|
if mount_docker_socket:
|
|
53
71
|
self.__print_docker_socket_warning()
|
|
54
72
|
additional_folder_volume_arguments = additional_folder_volume_arguments + ["--volume", "/var/run/docker.sock:/var/run/docker.sock"]
|
|
55
|
-
|
|
73
|
+
build_environment: SandboxBuildEnvironment = None
|
|
74
|
+
if agentRunnerConfiguration.provide_build_environment:
|
|
75
|
+
build_environment = self.__prepare_build_environment(repository)
|
|
76
|
+
return self.__run_sandbox(repository, agentRunnerConfiguration, prompt, interactive, initialization_script, additional_folder_volume_arguments, build_environment)
|
|
56
77
|
|
|
57
78
|
@GeneralUtilities.check_arguments
|
|
58
79
|
def __print_docker_socket_warning(self) -> None:
|
|
59
80
|
GeneralUtilities.write_message_to_stderr("Warning: Mounting the docker-socket of the host into the sandbox is dangerous because it makes the sandbox ineffective.")
|
|
60
81
|
|
|
82
|
+
@GeneralUtilities.check_arguments
|
|
83
|
+
def __prepare_build_environment(self, repository: str) -> SandboxBuildEnvironment:
|
|
84
|
+
# The sandbox gets the same build-environment as a build-container of "scbuildcodeunits -c" (see ContainerBuildEnvironment), so the agent can
|
|
85
|
+
# build the repository and restore its dependencies (also from private package-sources). The host-hook runs first, like for "scbuildcodeunits -c",
|
|
86
|
+
# because it can create the files the values of the environment-variables are read from. A sandbox has no target-environment of its own;
|
|
87
|
+
# "Development" is passed because the hook expects a valid one and an agent develops the repository.
|
|
88
|
+
sc = ScriptCollectionCore()
|
|
89
|
+
container_build_environment = ContainerBuildEnvironment(sc, TFCPS_Tools_General(sc), repository)
|
|
90
|
+
self.__print_build_environment_warning(sc.get_tfcps_configuration_folder())
|
|
91
|
+
verbosity = LogLevel.Debug if self.verbose else LogLevel.Information
|
|
92
|
+
container_build_environment.run_host_preparation_hook(["--repository", repository, "--targetenvironmenttype", "Development", "--verbosity", str(int(verbosity))])
|
|
93
|
+
volume_arguments: list[str] = []
|
|
94
|
+
for mount in container_build_environment.get_mounts():
|
|
95
|
+
volume_arguments = volume_arguments + ["--volume", f"{mount.source.replace(os.sep, '/')}:{mount.target}{':ro' if mount.read_only else ''}"]
|
|
96
|
+
return SandboxBuildEnvironment(volume_arguments, container_build_environment.get_environment_variables())
|
|
97
|
+
|
|
98
|
+
@GeneralUtilities.check_arguments
|
|
99
|
+
def __print_build_environment_warning(self, tfcps_configuration_folder: str) -> None:
|
|
100
|
+
GeneralUtilities.write_message_to_stderr(f"Warning: The build-environment is provided to the sandbox. The agent can read every secret of \"{tfcps_configuration_folder}\" and the values of the required environment-variables of the repository.")
|
|
101
|
+
|
|
61
102
|
@GeneralUtilities.check_arguments
|
|
62
103
|
def __get_additional_folder_volume_arguments(self, additional_folders: list[str]) -> list[str]:
|
|
63
104
|
# Every additional folder is mounted under "/Workspace/Other/AdditionalResources/<folder-name>" (preserving the folder-name).
|
|
@@ -126,10 +167,20 @@ class AgentRunner:
|
|
|
126
167
|
|
|
127
168
|
for name, value in [("codingagent", agentRunnerConfiguration.codingagent), ("image", agentRunnerConfiguration.image), ("drop_capabilities", agentRunnerConfiguration.drop_capabilities)]:
|
|
128
169
|
GeneralUtilities.assert_condition(value is not None, f"The value '{name}' is neither given as argument nor defined in '{agent_sandbox_configuration_file}'.")
|
|
170
|
+
|
|
171
|
+
# Unlike the values above, provide_build_environment is not required: the build-environment is opt-in, so it is only provided if it is enabled
|
|
172
|
+
# explicitly. Requiring it would break every existing configuration-file.
|
|
173
|
+
if configuration_overrides.provide_build_environment is not None:
|
|
174
|
+
agentRunnerConfiguration.provide_build_environment = configuration_overrides.provide_build_environment
|
|
175
|
+
elif config_parser.has_option(section, "provide_build_environment"):
|
|
176
|
+
agentRunnerConfiguration.provide_build_environment = config_parser.getboolean(section, "provide_build_environment")
|
|
177
|
+
else:
|
|
178
|
+
agentRunnerConfiguration.provide_build_environment = False
|
|
129
179
|
return agentRunnerConfiguration
|
|
130
180
|
|
|
131
181
|
@GeneralUtilities.check_arguments
|
|
132
|
-
def __run_sandbox(self, repository: str, agentRunnerConfiguration: AgentRunnerConfiguration, prompt: str, interactive: bool, initialization_script: str, additional_folder_volume_arguments: list[str]) -> int:
|
|
182
|
+
def __run_sandbox(self, repository: str, agentRunnerConfiguration: AgentRunnerConfiguration, prompt: str, interactive: bool, initialization_script: str, additional_folder_volume_arguments: list[str], build_environment: SandboxBuildEnvironment) -> int:
|
|
183
|
+
# build_environment is None if the build-environment is not provided.
|
|
133
184
|
# The container always runs Linux.
|
|
134
185
|
# The repository is the folder given by the caller (which was asserted to be the git-root-directory before).
|
|
135
186
|
# It is mounted into the container under "/Workspace/<repository-name>" (preserving the folder-name) and used as working-directory of the agent.
|
|
@@ -137,8 +188,11 @@ class AgentRunner:
|
|
|
137
188
|
repository_name = os.path.basename(os.path.normpath(host_repository_folder))
|
|
138
189
|
container_workspace_folder = f"/Workspace/{repository_name}"
|
|
139
190
|
agent_command = self.__get_agent_command(agentRunnerConfiguration.codingagent, prompt, interactive)
|
|
191
|
+
# The status of the agent is reported on the same way for every coding-agent. The session is None if there is nothing to report to.
|
|
192
|
+
status_session = AgentStatusReportingSession.try_create(dict(os.environ), f"scrais-{agentRunnerConfiguration.codingagent.value}", self.verbose)
|
|
193
|
+
status_start_line = None if status_session is None else status_session.get_start_report_line(prompt is not None)
|
|
140
194
|
# The entrypoint is always used: if an initialization-script is present it runs it first, and then always starts the agent.
|
|
141
|
-
entrypoint_script_content = self.__get_entrypoint_script_content(agent_command, initialization_script)
|
|
195
|
+
entrypoint_script_content = self.__get_entrypoint_script_content(agent_command, initialization_script, status_start_line, build_environment is not None, container_workspace_folder)
|
|
142
196
|
with tempfile.NamedTemporaryFile("w", suffix=".sh", delete=False, newline="\n", encoding="utf-8") as entrypoint_file_handle:
|
|
143
197
|
host_entrypoint_file = entrypoint_file_handle.name
|
|
144
198
|
entrypoint_file_handle.write(entrypoint_script_content)
|
|
@@ -160,13 +214,35 @@ class AgentRunner:
|
|
|
160
214
|
initialization_script_absolute_path = os.path.abspath(initialization_script)
|
|
161
215
|
os.chmod(initialization_script_absolute_path, 0o755)
|
|
162
216
|
docker_command = docker_command + ["--volume", f"{initialization_script_absolute_path.replace(os.sep, '/')}:/Workspace/Temp/InitializationScript.sh:ro"]
|
|
217
|
+
if status_session is not None:
|
|
218
|
+
docker_command = docker_command + status_session.get_volume_arguments()
|
|
219
|
+
# The environment of the docker-client; it differs from the one of this process only if the build-environment is provided.
|
|
220
|
+
docker_client_environment: dict[str, str] = None
|
|
221
|
+
if build_environment is not None:
|
|
222
|
+
docker_command = docker_command + build_environment.volume_arguments
|
|
223
|
+
# Only the names are passed as arguments ("--env <name>" takes the value from the environment of the docker-client), because
|
|
224
|
+
# arguments are visible in the process-list of the host and the values are typically secrets.
|
|
225
|
+
for environment_variable_name in build_environment.environment_variables:
|
|
226
|
+
docker_command = docker_command + ["--env", environment_variable_name]
|
|
227
|
+
docker_client_environment = {**os.environ, **build_environment.environment_variables}
|
|
163
228
|
image = self.__get_image_to_use(agentRunnerConfiguration.image, host_repository_folder)
|
|
164
|
-
docker_command = docker_command + ["--workdir", container_workspace_folder
|
|
229
|
+
docker_command = docker_command + ["--workdir", container_workspace_folder]
|
|
230
|
+
if build_environment is None:
|
|
231
|
+
docker_command = docker_command + ["--entrypoint", "sh", image, "/Workspace/Entrypoint.sh"]
|
|
232
|
+
else:
|
|
233
|
+
# With the build-environment the entrypoint of the image is kept and runs the generated entrypoint as its command: the entrypoint of the
|
|
234
|
+
# SCBuilder-image installs the trusted root-CA-certificates of the build-environment (and then runs "exec \"$@\""), exactly like in a
|
|
235
|
+
# build-container of "scbuildcodeunits -c". Without the build-environment the entrypoint of the image is replaced as before.
|
|
236
|
+
docker_command = docker_command + [image, "sh", "/Workspace/Entrypoint.sh"]
|
|
165
237
|
# Nothing is installed in the container; the coding-agent-CLI is assumed to be available in the image.
|
|
166
238
|
# subprocess.run blocks until the container has exited, which happens when the agent is finished (or was terminated by the user in interactive mode).
|
|
167
239
|
# The exit-code of the container is returned (and not checked here), so that the caller can pass it through.
|
|
168
|
-
|
|
240
|
+
if status_session is not None:
|
|
241
|
+
status_session.start()
|
|
242
|
+
return subprocess.run(docker_command, check=False, env=docker_client_environment).returncode
|
|
169
243
|
finally:
|
|
244
|
+
if status_session is not None:
|
|
245
|
+
status_session.stop()
|
|
170
246
|
GeneralUtilities.ensure_file_does_not_exist(host_entrypoint_file)
|
|
171
247
|
|
|
172
248
|
@GeneralUtilities.check_arguments
|
|
@@ -183,13 +259,20 @@ class AgentRunner:
|
|
|
183
259
|
return configured_image
|
|
184
260
|
|
|
185
261
|
@GeneralUtilities.check_arguments
|
|
186
|
-
def __get_entrypoint_script_content(self, agent_command: list[str], initialization_script: str) -> str:
|
|
262
|
+
def __get_entrypoint_script_content(self, agent_command: list[str], initialization_script: str, status_start_line: str, provide_build_environment: bool, container_workspace_folder: str) -> str:
|
|
187
263
|
# The entrypoint-script is always used as the container's entrypoint (located at "/Workspace/Entrypoint.sh").
|
|
188
264
|
# It is regenerated before every container-start, so the decision whether the initialization-script is called is made here in python
|
|
189
265
|
# (no runtime-check in the script): if an initialization-script is present its call is included (and the script is removed afterwards),
|
|
190
266
|
# otherwise it is omitted. Afterwards the agent is always started (via exec, so signals reach the agent).
|
|
267
|
+
# status_start_line is None if the status is not reported; it is the only runtime-check, because only the container knows whether its image supports the status-report.
|
|
191
268
|
agent_command_line = shlex.join(agent_command)
|
|
192
269
|
lines = ["#!/bin/sh", "set -e"]
|
|
270
|
+
if provide_build_environment:
|
|
271
|
+
# The declared package-sources are registered before the initialization-script, so that script and the agent can already use them directly
|
|
272
|
+
# (for example with "dotnet restore") and not only through "scbuildcodeunits". The command comes from the ScriptCollection of the image; an
|
|
273
|
+
# image without it is rejected instead of starting the agent without the package-sources.
|
|
274
|
+
lines.append('if ! command -v scregisterpackagesources >/dev/null 2>&1; then echo "The build-environment requires the command \\"scregisterpackagesources\\" of ScriptCollection, which is not available in this image. Use a newer image." >&2; exit 1; fi')
|
|
275
|
+
lines.append(f"scregisterpackagesources -r {shlex.quote(container_workspace_folder)}")
|
|
193
276
|
if initialization_script is not None:
|
|
194
277
|
# A line is written to stdout before the initialization-script runs, so it is unambiguous in the container-output that an initialization-script was configured and executed (without it one cannot tell whether the agent started unconfigured because no script was declared or because a declared script did nothing).
|
|
195
278
|
lines.append('echo "Running initialization-script for the agent-sandbox..."')
|
|
@@ -199,6 +282,8 @@ class AgentRunner:
|
|
|
199
282
|
else:
|
|
200
283
|
# The counterpart of the message above: it makes clear that the agent is started unconfigured on purpose because no initialization-script is declared for this repository (see __get_initialization_script).
|
|
201
284
|
lines.append('echo "No initialization-script is configured for this repository; the agent is started without an initialization-script."')
|
|
285
|
+
if status_start_line is not None:
|
|
286
|
+
lines.append(status_start_line)
|
|
202
287
|
lines.append(f"exec {agent_command_line}")
|
|
203
288
|
return "\n".join(lines) + "\n"
|
|
204
289
|
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class AgentStatus(Enum):
|
|
5
|
+
"""The closed vocabulary of the channel between the container and the host."""
|
|
6
|
+
Working = "working"
|
|
7
|
+
Idle = "idle"
|
|
8
|
+
Blocked = "blocked"
|
|
9
|
+
|
|
10
|
+
@staticmethod
|
|
11
|
+
def parse_token(line: bytes) -> "AgentStatus":
|
|
12
|
+
# The line comes from a file which the container can write, so it is untrusted: only an exact token is accepted and everything else results in None.
|
|
13
|
+
for status in AgentStatus:
|
|
14
|
+
if line == status.value.encode("ascii"):
|
|
15
|
+
return status
|
|
16
|
+
return None
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import tempfile
|
|
3
|
+
from ..GeneralUtilities import GeneralUtilities
|
|
4
|
+
from .AgentStatus import AgentStatus
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class AgentStatusEventChannel:
|
|
8
|
+
"""The channel from the container to the host: one event-file in a dedicated host-folder which is mounted into the container."""
|
|
9
|
+
container_folder: str = "/Workspace/Temp/AgentStatus"
|
|
10
|
+
status_helper: str = "scrais-agent-status"
|
|
11
|
+
__host_folder: str = None
|
|
12
|
+
|
|
13
|
+
@GeneralUtilities.check_arguments
|
|
14
|
+
def __init__(self):
|
|
15
|
+
self.__host_folder = tempfile.mkdtemp(prefix="ScraisAgentStatus")
|
|
16
|
+
# The user of the container is unknown and it can be a user without the capability to bypass file-permissions (see drop_capabilities), so everybody must be able to write.
|
|
17
|
+
os.chmod(self.__host_folder, 0o777)
|
|
18
|
+
GeneralUtilities.ensure_file_exists(self.get_events_file())
|
|
19
|
+
os.chmod(self.get_events_file(), 0o666)
|
|
20
|
+
|
|
21
|
+
@GeneralUtilities.check_arguments
|
|
22
|
+
def get_events_file(self) -> str:
|
|
23
|
+
return os.path.join(self.__host_folder, "events")
|
|
24
|
+
|
|
25
|
+
@GeneralUtilities.check_arguments
|
|
26
|
+
def get_volume_arguments(self) -> list[str]:
|
|
27
|
+
return ["--volume", f"{self.__host_folder.replace(os.sep, '/')}:{self.container_folder}"]
|
|
28
|
+
|
|
29
|
+
@GeneralUtilities.check_arguments
|
|
30
|
+
def get_start_report_line(self, status: AgentStatus) -> str:
|
|
31
|
+
# An image without the status-helper does not report anything. The "|| true" prevents that this line can stop the entrypoint, which runs with "set -e".
|
|
32
|
+
return f"command -v {self.status_helper} >/dev/null 2>&1 && {self.status_helper} {status.value} || true"
|
|
33
|
+
|
|
34
|
+
@GeneralUtilities.check_arguments
|
|
35
|
+
def remove(self) -> None:
|
|
36
|
+
GeneralUtilities.ensure_directory_does_not_exist(self.__host_folder)
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import threading
|
|
2
|
+
from ..GeneralUtilities import GeneralUtilities
|
|
3
|
+
from .AgentStatus import AgentStatus
|
|
4
|
+
from .AgentStatusReporter import AgentStatusReporter
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class AgentStatusRelay:
|
|
8
|
+
"""Forwards the valid states from the event-file to a reporter. The file is written by the container, so its content is untrusted.
|
|
9
|
+
To limit what a container can cause, a read is limited in size and it results in at most one report, which is the latest state of that read."""
|
|
10
|
+
poll_interval_in_seconds: float = 0.25
|
|
11
|
+
maximal_line_length: int = 64
|
|
12
|
+
maximal_read_size: int = 65536
|
|
13
|
+
__events_file: str = None
|
|
14
|
+
__reporter: AgentStatusReporter = None
|
|
15
|
+
__verbose: bool = None
|
|
16
|
+
__offset: int = 0
|
|
17
|
+
__unfinished_line: bytes = b""
|
|
18
|
+
__skip_rest_of_line: bool = False
|
|
19
|
+
__last_status: AgentStatus = None
|
|
20
|
+
__stop_requested: threading.Event = None
|
|
21
|
+
__thread: threading.Thread = None
|
|
22
|
+
|
|
23
|
+
@GeneralUtilities.check_arguments
|
|
24
|
+
def __init__(self, events_file: str, reporter: AgentStatusReporter, verbose: bool):
|
|
25
|
+
self.__events_file = events_file
|
|
26
|
+
self.__reporter = reporter
|
|
27
|
+
self.__verbose = verbose
|
|
28
|
+
self.__offset = 0
|
|
29
|
+
self.__unfinished_line = b""
|
|
30
|
+
self.__skip_rest_of_line = False
|
|
31
|
+
self.__last_status = None
|
|
32
|
+
self.__stop_requested = threading.Event()
|
|
33
|
+
self.__thread = threading.Thread(target=self.__poll, daemon=True)
|
|
34
|
+
|
|
35
|
+
@GeneralUtilities.check_arguments
|
|
36
|
+
def has_reported(self) -> bool:
|
|
37
|
+
return self.__last_status is not None
|
|
38
|
+
|
|
39
|
+
@GeneralUtilities.check_arguments
|
|
40
|
+
def start(self) -> None:
|
|
41
|
+
self.__thread.start()
|
|
42
|
+
|
|
43
|
+
@GeneralUtilities.check_arguments
|
|
44
|
+
def stop(self) -> None:
|
|
45
|
+
# The final read gets what the agent has written between the last poll and the end of the container.
|
|
46
|
+
self.__stop_requested.set()
|
|
47
|
+
if self.__thread.is_alive():
|
|
48
|
+
self.__thread.join()
|
|
49
|
+
self.process_new_events()
|
|
50
|
+
|
|
51
|
+
@GeneralUtilities.check_arguments
|
|
52
|
+
def process_new_events(self) -> None:
|
|
53
|
+
status = self.__read_latest_status()
|
|
54
|
+
if status is not None and status != self.__last_status:
|
|
55
|
+
self.__last_status = status
|
|
56
|
+
self.__reporter.report(status)
|
|
57
|
+
|
|
58
|
+
@GeneralUtilities.check_arguments
|
|
59
|
+
def __poll(self) -> None:
|
|
60
|
+
try:
|
|
61
|
+
while not self.__stop_requested.wait(self.poll_interval_in_seconds):
|
|
62
|
+
self.process_new_events()
|
|
63
|
+
except Exception as exception: # pylint: disable=broad-exception-caught
|
|
64
|
+
# This runs in an own thread beside the agent, so a failure only ends the reporting.
|
|
65
|
+
if self.__verbose:
|
|
66
|
+
GeneralUtilities.write_message_to_stderr(f"Reading the agent-status failed: {exception}")
|
|
67
|
+
|
|
68
|
+
@GeneralUtilities.check_arguments
|
|
69
|
+
def __read_latest_status(self) -> AgentStatus:
|
|
70
|
+
with open(self.__events_file, "rb") as events_file:
|
|
71
|
+
events_file.seek(self.__offset)
|
|
72
|
+
new_content = events_file.read(self.maximal_read_size)
|
|
73
|
+
self.__offset = self.__offset + len(new_content)
|
|
74
|
+
lines = (self.__unfinished_line + new_content).split(b"\n")
|
|
75
|
+
self.__unfinished_line = lines.pop()
|
|
76
|
+
latest_status: AgentStatus = None
|
|
77
|
+
for line in lines:
|
|
78
|
+
if self.__skip_rest_of_line:
|
|
79
|
+
self.__skip_rest_of_line = False
|
|
80
|
+
continue
|
|
81
|
+
status = AgentStatus.parse_token(line)
|
|
82
|
+
if status is not None:
|
|
83
|
+
latest_status = status
|
|
84
|
+
if self.maximal_line_length < len(self.__unfinished_line):
|
|
85
|
+
self.__unfinished_line = b""
|
|
86
|
+
self.__skip_rest_of_line = True
|
|
87
|
+
return latest_status
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from .AgentStatus import AgentStatus
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class AgentStatusReporter(ABC):
|
|
6
|
+
"""Receives the state of the agent. An implementation must not raise on a failed report, because the agent has to keep running in that case."""
|
|
7
|
+
|
|
8
|
+
@abstractmethod
|
|
9
|
+
def report(self, status: AgentStatus) -> None:
|
|
10
|
+
raise NotImplementedError
|
|
11
|
+
|
|
12
|
+
@abstractmethod
|
|
13
|
+
def release(self) -> None:
|
|
14
|
+
raise NotImplementedError
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
from ..GeneralUtilities import GeneralUtilities
|
|
2
|
+
from .AgentStatus import AgentStatus
|
|
3
|
+
from .AgentStatusEventChannel import AgentStatusEventChannel
|
|
4
|
+
from .AgentStatusRelay import AgentStatusRelay
|
|
5
|
+
from .AgentStatusReporter import AgentStatusReporter
|
|
6
|
+
from .HerdrAgentStatusReporter import HerdrAgentStatusReporter
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class AgentStatusReportingSession:
|
|
10
|
+
"""The status-report of one run of a container. It knows no concrete coding-agent: the image connects its agents to the channel.
|
|
11
|
+
Nothing in here can stop the agent: a failure disables the report for this run and is shown in verbose output only."""
|
|
12
|
+
__channel: AgentStatusEventChannel = None
|
|
13
|
+
__relay: AgentStatusRelay = None
|
|
14
|
+
__reporter: AgentStatusReporter = None
|
|
15
|
+
__verbose: bool = None
|
|
16
|
+
|
|
17
|
+
@GeneralUtilities.check_arguments
|
|
18
|
+
def __init__(self, channel: AgentStatusEventChannel, relay: AgentStatusRelay, reporter: AgentStatusReporter, verbose: bool):
|
|
19
|
+
self.__channel = channel
|
|
20
|
+
self.__relay = relay
|
|
21
|
+
self.__reporter = reporter
|
|
22
|
+
self.__verbose = verbose
|
|
23
|
+
|
|
24
|
+
@staticmethod
|
|
25
|
+
@GeneralUtilities.check_arguments
|
|
26
|
+
def try_create(environment: dict[str, str], agent_name: str, verbose: bool) -> "AgentStatusReportingSession":
|
|
27
|
+
# Returns None if there is nothing to report to or if the channel can not be created.
|
|
28
|
+
reporter = HerdrAgentStatusReporter.try_create(environment, agent_name, verbose)
|
|
29
|
+
if reporter is None:
|
|
30
|
+
return None
|
|
31
|
+
try:
|
|
32
|
+
channel = AgentStatusEventChannel()
|
|
33
|
+
except OSError as exception:
|
|
34
|
+
AgentStatusReportingSession.__write_verbose_message(verbose, f"The agent-status is not reported because its channel can not be created: {exception}")
|
|
35
|
+
return None
|
|
36
|
+
return AgentStatusReportingSession(channel, AgentStatusRelay(channel.get_events_file(), reporter, verbose), reporter, verbose)
|
|
37
|
+
|
|
38
|
+
@GeneralUtilities.check_arguments
|
|
39
|
+
def get_volume_arguments(self) -> list[str]:
|
|
40
|
+
return self.__channel.get_volume_arguments()
|
|
41
|
+
|
|
42
|
+
@GeneralUtilities.check_arguments
|
|
43
|
+
def get_start_report_line(self, prompt_is_given: bool) -> str:
|
|
44
|
+
return self.__channel.get_start_report_line(AgentStatus.Working if prompt_is_given else AgentStatus.Idle)
|
|
45
|
+
|
|
46
|
+
@GeneralUtilities.check_arguments
|
|
47
|
+
def start(self) -> None:
|
|
48
|
+
try:
|
|
49
|
+
self.__relay.start()
|
|
50
|
+
except RuntimeError as exception:
|
|
51
|
+
AgentStatusReportingSession.__write_verbose_message(self.__verbose, f"The agent-status is not reported because reading it can not be started: {exception}")
|
|
52
|
+
|
|
53
|
+
@GeneralUtilities.check_arguments
|
|
54
|
+
def stop(self) -> None:
|
|
55
|
+
try:
|
|
56
|
+
self.__finish()
|
|
57
|
+
except Exception as exception: # pylint: disable=broad-exception-caught
|
|
58
|
+
# This runs after the agent has ended, so a failure must not replace its exit-code.
|
|
59
|
+
AgentStatusReportingSession.__write_verbose_message(self.__verbose, f"Finishing the report of the agent-status failed: {exception}")
|
|
60
|
+
|
|
61
|
+
@GeneralUtilities.check_arguments
|
|
62
|
+
def __finish(self) -> None:
|
|
63
|
+
try:
|
|
64
|
+
self.__relay.stop()
|
|
65
|
+
if self.__relay.has_reported():
|
|
66
|
+
self.__reporter.release()
|
|
67
|
+
else:
|
|
68
|
+
AgentStatusReportingSession.__write_verbose_message(self.__verbose, f"No agent-status was received. An image only reports the status if it contains '{AgentStatusEventChannel.status_helper}'.")
|
|
69
|
+
finally:
|
|
70
|
+
self.__channel.remove()
|
|
71
|
+
|
|
72
|
+
@staticmethod
|
|
73
|
+
@GeneralUtilities.check_arguments
|
|
74
|
+
def __write_verbose_message(verbose: bool, message: str) -> None:
|
|
75
|
+
if verbose:
|
|
76
|
+
GeneralUtilities.write_message_to_stderr(message)
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import subprocess
|
|
2
|
+
import time
|
|
3
|
+
from ..GeneralUtilities import GeneralUtilities
|
|
4
|
+
from .AgentStatus import AgentStatus
|
|
5
|
+
from .AgentStatusReporter import AgentStatusReporter
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class HerdrAgentStatusReporter(AgentStatusReporter):
|
|
9
|
+
"""Reports the state of the agent with the herdr-CLI."""
|
|
10
|
+
source: str = "scrais"
|
|
11
|
+
timeout_in_seconds: int = 5
|
|
12
|
+
__herdr_program: str = None
|
|
13
|
+
__pane_id: str = None
|
|
14
|
+
__agent_name: str = None
|
|
15
|
+
__verbose: bool = None
|
|
16
|
+
__last_sequence_number: int = 0
|
|
17
|
+
|
|
18
|
+
@GeneralUtilities.check_arguments
|
|
19
|
+
def __init__(self, herdr_program: str, pane_id: str, agent_name: str, verbose: bool):
|
|
20
|
+
self.__herdr_program = herdr_program
|
|
21
|
+
self.__pane_id = pane_id
|
|
22
|
+
self.__agent_name = agent_name
|
|
23
|
+
self.__verbose = verbose
|
|
24
|
+
self.__last_sequence_number = 0
|
|
25
|
+
|
|
26
|
+
@staticmethod
|
|
27
|
+
@GeneralUtilities.check_arguments
|
|
28
|
+
def try_create(environment: dict[str, str], agent_name: str, verbose: bool) -> "HerdrAgentStatusReporter":
|
|
29
|
+
# Returns None if scrais does not run in a herdr-pane.
|
|
30
|
+
pane_id = environment.get("HERDR_PANE_ID")
|
|
31
|
+
herdr_program = environment.get("HERDR_BIN_PATH")
|
|
32
|
+
if environment.get("HERDR_ENV") != "1" or not pane_id or not herdr_program:
|
|
33
|
+
return None
|
|
34
|
+
return HerdrAgentStatusReporter(herdr_program, pane_id, agent_name, verbose)
|
|
35
|
+
|
|
36
|
+
@GeneralUtilities.check_arguments
|
|
37
|
+
def get_report_command(self, status: AgentStatus) -> list[str]:
|
|
38
|
+
return [self.__herdr_program, "pane", "report-agent", self.__pane_id, "--source", self.source, "--agent", self.__agent_name, "--state", status.value, "--seq", str(self.__get_next_sequence_number())]
|
|
39
|
+
|
|
40
|
+
@GeneralUtilities.check_arguments
|
|
41
|
+
def get_release_command(self) -> list[str]:
|
|
42
|
+
return [self.__herdr_program, "pane", "release-agent", self.__pane_id, "--source", self.source, "--agent", self.__agent_name, "--seq", str(self.__get_next_sequence_number())]
|
|
43
|
+
|
|
44
|
+
@GeneralUtilities.check_arguments
|
|
45
|
+
def report(self, status: AgentStatus) -> None:
|
|
46
|
+
self.__run(self.get_report_command(status))
|
|
47
|
+
|
|
48
|
+
@GeneralUtilities.check_arguments
|
|
49
|
+
def release(self) -> None:
|
|
50
|
+
self.__run(self.get_release_command())
|
|
51
|
+
|
|
52
|
+
@GeneralUtilities.check_arguments
|
|
53
|
+
def __get_next_sequence_number(self) -> int:
|
|
54
|
+
# time_ns makes the number increase across runs; the comparison keeps it strictly increasing if the resolution of the clock is too coarse for two reports in quick succession.
|
|
55
|
+
self.__last_sequence_number = max(time.time_ns(), self.__last_sequence_number + 1)
|
|
56
|
+
return self.__last_sequence_number
|
|
57
|
+
|
|
58
|
+
@GeneralUtilities.check_arguments
|
|
59
|
+
def __run(self, command: list[str]) -> None:
|
|
60
|
+
# The command is an argument-list without shell, so that paths with spaces work and nothing can be injected.
|
|
61
|
+
# Stdin and the output are detached, because the terminal belongs to the agent.
|
|
62
|
+
try:
|
|
63
|
+
result = subprocess.run(command, check=False, stdin=subprocess.DEVNULL, capture_output=True, timeout=self.timeout_in_seconds)
|
|
64
|
+
if result.returncode != 0:
|
|
65
|
+
self.__write_failure(f"exit-code {result.returncode}")
|
|
66
|
+
except (OSError, subprocess.SubprocessError) as exception:
|
|
67
|
+
self.__write_failure(str(exception))
|
|
68
|
+
|
|
69
|
+
@GeneralUtilities.check_arguments
|
|
70
|
+
def __write_failure(self, reason: str) -> None:
|
|
71
|
+
if self.__verbose:
|
|
72
|
+
GeneralUtilities.write_message_to_stderr(f"Reporting the agent-status to herdr failed: {reason}")
|
|
File without changes
|
ScriptCollection/Executables.py
CHANGED
|
@@ -14,6 +14,7 @@ from .SCLog import LogLevel, SCLog
|
|
|
14
14
|
from .TFCPS.TFCPS_CodeUnit_BuildCodeUnits import TFCPS_CodeUnit_BuildCodeUnits
|
|
15
15
|
from .TFCPS.TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
|
|
16
16
|
from .TFCPS.TFCPS_Tools_General import TFCPS_Tools_General
|
|
17
|
+
from .TFCPS.DotNet.NuGetPackageSourceRegistration import NuGetPackageSourceRegistration
|
|
17
18
|
from .OCIImages.OCIImageManager import OCIImageManager
|
|
18
19
|
from .AgentRunner import AgentRunner, AgentRunnerConfiguration, CodingAgent
|
|
19
20
|
|
|
@@ -28,6 +29,7 @@ def RunAgentInSandbox() -> int:
|
|
|
28
29
|
parser.add_argument('--image', default=None, required=False, help='OCI-image which should be used for the sandbox: either an image-name defined in ".ScriptCollection/OCIImages/ImageDefinition.csv" or an image-address including tag. Overrides the value from ".ScriptCollection/AgentSandboxConfiguration.ini".')
|
|
29
30
|
parser.add_argument('-s','--dropcapabilities', default=None, required=False, choices=["true", "false"], help='Defines whether all capabilities should be dropped in the sandbox. Overrides the value from ".ScriptCollection/AgentSandboxConfiguration.ini".')
|
|
30
31
|
parser.add_argument('-a', '--additionalfolder', action='append', default=None, required=False, help='Folder which should additionally be mounted (read-write) into the sandbox under "/Workspace/Other/AdditionalResources/<folder-name>". Can be given multiple times.')
|
|
32
|
+
parser.add_argument('-b','--providebuildenvironment', default=None, required=False, choices=["true", "false"], help='Defines whether the sandbox gets the same build-environment as a build-container of "scbuildcodeunits -c" (required environment-variables, configuration-folder, custom scripts, image-registries, trusted root-CAs and registered package-sources). The agent can then read every secret of the TFCPS-configuration-folder. Overrides the value from ".ScriptCollection/AgentSandboxConfiguration.ini".')
|
|
31
33
|
parser.add_argument('-d','--mountdockersocket', action='store_true', required=False, default=False, help='Mounts the docker-socket of the host (/var/run/docker.sock) into the sandbox so that the agent can use docker. DANGEROUS: this effectively gives the agent administrator-access to the docker-daemon of the host and makes the restrictions of the sandbox ineffective. Can only be set via this argument (not in the configuration-file); a warning is printed to stderr.')
|
|
32
34
|
args = parser.parse_args()
|
|
33
35
|
repo=os.getcwd()
|
|
@@ -40,6 +42,8 @@ def RunAgentInSandbox() -> int:
|
|
|
40
42
|
configuration_overrides.image = args.image
|
|
41
43
|
if args.dropcapabilities is not None:
|
|
42
44
|
configuration_overrides.drop_capabilities = args.dropcapabilities == "true"
|
|
45
|
+
if args.providebuildenvironment is not None:
|
|
46
|
+
configuration_overrides.provide_build_environment = args.providebuildenvironment == "true"
|
|
43
47
|
additional_folders: list[str] = []
|
|
44
48
|
if args.additionalfolder is not None:
|
|
45
49
|
additional_folders = [GeneralUtilities.resolve_relative_path(additional_folder, os.getcwd()) for additional_folder in args.additionalfolder]
|
|
@@ -1352,6 +1356,16 @@ def ShowProjectVersion() -> int:
|
|
|
1352
1356
|
return 0
|
|
1353
1357
|
|
|
1354
1358
|
|
|
1359
|
+
def RegisterPackageSources() -> int:
|
|
1360
|
+
parser = argparse.ArgumentParser(description="Registers every package-source which is declared in the environment of this process by the environment-variables \"Dependency_<technology>_<name>_URL\" (and optionally \"_Username\" and \"_Password\") with its credentials on this machine, exactly like a codeunit-build does it. Package-sources which are not declared are not changed.")
|
|
1361
|
+
parser.add_argument('-r', '--folder', required=False, default=None, help="Folder in which the tools of the technologies (for example the dotnet-cli) are run. Defaults to the current working directory.")
|
|
1362
|
+
args = parser.parse_args()
|
|
1363
|
+
folder = GeneralUtilities.resolve_relative_path(args.folder, os.getcwd()) if args.folder is not None else os.getcwd()
|
|
1364
|
+
sc: ScriptCollectionCore = ScriptCollectionCore()
|
|
1365
|
+
NuGetPackageSourceRegistration(sc, TFCPS_Tools_General(sc)).register_declared_package_sources(folder)
|
|
1366
|
+
return 0
|
|
1367
|
+
|
|
1368
|
+
|
|
1355
1369
|
def RunCommandInFolder() -> int:
|
|
1356
1370
|
parser = argparse.ArgumentParser(description="Runs a command in the base-folder if (and only if) the actual folder is located inside the base-folder and not inside an excluded folder. The command is always executed in the base-folder; the actual folder is only passed to the command by using the placeholder \"{actual_folder}\" in the arguments (for example: -c git -a \"-C {actual_folder} status\").")
|
|
1357
1371
|
parser.add_argument('-b', '--basefolder', required=True, help="Folder in which the command is executed.")
|