ScriptCollection 4.4.61__py3-none-any.whl → 4.4.63__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,229 @@
1
+ import os
2
+ import subprocess
3
+ import configparser
4
+ import shlex
5
+ import tempfile
6
+ from enum import Enum
7
+ from .GeneralUtilities import GeneralUtilities
8
+ from .ScriptCollectionCore import ScriptCollectionCore
9
+ from .OCIImages.OCIImageManager import OCIImageManager
10
+
11
+
12
+ class CodingAgent(Enum):
13
+ Copilot = "copilot"
14
+ Claude = "claude"
15
+ OpenCode = "opencode"
16
+ Vibe = "vibe"
17
+ Codex = "codex"
18
+
19
+
20
+ class AgentRunnerConfiguration:
21
+ codingagent: CodingAgent = None
22
+ image: str = None
23
+ drop_capabilities: bool = None
24
+
25
+ @GeneralUtilities.check_arguments
26
+ def __init__(self):
27
+ self.codingagent = None
28
+ self.image = None
29
+ self.drop_capabilities = None
30
+
31
+
32
+ class AgentRunner:
33
+ """AgentRunner ("rais" stands for "run agent in sandbox") runs an AI-coding-agent (for example the claude-CLI) interactively in a sandboxed environment."""
34
+ verbose: bool = None
35
+
36
+ @GeneralUtilities.check_arguments
37
+ def __init__(self, verbose: bool):
38
+ self.verbose = verbose
39
+
40
+ @GeneralUtilities.check_arguments
41
+ def run(self, prompt: str, interactive: bool, repository: str, configuration_overrides: AgentRunnerConfiguration, additional_folders: list[str]) -> int:
42
+ # Returns the exit-code of the container, which is the exit-code of the initialization-script if it failed and the exit-code of the agent otherwise.
43
+ # In interactive mode the prompt is optional: without a prompt the agent is started without an initial instruction and the user can instruct it interactively.
44
+ # Every value of configuration_overrides which is not None overrides the corresponding value from the configuration-file.
45
+ # additional_folders are absolute host-folders which are additionally mounted (read-write) under "/Workspace/Other/AdditionalResources/<folder-name>".
46
+ GeneralUtilities.assert_condition(interactive or prompt is not None, "A prompt is required when the agent is not run in interactive mode.")
47
+ self.__assert_is_git_root_dir(repository)
48
+ agentRunnerConfiguration: AgentRunnerConfiguration = self.__load_configuration(repository, configuration_overrides)
49
+ initialization_script = self.__get_initialization_script(repository)
50
+ additional_folder_volume_arguments = self.__get_additional_folder_volume_arguments(additional_folders)
51
+ return self.__run_sandbox(repository, agentRunnerConfiguration, prompt, interactive, initialization_script, additional_folder_volume_arguments)
52
+
53
+ @GeneralUtilities.check_arguments
54
+ def __get_additional_folder_volume_arguments(self, additional_folders: list[str]) -> list[str]:
55
+ # Every additional folder is mounted under "/Workspace/Other/AdditionalResources/<folder-name>" (preserving the folder-name).
56
+ # Because the folder-name is the mount-target, folder-names must be unique.
57
+ result: list[str] = []
58
+ used_folder_names: list[str] = []
59
+ for additional_folder in additional_folders:
60
+ GeneralUtilities.assert_condition(os.path.isdir(additional_folder), f"The additional folder '{additional_folder}' does not exist.")
61
+ folder_name = os.path.basename(os.path.normpath(additional_folder))
62
+ GeneralUtilities.assert_condition(folder_name != "", f"The additional folder '{additional_folder}' has no folder-name and can therefore not be mounted.")
63
+ GeneralUtilities.assert_condition(folder_name not in used_folder_names, f"The folder-name '{folder_name}' is used by more than one additional folder.")
64
+ used_folder_names.append(folder_name)
65
+ result = result + ["--volume", f"{additional_folder.replace(os.sep, '/')}:/Workspace/Other/AdditionalResources/{folder_name}"]
66
+ return result
67
+
68
+ @GeneralUtilities.check_arguments
69
+ def __get_initialization_script(self, repository: str) -> str:
70
+ # The user can define per-repository initialization-scripts in "~/.ScriptCollection/RunAIInSandbox/InitializationScripts.csv".
71
+ # The expected structure is "repository;scriptfile" (one row per repository, acting as a dictionary from repository to initialization-script).
72
+ # If the given repository is listed in the first column (compared as absolute paths), the script-file given in the second column is returned, otherwise None.
73
+ initialization_scripts_file = os.path.join(os.path.expanduser("~"), ".ScriptCollection", "RunAIInSandbox", "InitializationScripts.csv")
74
+ if not os.path.isfile(initialization_scripts_file):
75
+ return None
76
+ normalized_repository = os.path.normcase(os.path.realpath(repository))
77
+ for row in GeneralUtilities.read_csv_file(initialization_scripts_file, ignore_first_line=True):
78
+ if len(row) < 2:
79
+ continue
80
+ if os.path.normcase(os.path.realpath(row[0])) == normalized_repository:
81
+ return row[1]
82
+ return None
83
+
84
+ @GeneralUtilities.check_arguments
85
+ def __assert_is_git_root_dir(self, repository: str) -> None:
86
+ ScriptCollectionCore().assert_is_git_repository(repository)
87
+ repository_root = subprocess.run(["git", "rev-parse", "--show-toplevel"], cwd=repository, capture_output=True, text=True, check=False)
88
+ GeneralUtilities.assert_condition(repository_root.returncode == 0 and os.path.normcase(os.path.realpath(repository_root.stdout.strip())) == os.path.normcase(os.path.realpath(repository)), f"The folder '{repository}' is not the top-level-directory of its git-repository.")
89
+
90
+ @GeneralUtilities.check_arguments
91
+ def __load_configuration(self, repository: str, configuration_overrides: AgentRunnerConfiguration) -> AgentRunnerConfiguration:
92
+ # The configuration-file is optional. Each value is taken from configuration_overrides if it is set there, otherwise from the configuration-file.
93
+ # If a value is neither set in configuration_overrides nor in the configuration-file, an error is raised.
94
+ section = "AgentSandboxConfiguration"
95
+ agent_sandbox_configuration_file = os.path.join(repository, ".ScriptCollection", "AgentSandboxConfiguration.ini")
96
+ config_parser = configparser.ConfigParser()
97
+ if os.path.isfile(agent_sandbox_configuration_file):
98
+ config_parser.read(agent_sandbox_configuration_file, encoding="utf-8")
99
+ agentRunnerConfiguration: AgentRunnerConfiguration = AgentRunnerConfiguration()
100
+
101
+ if configuration_overrides.codingagent is not None:
102
+ agentRunnerConfiguration.codingagent = configuration_overrides.codingagent
103
+ elif config_parser.has_option(section, "codingagent"):
104
+ codingagent_value = config_parser.get(section, "codingagent")
105
+ allowed_codingagent_values = [coding_agent.value for coding_agent in CodingAgent]
106
+ GeneralUtilities.assert_condition(codingagent_value in allowed_codingagent_values, f"Invalid value '{codingagent_value}' for 'codingagent'. Allowed values are: {', '.join(allowed_codingagent_values)}.")
107
+ agentRunnerConfiguration.codingagent = CodingAgent(codingagent_value)
108
+
109
+ if configuration_overrides.image is not None:
110
+ agentRunnerConfiguration.image = configuration_overrides.image
111
+ elif config_parser.has_option(section, "image"):
112
+ agentRunnerConfiguration.image = config_parser.get(section, "image")
113
+
114
+ if configuration_overrides.drop_capabilities is not None:
115
+ agentRunnerConfiguration.drop_capabilities = configuration_overrides.drop_capabilities
116
+ elif config_parser.has_option(section, "drop_capabilities"):
117
+ agentRunnerConfiguration.drop_capabilities = config_parser.getboolean(section, "drop_capabilities")
118
+
119
+ for name, value in [("codingagent", agentRunnerConfiguration.codingagent), ("image", agentRunnerConfiguration.image), ("drop_capabilities", agentRunnerConfiguration.drop_capabilities)]:
120
+ GeneralUtilities.assert_condition(value is not None, f"The value '{name}' is neither given as argument nor defined in '{agent_sandbox_configuration_file}'.")
121
+ return agentRunnerConfiguration
122
+
123
+ @GeneralUtilities.check_arguments
124
+ def __run_sandbox(self, repository: str, agentRunnerConfiguration: AgentRunnerConfiguration, prompt: str, interactive: bool, initialization_script: str, additional_folder_volume_arguments: list[str]) -> int:
125
+ # The container always runs Linux.
126
+ # The repository is the folder given by the caller (which was asserted to be the git-root-directory before).
127
+ # It is mounted into the container under "/Workspace/<repository-name>" (preserving the folder-name) and used as working-directory of the agent.
128
+ host_repository_folder = repository
129
+ repository_name = os.path.basename(os.path.normpath(host_repository_folder))
130
+ container_workspace_folder = f"/Workspace/{repository_name}"
131
+ agent_command = self.__get_agent_command(agentRunnerConfiguration.codingagent, prompt, interactive)
132
+ # The entrypoint is always used: if an initialization-script is present it runs it first, and then always starts the agent.
133
+ entrypoint_script_content = self.__get_entrypoint_script_content(agent_command, initialization_script)
134
+ entrypoint_file_handle = tempfile.NamedTemporaryFile("w", suffix=".sh", delete=False, newline="\n", encoding="utf-8")
135
+ host_entrypoint_file = entrypoint_file_handle.name
136
+ try:
137
+ entrypoint_file_handle.write(entrypoint_script_content)
138
+ entrypoint_file_handle.close()
139
+ # Make the entrypoint-script executable (chmod +x). On Linux-/macOS-hosts the execute-bit propagates into the container; on Windows-hosts this has no effect, which is why the script is additionally invoked via "sh" below.
140
+ os.chmod(host_entrypoint_file, 0o755)
141
+ docker_command = ["docker", "run", "--rm"]
142
+ if interactive:
143
+ # Interactive mode: allocate a pseudo-TTY and keep STDIN open so the agent runs interactively while its STDIO is inherited (variant A).
144
+ docker_command = docker_command + ["--interactive", "--tty"]
145
+ if agentRunnerConfiguration.drop_capabilities:
146
+ docker_command = docker_command + ["--cap-drop", "all", "--security-opt", "no-new-privileges:true"]
147
+ docker_command = docker_command + ["--volume", f"{host_repository_folder.replace(os.sep, '/')}:{container_workspace_folder}"]
148
+ docker_command = docker_command + ["--volume", f"{host_entrypoint_file.replace(os.sep, '/')}:/Workspace/Entrypoint.sh:ro"]
149
+ docker_command = docker_command + additional_folder_volume_arguments
150
+ if initialization_script is not None:
151
+ # The initialization-script is mounted under the fixed path "/Workspace/Temp/InitializationScript.sh"; the entrypoint runs and then removes it before the agent is started.
152
+ # Make the initialization-script executable (chmod +x) too. As for the entrypoint this is effective on Linux-/macOS-hosts; on Windows-hosts it has no effect, which is why it is invoked via "bash" in the entrypoint.
153
+ initialization_script_absolute_path = os.path.abspath(initialization_script)
154
+ os.chmod(initialization_script_absolute_path, 0o755)
155
+ docker_command = docker_command + ["--volume", f"{initialization_script_absolute_path.replace(os.sep, '/')}:/Workspace/Temp/InitializationScript.sh:ro"]
156
+ image = self.__get_image_to_use(agentRunnerConfiguration.image, host_repository_folder)
157
+ docker_command = docker_command + ["--workdir", container_workspace_folder, "--entrypoint", "sh", image, "/Workspace/Entrypoint.sh"]
158
+ # Nothing is installed in the container; the coding-agent-CLI is assumed to be available in the image.
159
+ # subprocess.run blocks until the container has exited, which happens when the agent is finished (or was terminated by the user in interactive mode).
160
+ # The exit-code of the container is returned (and not checked here), so that the caller can pass it through.
161
+ return subprocess.run(docker_command, check=False).returncode
162
+ finally:
163
+ GeneralUtilities.ensure_file_does_not_exist(host_entrypoint_file)
164
+
165
+ @GeneralUtilities.check_arguments
166
+ def __get_image_to_use(self, configured_image: str, repository: str) -> str:
167
+ # If the configured image is an image-name defined in "<repository>/.ScriptCollection/OCIImages/ImageDefinition.csv" then the image (address including tag) is resolved
168
+ # using the usual image-mechanism of ScriptCollection: the custom registry from "~/.ScriptCollection/GlobalCache/OCIImages/ImageRegistries.csv" is preferred
169
+ # if it provides the image, otherwise the registry from ImageDefinition.csv is used; the tag is always the DefaultTag from ImageDefinition.csv.
170
+ # Otherwise the configured image is interpreted directly as image-address including tag.
171
+ oci_image_manager = OCIImageManager(ScriptCollectionCore())
172
+ if os.path.isfile(oci_image_manager.get_repository_image_definition_file(repository)) and configured_image in oci_image_manager.get_used_images_in_repository(repository):
173
+ return oci_image_manager.get_registry_address_for_image_with_default_tag(repository, configured_image)
174
+ last_colon_index = configured_image.rfind(":")
175
+ GeneralUtilities.assert_condition(last_colon_index > configured_image.rfind("/") and last_colon_index < len(configured_image) - 1, f"The image '{configured_image}' is neither an image-name defined in the image-definition-file of the repository nor an image-address including a tag.")
176
+ return configured_image
177
+
178
+ @GeneralUtilities.check_arguments
179
+ def __get_entrypoint_script_content(self, agent_command: list[str], initialization_script: str) -> str:
180
+ # The entrypoint-script is always used as the container's entrypoint (located at "/Workspace/Entrypoint.sh").
181
+ # It is regenerated before every container-start, so the decision whether the initialization-script is called is made here in python
182
+ # (no runtime-check in the script): if an initialization-script is present its call is included (and the script is removed afterwards),
183
+ # otherwise it is omitted. Afterwards the agent is always started (via exec, so signals reach the agent).
184
+ agent_command_line = shlex.join(agent_command)
185
+ lines = ["#!/bin/sh", "set -e"]
186
+ if initialization_script is not None:
187
+ # A line is written to stdout before the initialization-script runs, so it is unambiguous in the container-output that an initialization-script was configured and executed (without it one cannot tell whether the agent started unconfigured because no script was declared or because a declared script did nothing).
188
+ lines.append('echo "Running initialization-script for the agent-sandbox..."')
189
+ # The initialization-script is run with bash (and not with sh), so it can use bash-syntax; the image therefore has to contain bash if an initialization-script is used.
190
+ lines.append("bash /Workspace/Temp/InitializationScript.sh")
191
+ lines.append("rm -f /Workspace/Temp/InitializationScript.sh 2>/dev/null || true")
192
+ else:
193
+ # The counterpart of the message above: it makes clear that the agent is started unconfigured on purpose because no initialization-script is declared for this repository (see __get_initialization_script).
194
+ lines.append('echo "No initialization-script is configured for this repository; the agent is started without an initialization-script."')
195
+ lines.append(f"exec {agent_command_line}")
196
+ return "\n".join(lines) + "\n"
197
+
198
+ @GeneralUtilities.check_arguments
199
+ def __get_agent_command(self, codingagent: CodingAgent, prompt: str, interactive: bool) -> list[str]:
200
+ # Returns the command (including the prompt and the agent's "approve-all"-switch) which starts the coding-agent-CLI inside the container.
201
+ # Non-interactive mode runs the agent with its switch to approve everything, so it can do whatever it wants inside the (sandboxed) container.
202
+ # Claude and Copilot also get their "approve-all"-switch in interactive mode, so the user is only asked for decisions and not for permissions.
203
+ # For most agents the prompt is passed as the last positional argument; Mistral Vibe is an exception in non-interactive mode (see below).
204
+ # If no prompt is given (only allowed in interactive mode) the agent is started without an initial prompt.
205
+ if codingagent == CodingAgent.Claude:
206
+ agent_command = ["claude", "--dangerously-skip-permissions"] if interactive else ["claude", "--print", "--dangerously-skip-permissions"]
207
+ elif codingagent == CodingAgent.Codex:
208
+ agent_command = ["codex"] if interactive else ["codex", "exec", "--dangerously-bypass-approvals-and-sandbox"]
209
+ elif codingagent == CodingAgent.Copilot:
210
+ agent_command = ["copilot", "--allow-all-tools", "--allow-all-paths"]
211
+ elif codingagent == CodingAgent.OpenCode:
212
+ agent_command = ["opencode"] if interactive else ["opencode", "run"]
213
+ elif codingagent == CodingAgent.Vibe:
214
+ # Mistral Vibe: interactive mode takes the initial prompt as a positional argument, while programmatic (non-interactive) mode
215
+ # expects the prompt via the --prompt-flag and uses --auto-approve to approve all tool-calls.
216
+ if interactive:
217
+ agent_command = ["vibe"]
218
+ if prompt is not None:
219
+ agent_command = agent_command + [prompt]
220
+ else:
221
+ agent_command = ["vibe", "--auto-approve"]
222
+ if prompt is not None:
223
+ agent_command = agent_command + ["--prompt", prompt]
224
+ return agent_command
225
+ else:
226
+ raise ValueError(f"Unsupported coding-agent '{codingagent}'.")
227
+ if prompt is not None:
228
+ agent_command = agent_command + [prompt]
229
+ return agent_command
@@ -15,6 +15,36 @@ from .TFCPS.TFCPS_CodeUnit_BuildCodeUnits import TFCPS_CodeUnit_BuildCodeUnits
15
15
  from .TFCPS.TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
16
16
  from .TFCPS.TFCPS_Tools_General import TFCPS_Tools_General
17
17
  from .OCIImages.OCIImageManager import OCIImageManager
18
+ from .AgentRunner import AgentRunner, AgentRunnerConfiguration, CodingAgent
19
+
20
+
21
+ def RunAgentInSandbox() -> int:
22
+ parser = argparse.ArgumentParser(description='scrais ("rais" stands for "run agent in sandbox") runs an AI-coding-agent (for example the claude-CLI) interactively in a sandboxed environment.')
23
+ parser.add_argument('-p','--prompt',default=None,required=False, help='Prompt-string which should be passed to the agent.')
24
+ parser.add_argument('-r','--repository',default=None, required=False, help='Repository which should be worked in.')
25
+ parser.add_argument('-i', '--interactive', action='store_true', required=False, default=False, help='Runs the agent in interactive mode.')
26
+ parser.add_argument('-v', '--verbose', action='store_true', required=False, default=False, help='Enables verbose output.')
27
+ parser.add_argument('-c', '--codingagent', default=None, required=False, choices=[coding_agent.value for coding_agent in CodingAgent], help='Coding-agent which should be used. Overrides the value from ".ScriptCollection/AgentSandboxConfiguration.ini".')
28
+ parser.add_argument('--image', default=None, required=False, help='OCI-image which should be used for the sandbox: either an image-name defined in ".ScriptCollection/OCIImages/ImageDefinition.csv" or an image-address including tag. Overrides the value from ".ScriptCollection/AgentSandboxConfiguration.ini".')
29
+ parser.add_argument('--dropcapabilities', default=None, required=False, choices=["true", "false"], help='Defines whether all capabilities should be dropped in the sandbox. Overrides the value from ".ScriptCollection/AgentSandboxConfiguration.ini".')
30
+ parser.add_argument('-a', '--additionalfolder', action='append', default=None, required=False, help='Folder which should additionally be mounted (read-write) into the sandbox under "/Workspace/Other/AdditionalResources/<folder-name>". Can be given multiple times.')
31
+ args = parser.parse_args()
32
+ repo=os.getcwd()
33
+ if args.repository is not None:
34
+ repo= args.repository
35
+ repo=GeneralUtilities.resolve_relative_path(repo, os.getcwd())
36
+ configuration_overrides = AgentRunnerConfiguration()
37
+ if args.codingagent is not None:
38
+ configuration_overrides.codingagent = CodingAgent(args.codingagent)
39
+ configuration_overrides.image = args.image
40
+ if args.dropcapabilities is not None:
41
+ configuration_overrides.drop_capabilities = args.dropcapabilities == "true"
42
+ additional_folders: list[str] = []
43
+ if args.additionalfolder is not None:
44
+ additional_folders = [GeneralUtilities.resolve_relative_path(additional_folder, os.getcwd()) for additional_folder in args.additionalfolder]
45
+ ar = AgentRunner(args.verbose)
46
+ return ar.run(args.prompt, args.interactive, repo, configuration_overrides, additional_folders)
47
+
18
48
 
19
49
  def FilenameObfuscator() -> int:
20
50
  parser = argparse.ArgumentParser(description=''''Obfuscates the names of all files in the given folder.
@@ -185,6 +215,20 @@ def ShowMissingFiles() -> int:
185
215
  return 0
186
216
 
187
217
 
218
+ def CheckMediaFile() -> int:
219
+ parser = argparse.ArgumentParser(description='Checks whether a media-file (for example mp4 or mkv) can be played completely. Exit-code 0 if and only if no inconsistency was found.')
220
+ parser.add_argument('file', help='Media-file to check')
221
+ parser.add_argument('--decode', action='store_true', help='Additionally decodes all video- and audio-streams completely (slow)')
222
+ args = parser.parse_args()
223
+ findings = GeneralUtilities.get_media_file_inconsistencies(args.file, args.decode)
224
+ for finding in findings:
225
+ GeneralUtilities.write_message_to_stderr(finding)
226
+ if 0 < len(findings):
227
+ return 1
228
+ GeneralUtilities.write_message_to_stdout(f"No inconsistencies found in '{args.file}'.")
229
+ return 0
230
+
231
+
188
232
  def ExtractPDFPages() -> int:
189
233
  parser = argparse.ArgumentParser(description='Extract pages from PDF-file')
190
234
  parser.add_argument('file', help='Input file')
@@ -1356,6 +1356,98 @@ class GeneralUtilities:
1356
1356
  exit_code = process.wait()
1357
1357
  return (exit_code, stdout, stderr)
1358
1358
 
1359
+ media_check_tolerance_in_seconds: float = 2.0
1360
+
1361
+ @staticmethod
1362
+ @check_arguments
1363
+ def __run_media_tool(tool: str, arguments: list[str]) -> tuple[int, str, str]:
1364
+ result = GeneralUtilities.run_program_simple(tool, arguments)
1365
+ return (result[0], result[1].decode("utf-8", errors="replace"), result[2].decode("utf-8", errors="replace"))
1366
+
1367
+ @staticmethod
1368
+ @check_arguments
1369
+ def __read_media_packets(file: str) -> tuple[list[str], dict[int, list[tuple[float, float]]]]:
1370
+ """Reads all packets of the file (so the whole file is demuxed) and returns the problems reported by ffprobe and the list of (start, duration) in seconds of the packets per stream-index."""
1371
+ result = GeneralUtilities.__run_media_tool("ffprobe", ["-v", "error", "-show_entries", "packet=stream_index,pts_time,dts_time,duration_time", "-of", "compact=p=0", file])
1372
+ problems: list[str] = []
1373
+ if result[0] != 0 or result[2].strip():
1374
+ problems.append(f"Reading all packets of the file failed (exit-code: {result[0]}): {result[2].strip()}")
1375
+ packets: dict[int, list[tuple[float, float]]] = {}
1376
+ for line in GeneralUtilities.string_to_lines(result[1]):
1377
+ if not line.strip():
1378
+ continue
1379
+ values = dict(part.split("=", 1) for part in line.split("|") if "=" in part)
1380
+ start = values.get("pts_time", "N/A")
1381
+ if start == "N/A":
1382
+ start = values.get("dts_time", "N/A")
1383
+ if start == "N/A":
1384
+ continue
1385
+ duration = values.get("duration_time", "N/A")
1386
+ packets.setdefault(int(values["stream_index"]), []).append((float(start), 0.0 if duration == "N/A" else float(duration)))
1387
+ return (problems, packets)
1388
+
1389
+ @staticmethod
1390
+ @check_arguments
1391
+ def get_media_file_inconsistencies(file: str, decode: bool) -> list[str]:
1392
+ """Checks whether a media-file (for example mp4 or mkv) can be played completely. Returns the found inconsistencies; an empty list means that none was found.
1393
+ Checked are: The container is readable, it declares a duration, every video- and audio-stream contains packets, starts and ends in line with the declared duration
1394
+ (tolerance: media_check_tolerance_in_seconds) and has no gap longer than that tolerance. If decode is true then all video- and audio-streams are additionally decoded completely (slow).
1395
+ This function needs ffprobe (and ffmpeg if decode is true)."""
1396
+ GeneralUtilities.assert_file_exists(file)
1397
+ GeneralUtilities.assert_condition(GeneralUtilities.tool_is_available("ffprobe"), "ffprobe is not available.")
1398
+ GeneralUtilities.assert_condition(not decode or GeneralUtilities.tool_is_available("ffmpeg"), "ffmpeg is not available.")
1399
+ if os.path.getsize(file) == 0:
1400
+ return ["The file is empty."]
1401
+ tolerance = GeneralUtilities.media_check_tolerance_in_seconds
1402
+
1403
+ probe = GeneralUtilities.__run_media_tool("ffprobe", ["-v", "error", "-show_format", "-show_streams", "-of", "json", file])
1404
+ if probe[0] != 0 or probe[2].strip():
1405
+ return [f"The container is not readable (exit-code: {probe[0]}): {probe[2].strip()}"]
1406
+ info = json.loads(probe[1])
1407
+ media_format: dict = info.get("format", {})
1408
+ streams = [stream for stream in info.get("streams", []) if stream.get("codec_type") in ("video", "audio") and not stream.get("disposition", {}).get("attached_pic", 0)]
1409
+ if len(streams) == 0:
1410
+ return ["The file contains no video- or audio-stream."]
1411
+ if "duration" not in media_format:
1412
+ return ["The container does not declare a duration, so completeness can not be verified."]
1413
+ declared_duration = float(media_format["duration"])
1414
+ declared_start = float(media_format.get("start_time", 0))
1415
+
1416
+ findings: list[str] = []
1417
+ problems, packets = GeneralUtilities.__read_media_packets(file)
1418
+ findings.extend(problems)
1419
+ for stream in streams:
1420
+ index = int(stream["index"])
1421
+ name = f"{stream['codec_type']}-stream {index} ({stream.get('codec_name', 'unknown')})"
1422
+ stream_packets = sorted(packets.get(index, []))
1423
+ if len(stream_packets) == 0:
1424
+ findings.append(f"The {name} contains no packets.")
1425
+ continue
1426
+ first_start = stream_packets[0][0]
1427
+ end = max(start+duration for start, duration in stream_packets)
1428
+ if declared_start+tolerance < first_start:
1429
+ findings.append(f"The {name} starts {first_start-declared_start:.1f}s after the start of the file.")
1430
+ if end < declared_start+declared_duration-tolerance:
1431
+ findings.append(f"The {name} contains only {end-declared_start:.1f}s of the declared {declared_duration:.1f}s.")
1432
+ if declared_start+declared_duration+tolerance < end:
1433
+ findings.append(f"The {name} is longer ({end-declared_start:.1f}s) than the declared {declared_duration:.1f}s.")
1434
+ gaps = [(stream_packets[i][0], stream_packets[i+1][0]-(stream_packets[i][0]+stream_packets[i][1])) for i in range(len(stream_packets)-1)]
1435
+ gaps = [gap for gap in gaps if tolerance < gap[1]]
1436
+ if 0 < len(gaps):
1437
+ findings.append(f"The {name} has {len(gaps)} gap(s) longer than {tolerance:.1f}s, the first one of {gaps[0][1]:.1f}s after {gaps[0][0]:.1f}s.")
1438
+
1439
+ if decode:
1440
+ decoded = GeneralUtilities.__run_media_tool("ffmpeg", ["-v", "error", "-nostdin", "-i", file, "-map", "0:v?", "-map", "0:a?", "-f", "null", "-"])
1441
+ if decoded[0] != 0 or decoded[2].strip():
1442
+ findings.append(f"Decoding failed (exit-code: {decoded[0]}): {decoded[2].strip()}")
1443
+ return findings
1444
+
1445
+ @staticmethod
1446
+ @check_arguments
1447
+ def assert_media_file_is_consistent(file: str, decode: bool) -> None:
1448
+ findings = GeneralUtilities.get_media_file_inconsistencies(file, decode)
1449
+ GeneralUtilities.assert_condition(len(findings) == 0, f"The media-file '{file}' is inconsistent: "+" ".join(findings))
1450
+
1359
1451
  @staticmethod
1360
1452
  @check_arguments
1361
1453
  def assert_file_exists(file: str,message=None) -> None:
@@ -40,7 +40,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
40
40
  from .ProgramRunnerPopen import ProgramRunnerPopen
41
41
  from .SCLog import SCLog, LogLevel
42
42
 
43
- version = "4.4.61"
43
+ version = "4.4.63"
44
44
  __version__ = version
45
45
 
46
46
  class VSCodeWorkspaceShellTask:
@@ -884,7 +884,7 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
884
884
  latest to the oldest. Only plain three-part-versions (major.minor.patch) are kept: NuGet also lists four-part-
885
885
  versions (for example "1.8.0.7") and pre-release-versions (for example "2.8.0-preview4" or "2.9.2-ci-210"), which
886
886
  the rest of the update-logic does not support (it asserts a three-part-format) and which "packaging.version.Version"
887
- can not always parse. Without this strict filter such an unparseable string would let the sorting below raise, which
887
+ can not always parse. Without this strict filter such an unparsable string would let the sorting below raise, which
888
888
  the caller turns into an empty version-list and therefore into the misleading error "Version-list can not be empty"
889
889
  for a dependency whose registry actually offers usable versions."""
890
890
  three_part_versions = [version for version in versions if re.match(r"^\d+\.\d+\.\d+$", version)]
@@ -10,6 +10,7 @@ from ..SCLog import LogLevel
10
10
  from .TFCPS_BuildCodeUnitsHook import TFCPS_BuildCodeUnitsHook
11
11
  from .TFCPS_CodeUnit_BuildCodeUnit import TFCPS_CodeUnit_BuildCodeUnit
12
12
  from .TFCPS_CodeSpellCheck import TFCPS_CodeSpellCheck
13
+ from .TFCPS_FindingsSorter import TFCPS_FindingsSorter
13
14
  from .TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
14
15
  from .TFCPS_Tools_General import TFCPS_Tools_General, CommonProjectStructureVersion
15
16
 
@@ -241,13 +242,19 @@ class TFCPS_CodeUnit_BuildCodeUnits:
241
242
  def __do_project_linting(self, repository_folder: str) -> None:
242
243
  self.sc.log.log("Do project-linting...")
243
244
  self.__do_project_linting_using_codespell(repository_folder)
245
+ self.__sort_findings(repository_folder)
246
+
247
+ @GeneralUtilities.check_arguments
248
+ def __sort_findings(self, repository_folder: str) -> None:
249
+ self.sc.log.log("Sort findings...",LogLevel.Debug)
250
+ TFCPS_FindingsSorter().sort_findings(repository_folder)
244
251
 
245
252
  @GeneralUtilities.check_arguments
246
253
  def __do_project_linting_using_codespell(self, repository_folder: str) -> None:
247
254
  """Searches for typos in the repository using codespell. Findings are only reported as warnings and do not let the build fail.
248
255
  Only files which are not git-ignored are checked. See TFCPS_CodeSpellCheck for details."""
249
256
  try:
250
- self.sc.log.log("Search for typos using codespell...")
257
+ self.sc.log.log("Search for typos using codespell...",LogLevel.Debug)
251
258
  findings: list[str] = TFCPS_CodeSpellCheck(self.sc).search_for_typos(repository_folder)
252
259
  for finding in findings:
253
260
  self.sc.log.log(f"Possible typo: {finding}", LogLevel.Warning)
@@ -0,0 +1,89 @@
1
+ import os
2
+ import re
3
+ from ..GeneralUtilities import GeneralUtilities
4
+
5
+
6
+ class TFCPS_FindingsSorter:
7
+ """Sorts the rows of the findings-table in the section "## Findings" in "<repository>/Other/Reference/Reference.md".
8
+
9
+ The findings-table is specified in the specification "common-projects-structure-findings".
10
+ Only the rows of the table are moved; no row is edited and everything else in the file stays untouched.
11
+ The rows of not completed findings come first, sorted by priority (descending, from "very high" to "low") and then by id (ascending).
12
+ The rows of completed findings (state "fixed" or "accepted") come after that, sorted by id (ascending).
13
+ If the file, the section or the table does not exist then nothing is done."""
14
+
15
+ findings_section_heading: str = "## Findings"
16
+ findings_table_columns: list[str] = ["Id", "Priority", "State", "Description", "Impact", "Notes"]
17
+ completed_states: list[str] = ["fixed", "accepted"]
18
+ priorities_in_sort_order: list[str] = ["very high", "high", "mid", "low"]
19
+
20
+ @GeneralUtilities.check_arguments
21
+ def sort_findings(self, repository_folder: str) -> None:
22
+ reference_file = os.path.join(repository_folder, "Other", "Reference", "Reference.md")
23
+ if not os.path.isfile(reference_file):
24
+ return
25
+ content = GeneralUtilities.read_text_from_file(reference_file)
26
+ sorted_content = self.sort_findings_in_content(content)
27
+ if sorted_content != content:
28
+ GeneralUtilities.write_text_to_file(reference_file, sorted_content)
29
+
30
+ @GeneralUtilities.check_arguments
31
+ def sort_findings_in_content(self, content: str) -> str:
32
+ """Returns the given markdown-content with sorted rows of the findings-table."""
33
+ lines = content.split("\n")
34
+ rows_range = self.__get_rows_range(lines)
35
+ if rows_range is None:
36
+ return content
37
+ (first_row_index, end_index) = rows_range
38
+ rows = lines[first_row_index:end_index]
39
+ uncompleted_rows = [row for row in rows if not self.__is_completed(row)]
40
+ completed_rows = [row for row in rows if self.__is_completed(row)]
41
+ uncompleted_rows.sort(key=lambda row: (self.__get_priority_rank(row), self.__get_id_sort_key(row)))
42
+ completed_rows.sort(key=self.__get_id_sort_key)
43
+ lines[first_row_index:end_index] = uncompleted_rows+completed_rows
44
+ return "\n".join(lines)
45
+
46
+ def __get_rows_range(self, lines: list[str]) -> tuple[int, int] | None:
47
+ """Returns the index of the first row and the index after the last row of the findings-table, or None if there is no findings-table."""
48
+ section_start_index = None
49
+ for index, line in enumerate(lines):
50
+ if line.strip() == self.findings_section_heading:
51
+ section_start_index = index
52
+ break
53
+ if section_start_index is None:
54
+ return None
55
+ index = section_start_index+1
56
+ while index < len(lines) and not re.match(r"^#{1,2} ", lines[index]):
57
+ if self.__is_table_line(lines[index]) and self.__get_cells(lines[index]) == self.findings_table_columns:
58
+ first_row_index = index+2 # skip the header-line and the separator-line
59
+ end_index = first_row_index
60
+ while end_index < len(lines) and self.__is_table_line(lines[end_index]):
61
+ end_index = end_index+1
62
+ return (first_row_index, end_index)
63
+ index = index+1
64
+ return None
65
+
66
+ def __is_table_line(self, line: str) -> bool:
67
+ return line.strip().startswith("|")
68
+
69
+ def __get_cells(self, line: str) -> list[str]:
70
+ # a "|" which is escaped as "\|" is part of the content of a cell and not a cell-separator
71
+ cells = re.split(r"(?<!\\)\|", line.strip())
72
+ return [cell.strip() for cell in cells[1:-1]]
73
+
74
+ def __is_completed(self, row: str) -> bool:
75
+ return self.__get_cells(row)[2] in self.completed_states
76
+
77
+ def __get_priority_rank(self, row: str) -> int:
78
+ priority = self.__get_cells(row)[1]
79
+ if priority not in self.priorities_in_sort_order:
80
+ raise ValueError(f"Finding \"{self.__get_cells(row)[0]}\" has the unknown priority \"{priority}\". Allowed priorities: {', '.join(self.priorities_in_sort_order)}")
81
+ return self.priorities_in_sort_order.index(priority)
82
+
83
+ def __get_id_sort_key(self, row: str) -> tuple[str, int]:
84
+ # a fixed finding has a struck through id like "~~SEC-2~~"
85
+ finding_id = self.__get_cells(row)[0].replace("~~", "")
86
+ match = re.match(r"^([A-Za-z]+)-(\d+)$", finding_id)
87
+ if match is None:
88
+ raise ValueError(f"Finding-id \"{finding_id}\" does not have the format \"<prefix>-<number>\".")
89
+ return (match.group(1), int(match.group(2)))
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptcollection
3
- Version: 4.4.61
3
+ Version: 4.4.63
4
4
  Summary: The ScriptCollection is the place for reusable scripts.
5
5
  Author-email: Marius Göcke <marius.goecke@gmail.com>
6
6
  License-Expression: GPL-3.0-only
@@ -1,7 +1,8 @@
1
+ ScriptCollection/AgentRunner.py,sha256=eVKKsSHywH9hiEkj9l0vT34vETImK7E-ajPi7XoOsKU,18062
1
2
  ScriptCollection/ArbTranslationsOrganizer.py,sha256=3TO4fjkYADcHGyUXdiQk-RtfCxtf9ojuBS3CbZtgXBE,12218
2
3
  ScriptCollection/CertificateUpdater.py,sha256=yZc7KESc1F_Mc6HP2UfLmNHRc2MvtVqp0t60J_CqY3s,9755
3
- ScriptCollection/Executables.py,sha256=j46g6m5Xpn5Jm_qQ-4QrtOGOhAKc37veC8hGPQGB7KE,77554
4
- ScriptCollection/GeneralUtilities.py,sha256=2Sf7H0cEdhvjY5gmi4QN-isasufHVIb9KltXKfxt7e4,73909
4
+ ScriptCollection/Executables.py,sha256=0rJTjsOvvSNWG9Qz5e27rU10XwphalRxvf014wCwt8E,81132
5
+ ScriptCollection/GeneralUtilities.py,sha256=kW6y52rdD-dJrjCtIgsdwYinz6FtclGj-xc_4tL1MUg,80022
5
6
  ScriptCollection/HTTPMaintenanceOverheadHelper.py,sha256=TToNtyO1XzsMbBsTBf3o0xgOK0v4Jf03qw2Z0xb2nCk,2007
6
7
  ScriptCollection/ProcessesRunner.py,sha256=wtNFdmj9klETTIDUSE90vaR5ClEX4V7dgJNVbp9_wi0,1381
7
8
  ScriptCollection/ProgramRunnerBase.py,sha256=eZenpiIFW1dhvSVCG7cfBs5hO4_KLlOxR7oXoSxUDmk,2286
@@ -11,7 +12,7 @@ ScriptCollection/ProgramRunnerSudo.py,sha256=vwKq-MOcqsgiKhSf-MBUDBC_UyaU8evP3Ro
11
12
  ScriptCollection/RepositoryDiagramGenerator.py,sha256=K_u2NW_p4ozdWD50irJEkkcDVa70TcUJpZN63TeXSLw,7092
12
13
  ScriptCollection/RepositoryHistoryAnalyzer.py,sha256=u9141B9D-Kef-9uo3yR7BXKUMzIKLVsesnYk7H-cWm4,6157
13
14
  ScriptCollection/SCLog.py,sha256=Fjf8Pdi63cXHGMxL_aI6A0dwGm7QIfrv3czjvpSUlaM,4705
14
- ScriptCollection/ScriptCollectionCore.py,sha256=iiOvej2wdxKVh6dRp-2HqIsm9i2KTiG35ca6WP0aYPk,310633
15
+ ScriptCollection/ScriptCollectionCore.py,sha256=U6SsZtxPPnELu5DTQa-ypGXj3nL2kd_TMCBEg8d_8HE,310633
15
16
  ScriptCollection/TestKind.py,sha256=jqLJwNvQeyea7SZf-UFzMC9pM0znv4do7FIC1xq8u9M,1672
16
17
  ScriptCollection/UnassignedVariablesCheck.py,sha256=8xNcp1D5HeOLO1a1NEerq2TsnpFhw7yzYPkaay-oisY,3804
17
18
  ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -39,8 +40,9 @@ ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py,sha256=67_DQwoH5bhxxSmwtuAeoW
39
40
  ScriptCollection/TFCPS/TFCPS_CodeSpellCheck.py,sha256=zV62O7UiGjewx1yQUIWmUJ9D-5ipkV_f28uNhp-JWUs,5306
40
41
  ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=Rh7S84B8QbeWn5qJxRItoI7jquJYvFnNRi_WiX3FjWo,40414
41
42
  ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=CB0K8OZNtQESdS2pqX9RQqN-lI5M-cbmXYOR0rbuRlk,9741
42
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=j7HMOG0kttgmnsUNMZErpyufjsrm3gF8Cm-5luEQFZk,63640
43
+ ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=n_jkEWkZBRL-7id-OLAZ6HiAh2nh9to3lOLFg9ujGNc,63983
43
44
  ScriptCollection/TFCPS/TFCPS_CreateRelease.py,sha256=BVK3fpEyjcXYVHIb6YYLH7bWe6iKRaCtPrJ3tXAemYw,6582
45
+ ScriptCollection/TFCPS/TFCPS_FindingsSorter.py,sha256=3ChuKf-Myr3sBtQhYGw6dHlk93HO6PAH-Jq1oPYdW1s,4804
44
46
  ScriptCollection/TFCPS/TFCPS_Generic.py,sha256=mRBcf7XuNWOSXbZYc3NuJow18AMZ30D6Z-BtyNYfNcM,2072
45
47
  ScriptCollection/TFCPS/TFCPS_MergeToMain.py,sha256=Krt5oyGNcRAN3szsgU7LoTh8yVTOdHW-Kq7R9MDDV-Q,7673
46
48
  ScriptCollection/TFCPS/TFCPS_MergeToStable.py,sha256=oYEhz5ZWMnbu97_CcYEqdDxakUFVppyOVQiJzADq59c,22434
@@ -57,7 +59,7 @@ ScriptCollection/TFCPS/Docker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5N
57
59
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py,sha256=bT6Gd5pQpZCw4OQz6HWkPCSn5z__eUUEisABLDSxd0o,200
58
60
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py,sha256=QyjOfMY22JWCvKjMelHiDWbJiWqotOfebpJpgDUaoO4,237
59
61
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py,sha256=i0zEGehj0sttxjjZtoq2KFSKp_ulxVyWp_ZgAhIY_So,241
60
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=RfvRHlB_sQFmnV-NYdluz8OstROMVczTSBzvcYOxLCc,71171
62
+ ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=-vdQfhPcf69R80Lfsos3k04JMe2hUsClW9XUNu2KHv0,71170
61
63
  ScriptCollection/TFCPS/DotNet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
62
64
  ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=9PMxJFB4nyKGzl916UOCvHB5toIyLcWk-ZAM8vPdLxw,64571
63
65
  ScriptCollection/TFCPS/Flutter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -73,8 +75,8 @@ ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=yyjWPMHQjPLkML
73
75
  ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
74
76
  ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py,sha256=ILVhJHvwG5TvPBBqSvyACYb7W9QnWrJpxKgHu8IQu4g,9249
75
77
  ScriptCollection/TFCPS/TypeScript/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
76
- scriptcollection-4.4.61.dist-info/METADATA,sha256=TRCqRXBsDxdQhL3dk4o5EAWEBgTr_1PJBQ44PhtpL04,8032
77
- scriptcollection-4.4.61.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
78
- scriptcollection-4.4.61.dist-info/entry_points.txt,sha256=--Xk3EcTa14uk1YnUD8CQ2naQjNKJFohScd1ONy4CeY,6135
79
- scriptcollection-4.4.61.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
80
- scriptcollection-4.4.61.dist-info/RECORD,,
78
+ scriptcollection-4.4.63.dist-info/METADATA,sha256=WTHUJeoQlXMiqpJN4pFzcYd2xcFFFn45OLClRblrBfs,8032
79
+ scriptcollection-4.4.63.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
80
+ scriptcollection-4.4.63.dist-info/entry_points.txt,sha256=kI-HqAydo8a0FGjGeXh6i1pGU73Pit0xACdjxBxNkis,6254
81
+ scriptcollection-4.4.63.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
82
+ scriptcollection-4.4.63.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (84.0.0)
2
+ Generator: setuptools (84.2.1)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5
 
@@ -6,6 +6,7 @@ scbuildcodeunitsc = ScriptCollection.Executables:BuildCodeUnitsC
6
6
  sccalculatebitcoinblockhash = ScriptCollection.Executables:CalculateBitcoinBlockHash
7
7
  scchangefileextension = ScriptCollection.Executables:ChangeFileExtensions
8
8
  scchangehashofprogram = ScriptCollection.Executables:ChangeHashOfProgram
9
+ sccheckmediafile = ScriptCollection.Executables:CheckMediaFile
9
10
  sccheckpythonast = ScriptCollection.Executables:CheckPythonAst
10
11
  sccleantoolscache = ScriptCollection.Executables:CleanToolsCache
11
12
  scconvertgitrepositorytobarerepository = ScriptCollection.Executables:ConvertGitRepositoryToBareRepository
@@ -56,6 +57,7 @@ scprintcurrentworkingdirectory = ScriptCollection.Executables:PrintCurrecntWorki
56
57
  scprintfilecontent = ScriptCollection.Executables:PrintFileContent
57
58
  scprintfilesize = ScriptCollection.Executables:PrintFileSize
58
59
  scprintosname = ScriptCollection.Executables:PrintOSName
60
+ scrais = ScriptCollection.Executables:RunAgentInSandbox
59
61
  screclaimspacefromdocker = ScriptCollection.Executables:ReclaimSpaceFromDocker
60
62
  scregexreplaceinfile = ScriptCollection.Executables:RegexReplaceInFile
61
63
  scremovefile = ScriptCollection.Executables:RemoveFile