ScriptCollection 4.4.72__py3-none-any.whl → 4.4.73__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. ScriptCollection/Executables.py +19 -0
  2. ScriptCollection/LinuxServerSecurityCheck/Checks/AccountChecks.py +71 -0
  3. ScriptCollection/LinuxServerSecurityCheck/Checks/ContainerChecks.py +34 -0
  4. ScriptCollection/LinuxServerSecurityCheck/Checks/FileSystemChecks.py +141 -0
  5. ScriptCollection/LinuxServerSecurityCheck/Checks/KernelChecks.py +49 -0
  6. ScriptCollection/LinuxServerSecurityCheck/Checks/LogChecks.py +39 -0
  7. ScriptCollection/LinuxServerSecurityCheck/Checks/NetworkChecks.py +67 -0
  8. ScriptCollection/LinuxServerSecurityCheck/Checks/PackageIntegrityChecks.py +62 -0
  9. ScriptCollection/LinuxServerSecurityCheck/Checks/PersistenceChecks.py +99 -0
  10. ScriptCollection/LinuxServerSecurityCheck/Checks/ProcessChecks.py +88 -0
  11. ScriptCollection/LinuxServerSecurityCheck/Checks/SSHChecks.py +56 -0
  12. ScriptCollection/LinuxServerSecurityCheck/Checks/__init__.py +0 -0
  13. ScriptCollection/LinuxServerSecurityCheck/LinuxServerSecurityChecker.py +90 -0
  14. ScriptCollection/LinuxServerSecurityCheck/LinuxSystemAccess.py +85 -0
  15. ScriptCollection/LinuxServerSecurityCheck/PackageOwnershipLookup.py +42 -0
  16. ScriptCollection/LinuxServerSecurityCheck/SecurityCheckBase.py +19 -0
  17. ScriptCollection/LinuxServerSecurityCheck/SecurityFinding.py +19 -0
  18. ScriptCollection/LinuxServerSecurityCheck/SuspiciousCommandPatterns.py +46 -0
  19. ScriptCollection/LinuxServerSecurityCheck/SystemAccessBase.py +244 -0
  20. ScriptCollection/LinuxServerSecurityCheck/SystemEntities.py +45 -0
  21. ScriptCollection/LinuxServerSecurityCheck/__init__.py +0 -0
  22. ScriptCollection/ScriptCollectionCore.py +1 -1
  23. ScriptCollection/TFCPS/Android/TFCPS_CodeUnitSpecific_Android.py +109 -5
  24. ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py +1 -1
  25. ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +4 -4
  26. ScriptCollection/TFCPS/SCTaskRunnerServer.py +55 -9
  27. ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +5 -2
  28. ScriptCollection/TFCPS/TFCPS_RemoteBuild.py +39 -11
  29. ScriptCollection/TFCPS/TFCPS_Tools_General.py +19 -0
  30. {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.73.dist-info}/METADATA +177 -177
  31. {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.73.dist-info}/RECORD +34 -14
  32. {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.73.dist-info}/entry_points.txt +1 -0
  33. {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.73.dist-info}/WHEEL +0 -0
  34. {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.73.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,244 @@
1
+ import ipaddress
2
+ import os
3
+ import posixpath
4
+ import stat
5
+ import sys
6
+ from abc import ABC, abstractmethod
7
+ from collections.abc import Iterator
8
+ from .SystemEntities import UserAccount, ProcessInformation, ProgramResult, SocketInformation
9
+
10
+
11
+ class SystemAccessBase(ABC):
12
+ """Read-only access to the system which is checked.
13
+ All paths are absolute paths of the checked system.
14
+ The abstract methods are the primitives which access the system; everything else is derived from them,
15
+ so that a testcase only has to replace the primitives."""
16
+
17
+ maximal_size_of_text_files: int = 50 * 1024 * 1024
18
+ seconds_per_day: int = 24 * 60 * 60
19
+
20
+ @abstractmethod
21
+ def current_user_is_root(self) -> bool:
22
+ raise NotImplementedError
23
+
24
+ @abstractmethod
25
+ def read_bytes(self, path: str, maximal_amount_of_bytes: int) -> bytes | None:
26
+ """Returns the first bytes of the file or None if the file does not exist (anymore)."""
27
+ raise NotImplementedError
28
+
29
+ @abstractmethod
30
+ def list_folder(self, path: str) -> list[str] | None:
31
+ """Returns the names of the entries of the folder or None if the folder does not exist (anymore)."""
32
+ raise NotImplementedError
33
+
34
+ @abstractmethod
35
+ def get_lstat(self, path: str) -> os.stat_result | None:
36
+ """Returns the stat-information of the path itself (a symlink is not followed) or None if the path does not exist (anymore)."""
37
+ raise NotImplementedError
38
+
39
+ @abstractmethod
40
+ def path_exists(self, path: str) -> bool:
41
+ """Returns whether the path exists. Symlinks are followed."""
42
+ raise NotImplementedError
43
+
44
+ @abstractmethod
45
+ def read_link(self, path: str) -> str | None:
46
+ """Returns the target of the symlink or None if the symlink does not exist (anymore) or has no target (for example /proc/<pid>/exe of a kernel-thread)."""
47
+ raise NotImplementedError
48
+
49
+ @abstractmethod
50
+ def program_is_available(self, program: str) -> bool:
51
+ raise NotImplementedError
52
+
53
+ @abstractmethod
54
+ def run_program(self, program: str, arguments: list[str]) -> ProgramResult:
55
+ """Runs a program which must not change anything on the system."""
56
+ raise NotImplementedError
57
+
58
+ @abstractmethod
59
+ def get_file_attribute_flags(self, path: str) -> int | None:
60
+ """Returns the inode-flags (as shown by lsattr) of a regular file or None if the filesystem does not support them."""
61
+ raise NotImplementedError
62
+
63
+ @abstractmethod
64
+ def get_current_time(self) -> float:
65
+ raise NotImplementedError
66
+
67
+ def read_text(self, path: str) -> str | None:
68
+ content = self.read_bytes(path, SystemAccessBase.maximal_size_of_text_files)
69
+ if content is None:
70
+ return None
71
+ return content.decode("utf-8", errors="replace")
72
+
73
+ def is_regular_file(self, path: str) -> bool:
74
+ file_stat = self.get_lstat(path)
75
+ return file_stat is not None and stat.S_ISREG(file_stat.st_mode)
76
+
77
+ def is_symlink(self, path: str) -> bool:
78
+ file_stat = self.get_lstat(path)
79
+ return file_stat is not None and stat.S_ISLNK(file_stat.st_mode)
80
+
81
+ def is_folder(self, path: str) -> bool:
82
+ file_stat = self.get_lstat(path)
83
+ return file_stat is not None and stat.S_ISDIR(file_stat.st_mode)
84
+
85
+ def is_binary_content(self, content: bytes) -> bool:
86
+ return b"\0" in content[:8192]
87
+
88
+ def list_files_recursively(self, folder: str, excluded_folders: set[str]) -> Iterator[tuple[str, os.stat_result]]:
89
+ """Yields every non-folder-entry (regular files, symlinks, ...) below the folder together with its lstat-information.
90
+ Symlinks to folders are not followed. Folders contained in excluded_folders are not entered."""
91
+ folders_to_visit: list[str] = [folder]
92
+ while len(folders_to_visit) > 0:
93
+ current_folder = folders_to_visit.pop()
94
+ entries = self.list_folder(current_folder)
95
+ if entries is None:
96
+ continue
97
+ for entry in sorted(entries):
98
+ path = posixpath.join(current_folder, entry)
99
+ entry_stat = self.get_lstat(path)
100
+ if entry_stat is None:
101
+ continue
102
+ if stat.S_ISDIR(entry_stat.st_mode):
103
+ if path not in excluded_folders:
104
+ folders_to_visit.append(path)
105
+ else:
106
+ yield path, entry_stat
107
+
108
+ def list_regular_files_recursively(self, folder: str) -> list[str]:
109
+ return [path for path, file_stat in self.list_files_recursively(folder, set()) if stat.S_ISREG(file_stat.st_mode)]
110
+
111
+ def path_is_alias(self, path: str) -> bool:
112
+ """Returns whether the path is reachable only through a symlinked top-level-folder (for example /lib/systemd on a system with merged /usr),
113
+ so that its content is already reachable through another path."""
114
+ parts = [part for part in path.split("/") if part != ""]
115
+ current = ""
116
+ for part in parts[:-1]:
117
+ current = current + "/" + part
118
+ if self.is_symlink(current):
119
+ return True
120
+ return False
121
+
122
+ def get_users(self) -> list[UserAccount]:
123
+ content = self.read_text("/etc/passwd")
124
+ if content is None:
125
+ raise FileNotFoundError("/etc/passwd does not exist.")
126
+ result: list[UserAccount] = []
127
+ for line in content.splitlines():
128
+ if line.strip() == "" or line.startswith("#"):
129
+ continue
130
+ fields = line.split(":")
131
+ if len(fields) < 7:
132
+ continue
133
+ result.append(UserAccount(fields[0], int(fields[2]), int(fields[3]), fields[5], fields[6]))
134
+ return result
135
+
136
+ def get_home_folders(self) -> list[str]:
137
+ result: list[str] = []
138
+ for user in self.get_users():
139
+ if user.home_folder not in result and user.home_folder.startswith("/") and self.is_folder(user.home_folder):
140
+ result.append(user.home_folder)
141
+ return result
142
+
143
+ def get_minimal_uid_of_regular_users(self) -> int:
144
+ content = self.read_text("/etc/login.defs")
145
+ if content is not None:
146
+ for line in content.splitlines():
147
+ fields = line.split()
148
+ if len(fields) == 2 and fields[0] == "UID_MIN":
149
+ return int(fields[1])
150
+ # 1000 is the value which shadow-utils uses when UID_MIN is not defined in /etc/login.defs.
151
+ return 1000
152
+
153
+ def get_process_ids(self) -> list[int]:
154
+ entries = self.list_folder("/proc")
155
+ if entries is None:
156
+ raise FileNotFoundError("/proc does not exist.")
157
+ return sorted(int(entry) for entry in entries if entry.isdigit())
158
+
159
+ def get_process(self, pid: int) -> ProcessInformation | None:
160
+ """Returns None if the process does not exist (anymore)."""
161
+ status = self.read_text(f"/proc/{pid}/status")
162
+ if status is None:
163
+ return None
164
+ name = ""
165
+ uid = -1
166
+ for line in status.splitlines():
167
+ if line.startswith("Name:"):
168
+ name = line[len("Name:"):].strip()
169
+ elif line.startswith("Uid:"):
170
+ uid = int(line.split()[1])
171
+ commandline_bytes = self.read_bytes(f"/proc/{pid}/cmdline", 65536)
172
+ if commandline_bytes is None:
173
+ return None
174
+ commandline = commandline_bytes.replace(b"\0", b" ").decode("utf-8", errors="replace").strip()
175
+ executable = self.read_link(f"/proc/{pid}/exe")
176
+ return ProcessInformation(pid, name, uid, executable, commandline)
177
+
178
+ def get_processes(self) -> list[ProcessInformation]:
179
+ result: list[ProcessInformation] = []
180
+ for pid in self.get_process_ids():
181
+ process = self.get_process(pid)
182
+ if process is not None:
183
+ result.append(process)
184
+ return result
185
+
186
+ def get_processes_by_socket_inode(self) -> dict[int, list[ProcessInformation]]:
187
+ result: dict[int, list[ProcessInformation]] = {}
188
+ for process in self.get_processes():
189
+ file_descriptors = self.list_folder(f"/proc/{process.pid}/fd")
190
+ if file_descriptors is None:
191
+ continue
192
+ for file_descriptor in file_descriptors:
193
+ target = self.read_link(f"/proc/{process.pid}/fd/{file_descriptor}")
194
+ if target is not None and target.startswith("socket:[") and target.endswith("]"):
195
+ inode = int(target[len("socket:["):-1])
196
+ processes = result.setdefault(inode, [])
197
+ if process not in processes:
198
+ processes.append(process)
199
+ return result
200
+
201
+ def get_sockets(self) -> list[SocketInformation]:
202
+ """Returns the tcp- and udp-sockets of all network-namespaces (so also the sockets of containers)."""
203
+ socket_tables: list[str] = []
204
+ known_namespaces: set[str] = set()
205
+ for pid in self.get_process_ids():
206
+ namespace = self.read_link(f"/proc/{pid}/ns/net")
207
+ if namespace is not None and namespace not in known_namespaces:
208
+ known_namespaces.add(namespace)
209
+ socket_tables.append(f"/proc/{pid}/net")
210
+ if len(socket_tables) == 0:
211
+ socket_tables.append("/proc/net")
212
+ result: list[SocketInformation] = []
213
+ known_inodes: set[int] = set()
214
+ for socket_table in socket_tables:
215
+ for protocol in ["tcp", "tcp6", "udp", "udp6"]:
216
+ content = self.read_text(f"{socket_table}/{protocol}")
217
+ if content is None:
218
+ continue
219
+ for socket_information in SystemAccessBase.parse_socket_table(protocol, content):
220
+ if socket_information.inode == 0 or socket_information.inode not in known_inodes:
221
+ known_inodes.add(socket_information.inode)
222
+ result.append(socket_information)
223
+ return result
224
+
225
+ @staticmethod
226
+ def parse_socket_table(protocol: str, content: str) -> list[SocketInformation]:
227
+ result: list[SocketInformation] = []
228
+ for line in content.splitlines()[1:]:
229
+ fields = line.split()
230
+ if len(fields) < 10:
231
+ continue
232
+ local_address, local_port = SystemAccessBase.decode_socket_address(fields[1])
233
+ remote_address, remote_port = SystemAccessBase.decode_socket_address(fields[2])
234
+ result.append(SocketInformation(protocol, local_address, local_port, remote_address, remote_port, fields[3].upper(), int(fields[9])))
235
+ return result
236
+
237
+ @staticmethod
238
+ def decode_socket_address(hex_address: str) -> tuple[str, int]:
239
+ """Decodes an address like "0100007F:0035" from /proc/net/tcp. The address consists of 32-bit-words in host-byte-order."""
240
+ address_part, port_part = hex_address.split(":")
241
+ raw_address = bytes.fromhex(address_part)
242
+ if sys.byteorder == "little":
243
+ raw_address = b"".join(raw_address[index:index + 4][::-1] for index in range(0, len(raw_address), 4))
244
+ return str(ipaddress.ip_address(raw_address)), int(port_part, 16)
@@ -0,0 +1,45 @@
1
+ from dataclasses import dataclass
2
+
3
+
4
+ @dataclass(frozen=True)
5
+ class UserAccount:
6
+ name: str
7
+ uid: int
8
+ gid: int
9
+ home_folder: str
10
+ shell: str
11
+
12
+
13
+ @dataclass(frozen=True)
14
+ class ProcessInformation:
15
+ pid: int
16
+ name: str
17
+ uid: int
18
+ executable: str | None
19
+ """Target of /proc/<pid>/exe as seen from the mount-namespace of the process. None for kernel-threads and zombies."""
20
+ commandline: str
21
+
22
+ def is_kernel_thread_or_zombie(self) -> bool:
23
+ return self.executable is None and self.commandline == ""
24
+
25
+ def describe(self) -> str:
26
+ return f"pid {self.pid} (name \"{self.name}\", uid {self.uid}, executable \"{self.executable}\", commandline \"{self.commandline[:300]}\")"
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class ProgramResult:
31
+ exit_code: int
32
+ stdout: str
33
+ stderr: str
34
+
35
+
36
+ @dataclass(frozen=True)
37
+ class SocketInformation:
38
+ protocol: str
39
+ local_address: str
40
+ local_port: int
41
+ remote_address: str
42
+ remote_port: int
43
+ state: str
44
+ """Hex-state as written in /proc/net/tcp, for example "0A" for listening tcp-sockets."""
45
+ inode: int
File without changes
@@ -41,7 +41,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
41
41
  from .ProgramRunnerPopen import ProgramRunnerPopen
42
42
  from .SCLog import SCLog, LogLevel
43
43
 
44
- version = "4.4.72"
44
+ version = "4.4.73"
45
45
  __version__ = version
46
46
 
47
47
  class VSCodeWorkspaceShellTask:
@@ -1,5 +1,8 @@
1
1
  import os
2
2
  import re
3
+ import base64
4
+ import binascii
5
+ import zipfile
3
6
  import shlex
4
7
  import shutil
5
8
  import requests
@@ -32,7 +35,29 @@ class TFCPS_CodeUnitSpecific_Android_Functions(TFCPS_CodeUnitSpecific_Base):
32
35
  - Only if the codeunit has visual-regression-tests (see run_visual_regression_tests): it applies the plugin
33
36
  "io.github.takahirom.roborazzi" and configures "roborazzi { outputDir = file('../Other/Resources/VisualRegressionBaselines');
34
37
  compare { outputDir = layout.buildDirectory.dir('outputs/roborazzi-comparison') } }" (see
35
- get_visual_regression_baselines_folder and get_visual_regression_comparison_folder_relative_to_build_folder)."""
38
+ get_visual_regression_baselines_folder and get_visual_regression_comparison_folder_relative_to_build_folder).
39
+ - The release-build-type is signed with the upload-key if the environment-variables "SC_ANDROID_SIGNING_STORE_FILE",
40
+ "SC_ANDROID_SIGNING_STORE_PASSWORD", "SC_ANDROID_SIGNING_KEY_ALIAS" and "SC_ANDROID_SIGNING_KEY_PASSWORD" are set,
41
+ and with the debug-key otherwise (see build), for example:
42
+ "signingConfigs { release { if (System.getenv('SC_ANDROID_SIGNING_STORE_FILE') != null) { storeFile =
43
+ file(System.getenv('SC_ANDROID_SIGNING_STORE_FILE')); storePassword = System.getenv('SC_ANDROID_SIGNING_STORE_PASSWORD');
44
+ keyAlias = System.getenv('SC_ANDROID_SIGNING_KEY_ALIAS'); keyPassword = System.getenv('SC_ANDROID_SIGNING_KEY_PASSWORD') } } }"
45
+ and "release { signingConfig = System.getenv('SC_ANDROID_SIGNING_STORE_FILE') != null ? signingConfigs.release :
46
+ signingConfigs.debug }"."""
47
+
48
+ # The environment-variables (of the machine which runs the build, for example the build-pipeline) which contain the
49
+ # upload-key of a productive build, mapped to the environment-variables of the gradle-run on the runner which
50
+ # receive them (see build). The keystore is given as the base64-encoding of the keystore-file, because an
51
+ # environment-variable can not contain a file.
52
+ __signing_environment_variables: dict[str, str] = {
53
+ "Android_Signing_Keystore": "SC_ANDROID_SIGNING_KEYSTORE_BASE64",
54
+ "Android_Signing_StorePassword": "SC_ANDROID_SIGNING_STORE_PASSWORD",
55
+ "Android_Signing_KeyAlias": "SC_ANDROID_SIGNING_KEY_ALIAS",
56
+ "Android_Signing_KeyPassword": "SC_ANDROID_SIGNING_KEY_PASSWORD",
57
+ }
58
+
59
+ # The name the subject of the debug-certificate of the android-sdk contains ("CN=Android Debug,O=Android,C=US").
60
+ __subject_of_the_debug_certificate: bytes = b"Android Debug"
36
61
 
37
62
  # The repositories in which a dependency or a gradle-plugin of an android-app is published. Gradle itself searches
38
63
  # all of them (they are the repositories a project declares by "google()", "mavenCentral()" and
@@ -94,16 +119,38 @@ class TFCPS_CodeUnitSpecific_Android_Functions(TFCPS_CodeUnitSpecific_Base):
94
119
  def build(self) -> None:
95
120
  """Builds the app-bundle (which is what is uploaded to a store) and the apk (which can be installed directly) of the
96
121
  release-build-type and generates the bill-of-materials. All of them are generated by one gradle-run, so the runner
97
- has to be used only once for them."""
122
+ has to be used only once for them.
123
+
124
+ A productive build (target-environment-type "Productive") is signed with the upload-key, so its app-bundle can be
125
+ uploaded to Google Play. The upload-key is taken from the environment-variables "Android_Signing_Keystore" (the
126
+ base64-encoding of the keystore-file), "Android_Signing_StorePassword", "Android_Signing_KeyAlias" and
127
+ "Android_Signing_KeyPassword", which are required for a productive build. They are given to the gradle-run on the
128
+ runner as environment-variables only (see TFCPS_RemoteBuild.run_program_on_runner); on the runner the keystore
129
+ is written into a temporary file outside of the workspace which is removed when gradle ends. Afterwards it is
130
+ verified that the app-bundle is signed, but not with the debug-key.
131
+ Every other build is signed with the debug-key of the runner, so a developer can build and install the app
132
+ without the upload-key."""
98
133
  codeunit_name = self.get_codeunit_name()
99
134
  codeunit_version: str = self.tfcps_Tools_General.get_version_of_codeunit(self.get_codeunit_file())
100
135
  outputs_folder = os.path.join(self.get_build_folder(), "outputs")
101
136
  GeneralUtilities.ensure_directory_does_not_exist(outputs_folder)
102
- self.__run_gradle_on_runner(["bundleRelease", "assembleRelease", "cyclonedxBom"], outputs_folder)
137
+ gradle_tasks = ["bundleRelease", "assembleRelease", "cyclonedxBom"]
138
+ is_signed_with_upload_key = self.get_target_environment_type() == "Productive"
139
+ if is_signed_with_upload_key:
140
+ self._protected_sc.log.log("Build the release-build signed with the upload-key...")
141
+ signing_environment_variables = TFCPS_CodeUnitSpecific_Android_Functions.get_signing_environment_variables_for_runner(dict(os.environ))
142
+ script = TFCPS_CodeUnitSpecific_Android_Functions.get_script_which_signs_with_upload_key(self.__get_gradle_arguments(gradle_tasks))
143
+ self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", ["-c", script], self.get_gradle_project_folder(), outputs_folder, self.__get_folders_which_are_not_transferred(), signing_environment_variables)
144
+ else:
145
+ self._protected_sc.log.log(f'Build the release-build signed with the debug-key of the runner (only a build with target-environment-type "Productive" is signed with the upload-key, this one has "{self.get_target_environment_type()}")...')
146
+ self.__run_gradle_on_runner(gradle_tasks, outputs_folder)
103
147
 
104
148
  aab_folder = os.path.join(self.get_artifacts_folder(), "BuildResult_AAB")
105
149
  GeneralUtilities.ensure_folder_exists_and_is_empty(aab_folder)
106
150
  aab_file_original = self._protected_sc.find_file_by_extension(os.path.join(outputs_folder, "bundle", "release"), "aab")
151
+ if is_signed_with_upload_key:
152
+ GeneralUtilities.assert_condition(TFCPS_CodeUnitSpecific_Android_Functions.is_signed(aab_file_original), f'The app-bundle "{aab_file_original}" is not signed. The release-build-type of "{self.get_build_file()}" has to use the upload-key which is given by the environment-variables "SC_ANDROID_SIGNING_*" (see the docstring of TFCPS_CodeUnitSpecific_Android_Functions).')
153
+ GeneralUtilities.assert_condition(not TFCPS_CodeUnitSpecific_Android_Functions.is_signed_with_debug_key(aab_file_original), f'The app-bundle "{aab_file_original}" is signed with the debug-key, so Google Play would reject it. The release-build-type of "{self.get_build_file()}" has to use the upload-key which is given by the environment-variables "SC_ANDROID_SIGNING_*" (see the docstring of TFCPS_CodeUnitSpecific_Android_Functions).')
107
154
  shutil.copyfile(aab_file_original, os.path.join(aab_folder, f"{codeunit_name}.aab"))
108
155
 
109
156
  apk_folder = os.path.join(self.get_artifacts_folder(), "BuildResult_APK")
@@ -214,7 +261,7 @@ class TFCPS_CodeUnitSpecific_Android_Functions(TFCPS_CodeUnitSpecific_Base):
214
261
  def __run_gradle_on_runner(self, tasks: list[str], result_folder: str) -> None:
215
262
  """Runs the given gradle-tasks on an android-task-runner and writes the content of result_folder (which must be a
216
263
  folder inside this codeunit) back into this codeunit. A failure of gradle raises an exception."""
217
- self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", self.__get_gradle_arguments(tasks), self.get_gradle_project_folder(), result_folder, self.__get_folders_which_are_not_transferred())
264
+ self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", self.__get_gradle_arguments(tasks), self.get_gradle_project_folder(), result_folder, self.__get_folders_which_are_not_transferred(), {})
218
265
 
219
266
  @GeneralUtilities.check_arguments
220
267
  def __run_gradle_on_runner_and_get_exit_code(self, tasks: list[str], result_folder: str) -> int:
@@ -225,11 +272,68 @@ class TFCPS_CodeUnitSpecific_Android_Functions(TFCPS_CodeUnitSpecific_Base):
225
272
  succeeds."""
226
273
  result_folder_relative_to_project = os.path.relpath(result_folder, self.get_gradle_project_folder()).replace("\\", "/")
227
274
  script = TFCPS_CodeUnitSpecific_Android_Functions.get_script_which_preserves_exit_code(self.__get_gradle_arguments(tasks), result_folder_relative_to_project, TFCPS_CodeUnitSpecific_Android_Functions.__exit_code_file_name)
228
- self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", ["-c", script], self.get_gradle_project_folder(), result_folder, self.__get_folders_which_are_not_transferred())
275
+ self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", ["-c", script], self.get_gradle_project_folder(), result_folder, self.__get_folders_which_are_not_transferred(), {})
229
276
  local_exit_code_file = os.path.join(result_folder, TFCPS_CodeUnitSpecific_Android_Functions.__exit_code_file_name)
230
277
  GeneralUtilities.assert_file_exists(local_exit_code_file)
231
278
  return int(GeneralUtilities.read_text_from_file(local_exit_code_file).strip())
232
279
 
280
+ @staticmethod
281
+ @GeneralUtilities.check_arguments
282
+ def get_signing_environment_variables_for_runner(environment: dict[str, str]) -> dict[str, str]:
283
+ """Returns the environment-variables which give the upload-key to the gradle-run on the runner, taken from the
284
+ given environment-variables of this machine (see build). The names are compared case-insensitively, because
285
+ Windows exposes the names of environment-variables upper-cased via os.environ."""
286
+ environment_upper = {name.upper(): value for name, value in environment.items()}
287
+ missing_variables = [name for name in TFCPS_CodeUnitSpecific_Android_Functions.__signing_environment_variables if len((environment_upper.get(name.upper()) or GeneralUtilities.empty_string).strip()) == 0]
288
+ GeneralUtilities.assert_condition(len(missing_variables) == 0, f'A productive build of an android-app is signed with the upload-key, which requires the environment-variable(s) {", ".join(missing_variables)}. "Android_Signing_Keystore" contains the base64-encoding of the keystore-file.')
289
+ result: dict[str, str] = {}
290
+ for name, name_on_runner in TFCPS_CodeUnitSpecific_Android_Functions.__signing_environment_variables.items():
291
+ result[name_on_runner] = environment_upper[name.upper()]
292
+ # A base64-encoding is often wrapped into several lines (for example by the base64-tool), which is not part of it.
293
+ keystore_base64 = "".join(result["SC_ANDROID_SIGNING_KEYSTORE_BASE64"].split())
294
+ try:
295
+ base64.b64decode(keystore_base64, validate=True)
296
+ except binascii.Error as exception:
297
+ raise ValueError('The environment-variable "Android_Signing_Keystore" does not contain a valid base64-encoding of the keystore-file.') from exception
298
+ result["SC_ANDROID_SIGNING_KEYSTORE_BASE64"] = keystore_base64
299
+ return result
300
+
301
+ @staticmethod
302
+ @GeneralUtilities.check_arguments
303
+ def get_script_which_signs_with_upload_key(gradle_arguments: list[str]) -> str:
304
+ """Returns the bash-script which runs gradle on the runner with the upload-key: it writes the keystore (which it
305
+ gets base64-encoded in an environment-variable) into a temporary file outside of the workspace of the job, gives
306
+ that file to gradle via "SC_ANDROID_SIGNING_STORE_FILE" and removes it when the script ends, regardless of how it
307
+ ends."""
308
+ return "set -e; " \
309
+ + 'keystore_file="$(mktemp)"; ' \
310
+ + """trap 'rm -f "$keystore_file"' EXIT; """ \
311
+ + """printf '%s' "$SC_ANDROID_SIGNING_KEYSTORE_BASE64" | base64 -d > "$keystore_file"; """ \
312
+ + "unset SC_ANDROID_SIGNING_KEYSTORE_BASE64; " \
313
+ + 'export SC_ANDROID_SIGNING_STORE_FILE="$keystore_file"; ' \
314
+ + "bash " + shlex.join(gradle_arguments)
315
+
316
+ @staticmethod
317
+ @GeneralUtilities.check_arguments
318
+ def __get_signature_blocks(signed_file: str) -> list[bytes]:
319
+ """Returns the signature-blocks of a jar-signed file (an app-bundle is jar-signed): the files "META-INF/*.RSA",
320
+ "META-INF/*.DSA" and "META-INF/*.EC", which contain the certificate of the signer."""
321
+ with zipfile.ZipFile(signed_file) as archive:
322
+ return [archive.read(name) for name in archive.namelist() if re.fullmatch(r"META-INF/[^/]+\.(RSA|DSA|EC)", name, re.IGNORECASE)]
323
+
324
+ @staticmethod
325
+ @GeneralUtilities.check_arguments
326
+ def is_signed(signed_file: str) -> bool:
327
+ """Whether the given app-bundle is signed."""
328
+ return len(TFCPS_CodeUnitSpecific_Android_Functions.__get_signature_blocks(signed_file)) > 0
329
+
330
+ @staticmethod
331
+ @GeneralUtilities.check_arguments
332
+ def is_signed_with_debug_key(signed_file: str) -> bool:
333
+ """Whether the given app-bundle is signed with the debug-key of an android-sdk. The certificate of that key is
334
+ always issued to "CN=Android Debug,O=Android,C=US", whose name is contained in the signature-block as text."""
335
+ return any(TFCPS_CodeUnitSpecific_Android_Functions.__subject_of_the_debug_certificate in block for block in TFCPS_CodeUnitSpecific_Android_Functions.__get_signature_blocks(signed_file))
336
+
233
337
  @staticmethod
234
338
  @GeneralUtilities.check_arguments
235
339
  def get_script_which_preserves_exit_code(gradle_arguments: list[str], result_folder_relative_to_project: str, exit_code_file_name: str) -> str:
@@ -169,7 +169,7 @@ class TFCPS_CodeUnitSpecific_CPP_Functions(TFCPS_CodeUnitSpecific_Base):
169
169
  #run in - which is also the folder __collect_output_files searches afterwards.
170
170
  #Nothing of a C/C++-codeunit is left out of the archive the runner gets: where a build-command puts its
171
171
  #output is defined by that command, so there is no folder which is known to contain state of this machine.
172
- self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "/bin/sh", ["-c", " && ".join(self.macos_build_commands)], source_folder, source_folder, [])
172
+ self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "/bin/sh", ["-c", " && ".join(self.macos_build_commands)], source_folder, source_folder, [], {})
173
173
  self.__collect_output_files(source_folder, self.macos_output_file_patterns, "BuildResult_MacOS")
174
174
 
175
175
  @GeneralUtilities.check_arguments
@@ -112,7 +112,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
112
112
  "environment as a runner-built one.", LogLevel.Warning)
113
113
  self._protected_sc.run_with_epew("flutter", "build windows", src_folder)
114
114
  else:
115
- self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder, windows_release_folder, folders_which_are_not_transferred)
115
+ self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder, windows_release_folder, folders_which_are_not_transferred, {})
116
116
  windows_folder = os.path.join(artifacts_folder, "BuildResult_Windows")
117
117
  GeneralUtilities.ensure_directory_does_not_exist(windows_folder)
118
118
  GeneralUtilities.ensure_directory_exists(windows_folder)
@@ -134,7 +134,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
134
134
  # builds), analogous to how ios-builds always run on the iOS-task-runner. See SCTaskRunnerMacOS and
135
135
  # the remote-build-article in the reference.
136
136
  macos_release_folder = os.path.join(src_folder, "build/macos/Build/Products/Release")
137
- self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder, macos_release_folder, folders_which_are_not_transferred)
137
+ self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder, macos_release_folder, folders_which_are_not_transferred, {})
138
138
  macos_folder = os.path.join(artifacts_folder, "BuildResult_MacOS")
139
139
  GeneralUtilities.ensure_directory_does_not_exist(macos_folder)
140
140
  GeneralUtilities.ensure_directory_exists(macos_folder)
@@ -143,7 +143,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
143
143
  # iOS-builds must run on macOS and therefore always run on the dedicated iOS-task-runner (uniform
144
144
  # builds). See SCTaskRunnerIOS and the remote-build-article in the reference.
145
145
  ios_release_folder = os.path.join(src_folder, "build/ios/iphoneos")
146
- self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder, ios_release_folder, folders_which_are_not_transferred)
146
+ self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder, ios_release_folder, folders_which_are_not_transferred, {})
147
147
  ios_folder = os.path.join(artifacts_folder, "BuildResult_IOS")
148
148
  GeneralUtilities.ensure_directory_does_not_exist(ios_folder)
149
149
  GeneralUtilities.ensure_directory_exists(ios_folder)
@@ -153,7 +153,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
153
153
  # remote-build-article in the reference), analogous to how ios-builds always run on a macOS-task-runner:
154
154
  # the Android-SDK/NDK-toolchain no longer lives in SCBuilder itself, it moved into SCTaskRunnerAndroid.
155
155
  aab_release_folder = os.path.join(src_folder, "build/app/outputs/bundle/release")
156
- self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder, aab_release_folder, folders_which_are_not_transferred)
156
+ self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder, aab_release_folder, folders_which_are_not_transferred, {})
157
157
  # The app-bundle which came back from the runner is published as the build-result of this codeunit,
158
158
  # analogous to the targets above: it is the artifact of an android-build, and every codeunit has to have
159
159
  # an artifact whose name matches "BuildResult_.+" (see TFCPS_CodeUnit_BuildCodeUnit.build_codeunit).
@@ -103,8 +103,13 @@ class SCTaskRunnerServer:
103
103
  self.__sc.log.log(f"SCTaskRunner for operating-system '{self.operating_system_name}' is listening on {protocol}://{host}:{port}.")
104
104
  server.serve_forever()
105
105
 
106
+ @staticmethod
107
+ def is_valid_set_of_environment_variables(environment_variables) -> bool:
108
+ """Whether the given (deserialized) value is a set of environment-variables a job can be started with."""
109
+ return isinstance(environment_variables, dict) and all(isinstance(name, str) and isinstance(value, str) and len(name) > 0 and "=" not in name for name, value in environment_variables.items())
110
+
106
111
  @GeneralUtilities.check_arguments
107
- def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
112
+ def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, environment_variables: dict[str, str]) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
108
113
  """Starts a job for the repository-archive which can be read from archive_stream (the body of the request which
109
114
  submitted the job; archive_size is its announced length). The archive is taken as a stream instead of as bytes
110
115
  because it contains the complete repository of the client and is therefore regularly several hundred megabytes,
@@ -128,13 +133,15 @@ class SCTaskRunnerServer:
128
133
  job = _RunnerJob(job_id, workspace_folder, result_folder)
129
134
  with self.__jobs_lock:
130
135
  self.__jobs[job_id] = job
131
- thread = threading.Thread(target=self.__extract_payload_and_run_job, args=(job, archive_file, program, arguments, working_directory), daemon=True)
136
+ thread = threading.Thread(target=self.__extract_payload_and_run_job, args=(job, archive_file, program, arguments, working_directory, environment_variables), daemon=True)
132
137
  thread.start()
133
- self.__sc.log.log(f"Accepted job '{job_id}': run '{program} {' '.join(arguments)}' in '{working_directory}' of codeunit '{codeunit_name}', result-folder '{result_folder}' ({archive_size} transferred bytes of repository-content).")
138
+ #Only the names of the environment-variables are logged: their values are secrets of the client (for example the
139
+ #key which signs an app for a store).
140
+ self.__sc.log.log(f"Accepted job '{job_id}': run '{program} {' '.join(arguments)}' in '{working_directory}' of codeunit '{codeunit_name}', result-folder '{result_folder}', environment-variables [{', '.join(sorted(environment_variables.keys()))}] ({archive_size} transferred bytes of repository-content).")
134
141
  return job_id
135
142
 
136
143
  @GeneralUtilities.check_arguments
137
- def __extract_payload_and_run_job(self, job: _RunnerJob, archive_file: str, program: str, arguments: list[str], working_directory: str) -> None:
144
+ def __extract_payload_and_run_job(self, job: _RunnerJob, archive_file: str, program: str, arguments: list[str], working_directory: str, environment_variables: dict[str, str]) -> None:
138
145
  """Extracts the transferred repository-archive into the workspace of the job and runs the requested program
139
146
  afterwards. The extraction happens here - in the thread of the job - and not in the request which submitted the
140
147
  job, because extracting the repository of a large codeunit takes minutes during which no data flows: the client
@@ -151,7 +158,7 @@ class SCTaskRunnerServer:
151
158
  finally:
152
159
  GeneralUtilities.ensure_file_does_not_exist(archive_file)
153
160
  self.__sc.log.log(f"Job '{job.job_id}' works on the repository '{self.__get_origin_of_the_transferred_repository(job.workspace_folder)}'.")
154
- self.__run_job(job, program, arguments, working_directory)
161
+ self.__run_job(job, program, arguments, working_directory, environment_variables)
155
162
 
156
163
  @GeneralUtilities.check_arguments
157
164
  def __get_origin_of_the_transferred_repository(self, workspace_folder: str) -> str:
@@ -204,14 +211,18 @@ class SCTaskRunnerServer:
204
211
  remaining_amount_of_bytes = remaining_amount_of_bytes-len(chunk)
205
212
 
206
213
  @GeneralUtilities.check_arguments
207
- def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str) -> None:
214
+ def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str, environment_variables: dict[str, str]) -> None:
215
+ """Runs the program of the job. The environment-variables of the job are set in addition to the ones of this
216
+ runner and only for this program."""
208
217
  command_folder = os.path.join(job.workspace_folder, working_directory)
209
218
  try:
210
219
  self.__sc.log.log(f"Job '{job.job_id}' runs '{program} {' '.join(arguments)}' in '{command_folder}'...")
211
220
  moment_of_the_start = time.time()
212
221
  moment_of_the_last_progress_log = moment_of_the_start
213
222
  amount_of_lines = 0
214
- with subprocess.Popen([program] + arguments, cwd=command_folder, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace") as process:
223
+ environment_of_the_program = dict(os.environ)
224
+ environment_of_the_program.update(environment_variables)
225
+ with subprocess.Popen([program] + arguments, cwd=command_folder, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", env=environment_of_the_program) as process:
215
226
  for line in process.stdout:
216
227
  with job.lock:
217
228
  job.log = job.log + line
@@ -293,6 +304,30 @@ class SCTaskRunnerServer:
293
304
  raise ValueError(f"The result-folder '{job.result_folder}' of job '{job.job_id}' is not inside the workspace of that job.")
294
305
  return result_folder
295
306
 
307
+ def __get_amount_of_running_jobs(self) -> int:
308
+ """Returns the amount of jobs which are currently running. A job counts as running from its submit until its
309
+ program has ended, which includes the extraction of its repository."""
310
+ with self.__jobs_lock:
311
+ jobs = list(self.__jobs.values())
312
+ amount_of_running_jobs = 0
313
+ for job in jobs:
314
+ with job.lock:
315
+ if job.state == "running":
316
+ amount_of_running_jobs = amount_of_running_jobs+1
317
+ return amount_of_running_jobs
318
+
319
+ def __get_metrics(self) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
320
+ """Returns the metrics of this runner in the text-based exposition-format of Prometheus, so that the utilization
321
+ of a runner can be monitored by a monitoring-system which scrapes it. Only the amount of running jobs is exposed:
322
+ anything about a job itself (its repository, its program) is information of the client which submitted it."""
323
+ metric_name = "sctaskrunner_running_jobs"
324
+ lines = [
325
+ f"# HELP {metric_name} Amount of jobs which are currently running on this runner.",
326
+ f"# TYPE {metric_name} gauge",
327
+ f"{metric_name} {self.__get_amount_of_running_jobs()}",
328
+ ]
329
+ return "\n".join(lines)+"\n"
330
+
296
331
  @GeneralUtilities.check_arguments
297
332
  def __delete_job(self, job_id: str) -> None: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
298
333
  with self.__jobs_lock:
@@ -381,6 +416,9 @@ class SCTaskRunnerServer:
381
416
  if self.path == "/os":
382
417
  self.__send_text(200, outer.operating_system_name)
383
418
  return
419
+ if self.path == "/metrics":
420
+ self.__send(200, outer._SCTaskRunnerServer__get_metrics().encode("utf-8"), "text/plain; version=0.0.4; charset=utf-8")
421
+ return
384
422
  parts = self.path.strip("/").split("/")
385
423
  if len(parts) >= 2 and parts[0] == "jobs":
386
424
  self.__handle_job_get(parts)
@@ -447,9 +485,17 @@ class SCTaskRunnerServer:
447
485
  arguments = json.loads(base64.b64decode(self.headers.get("X-Arguments")).decode("utf-8"))
448
486
  working_directory = self.headers.get("X-Working-Directory", ".")
449
487
  result_folder = self.headers.get("X-Result-Folder")
488
+ #A client of a version which does not know environment-variables does not send this header.
489
+ environment_variables_header = self.headers.get("X-Environment-Variables")
490
+ environment_variables = {} if environment_variables_header is None else json.loads(base64.b64decode(environment_variables_header).decode("utf-8"))
491
+ if not SCTaskRunnerServer.is_valid_set_of_environment_variables(environment_variables):
492
+ self.__send_text(400, "The header X-Environment-Variables must contain a json-object whose names and values are texts.")
493
+ return
450
494
  #the body is handed over as a stream (and not read here) so the archive is written to disk chunk-wise, see __start_job.
451
- job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory, result_folder)
452
- self.__send_json(200, {"job_id": job_id})
495
+ job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory, result_folder, environment_variables)
496
+ #The names of the applied environment-variables are part of the answer, so that a client can tell this runner
497
+ #from one which does not support environment-variables and would ignore them (see TFCPS_RemoteBuild).
498
+ self.__send_json(200, {"job_id": job_id, "applied_environment_variables": sorted(environment_variables.keys())})
453
499
 
454
500
  def do_DELETE(self): # pylint:disable=invalid-name
455
501
  self.__handle(self.__delete)