ScriptCollection 4.4.72__py3-none-any.whl → 4.4.73__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/Executables.py +19 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/AccountChecks.py +71 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/ContainerChecks.py +34 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/FileSystemChecks.py +141 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/KernelChecks.py +49 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/LogChecks.py +39 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/NetworkChecks.py +67 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/PackageIntegrityChecks.py +62 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/PersistenceChecks.py +99 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/ProcessChecks.py +88 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/SSHChecks.py +56 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/__init__.py +0 -0
- ScriptCollection/LinuxServerSecurityCheck/LinuxServerSecurityChecker.py +90 -0
- ScriptCollection/LinuxServerSecurityCheck/LinuxSystemAccess.py +85 -0
- ScriptCollection/LinuxServerSecurityCheck/PackageOwnershipLookup.py +42 -0
- ScriptCollection/LinuxServerSecurityCheck/SecurityCheckBase.py +19 -0
- ScriptCollection/LinuxServerSecurityCheck/SecurityFinding.py +19 -0
- ScriptCollection/LinuxServerSecurityCheck/SuspiciousCommandPatterns.py +46 -0
- ScriptCollection/LinuxServerSecurityCheck/SystemAccessBase.py +244 -0
- ScriptCollection/LinuxServerSecurityCheck/SystemEntities.py +45 -0
- ScriptCollection/LinuxServerSecurityCheck/__init__.py +0 -0
- ScriptCollection/ScriptCollectionCore.py +1 -1
- ScriptCollection/TFCPS/Android/TFCPS_CodeUnitSpecific_Android.py +109 -5
- ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py +1 -1
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +4 -4
- ScriptCollection/TFCPS/SCTaskRunnerServer.py +55 -9
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +5 -2
- ScriptCollection/TFCPS/TFCPS_RemoteBuild.py +39 -11
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +19 -0
- {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.73.dist-info}/METADATA +177 -177
- {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.73.dist-info}/RECORD +34 -14
- {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.73.dist-info}/entry_points.txt +1 -0
- {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.73.dist-info}/WHEEL +0 -0
- {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.73.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
import ipaddress
|
|
2
|
+
import os
|
|
3
|
+
import posixpath
|
|
4
|
+
import stat
|
|
5
|
+
import sys
|
|
6
|
+
from abc import ABC, abstractmethod
|
|
7
|
+
from collections.abc import Iterator
|
|
8
|
+
from .SystemEntities import UserAccount, ProcessInformation, ProgramResult, SocketInformation
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class SystemAccessBase(ABC):
|
|
12
|
+
"""Read-only access to the system which is checked.
|
|
13
|
+
All paths are absolute paths of the checked system.
|
|
14
|
+
The abstract methods are the primitives which access the system; everything else is derived from them,
|
|
15
|
+
so that a testcase only has to replace the primitives."""
|
|
16
|
+
|
|
17
|
+
maximal_size_of_text_files: int = 50 * 1024 * 1024
|
|
18
|
+
seconds_per_day: int = 24 * 60 * 60
|
|
19
|
+
|
|
20
|
+
@abstractmethod
|
|
21
|
+
def current_user_is_root(self) -> bool:
|
|
22
|
+
raise NotImplementedError
|
|
23
|
+
|
|
24
|
+
@abstractmethod
|
|
25
|
+
def read_bytes(self, path: str, maximal_amount_of_bytes: int) -> bytes | None:
|
|
26
|
+
"""Returns the first bytes of the file or None if the file does not exist (anymore)."""
|
|
27
|
+
raise NotImplementedError
|
|
28
|
+
|
|
29
|
+
@abstractmethod
|
|
30
|
+
def list_folder(self, path: str) -> list[str] | None:
|
|
31
|
+
"""Returns the names of the entries of the folder or None if the folder does not exist (anymore)."""
|
|
32
|
+
raise NotImplementedError
|
|
33
|
+
|
|
34
|
+
@abstractmethod
|
|
35
|
+
def get_lstat(self, path: str) -> os.stat_result | None:
|
|
36
|
+
"""Returns the stat-information of the path itself (a symlink is not followed) or None if the path does not exist (anymore)."""
|
|
37
|
+
raise NotImplementedError
|
|
38
|
+
|
|
39
|
+
@abstractmethod
|
|
40
|
+
def path_exists(self, path: str) -> bool:
|
|
41
|
+
"""Returns whether the path exists. Symlinks are followed."""
|
|
42
|
+
raise NotImplementedError
|
|
43
|
+
|
|
44
|
+
@abstractmethod
|
|
45
|
+
def read_link(self, path: str) -> str | None:
|
|
46
|
+
"""Returns the target of the symlink or None if the symlink does not exist (anymore) or has no target (for example /proc/<pid>/exe of a kernel-thread)."""
|
|
47
|
+
raise NotImplementedError
|
|
48
|
+
|
|
49
|
+
@abstractmethod
|
|
50
|
+
def program_is_available(self, program: str) -> bool:
|
|
51
|
+
raise NotImplementedError
|
|
52
|
+
|
|
53
|
+
@abstractmethod
|
|
54
|
+
def run_program(self, program: str, arguments: list[str]) -> ProgramResult:
|
|
55
|
+
"""Runs a program which must not change anything on the system."""
|
|
56
|
+
raise NotImplementedError
|
|
57
|
+
|
|
58
|
+
@abstractmethod
|
|
59
|
+
def get_file_attribute_flags(self, path: str) -> int | None:
|
|
60
|
+
"""Returns the inode-flags (as shown by lsattr) of a regular file or None if the filesystem does not support them."""
|
|
61
|
+
raise NotImplementedError
|
|
62
|
+
|
|
63
|
+
@abstractmethod
|
|
64
|
+
def get_current_time(self) -> float:
|
|
65
|
+
raise NotImplementedError
|
|
66
|
+
|
|
67
|
+
def read_text(self, path: str) -> str | None:
|
|
68
|
+
content = self.read_bytes(path, SystemAccessBase.maximal_size_of_text_files)
|
|
69
|
+
if content is None:
|
|
70
|
+
return None
|
|
71
|
+
return content.decode("utf-8", errors="replace")
|
|
72
|
+
|
|
73
|
+
def is_regular_file(self, path: str) -> bool:
|
|
74
|
+
file_stat = self.get_lstat(path)
|
|
75
|
+
return file_stat is not None and stat.S_ISREG(file_stat.st_mode)
|
|
76
|
+
|
|
77
|
+
def is_symlink(self, path: str) -> bool:
|
|
78
|
+
file_stat = self.get_lstat(path)
|
|
79
|
+
return file_stat is not None and stat.S_ISLNK(file_stat.st_mode)
|
|
80
|
+
|
|
81
|
+
def is_folder(self, path: str) -> bool:
|
|
82
|
+
file_stat = self.get_lstat(path)
|
|
83
|
+
return file_stat is not None and stat.S_ISDIR(file_stat.st_mode)
|
|
84
|
+
|
|
85
|
+
def is_binary_content(self, content: bytes) -> bool:
|
|
86
|
+
return b"\0" in content[:8192]
|
|
87
|
+
|
|
88
|
+
def list_files_recursively(self, folder: str, excluded_folders: set[str]) -> Iterator[tuple[str, os.stat_result]]:
|
|
89
|
+
"""Yields every non-folder-entry (regular files, symlinks, ...) below the folder together with its lstat-information.
|
|
90
|
+
Symlinks to folders are not followed. Folders contained in excluded_folders are not entered."""
|
|
91
|
+
folders_to_visit: list[str] = [folder]
|
|
92
|
+
while len(folders_to_visit) > 0:
|
|
93
|
+
current_folder = folders_to_visit.pop()
|
|
94
|
+
entries = self.list_folder(current_folder)
|
|
95
|
+
if entries is None:
|
|
96
|
+
continue
|
|
97
|
+
for entry in sorted(entries):
|
|
98
|
+
path = posixpath.join(current_folder, entry)
|
|
99
|
+
entry_stat = self.get_lstat(path)
|
|
100
|
+
if entry_stat is None:
|
|
101
|
+
continue
|
|
102
|
+
if stat.S_ISDIR(entry_stat.st_mode):
|
|
103
|
+
if path not in excluded_folders:
|
|
104
|
+
folders_to_visit.append(path)
|
|
105
|
+
else:
|
|
106
|
+
yield path, entry_stat
|
|
107
|
+
|
|
108
|
+
def list_regular_files_recursively(self, folder: str) -> list[str]:
|
|
109
|
+
return [path for path, file_stat in self.list_files_recursively(folder, set()) if stat.S_ISREG(file_stat.st_mode)]
|
|
110
|
+
|
|
111
|
+
def path_is_alias(self, path: str) -> bool:
|
|
112
|
+
"""Returns whether the path is reachable only through a symlinked top-level-folder (for example /lib/systemd on a system with merged /usr),
|
|
113
|
+
so that its content is already reachable through another path."""
|
|
114
|
+
parts = [part for part in path.split("/") if part != ""]
|
|
115
|
+
current = ""
|
|
116
|
+
for part in parts[:-1]:
|
|
117
|
+
current = current + "/" + part
|
|
118
|
+
if self.is_symlink(current):
|
|
119
|
+
return True
|
|
120
|
+
return False
|
|
121
|
+
|
|
122
|
+
def get_users(self) -> list[UserAccount]:
|
|
123
|
+
content = self.read_text("/etc/passwd")
|
|
124
|
+
if content is None:
|
|
125
|
+
raise FileNotFoundError("/etc/passwd does not exist.")
|
|
126
|
+
result: list[UserAccount] = []
|
|
127
|
+
for line in content.splitlines():
|
|
128
|
+
if line.strip() == "" or line.startswith("#"):
|
|
129
|
+
continue
|
|
130
|
+
fields = line.split(":")
|
|
131
|
+
if len(fields) < 7:
|
|
132
|
+
continue
|
|
133
|
+
result.append(UserAccount(fields[0], int(fields[2]), int(fields[3]), fields[5], fields[6]))
|
|
134
|
+
return result
|
|
135
|
+
|
|
136
|
+
def get_home_folders(self) -> list[str]:
|
|
137
|
+
result: list[str] = []
|
|
138
|
+
for user in self.get_users():
|
|
139
|
+
if user.home_folder not in result and user.home_folder.startswith("/") and self.is_folder(user.home_folder):
|
|
140
|
+
result.append(user.home_folder)
|
|
141
|
+
return result
|
|
142
|
+
|
|
143
|
+
def get_minimal_uid_of_regular_users(self) -> int:
|
|
144
|
+
content = self.read_text("/etc/login.defs")
|
|
145
|
+
if content is not None:
|
|
146
|
+
for line in content.splitlines():
|
|
147
|
+
fields = line.split()
|
|
148
|
+
if len(fields) == 2 and fields[0] == "UID_MIN":
|
|
149
|
+
return int(fields[1])
|
|
150
|
+
# 1000 is the value which shadow-utils uses when UID_MIN is not defined in /etc/login.defs.
|
|
151
|
+
return 1000
|
|
152
|
+
|
|
153
|
+
def get_process_ids(self) -> list[int]:
|
|
154
|
+
entries = self.list_folder("/proc")
|
|
155
|
+
if entries is None:
|
|
156
|
+
raise FileNotFoundError("/proc does not exist.")
|
|
157
|
+
return sorted(int(entry) for entry in entries if entry.isdigit())
|
|
158
|
+
|
|
159
|
+
def get_process(self, pid: int) -> ProcessInformation | None:
|
|
160
|
+
"""Returns None if the process does not exist (anymore)."""
|
|
161
|
+
status = self.read_text(f"/proc/{pid}/status")
|
|
162
|
+
if status is None:
|
|
163
|
+
return None
|
|
164
|
+
name = ""
|
|
165
|
+
uid = -1
|
|
166
|
+
for line in status.splitlines():
|
|
167
|
+
if line.startswith("Name:"):
|
|
168
|
+
name = line[len("Name:"):].strip()
|
|
169
|
+
elif line.startswith("Uid:"):
|
|
170
|
+
uid = int(line.split()[1])
|
|
171
|
+
commandline_bytes = self.read_bytes(f"/proc/{pid}/cmdline", 65536)
|
|
172
|
+
if commandline_bytes is None:
|
|
173
|
+
return None
|
|
174
|
+
commandline = commandline_bytes.replace(b"\0", b" ").decode("utf-8", errors="replace").strip()
|
|
175
|
+
executable = self.read_link(f"/proc/{pid}/exe")
|
|
176
|
+
return ProcessInformation(pid, name, uid, executable, commandline)
|
|
177
|
+
|
|
178
|
+
def get_processes(self) -> list[ProcessInformation]:
|
|
179
|
+
result: list[ProcessInformation] = []
|
|
180
|
+
for pid in self.get_process_ids():
|
|
181
|
+
process = self.get_process(pid)
|
|
182
|
+
if process is not None:
|
|
183
|
+
result.append(process)
|
|
184
|
+
return result
|
|
185
|
+
|
|
186
|
+
def get_processes_by_socket_inode(self) -> dict[int, list[ProcessInformation]]:
|
|
187
|
+
result: dict[int, list[ProcessInformation]] = {}
|
|
188
|
+
for process in self.get_processes():
|
|
189
|
+
file_descriptors = self.list_folder(f"/proc/{process.pid}/fd")
|
|
190
|
+
if file_descriptors is None:
|
|
191
|
+
continue
|
|
192
|
+
for file_descriptor in file_descriptors:
|
|
193
|
+
target = self.read_link(f"/proc/{process.pid}/fd/{file_descriptor}")
|
|
194
|
+
if target is not None and target.startswith("socket:[") and target.endswith("]"):
|
|
195
|
+
inode = int(target[len("socket:["):-1])
|
|
196
|
+
processes = result.setdefault(inode, [])
|
|
197
|
+
if process not in processes:
|
|
198
|
+
processes.append(process)
|
|
199
|
+
return result
|
|
200
|
+
|
|
201
|
+
def get_sockets(self) -> list[SocketInformation]:
|
|
202
|
+
"""Returns the tcp- and udp-sockets of all network-namespaces (so also the sockets of containers)."""
|
|
203
|
+
socket_tables: list[str] = []
|
|
204
|
+
known_namespaces: set[str] = set()
|
|
205
|
+
for pid in self.get_process_ids():
|
|
206
|
+
namespace = self.read_link(f"/proc/{pid}/ns/net")
|
|
207
|
+
if namespace is not None and namespace not in known_namespaces:
|
|
208
|
+
known_namespaces.add(namespace)
|
|
209
|
+
socket_tables.append(f"/proc/{pid}/net")
|
|
210
|
+
if len(socket_tables) == 0:
|
|
211
|
+
socket_tables.append("/proc/net")
|
|
212
|
+
result: list[SocketInformation] = []
|
|
213
|
+
known_inodes: set[int] = set()
|
|
214
|
+
for socket_table in socket_tables:
|
|
215
|
+
for protocol in ["tcp", "tcp6", "udp", "udp6"]:
|
|
216
|
+
content = self.read_text(f"{socket_table}/{protocol}")
|
|
217
|
+
if content is None:
|
|
218
|
+
continue
|
|
219
|
+
for socket_information in SystemAccessBase.parse_socket_table(protocol, content):
|
|
220
|
+
if socket_information.inode == 0 or socket_information.inode not in known_inodes:
|
|
221
|
+
known_inodes.add(socket_information.inode)
|
|
222
|
+
result.append(socket_information)
|
|
223
|
+
return result
|
|
224
|
+
|
|
225
|
+
@staticmethod
|
|
226
|
+
def parse_socket_table(protocol: str, content: str) -> list[SocketInformation]:
|
|
227
|
+
result: list[SocketInformation] = []
|
|
228
|
+
for line in content.splitlines()[1:]:
|
|
229
|
+
fields = line.split()
|
|
230
|
+
if len(fields) < 10:
|
|
231
|
+
continue
|
|
232
|
+
local_address, local_port = SystemAccessBase.decode_socket_address(fields[1])
|
|
233
|
+
remote_address, remote_port = SystemAccessBase.decode_socket_address(fields[2])
|
|
234
|
+
result.append(SocketInformation(protocol, local_address, local_port, remote_address, remote_port, fields[3].upper(), int(fields[9])))
|
|
235
|
+
return result
|
|
236
|
+
|
|
237
|
+
@staticmethod
|
|
238
|
+
def decode_socket_address(hex_address: str) -> tuple[str, int]:
|
|
239
|
+
"""Decodes an address like "0100007F:0035" from /proc/net/tcp. The address consists of 32-bit-words in host-byte-order."""
|
|
240
|
+
address_part, port_part = hex_address.split(":")
|
|
241
|
+
raw_address = bytes.fromhex(address_part)
|
|
242
|
+
if sys.byteorder == "little":
|
|
243
|
+
raw_address = b"".join(raw_address[index:index + 4][::-1] for index in range(0, len(raw_address), 4))
|
|
244
|
+
return str(ipaddress.ip_address(raw_address)), int(port_part, 16)
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
@dataclass(frozen=True)
|
|
5
|
+
class UserAccount:
|
|
6
|
+
name: str
|
|
7
|
+
uid: int
|
|
8
|
+
gid: int
|
|
9
|
+
home_folder: str
|
|
10
|
+
shell: str
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class ProcessInformation:
|
|
15
|
+
pid: int
|
|
16
|
+
name: str
|
|
17
|
+
uid: int
|
|
18
|
+
executable: str | None
|
|
19
|
+
"""Target of /proc/<pid>/exe as seen from the mount-namespace of the process. None for kernel-threads and zombies."""
|
|
20
|
+
commandline: str
|
|
21
|
+
|
|
22
|
+
def is_kernel_thread_or_zombie(self) -> bool:
|
|
23
|
+
return self.executable is None and self.commandline == ""
|
|
24
|
+
|
|
25
|
+
def describe(self) -> str:
|
|
26
|
+
return f"pid {self.pid} (name \"{self.name}\", uid {self.uid}, executable \"{self.executable}\", commandline \"{self.commandline[:300]}\")"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class ProgramResult:
|
|
31
|
+
exit_code: int
|
|
32
|
+
stdout: str
|
|
33
|
+
stderr: str
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class SocketInformation:
|
|
38
|
+
protocol: str
|
|
39
|
+
local_address: str
|
|
40
|
+
local_port: int
|
|
41
|
+
remote_address: str
|
|
42
|
+
remote_port: int
|
|
43
|
+
state: str
|
|
44
|
+
"""Hex-state as written in /proc/net/tcp, for example "0A" for listening tcp-sockets."""
|
|
45
|
+
inode: int
|
|
File without changes
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import re
|
|
3
|
+
import base64
|
|
4
|
+
import binascii
|
|
5
|
+
import zipfile
|
|
3
6
|
import shlex
|
|
4
7
|
import shutil
|
|
5
8
|
import requests
|
|
@@ -32,7 +35,29 @@ class TFCPS_CodeUnitSpecific_Android_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
32
35
|
- Only if the codeunit has visual-regression-tests (see run_visual_regression_tests): it applies the plugin
|
|
33
36
|
"io.github.takahirom.roborazzi" and configures "roborazzi { outputDir = file('../Other/Resources/VisualRegressionBaselines');
|
|
34
37
|
compare { outputDir = layout.buildDirectory.dir('outputs/roborazzi-comparison') } }" (see
|
|
35
|
-
get_visual_regression_baselines_folder and get_visual_regression_comparison_folder_relative_to_build_folder).
|
|
38
|
+
get_visual_regression_baselines_folder and get_visual_regression_comparison_folder_relative_to_build_folder).
|
|
39
|
+
- The release-build-type is signed with the upload-key if the environment-variables "SC_ANDROID_SIGNING_STORE_FILE",
|
|
40
|
+
"SC_ANDROID_SIGNING_STORE_PASSWORD", "SC_ANDROID_SIGNING_KEY_ALIAS" and "SC_ANDROID_SIGNING_KEY_PASSWORD" are set,
|
|
41
|
+
and with the debug-key otherwise (see build), for example:
|
|
42
|
+
"signingConfigs { release { if (System.getenv('SC_ANDROID_SIGNING_STORE_FILE') != null) { storeFile =
|
|
43
|
+
file(System.getenv('SC_ANDROID_SIGNING_STORE_FILE')); storePassword = System.getenv('SC_ANDROID_SIGNING_STORE_PASSWORD');
|
|
44
|
+
keyAlias = System.getenv('SC_ANDROID_SIGNING_KEY_ALIAS'); keyPassword = System.getenv('SC_ANDROID_SIGNING_KEY_PASSWORD') } } }"
|
|
45
|
+
and "release { signingConfig = System.getenv('SC_ANDROID_SIGNING_STORE_FILE') != null ? signingConfigs.release :
|
|
46
|
+
signingConfigs.debug }"."""
|
|
47
|
+
|
|
48
|
+
# The environment-variables (of the machine which runs the build, for example the build-pipeline) which contain the
|
|
49
|
+
# upload-key of a productive build, mapped to the environment-variables of the gradle-run on the runner which
|
|
50
|
+
# receive them (see build). The keystore is given as the base64-encoding of the keystore-file, because an
|
|
51
|
+
# environment-variable can not contain a file.
|
|
52
|
+
__signing_environment_variables: dict[str, str] = {
|
|
53
|
+
"Android_Signing_Keystore": "SC_ANDROID_SIGNING_KEYSTORE_BASE64",
|
|
54
|
+
"Android_Signing_StorePassword": "SC_ANDROID_SIGNING_STORE_PASSWORD",
|
|
55
|
+
"Android_Signing_KeyAlias": "SC_ANDROID_SIGNING_KEY_ALIAS",
|
|
56
|
+
"Android_Signing_KeyPassword": "SC_ANDROID_SIGNING_KEY_PASSWORD",
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
# The name the subject of the debug-certificate of the android-sdk contains ("CN=Android Debug,O=Android,C=US").
|
|
60
|
+
__subject_of_the_debug_certificate: bytes = b"Android Debug"
|
|
36
61
|
|
|
37
62
|
# The repositories in which a dependency or a gradle-plugin of an android-app is published. Gradle itself searches
|
|
38
63
|
# all of them (they are the repositories a project declares by "google()", "mavenCentral()" and
|
|
@@ -94,16 +119,38 @@ class TFCPS_CodeUnitSpecific_Android_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
94
119
|
def build(self) -> None:
|
|
95
120
|
"""Builds the app-bundle (which is what is uploaded to a store) and the apk (which can be installed directly) of the
|
|
96
121
|
release-build-type and generates the bill-of-materials. All of them are generated by one gradle-run, so the runner
|
|
97
|
-
has to be used only once for them.
|
|
122
|
+
has to be used only once for them.
|
|
123
|
+
|
|
124
|
+
A productive build (target-environment-type "Productive") is signed with the upload-key, so its app-bundle can be
|
|
125
|
+
uploaded to Google Play. The upload-key is taken from the environment-variables "Android_Signing_Keystore" (the
|
|
126
|
+
base64-encoding of the keystore-file), "Android_Signing_StorePassword", "Android_Signing_KeyAlias" and
|
|
127
|
+
"Android_Signing_KeyPassword", which are required for a productive build. They are given to the gradle-run on the
|
|
128
|
+
runner as environment-variables only (see TFCPS_RemoteBuild.run_program_on_runner); on the runner the keystore
|
|
129
|
+
is written into a temporary file outside of the workspace which is removed when gradle ends. Afterwards it is
|
|
130
|
+
verified that the app-bundle is signed, but not with the debug-key.
|
|
131
|
+
Every other build is signed with the debug-key of the runner, so a developer can build and install the app
|
|
132
|
+
without the upload-key."""
|
|
98
133
|
codeunit_name = self.get_codeunit_name()
|
|
99
134
|
codeunit_version: str = self.tfcps_Tools_General.get_version_of_codeunit(self.get_codeunit_file())
|
|
100
135
|
outputs_folder = os.path.join(self.get_build_folder(), "outputs")
|
|
101
136
|
GeneralUtilities.ensure_directory_does_not_exist(outputs_folder)
|
|
102
|
-
|
|
137
|
+
gradle_tasks = ["bundleRelease", "assembleRelease", "cyclonedxBom"]
|
|
138
|
+
is_signed_with_upload_key = self.get_target_environment_type() == "Productive"
|
|
139
|
+
if is_signed_with_upload_key:
|
|
140
|
+
self._protected_sc.log.log("Build the release-build signed with the upload-key...")
|
|
141
|
+
signing_environment_variables = TFCPS_CodeUnitSpecific_Android_Functions.get_signing_environment_variables_for_runner(dict(os.environ))
|
|
142
|
+
script = TFCPS_CodeUnitSpecific_Android_Functions.get_script_which_signs_with_upload_key(self.__get_gradle_arguments(gradle_tasks))
|
|
143
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", ["-c", script], self.get_gradle_project_folder(), outputs_folder, self.__get_folders_which_are_not_transferred(), signing_environment_variables)
|
|
144
|
+
else:
|
|
145
|
+
self._protected_sc.log.log(f'Build the release-build signed with the debug-key of the runner (only a build with target-environment-type "Productive" is signed with the upload-key, this one has "{self.get_target_environment_type()}")...')
|
|
146
|
+
self.__run_gradle_on_runner(gradle_tasks, outputs_folder)
|
|
103
147
|
|
|
104
148
|
aab_folder = os.path.join(self.get_artifacts_folder(), "BuildResult_AAB")
|
|
105
149
|
GeneralUtilities.ensure_folder_exists_and_is_empty(aab_folder)
|
|
106
150
|
aab_file_original = self._protected_sc.find_file_by_extension(os.path.join(outputs_folder, "bundle", "release"), "aab")
|
|
151
|
+
if is_signed_with_upload_key:
|
|
152
|
+
GeneralUtilities.assert_condition(TFCPS_CodeUnitSpecific_Android_Functions.is_signed(aab_file_original), f'The app-bundle "{aab_file_original}" is not signed. The release-build-type of "{self.get_build_file()}" has to use the upload-key which is given by the environment-variables "SC_ANDROID_SIGNING_*" (see the docstring of TFCPS_CodeUnitSpecific_Android_Functions).')
|
|
153
|
+
GeneralUtilities.assert_condition(not TFCPS_CodeUnitSpecific_Android_Functions.is_signed_with_debug_key(aab_file_original), f'The app-bundle "{aab_file_original}" is signed with the debug-key, so Google Play would reject it. The release-build-type of "{self.get_build_file()}" has to use the upload-key which is given by the environment-variables "SC_ANDROID_SIGNING_*" (see the docstring of TFCPS_CodeUnitSpecific_Android_Functions).')
|
|
107
154
|
shutil.copyfile(aab_file_original, os.path.join(aab_folder, f"{codeunit_name}.aab"))
|
|
108
155
|
|
|
109
156
|
apk_folder = os.path.join(self.get_artifacts_folder(), "BuildResult_APK")
|
|
@@ -214,7 +261,7 @@ class TFCPS_CodeUnitSpecific_Android_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
214
261
|
def __run_gradle_on_runner(self, tasks: list[str], result_folder: str) -> None:
|
|
215
262
|
"""Runs the given gradle-tasks on an android-task-runner and writes the content of result_folder (which must be a
|
|
216
263
|
folder inside this codeunit) back into this codeunit. A failure of gradle raises an exception."""
|
|
217
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", self.__get_gradle_arguments(tasks), self.get_gradle_project_folder(), result_folder, self.__get_folders_which_are_not_transferred())
|
|
264
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", self.__get_gradle_arguments(tasks), self.get_gradle_project_folder(), result_folder, self.__get_folders_which_are_not_transferred(), {})
|
|
218
265
|
|
|
219
266
|
@GeneralUtilities.check_arguments
|
|
220
267
|
def __run_gradle_on_runner_and_get_exit_code(self, tasks: list[str], result_folder: str) -> int:
|
|
@@ -225,11 +272,68 @@ class TFCPS_CodeUnitSpecific_Android_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
225
272
|
succeeds."""
|
|
226
273
|
result_folder_relative_to_project = os.path.relpath(result_folder, self.get_gradle_project_folder()).replace("\\", "/")
|
|
227
274
|
script = TFCPS_CodeUnitSpecific_Android_Functions.get_script_which_preserves_exit_code(self.__get_gradle_arguments(tasks), result_folder_relative_to_project, TFCPS_CodeUnitSpecific_Android_Functions.__exit_code_file_name)
|
|
228
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", ["-c", script], self.get_gradle_project_folder(), result_folder, self.__get_folders_which_are_not_transferred())
|
|
275
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", ["-c", script], self.get_gradle_project_folder(), result_folder, self.__get_folders_which_are_not_transferred(), {})
|
|
229
276
|
local_exit_code_file = os.path.join(result_folder, TFCPS_CodeUnitSpecific_Android_Functions.__exit_code_file_name)
|
|
230
277
|
GeneralUtilities.assert_file_exists(local_exit_code_file)
|
|
231
278
|
return int(GeneralUtilities.read_text_from_file(local_exit_code_file).strip())
|
|
232
279
|
|
|
280
|
+
@staticmethod
|
|
281
|
+
@GeneralUtilities.check_arguments
|
|
282
|
+
def get_signing_environment_variables_for_runner(environment: dict[str, str]) -> dict[str, str]:
|
|
283
|
+
"""Returns the environment-variables which give the upload-key to the gradle-run on the runner, taken from the
|
|
284
|
+
given environment-variables of this machine (see build). The names are compared case-insensitively, because
|
|
285
|
+
Windows exposes the names of environment-variables upper-cased via os.environ."""
|
|
286
|
+
environment_upper = {name.upper(): value for name, value in environment.items()}
|
|
287
|
+
missing_variables = [name for name in TFCPS_CodeUnitSpecific_Android_Functions.__signing_environment_variables if len((environment_upper.get(name.upper()) or GeneralUtilities.empty_string).strip()) == 0]
|
|
288
|
+
GeneralUtilities.assert_condition(len(missing_variables) == 0, f'A productive build of an android-app is signed with the upload-key, which requires the environment-variable(s) {", ".join(missing_variables)}. "Android_Signing_Keystore" contains the base64-encoding of the keystore-file.')
|
|
289
|
+
result: dict[str, str] = {}
|
|
290
|
+
for name, name_on_runner in TFCPS_CodeUnitSpecific_Android_Functions.__signing_environment_variables.items():
|
|
291
|
+
result[name_on_runner] = environment_upper[name.upper()]
|
|
292
|
+
# A base64-encoding is often wrapped into several lines (for example by the base64-tool), which is not part of it.
|
|
293
|
+
keystore_base64 = "".join(result["SC_ANDROID_SIGNING_KEYSTORE_BASE64"].split())
|
|
294
|
+
try:
|
|
295
|
+
base64.b64decode(keystore_base64, validate=True)
|
|
296
|
+
except binascii.Error as exception:
|
|
297
|
+
raise ValueError('The environment-variable "Android_Signing_Keystore" does not contain a valid base64-encoding of the keystore-file.') from exception
|
|
298
|
+
result["SC_ANDROID_SIGNING_KEYSTORE_BASE64"] = keystore_base64
|
|
299
|
+
return result
|
|
300
|
+
|
|
301
|
+
@staticmethod
|
|
302
|
+
@GeneralUtilities.check_arguments
|
|
303
|
+
def get_script_which_signs_with_upload_key(gradle_arguments: list[str]) -> str:
|
|
304
|
+
"""Returns the bash-script which runs gradle on the runner with the upload-key: it writes the keystore (which it
|
|
305
|
+
gets base64-encoded in an environment-variable) into a temporary file outside of the workspace of the job, gives
|
|
306
|
+
that file to gradle via "SC_ANDROID_SIGNING_STORE_FILE" and removes it when the script ends, regardless of how it
|
|
307
|
+
ends."""
|
|
308
|
+
return "set -e; " \
|
|
309
|
+
+ 'keystore_file="$(mktemp)"; ' \
|
|
310
|
+
+ """trap 'rm -f "$keystore_file"' EXIT; """ \
|
|
311
|
+
+ """printf '%s' "$SC_ANDROID_SIGNING_KEYSTORE_BASE64" | base64 -d > "$keystore_file"; """ \
|
|
312
|
+
+ "unset SC_ANDROID_SIGNING_KEYSTORE_BASE64; " \
|
|
313
|
+
+ 'export SC_ANDROID_SIGNING_STORE_FILE="$keystore_file"; ' \
|
|
314
|
+
+ "bash " + shlex.join(gradle_arguments)
|
|
315
|
+
|
|
316
|
+
@staticmethod
|
|
317
|
+
@GeneralUtilities.check_arguments
|
|
318
|
+
def __get_signature_blocks(signed_file: str) -> list[bytes]:
|
|
319
|
+
"""Returns the signature-blocks of a jar-signed file (an app-bundle is jar-signed): the files "META-INF/*.RSA",
|
|
320
|
+
"META-INF/*.DSA" and "META-INF/*.EC", which contain the certificate of the signer."""
|
|
321
|
+
with zipfile.ZipFile(signed_file) as archive:
|
|
322
|
+
return [archive.read(name) for name in archive.namelist() if re.fullmatch(r"META-INF/[^/]+\.(RSA|DSA|EC)", name, re.IGNORECASE)]
|
|
323
|
+
|
|
324
|
+
@staticmethod
|
|
325
|
+
@GeneralUtilities.check_arguments
|
|
326
|
+
def is_signed(signed_file: str) -> bool:
|
|
327
|
+
"""Whether the given app-bundle is signed."""
|
|
328
|
+
return len(TFCPS_CodeUnitSpecific_Android_Functions.__get_signature_blocks(signed_file)) > 0
|
|
329
|
+
|
|
330
|
+
@staticmethod
|
|
331
|
+
@GeneralUtilities.check_arguments
|
|
332
|
+
def is_signed_with_debug_key(signed_file: str) -> bool:
|
|
333
|
+
"""Whether the given app-bundle is signed with the debug-key of an android-sdk. The certificate of that key is
|
|
334
|
+
always issued to "CN=Android Debug,O=Android,C=US", whose name is contained in the signature-block as text."""
|
|
335
|
+
return any(TFCPS_CodeUnitSpecific_Android_Functions.__subject_of_the_debug_certificate in block for block in TFCPS_CodeUnitSpecific_Android_Functions.__get_signature_blocks(signed_file))
|
|
336
|
+
|
|
233
337
|
@staticmethod
|
|
234
338
|
@GeneralUtilities.check_arguments
|
|
235
339
|
def get_script_which_preserves_exit_code(gradle_arguments: list[str], result_folder_relative_to_project: str, exit_code_file_name: str) -> str:
|
|
@@ -169,7 +169,7 @@ class TFCPS_CodeUnitSpecific_CPP_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
169
169
|
#run in - which is also the folder __collect_output_files searches afterwards.
|
|
170
170
|
#Nothing of a C/C++-codeunit is left out of the archive the runner gets: where a build-command puts its
|
|
171
171
|
#output is defined by that command, so there is no folder which is known to contain state of this machine.
|
|
172
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "/bin/sh", ["-c", " && ".join(self.macos_build_commands)], source_folder, source_folder, [])
|
|
172
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "/bin/sh", ["-c", " && ".join(self.macos_build_commands)], source_folder, source_folder, [], {})
|
|
173
173
|
self.__collect_output_files(source_folder, self.macos_output_file_patterns, "BuildResult_MacOS")
|
|
174
174
|
|
|
175
175
|
@GeneralUtilities.check_arguments
|
|
@@ -112,7 +112,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
112
112
|
"environment as a runner-built one.", LogLevel.Warning)
|
|
113
113
|
self._protected_sc.run_with_epew("flutter", "build windows", src_folder)
|
|
114
114
|
else:
|
|
115
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder, windows_release_folder, folders_which_are_not_transferred)
|
|
115
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder, windows_release_folder, folders_which_are_not_transferred, {})
|
|
116
116
|
windows_folder = os.path.join(artifacts_folder, "BuildResult_Windows")
|
|
117
117
|
GeneralUtilities.ensure_directory_does_not_exist(windows_folder)
|
|
118
118
|
GeneralUtilities.ensure_directory_exists(windows_folder)
|
|
@@ -134,7 +134,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
134
134
|
# builds), analogous to how ios-builds always run on the iOS-task-runner. See SCTaskRunnerMacOS and
|
|
135
135
|
# the remote-build-article in the reference.
|
|
136
136
|
macos_release_folder = os.path.join(src_folder, "build/macos/Build/Products/Release")
|
|
137
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder, macos_release_folder, folders_which_are_not_transferred)
|
|
137
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder, macos_release_folder, folders_which_are_not_transferred, {})
|
|
138
138
|
macos_folder = os.path.join(artifacts_folder, "BuildResult_MacOS")
|
|
139
139
|
GeneralUtilities.ensure_directory_does_not_exist(macos_folder)
|
|
140
140
|
GeneralUtilities.ensure_directory_exists(macos_folder)
|
|
@@ -143,7 +143,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
143
143
|
# iOS-builds must run on macOS and therefore always run on the dedicated iOS-task-runner (uniform
|
|
144
144
|
# builds). See SCTaskRunnerIOS and the remote-build-article in the reference.
|
|
145
145
|
ios_release_folder = os.path.join(src_folder, "build/ios/iphoneos")
|
|
146
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder, ios_release_folder, folders_which_are_not_transferred)
|
|
146
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder, ios_release_folder, folders_which_are_not_transferred, {})
|
|
147
147
|
ios_folder = os.path.join(artifacts_folder, "BuildResult_IOS")
|
|
148
148
|
GeneralUtilities.ensure_directory_does_not_exist(ios_folder)
|
|
149
149
|
GeneralUtilities.ensure_directory_exists(ios_folder)
|
|
@@ -153,7 +153,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
153
153
|
# remote-build-article in the reference), analogous to how ios-builds always run on a macOS-task-runner:
|
|
154
154
|
# the Android-SDK/NDK-toolchain no longer lives in SCBuilder itself, it moved into SCTaskRunnerAndroid.
|
|
155
155
|
aab_release_folder = os.path.join(src_folder, "build/app/outputs/bundle/release")
|
|
156
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder, aab_release_folder, folders_which_are_not_transferred)
|
|
156
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder, aab_release_folder, folders_which_are_not_transferred, {})
|
|
157
157
|
# The app-bundle which came back from the runner is published as the build-result of this codeunit,
|
|
158
158
|
# analogous to the targets above: it is the artifact of an android-build, and every codeunit has to have
|
|
159
159
|
# an artifact whose name matches "BuildResult_.+" (see TFCPS_CodeUnit_BuildCodeUnit.build_codeunit).
|
|
@@ -103,8 +103,13 @@ class SCTaskRunnerServer:
|
|
|
103
103
|
self.__sc.log.log(f"SCTaskRunner for operating-system '{self.operating_system_name}' is listening on {protocol}://{host}:{port}.")
|
|
104
104
|
server.serve_forever()
|
|
105
105
|
|
|
106
|
+
@staticmethod
|
|
107
|
+
def is_valid_set_of_environment_variables(environment_variables) -> bool:
|
|
108
|
+
"""Whether the given (deserialized) value is a set of environment-variables a job can be started with."""
|
|
109
|
+
return isinstance(environment_variables, dict) and all(isinstance(name, str) and isinstance(value, str) and len(name) > 0 and "=" not in name for name, value in environment_variables.items())
|
|
110
|
+
|
|
106
111
|
@GeneralUtilities.check_arguments
|
|
107
|
-
def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
112
|
+
def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, environment_variables: dict[str, str]) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
108
113
|
"""Starts a job for the repository-archive which can be read from archive_stream (the body of the request which
|
|
109
114
|
submitted the job; archive_size is its announced length). The archive is taken as a stream instead of as bytes
|
|
110
115
|
because it contains the complete repository of the client and is therefore regularly several hundred megabytes,
|
|
@@ -128,13 +133,15 @@ class SCTaskRunnerServer:
|
|
|
128
133
|
job = _RunnerJob(job_id, workspace_folder, result_folder)
|
|
129
134
|
with self.__jobs_lock:
|
|
130
135
|
self.__jobs[job_id] = job
|
|
131
|
-
thread = threading.Thread(target=self.__extract_payload_and_run_job, args=(job, archive_file, program, arguments, working_directory), daemon=True)
|
|
136
|
+
thread = threading.Thread(target=self.__extract_payload_and_run_job, args=(job, archive_file, program, arguments, working_directory, environment_variables), daemon=True)
|
|
132
137
|
thread.start()
|
|
133
|
-
|
|
138
|
+
#Only the names of the environment-variables are logged: their values are secrets of the client (for example the
|
|
139
|
+
#key which signs an app for a store).
|
|
140
|
+
self.__sc.log.log(f"Accepted job '{job_id}': run '{program} {' '.join(arguments)}' in '{working_directory}' of codeunit '{codeunit_name}', result-folder '{result_folder}', environment-variables [{', '.join(sorted(environment_variables.keys()))}] ({archive_size} transferred bytes of repository-content).")
|
|
134
141
|
return job_id
|
|
135
142
|
|
|
136
143
|
@GeneralUtilities.check_arguments
|
|
137
|
-
def __extract_payload_and_run_job(self, job: _RunnerJob, archive_file: str, program: str, arguments: list[str], working_directory: str) -> None:
|
|
144
|
+
def __extract_payload_and_run_job(self, job: _RunnerJob, archive_file: str, program: str, arguments: list[str], working_directory: str, environment_variables: dict[str, str]) -> None:
|
|
138
145
|
"""Extracts the transferred repository-archive into the workspace of the job and runs the requested program
|
|
139
146
|
afterwards. The extraction happens here - in the thread of the job - and not in the request which submitted the
|
|
140
147
|
job, because extracting the repository of a large codeunit takes minutes during which no data flows: the client
|
|
@@ -151,7 +158,7 @@ class SCTaskRunnerServer:
|
|
|
151
158
|
finally:
|
|
152
159
|
GeneralUtilities.ensure_file_does_not_exist(archive_file)
|
|
153
160
|
self.__sc.log.log(f"Job '{job.job_id}' works on the repository '{self.__get_origin_of_the_transferred_repository(job.workspace_folder)}'.")
|
|
154
|
-
self.__run_job(job, program, arguments, working_directory)
|
|
161
|
+
self.__run_job(job, program, arguments, working_directory, environment_variables)
|
|
155
162
|
|
|
156
163
|
@GeneralUtilities.check_arguments
|
|
157
164
|
def __get_origin_of_the_transferred_repository(self, workspace_folder: str) -> str:
|
|
@@ -204,14 +211,18 @@ class SCTaskRunnerServer:
|
|
|
204
211
|
remaining_amount_of_bytes = remaining_amount_of_bytes-len(chunk)
|
|
205
212
|
|
|
206
213
|
@GeneralUtilities.check_arguments
|
|
207
|
-
def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str) -> None:
|
|
214
|
+
def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str, environment_variables: dict[str, str]) -> None:
|
|
215
|
+
"""Runs the program of the job. The environment-variables of the job are set in addition to the ones of this
|
|
216
|
+
runner and only for this program."""
|
|
208
217
|
command_folder = os.path.join(job.workspace_folder, working_directory)
|
|
209
218
|
try:
|
|
210
219
|
self.__sc.log.log(f"Job '{job.job_id}' runs '{program} {' '.join(arguments)}' in '{command_folder}'...")
|
|
211
220
|
moment_of_the_start = time.time()
|
|
212
221
|
moment_of_the_last_progress_log = moment_of_the_start
|
|
213
222
|
amount_of_lines = 0
|
|
214
|
-
|
|
223
|
+
environment_of_the_program = dict(os.environ)
|
|
224
|
+
environment_of_the_program.update(environment_variables)
|
|
225
|
+
with subprocess.Popen([program] + arguments, cwd=command_folder, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", env=environment_of_the_program) as process:
|
|
215
226
|
for line in process.stdout:
|
|
216
227
|
with job.lock:
|
|
217
228
|
job.log = job.log + line
|
|
@@ -293,6 +304,30 @@ class SCTaskRunnerServer:
|
|
|
293
304
|
raise ValueError(f"The result-folder '{job.result_folder}' of job '{job.job_id}' is not inside the workspace of that job.")
|
|
294
305
|
return result_folder
|
|
295
306
|
|
|
307
|
+
def __get_amount_of_running_jobs(self) -> int:
|
|
308
|
+
"""Returns the amount of jobs which are currently running. A job counts as running from its submit until its
|
|
309
|
+
program has ended, which includes the extraction of its repository."""
|
|
310
|
+
with self.__jobs_lock:
|
|
311
|
+
jobs = list(self.__jobs.values())
|
|
312
|
+
amount_of_running_jobs = 0
|
|
313
|
+
for job in jobs:
|
|
314
|
+
with job.lock:
|
|
315
|
+
if job.state == "running":
|
|
316
|
+
amount_of_running_jobs = amount_of_running_jobs+1
|
|
317
|
+
return amount_of_running_jobs
|
|
318
|
+
|
|
319
|
+
def __get_metrics(self) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
320
|
+
"""Returns the metrics of this runner in the text-based exposition-format of Prometheus, so that the utilization
|
|
321
|
+
of a runner can be monitored by a monitoring-system which scrapes it. Only the amount of running jobs is exposed:
|
|
322
|
+
anything about a job itself (its repository, its program) is information of the client which submitted it."""
|
|
323
|
+
metric_name = "sctaskrunner_running_jobs"
|
|
324
|
+
lines = [
|
|
325
|
+
f"# HELP {metric_name} Amount of jobs which are currently running on this runner.",
|
|
326
|
+
f"# TYPE {metric_name} gauge",
|
|
327
|
+
f"{metric_name} {self.__get_amount_of_running_jobs()}",
|
|
328
|
+
]
|
|
329
|
+
return "\n".join(lines)+"\n"
|
|
330
|
+
|
|
296
331
|
@GeneralUtilities.check_arguments
|
|
297
332
|
def __delete_job(self, job_id: str) -> None: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
298
333
|
with self.__jobs_lock:
|
|
@@ -381,6 +416,9 @@ class SCTaskRunnerServer:
|
|
|
381
416
|
if self.path == "/os":
|
|
382
417
|
self.__send_text(200, outer.operating_system_name)
|
|
383
418
|
return
|
|
419
|
+
if self.path == "/metrics":
|
|
420
|
+
self.__send(200, outer._SCTaskRunnerServer__get_metrics().encode("utf-8"), "text/plain; version=0.0.4; charset=utf-8")
|
|
421
|
+
return
|
|
384
422
|
parts = self.path.strip("/").split("/")
|
|
385
423
|
if len(parts) >= 2 and parts[0] == "jobs":
|
|
386
424
|
self.__handle_job_get(parts)
|
|
@@ -447,9 +485,17 @@ class SCTaskRunnerServer:
|
|
|
447
485
|
arguments = json.loads(base64.b64decode(self.headers.get("X-Arguments")).decode("utf-8"))
|
|
448
486
|
working_directory = self.headers.get("X-Working-Directory", ".")
|
|
449
487
|
result_folder = self.headers.get("X-Result-Folder")
|
|
488
|
+
#A client of a version which does not know environment-variables does not send this header.
|
|
489
|
+
environment_variables_header = self.headers.get("X-Environment-Variables")
|
|
490
|
+
environment_variables = {} if environment_variables_header is None else json.loads(base64.b64decode(environment_variables_header).decode("utf-8"))
|
|
491
|
+
if not SCTaskRunnerServer.is_valid_set_of_environment_variables(environment_variables):
|
|
492
|
+
self.__send_text(400, "The header X-Environment-Variables must contain a json-object whose names and values are texts.")
|
|
493
|
+
return
|
|
450
494
|
#the body is handed over as a stream (and not read here) so the archive is written to disk chunk-wise, see __start_job.
|
|
451
|
-
job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory, result_folder)
|
|
452
|
-
|
|
495
|
+
job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory, result_folder, environment_variables)
|
|
496
|
+
#The names of the applied environment-variables are part of the answer, so that a client can tell this runner
|
|
497
|
+
#from one which does not support environment-variables and would ignore them (see TFCPS_RemoteBuild).
|
|
498
|
+
self.__send_json(200, {"job_id": job_id, "applied_environment_variables": sorted(environment_variables.keys())})
|
|
453
499
|
|
|
454
500
|
def do_DELETE(self): # pylint:disable=invalid-name
|
|
455
501
|
self.__handle(self.__delete)
|