ScriptCollection 4.4.72__py3-none-any.whl → 4.4.74__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. ScriptCollection/Executables.py +19 -0
  2. ScriptCollection/LinuxServerSecurityCheck/Checks/AccountChecks.py +71 -0
  3. ScriptCollection/LinuxServerSecurityCheck/Checks/ContainerChecks.py +34 -0
  4. ScriptCollection/LinuxServerSecurityCheck/Checks/FileSystemChecks.py +141 -0
  5. ScriptCollection/LinuxServerSecurityCheck/Checks/KernelChecks.py +49 -0
  6. ScriptCollection/LinuxServerSecurityCheck/Checks/LogChecks.py +39 -0
  7. ScriptCollection/LinuxServerSecurityCheck/Checks/NetworkChecks.py +67 -0
  8. ScriptCollection/LinuxServerSecurityCheck/Checks/PackageIntegrityChecks.py +62 -0
  9. ScriptCollection/LinuxServerSecurityCheck/Checks/PersistenceChecks.py +99 -0
  10. ScriptCollection/LinuxServerSecurityCheck/Checks/ProcessChecks.py +88 -0
  11. ScriptCollection/LinuxServerSecurityCheck/Checks/SSHChecks.py +56 -0
  12. ScriptCollection/LinuxServerSecurityCheck/Checks/__init__.py +0 -0
  13. ScriptCollection/LinuxServerSecurityCheck/LinuxServerSecurityChecker.py +90 -0
  14. ScriptCollection/LinuxServerSecurityCheck/LinuxSystemAccess.py +85 -0
  15. ScriptCollection/LinuxServerSecurityCheck/PackageOwnershipLookup.py +42 -0
  16. ScriptCollection/LinuxServerSecurityCheck/SecurityCheckBase.py +19 -0
  17. ScriptCollection/LinuxServerSecurityCheck/SecurityFinding.py +19 -0
  18. ScriptCollection/LinuxServerSecurityCheck/SuspiciousCommandPatterns.py +46 -0
  19. ScriptCollection/LinuxServerSecurityCheck/SystemAccessBase.py +244 -0
  20. ScriptCollection/LinuxServerSecurityCheck/SystemEntities.py +45 -0
  21. ScriptCollection/LinuxServerSecurityCheck/__init__.py +0 -0
  22. ScriptCollection/ProgramRunnerPopen.py +4 -2
  23. ScriptCollection/ProgramRunnerSudo.py +11 -5
  24. ScriptCollection/ScriptCollectionCore.py +18 -6
  25. ScriptCollection/TFCPS/Android/TFCPS_CodeUnitSpecific_Android.py +109 -5
  26. ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py +1 -1
  27. ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +4 -4
  28. ScriptCollection/TFCPS/SCTaskRunnerServer.py +55 -9
  29. ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +5 -2
  30. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +158 -2
  31. ScriptCollection/TFCPS/TFCPS_RemoteBuild.py +39 -11
  32. ScriptCollection/TFCPS/TFCPS_Tools_General.py +19 -0
  33. {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.74.dist-info}/METADATA +177 -177
  34. {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.74.dist-info}/RECORD +37 -17
  35. {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.74.dist-info}/entry_points.txt +1 -0
  36. {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.74.dist-info}/WHEEL +0 -0
  37. {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.74.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,244 @@
1
+ import ipaddress
2
+ import os
3
+ import posixpath
4
+ import stat
5
+ import sys
6
+ from abc import ABC, abstractmethod
7
+ from collections.abc import Iterator
8
+ from .SystemEntities import UserAccount, ProcessInformation, ProgramResult, SocketInformation
9
+
10
+
11
+ class SystemAccessBase(ABC):
12
+ """Read-only access to the system which is checked.
13
+ All paths are absolute paths of the checked system.
14
+ The abstract methods are the primitives which access the system; everything else is derived from them,
15
+ so that a testcase only has to replace the primitives."""
16
+
17
+ maximal_size_of_text_files: int = 50 * 1024 * 1024
18
+ seconds_per_day: int = 24 * 60 * 60
19
+
20
+ @abstractmethod
21
+ def current_user_is_root(self) -> bool:
22
+ raise NotImplementedError
23
+
24
+ @abstractmethod
25
+ def read_bytes(self, path: str, maximal_amount_of_bytes: int) -> bytes | None:
26
+ """Returns the first bytes of the file or None if the file does not exist (anymore)."""
27
+ raise NotImplementedError
28
+
29
+ @abstractmethod
30
+ def list_folder(self, path: str) -> list[str] | None:
31
+ """Returns the names of the entries of the folder or None if the folder does not exist (anymore)."""
32
+ raise NotImplementedError
33
+
34
+ @abstractmethod
35
+ def get_lstat(self, path: str) -> os.stat_result | None:
36
+ """Returns the stat-information of the path itself (a symlink is not followed) or None if the path does not exist (anymore)."""
37
+ raise NotImplementedError
38
+
39
+ @abstractmethod
40
+ def path_exists(self, path: str) -> bool:
41
+ """Returns whether the path exists. Symlinks are followed."""
42
+ raise NotImplementedError
43
+
44
+ @abstractmethod
45
+ def read_link(self, path: str) -> str | None:
46
+ """Returns the target of the symlink or None if the symlink does not exist (anymore) or has no target (for example /proc/<pid>/exe of a kernel-thread)."""
47
+ raise NotImplementedError
48
+
49
+ @abstractmethod
50
+ def program_is_available(self, program: str) -> bool:
51
+ raise NotImplementedError
52
+
53
+ @abstractmethod
54
+ def run_program(self, program: str, arguments: list[str]) -> ProgramResult:
55
+ """Runs a program which must not change anything on the system."""
56
+ raise NotImplementedError
57
+
58
+ @abstractmethod
59
+ def get_file_attribute_flags(self, path: str) -> int | None:
60
+ """Returns the inode-flags (as shown by lsattr) of a regular file or None if the filesystem does not support them."""
61
+ raise NotImplementedError
62
+
63
+ @abstractmethod
64
+ def get_current_time(self) -> float:
65
+ raise NotImplementedError
66
+
67
+ def read_text(self, path: str) -> str | None:
68
+ content = self.read_bytes(path, SystemAccessBase.maximal_size_of_text_files)
69
+ if content is None:
70
+ return None
71
+ return content.decode("utf-8", errors="replace")
72
+
73
+ def is_regular_file(self, path: str) -> bool:
74
+ file_stat = self.get_lstat(path)
75
+ return file_stat is not None and stat.S_ISREG(file_stat.st_mode)
76
+
77
+ def is_symlink(self, path: str) -> bool:
78
+ file_stat = self.get_lstat(path)
79
+ return file_stat is not None and stat.S_ISLNK(file_stat.st_mode)
80
+
81
+ def is_folder(self, path: str) -> bool:
82
+ file_stat = self.get_lstat(path)
83
+ return file_stat is not None and stat.S_ISDIR(file_stat.st_mode)
84
+
85
+ def is_binary_content(self, content: bytes) -> bool:
86
+ return b"\0" in content[:8192]
87
+
88
+ def list_files_recursively(self, folder: str, excluded_folders: set[str]) -> Iterator[tuple[str, os.stat_result]]:
89
+ """Yields every non-folder-entry (regular files, symlinks, ...) below the folder together with its lstat-information.
90
+ Symlinks to folders are not followed. Folders contained in excluded_folders are not entered."""
91
+ folders_to_visit: list[str] = [folder]
92
+ while len(folders_to_visit) > 0:
93
+ current_folder = folders_to_visit.pop()
94
+ entries = self.list_folder(current_folder)
95
+ if entries is None:
96
+ continue
97
+ for entry in sorted(entries):
98
+ path = posixpath.join(current_folder, entry)
99
+ entry_stat = self.get_lstat(path)
100
+ if entry_stat is None:
101
+ continue
102
+ if stat.S_ISDIR(entry_stat.st_mode):
103
+ if path not in excluded_folders:
104
+ folders_to_visit.append(path)
105
+ else:
106
+ yield path, entry_stat
107
+
108
+ def list_regular_files_recursively(self, folder: str) -> list[str]:
109
+ return [path for path, file_stat in self.list_files_recursively(folder, set()) if stat.S_ISREG(file_stat.st_mode)]
110
+
111
+ def path_is_alias(self, path: str) -> bool:
112
+ """Returns whether the path is reachable only through a symlinked top-level-folder (for example /lib/systemd on a system with merged /usr),
113
+ so that its content is already reachable through another path."""
114
+ parts = [part for part in path.split("/") if part != ""]
115
+ current = ""
116
+ for part in parts[:-1]:
117
+ current = current + "/" + part
118
+ if self.is_symlink(current):
119
+ return True
120
+ return False
121
+
122
+ def get_users(self) -> list[UserAccount]:
123
+ content = self.read_text("/etc/passwd")
124
+ if content is None:
125
+ raise FileNotFoundError("/etc/passwd does not exist.")
126
+ result: list[UserAccount] = []
127
+ for line in content.splitlines():
128
+ if line.strip() == "" or line.startswith("#"):
129
+ continue
130
+ fields = line.split(":")
131
+ if len(fields) < 7:
132
+ continue
133
+ result.append(UserAccount(fields[0], int(fields[2]), int(fields[3]), fields[5], fields[6]))
134
+ return result
135
+
136
+ def get_home_folders(self) -> list[str]:
137
+ result: list[str] = []
138
+ for user in self.get_users():
139
+ if user.home_folder not in result and user.home_folder.startswith("/") and self.is_folder(user.home_folder):
140
+ result.append(user.home_folder)
141
+ return result
142
+
143
+ def get_minimal_uid_of_regular_users(self) -> int:
144
+ content = self.read_text("/etc/login.defs")
145
+ if content is not None:
146
+ for line in content.splitlines():
147
+ fields = line.split()
148
+ if len(fields) == 2 and fields[0] == "UID_MIN":
149
+ return int(fields[1])
150
+ # 1000 is the value which shadow-utils uses when UID_MIN is not defined in /etc/login.defs.
151
+ return 1000
152
+
153
+ def get_process_ids(self) -> list[int]:
154
+ entries = self.list_folder("/proc")
155
+ if entries is None:
156
+ raise FileNotFoundError("/proc does not exist.")
157
+ return sorted(int(entry) for entry in entries if entry.isdigit())
158
+
159
+ def get_process(self, pid: int) -> ProcessInformation | None:
160
+ """Returns None if the process does not exist (anymore)."""
161
+ status = self.read_text(f"/proc/{pid}/status")
162
+ if status is None:
163
+ return None
164
+ name = ""
165
+ uid = -1
166
+ for line in status.splitlines():
167
+ if line.startswith("Name:"):
168
+ name = line[len("Name:"):].strip()
169
+ elif line.startswith("Uid:"):
170
+ uid = int(line.split()[1])
171
+ commandline_bytes = self.read_bytes(f"/proc/{pid}/cmdline", 65536)
172
+ if commandline_bytes is None:
173
+ return None
174
+ commandline = commandline_bytes.replace(b"\0", b" ").decode("utf-8", errors="replace").strip()
175
+ executable = self.read_link(f"/proc/{pid}/exe")
176
+ return ProcessInformation(pid, name, uid, executable, commandline)
177
+
178
+ def get_processes(self) -> list[ProcessInformation]:
179
+ result: list[ProcessInformation] = []
180
+ for pid in self.get_process_ids():
181
+ process = self.get_process(pid)
182
+ if process is not None:
183
+ result.append(process)
184
+ return result
185
+
186
+ def get_processes_by_socket_inode(self) -> dict[int, list[ProcessInformation]]:
187
+ result: dict[int, list[ProcessInformation]] = {}
188
+ for process in self.get_processes():
189
+ file_descriptors = self.list_folder(f"/proc/{process.pid}/fd")
190
+ if file_descriptors is None:
191
+ continue
192
+ for file_descriptor in file_descriptors:
193
+ target = self.read_link(f"/proc/{process.pid}/fd/{file_descriptor}")
194
+ if target is not None and target.startswith("socket:[") and target.endswith("]"):
195
+ inode = int(target[len("socket:["):-1])
196
+ processes = result.setdefault(inode, [])
197
+ if process not in processes:
198
+ processes.append(process)
199
+ return result
200
+
201
+ def get_sockets(self) -> list[SocketInformation]:
202
+ """Returns the tcp- and udp-sockets of all network-namespaces (so also the sockets of containers)."""
203
+ socket_tables: list[str] = []
204
+ known_namespaces: set[str] = set()
205
+ for pid in self.get_process_ids():
206
+ namespace = self.read_link(f"/proc/{pid}/ns/net")
207
+ if namespace is not None and namespace not in known_namespaces:
208
+ known_namespaces.add(namespace)
209
+ socket_tables.append(f"/proc/{pid}/net")
210
+ if len(socket_tables) == 0:
211
+ socket_tables.append("/proc/net")
212
+ result: list[SocketInformation] = []
213
+ known_inodes: set[int] = set()
214
+ for socket_table in socket_tables:
215
+ for protocol in ["tcp", "tcp6", "udp", "udp6"]:
216
+ content = self.read_text(f"{socket_table}/{protocol}")
217
+ if content is None:
218
+ continue
219
+ for socket_information in SystemAccessBase.parse_socket_table(protocol, content):
220
+ if socket_information.inode == 0 or socket_information.inode not in known_inodes:
221
+ known_inodes.add(socket_information.inode)
222
+ result.append(socket_information)
223
+ return result
224
+
225
+ @staticmethod
226
+ def parse_socket_table(protocol: str, content: str) -> list[SocketInformation]:
227
+ result: list[SocketInformation] = []
228
+ for line in content.splitlines()[1:]:
229
+ fields = line.split()
230
+ if len(fields) < 10:
231
+ continue
232
+ local_address, local_port = SystemAccessBase.decode_socket_address(fields[1])
233
+ remote_address, remote_port = SystemAccessBase.decode_socket_address(fields[2])
234
+ result.append(SocketInformation(protocol, local_address, local_port, remote_address, remote_port, fields[3].upper(), int(fields[9])))
235
+ return result
236
+
237
+ @staticmethod
238
+ def decode_socket_address(hex_address: str) -> tuple[str, int]:
239
+ """Decodes an address like "0100007F:0035" from /proc/net/tcp. The address consists of 32-bit-words in host-byte-order."""
240
+ address_part, port_part = hex_address.split(":")
241
+ raw_address = bytes.fromhex(address_part)
242
+ if sys.byteorder == "little":
243
+ raw_address = b"".join(raw_address[index:index + 4][::-1] for index in range(0, len(raw_address), 4))
244
+ return str(ipaddress.ip_address(raw_address)), int(port_part, 16)
@@ -0,0 +1,45 @@
1
+ from dataclasses import dataclass
2
+
3
+
4
+ @dataclass(frozen=True)
5
+ class UserAccount:
6
+ name: str
7
+ uid: int
8
+ gid: int
9
+ home_folder: str
10
+ shell: str
11
+
12
+
13
+ @dataclass(frozen=True)
14
+ class ProcessInformation:
15
+ pid: int
16
+ name: str
17
+ uid: int
18
+ executable: str | None
19
+ """Target of /proc/<pid>/exe as seen from the mount-namespace of the process. None for kernel-threads and zombies."""
20
+ commandline: str
21
+
22
+ def is_kernel_thread_or_zombie(self) -> bool:
23
+ return self.executable is None and self.commandline == ""
24
+
25
+ def describe(self) -> str:
26
+ return f"pid {self.pid} (name \"{self.name}\", uid {self.uid}, executable \"{self.executable}\", commandline \"{self.commandline[:300]}\")"
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class ProgramResult:
31
+ exit_code: int
32
+ stdout: str
33
+ stderr: str
34
+
35
+
36
+ @dataclass(frozen=True)
37
+ class SocketInformation:
38
+ protocol: str
39
+ local_address: str
40
+ local_port: int
41
+ remote_address: str
42
+ remote_port: int
43
+ state: str
44
+ """Hex-state as written in /proc/net/tcp, for example "0A" for listening tcp-sockets."""
45
+ inode: int
File without changes
@@ -26,10 +26,12 @@ class ProgramRunnerPopen(ProgramRunnerBase):
26
26
  def run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, custom_argument: object = None, interactive: bool = False, env_vars: dict = None) -> Popen:
27
27
  arguments_for_process, env = self.__get_process_start_information(program, arguments_as_array, env_vars)
28
28
  try:
29
+ # errors="replace": a program can write bytes which are not valid in the used encoding (for example gpgv writes a name with
30
+ # an umlaut in latin-1). Without replacing them, reading the output would fail at the first such byte and the output would be lost.
29
31
  if interactive:
30
- result = Popen(arguments_for_process, cwd=working_directory, stdout=PIPE, stderr=PIPE, shell=False, text=True, stdin=sys.stdin, env=env) # pylint: disable=consider-using-with
32
+ result = Popen(arguments_for_process, cwd=working_directory, stdout=PIPE, stderr=PIPE, shell=False, text=True, errors="replace", stdin=sys.stdin, env=env) # pylint: disable=consider-using-with
31
33
  else:
32
- result = Popen(arguments_for_process, cwd=working_directory, stdout=PIPE, stderr=PIPE, shell=False, text=True, env=env) # pylint: disable=consider-using-with
34
+ result = Popen(arguments_for_process, cwd=working_directory, stdout=PIPE, stderr=PIPE, shell=False, text=True, errors="replace", env=env) # pylint: disable=consider-using-with
33
35
  except FileNotFoundError as fileNotFoundError:
34
36
  raise FileNotFoundError(f"Starting '{program}' in '{working_directory}' resulted in a FileNotFoundError: '{str(fileNotFoundError)}'")
35
37
  except NotADirectoryError as notADirectoryError:
@@ -1,3 +1,4 @@
1
+ import io
1
2
  from subprocess import Popen
2
3
  from .GeneralUtilities import GeneralUtilities
3
4
  from .ProgramRunnerBase import ProgramRunnerBase
@@ -18,23 +19,25 @@ class SudoPopenReader:
18
19
 
19
20
 
20
21
  class SudoPopen:
22
+ """Represents an already terminated program-run with the interface of a subprocess.Popen which was created with text=True.
23
+ stdout and stderr are text-streams (and not bytes), because ScriptCollectionCore reads them line by line with readline until it gets an empty string."""
21
24
  returncode: int = None
22
25
  stdout_str: str = None
23
26
  stderr_str: str = None
24
27
  pid: int = None
25
- stdout: bytes = None
26
- stderr: bytes = None
28
+ stdout: io.StringIO = None
29
+ stderr: io.StringIO = None
27
30
 
28
31
  def __init__(self, exitcode: int, stdout: str, stderr: str, pid: int):
29
32
  self.returncode: int = exitcode
30
33
  self.stdout_str: str = stdout
31
- self.stdout = str.encode(self.stdout_str)
34
+ self.stdout = io.StringIO(self.stdout_str)
32
35
  self.stderr_str: str = stderr
33
- self.stderr = str.encode(self.stderr_str)
36
+ self.stderr = io.StringIO(self.stderr_str)
34
37
  self.pid = pid
35
38
 
36
39
  def communicate(self):
37
- return (self.stdout, self.stderr)
40
+ return (self.stdout_str, self.stderr_str)
38
41
 
39
42
  def wait(self):
40
43
  return self.returncode
@@ -42,6 +45,9 @@ class SudoPopen:
42
45
  def poll(self) -> object:
43
46
  return self.returncode
44
47
 
48
+ def kill(self) -> None:
49
+ pass # the program-run is already terminated when this object is created, so there is nothing to kill.
50
+
45
51
  def __enter__(self):
46
52
  return self
47
53
 
@@ -41,7 +41,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
41
41
  from .ProgramRunnerPopen import ProgramRunnerPopen
42
42
  from .SCLog import SCLog, LogLevel
43
43
 
44
- version = "4.4.72"
44
+ version = "4.4.74"
45
45
  __version__ = version
46
46
 
47
47
  class VSCodeWorkspaceShellTask:
@@ -1521,13 +1521,13 @@ class ScriptCollectionCore:
1521
1521
  arguments = arguments+["--excludedirectories"]
1522
1522
  if printonlynamewithoutpath:
1523
1523
  arguments = arguments+["--printonlynamewithoutpath"]
1524
- exit_code, stdout, stderr, _ = self.run_program_argsasarray("sclistfoldercontent", arguments)
1524
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("sclistfoldercontent", arguments, throw_exception_if_exitcode_is_not_zero=False)
1525
1525
  if exit_code == 0:
1526
1526
  for line in stdout.split("\n"):
1527
1527
  normalized_line = line.replace("\r", "")
1528
1528
  result.append(normalized_line)
1529
1529
  else:
1530
- raise ValueError(f"Fatal error occurred while checking whether file '{path}' exists. StdErr: '{stderr}'")
1530
+ raise ValueError(f"Fatal error occurred while listing the content of folder '{path}'. Exitcode: {exit_code}, StdErr: '{stderr}'")
1531
1531
  result = [item for item in result if GeneralUtilities.string_has_nonwhitespace_content(item)]
1532
1532
  return result
1533
1533
 
@@ -2626,9 +2626,21 @@ resolving a name requires fontconfig, which does not exist on every system ffmpe
2626
2626
  ScriptCollectionCore.__process_stdout_lines(q_stdout, stdout_result, print_live_output, log)
2627
2627
  ScriptCollectionCore.__process_stderr_lines(q_stderr, stderr_result, print_live_output, print_errors_as_information, log)
2628
2628
 
2629
- if timed_out:
2630
- raise TimeoutError(f"The process with process-id {p_id} did not finish within the configured timeout of {timeoutInSeconds} second(s) and was killed.")
2631
- return (stdout_result, stderr_result)
2629
+ # Leaving the with-block waits until both reader-threads are terminated, so their result is final here.
2630
+ if timed_out:
2631
+ raise TimeoutError(f"The process with process-id {p_id} did not finish within the configured timeout of {timeoutInSeconds} second(s) and was killed.")
2632
+ ScriptCollectionCore.__raise_if_reading_failed(readers, p_id)
2633
+ return (stdout_result, stderr_result)
2634
+
2635
+ @staticmethod
2636
+ def __raise_if_reading_failed(readers: list, p_id: int) -> None:
2637
+ """Raises the error of a reader-thread which failed.
2638
+ Without this check such an error (for example a program-runner whose popen-object does not provide readable text-streams)
2639
+ would be swallowed by the thread-pool, so that the program-run would look like a successful run without any output."""
2640
+ for reader in readers:
2641
+ reader_error: BaseException = reader.exception()
2642
+ if reader_error is not None:
2643
+ raise ValueError(f"Reading the output of the process with process-id {p_id} failed.") from reader_error
2632
2644
 
2633
2645
  @GeneralUtilities.check_arguments
2634
2646
  def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = None, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False, env_vars: dict = None) -> tuple[int, str, str, int]:
@@ -1,5 +1,8 @@
1
1
  import os
2
2
  import re
3
+ import base64
4
+ import binascii
5
+ import zipfile
3
6
  import shlex
4
7
  import shutil
5
8
  import requests
@@ -32,7 +35,29 @@ class TFCPS_CodeUnitSpecific_Android_Functions(TFCPS_CodeUnitSpecific_Base):
32
35
  - Only if the codeunit has visual-regression-tests (see run_visual_regression_tests): it applies the plugin
33
36
  "io.github.takahirom.roborazzi" and configures "roborazzi { outputDir = file('../Other/Resources/VisualRegressionBaselines');
34
37
  compare { outputDir = layout.buildDirectory.dir('outputs/roborazzi-comparison') } }" (see
35
- get_visual_regression_baselines_folder and get_visual_regression_comparison_folder_relative_to_build_folder)."""
38
+ get_visual_regression_baselines_folder and get_visual_regression_comparison_folder_relative_to_build_folder).
39
+ - The release-build-type is signed with the upload-key if the environment-variables "SC_ANDROID_SIGNING_STORE_FILE",
40
+ "SC_ANDROID_SIGNING_STORE_PASSWORD", "SC_ANDROID_SIGNING_KEY_ALIAS" and "SC_ANDROID_SIGNING_KEY_PASSWORD" are set,
41
+ and with the debug-key otherwise (see build), for example:
42
+ "signingConfigs { release { if (System.getenv('SC_ANDROID_SIGNING_STORE_FILE') != null) { storeFile =
43
+ file(System.getenv('SC_ANDROID_SIGNING_STORE_FILE')); storePassword = System.getenv('SC_ANDROID_SIGNING_STORE_PASSWORD');
44
+ keyAlias = System.getenv('SC_ANDROID_SIGNING_KEY_ALIAS'); keyPassword = System.getenv('SC_ANDROID_SIGNING_KEY_PASSWORD') } } }"
45
+ and "release { signingConfig = System.getenv('SC_ANDROID_SIGNING_STORE_FILE') != null ? signingConfigs.release :
46
+ signingConfigs.debug }"."""
47
+
48
+ # The environment-variables (of the machine which runs the build, for example the build-pipeline) which contain the
49
+ # upload-key of a productive build, mapped to the environment-variables of the gradle-run on the runner which
50
+ # receive them (see build). The keystore is given as the base64-encoding of the keystore-file, because an
51
+ # environment-variable can not contain a file.
52
+ __signing_environment_variables: dict[str, str] = {
53
+ "Android_Signing_Keystore": "SC_ANDROID_SIGNING_KEYSTORE_BASE64",
54
+ "Android_Signing_StorePassword": "SC_ANDROID_SIGNING_STORE_PASSWORD",
55
+ "Android_Signing_KeyAlias": "SC_ANDROID_SIGNING_KEY_ALIAS",
56
+ "Android_Signing_KeyPassword": "SC_ANDROID_SIGNING_KEY_PASSWORD",
57
+ }
58
+
59
+ # The name the subject of the debug-certificate of the android-sdk contains ("CN=Android Debug,O=Android,C=US").
60
+ __subject_of_the_debug_certificate: bytes = b"Android Debug"
36
61
 
37
62
  # The repositories in which a dependency or a gradle-plugin of an android-app is published. Gradle itself searches
38
63
  # all of them (they are the repositories a project declares by "google()", "mavenCentral()" and
@@ -94,16 +119,38 @@ class TFCPS_CodeUnitSpecific_Android_Functions(TFCPS_CodeUnitSpecific_Base):
94
119
  def build(self) -> None:
95
120
  """Builds the app-bundle (which is what is uploaded to a store) and the apk (which can be installed directly) of the
96
121
  release-build-type and generates the bill-of-materials. All of them are generated by one gradle-run, so the runner
97
- has to be used only once for them."""
122
+ has to be used only once for them.
123
+
124
+ A productive build (target-environment-type "Productive") is signed with the upload-key, so its app-bundle can be
125
+ uploaded to Google Play. The upload-key is taken from the environment-variables "Android_Signing_Keystore" (the
126
+ base64-encoding of the keystore-file), "Android_Signing_StorePassword", "Android_Signing_KeyAlias" and
127
+ "Android_Signing_KeyPassword", which are required for a productive build. They are given to the gradle-run on the
128
+ runner as environment-variables only (see TFCPS_RemoteBuild.run_program_on_runner); on the runner the keystore
129
+ is written into a temporary file outside of the workspace which is removed when gradle ends. Afterwards it is
130
+ verified that the app-bundle is signed, but not with the debug-key.
131
+ Every other build is signed with the debug-key of the runner, so a developer can build and install the app
132
+ without the upload-key."""
98
133
  codeunit_name = self.get_codeunit_name()
99
134
  codeunit_version: str = self.tfcps_Tools_General.get_version_of_codeunit(self.get_codeunit_file())
100
135
  outputs_folder = os.path.join(self.get_build_folder(), "outputs")
101
136
  GeneralUtilities.ensure_directory_does_not_exist(outputs_folder)
102
- self.__run_gradle_on_runner(["bundleRelease", "assembleRelease", "cyclonedxBom"], outputs_folder)
137
+ gradle_tasks = ["bundleRelease", "assembleRelease", "cyclonedxBom"]
138
+ is_signed_with_upload_key = self.get_target_environment_type() == "Productive"
139
+ if is_signed_with_upload_key:
140
+ self._protected_sc.log.log("Build the release-build signed with the upload-key...")
141
+ signing_environment_variables = TFCPS_CodeUnitSpecific_Android_Functions.get_signing_environment_variables_for_runner(dict(os.environ))
142
+ script = TFCPS_CodeUnitSpecific_Android_Functions.get_script_which_signs_with_upload_key(self.__get_gradle_arguments(gradle_tasks))
143
+ self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", ["-c", script], self.get_gradle_project_folder(), outputs_folder, self.__get_folders_which_are_not_transferred(), signing_environment_variables)
144
+ else:
145
+ self._protected_sc.log.log(f'Build the release-build signed with the debug-key of the runner (only a build with target-environment-type "Productive" is signed with the upload-key, this one has "{self.get_target_environment_type()}")...')
146
+ self.__run_gradle_on_runner(gradle_tasks, outputs_folder)
103
147
 
104
148
  aab_folder = os.path.join(self.get_artifacts_folder(), "BuildResult_AAB")
105
149
  GeneralUtilities.ensure_folder_exists_and_is_empty(aab_folder)
106
150
  aab_file_original = self._protected_sc.find_file_by_extension(os.path.join(outputs_folder, "bundle", "release"), "aab")
151
+ if is_signed_with_upload_key:
152
+ GeneralUtilities.assert_condition(TFCPS_CodeUnitSpecific_Android_Functions.is_signed(aab_file_original), f'The app-bundle "{aab_file_original}" is not signed. The release-build-type of "{self.get_build_file()}" has to use the upload-key which is given by the environment-variables "SC_ANDROID_SIGNING_*" (see the docstring of TFCPS_CodeUnitSpecific_Android_Functions).')
153
+ GeneralUtilities.assert_condition(not TFCPS_CodeUnitSpecific_Android_Functions.is_signed_with_debug_key(aab_file_original), f'The app-bundle "{aab_file_original}" is signed with the debug-key, so Google Play would reject it. The release-build-type of "{self.get_build_file()}" has to use the upload-key which is given by the environment-variables "SC_ANDROID_SIGNING_*" (see the docstring of TFCPS_CodeUnitSpecific_Android_Functions).')
107
154
  shutil.copyfile(aab_file_original, os.path.join(aab_folder, f"{codeunit_name}.aab"))
108
155
 
109
156
  apk_folder = os.path.join(self.get_artifacts_folder(), "BuildResult_APK")
@@ -214,7 +261,7 @@ class TFCPS_CodeUnitSpecific_Android_Functions(TFCPS_CodeUnitSpecific_Base):
214
261
  def __run_gradle_on_runner(self, tasks: list[str], result_folder: str) -> None:
215
262
  """Runs the given gradle-tasks on an android-task-runner and writes the content of result_folder (which must be a
216
263
  folder inside this codeunit) back into this codeunit. A failure of gradle raises an exception."""
217
- self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", self.__get_gradle_arguments(tasks), self.get_gradle_project_folder(), result_folder, self.__get_folders_which_are_not_transferred())
264
+ self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", self.__get_gradle_arguments(tasks), self.get_gradle_project_folder(), result_folder, self.__get_folders_which_are_not_transferred(), {})
218
265
 
219
266
  @GeneralUtilities.check_arguments
220
267
  def __run_gradle_on_runner_and_get_exit_code(self, tasks: list[str], result_folder: str) -> int:
@@ -225,11 +272,68 @@ class TFCPS_CodeUnitSpecific_Android_Functions(TFCPS_CodeUnitSpecific_Base):
225
272
  succeeds."""
226
273
  result_folder_relative_to_project = os.path.relpath(result_folder, self.get_gradle_project_folder()).replace("\\", "/")
227
274
  script = TFCPS_CodeUnitSpecific_Android_Functions.get_script_which_preserves_exit_code(self.__get_gradle_arguments(tasks), result_folder_relative_to_project, TFCPS_CodeUnitSpecific_Android_Functions.__exit_code_file_name)
228
- self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", ["-c", script], self.get_gradle_project_folder(), result_folder, self.__get_folders_which_are_not_transferred())
275
+ self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "bash", ["-c", script], self.get_gradle_project_folder(), result_folder, self.__get_folders_which_are_not_transferred(), {})
229
276
  local_exit_code_file = os.path.join(result_folder, TFCPS_CodeUnitSpecific_Android_Functions.__exit_code_file_name)
230
277
  GeneralUtilities.assert_file_exists(local_exit_code_file)
231
278
  return int(GeneralUtilities.read_text_from_file(local_exit_code_file).strip())
232
279
 
280
+ @staticmethod
281
+ @GeneralUtilities.check_arguments
282
+ def get_signing_environment_variables_for_runner(environment: dict[str, str]) -> dict[str, str]:
283
+ """Returns the environment-variables which give the upload-key to the gradle-run on the runner, taken from the
284
+ given environment-variables of this machine (see build). The names are compared case-insensitively, because
285
+ Windows exposes the names of environment-variables upper-cased via os.environ."""
286
+ environment_upper = {name.upper(): value for name, value in environment.items()}
287
+ missing_variables = [name for name in TFCPS_CodeUnitSpecific_Android_Functions.__signing_environment_variables if len((environment_upper.get(name.upper()) or GeneralUtilities.empty_string).strip()) == 0]
288
+ GeneralUtilities.assert_condition(len(missing_variables) == 0, f'A productive build of an android-app is signed with the upload-key, which requires the environment-variable(s) {", ".join(missing_variables)}. "Android_Signing_Keystore" contains the base64-encoding of the keystore-file.')
289
+ result: dict[str, str] = {}
290
+ for name, name_on_runner in TFCPS_CodeUnitSpecific_Android_Functions.__signing_environment_variables.items():
291
+ result[name_on_runner] = environment_upper[name.upper()]
292
+ # A base64-encoding is often wrapped into several lines (for example by the base64-tool), which is not part of it.
293
+ keystore_base64 = "".join(result["SC_ANDROID_SIGNING_KEYSTORE_BASE64"].split())
294
+ try:
295
+ base64.b64decode(keystore_base64, validate=True)
296
+ except binascii.Error as exception:
297
+ raise ValueError('The environment-variable "Android_Signing_Keystore" does not contain a valid base64-encoding of the keystore-file.') from exception
298
+ result["SC_ANDROID_SIGNING_KEYSTORE_BASE64"] = keystore_base64
299
+ return result
300
+
301
+ @staticmethod
302
+ @GeneralUtilities.check_arguments
303
+ def get_script_which_signs_with_upload_key(gradle_arguments: list[str]) -> str:
304
+ """Returns the bash-script which runs gradle on the runner with the upload-key: it writes the keystore (which it
305
+ gets base64-encoded in an environment-variable) into a temporary file outside of the workspace of the job, gives
306
+ that file to gradle via "SC_ANDROID_SIGNING_STORE_FILE" and removes it when the script ends, regardless of how it
307
+ ends."""
308
+ return "set -e; " \
309
+ + 'keystore_file="$(mktemp)"; ' \
310
+ + """trap 'rm -f "$keystore_file"' EXIT; """ \
311
+ + """printf '%s' "$SC_ANDROID_SIGNING_KEYSTORE_BASE64" | base64 -d > "$keystore_file"; """ \
312
+ + "unset SC_ANDROID_SIGNING_KEYSTORE_BASE64; " \
313
+ + 'export SC_ANDROID_SIGNING_STORE_FILE="$keystore_file"; ' \
314
+ + "bash " + shlex.join(gradle_arguments)
315
+
316
+ @staticmethod
317
+ @GeneralUtilities.check_arguments
318
+ def __get_signature_blocks(signed_file: str) -> list[bytes]:
319
+ """Returns the signature-blocks of a jar-signed file (an app-bundle is jar-signed): the files "META-INF/*.RSA",
320
+ "META-INF/*.DSA" and "META-INF/*.EC", which contain the certificate of the signer."""
321
+ with zipfile.ZipFile(signed_file) as archive:
322
+ return [archive.read(name) for name in archive.namelist() if re.fullmatch(r"META-INF/[^/]+\.(RSA|DSA|EC)", name, re.IGNORECASE)]
323
+
324
+ @staticmethod
325
+ @GeneralUtilities.check_arguments
326
+ def is_signed(signed_file: str) -> bool:
327
+ """Whether the given app-bundle is signed."""
328
+ return len(TFCPS_CodeUnitSpecific_Android_Functions.__get_signature_blocks(signed_file)) > 0
329
+
330
+ @staticmethod
331
+ @GeneralUtilities.check_arguments
332
+ def is_signed_with_debug_key(signed_file: str) -> bool:
333
+ """Whether the given app-bundle is signed with the debug-key of an android-sdk. The certificate of that key is
334
+ always issued to "CN=Android Debug,O=Android,C=US", whose name is contained in the signature-block as text."""
335
+ return any(TFCPS_CodeUnitSpecific_Android_Functions.__subject_of_the_debug_certificate in block for block in TFCPS_CodeUnitSpecific_Android_Functions.__get_signature_blocks(signed_file))
336
+
233
337
  @staticmethod
234
338
  @GeneralUtilities.check_arguments
235
339
  def get_script_which_preserves_exit_code(gradle_arguments: list[str], result_folder_relative_to_project: str, exit_code_file_name: str) -> str:
@@ -169,7 +169,7 @@ class TFCPS_CodeUnitSpecific_CPP_Functions(TFCPS_CodeUnitSpecific_Base):
169
169
  #run in - which is also the folder __collect_output_files searches afterwards.
170
170
  #Nothing of a C/C++-codeunit is left out of the archive the runner gets: where a build-command puts its
171
171
  #output is defined by that command, so there is no folder which is known to contain state of this machine.
172
- self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "/bin/sh", ["-c", " && ".join(self.macos_build_commands)], source_folder, source_folder, [])
172
+ self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "/bin/sh", ["-c", " && ".join(self.macos_build_commands)], source_folder, source_folder, [], {})
173
173
  self.__collect_output_files(source_folder, self.macos_output_file_patterns, "BuildResult_MacOS")
174
174
 
175
175
  @GeneralUtilities.check_arguments
@@ -112,7 +112,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
112
112
  "environment as a runner-built one.", LogLevel.Warning)
113
113
  self._protected_sc.run_with_epew("flutter", "build windows", src_folder)
114
114
  else:
115
- self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder, windows_release_folder, folders_which_are_not_transferred)
115
+ self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder, windows_release_folder, folders_which_are_not_transferred, {})
116
116
  windows_folder = os.path.join(artifacts_folder, "BuildResult_Windows")
117
117
  GeneralUtilities.ensure_directory_does_not_exist(windows_folder)
118
118
  GeneralUtilities.ensure_directory_exists(windows_folder)
@@ -134,7 +134,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
134
134
  # builds), analogous to how ios-builds always run on the iOS-task-runner. See SCTaskRunnerMacOS and
135
135
  # the remote-build-article in the reference.
136
136
  macos_release_folder = os.path.join(src_folder, "build/macos/Build/Products/Release")
137
- self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder, macos_release_folder, folders_which_are_not_transferred)
137
+ self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder, macos_release_folder, folders_which_are_not_transferred, {})
138
138
  macos_folder = os.path.join(artifacts_folder, "BuildResult_MacOS")
139
139
  GeneralUtilities.ensure_directory_does_not_exist(macos_folder)
140
140
  GeneralUtilities.ensure_directory_exists(macos_folder)
@@ -143,7 +143,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
143
143
  # iOS-builds must run on macOS and therefore always run on the dedicated iOS-task-runner (uniform
144
144
  # builds). See SCTaskRunnerIOS and the remote-build-article in the reference.
145
145
  ios_release_folder = os.path.join(src_folder, "build/ios/iphoneos")
146
- self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder, ios_release_folder, folders_which_are_not_transferred)
146
+ self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder, ios_release_folder, folders_which_are_not_transferred, {})
147
147
  ios_folder = os.path.join(artifacts_folder, "BuildResult_IOS")
148
148
  GeneralUtilities.ensure_directory_does_not_exist(ios_folder)
149
149
  GeneralUtilities.ensure_directory_exists(ios_folder)
@@ -153,7 +153,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
153
153
  # remote-build-article in the reference), analogous to how ios-builds always run on a macOS-task-runner:
154
154
  # the Android-SDK/NDK-toolchain no longer lives in SCBuilder itself, it moved into SCTaskRunnerAndroid.
155
155
  aab_release_folder = os.path.join(src_folder, "build/app/outputs/bundle/release")
156
- self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder, aab_release_folder, folders_which_are_not_transferred)
156
+ self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder, aab_release_folder, folders_which_are_not_transferred, {})
157
157
  # The app-bundle which came back from the runner is published as the build-result of this codeunit,
158
158
  # analogous to the targets above: it is the artifact of an android-build, and every codeunit has to have
159
159
  # an artifact whose name matches "BuildResult_.+" (see TFCPS_CodeUnit_BuildCodeUnit.build_codeunit).