ScriptCollection 4.4.71__py3-none-any.whl → 4.4.73__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. ScriptCollection/Executables.py +22 -3
  2. ScriptCollection/GeneralUtilities.py +42 -0
  3. ScriptCollection/LinuxServerSecurityCheck/Checks/AccountChecks.py +71 -0
  4. ScriptCollection/LinuxServerSecurityCheck/Checks/ContainerChecks.py +34 -0
  5. ScriptCollection/LinuxServerSecurityCheck/Checks/FileSystemChecks.py +141 -0
  6. ScriptCollection/LinuxServerSecurityCheck/Checks/KernelChecks.py +49 -0
  7. ScriptCollection/LinuxServerSecurityCheck/Checks/LogChecks.py +39 -0
  8. ScriptCollection/LinuxServerSecurityCheck/Checks/NetworkChecks.py +67 -0
  9. ScriptCollection/LinuxServerSecurityCheck/Checks/PackageIntegrityChecks.py +62 -0
  10. ScriptCollection/LinuxServerSecurityCheck/Checks/PersistenceChecks.py +99 -0
  11. ScriptCollection/LinuxServerSecurityCheck/Checks/ProcessChecks.py +88 -0
  12. ScriptCollection/LinuxServerSecurityCheck/Checks/SSHChecks.py +56 -0
  13. ScriptCollection/LinuxServerSecurityCheck/Checks/__init__.py +0 -0
  14. ScriptCollection/LinuxServerSecurityCheck/LinuxServerSecurityChecker.py +90 -0
  15. ScriptCollection/LinuxServerSecurityCheck/LinuxSystemAccess.py +85 -0
  16. ScriptCollection/LinuxServerSecurityCheck/PackageOwnershipLookup.py +42 -0
  17. ScriptCollection/LinuxServerSecurityCheck/SecurityCheckBase.py +19 -0
  18. ScriptCollection/LinuxServerSecurityCheck/SecurityFinding.py +19 -0
  19. ScriptCollection/LinuxServerSecurityCheck/SuspiciousCommandPatterns.py +46 -0
  20. ScriptCollection/LinuxServerSecurityCheck/SystemAccessBase.py +244 -0
  21. ScriptCollection/LinuxServerSecurityCheck/SystemEntities.py +45 -0
  22. ScriptCollection/LinuxServerSecurityCheck/__init__.py +0 -0
  23. ScriptCollection/OCIImages/AbstractImageHandler.py +14 -26
  24. ScriptCollection/OCIImages/AbstractTagPatternImageHandler.py +46 -0
  25. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py +5 -14
  26. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py +5 -14
  27. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGeneric.py +6 -15
  28. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGenericV.py +6 -15
  29. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabCE.py +7 -8
  30. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabEE.py +7 -8
  31. ScriptCollection/OCIImages/OCIImageManager.py +73 -68
  32. ScriptCollection/OCIImages/OCIRegistryClient.py +117 -0
  33. ScriptCollection/ScriptCollectionCore.py +9 -4
  34. ScriptCollection/TFCPS/Android/TFCPS_CodeUnitSpecific_Android.py +479 -0
  35. ScriptCollection/TFCPS/Android/__init__.py +0 -0
  36. ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py +6 -5
  37. ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +38 -4
  38. ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +4 -4
  39. ScriptCollection/TFCPS/Maven/TFCPS_CodeUnitSpecific_Maven.py +1 -0
  40. ScriptCollection/TFCPS/Rust/CargoLockFile.py +151 -0
  41. ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py +5 -0
  42. ScriptCollection/TFCPS/SCTaskRunnerServer.py +55 -9
  43. ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +6 -2
  44. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +14 -0
  45. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +19 -0
  46. ScriptCollection/TFCPS/TFCPS_RemoteBuild.py +39 -11
  47. ScriptCollection/TFCPS/TFCPS_Tools_General.py +34 -2
  48. {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.73.dist-info}/METADATA +3 -3
  49. {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.73.dist-info}/RECORD +52 -27
  50. {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.73.dist-info}/entry_points.txt +1 -0
  51. {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.73.dist-info}/WHEEL +0 -0
  52. {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.73.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,151 @@
1
+ import os
2
+ import re
3
+ import tomllib
4
+ from ...GeneralUtilities import GeneralUtilities
5
+
6
+
7
+ class CargoLockFile:
8
+ """Synchronizes the versions of the local packages in a "Cargo.lock"-file with the versions which their
9
+ "Cargo.toml"-files declare.
10
+
11
+ A local package is a package which is not resolved from a registry or a git-repository: the cargo-project itself
12
+ and every path-dependency (directly or transitively). Cargo writes such a package into the lock-file without a
13
+ "source"-entry, together with the version its cargo-file declared at the time the lock-file was written. When the
14
+ version in a cargo-file changes (which happens on every new version of a codeunit), the lock-file is outdated and
15
+ a build with "--locked" fails. Cargo can not update the lock-file without being allowed to resolve the dependencies
16
+ again, so the versions of the local packages are adjusted textually here: everything else in the lock-file stays
17
+ exactly as it is, no network-access is required and running it again does not change anything anymore."""
18
+
19
+ __dependency_sections: tuple = ("dependencies", "dev-dependencies", "build-dependencies")
20
+ # Cargo uses this version for a package whose cargo-file does not declare a version
21
+ # (see https://doc.rust-lang.org/cargo/reference/manifest.html#the-version-field).
22
+ __version_of_package_without_declared_version: str = "0.0.0"
23
+
24
+ @staticmethod
25
+ @GeneralUtilities.check_arguments
26
+ def synchronize_versions_of_local_packages(cargo_file: str) -> None:
27
+ """Sets the version of every local package in the lock-file next to the given cargo-file to the version
28
+ which the cargo-file of that package declares. A cargo-project without lock-file (for example a library,
29
+ which usually does not commit its lock-file) is left as it is."""
30
+ lock_file = os.path.join(os.path.dirname(cargo_file), "Cargo.lock")
31
+ if os.path.isfile(lock_file):
32
+ CargoLockFile.set_versions_of_local_packages(lock_file, CargoLockFile.get_versions_of_local_packages(cargo_file))
33
+
34
+ @staticmethod
35
+ @GeneralUtilities.check_arguments
36
+ def get_versions_of_local_packages(cargo_file: str) -> dict[str, str]:
37
+ """Returns the names and the versions of the package of the given cargo-file and of all of its (also
38
+ transitive) path-dependencies, read from their cargo-files."""
39
+ result: dict[str, str] = {}
40
+ CargoLockFile.__add_versions_of_local_packages(os.path.abspath(cargo_file), result, set())
41
+ return result
42
+
43
+ @staticmethod
44
+ @GeneralUtilities.check_arguments
45
+ def __add_versions_of_local_packages(cargo_file: str, result: dict[str, str], visited_cargo_files: set[str]) -> None:
46
+ if cargo_file in visited_cargo_files:
47
+ return
48
+ visited_cargo_files.add(cargo_file)
49
+ GeneralUtilities.assert_file_exists(cargo_file, f'The cargo-file "{cargo_file}" does not exist. If it belongs to a dependent codeunit, then this codeunit must be built before.')
50
+ content: dict = tomllib.loads(GeneralUtilities.read_text_from_file(cargo_file))
51
+ package: dict = content.get("package")
52
+ GeneralUtilities.assert_condition(package is not None, f'The cargo-file "{cargo_file}" does not contain a package-section.')
53
+ version = package.get("version", CargoLockFile.__version_of_package_without_declared_version)
54
+ GeneralUtilities.assert_condition(isinstance(version, str), f'The version of the package in "{cargo_file}" is not declared as plain version-string, which is not supported.')
55
+ result[package["name"]] = version
56
+ cargo_project_folder = os.path.dirname(cargo_file)
57
+ for relative_path in CargoLockFile.__get_paths_of_path_dependencies(content):
58
+ dependency_cargo_file = os.path.abspath(os.path.join(cargo_project_folder, relative_path, "Cargo.toml"))
59
+ CargoLockFile.__add_versions_of_local_packages(dependency_cargo_file, result, visited_cargo_files)
60
+
61
+ @staticmethod
62
+ @GeneralUtilities.check_arguments
63
+ def __get_paths_of_path_dependencies(cargo_file_content: dict) -> list[str]:
64
+ # Dependencies can be declared in the top-level-sections and in the target-specific sections
65
+ # (for example "[target.'cfg(windows)'.dependencies]").
66
+ dependency_tables: list[dict] = [cargo_file_content.get(section, {}) for section in CargoLockFile.__dependency_sections]
67
+ for target_content in cargo_file_content.get("target", {}).values():
68
+ dependency_tables.extend(target_content.get(section, {}) for section in CargoLockFile.__dependency_sections)
69
+ result: list[str] = []
70
+ for dependency_table in dependency_tables:
71
+ for declaration in dependency_table.values():
72
+ if isinstance(declaration, dict) and "path" in declaration:
73
+ result.append(declaration["path"])
74
+ return result
75
+
76
+ @staticmethod
77
+ @GeneralUtilities.check_arguments
78
+ def set_versions_of_local_packages(lock_file: str, versions_of_local_packages: dict[str, str]) -> None:
79
+ """Sets the version of every package of the lock-file which has no source and whose name is contained in
80
+ the given dictionary to the version stated there. A package of the lock-file which is not contained in the
81
+ dictionary is not changed. The line-endings and all other lines of the lock-file are preserved."""
82
+ lines: list[str] = GeneralUtilities.read_text_from_file(lock_file).splitlines(keepends=True)
83
+ packages = CargoLockFile.__parse_packages(lines)
84
+ changed_references: dict[str, str] = {}
85
+ for package in packages:
86
+ name: str = package["name"]
87
+ if package["has_source"] or name not in versions_of_local_packages or package["version_line_index"] is None:
88
+ continue
89
+ old_version: str = package["version"]
90
+ new_version: str = versions_of_local_packages[name]
91
+ if old_version != new_version:
92
+ line_index: int = package["version_line_index"]
93
+ lines[line_index] = CargoLockFile.__replace_quoted_value(lines[line_index], old_version, new_version)
94
+ # A dependency is referenced as "<name> <version>" instead of "<name>" when the lock-file contains
95
+ # several packages with this name. Such a reference must be changed together with the version. When a
96
+ # package from a registry has the same name and the same old version, then the reference can not be
97
+ # assigned unambiguously to the local package, so it is not changed.
98
+ if not any(other["has_source"] and other["name"] == name and other["version"] == old_version for other in packages):
99
+ changed_references[f"{name} {old_version}"] = f"{name} {new_version}"
100
+ if len(changed_references) > 0:
101
+ for package in packages:
102
+ for line_index in range(package["start_line_index"], package["end_line_index"]):
103
+ lines[line_index] = CargoLockFile.__replace_dependency_reference(lines[line_index], changed_references)
104
+ GeneralUtilities.write_text_to_file(lock_file, "".join(lines))
105
+
106
+ @staticmethod
107
+ @GeneralUtilities.check_arguments
108
+ def __parse_packages(lines: list[str]) -> list[dict]:
109
+ """Returns the "[[package]]"-entries of the lines of a lock-file with their name, their version, the index of
110
+ the line which contains the version, the information whether they have a source and the range of their
111
+ lines (the end-index is exclusive)."""
112
+ result: list[dict] = []
113
+ current_package: dict = None
114
+ for line_index, line in enumerate(lines):
115
+ stripped_line = line.strip()
116
+ if stripped_line.startswith("["):
117
+ if current_package is not None:
118
+ current_package["end_line_index"] = line_index
119
+ result.append(current_package)
120
+ current_package = None
121
+ if stripped_line == "[[package]]":
122
+ current_package = {"name": None, "version": None, "version_line_index": None, "has_source": False, "start_line_index": line_index}
123
+ continue
124
+ if current_package is None:
125
+ continue
126
+ match = re.match(r'^(name|version|source)\s*=\s*"([^"]*)"$', stripped_line)
127
+ if match is not None:
128
+ key = match.group(1)
129
+ if key == "source":
130
+ current_package["has_source"] = True
131
+ else:
132
+ current_package[key] = match.group(2)
133
+ if key == "version":
134
+ current_package["version_line_index"] = line_index
135
+ if current_package is not None:
136
+ current_package["end_line_index"] = len(lines)
137
+ result.append(current_package)
138
+ return result
139
+
140
+ @staticmethod
141
+ @GeneralUtilities.check_arguments
142
+ def __replace_quoted_value(line: str, old_value: str, new_value: str) -> str:
143
+ return line.replace(f'"{old_value}"', f'"{new_value}"', 1)
144
+
145
+ @staticmethod
146
+ @GeneralUtilities.check_arguments
147
+ def __replace_dependency_reference(line: str, changed_references: dict[str, str]) -> str:
148
+ match = re.match(r'^(\s*)"([^"]+)"(.*)$', line, re.DOTALL)
149
+ if match is not None and match.group(2) in changed_references:
150
+ return f'{match.group(1)}"{changed_references[match.group(2)]}"{match.group(3)}'
151
+ return line
@@ -6,6 +6,7 @@ from lxml import etree
6
6
  from ...GeneralUtilities import GeneralUtilities, Dependency
7
7
  from ...SCLog import LogLevel
8
8
  from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base, TFCPS_CodeUnitSpecific_Base_CLI
9
+ from .CargoLockFile import CargoLockFile
9
10
 
10
11
  class TFCPS_CodeUnitSpecific_Rust_Functions(TFCPS_CodeUnitSpecific_Base):
11
12
  """Implements the tasks of a codeunit which contains a cargo-project.
@@ -65,6 +66,7 @@ class TFCPS_CodeUnitSpecific_Rust_Functions(TFCPS_CodeUnitSpecific_Base):
65
66
 
66
67
  @GeneralUtilities.check_arguments
67
68
  def linting(self) -> None:
69
+ self._protected_sc.normalize_invisible_characters_of_files_in_folder(self.get_codeunit_folder(), ["rs"])
68
70
  # "--all-targets" includes the testcases in the check, and "-D warnings" makes clippy exit with a non-zero
69
71
  # exit-code for a warning as well, because a warning is a linting-issue too.
70
72
  self.__run_cargo("clippy --all-targets --all-features -- -D warnings")
@@ -73,6 +75,9 @@ class TFCPS_CodeUnitSpecific_Rust_Functions(TFCPS_CodeUnitSpecific_Base):
73
75
  def do_common_tasks(self, current_codeunit_version: str) -> None:
74
76
  self.do_common_tasks_base(current_codeunit_version)
75
77
  self.__set_version_in_cargo_file(self.get_version_of_project())
78
+ # The lock-file contains the versions of the local packages (this project and its path-dependencies, for
79
+ # example the copies of dependent codeunits), so it must be adjusted to the versions of their cargo-files.
80
+ CargoLockFile.synchronize_versions_of_local_packages(self.get_cargo_file())
76
81
 
77
82
  @GeneralUtilities.check_arguments
78
83
  def generate_reference(self) -> None:
@@ -103,8 +103,13 @@ class SCTaskRunnerServer:
103
103
  self.__sc.log.log(f"SCTaskRunner for operating-system '{self.operating_system_name}' is listening on {protocol}://{host}:{port}.")
104
104
  server.serve_forever()
105
105
 
106
+ @staticmethod
107
+ def is_valid_set_of_environment_variables(environment_variables) -> bool:
108
+ """Whether the given (deserialized) value is a set of environment-variables a job can be started with."""
109
+ return isinstance(environment_variables, dict) and all(isinstance(name, str) and isinstance(value, str) and len(name) > 0 and "=" not in name for name, value in environment_variables.items())
110
+
106
111
  @GeneralUtilities.check_arguments
107
- def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
112
+ def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, environment_variables: dict[str, str]) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
108
113
  """Starts a job for the repository-archive which can be read from archive_stream (the body of the request which
109
114
  submitted the job; archive_size is its announced length). The archive is taken as a stream instead of as bytes
110
115
  because it contains the complete repository of the client and is therefore regularly several hundred megabytes,
@@ -128,13 +133,15 @@ class SCTaskRunnerServer:
128
133
  job = _RunnerJob(job_id, workspace_folder, result_folder)
129
134
  with self.__jobs_lock:
130
135
  self.__jobs[job_id] = job
131
- thread = threading.Thread(target=self.__extract_payload_and_run_job, args=(job, archive_file, program, arguments, working_directory), daemon=True)
136
+ thread = threading.Thread(target=self.__extract_payload_and_run_job, args=(job, archive_file, program, arguments, working_directory, environment_variables), daemon=True)
132
137
  thread.start()
133
- self.__sc.log.log(f"Accepted job '{job_id}': run '{program} {' '.join(arguments)}' in '{working_directory}' of codeunit '{codeunit_name}', result-folder '{result_folder}' ({archive_size} transferred bytes of repository-content).")
138
+ #Only the names of the environment-variables are logged: their values are secrets of the client (for example the
139
+ #key which signs an app for a store).
140
+ self.__sc.log.log(f"Accepted job '{job_id}': run '{program} {' '.join(arguments)}' in '{working_directory}' of codeunit '{codeunit_name}', result-folder '{result_folder}', environment-variables [{', '.join(sorted(environment_variables.keys()))}] ({archive_size} transferred bytes of repository-content).")
134
141
  return job_id
135
142
 
136
143
  @GeneralUtilities.check_arguments
137
- def __extract_payload_and_run_job(self, job: _RunnerJob, archive_file: str, program: str, arguments: list[str], working_directory: str) -> None:
144
+ def __extract_payload_and_run_job(self, job: _RunnerJob, archive_file: str, program: str, arguments: list[str], working_directory: str, environment_variables: dict[str, str]) -> None:
138
145
  """Extracts the transferred repository-archive into the workspace of the job and runs the requested program
139
146
  afterwards. The extraction happens here - in the thread of the job - and not in the request which submitted the
140
147
  job, because extracting the repository of a large codeunit takes minutes during which no data flows: the client
@@ -151,7 +158,7 @@ class SCTaskRunnerServer:
151
158
  finally:
152
159
  GeneralUtilities.ensure_file_does_not_exist(archive_file)
153
160
  self.__sc.log.log(f"Job '{job.job_id}' works on the repository '{self.__get_origin_of_the_transferred_repository(job.workspace_folder)}'.")
154
- self.__run_job(job, program, arguments, working_directory)
161
+ self.__run_job(job, program, arguments, working_directory, environment_variables)
155
162
 
156
163
  @GeneralUtilities.check_arguments
157
164
  def __get_origin_of_the_transferred_repository(self, workspace_folder: str) -> str:
@@ -204,14 +211,18 @@ class SCTaskRunnerServer:
204
211
  remaining_amount_of_bytes = remaining_amount_of_bytes-len(chunk)
205
212
 
206
213
  @GeneralUtilities.check_arguments
207
- def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str) -> None:
214
+ def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str, environment_variables: dict[str, str]) -> None:
215
+ """Runs the program of the job. The environment-variables of the job are set in addition to the ones of this
216
+ runner and only for this program."""
208
217
  command_folder = os.path.join(job.workspace_folder, working_directory)
209
218
  try:
210
219
  self.__sc.log.log(f"Job '{job.job_id}' runs '{program} {' '.join(arguments)}' in '{command_folder}'...")
211
220
  moment_of_the_start = time.time()
212
221
  moment_of_the_last_progress_log = moment_of_the_start
213
222
  amount_of_lines = 0
214
- with subprocess.Popen([program] + arguments, cwd=command_folder, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace") as process:
223
+ environment_of_the_program = dict(os.environ)
224
+ environment_of_the_program.update(environment_variables)
225
+ with subprocess.Popen([program] + arguments, cwd=command_folder, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", env=environment_of_the_program) as process:
215
226
  for line in process.stdout:
216
227
  with job.lock:
217
228
  job.log = job.log + line
@@ -293,6 +304,30 @@ class SCTaskRunnerServer:
293
304
  raise ValueError(f"The result-folder '{job.result_folder}' of job '{job.job_id}' is not inside the workspace of that job.")
294
305
  return result_folder
295
306
 
307
+ def __get_amount_of_running_jobs(self) -> int:
308
+ """Returns the amount of jobs which are currently running. A job counts as running from its submit until its
309
+ program has ended, which includes the extraction of its repository."""
310
+ with self.__jobs_lock:
311
+ jobs = list(self.__jobs.values())
312
+ amount_of_running_jobs = 0
313
+ for job in jobs:
314
+ with job.lock:
315
+ if job.state == "running":
316
+ amount_of_running_jobs = amount_of_running_jobs+1
317
+ return amount_of_running_jobs
318
+
319
+ def __get_metrics(self) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
320
+ """Returns the metrics of this runner in the text-based exposition-format of Prometheus, so that the utilization
321
+ of a runner can be monitored by a monitoring-system which scrapes it. Only the amount of running jobs is exposed:
322
+ anything about a job itself (its repository, its program) is information of the client which submitted it."""
323
+ metric_name = "sctaskrunner_running_jobs"
324
+ lines = [
325
+ f"# HELP {metric_name} Amount of jobs which are currently running on this runner.",
326
+ f"# TYPE {metric_name} gauge",
327
+ f"{metric_name} {self.__get_amount_of_running_jobs()}",
328
+ ]
329
+ return "\n".join(lines)+"\n"
330
+
296
331
  @GeneralUtilities.check_arguments
297
332
  def __delete_job(self, job_id: str) -> None: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
298
333
  with self.__jobs_lock:
@@ -381,6 +416,9 @@ class SCTaskRunnerServer:
381
416
  if self.path == "/os":
382
417
  self.__send_text(200, outer.operating_system_name)
383
418
  return
419
+ if self.path == "/metrics":
420
+ self.__send(200, outer._SCTaskRunnerServer__get_metrics().encode("utf-8"), "text/plain; version=0.0.4; charset=utf-8")
421
+ return
384
422
  parts = self.path.strip("/").split("/")
385
423
  if len(parts) >= 2 and parts[0] == "jobs":
386
424
  self.__handle_job_get(parts)
@@ -447,9 +485,17 @@ class SCTaskRunnerServer:
447
485
  arguments = json.loads(base64.b64decode(self.headers.get("X-Arguments")).decode("utf-8"))
448
486
  working_directory = self.headers.get("X-Working-Directory", ".")
449
487
  result_folder = self.headers.get("X-Result-Folder")
488
+ #A client of a version which does not know environment-variables does not send this header.
489
+ environment_variables_header = self.headers.get("X-Environment-Variables")
490
+ environment_variables = {} if environment_variables_header is None else json.loads(base64.b64decode(environment_variables_header).decode("utf-8"))
491
+ if not SCTaskRunnerServer.is_valid_set_of_environment_variables(environment_variables):
492
+ self.__send_text(400, "The header X-Environment-Variables must contain a json-object whose names and values are texts.")
493
+ return
450
494
  #the body is handed over as a stream (and not read here) so the archive is written to disk chunk-wise, see __start_job.
451
- job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory, result_folder)
452
- self.__send_json(200, {"job_id": job_id})
495
+ job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory, result_folder, environment_variables)
496
+ #The names of the applied environment-variables are part of the answer, so that a client can tell this runner
497
+ #from one which does not support environment-variables and would ignore them (see TFCPS_RemoteBuild).
498
+ self.__send_json(200, {"job_id": job_id, "applied_environment_variables": sorted(environment_variables.keys())})
453
499
 
454
500
  def do_DELETE(self): # pylint:disable=invalid-name
455
501
  self.__handle(self.__delete)
@@ -102,6 +102,7 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
102
102
  if len(dependency_versions)==1:
103
103
  GeneralUtilities.write_message_to_stdout("Update dependency "+dependencyname+" (which is currently used in version "+dependency_version+") to version "+desired_version+".")
104
104
  self.set_dependency_version(dependencyname,desired_version)
105
+ self.tfcps_Tools_General.log_newer_major_versions_of_dependency(dependencyname,available_versions,desired_version)
105
106
  except Exception:
106
107
  GeneralUtilities.write_message_to_stderr(f"Error while updating {dependencyname}.")
107
108
  raise
@@ -364,7 +365,7 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
364
365
  return self.__use_cache
365
366
 
366
367
  @GeneralUtilities.check_arguments
367
- def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str]) -> None:
368
+ def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str], environment_variables: dict[str, str]) -> None:
368
369
  """Runs the given program for this codeunit on a remote task-runner that provides the given operating-system (see
369
370
  TFCPS_RemoteBuild and the SCTaskRunnerWindows/SCTaskRunnerMacOS-codeunits). This is used for operating-system-bound
370
371
  build-steps (e.g. flutter-windows- or flutter-ios-builds) which can not run on the current operating-system. After
@@ -373,8 +374,11 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
373
374
  working_directory and result_folder must be folders inside this codeunit.
374
375
  folders_which_are_not_transferred are the folders of this codeunit whose content only applies to this machine
375
376
  and which are therefore left out of the repository-archive the runner gets (see
377
+ TFCPS_RemoteBuild.run_program_on_runner).
378
+ environment_variables are set for the program on the runner only; they are the way to give a secret (for example
379
+ a signing-key) to the build-step without transferring it as part of the repository (see
376
380
  TFCPS_RemoteBuild.run_program_on_runner)."""
377
- TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory, result_folder, folders_which_are_not_transferred)
381
+ TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory, result_folder, folders_which_are_not_transferred, environment_variables)
378
382
 
379
383
  @GeneralUtilities.check_arguments
380
384
  def get_codeunit_folder(self)->str:
@@ -34,6 +34,19 @@ class TFCPS_CodeUnit_BuildCodeUnit:
34
34
  self.is_pre_merge=is_pre_merge
35
35
  self.fast_lane=fast_lane
36
36
 
37
+ def __check_article_links(self) -> None:
38
+ reference_content_folder: str = os.path.join(self.codeunit_folder, "Other", "Reference", "ReferenceContent")
39
+ articles_folder: str = os.path.join(reference_content_folder, "Articles")
40
+ if not os.path.isdir(articles_folder):
41
+ return
42
+ index_file: str = os.path.join(reference_content_folder, "index.md")
43
+ index_content: str = GeneralUtilities.read_text_from_file(index_file)
44
+ GeneralUtilities.assert_file_exists(index_file)
45
+ for article_file in sorted(os.listdir(articles_folder)):
46
+ if article_file.lower().endswith(".md") and os.path.isfile(os.path.join(articles_folder, article_file)):
47
+ if f"(./Articles/{article_file})" not in index_content:
48
+ self.sc.log.log(f"Article \"{article_file}\" is not linked in \"{index_file}\".", LogLevel.Warning)
49
+
37
50
  @GeneralUtilities.check_arguments
38
51
  def build_codeunit(self) -> None:
39
52
  codeunit_file: str = str(os.path.join(self.codeunit_folder, f"{self.codeunit_name}.codeunit.xml"))
@@ -77,6 +90,7 @@ class TFCPS_CodeUnit_BuildCodeUnit:
77
90
  self.sc.log.log(line, LogLevel.Warning)
78
91
  for line in GeneralUtilities.string_to_lines(linting_result[2]):
79
92
  self.sc.log.log(line, LogLevel.Warning)
93
+ self.__check_article_links()
80
94
 
81
95
  self.sc.log.log("Generate reference...")
82
96
  try:
@@ -441,6 +441,25 @@ class TFCPS_CodeUnit_BuildCodeUnits:
441
441
  if os.path.isfile(openspec_configuration_file):
442
442
  self.sc.log.log("Update openspec-instruction-files...")
443
443
  self.sc.run_with_epew("openspec", "update --force", self.repository)
444
+ self.__replace_openspec_skill_links_by_copies()
445
+
446
+ @GeneralUtilities.check_arguments
447
+ def __replace_openspec_skill_links_by_copies(self) -> None:
448
+ """openspec (since v2.0.0) writes its skills to '.agents/skills' and links them into the skill-folders of the configured tools
449
+ (for example '.claude/skills') if the operating-system allows to create symbolic links, otherwise it writes copies. openspec has no
450
+ option to always write copies, so the links are replaced by copies here. This way the content of the repository does not depend on
451
+ the operating-system on which the build ran.
452
+ The skill-folders of all tools are located in folders in the root of the repository whose names start with a dot, so only these
453
+ folders are searched (and not for example the codeunit-folders, which may contain large folders like 'node_modules').
454
+ Submodules are not changed, because their content belongs to another repository."""
455
+ shared_skills_folder: str = os.path.join(self.repository, ".agents", "skills")
456
+ if os.path.isdir(shared_skills_folder):
457
+ submodule_folders: list[str] = [os.path.join(self.repository, submodule) for submodule in self.sc.get_git_submodules(self.repository)]
458
+ for name in os.listdir(self.repository):
459
+ tool_folder: str = os.path.join(self.repository, name)
460
+ if name.startswith(".") and name not in [".git", ".agents"] and os.path.isdir(tool_folder) and not os.path.islink(tool_folder):
461
+ for replaced_link in GeneralUtilities.replace_symlinks_by_copies(tool_folder, shared_skills_folder, submodule_folders):
462
+ self.sc.log.log(f"Replaced symbolic link \"{replaced_link}\" by a copy of its target.", LogLevel.Debug)
444
463
 
445
464
  @GeneralUtilities.check_arguments
446
465
  def __validate_openspec(self,repository:str) -> None:
@@ -93,14 +93,20 @@ class TFCPS_RemoteBuild:
93
93
  return len(self.__load_runner_endpoints()) > 0
94
94
 
95
95
  @GeneralUtilities.check_arguments
96
- def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str], poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
96
+ def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str], environment_variables: dict[str, str], poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
97
97
  """Runs the given program on a runner which provides the given operating-system, see the docstring of this class.
98
98
 
99
99
  'folders_which_are_not_transferred' are folders (as paths inside the repository) which are left out of the
100
100
  archive which is sent to the runner, because their content only applies to this machine (for example the
101
101
  '.dart_tool'-folder of a flutter-package, whose 'package_config.json' states the folder of every package of
102
102
  the app as an absolute path of this machine). Such a folder is generated again by the toolchain of the
103
- runner while it builds; transferring it makes the runner use the paths of this machine instead."""
103
+ runner while it builds; transferring it makes the runner use the paths of this machine instead.
104
+
105
+ 'environment_variables' are set (in addition to the ones of the runner) for the program on the runner only. They
106
+ are the way to give a secret to a build-step (for example the key which signs an app for a store): they are
107
+ neither part of the transferred repository nor of the result nor of a log - the runner logs only their names.
108
+ A runner which does not support them is refused if environment-variables are given (see
109
+ assert_runner_applied_environment_variables)."""
104
110
  endpoints = self.__load_runner_endpoints()
105
111
  if len(endpoints) == 0:
106
112
  raise ValueError("No remote-build-runner is configured. Define runners either in "
@@ -115,18 +121,13 @@ class TFCPS_RemoteBuild:
115
121
  archive_file = self.__create_repository_archive(repository_folder, folders_which_are_not_transferred_relative)
116
122
  job_id: str = None
117
123
  try:
118
- metadata_headers = {
119
- "Content-Type": "application/octet-stream",
120
- "X-Codeunit-Name": codeunit_name,
121
- "X-Program": program,
122
- "X-Arguments": base64.b64encode(json.dumps(arguments).encode("utf-8")).decode("ascii"),
123
- "X-Working-Directory": working_directory_relative,
124
- "X-Result-Folder": result_folder_relative,
125
- }
124
+ metadata_headers = TFCPS_RemoteBuild.get_headers_of_job_submission(codeunit_name, program, arguments, working_directory_relative, result_folder_relative, environment_variables)
126
125
  with open(archive_file, "rb") as archive_content:
127
126
  archive_bytes = archive_content.read()
128
127
  _, submit_response = self.__http(endpoint, "POST", "/jobs", _timeout_of_a_request_in_seconds, metadata_headers, archive_bytes)
129
- job_id = json.loads(submit_response.decode("utf-8"))["job_id"]
128
+ submit_answer = json.loads(submit_response.decode("utf-8"))
129
+ job_id = submit_answer["job_id"]
130
+ TFCPS_RemoteBuild.assert_runner_applied_environment_variables(submit_answer, environment_variables, endpoint.url)
130
131
  self.__wait_for_job(endpoint, job_id, required_os, poll_interval_in_seconds, timeout_in_seconds)
131
132
  #The result-archive is packed completely before it is sent, which transports nothing while it runs - hence
132
133
  #the timeout of a long-running request here.
@@ -141,6 +142,33 @@ class TFCPS_RemoteBuild:
141
142
  self.__sc.log.log_exception(f"Could not delete remote job '{job_id}' on runner '{endpoint.url}'.", exception, LogLevel.Warning)
142
143
  GeneralUtilities.ensure_file_does_not_exist(archive_file)
143
144
 
145
+ @staticmethod
146
+ @GeneralUtilities.check_arguments
147
+ def get_headers_of_job_submission(codeunit_name: str, program: str, arguments: list[str], working_directory_relative: str, result_folder_relative: str, environment_variables: dict[str, str]) -> dict[str, str]:
148
+ """Returns the headers of the request which submits a job (its body is the repository-archive)."""
149
+ return {
150
+ "Content-Type": "application/octet-stream",
151
+ "X-Codeunit-Name": codeunit_name,
152
+ "X-Program": program,
153
+ "X-Arguments": base64.b64encode(json.dumps(arguments).encode("utf-8")).decode("ascii"),
154
+ "X-Working-Directory": working_directory_relative,
155
+ "X-Result-Folder": result_folder_relative,
156
+ "X-Environment-Variables": base64.b64encode(json.dumps(environment_variables).encode("utf-8")).decode("ascii"),
157
+ }
158
+
159
+ @staticmethod
160
+ @GeneralUtilities.check_arguments
161
+ def assert_runner_applied_environment_variables(submit_answer: dict, environment_variables: dict[str, str], runner_url: str) -> None:
162
+ """A runner states in the answer of a submit the names of the environment-variables it sets for the job. A runner
163
+ which does not support environment-variables ignores them without saying so, and the job would then run without
164
+ them (for example an app which is meant to be signed for a store would be built unsigned). That is refused here.
165
+ A job without environment-variables runs on such a runner as before."""
166
+ if len(environment_variables) == 0:
167
+ return
168
+ applied_environment_variables = submit_answer.get("applied_environment_variables")
169
+ GeneralUtilities.assert_condition(applied_environment_variables is not None, f"The runner '{runner_url}' does not support environment-variables for a job, so it would run the job without them. Update the runner to a version of ScriptCollection which supports them.")
170
+ GeneralUtilities.assert_condition(sorted(applied_environment_variables) == sorted(environment_variables.keys()), f"The runner '{runner_url}' did not apply exactly the requested environment-variables (requested: {', '.join(sorted(environment_variables.keys()))}; applied: {', '.join(sorted(applied_environment_variables))}).")
171
+
144
172
  @GeneralUtilities.check_arguments
145
173
  def __wait_for_job(self, endpoint: RunnerEndpoint, job_id: str, required_os: RunnerOperatingSystem, poll_interval_in_seconds: int, timeout_in_seconds: int) -> None:
146
174
  start_time = time.time()
@@ -16,6 +16,7 @@ import urllib.request
16
16
  import xml.etree.ElementTree as ET
17
17
  from packaging import version
18
18
  import requests
19
+ import yaml
19
20
  from lxml import etree
20
21
  from ..GeneralUtilities import GeneralUtilities,Platform
21
22
  from ..ScriptCollectionCore import ScriptCollectionCore,VSCodeWorkspaceShellTask
@@ -1109,11 +1110,29 @@ class TFCPS_Tools_General:
1109
1110
  result["MacOS"]=download_and_extract("MacOS", "darwin", "tar.gz")
1110
1111
  return result
1111
1112
 
1113
+ @GeneralUtilities.check_arguments
1114
+ def __check_for_tasks_which_will_be_overwritten(self, repository_folder: str) -> None:
1115
+ """Asserts that there is no task in the existing taskfile which would be lost when the taskfile is re-generated from the code-workspace-file."""
1116
+ task_file: str = os.path.join(repository_folder, "Taskfile.yml")
1117
+ if not os.path.isfile(task_file):
1118
+ return
1119
+ task_file_content = yaml.safe_load(GeneralUtilities.read_text_from_file(task_file))
1120
+ if not isinstance(task_file_content, dict):
1121
+ return
1122
+ existing_tasks = task_file_content.get("tasks")
1123
+ if not isinstance(existing_tasks, dict):
1124
+ return
1125
+ workspace_file: str = self.__sc.find_file_by_extension(repository_folder, "code-workspace")
1126
+ task_labels_in_workspace_file: set[str] = set(task.label for task in self.__sc.parse_tasks_from_codeworkspace_file(workspace_file))
1127
+ for task_name in existing_tasks.keys():
1128
+ GeneralUtilities.assert_condition(str(task_name) in task_labels_in_workspace_file, f"The task \"{task_name}\" in the taskfile \"{task_file}\" is not defined in the code-workspace-file \"{workspace_file}\" and would be deleted when re-generating the taskfile based on the code-workspace file. Add the task to the code-workspace file to prevent this error.")
1129
+
1112
1130
  @GeneralUtilities.check_arguments
1113
1131
  def generate_tasksfile_from_workspace_file(self, repository_folder: str, append_cli_args_at_end: bool = False) -> None:
1114
1132
  """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1115
1133
  if self.__sc.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
1116
1134
  self.__sc.log.log("Generate taskfile from code-workspace-file...")
1135
+ self.__check_for_tasks_which_will_be_overwritten(repository_folder)
1117
1136
  self.__sc.assert_is_git_repository(repository_folder)
1118
1137
  workspace_file: str = self.__sc.find_file_by_extension(repository_folder, "code-workspace")
1119
1138
  task_file: str = repository_folder + "/Taskfile.yml"
@@ -2091,11 +2110,11 @@ class TFCPS_Tools_General:
2091
2110
  changelog_file = os.path.join(repository_folder, "Other", "Resources", "Changelog", f"v{current_version}.md")
2092
2111
  if (not os.path.isfile(changelog_file)):
2093
2112
  GeneralUtilities.ensure_file_exists(changelog_file)
2094
- GeneralUtilities.write_text_to_file(changelog_file, """# Release notes
2113
+ GeneralUtilities.write_text_to_file(changelog_file, f"""# Release notes
2095
2114
 
2096
2115
  ## Changes
2097
2116
 
2098
- - Updated geo-ip-database.
2117
+ - Updated submodule {submodule_name}.
2099
2118
  """)
2100
2119
 
2101
2120
  @GeneralUtilities.check_arguments
@@ -2111,6 +2130,19 @@ class TFCPS_Tools_General:
2111
2130
  if current_version != latest_version:
2112
2131
  GeneralUtilities.write_text_to_file(resrepo_commit_id_file, latest_version)
2113
2132
 
2133
+ @GeneralUtilities.check_arguments
2134
+ def log_newer_major_versions_of_dependency(self, dependency_name: str, available_versions: list[str], used_version: str) -> None:
2135
+ """Logs if a newer major-version of the given dependency is available than the used one.
2136
+ An update with an echolon like LatestPatchOrLatestMinor does not take over a new major-version, so without this message it would not be visible that the dependency falls behind.
2137
+ Being 1 major-version behind is logged as information, being 2 or more major-versions behind is logged as warning.
2138
+ used_version is the version which is used after the update (if an update was done)."""
2139
+ amount_of_major_versions_behind: int = GeneralUtilities.get_amount_of_major_versions_behind(available_versions, used_version)
2140
+ if amount_of_major_versions_behind == 0:
2141
+ return
2142
+ loglevel: LogLevel = LogLevel.Information if amount_of_major_versions_behind == 1 else LogLevel.Warning
2143
+ latest_version: str = GeneralUtilities.get_latest_version(available_versions)
2144
+ self.__sc.log.log(f"Dependency {dependency_name} is used in version {used_version}, which is {amount_of_major_versions_behind} major-version(s) behind the latest available version {latest_version}. A new major-version has to be taken over deliberately.", loglevel)
2145
+
2114
2146
  @GeneralUtilities.check_arguments
2115
2147
  def get_dependencies_which_are_ignored_from_updates(self, codeunit_folder: str) -> list[str]:
2116
2148
  self.assert_is_codeunit_folder(codeunit_folder)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptcollection
3
- Version: 4.4.71
3
+ Version: 4.4.73
4
4
  Summary: The ScriptCollection is the place for reusable scripts.
5
5
  Author-email: Marius Göcke <marius.goecke@gmail.com>
6
6
  License-Expression: GPL-3.0-only
@@ -35,7 +35,7 @@ Requires-Dist: Pillow>=11.3.0
35
35
  Requires-Dist: psutil>=7.2.2
36
36
  Requires-Dist: pycdlib>=1.22.0
37
37
  Requires-Dist: Pygments>=2.21.0
38
- Requires-Dist: pylint>=4.1.3
38
+ Requires-Dist: pylint>=4.1.4
39
39
  Requires-Dist: pyOpenSSL>=26.4.0
40
40
  Requires-Dist: PyPDF>=6.21.1
41
41
  Requires-Dist: pytest>=8.4.2
@@ -155,7 +155,7 @@ Afterwards `task dl` installs the built wheel locally.
155
155
  In this repository [pylint](https://pylint.org/) will be used to report linting-issues.
156
156
  If you change code in this repository please ensure pylint does not find any issues before creating a pull-request.
157
157
 
158
- If linting-issues exist in the current code-base can be checked by running `python ./ScriptCollection/Other/QualityCheck/Linting.py`.
158
+ Whether linting-issues exist in the current code-base can be checked by running `python ./ScriptCollection/Other/QualityCheck/Linting.py`.
159
159
 
160
160
  ## Runtime dependencies
161
161