ScriptCollection 4.4.72__py3-none-any.whl → 4.4.74__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. ScriptCollection/Executables.py +19 -0
  2. ScriptCollection/LinuxServerSecurityCheck/Checks/AccountChecks.py +71 -0
  3. ScriptCollection/LinuxServerSecurityCheck/Checks/ContainerChecks.py +34 -0
  4. ScriptCollection/LinuxServerSecurityCheck/Checks/FileSystemChecks.py +141 -0
  5. ScriptCollection/LinuxServerSecurityCheck/Checks/KernelChecks.py +49 -0
  6. ScriptCollection/LinuxServerSecurityCheck/Checks/LogChecks.py +39 -0
  7. ScriptCollection/LinuxServerSecurityCheck/Checks/NetworkChecks.py +67 -0
  8. ScriptCollection/LinuxServerSecurityCheck/Checks/PackageIntegrityChecks.py +62 -0
  9. ScriptCollection/LinuxServerSecurityCheck/Checks/PersistenceChecks.py +99 -0
  10. ScriptCollection/LinuxServerSecurityCheck/Checks/ProcessChecks.py +88 -0
  11. ScriptCollection/LinuxServerSecurityCheck/Checks/SSHChecks.py +56 -0
  12. ScriptCollection/LinuxServerSecurityCheck/Checks/__init__.py +0 -0
  13. ScriptCollection/LinuxServerSecurityCheck/LinuxServerSecurityChecker.py +90 -0
  14. ScriptCollection/LinuxServerSecurityCheck/LinuxSystemAccess.py +85 -0
  15. ScriptCollection/LinuxServerSecurityCheck/PackageOwnershipLookup.py +42 -0
  16. ScriptCollection/LinuxServerSecurityCheck/SecurityCheckBase.py +19 -0
  17. ScriptCollection/LinuxServerSecurityCheck/SecurityFinding.py +19 -0
  18. ScriptCollection/LinuxServerSecurityCheck/SuspiciousCommandPatterns.py +46 -0
  19. ScriptCollection/LinuxServerSecurityCheck/SystemAccessBase.py +244 -0
  20. ScriptCollection/LinuxServerSecurityCheck/SystemEntities.py +45 -0
  21. ScriptCollection/LinuxServerSecurityCheck/__init__.py +0 -0
  22. ScriptCollection/ProgramRunnerPopen.py +4 -2
  23. ScriptCollection/ProgramRunnerSudo.py +11 -5
  24. ScriptCollection/ScriptCollectionCore.py +18 -6
  25. ScriptCollection/TFCPS/Android/TFCPS_CodeUnitSpecific_Android.py +109 -5
  26. ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py +1 -1
  27. ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +4 -4
  28. ScriptCollection/TFCPS/SCTaskRunnerServer.py +55 -9
  29. ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +5 -2
  30. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +158 -2
  31. ScriptCollection/TFCPS/TFCPS_RemoteBuild.py +39 -11
  32. ScriptCollection/TFCPS/TFCPS_Tools_General.py +19 -0
  33. {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.74.dist-info}/METADATA +177 -177
  34. {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.74.dist-info}/RECORD +37 -17
  35. {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.74.dist-info}/entry_points.txt +1 -0
  36. {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.74.dist-info}/WHEEL +0 -0
  37. {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.74.dist-info}/top_level.txt +0 -0
@@ -103,8 +103,13 @@ class SCTaskRunnerServer:
103
103
  self.__sc.log.log(f"SCTaskRunner for operating-system '{self.operating_system_name}' is listening on {protocol}://{host}:{port}.")
104
104
  server.serve_forever()
105
105
 
106
+ @staticmethod
107
+ def is_valid_set_of_environment_variables(environment_variables) -> bool:
108
+ """Whether the given (deserialized) value is a set of environment-variables a job can be started with."""
109
+ return isinstance(environment_variables, dict) and all(isinstance(name, str) and isinstance(value, str) and len(name) > 0 and "=" not in name for name, value in environment_variables.items())
110
+
106
111
  @GeneralUtilities.check_arguments
107
- def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
112
+ def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, environment_variables: dict[str, str]) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
108
113
  """Starts a job for the repository-archive which can be read from archive_stream (the body of the request which
109
114
  submitted the job; archive_size is its announced length). The archive is taken as a stream instead of as bytes
110
115
  because it contains the complete repository of the client and is therefore regularly several hundred megabytes,
@@ -128,13 +133,15 @@ class SCTaskRunnerServer:
128
133
  job = _RunnerJob(job_id, workspace_folder, result_folder)
129
134
  with self.__jobs_lock:
130
135
  self.__jobs[job_id] = job
131
- thread = threading.Thread(target=self.__extract_payload_and_run_job, args=(job, archive_file, program, arguments, working_directory), daemon=True)
136
+ thread = threading.Thread(target=self.__extract_payload_and_run_job, args=(job, archive_file, program, arguments, working_directory, environment_variables), daemon=True)
132
137
  thread.start()
133
- self.__sc.log.log(f"Accepted job '{job_id}': run '{program} {' '.join(arguments)}' in '{working_directory}' of codeunit '{codeunit_name}', result-folder '{result_folder}' ({archive_size} transferred bytes of repository-content).")
138
+ #Only the names of the environment-variables are logged: their values are secrets of the client (for example the
139
+ #key which signs an app for a store).
140
+ self.__sc.log.log(f"Accepted job '{job_id}': run '{program} {' '.join(arguments)}' in '{working_directory}' of codeunit '{codeunit_name}', result-folder '{result_folder}', environment-variables [{', '.join(sorted(environment_variables.keys()))}] ({archive_size} transferred bytes of repository-content).")
134
141
  return job_id
135
142
 
136
143
  @GeneralUtilities.check_arguments
137
- def __extract_payload_and_run_job(self, job: _RunnerJob, archive_file: str, program: str, arguments: list[str], working_directory: str) -> None:
144
+ def __extract_payload_and_run_job(self, job: _RunnerJob, archive_file: str, program: str, arguments: list[str], working_directory: str, environment_variables: dict[str, str]) -> None:
138
145
  """Extracts the transferred repository-archive into the workspace of the job and runs the requested program
139
146
  afterwards. The extraction happens here - in the thread of the job - and not in the request which submitted the
140
147
  job, because extracting the repository of a large codeunit takes minutes during which no data flows: the client
@@ -151,7 +158,7 @@ class SCTaskRunnerServer:
151
158
  finally:
152
159
  GeneralUtilities.ensure_file_does_not_exist(archive_file)
153
160
  self.__sc.log.log(f"Job '{job.job_id}' works on the repository '{self.__get_origin_of_the_transferred_repository(job.workspace_folder)}'.")
154
- self.__run_job(job, program, arguments, working_directory)
161
+ self.__run_job(job, program, arguments, working_directory, environment_variables)
155
162
 
156
163
  @GeneralUtilities.check_arguments
157
164
  def __get_origin_of_the_transferred_repository(self, workspace_folder: str) -> str:
@@ -204,14 +211,18 @@ class SCTaskRunnerServer:
204
211
  remaining_amount_of_bytes = remaining_amount_of_bytes-len(chunk)
205
212
 
206
213
  @GeneralUtilities.check_arguments
207
- def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str) -> None:
214
+ def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str, environment_variables: dict[str, str]) -> None:
215
+ """Runs the program of the job. The environment-variables of the job are set in addition to the ones of this
216
+ runner and only for this program."""
208
217
  command_folder = os.path.join(job.workspace_folder, working_directory)
209
218
  try:
210
219
  self.__sc.log.log(f"Job '{job.job_id}' runs '{program} {' '.join(arguments)}' in '{command_folder}'...")
211
220
  moment_of_the_start = time.time()
212
221
  moment_of_the_last_progress_log = moment_of_the_start
213
222
  amount_of_lines = 0
214
- with subprocess.Popen([program] + arguments, cwd=command_folder, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace") as process:
223
+ environment_of_the_program = dict(os.environ)
224
+ environment_of_the_program.update(environment_variables)
225
+ with subprocess.Popen([program] + arguments, cwd=command_folder, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", env=environment_of_the_program) as process:
215
226
  for line in process.stdout:
216
227
  with job.lock:
217
228
  job.log = job.log + line
@@ -293,6 +304,30 @@ class SCTaskRunnerServer:
293
304
  raise ValueError(f"The result-folder '{job.result_folder}' of job '{job.job_id}' is not inside the workspace of that job.")
294
305
  return result_folder
295
306
 
307
+ def __get_amount_of_running_jobs(self) -> int:
308
+ """Returns the amount of jobs which are currently running. A job counts as running from its submit until its
309
+ program has ended, which includes the extraction of its repository."""
310
+ with self.__jobs_lock:
311
+ jobs = list(self.__jobs.values())
312
+ amount_of_running_jobs = 0
313
+ for job in jobs:
314
+ with job.lock:
315
+ if job.state == "running":
316
+ amount_of_running_jobs = amount_of_running_jobs+1
317
+ return amount_of_running_jobs
318
+
319
+ def __get_metrics(self) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
320
+ """Returns the metrics of this runner in the text-based exposition-format of Prometheus, so that the utilization
321
+ of a runner can be monitored by a monitoring-system which scrapes it. Only the amount of running jobs is exposed:
322
+ anything about a job itself (its repository, its program) is information of the client which submitted it."""
323
+ metric_name = "sctaskrunner_running_jobs"
324
+ lines = [
325
+ f"# HELP {metric_name} Amount of jobs which are currently running on this runner.",
326
+ f"# TYPE {metric_name} gauge",
327
+ f"{metric_name} {self.__get_amount_of_running_jobs()}",
328
+ ]
329
+ return "\n".join(lines)+"\n"
330
+
296
331
  @GeneralUtilities.check_arguments
297
332
  def __delete_job(self, job_id: str) -> None: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
298
333
  with self.__jobs_lock:
@@ -381,6 +416,9 @@ class SCTaskRunnerServer:
381
416
  if self.path == "/os":
382
417
  self.__send_text(200, outer.operating_system_name)
383
418
  return
419
+ if self.path == "/metrics":
420
+ self.__send(200, outer._SCTaskRunnerServer__get_metrics().encode("utf-8"), "text/plain; version=0.0.4; charset=utf-8")
421
+ return
384
422
  parts = self.path.strip("/").split("/")
385
423
  if len(parts) >= 2 and parts[0] == "jobs":
386
424
  self.__handle_job_get(parts)
@@ -447,9 +485,17 @@ class SCTaskRunnerServer:
447
485
  arguments = json.loads(base64.b64decode(self.headers.get("X-Arguments")).decode("utf-8"))
448
486
  working_directory = self.headers.get("X-Working-Directory", ".")
449
487
  result_folder = self.headers.get("X-Result-Folder")
488
+ #A client of a version which does not know environment-variables does not send this header.
489
+ environment_variables_header = self.headers.get("X-Environment-Variables")
490
+ environment_variables = {} if environment_variables_header is None else json.loads(base64.b64decode(environment_variables_header).decode("utf-8"))
491
+ if not SCTaskRunnerServer.is_valid_set_of_environment_variables(environment_variables):
492
+ self.__send_text(400, "The header X-Environment-Variables must contain a json-object whose names and values are texts.")
493
+ return
450
494
  #the body is handed over as a stream (and not read here) so the archive is written to disk chunk-wise, see __start_job.
451
- job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory, result_folder)
452
- self.__send_json(200, {"job_id": job_id})
495
+ job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory, result_folder, environment_variables)
496
+ #The names of the applied environment-variables are part of the answer, so that a client can tell this runner
497
+ #from one which does not support environment-variables and would ignore them (see TFCPS_RemoteBuild).
498
+ self.__send_json(200, {"job_id": job_id, "applied_environment_variables": sorted(environment_variables.keys())})
453
499
 
454
500
  def do_DELETE(self): # pylint:disable=invalid-name
455
501
  self.__handle(self.__delete)
@@ -365,7 +365,7 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
365
365
  return self.__use_cache
366
366
 
367
367
  @GeneralUtilities.check_arguments
368
- def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str]) -> None:
368
+ def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str], environment_variables: dict[str, str]) -> None:
369
369
  """Runs the given program for this codeunit on a remote task-runner that provides the given operating-system (see
370
370
  TFCPS_RemoteBuild and the SCTaskRunnerWindows/SCTaskRunnerMacOS-codeunits). This is used for operating-system-bound
371
371
  build-steps (e.g. flutter-windows- or flutter-ios-builds) which can not run on the current operating-system. After
@@ -374,8 +374,11 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
374
374
  working_directory and result_folder must be folders inside this codeunit.
375
375
  folders_which_are_not_transferred are the folders of this codeunit whose content only applies to this machine
376
376
  and which are therefore left out of the repository-archive the runner gets (see
377
+ TFCPS_RemoteBuild.run_program_on_runner).
378
+ environment_variables are set for the program on the runner only; they are the way to give a secret (for example
379
+ a signing-key) to the build-step without transferring it as part of the repository (see
377
380
  TFCPS_RemoteBuild.run_program_on_runner)."""
378
- TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory, result_folder, folders_which_are_not_transferred)
381
+ TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory, result_folder, folders_which_are_not_transferred, environment_variables)
379
382
 
380
383
  @GeneralUtilities.check_arguments
381
384
  def get_codeunit_folder(self)->str:
@@ -1,9 +1,11 @@
1
+ import base64
1
2
  import os
2
3
  import json
3
4
  import re
4
5
  import shutil
5
6
  import tempfile
6
7
  from datetime import datetime, timedelta,timezone
8
+ import requests
7
9
  import xmlschema
8
10
  from packaging.version import Version
9
11
  from ..GeneralUtilities import GeneralUtilities
@@ -76,6 +78,8 @@ class TFCPS_CodeUnit_BuildCodeUnits:
76
78
  __assert_no_new_changes:bool = None
77
79
  __add_ready_to_merge_flag:bool = None
78
80
  __fast_lane:bool = None
81
+ # The tools for which openspec is initialized in a repository which uses openspec but in which openspec is not initialized yet.
82
+ __openspec_tools:list[str] = ["claude", "cline", "github-copilot", "vibe", "opencode", "codex", "gemini", "cursor"]
79
83
 
80
84
  def __init__(self,repository:str,loglevel:LogLevel,target_environment_type:str,additionalargumentsfile:str,use_cache:bool,is_pre_merge:bool,assertnonewchanges:bool,add_ready_to_merge_flag:bool=False,fast_lane:bool=False):
81
85
  self.sc=ScriptCollectionCore()
@@ -287,12 +291,145 @@ class TFCPS_CodeUnit_BuildCodeUnits:
287
291
  if updated_lines != lines:
288
292
  GeneralUtilities.write_lines_to_file(agent_sandbox_configuration_file, updated_lines)
289
293
 
294
+ @GeneralUtilities.check_arguments
295
+ def __check_security_txt_file(self, repository_folder: str) -> None:
296
+ """Checks the file '<repository>/.well-known/security.txt' (see RFC 9116). Every failed check is only reported as warning and does
297
+ not let the build fail. The URLs of the file are only checked if an internet-connection is available."""
298
+ security_txt_file: str = os.path.join(repository_folder, ".well-known", "security.txt")
299
+ if not os.path.isfile(security_txt_file):
300
+ self.sc.log.log(f"The file '{security_txt_file}' does not exist.", LogLevel.Warning)
301
+ return
302
+ self.sc.log.log("Check security.txt-file...", LogLevel.Debug)
303
+ fields: dict[str, list[str]] = TFCPS_CodeUnit_BuildCodeUnits.__get_security_txt_fields(security_txt_file)
304
+ internet_connection_is_available: bool = GeneralUtilities.internet_connection_is_available()
305
+ self.__check_security_txt_signature(security_txt_file, fields, internet_connection_is_available)
306
+ self.__check_security_txt_expiration(security_txt_file, fields, GeneralUtilities.get_now())
307
+ if internet_connection_is_available:
308
+ self.__check_security_txt_urls_are_reachable(security_txt_file, fields, "Canonical")
309
+ self.__check_security_txt_urls_are_reachable(security_txt_file, fields, "Encryption")
310
+ else:
311
+ self.sc.log.log(f"The URLs of '{security_txt_file}' are not checked because no internet-connection is available.", LogLevel.Debug)
312
+
313
+ @staticmethod
314
+ @GeneralUtilities.check_arguments
315
+ def __get_security_txt_fields(security_txt_file: str) -> dict[str, list[str]]:
316
+ """Returns the values of the fields of the given security.txt-file, grouped by the lowercase field-name (field-names are
317
+ case-insensitive according to RFC 9116). A field can occur multiple times, so every field-name maps to a list of values.
318
+ Comments and the lines of the PGP-envelope are ignored."""
319
+ result: dict[str, list[str]] = {}
320
+ for line in GeneralUtilities.read_lines_from_file(security_txt_file):
321
+ match = re.match(r"^([A-Za-z0-9-]+):\s*(.*?)\s*$", line)
322
+ if match is not None:
323
+ result.setdefault(match.group(1).lower(), []).append(match.group(2))
324
+ return result
325
+
326
+ @GeneralUtilities.check_arguments
327
+ def __check_security_txt_signature(self, security_txt_file: str, fields: dict[str, list[str]], internet_connection_is_available: bool) -> None:
328
+ """Checks whether the given security.txt-file has a PGP-signature and whether it is valid.
329
+ Only the validity of the signature is checked and not whether the key of the signer is trusted. Therefore the signature is
330
+ verified with gpgv against a temporary keyring which only contains the public key which is downloaded from the first
331
+ http(s)-URL of the 'Encryption'-field. This way the result does not depend on the keyring of the current user, so the check
332
+ works the same way in a build-pipeline. Without internet-connection the public key can not be downloaded, so the signature
333
+ can not be verified then."""
334
+ if "-----BEGIN PGP SIGNED MESSAGE-----" not in GeneralUtilities.read_text_from_file(security_txt_file):
335
+ self.sc.log.log(f"The file '{security_txt_file}' is not signed.", LogLevel.Warning)
336
+ return
337
+ if not GeneralUtilities.tool_is_available("gpgv"):
338
+ self.sc.log.log(f"The signature of '{security_txt_file}' can not be verified because gpgv is not available.", LogLevel.Warning)
339
+ return
340
+ if not internet_connection_is_available:
341
+ self.sc.log.log(f"The signature of '{security_txt_file}' can not be verified because the public key can not be downloaded without internet-connection.", LogLevel.Warning)
342
+ return
343
+ key_urls: list[str] = [url for url in fields.get("encryption", []) if url.lower().startswith(("https://", "http://"))]
344
+ if len(key_urls) == 0:
345
+ self.sc.log.log(f"The signature of '{security_txt_file}' can not be verified because the file does not contain an 'Encryption'-field with an http(s)-URL of the public key.", LogLevel.Warning)
346
+ return
347
+ key_url: str = key_urls[0]
348
+ try:
349
+ response = requests.get(key_url, timeout=30, allow_redirects=True)
350
+ except requests.RequestException as exception:
351
+ self.sc.log.log(f"The signature of '{security_txt_file}' can not be verified because the public key can not be downloaded from '{key_url}': {exception}", LogLevel.Warning)
352
+ return
353
+ if not response.ok:
354
+ self.sc.log.log(f"The signature of '{security_txt_file}' can not be verified because the public key can not be downloaded from '{key_url}' (HTTP-status {response.status_code}).", LogLevel.Warning)
355
+ return
356
+ public_key: bytes = TFCPS_CodeUnit_BuildCodeUnits.__dearmor_pgp_public_key(response.text)
357
+ if public_key is None:
358
+ self.sc.log.log(f"The signature of '{security_txt_file}' can not be verified because '{key_url}' does not contain a PGP-public-key-block.", LogLevel.Warning)
359
+ return
360
+ with tempfile.TemporaryDirectory() as keyring_folder:
361
+ keyring_file: str = os.path.join(keyring_folder, "PublicKey.gpg")
362
+ with open(keyring_file, "wb") as keyring_file_stream:
363
+ keyring_file_stream.write(public_key)
364
+ # The keyring-file is passed as path relative to the working-directory with a leading "./": gpgv searches a path without
365
+ # a slash in its home-folder, and the gpgv of git-for-windows does not accept an absolute windows-path as keyring.
366
+ exit_code, _, stderr, _ = self.sc.run_program_argsasarray("gpgv", ["--keyring", f"./{os.path.basename(keyring_file)}", os.path.abspath(security_txt_file)], keyring_folder, throw_exception_if_exitcode_is_not_zero=False)
367
+ if exit_code != 0:
368
+ self.sc.log.log(f"The signature of '{security_txt_file}' is not valid for the public key from '{key_url}' (exitcode of gpgv: {exit_code}). StdErr: '{stderr}'", LogLevel.Warning)
369
+
370
+ @staticmethod
371
+ @GeneralUtilities.check_arguments
372
+ def __dearmor_pgp_public_key(armored_content: str) -> bytes:
373
+ """Returns the binary content of the first ASCII-armored PGP-public-key-block of the given content (the format which gpgv
374
+ expects for a keyring-file), or None if the content does not contain such a block. The armor-headers (for example 'Comment:')
375
+ and the checksum-line (which starts with '=') are not part of the binary content."""
376
+ match = re.search(r"-----BEGIN PGP PUBLIC KEY BLOCK-----\r?\n(.*?)-----END PGP PUBLIC KEY BLOCK-----", armored_content, re.DOTALL)
377
+ if match is None:
378
+ return None
379
+ lines: list[str] = [line.strip() for line in match.group(1).splitlines()]
380
+ if GeneralUtilities.empty_string in lines:
381
+ lines = lines[lines.index(GeneralUtilities.empty_string)+1:] # the armor-headers end with the first empty line
382
+ return base64.b64decode("".join(line for line in lines if GeneralUtilities.string_has_content(line) and not line.startswith("=")))
383
+
384
+ @GeneralUtilities.check_arguments
385
+ def __check_security_txt_expiration(self, security_txt_file: str, fields: dict[str, list[str]], now: datetime) -> None:
386
+ """Checks whether the given security.txt-file contains exactly one 'Expires'-field (RFC 9116 does not allow more than one) whose
387
+ value is not expired and which is valid for at least the next 3 months. A value without timezone is treated as UTC."""
388
+ expires_values: list[str] = fields.get("expires", [])
389
+ if len(expires_values) == 0:
390
+ self.sc.log.log(f"The file '{security_txt_file}' does not contain an 'Expires'-field.", LogLevel.Warning)
391
+ return
392
+ if 1 < len(expires_values):
393
+ self.sc.log.log(f"The file '{security_txt_file}' contains more than one 'Expires'-field.", LogLevel.Warning)
394
+ return
395
+ try:
396
+ expires: datetime = datetime.fromisoformat(expires_values[0])
397
+ except ValueError:
398
+ self.sc.log.log(f"The 'Expires'-value '{expires_values[0]}' of '{security_txt_file}' is not a valid timestamp.", LogLevel.Warning)
399
+ return
400
+ if expires.tzinfo is None:
401
+ expires = expires.replace(tzinfo=timezone.utc)
402
+ if expires <= now:
403
+ self.sc.log.log(f"The file '{security_txt_file}' is expired since {expires_values[0]}.", LogLevel.Warning)
404
+ elif expires - now < timedelta(days=30*3):
405
+ self.sc.log.log(f"The file '{security_txt_file}' expires within the next 3 months ({expires_values[0]}).", LogLevel.Warning)
406
+
407
+ @GeneralUtilities.check_arguments
408
+ def __check_security_txt_urls_are_reachable(self, security_txt_file: str, fields: dict[str, list[str]], field_name: str) -> None:
409
+ """Checks whether the URLs of the given field of the given security.txt-file are reachable. Values which are no http(s)-URL (for
410
+ example an 'openpgp4fpr:'-URI in the 'Encryption'-field) can not be requested and are therefore not checked."""
411
+ urls: list[str] = fields.get(field_name.lower(), [])
412
+ if len(urls) == 0:
413
+ self.sc.log.log(f"The file '{security_txt_file}' does not contain a '{field_name}'-field.", LogLevel.Warning)
414
+ return
415
+ for url in urls:
416
+ if not url.lower().startswith(("https://", "http://")):
417
+ self.sc.log.log(f"The {field_name}-value '{url}' of '{security_txt_file}' is not checked because it is not an http(s)-URL.", LogLevel.Debug)
418
+ continue
419
+ try:
420
+ with requests.get(url, timeout=30, stream=True, allow_redirects=True) as response:
421
+ if not response.ok:
422
+ self.sc.log.log(f"The {field_name}-URL '{url}' of '{security_txt_file}' is not reachable (HTTP-status {response.status_code}).", LogLevel.Warning)
423
+ except requests.RequestException as exception:
424
+ self.sc.log.log(f"The {field_name}-URL '{url}' of '{security_txt_file}' is not reachable: {exception}", LogLevel.Warning)
425
+
290
426
  @GeneralUtilities.check_arguments
291
427
  def __do_project_linting(self, repository_folder: str) -> None:
292
428
  self.sc.log.log("Do project-linting...")
293
- self.__do_project_linting_using_codespell(repository_folder)
429
+ self.__run_codespell(repository_folder)
294
430
  self.__sort_findings(repository_folder)
295
431
  self.__validate_openspec(repository_folder)
432
+ self.__check_security_txt_file(repository_folder)
296
433
 
297
434
  @GeneralUtilities.check_arguments
298
435
  def __sort_findings(self, repository_folder: str) -> None:
@@ -300,7 +437,7 @@ class TFCPS_CodeUnit_BuildCodeUnits:
300
437
  TFCPS_FindingsSorter().sort_findings(repository_folder)
301
438
 
302
439
  @GeneralUtilities.check_arguments
303
- def __do_project_linting_using_codespell(self, repository_folder: str) -> None:
440
+ def __run_codespell(self, repository_folder: str) -> None:
304
441
  """Searches for typos in the repository using codespell. Findings are only reported as warnings and do not let the build fail.
305
442
  Only files which are not git-ignored are checked. See TFCPS_CodeSpellCheck for details."""
306
443
  try:
@@ -439,10 +576,29 @@ class TFCPS_CodeUnit_BuildCodeUnits:
439
576
  def __update_openspec(self) -> None:
440
577
  openspec_configuration_file: str = os.path.join(self.repository, "openspec", "config.yaml")
441
578
  if os.path.isfile(openspec_configuration_file):
579
+ if not self.__openspec_is_initialized():
580
+ #"openspec update" only updates the tools which are already set up in the repository, so a repository which only contains
581
+ #"openspec/config.yaml" would never get any openspec-skill without this initialization. "openspec init" keeps an existing config.yaml.
582
+ self.sc.log.log("Initialize openspec...")
583
+ self.sc.run_with_epew("openspec", f"init --tools {','.join(TFCPS_CodeUnit_BuildCodeUnits.__openspec_tools)} --force", self.repository)
442
584
  self.sc.log.log("Update openspec-instruction-files...")
443
585
  self.sc.run_with_epew("openspec", "update --force", self.repository)
444
586
  self.__replace_openspec_skill_links_by_copies()
445
587
 
588
+ @GeneralUtilities.check_arguments
589
+ def __openspec_is_initialized(self) -> bool:
590
+ """Returns whether openspec has already deployed a skill in the repository. openspec (since v2.0.0) deploys its skills to
591
+ '.agents/skills', older versions deployed them to the skill-folders of the tools (for example '.claude/skills'). These folders are
592
+ located in folders in the root of the repository whose names start with a dot. The names of the skills which openspec deploys
593
+ start with 'openspec-'."""
594
+ for name in os.listdir(self.repository):
595
+ skills_folder: str = os.path.join(self.repository, name, "skills")
596
+ if name.startswith(".") and name != ".git" and os.path.isdir(skills_folder):
597
+ for skill_name in os.listdir(skills_folder):
598
+ if skill_name.startswith("openspec-") and os.path.isfile(os.path.join(skills_folder, skill_name, "SKILL.md")):
599
+ return True
600
+ return False
601
+
446
602
  @GeneralUtilities.check_arguments
447
603
  def __replace_openspec_skill_links_by_copies(self) -> None:
448
604
  """openspec (since v2.0.0) writes its skills to '.agents/skills' and links them into the skill-folders of the configured tools
@@ -93,14 +93,20 @@ class TFCPS_RemoteBuild:
93
93
  return len(self.__load_runner_endpoints()) > 0
94
94
 
95
95
  @GeneralUtilities.check_arguments
96
- def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str], poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
96
+ def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str], environment_variables: dict[str, str], poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
97
97
  """Runs the given program on a runner which provides the given operating-system, see the docstring of this class.
98
98
 
99
99
  'folders_which_are_not_transferred' are folders (as paths inside the repository) which are left out of the
100
100
  archive which is sent to the runner, because their content only applies to this machine (for example the
101
101
  '.dart_tool'-folder of a flutter-package, whose 'package_config.json' states the folder of every package of
102
102
  the app as an absolute path of this machine). Such a folder is generated again by the toolchain of the
103
- runner while it builds; transferring it makes the runner use the paths of this machine instead."""
103
+ runner while it builds; transferring it makes the runner use the paths of this machine instead.
104
+
105
+ 'environment_variables' are set (in addition to the ones of the runner) for the program on the runner only. They
106
+ are the way to give a secret to a build-step (for example the key which signs an app for a store): they are
107
+ neither part of the transferred repository nor of the result nor of a log - the runner logs only their names.
108
+ A runner which does not support them is refused if environment-variables are given (see
109
+ assert_runner_applied_environment_variables)."""
104
110
  endpoints = self.__load_runner_endpoints()
105
111
  if len(endpoints) == 0:
106
112
  raise ValueError("No remote-build-runner is configured. Define runners either in "
@@ -115,18 +121,13 @@ class TFCPS_RemoteBuild:
115
121
  archive_file = self.__create_repository_archive(repository_folder, folders_which_are_not_transferred_relative)
116
122
  job_id: str = None
117
123
  try:
118
- metadata_headers = {
119
- "Content-Type": "application/octet-stream",
120
- "X-Codeunit-Name": codeunit_name,
121
- "X-Program": program,
122
- "X-Arguments": base64.b64encode(json.dumps(arguments).encode("utf-8")).decode("ascii"),
123
- "X-Working-Directory": working_directory_relative,
124
- "X-Result-Folder": result_folder_relative,
125
- }
124
+ metadata_headers = TFCPS_RemoteBuild.get_headers_of_job_submission(codeunit_name, program, arguments, working_directory_relative, result_folder_relative, environment_variables)
126
125
  with open(archive_file, "rb") as archive_content:
127
126
  archive_bytes = archive_content.read()
128
127
  _, submit_response = self.__http(endpoint, "POST", "/jobs", _timeout_of_a_request_in_seconds, metadata_headers, archive_bytes)
129
- job_id = json.loads(submit_response.decode("utf-8"))["job_id"]
128
+ submit_answer = json.loads(submit_response.decode("utf-8"))
129
+ job_id = submit_answer["job_id"]
130
+ TFCPS_RemoteBuild.assert_runner_applied_environment_variables(submit_answer, environment_variables, endpoint.url)
130
131
  self.__wait_for_job(endpoint, job_id, required_os, poll_interval_in_seconds, timeout_in_seconds)
131
132
  #The result-archive is packed completely before it is sent, which transports nothing while it runs - hence
132
133
  #the timeout of a long-running request here.
@@ -141,6 +142,33 @@ class TFCPS_RemoteBuild:
141
142
  self.__sc.log.log_exception(f"Could not delete remote job '{job_id}' on runner '{endpoint.url}'.", exception, LogLevel.Warning)
142
143
  GeneralUtilities.ensure_file_does_not_exist(archive_file)
143
144
 
145
+ @staticmethod
146
+ @GeneralUtilities.check_arguments
147
+ def get_headers_of_job_submission(codeunit_name: str, program: str, arguments: list[str], working_directory_relative: str, result_folder_relative: str, environment_variables: dict[str, str]) -> dict[str, str]:
148
+ """Returns the headers of the request which submits a job (its body is the repository-archive)."""
149
+ return {
150
+ "Content-Type": "application/octet-stream",
151
+ "X-Codeunit-Name": codeunit_name,
152
+ "X-Program": program,
153
+ "X-Arguments": base64.b64encode(json.dumps(arguments).encode("utf-8")).decode("ascii"),
154
+ "X-Working-Directory": working_directory_relative,
155
+ "X-Result-Folder": result_folder_relative,
156
+ "X-Environment-Variables": base64.b64encode(json.dumps(environment_variables).encode("utf-8")).decode("ascii"),
157
+ }
158
+
159
+ @staticmethod
160
+ @GeneralUtilities.check_arguments
161
+ def assert_runner_applied_environment_variables(submit_answer: dict, environment_variables: dict[str, str], runner_url: str) -> None:
162
+ """A runner states in the answer of a submit the names of the environment-variables it sets for the job. A runner
163
+ which does not support environment-variables ignores them without saying so, and the job would then run without
164
+ them (for example an app which is meant to be signed for a store would be built unsigned). That is refused here.
165
+ A job without environment-variables runs on such a runner as before."""
166
+ if len(environment_variables) == 0:
167
+ return
168
+ applied_environment_variables = submit_answer.get("applied_environment_variables")
169
+ GeneralUtilities.assert_condition(applied_environment_variables is not None, f"The runner '{runner_url}' does not support environment-variables for a job, so it would run the job without them. Update the runner to a version of ScriptCollection which supports them.")
170
+ GeneralUtilities.assert_condition(sorted(applied_environment_variables) == sorted(environment_variables.keys()), f"The runner '{runner_url}' did not apply exactly the requested environment-variables (requested: {', '.join(sorted(environment_variables.keys()))}; applied: {', '.join(sorted(applied_environment_variables))}).")
171
+
144
172
  @GeneralUtilities.check_arguments
145
173
  def __wait_for_job(self, endpoint: RunnerEndpoint, job_id: str, required_os: RunnerOperatingSystem, poll_interval_in_seconds: int, timeout_in_seconds: int) -> None:
146
174
  start_time = time.time()
@@ -16,6 +16,7 @@ import urllib.request
16
16
  import xml.etree.ElementTree as ET
17
17
  from packaging import version
18
18
  import requests
19
+ import yaml
19
20
  from lxml import etree
20
21
  from ..GeneralUtilities import GeneralUtilities,Platform
21
22
  from ..ScriptCollectionCore import ScriptCollectionCore,VSCodeWorkspaceShellTask
@@ -1109,11 +1110,29 @@ class TFCPS_Tools_General:
1109
1110
  result["MacOS"]=download_and_extract("MacOS", "darwin", "tar.gz")
1110
1111
  return result
1111
1112
 
1113
+ @GeneralUtilities.check_arguments
1114
+ def __check_for_tasks_which_will_be_overwritten(self, repository_folder: str) -> None:
1115
+ """Asserts that there is no task in the existing taskfile which would be lost when the taskfile is re-generated from the code-workspace-file."""
1116
+ task_file: str = os.path.join(repository_folder, "Taskfile.yml")
1117
+ if not os.path.isfile(task_file):
1118
+ return
1119
+ task_file_content = yaml.safe_load(GeneralUtilities.read_text_from_file(task_file))
1120
+ if not isinstance(task_file_content, dict):
1121
+ return
1122
+ existing_tasks = task_file_content.get("tasks")
1123
+ if not isinstance(existing_tasks, dict):
1124
+ return
1125
+ workspace_file: str = self.__sc.find_file_by_extension(repository_folder, "code-workspace")
1126
+ task_labels_in_workspace_file: set[str] = set(task.label for task in self.__sc.parse_tasks_from_codeworkspace_file(workspace_file))
1127
+ for task_name in existing_tasks.keys():
1128
+ GeneralUtilities.assert_condition(str(task_name) in task_labels_in_workspace_file, f"The task \"{task_name}\" in the taskfile \"{task_file}\" is not defined in the code-workspace-file \"{workspace_file}\" and would be deleted when re-generating the taskfile based on the code-workspace file. Add the task to the code-workspace file to prevent this error.")
1129
+
1112
1130
  @GeneralUtilities.check_arguments
1113
1131
  def generate_tasksfile_from_workspace_file(self, repository_folder: str, append_cli_args_at_end: bool = False) -> None:
1114
1132
  """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1115
1133
  if self.__sc.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
1116
1134
  self.__sc.log.log("Generate taskfile from code-workspace-file...")
1135
+ self.__check_for_tasks_which_will_be_overwritten(repository_folder)
1117
1136
  self.__sc.assert_is_git_repository(repository_folder)
1118
1137
  workspace_file: str = self.__sc.find_file_by_extension(repository_folder, "code-workspace")
1119
1138
  task_file: str = repository_folder + "/Taskfile.yml"