ScriptCollection 4.4.72__py3-none-any.whl → 4.4.74__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/Executables.py +19 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/AccountChecks.py +71 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/ContainerChecks.py +34 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/FileSystemChecks.py +141 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/KernelChecks.py +49 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/LogChecks.py +39 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/NetworkChecks.py +67 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/PackageIntegrityChecks.py +62 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/PersistenceChecks.py +99 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/ProcessChecks.py +88 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/SSHChecks.py +56 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/__init__.py +0 -0
- ScriptCollection/LinuxServerSecurityCheck/LinuxServerSecurityChecker.py +90 -0
- ScriptCollection/LinuxServerSecurityCheck/LinuxSystemAccess.py +85 -0
- ScriptCollection/LinuxServerSecurityCheck/PackageOwnershipLookup.py +42 -0
- ScriptCollection/LinuxServerSecurityCheck/SecurityCheckBase.py +19 -0
- ScriptCollection/LinuxServerSecurityCheck/SecurityFinding.py +19 -0
- ScriptCollection/LinuxServerSecurityCheck/SuspiciousCommandPatterns.py +46 -0
- ScriptCollection/LinuxServerSecurityCheck/SystemAccessBase.py +244 -0
- ScriptCollection/LinuxServerSecurityCheck/SystemEntities.py +45 -0
- ScriptCollection/LinuxServerSecurityCheck/__init__.py +0 -0
- ScriptCollection/ProgramRunnerPopen.py +4 -2
- ScriptCollection/ProgramRunnerSudo.py +11 -5
- ScriptCollection/ScriptCollectionCore.py +18 -6
- ScriptCollection/TFCPS/Android/TFCPS_CodeUnitSpecific_Android.py +109 -5
- ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py +1 -1
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +4 -4
- ScriptCollection/TFCPS/SCTaskRunnerServer.py +55 -9
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +5 -2
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +158 -2
- ScriptCollection/TFCPS/TFCPS_RemoteBuild.py +39 -11
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +19 -0
- {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.74.dist-info}/METADATA +177 -177
- {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.74.dist-info}/RECORD +37 -17
- {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.74.dist-info}/entry_points.txt +1 -0
- {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.74.dist-info}/WHEEL +0 -0
- {scriptcollection-4.4.72.dist-info → scriptcollection-4.4.74.dist-info}/top_level.txt +0 -0
|
@@ -103,8 +103,13 @@ class SCTaskRunnerServer:
|
|
|
103
103
|
self.__sc.log.log(f"SCTaskRunner for operating-system '{self.operating_system_name}' is listening on {protocol}://{host}:{port}.")
|
|
104
104
|
server.serve_forever()
|
|
105
105
|
|
|
106
|
+
@staticmethod
|
|
107
|
+
def is_valid_set_of_environment_variables(environment_variables) -> bool:
|
|
108
|
+
"""Whether the given (deserialized) value is a set of environment-variables a job can be started with."""
|
|
109
|
+
return isinstance(environment_variables, dict) and all(isinstance(name, str) and isinstance(value, str) and len(name) > 0 and "=" not in name for name, value in environment_variables.items())
|
|
110
|
+
|
|
106
111
|
@GeneralUtilities.check_arguments
|
|
107
|
-
def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
112
|
+
def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, environment_variables: dict[str, str]) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
108
113
|
"""Starts a job for the repository-archive which can be read from archive_stream (the body of the request which
|
|
109
114
|
submitted the job; archive_size is its announced length). The archive is taken as a stream instead of as bytes
|
|
110
115
|
because it contains the complete repository of the client and is therefore regularly several hundred megabytes,
|
|
@@ -128,13 +133,15 @@ class SCTaskRunnerServer:
|
|
|
128
133
|
job = _RunnerJob(job_id, workspace_folder, result_folder)
|
|
129
134
|
with self.__jobs_lock:
|
|
130
135
|
self.__jobs[job_id] = job
|
|
131
|
-
thread = threading.Thread(target=self.__extract_payload_and_run_job, args=(job, archive_file, program, arguments, working_directory), daemon=True)
|
|
136
|
+
thread = threading.Thread(target=self.__extract_payload_and_run_job, args=(job, archive_file, program, arguments, working_directory, environment_variables), daemon=True)
|
|
132
137
|
thread.start()
|
|
133
|
-
|
|
138
|
+
#Only the names of the environment-variables are logged: their values are secrets of the client (for example the
|
|
139
|
+
#key which signs an app for a store).
|
|
140
|
+
self.__sc.log.log(f"Accepted job '{job_id}': run '{program} {' '.join(arguments)}' in '{working_directory}' of codeunit '{codeunit_name}', result-folder '{result_folder}', environment-variables [{', '.join(sorted(environment_variables.keys()))}] ({archive_size} transferred bytes of repository-content).")
|
|
134
141
|
return job_id
|
|
135
142
|
|
|
136
143
|
@GeneralUtilities.check_arguments
|
|
137
|
-
def __extract_payload_and_run_job(self, job: _RunnerJob, archive_file: str, program: str, arguments: list[str], working_directory: str) -> None:
|
|
144
|
+
def __extract_payload_and_run_job(self, job: _RunnerJob, archive_file: str, program: str, arguments: list[str], working_directory: str, environment_variables: dict[str, str]) -> None:
|
|
138
145
|
"""Extracts the transferred repository-archive into the workspace of the job and runs the requested program
|
|
139
146
|
afterwards. The extraction happens here - in the thread of the job - and not in the request which submitted the
|
|
140
147
|
job, because extracting the repository of a large codeunit takes minutes during which no data flows: the client
|
|
@@ -151,7 +158,7 @@ class SCTaskRunnerServer:
|
|
|
151
158
|
finally:
|
|
152
159
|
GeneralUtilities.ensure_file_does_not_exist(archive_file)
|
|
153
160
|
self.__sc.log.log(f"Job '{job.job_id}' works on the repository '{self.__get_origin_of_the_transferred_repository(job.workspace_folder)}'.")
|
|
154
|
-
self.__run_job(job, program, arguments, working_directory)
|
|
161
|
+
self.__run_job(job, program, arguments, working_directory, environment_variables)
|
|
155
162
|
|
|
156
163
|
@GeneralUtilities.check_arguments
|
|
157
164
|
def __get_origin_of_the_transferred_repository(self, workspace_folder: str) -> str:
|
|
@@ -204,14 +211,18 @@ class SCTaskRunnerServer:
|
|
|
204
211
|
remaining_amount_of_bytes = remaining_amount_of_bytes-len(chunk)
|
|
205
212
|
|
|
206
213
|
@GeneralUtilities.check_arguments
|
|
207
|
-
def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str) -> None:
|
|
214
|
+
def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str, environment_variables: dict[str, str]) -> None:
|
|
215
|
+
"""Runs the program of the job. The environment-variables of the job are set in addition to the ones of this
|
|
216
|
+
runner and only for this program."""
|
|
208
217
|
command_folder = os.path.join(job.workspace_folder, working_directory)
|
|
209
218
|
try:
|
|
210
219
|
self.__sc.log.log(f"Job '{job.job_id}' runs '{program} {' '.join(arguments)}' in '{command_folder}'...")
|
|
211
220
|
moment_of_the_start = time.time()
|
|
212
221
|
moment_of_the_last_progress_log = moment_of_the_start
|
|
213
222
|
amount_of_lines = 0
|
|
214
|
-
|
|
223
|
+
environment_of_the_program = dict(os.environ)
|
|
224
|
+
environment_of_the_program.update(environment_variables)
|
|
225
|
+
with subprocess.Popen([program] + arguments, cwd=command_folder, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", env=environment_of_the_program) as process:
|
|
215
226
|
for line in process.stdout:
|
|
216
227
|
with job.lock:
|
|
217
228
|
job.log = job.log + line
|
|
@@ -293,6 +304,30 @@ class SCTaskRunnerServer:
|
|
|
293
304
|
raise ValueError(f"The result-folder '{job.result_folder}' of job '{job.job_id}' is not inside the workspace of that job.")
|
|
294
305
|
return result_folder
|
|
295
306
|
|
|
307
|
+
def __get_amount_of_running_jobs(self) -> int:
|
|
308
|
+
"""Returns the amount of jobs which are currently running. A job counts as running from its submit until its
|
|
309
|
+
program has ended, which includes the extraction of its repository."""
|
|
310
|
+
with self.__jobs_lock:
|
|
311
|
+
jobs = list(self.__jobs.values())
|
|
312
|
+
amount_of_running_jobs = 0
|
|
313
|
+
for job in jobs:
|
|
314
|
+
with job.lock:
|
|
315
|
+
if job.state == "running":
|
|
316
|
+
amount_of_running_jobs = amount_of_running_jobs+1
|
|
317
|
+
return amount_of_running_jobs
|
|
318
|
+
|
|
319
|
+
def __get_metrics(self) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
320
|
+
"""Returns the metrics of this runner in the text-based exposition-format of Prometheus, so that the utilization
|
|
321
|
+
of a runner can be monitored by a monitoring-system which scrapes it. Only the amount of running jobs is exposed:
|
|
322
|
+
anything about a job itself (its repository, its program) is information of the client which submitted it."""
|
|
323
|
+
metric_name = "sctaskrunner_running_jobs"
|
|
324
|
+
lines = [
|
|
325
|
+
f"# HELP {metric_name} Amount of jobs which are currently running on this runner.",
|
|
326
|
+
f"# TYPE {metric_name} gauge",
|
|
327
|
+
f"{metric_name} {self.__get_amount_of_running_jobs()}",
|
|
328
|
+
]
|
|
329
|
+
return "\n".join(lines)+"\n"
|
|
330
|
+
|
|
296
331
|
@GeneralUtilities.check_arguments
|
|
297
332
|
def __delete_job(self, job_id: str) -> None: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
298
333
|
with self.__jobs_lock:
|
|
@@ -381,6 +416,9 @@ class SCTaskRunnerServer:
|
|
|
381
416
|
if self.path == "/os":
|
|
382
417
|
self.__send_text(200, outer.operating_system_name)
|
|
383
418
|
return
|
|
419
|
+
if self.path == "/metrics":
|
|
420
|
+
self.__send(200, outer._SCTaskRunnerServer__get_metrics().encode("utf-8"), "text/plain; version=0.0.4; charset=utf-8")
|
|
421
|
+
return
|
|
384
422
|
parts = self.path.strip("/").split("/")
|
|
385
423
|
if len(parts) >= 2 and parts[0] == "jobs":
|
|
386
424
|
self.__handle_job_get(parts)
|
|
@@ -447,9 +485,17 @@ class SCTaskRunnerServer:
|
|
|
447
485
|
arguments = json.loads(base64.b64decode(self.headers.get("X-Arguments")).decode("utf-8"))
|
|
448
486
|
working_directory = self.headers.get("X-Working-Directory", ".")
|
|
449
487
|
result_folder = self.headers.get("X-Result-Folder")
|
|
488
|
+
#A client of a version which does not know environment-variables does not send this header.
|
|
489
|
+
environment_variables_header = self.headers.get("X-Environment-Variables")
|
|
490
|
+
environment_variables = {} if environment_variables_header is None else json.loads(base64.b64decode(environment_variables_header).decode("utf-8"))
|
|
491
|
+
if not SCTaskRunnerServer.is_valid_set_of_environment_variables(environment_variables):
|
|
492
|
+
self.__send_text(400, "The header X-Environment-Variables must contain a json-object whose names and values are texts.")
|
|
493
|
+
return
|
|
450
494
|
#the body is handed over as a stream (and not read here) so the archive is written to disk chunk-wise, see __start_job.
|
|
451
|
-
job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory, result_folder)
|
|
452
|
-
|
|
495
|
+
job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory, result_folder, environment_variables)
|
|
496
|
+
#The names of the applied environment-variables are part of the answer, so that a client can tell this runner
|
|
497
|
+
#from one which does not support environment-variables and would ignore them (see TFCPS_RemoteBuild).
|
|
498
|
+
self.__send_json(200, {"job_id": job_id, "applied_environment_variables": sorted(environment_variables.keys())})
|
|
453
499
|
|
|
454
500
|
def do_DELETE(self): # pylint:disable=invalid-name
|
|
455
501
|
self.__handle(self.__delete)
|
|
@@ -365,7 +365,7 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
|
|
|
365
365
|
return self.__use_cache
|
|
366
366
|
|
|
367
367
|
@GeneralUtilities.check_arguments
|
|
368
|
-
def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str]) -> None:
|
|
368
|
+
def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str], environment_variables: dict[str, str]) -> None:
|
|
369
369
|
"""Runs the given program for this codeunit on a remote task-runner that provides the given operating-system (see
|
|
370
370
|
TFCPS_RemoteBuild and the SCTaskRunnerWindows/SCTaskRunnerMacOS-codeunits). This is used for operating-system-bound
|
|
371
371
|
build-steps (e.g. flutter-windows- or flutter-ios-builds) which can not run on the current operating-system. After
|
|
@@ -374,8 +374,11 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
|
|
|
374
374
|
working_directory and result_folder must be folders inside this codeunit.
|
|
375
375
|
folders_which_are_not_transferred are the folders of this codeunit whose content only applies to this machine
|
|
376
376
|
and which are therefore left out of the repository-archive the runner gets (see
|
|
377
|
+
TFCPS_RemoteBuild.run_program_on_runner).
|
|
378
|
+
environment_variables are set for the program on the runner only; they are the way to give a secret (for example
|
|
379
|
+
a signing-key) to the build-step without transferring it as part of the repository (see
|
|
377
380
|
TFCPS_RemoteBuild.run_program_on_runner)."""
|
|
378
|
-
TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory, result_folder, folders_which_are_not_transferred)
|
|
381
|
+
TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory, result_folder, folders_which_are_not_transferred, environment_variables)
|
|
379
382
|
|
|
380
383
|
@GeneralUtilities.check_arguments
|
|
381
384
|
def get_codeunit_folder(self)->str:
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import base64
|
|
1
2
|
import os
|
|
2
3
|
import json
|
|
3
4
|
import re
|
|
4
5
|
import shutil
|
|
5
6
|
import tempfile
|
|
6
7
|
from datetime import datetime, timedelta,timezone
|
|
8
|
+
import requests
|
|
7
9
|
import xmlschema
|
|
8
10
|
from packaging.version import Version
|
|
9
11
|
from ..GeneralUtilities import GeneralUtilities
|
|
@@ -76,6 +78,8 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
76
78
|
__assert_no_new_changes:bool = None
|
|
77
79
|
__add_ready_to_merge_flag:bool = None
|
|
78
80
|
__fast_lane:bool = None
|
|
81
|
+
# The tools for which openspec is initialized in a repository which uses openspec but in which openspec is not initialized yet.
|
|
82
|
+
__openspec_tools:list[str] = ["claude", "cline", "github-copilot", "vibe", "opencode", "codex", "gemini", "cursor"]
|
|
79
83
|
|
|
80
84
|
def __init__(self,repository:str,loglevel:LogLevel,target_environment_type:str,additionalargumentsfile:str,use_cache:bool,is_pre_merge:bool,assertnonewchanges:bool,add_ready_to_merge_flag:bool=False,fast_lane:bool=False):
|
|
81
85
|
self.sc=ScriptCollectionCore()
|
|
@@ -287,12 +291,145 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
287
291
|
if updated_lines != lines:
|
|
288
292
|
GeneralUtilities.write_lines_to_file(agent_sandbox_configuration_file, updated_lines)
|
|
289
293
|
|
|
294
|
+
@GeneralUtilities.check_arguments
|
|
295
|
+
def __check_security_txt_file(self, repository_folder: str) -> None:
|
|
296
|
+
"""Checks the file '<repository>/.well-known/security.txt' (see RFC 9116). Every failed check is only reported as warning and does
|
|
297
|
+
not let the build fail. The URLs of the file are only checked if an internet-connection is available."""
|
|
298
|
+
security_txt_file: str = os.path.join(repository_folder, ".well-known", "security.txt")
|
|
299
|
+
if not os.path.isfile(security_txt_file):
|
|
300
|
+
self.sc.log.log(f"The file '{security_txt_file}' does not exist.", LogLevel.Warning)
|
|
301
|
+
return
|
|
302
|
+
self.sc.log.log("Check security.txt-file...", LogLevel.Debug)
|
|
303
|
+
fields: dict[str, list[str]] = TFCPS_CodeUnit_BuildCodeUnits.__get_security_txt_fields(security_txt_file)
|
|
304
|
+
internet_connection_is_available: bool = GeneralUtilities.internet_connection_is_available()
|
|
305
|
+
self.__check_security_txt_signature(security_txt_file, fields, internet_connection_is_available)
|
|
306
|
+
self.__check_security_txt_expiration(security_txt_file, fields, GeneralUtilities.get_now())
|
|
307
|
+
if internet_connection_is_available:
|
|
308
|
+
self.__check_security_txt_urls_are_reachable(security_txt_file, fields, "Canonical")
|
|
309
|
+
self.__check_security_txt_urls_are_reachable(security_txt_file, fields, "Encryption")
|
|
310
|
+
else:
|
|
311
|
+
self.sc.log.log(f"The URLs of '{security_txt_file}' are not checked because no internet-connection is available.", LogLevel.Debug)
|
|
312
|
+
|
|
313
|
+
@staticmethod
|
|
314
|
+
@GeneralUtilities.check_arguments
|
|
315
|
+
def __get_security_txt_fields(security_txt_file: str) -> dict[str, list[str]]:
|
|
316
|
+
"""Returns the values of the fields of the given security.txt-file, grouped by the lowercase field-name (field-names are
|
|
317
|
+
case-insensitive according to RFC 9116). A field can occur multiple times, so every field-name maps to a list of values.
|
|
318
|
+
Comments and the lines of the PGP-envelope are ignored."""
|
|
319
|
+
result: dict[str, list[str]] = {}
|
|
320
|
+
for line in GeneralUtilities.read_lines_from_file(security_txt_file):
|
|
321
|
+
match = re.match(r"^([A-Za-z0-9-]+):\s*(.*?)\s*$", line)
|
|
322
|
+
if match is not None:
|
|
323
|
+
result.setdefault(match.group(1).lower(), []).append(match.group(2))
|
|
324
|
+
return result
|
|
325
|
+
|
|
326
|
+
@GeneralUtilities.check_arguments
|
|
327
|
+
def __check_security_txt_signature(self, security_txt_file: str, fields: dict[str, list[str]], internet_connection_is_available: bool) -> None:
|
|
328
|
+
"""Checks whether the given security.txt-file has a PGP-signature and whether it is valid.
|
|
329
|
+
Only the validity of the signature is checked and not whether the key of the signer is trusted. Therefore the signature is
|
|
330
|
+
verified with gpgv against a temporary keyring which only contains the public key which is downloaded from the first
|
|
331
|
+
http(s)-URL of the 'Encryption'-field. This way the result does not depend on the keyring of the current user, so the check
|
|
332
|
+
works the same way in a build-pipeline. Without internet-connection the public key can not be downloaded, so the signature
|
|
333
|
+
can not be verified then."""
|
|
334
|
+
if "-----BEGIN PGP SIGNED MESSAGE-----" not in GeneralUtilities.read_text_from_file(security_txt_file):
|
|
335
|
+
self.sc.log.log(f"The file '{security_txt_file}' is not signed.", LogLevel.Warning)
|
|
336
|
+
return
|
|
337
|
+
if not GeneralUtilities.tool_is_available("gpgv"):
|
|
338
|
+
self.sc.log.log(f"The signature of '{security_txt_file}' can not be verified because gpgv is not available.", LogLevel.Warning)
|
|
339
|
+
return
|
|
340
|
+
if not internet_connection_is_available:
|
|
341
|
+
self.sc.log.log(f"The signature of '{security_txt_file}' can not be verified because the public key can not be downloaded without internet-connection.", LogLevel.Warning)
|
|
342
|
+
return
|
|
343
|
+
key_urls: list[str] = [url for url in fields.get("encryption", []) if url.lower().startswith(("https://", "http://"))]
|
|
344
|
+
if len(key_urls) == 0:
|
|
345
|
+
self.sc.log.log(f"The signature of '{security_txt_file}' can not be verified because the file does not contain an 'Encryption'-field with an http(s)-URL of the public key.", LogLevel.Warning)
|
|
346
|
+
return
|
|
347
|
+
key_url: str = key_urls[0]
|
|
348
|
+
try:
|
|
349
|
+
response = requests.get(key_url, timeout=30, allow_redirects=True)
|
|
350
|
+
except requests.RequestException as exception:
|
|
351
|
+
self.sc.log.log(f"The signature of '{security_txt_file}' can not be verified because the public key can not be downloaded from '{key_url}': {exception}", LogLevel.Warning)
|
|
352
|
+
return
|
|
353
|
+
if not response.ok:
|
|
354
|
+
self.sc.log.log(f"The signature of '{security_txt_file}' can not be verified because the public key can not be downloaded from '{key_url}' (HTTP-status {response.status_code}).", LogLevel.Warning)
|
|
355
|
+
return
|
|
356
|
+
public_key: bytes = TFCPS_CodeUnit_BuildCodeUnits.__dearmor_pgp_public_key(response.text)
|
|
357
|
+
if public_key is None:
|
|
358
|
+
self.sc.log.log(f"The signature of '{security_txt_file}' can not be verified because '{key_url}' does not contain a PGP-public-key-block.", LogLevel.Warning)
|
|
359
|
+
return
|
|
360
|
+
with tempfile.TemporaryDirectory() as keyring_folder:
|
|
361
|
+
keyring_file: str = os.path.join(keyring_folder, "PublicKey.gpg")
|
|
362
|
+
with open(keyring_file, "wb") as keyring_file_stream:
|
|
363
|
+
keyring_file_stream.write(public_key)
|
|
364
|
+
# The keyring-file is passed as path relative to the working-directory with a leading "./": gpgv searches a path without
|
|
365
|
+
# a slash in its home-folder, and the gpgv of git-for-windows does not accept an absolute windows-path as keyring.
|
|
366
|
+
exit_code, _, stderr, _ = self.sc.run_program_argsasarray("gpgv", ["--keyring", f"./{os.path.basename(keyring_file)}", os.path.abspath(security_txt_file)], keyring_folder, throw_exception_if_exitcode_is_not_zero=False)
|
|
367
|
+
if exit_code != 0:
|
|
368
|
+
self.sc.log.log(f"The signature of '{security_txt_file}' is not valid for the public key from '{key_url}' (exitcode of gpgv: {exit_code}). StdErr: '{stderr}'", LogLevel.Warning)
|
|
369
|
+
|
|
370
|
+
@staticmethod
|
|
371
|
+
@GeneralUtilities.check_arguments
|
|
372
|
+
def __dearmor_pgp_public_key(armored_content: str) -> bytes:
|
|
373
|
+
"""Returns the binary content of the first ASCII-armored PGP-public-key-block of the given content (the format which gpgv
|
|
374
|
+
expects for a keyring-file), or None if the content does not contain such a block. The armor-headers (for example 'Comment:')
|
|
375
|
+
and the checksum-line (which starts with '=') are not part of the binary content."""
|
|
376
|
+
match = re.search(r"-----BEGIN PGP PUBLIC KEY BLOCK-----\r?\n(.*?)-----END PGP PUBLIC KEY BLOCK-----", armored_content, re.DOTALL)
|
|
377
|
+
if match is None:
|
|
378
|
+
return None
|
|
379
|
+
lines: list[str] = [line.strip() for line in match.group(1).splitlines()]
|
|
380
|
+
if GeneralUtilities.empty_string in lines:
|
|
381
|
+
lines = lines[lines.index(GeneralUtilities.empty_string)+1:] # the armor-headers end with the first empty line
|
|
382
|
+
return base64.b64decode("".join(line for line in lines if GeneralUtilities.string_has_content(line) and not line.startswith("=")))
|
|
383
|
+
|
|
384
|
+
@GeneralUtilities.check_arguments
|
|
385
|
+
def __check_security_txt_expiration(self, security_txt_file: str, fields: dict[str, list[str]], now: datetime) -> None:
|
|
386
|
+
"""Checks whether the given security.txt-file contains exactly one 'Expires'-field (RFC 9116 does not allow more than one) whose
|
|
387
|
+
value is not expired and which is valid for at least the next 3 months. A value without timezone is treated as UTC."""
|
|
388
|
+
expires_values: list[str] = fields.get("expires", [])
|
|
389
|
+
if len(expires_values) == 0:
|
|
390
|
+
self.sc.log.log(f"The file '{security_txt_file}' does not contain an 'Expires'-field.", LogLevel.Warning)
|
|
391
|
+
return
|
|
392
|
+
if 1 < len(expires_values):
|
|
393
|
+
self.sc.log.log(f"The file '{security_txt_file}' contains more than one 'Expires'-field.", LogLevel.Warning)
|
|
394
|
+
return
|
|
395
|
+
try:
|
|
396
|
+
expires: datetime = datetime.fromisoformat(expires_values[0])
|
|
397
|
+
except ValueError:
|
|
398
|
+
self.sc.log.log(f"The 'Expires'-value '{expires_values[0]}' of '{security_txt_file}' is not a valid timestamp.", LogLevel.Warning)
|
|
399
|
+
return
|
|
400
|
+
if expires.tzinfo is None:
|
|
401
|
+
expires = expires.replace(tzinfo=timezone.utc)
|
|
402
|
+
if expires <= now:
|
|
403
|
+
self.sc.log.log(f"The file '{security_txt_file}' is expired since {expires_values[0]}.", LogLevel.Warning)
|
|
404
|
+
elif expires - now < timedelta(days=30*3):
|
|
405
|
+
self.sc.log.log(f"The file '{security_txt_file}' expires within the next 3 months ({expires_values[0]}).", LogLevel.Warning)
|
|
406
|
+
|
|
407
|
+
@GeneralUtilities.check_arguments
|
|
408
|
+
def __check_security_txt_urls_are_reachable(self, security_txt_file: str, fields: dict[str, list[str]], field_name: str) -> None:
|
|
409
|
+
"""Checks whether the URLs of the given field of the given security.txt-file are reachable. Values which are no http(s)-URL (for
|
|
410
|
+
example an 'openpgp4fpr:'-URI in the 'Encryption'-field) can not be requested and are therefore not checked."""
|
|
411
|
+
urls: list[str] = fields.get(field_name.lower(), [])
|
|
412
|
+
if len(urls) == 0:
|
|
413
|
+
self.sc.log.log(f"The file '{security_txt_file}' does not contain a '{field_name}'-field.", LogLevel.Warning)
|
|
414
|
+
return
|
|
415
|
+
for url in urls:
|
|
416
|
+
if not url.lower().startswith(("https://", "http://")):
|
|
417
|
+
self.sc.log.log(f"The {field_name}-value '{url}' of '{security_txt_file}' is not checked because it is not an http(s)-URL.", LogLevel.Debug)
|
|
418
|
+
continue
|
|
419
|
+
try:
|
|
420
|
+
with requests.get(url, timeout=30, stream=True, allow_redirects=True) as response:
|
|
421
|
+
if not response.ok:
|
|
422
|
+
self.sc.log.log(f"The {field_name}-URL '{url}' of '{security_txt_file}' is not reachable (HTTP-status {response.status_code}).", LogLevel.Warning)
|
|
423
|
+
except requests.RequestException as exception:
|
|
424
|
+
self.sc.log.log(f"The {field_name}-URL '{url}' of '{security_txt_file}' is not reachable: {exception}", LogLevel.Warning)
|
|
425
|
+
|
|
290
426
|
@GeneralUtilities.check_arguments
|
|
291
427
|
def __do_project_linting(self, repository_folder: str) -> None:
|
|
292
428
|
self.sc.log.log("Do project-linting...")
|
|
293
|
-
self.
|
|
429
|
+
self.__run_codespell(repository_folder)
|
|
294
430
|
self.__sort_findings(repository_folder)
|
|
295
431
|
self.__validate_openspec(repository_folder)
|
|
432
|
+
self.__check_security_txt_file(repository_folder)
|
|
296
433
|
|
|
297
434
|
@GeneralUtilities.check_arguments
|
|
298
435
|
def __sort_findings(self, repository_folder: str) -> None:
|
|
@@ -300,7 +437,7 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
300
437
|
TFCPS_FindingsSorter().sort_findings(repository_folder)
|
|
301
438
|
|
|
302
439
|
@GeneralUtilities.check_arguments
|
|
303
|
-
def
|
|
440
|
+
def __run_codespell(self, repository_folder: str) -> None:
|
|
304
441
|
"""Searches for typos in the repository using codespell. Findings are only reported as warnings and do not let the build fail.
|
|
305
442
|
Only files which are not git-ignored are checked. See TFCPS_CodeSpellCheck for details."""
|
|
306
443
|
try:
|
|
@@ -439,10 +576,29 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
439
576
|
def __update_openspec(self) -> None:
|
|
440
577
|
openspec_configuration_file: str = os.path.join(self.repository, "openspec", "config.yaml")
|
|
441
578
|
if os.path.isfile(openspec_configuration_file):
|
|
579
|
+
if not self.__openspec_is_initialized():
|
|
580
|
+
#"openspec update" only updates the tools which are already set up in the repository, so a repository which only contains
|
|
581
|
+
#"openspec/config.yaml" would never get any openspec-skill without this initialization. "openspec init" keeps an existing config.yaml.
|
|
582
|
+
self.sc.log.log("Initialize openspec...")
|
|
583
|
+
self.sc.run_with_epew("openspec", f"init --tools {','.join(TFCPS_CodeUnit_BuildCodeUnits.__openspec_tools)} --force", self.repository)
|
|
442
584
|
self.sc.log.log("Update openspec-instruction-files...")
|
|
443
585
|
self.sc.run_with_epew("openspec", "update --force", self.repository)
|
|
444
586
|
self.__replace_openspec_skill_links_by_copies()
|
|
445
587
|
|
|
588
|
+
@GeneralUtilities.check_arguments
|
|
589
|
+
def __openspec_is_initialized(self) -> bool:
|
|
590
|
+
"""Returns whether openspec has already deployed a skill in the repository. openspec (since v2.0.0) deploys its skills to
|
|
591
|
+
'.agents/skills', older versions deployed them to the skill-folders of the tools (for example '.claude/skills'). These folders are
|
|
592
|
+
located in folders in the root of the repository whose names start with a dot. The names of the skills which openspec deploys
|
|
593
|
+
start with 'openspec-'."""
|
|
594
|
+
for name in os.listdir(self.repository):
|
|
595
|
+
skills_folder: str = os.path.join(self.repository, name, "skills")
|
|
596
|
+
if name.startswith(".") and name != ".git" and os.path.isdir(skills_folder):
|
|
597
|
+
for skill_name in os.listdir(skills_folder):
|
|
598
|
+
if skill_name.startswith("openspec-") and os.path.isfile(os.path.join(skills_folder, skill_name, "SKILL.md")):
|
|
599
|
+
return True
|
|
600
|
+
return False
|
|
601
|
+
|
|
446
602
|
@GeneralUtilities.check_arguments
|
|
447
603
|
def __replace_openspec_skill_links_by_copies(self) -> None:
|
|
448
604
|
"""openspec (since v2.0.0) writes its skills to '.agents/skills' and links them into the skill-folders of the configured tools
|
|
@@ -93,14 +93,20 @@ class TFCPS_RemoteBuild:
|
|
|
93
93
|
return len(self.__load_runner_endpoints()) > 0
|
|
94
94
|
|
|
95
95
|
@GeneralUtilities.check_arguments
|
|
96
|
-
def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str], poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
|
|
96
|
+
def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str], environment_variables: dict[str, str], poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
|
|
97
97
|
"""Runs the given program on a runner which provides the given operating-system, see the docstring of this class.
|
|
98
98
|
|
|
99
99
|
'folders_which_are_not_transferred' are folders (as paths inside the repository) which are left out of the
|
|
100
100
|
archive which is sent to the runner, because their content only applies to this machine (for example the
|
|
101
101
|
'.dart_tool'-folder of a flutter-package, whose 'package_config.json' states the folder of every package of
|
|
102
102
|
the app as an absolute path of this machine). Such a folder is generated again by the toolchain of the
|
|
103
|
-
runner while it builds; transferring it makes the runner use the paths of this machine instead.
|
|
103
|
+
runner while it builds; transferring it makes the runner use the paths of this machine instead.
|
|
104
|
+
|
|
105
|
+
'environment_variables' are set (in addition to the ones of the runner) for the program on the runner only. They
|
|
106
|
+
are the way to give a secret to a build-step (for example the key which signs an app for a store): they are
|
|
107
|
+
neither part of the transferred repository nor of the result nor of a log - the runner logs only their names.
|
|
108
|
+
A runner which does not support them is refused if environment-variables are given (see
|
|
109
|
+
assert_runner_applied_environment_variables)."""
|
|
104
110
|
endpoints = self.__load_runner_endpoints()
|
|
105
111
|
if len(endpoints) == 0:
|
|
106
112
|
raise ValueError("No remote-build-runner is configured. Define runners either in "
|
|
@@ -115,18 +121,13 @@ class TFCPS_RemoteBuild:
|
|
|
115
121
|
archive_file = self.__create_repository_archive(repository_folder, folders_which_are_not_transferred_relative)
|
|
116
122
|
job_id: str = None
|
|
117
123
|
try:
|
|
118
|
-
metadata_headers =
|
|
119
|
-
"Content-Type": "application/octet-stream",
|
|
120
|
-
"X-Codeunit-Name": codeunit_name,
|
|
121
|
-
"X-Program": program,
|
|
122
|
-
"X-Arguments": base64.b64encode(json.dumps(arguments).encode("utf-8")).decode("ascii"),
|
|
123
|
-
"X-Working-Directory": working_directory_relative,
|
|
124
|
-
"X-Result-Folder": result_folder_relative,
|
|
125
|
-
}
|
|
124
|
+
metadata_headers = TFCPS_RemoteBuild.get_headers_of_job_submission(codeunit_name, program, arguments, working_directory_relative, result_folder_relative, environment_variables)
|
|
126
125
|
with open(archive_file, "rb") as archive_content:
|
|
127
126
|
archive_bytes = archive_content.read()
|
|
128
127
|
_, submit_response = self.__http(endpoint, "POST", "/jobs", _timeout_of_a_request_in_seconds, metadata_headers, archive_bytes)
|
|
129
|
-
|
|
128
|
+
submit_answer = json.loads(submit_response.decode("utf-8"))
|
|
129
|
+
job_id = submit_answer["job_id"]
|
|
130
|
+
TFCPS_RemoteBuild.assert_runner_applied_environment_variables(submit_answer, environment_variables, endpoint.url)
|
|
130
131
|
self.__wait_for_job(endpoint, job_id, required_os, poll_interval_in_seconds, timeout_in_seconds)
|
|
131
132
|
#The result-archive is packed completely before it is sent, which transports nothing while it runs - hence
|
|
132
133
|
#the timeout of a long-running request here.
|
|
@@ -141,6 +142,33 @@ class TFCPS_RemoteBuild:
|
|
|
141
142
|
self.__sc.log.log_exception(f"Could not delete remote job '{job_id}' on runner '{endpoint.url}'.", exception, LogLevel.Warning)
|
|
142
143
|
GeneralUtilities.ensure_file_does_not_exist(archive_file)
|
|
143
144
|
|
|
145
|
+
@staticmethod
|
|
146
|
+
@GeneralUtilities.check_arguments
|
|
147
|
+
def get_headers_of_job_submission(codeunit_name: str, program: str, arguments: list[str], working_directory_relative: str, result_folder_relative: str, environment_variables: dict[str, str]) -> dict[str, str]:
|
|
148
|
+
"""Returns the headers of the request which submits a job (its body is the repository-archive)."""
|
|
149
|
+
return {
|
|
150
|
+
"Content-Type": "application/octet-stream",
|
|
151
|
+
"X-Codeunit-Name": codeunit_name,
|
|
152
|
+
"X-Program": program,
|
|
153
|
+
"X-Arguments": base64.b64encode(json.dumps(arguments).encode("utf-8")).decode("ascii"),
|
|
154
|
+
"X-Working-Directory": working_directory_relative,
|
|
155
|
+
"X-Result-Folder": result_folder_relative,
|
|
156
|
+
"X-Environment-Variables": base64.b64encode(json.dumps(environment_variables).encode("utf-8")).decode("ascii"),
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
@staticmethod
|
|
160
|
+
@GeneralUtilities.check_arguments
|
|
161
|
+
def assert_runner_applied_environment_variables(submit_answer: dict, environment_variables: dict[str, str], runner_url: str) -> None:
|
|
162
|
+
"""A runner states in the answer of a submit the names of the environment-variables it sets for the job. A runner
|
|
163
|
+
which does not support environment-variables ignores them without saying so, and the job would then run without
|
|
164
|
+
them (for example an app which is meant to be signed for a store would be built unsigned). That is refused here.
|
|
165
|
+
A job without environment-variables runs on such a runner as before."""
|
|
166
|
+
if len(environment_variables) == 0:
|
|
167
|
+
return
|
|
168
|
+
applied_environment_variables = submit_answer.get("applied_environment_variables")
|
|
169
|
+
GeneralUtilities.assert_condition(applied_environment_variables is not None, f"The runner '{runner_url}' does not support environment-variables for a job, so it would run the job without them. Update the runner to a version of ScriptCollection which supports them.")
|
|
170
|
+
GeneralUtilities.assert_condition(sorted(applied_environment_variables) == sorted(environment_variables.keys()), f"The runner '{runner_url}' did not apply exactly the requested environment-variables (requested: {', '.join(sorted(environment_variables.keys()))}; applied: {', '.join(sorted(applied_environment_variables))}).")
|
|
171
|
+
|
|
144
172
|
@GeneralUtilities.check_arguments
|
|
145
173
|
def __wait_for_job(self, endpoint: RunnerEndpoint, job_id: str, required_os: RunnerOperatingSystem, poll_interval_in_seconds: int, timeout_in_seconds: int) -> None:
|
|
146
174
|
start_time = time.time()
|
|
@@ -16,6 +16,7 @@ import urllib.request
|
|
|
16
16
|
import xml.etree.ElementTree as ET
|
|
17
17
|
from packaging import version
|
|
18
18
|
import requests
|
|
19
|
+
import yaml
|
|
19
20
|
from lxml import etree
|
|
20
21
|
from ..GeneralUtilities import GeneralUtilities,Platform
|
|
21
22
|
from ..ScriptCollectionCore import ScriptCollectionCore,VSCodeWorkspaceShellTask
|
|
@@ -1109,11 +1110,29 @@ class TFCPS_Tools_General:
|
|
|
1109
1110
|
result["MacOS"]=download_and_extract("MacOS", "darwin", "tar.gz")
|
|
1110
1111
|
return result
|
|
1111
1112
|
|
|
1113
|
+
@GeneralUtilities.check_arguments
|
|
1114
|
+
def __check_for_tasks_which_will_be_overwritten(self, repository_folder: str) -> None:
|
|
1115
|
+
"""Asserts that there is no task in the existing taskfile which would be lost when the taskfile is re-generated from the code-workspace-file."""
|
|
1116
|
+
task_file: str = os.path.join(repository_folder, "Taskfile.yml")
|
|
1117
|
+
if not os.path.isfile(task_file):
|
|
1118
|
+
return
|
|
1119
|
+
task_file_content = yaml.safe_load(GeneralUtilities.read_text_from_file(task_file))
|
|
1120
|
+
if not isinstance(task_file_content, dict):
|
|
1121
|
+
return
|
|
1122
|
+
existing_tasks = task_file_content.get("tasks")
|
|
1123
|
+
if not isinstance(existing_tasks, dict):
|
|
1124
|
+
return
|
|
1125
|
+
workspace_file: str = self.__sc.find_file_by_extension(repository_folder, "code-workspace")
|
|
1126
|
+
task_labels_in_workspace_file: set[str] = set(task.label for task in self.__sc.parse_tasks_from_codeworkspace_file(workspace_file))
|
|
1127
|
+
for task_name in existing_tasks.keys():
|
|
1128
|
+
GeneralUtilities.assert_condition(str(task_name) in task_labels_in_workspace_file, f"The task \"{task_name}\" in the taskfile \"{task_file}\" is not defined in the code-workspace-file \"{workspace_file}\" and would be deleted when re-generating the taskfile based on the code-workspace file. Add the task to the code-workspace file to prevent this error.")
|
|
1129
|
+
|
|
1112
1130
|
@GeneralUtilities.check_arguments
|
|
1113
1131
|
def generate_tasksfile_from_workspace_file(self, repository_folder: str, append_cli_args_at_end: bool = False) -> None:
|
|
1114
1132
|
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
1115
1133
|
if self.__sc.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
|
|
1116
1134
|
self.__sc.log.log("Generate taskfile from code-workspace-file...")
|
|
1135
|
+
self.__check_for_tasks_which_will_be_overwritten(repository_folder)
|
|
1117
1136
|
self.__sc.assert_is_git_repository(repository_folder)
|
|
1118
1137
|
workspace_file: str = self.__sc.find_file_by_extension(repository_folder, "code-workspace")
|
|
1119
1138
|
task_file: str = repository_folder + "/Taskfile.yml"
|