ScriptCollection 4.4.71__py3-none-any.whl → 4.4.73__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/Executables.py +22 -3
- ScriptCollection/GeneralUtilities.py +42 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/AccountChecks.py +71 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/ContainerChecks.py +34 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/FileSystemChecks.py +141 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/KernelChecks.py +49 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/LogChecks.py +39 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/NetworkChecks.py +67 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/PackageIntegrityChecks.py +62 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/PersistenceChecks.py +99 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/ProcessChecks.py +88 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/SSHChecks.py +56 -0
- ScriptCollection/LinuxServerSecurityCheck/Checks/__init__.py +0 -0
- ScriptCollection/LinuxServerSecurityCheck/LinuxServerSecurityChecker.py +90 -0
- ScriptCollection/LinuxServerSecurityCheck/LinuxSystemAccess.py +85 -0
- ScriptCollection/LinuxServerSecurityCheck/PackageOwnershipLookup.py +42 -0
- ScriptCollection/LinuxServerSecurityCheck/SecurityCheckBase.py +19 -0
- ScriptCollection/LinuxServerSecurityCheck/SecurityFinding.py +19 -0
- ScriptCollection/LinuxServerSecurityCheck/SuspiciousCommandPatterns.py +46 -0
- ScriptCollection/LinuxServerSecurityCheck/SystemAccessBase.py +244 -0
- ScriptCollection/LinuxServerSecurityCheck/SystemEntities.py +45 -0
- ScriptCollection/LinuxServerSecurityCheck/__init__.py +0 -0
- ScriptCollection/OCIImages/AbstractImageHandler.py +14 -26
- ScriptCollection/OCIImages/AbstractTagPatternImageHandler.py +46 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py +5 -14
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py +5 -14
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGeneric.py +6 -15
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGenericV.py +6 -15
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabCE.py +7 -8
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabEE.py +7 -8
- ScriptCollection/OCIImages/OCIImageManager.py +73 -68
- ScriptCollection/OCIImages/OCIRegistryClient.py +117 -0
- ScriptCollection/ScriptCollectionCore.py +9 -4
- ScriptCollection/TFCPS/Android/TFCPS_CodeUnitSpecific_Android.py +479 -0
- ScriptCollection/TFCPS/Android/__init__.py +0 -0
- ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py +6 -5
- ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +38 -4
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +4 -4
- ScriptCollection/TFCPS/Maven/TFCPS_CodeUnitSpecific_Maven.py +1 -0
- ScriptCollection/TFCPS/Rust/CargoLockFile.py +151 -0
- ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py +5 -0
- ScriptCollection/TFCPS/SCTaskRunnerServer.py +55 -9
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +6 -2
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +14 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +19 -0
- ScriptCollection/TFCPS/TFCPS_RemoteBuild.py +39 -11
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +34 -2
- {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.73.dist-info}/METADATA +3 -3
- {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.73.dist-info}/RECORD +52 -27
- {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.73.dist-info}/entry_points.txt +1 -0
- {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.73.dist-info}/WHEEL +0 -0
- {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.73.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
import tomllib
|
|
4
|
+
from ...GeneralUtilities import GeneralUtilities
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class CargoLockFile:
|
|
8
|
+
"""Synchronizes the versions of the local packages in a "Cargo.lock"-file with the versions which their
|
|
9
|
+
"Cargo.toml"-files declare.
|
|
10
|
+
|
|
11
|
+
A local package is a package which is not resolved from a registry or a git-repository: the cargo-project itself
|
|
12
|
+
and every path-dependency (directly or transitively). Cargo writes such a package into the lock-file without a
|
|
13
|
+
"source"-entry, together with the version its cargo-file declared at the time the lock-file was written. When the
|
|
14
|
+
version in a cargo-file changes (which happens on every new version of a codeunit), the lock-file is outdated and
|
|
15
|
+
a build with "--locked" fails. Cargo can not update the lock-file without being allowed to resolve the dependencies
|
|
16
|
+
again, so the versions of the local packages are adjusted textually here: everything else in the lock-file stays
|
|
17
|
+
exactly as it is, no network-access is required and running it again does not change anything anymore."""
|
|
18
|
+
|
|
19
|
+
__dependency_sections: tuple = ("dependencies", "dev-dependencies", "build-dependencies")
|
|
20
|
+
# Cargo uses this version for a package whose cargo-file does not declare a version
|
|
21
|
+
# (see https://doc.rust-lang.org/cargo/reference/manifest.html#the-version-field).
|
|
22
|
+
__version_of_package_without_declared_version: str = "0.0.0"
|
|
23
|
+
|
|
24
|
+
@staticmethod
|
|
25
|
+
@GeneralUtilities.check_arguments
|
|
26
|
+
def synchronize_versions_of_local_packages(cargo_file: str) -> None:
|
|
27
|
+
"""Sets the version of every local package in the lock-file next to the given cargo-file to the version
|
|
28
|
+
which the cargo-file of that package declares. A cargo-project without lock-file (for example a library,
|
|
29
|
+
which usually does not commit its lock-file) is left as it is."""
|
|
30
|
+
lock_file = os.path.join(os.path.dirname(cargo_file), "Cargo.lock")
|
|
31
|
+
if os.path.isfile(lock_file):
|
|
32
|
+
CargoLockFile.set_versions_of_local_packages(lock_file, CargoLockFile.get_versions_of_local_packages(cargo_file))
|
|
33
|
+
|
|
34
|
+
@staticmethod
|
|
35
|
+
@GeneralUtilities.check_arguments
|
|
36
|
+
def get_versions_of_local_packages(cargo_file: str) -> dict[str, str]:
|
|
37
|
+
"""Returns the names and the versions of the package of the given cargo-file and of all of its (also
|
|
38
|
+
transitive) path-dependencies, read from their cargo-files."""
|
|
39
|
+
result: dict[str, str] = {}
|
|
40
|
+
CargoLockFile.__add_versions_of_local_packages(os.path.abspath(cargo_file), result, set())
|
|
41
|
+
return result
|
|
42
|
+
|
|
43
|
+
@staticmethod
|
|
44
|
+
@GeneralUtilities.check_arguments
|
|
45
|
+
def __add_versions_of_local_packages(cargo_file: str, result: dict[str, str], visited_cargo_files: set[str]) -> None:
|
|
46
|
+
if cargo_file in visited_cargo_files:
|
|
47
|
+
return
|
|
48
|
+
visited_cargo_files.add(cargo_file)
|
|
49
|
+
GeneralUtilities.assert_file_exists(cargo_file, f'The cargo-file "{cargo_file}" does not exist. If it belongs to a dependent codeunit, then this codeunit must be built before.')
|
|
50
|
+
content: dict = tomllib.loads(GeneralUtilities.read_text_from_file(cargo_file))
|
|
51
|
+
package: dict = content.get("package")
|
|
52
|
+
GeneralUtilities.assert_condition(package is not None, f'The cargo-file "{cargo_file}" does not contain a package-section.')
|
|
53
|
+
version = package.get("version", CargoLockFile.__version_of_package_without_declared_version)
|
|
54
|
+
GeneralUtilities.assert_condition(isinstance(version, str), f'The version of the package in "{cargo_file}" is not declared as plain version-string, which is not supported.')
|
|
55
|
+
result[package["name"]] = version
|
|
56
|
+
cargo_project_folder = os.path.dirname(cargo_file)
|
|
57
|
+
for relative_path in CargoLockFile.__get_paths_of_path_dependencies(content):
|
|
58
|
+
dependency_cargo_file = os.path.abspath(os.path.join(cargo_project_folder, relative_path, "Cargo.toml"))
|
|
59
|
+
CargoLockFile.__add_versions_of_local_packages(dependency_cargo_file, result, visited_cargo_files)
|
|
60
|
+
|
|
61
|
+
@staticmethod
|
|
62
|
+
@GeneralUtilities.check_arguments
|
|
63
|
+
def __get_paths_of_path_dependencies(cargo_file_content: dict) -> list[str]:
|
|
64
|
+
# Dependencies can be declared in the top-level-sections and in the target-specific sections
|
|
65
|
+
# (for example "[target.'cfg(windows)'.dependencies]").
|
|
66
|
+
dependency_tables: list[dict] = [cargo_file_content.get(section, {}) for section in CargoLockFile.__dependency_sections]
|
|
67
|
+
for target_content in cargo_file_content.get("target", {}).values():
|
|
68
|
+
dependency_tables.extend(target_content.get(section, {}) for section in CargoLockFile.__dependency_sections)
|
|
69
|
+
result: list[str] = []
|
|
70
|
+
for dependency_table in dependency_tables:
|
|
71
|
+
for declaration in dependency_table.values():
|
|
72
|
+
if isinstance(declaration, dict) and "path" in declaration:
|
|
73
|
+
result.append(declaration["path"])
|
|
74
|
+
return result
|
|
75
|
+
|
|
76
|
+
@staticmethod
|
|
77
|
+
@GeneralUtilities.check_arguments
|
|
78
|
+
def set_versions_of_local_packages(lock_file: str, versions_of_local_packages: dict[str, str]) -> None:
|
|
79
|
+
"""Sets the version of every package of the lock-file which has no source and whose name is contained in
|
|
80
|
+
the given dictionary to the version stated there. A package of the lock-file which is not contained in the
|
|
81
|
+
dictionary is not changed. The line-endings and all other lines of the lock-file are preserved."""
|
|
82
|
+
lines: list[str] = GeneralUtilities.read_text_from_file(lock_file).splitlines(keepends=True)
|
|
83
|
+
packages = CargoLockFile.__parse_packages(lines)
|
|
84
|
+
changed_references: dict[str, str] = {}
|
|
85
|
+
for package in packages:
|
|
86
|
+
name: str = package["name"]
|
|
87
|
+
if package["has_source"] or name not in versions_of_local_packages or package["version_line_index"] is None:
|
|
88
|
+
continue
|
|
89
|
+
old_version: str = package["version"]
|
|
90
|
+
new_version: str = versions_of_local_packages[name]
|
|
91
|
+
if old_version != new_version:
|
|
92
|
+
line_index: int = package["version_line_index"]
|
|
93
|
+
lines[line_index] = CargoLockFile.__replace_quoted_value(lines[line_index], old_version, new_version)
|
|
94
|
+
# A dependency is referenced as "<name> <version>" instead of "<name>" when the lock-file contains
|
|
95
|
+
# several packages with this name. Such a reference must be changed together with the version. When a
|
|
96
|
+
# package from a registry has the same name and the same old version, then the reference can not be
|
|
97
|
+
# assigned unambiguously to the local package, so it is not changed.
|
|
98
|
+
if not any(other["has_source"] and other["name"] == name and other["version"] == old_version for other in packages):
|
|
99
|
+
changed_references[f"{name} {old_version}"] = f"{name} {new_version}"
|
|
100
|
+
if len(changed_references) > 0:
|
|
101
|
+
for package in packages:
|
|
102
|
+
for line_index in range(package["start_line_index"], package["end_line_index"]):
|
|
103
|
+
lines[line_index] = CargoLockFile.__replace_dependency_reference(lines[line_index], changed_references)
|
|
104
|
+
GeneralUtilities.write_text_to_file(lock_file, "".join(lines))
|
|
105
|
+
|
|
106
|
+
@staticmethod
|
|
107
|
+
@GeneralUtilities.check_arguments
|
|
108
|
+
def __parse_packages(lines: list[str]) -> list[dict]:
|
|
109
|
+
"""Returns the "[[package]]"-entries of the lines of a lock-file with their name, their version, the index of
|
|
110
|
+
the line which contains the version, the information whether they have a source and the range of their
|
|
111
|
+
lines (the end-index is exclusive)."""
|
|
112
|
+
result: list[dict] = []
|
|
113
|
+
current_package: dict = None
|
|
114
|
+
for line_index, line in enumerate(lines):
|
|
115
|
+
stripped_line = line.strip()
|
|
116
|
+
if stripped_line.startswith("["):
|
|
117
|
+
if current_package is not None:
|
|
118
|
+
current_package["end_line_index"] = line_index
|
|
119
|
+
result.append(current_package)
|
|
120
|
+
current_package = None
|
|
121
|
+
if stripped_line == "[[package]]":
|
|
122
|
+
current_package = {"name": None, "version": None, "version_line_index": None, "has_source": False, "start_line_index": line_index}
|
|
123
|
+
continue
|
|
124
|
+
if current_package is None:
|
|
125
|
+
continue
|
|
126
|
+
match = re.match(r'^(name|version|source)\s*=\s*"([^"]*)"$', stripped_line)
|
|
127
|
+
if match is not None:
|
|
128
|
+
key = match.group(1)
|
|
129
|
+
if key == "source":
|
|
130
|
+
current_package["has_source"] = True
|
|
131
|
+
else:
|
|
132
|
+
current_package[key] = match.group(2)
|
|
133
|
+
if key == "version":
|
|
134
|
+
current_package["version_line_index"] = line_index
|
|
135
|
+
if current_package is not None:
|
|
136
|
+
current_package["end_line_index"] = len(lines)
|
|
137
|
+
result.append(current_package)
|
|
138
|
+
return result
|
|
139
|
+
|
|
140
|
+
@staticmethod
|
|
141
|
+
@GeneralUtilities.check_arguments
|
|
142
|
+
def __replace_quoted_value(line: str, old_value: str, new_value: str) -> str:
|
|
143
|
+
return line.replace(f'"{old_value}"', f'"{new_value}"', 1)
|
|
144
|
+
|
|
145
|
+
@staticmethod
|
|
146
|
+
@GeneralUtilities.check_arguments
|
|
147
|
+
def __replace_dependency_reference(line: str, changed_references: dict[str, str]) -> str:
|
|
148
|
+
match = re.match(r'^(\s*)"([^"]+)"(.*)$', line, re.DOTALL)
|
|
149
|
+
if match is not None and match.group(2) in changed_references:
|
|
150
|
+
return f'{match.group(1)}"{changed_references[match.group(2)]}"{match.group(3)}'
|
|
151
|
+
return line
|
|
@@ -6,6 +6,7 @@ from lxml import etree
|
|
|
6
6
|
from ...GeneralUtilities import GeneralUtilities, Dependency
|
|
7
7
|
from ...SCLog import LogLevel
|
|
8
8
|
from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base, TFCPS_CodeUnitSpecific_Base_CLI
|
|
9
|
+
from .CargoLockFile import CargoLockFile
|
|
9
10
|
|
|
10
11
|
class TFCPS_CodeUnitSpecific_Rust_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
11
12
|
"""Implements the tasks of a codeunit which contains a cargo-project.
|
|
@@ -65,6 +66,7 @@ class TFCPS_CodeUnitSpecific_Rust_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
65
66
|
|
|
66
67
|
@GeneralUtilities.check_arguments
|
|
67
68
|
def linting(self) -> None:
|
|
69
|
+
self._protected_sc.normalize_invisible_characters_of_files_in_folder(self.get_codeunit_folder(), ["rs"])
|
|
68
70
|
# "--all-targets" includes the testcases in the check, and "-D warnings" makes clippy exit with a non-zero
|
|
69
71
|
# exit-code for a warning as well, because a warning is a linting-issue too.
|
|
70
72
|
self.__run_cargo("clippy --all-targets --all-features -- -D warnings")
|
|
@@ -73,6 +75,9 @@ class TFCPS_CodeUnitSpecific_Rust_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
73
75
|
def do_common_tasks(self, current_codeunit_version: str) -> None:
|
|
74
76
|
self.do_common_tasks_base(current_codeunit_version)
|
|
75
77
|
self.__set_version_in_cargo_file(self.get_version_of_project())
|
|
78
|
+
# The lock-file contains the versions of the local packages (this project and its path-dependencies, for
|
|
79
|
+
# example the copies of dependent codeunits), so it must be adjusted to the versions of their cargo-files.
|
|
80
|
+
CargoLockFile.synchronize_versions_of_local_packages(self.get_cargo_file())
|
|
76
81
|
|
|
77
82
|
@GeneralUtilities.check_arguments
|
|
78
83
|
def generate_reference(self) -> None:
|
|
@@ -103,8 +103,13 @@ class SCTaskRunnerServer:
|
|
|
103
103
|
self.__sc.log.log(f"SCTaskRunner for operating-system '{self.operating_system_name}' is listening on {protocol}://{host}:{port}.")
|
|
104
104
|
server.serve_forever()
|
|
105
105
|
|
|
106
|
+
@staticmethod
|
|
107
|
+
def is_valid_set_of_environment_variables(environment_variables) -> bool:
|
|
108
|
+
"""Whether the given (deserialized) value is a set of environment-variables a job can be started with."""
|
|
109
|
+
return isinstance(environment_variables, dict) and all(isinstance(name, str) and isinstance(value, str) and len(name) > 0 and "=" not in name for name, value in environment_variables.items())
|
|
110
|
+
|
|
106
111
|
@GeneralUtilities.check_arguments
|
|
107
|
-
def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
112
|
+
def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, environment_variables: dict[str, str]) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
108
113
|
"""Starts a job for the repository-archive which can be read from archive_stream (the body of the request which
|
|
109
114
|
submitted the job; archive_size is its announced length). The archive is taken as a stream instead of as bytes
|
|
110
115
|
because it contains the complete repository of the client and is therefore regularly several hundred megabytes,
|
|
@@ -128,13 +133,15 @@ class SCTaskRunnerServer:
|
|
|
128
133
|
job = _RunnerJob(job_id, workspace_folder, result_folder)
|
|
129
134
|
with self.__jobs_lock:
|
|
130
135
|
self.__jobs[job_id] = job
|
|
131
|
-
thread = threading.Thread(target=self.__extract_payload_and_run_job, args=(job, archive_file, program, arguments, working_directory), daemon=True)
|
|
136
|
+
thread = threading.Thread(target=self.__extract_payload_and_run_job, args=(job, archive_file, program, arguments, working_directory, environment_variables), daemon=True)
|
|
132
137
|
thread.start()
|
|
133
|
-
|
|
138
|
+
#Only the names of the environment-variables are logged: their values are secrets of the client (for example the
|
|
139
|
+
#key which signs an app for a store).
|
|
140
|
+
self.__sc.log.log(f"Accepted job '{job_id}': run '{program} {' '.join(arguments)}' in '{working_directory}' of codeunit '{codeunit_name}', result-folder '{result_folder}', environment-variables [{', '.join(sorted(environment_variables.keys()))}] ({archive_size} transferred bytes of repository-content).")
|
|
134
141
|
return job_id
|
|
135
142
|
|
|
136
143
|
@GeneralUtilities.check_arguments
|
|
137
|
-
def __extract_payload_and_run_job(self, job: _RunnerJob, archive_file: str, program: str, arguments: list[str], working_directory: str) -> None:
|
|
144
|
+
def __extract_payload_and_run_job(self, job: _RunnerJob, archive_file: str, program: str, arguments: list[str], working_directory: str, environment_variables: dict[str, str]) -> None:
|
|
138
145
|
"""Extracts the transferred repository-archive into the workspace of the job and runs the requested program
|
|
139
146
|
afterwards. The extraction happens here - in the thread of the job - and not in the request which submitted the
|
|
140
147
|
job, because extracting the repository of a large codeunit takes minutes during which no data flows: the client
|
|
@@ -151,7 +158,7 @@ class SCTaskRunnerServer:
|
|
|
151
158
|
finally:
|
|
152
159
|
GeneralUtilities.ensure_file_does_not_exist(archive_file)
|
|
153
160
|
self.__sc.log.log(f"Job '{job.job_id}' works on the repository '{self.__get_origin_of_the_transferred_repository(job.workspace_folder)}'.")
|
|
154
|
-
self.__run_job(job, program, arguments, working_directory)
|
|
161
|
+
self.__run_job(job, program, arguments, working_directory, environment_variables)
|
|
155
162
|
|
|
156
163
|
@GeneralUtilities.check_arguments
|
|
157
164
|
def __get_origin_of_the_transferred_repository(self, workspace_folder: str) -> str:
|
|
@@ -204,14 +211,18 @@ class SCTaskRunnerServer:
|
|
|
204
211
|
remaining_amount_of_bytes = remaining_amount_of_bytes-len(chunk)
|
|
205
212
|
|
|
206
213
|
@GeneralUtilities.check_arguments
|
|
207
|
-
def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str) -> None:
|
|
214
|
+
def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str, environment_variables: dict[str, str]) -> None:
|
|
215
|
+
"""Runs the program of the job. The environment-variables of the job are set in addition to the ones of this
|
|
216
|
+
runner and only for this program."""
|
|
208
217
|
command_folder = os.path.join(job.workspace_folder, working_directory)
|
|
209
218
|
try:
|
|
210
219
|
self.__sc.log.log(f"Job '{job.job_id}' runs '{program} {' '.join(arguments)}' in '{command_folder}'...")
|
|
211
220
|
moment_of_the_start = time.time()
|
|
212
221
|
moment_of_the_last_progress_log = moment_of_the_start
|
|
213
222
|
amount_of_lines = 0
|
|
214
|
-
|
|
223
|
+
environment_of_the_program = dict(os.environ)
|
|
224
|
+
environment_of_the_program.update(environment_variables)
|
|
225
|
+
with subprocess.Popen([program] + arguments, cwd=command_folder, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", env=environment_of_the_program) as process:
|
|
215
226
|
for line in process.stdout:
|
|
216
227
|
with job.lock:
|
|
217
228
|
job.log = job.log + line
|
|
@@ -293,6 +304,30 @@ class SCTaskRunnerServer:
|
|
|
293
304
|
raise ValueError(f"The result-folder '{job.result_folder}' of job '{job.job_id}' is not inside the workspace of that job.")
|
|
294
305
|
return result_folder
|
|
295
306
|
|
|
307
|
+
def __get_amount_of_running_jobs(self) -> int:
|
|
308
|
+
"""Returns the amount of jobs which are currently running. A job counts as running from its submit until its
|
|
309
|
+
program has ended, which includes the extraction of its repository."""
|
|
310
|
+
with self.__jobs_lock:
|
|
311
|
+
jobs = list(self.__jobs.values())
|
|
312
|
+
amount_of_running_jobs = 0
|
|
313
|
+
for job in jobs:
|
|
314
|
+
with job.lock:
|
|
315
|
+
if job.state == "running":
|
|
316
|
+
amount_of_running_jobs = amount_of_running_jobs+1
|
|
317
|
+
return amount_of_running_jobs
|
|
318
|
+
|
|
319
|
+
def __get_metrics(self) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
320
|
+
"""Returns the metrics of this runner in the text-based exposition-format of Prometheus, so that the utilization
|
|
321
|
+
of a runner can be monitored by a monitoring-system which scrapes it. Only the amount of running jobs is exposed:
|
|
322
|
+
anything about a job itself (its repository, its program) is information of the client which submitted it."""
|
|
323
|
+
metric_name = "sctaskrunner_running_jobs"
|
|
324
|
+
lines = [
|
|
325
|
+
f"# HELP {metric_name} Amount of jobs which are currently running on this runner.",
|
|
326
|
+
f"# TYPE {metric_name} gauge",
|
|
327
|
+
f"{metric_name} {self.__get_amount_of_running_jobs()}",
|
|
328
|
+
]
|
|
329
|
+
return "\n".join(lines)+"\n"
|
|
330
|
+
|
|
296
331
|
@GeneralUtilities.check_arguments
|
|
297
332
|
def __delete_job(self, job_id: str) -> None: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
298
333
|
with self.__jobs_lock:
|
|
@@ -381,6 +416,9 @@ class SCTaskRunnerServer:
|
|
|
381
416
|
if self.path == "/os":
|
|
382
417
|
self.__send_text(200, outer.operating_system_name)
|
|
383
418
|
return
|
|
419
|
+
if self.path == "/metrics":
|
|
420
|
+
self.__send(200, outer._SCTaskRunnerServer__get_metrics().encode("utf-8"), "text/plain; version=0.0.4; charset=utf-8")
|
|
421
|
+
return
|
|
384
422
|
parts = self.path.strip("/").split("/")
|
|
385
423
|
if len(parts) >= 2 and parts[0] == "jobs":
|
|
386
424
|
self.__handle_job_get(parts)
|
|
@@ -447,9 +485,17 @@ class SCTaskRunnerServer:
|
|
|
447
485
|
arguments = json.loads(base64.b64decode(self.headers.get("X-Arguments")).decode("utf-8"))
|
|
448
486
|
working_directory = self.headers.get("X-Working-Directory", ".")
|
|
449
487
|
result_folder = self.headers.get("X-Result-Folder")
|
|
488
|
+
#A client of a version which does not know environment-variables does not send this header.
|
|
489
|
+
environment_variables_header = self.headers.get("X-Environment-Variables")
|
|
490
|
+
environment_variables = {} if environment_variables_header is None else json.loads(base64.b64decode(environment_variables_header).decode("utf-8"))
|
|
491
|
+
if not SCTaskRunnerServer.is_valid_set_of_environment_variables(environment_variables):
|
|
492
|
+
self.__send_text(400, "The header X-Environment-Variables must contain a json-object whose names and values are texts.")
|
|
493
|
+
return
|
|
450
494
|
#the body is handed over as a stream (and not read here) so the archive is written to disk chunk-wise, see __start_job.
|
|
451
|
-
job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory, result_folder)
|
|
452
|
-
|
|
495
|
+
job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory, result_folder, environment_variables)
|
|
496
|
+
#The names of the applied environment-variables are part of the answer, so that a client can tell this runner
|
|
497
|
+
#from one which does not support environment-variables and would ignore them (see TFCPS_RemoteBuild).
|
|
498
|
+
self.__send_json(200, {"job_id": job_id, "applied_environment_variables": sorted(environment_variables.keys())})
|
|
453
499
|
|
|
454
500
|
def do_DELETE(self): # pylint:disable=invalid-name
|
|
455
501
|
self.__handle(self.__delete)
|
|
@@ -102,6 +102,7 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
|
|
|
102
102
|
if len(dependency_versions)==1:
|
|
103
103
|
GeneralUtilities.write_message_to_stdout("Update dependency "+dependencyname+" (which is currently used in version "+dependency_version+") to version "+desired_version+".")
|
|
104
104
|
self.set_dependency_version(dependencyname,desired_version)
|
|
105
|
+
self.tfcps_Tools_General.log_newer_major_versions_of_dependency(dependencyname,available_versions,desired_version)
|
|
105
106
|
except Exception:
|
|
106
107
|
GeneralUtilities.write_message_to_stderr(f"Error while updating {dependencyname}.")
|
|
107
108
|
raise
|
|
@@ -364,7 +365,7 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
|
|
|
364
365
|
return self.__use_cache
|
|
365
366
|
|
|
366
367
|
@GeneralUtilities.check_arguments
|
|
367
|
-
def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str]) -> None:
|
|
368
|
+
def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str], environment_variables: dict[str, str]) -> None:
|
|
368
369
|
"""Runs the given program for this codeunit on a remote task-runner that provides the given operating-system (see
|
|
369
370
|
TFCPS_RemoteBuild and the SCTaskRunnerWindows/SCTaskRunnerMacOS-codeunits). This is used for operating-system-bound
|
|
370
371
|
build-steps (e.g. flutter-windows- or flutter-ios-builds) which can not run on the current operating-system. After
|
|
@@ -373,8 +374,11 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
|
|
|
373
374
|
working_directory and result_folder must be folders inside this codeunit.
|
|
374
375
|
folders_which_are_not_transferred are the folders of this codeunit whose content only applies to this machine
|
|
375
376
|
and which are therefore left out of the repository-archive the runner gets (see
|
|
377
|
+
TFCPS_RemoteBuild.run_program_on_runner).
|
|
378
|
+
environment_variables are set for the program on the runner only; they are the way to give a secret (for example
|
|
379
|
+
a signing-key) to the build-step without transferring it as part of the repository (see
|
|
376
380
|
TFCPS_RemoteBuild.run_program_on_runner)."""
|
|
377
|
-
TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory, result_folder, folders_which_are_not_transferred)
|
|
381
|
+
TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory, result_folder, folders_which_are_not_transferred, environment_variables)
|
|
378
382
|
|
|
379
383
|
@GeneralUtilities.check_arguments
|
|
380
384
|
def get_codeunit_folder(self)->str:
|
|
@@ -34,6 +34,19 @@ class TFCPS_CodeUnit_BuildCodeUnit:
|
|
|
34
34
|
self.is_pre_merge=is_pre_merge
|
|
35
35
|
self.fast_lane=fast_lane
|
|
36
36
|
|
|
37
|
+
def __check_article_links(self) -> None:
|
|
38
|
+
reference_content_folder: str = os.path.join(self.codeunit_folder, "Other", "Reference", "ReferenceContent")
|
|
39
|
+
articles_folder: str = os.path.join(reference_content_folder, "Articles")
|
|
40
|
+
if not os.path.isdir(articles_folder):
|
|
41
|
+
return
|
|
42
|
+
index_file: str = os.path.join(reference_content_folder, "index.md")
|
|
43
|
+
index_content: str = GeneralUtilities.read_text_from_file(index_file)
|
|
44
|
+
GeneralUtilities.assert_file_exists(index_file)
|
|
45
|
+
for article_file in sorted(os.listdir(articles_folder)):
|
|
46
|
+
if article_file.lower().endswith(".md") and os.path.isfile(os.path.join(articles_folder, article_file)):
|
|
47
|
+
if f"(./Articles/{article_file})" not in index_content:
|
|
48
|
+
self.sc.log.log(f"Article \"{article_file}\" is not linked in \"{index_file}\".", LogLevel.Warning)
|
|
49
|
+
|
|
37
50
|
@GeneralUtilities.check_arguments
|
|
38
51
|
def build_codeunit(self) -> None:
|
|
39
52
|
codeunit_file: str = str(os.path.join(self.codeunit_folder, f"{self.codeunit_name}.codeunit.xml"))
|
|
@@ -77,6 +90,7 @@ class TFCPS_CodeUnit_BuildCodeUnit:
|
|
|
77
90
|
self.sc.log.log(line, LogLevel.Warning)
|
|
78
91
|
for line in GeneralUtilities.string_to_lines(linting_result[2]):
|
|
79
92
|
self.sc.log.log(line, LogLevel.Warning)
|
|
93
|
+
self.__check_article_links()
|
|
80
94
|
|
|
81
95
|
self.sc.log.log("Generate reference...")
|
|
82
96
|
try:
|
|
@@ -441,6 +441,25 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
441
441
|
if os.path.isfile(openspec_configuration_file):
|
|
442
442
|
self.sc.log.log("Update openspec-instruction-files...")
|
|
443
443
|
self.sc.run_with_epew("openspec", "update --force", self.repository)
|
|
444
|
+
self.__replace_openspec_skill_links_by_copies()
|
|
445
|
+
|
|
446
|
+
@GeneralUtilities.check_arguments
|
|
447
|
+
def __replace_openspec_skill_links_by_copies(self) -> None:
|
|
448
|
+
"""openspec (since v2.0.0) writes its skills to '.agents/skills' and links them into the skill-folders of the configured tools
|
|
449
|
+
(for example '.claude/skills') if the operating-system allows to create symbolic links, otherwise it writes copies. openspec has no
|
|
450
|
+
option to always write copies, so the links are replaced by copies here. This way the content of the repository does not depend on
|
|
451
|
+
the operating-system on which the build ran.
|
|
452
|
+
The skill-folders of all tools are located in folders in the root of the repository whose names start with a dot, so only these
|
|
453
|
+
folders are searched (and not for example the codeunit-folders, which may contain large folders like 'node_modules').
|
|
454
|
+
Submodules are not changed, because their content belongs to another repository."""
|
|
455
|
+
shared_skills_folder: str = os.path.join(self.repository, ".agents", "skills")
|
|
456
|
+
if os.path.isdir(shared_skills_folder):
|
|
457
|
+
submodule_folders: list[str] = [os.path.join(self.repository, submodule) for submodule in self.sc.get_git_submodules(self.repository)]
|
|
458
|
+
for name in os.listdir(self.repository):
|
|
459
|
+
tool_folder: str = os.path.join(self.repository, name)
|
|
460
|
+
if name.startswith(".") and name not in [".git", ".agents"] and os.path.isdir(tool_folder) and not os.path.islink(tool_folder):
|
|
461
|
+
for replaced_link in GeneralUtilities.replace_symlinks_by_copies(tool_folder, shared_skills_folder, submodule_folders):
|
|
462
|
+
self.sc.log.log(f"Replaced symbolic link \"{replaced_link}\" by a copy of its target.", LogLevel.Debug)
|
|
444
463
|
|
|
445
464
|
@GeneralUtilities.check_arguments
|
|
446
465
|
def __validate_openspec(self,repository:str) -> None:
|
|
@@ -93,14 +93,20 @@ class TFCPS_RemoteBuild:
|
|
|
93
93
|
return len(self.__load_runner_endpoints()) > 0
|
|
94
94
|
|
|
95
95
|
@GeneralUtilities.check_arguments
|
|
96
|
-
def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str], poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
|
|
96
|
+
def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str], environment_variables: dict[str, str], poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
|
|
97
97
|
"""Runs the given program on a runner which provides the given operating-system, see the docstring of this class.
|
|
98
98
|
|
|
99
99
|
'folders_which_are_not_transferred' are folders (as paths inside the repository) which are left out of the
|
|
100
100
|
archive which is sent to the runner, because their content only applies to this machine (for example the
|
|
101
101
|
'.dart_tool'-folder of a flutter-package, whose 'package_config.json' states the folder of every package of
|
|
102
102
|
the app as an absolute path of this machine). Such a folder is generated again by the toolchain of the
|
|
103
|
-
runner while it builds; transferring it makes the runner use the paths of this machine instead.
|
|
103
|
+
runner while it builds; transferring it makes the runner use the paths of this machine instead.
|
|
104
|
+
|
|
105
|
+
'environment_variables' are set (in addition to the ones of the runner) for the program on the runner only. They
|
|
106
|
+
are the way to give a secret to a build-step (for example the key which signs an app for a store): they are
|
|
107
|
+
neither part of the transferred repository nor of the result nor of a log - the runner logs only their names.
|
|
108
|
+
A runner which does not support them is refused if environment-variables are given (see
|
|
109
|
+
assert_runner_applied_environment_variables)."""
|
|
104
110
|
endpoints = self.__load_runner_endpoints()
|
|
105
111
|
if len(endpoints) == 0:
|
|
106
112
|
raise ValueError("No remote-build-runner is configured. Define runners either in "
|
|
@@ -115,18 +121,13 @@ class TFCPS_RemoteBuild:
|
|
|
115
121
|
archive_file = self.__create_repository_archive(repository_folder, folders_which_are_not_transferred_relative)
|
|
116
122
|
job_id: str = None
|
|
117
123
|
try:
|
|
118
|
-
metadata_headers =
|
|
119
|
-
"Content-Type": "application/octet-stream",
|
|
120
|
-
"X-Codeunit-Name": codeunit_name,
|
|
121
|
-
"X-Program": program,
|
|
122
|
-
"X-Arguments": base64.b64encode(json.dumps(arguments).encode("utf-8")).decode("ascii"),
|
|
123
|
-
"X-Working-Directory": working_directory_relative,
|
|
124
|
-
"X-Result-Folder": result_folder_relative,
|
|
125
|
-
}
|
|
124
|
+
metadata_headers = TFCPS_RemoteBuild.get_headers_of_job_submission(codeunit_name, program, arguments, working_directory_relative, result_folder_relative, environment_variables)
|
|
126
125
|
with open(archive_file, "rb") as archive_content:
|
|
127
126
|
archive_bytes = archive_content.read()
|
|
128
127
|
_, submit_response = self.__http(endpoint, "POST", "/jobs", _timeout_of_a_request_in_seconds, metadata_headers, archive_bytes)
|
|
129
|
-
|
|
128
|
+
submit_answer = json.loads(submit_response.decode("utf-8"))
|
|
129
|
+
job_id = submit_answer["job_id"]
|
|
130
|
+
TFCPS_RemoteBuild.assert_runner_applied_environment_variables(submit_answer, environment_variables, endpoint.url)
|
|
130
131
|
self.__wait_for_job(endpoint, job_id, required_os, poll_interval_in_seconds, timeout_in_seconds)
|
|
131
132
|
#The result-archive is packed completely before it is sent, which transports nothing while it runs - hence
|
|
132
133
|
#the timeout of a long-running request here.
|
|
@@ -141,6 +142,33 @@ class TFCPS_RemoteBuild:
|
|
|
141
142
|
self.__sc.log.log_exception(f"Could not delete remote job '{job_id}' on runner '{endpoint.url}'.", exception, LogLevel.Warning)
|
|
142
143
|
GeneralUtilities.ensure_file_does_not_exist(archive_file)
|
|
143
144
|
|
|
145
|
+
@staticmethod
|
|
146
|
+
@GeneralUtilities.check_arguments
|
|
147
|
+
def get_headers_of_job_submission(codeunit_name: str, program: str, arguments: list[str], working_directory_relative: str, result_folder_relative: str, environment_variables: dict[str, str]) -> dict[str, str]:
|
|
148
|
+
"""Returns the headers of the request which submits a job (its body is the repository-archive)."""
|
|
149
|
+
return {
|
|
150
|
+
"Content-Type": "application/octet-stream",
|
|
151
|
+
"X-Codeunit-Name": codeunit_name,
|
|
152
|
+
"X-Program": program,
|
|
153
|
+
"X-Arguments": base64.b64encode(json.dumps(arguments).encode("utf-8")).decode("ascii"),
|
|
154
|
+
"X-Working-Directory": working_directory_relative,
|
|
155
|
+
"X-Result-Folder": result_folder_relative,
|
|
156
|
+
"X-Environment-Variables": base64.b64encode(json.dumps(environment_variables).encode("utf-8")).decode("ascii"),
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
@staticmethod
|
|
160
|
+
@GeneralUtilities.check_arguments
|
|
161
|
+
def assert_runner_applied_environment_variables(submit_answer: dict, environment_variables: dict[str, str], runner_url: str) -> None:
|
|
162
|
+
"""A runner states in the answer of a submit the names of the environment-variables it sets for the job. A runner
|
|
163
|
+
which does not support environment-variables ignores them without saying so, and the job would then run without
|
|
164
|
+
them (for example an app which is meant to be signed for a store would be built unsigned). That is refused here.
|
|
165
|
+
A job without environment-variables runs on such a runner as before."""
|
|
166
|
+
if len(environment_variables) == 0:
|
|
167
|
+
return
|
|
168
|
+
applied_environment_variables = submit_answer.get("applied_environment_variables")
|
|
169
|
+
GeneralUtilities.assert_condition(applied_environment_variables is not None, f"The runner '{runner_url}' does not support environment-variables for a job, so it would run the job without them. Update the runner to a version of ScriptCollection which supports them.")
|
|
170
|
+
GeneralUtilities.assert_condition(sorted(applied_environment_variables) == sorted(environment_variables.keys()), f"The runner '{runner_url}' did not apply exactly the requested environment-variables (requested: {', '.join(sorted(environment_variables.keys()))}; applied: {', '.join(sorted(applied_environment_variables))}).")
|
|
171
|
+
|
|
144
172
|
@GeneralUtilities.check_arguments
|
|
145
173
|
def __wait_for_job(self, endpoint: RunnerEndpoint, job_id: str, required_os: RunnerOperatingSystem, poll_interval_in_seconds: int, timeout_in_seconds: int) -> None:
|
|
146
174
|
start_time = time.time()
|
|
@@ -16,6 +16,7 @@ import urllib.request
|
|
|
16
16
|
import xml.etree.ElementTree as ET
|
|
17
17
|
from packaging import version
|
|
18
18
|
import requests
|
|
19
|
+
import yaml
|
|
19
20
|
from lxml import etree
|
|
20
21
|
from ..GeneralUtilities import GeneralUtilities,Platform
|
|
21
22
|
from ..ScriptCollectionCore import ScriptCollectionCore,VSCodeWorkspaceShellTask
|
|
@@ -1109,11 +1110,29 @@ class TFCPS_Tools_General:
|
|
|
1109
1110
|
result["MacOS"]=download_and_extract("MacOS", "darwin", "tar.gz")
|
|
1110
1111
|
return result
|
|
1111
1112
|
|
|
1113
|
+
@GeneralUtilities.check_arguments
|
|
1114
|
+
def __check_for_tasks_which_will_be_overwritten(self, repository_folder: str) -> None:
|
|
1115
|
+
"""Asserts that there is no task in the existing taskfile which would be lost when the taskfile is re-generated from the code-workspace-file."""
|
|
1116
|
+
task_file: str = os.path.join(repository_folder, "Taskfile.yml")
|
|
1117
|
+
if not os.path.isfile(task_file):
|
|
1118
|
+
return
|
|
1119
|
+
task_file_content = yaml.safe_load(GeneralUtilities.read_text_from_file(task_file))
|
|
1120
|
+
if not isinstance(task_file_content, dict):
|
|
1121
|
+
return
|
|
1122
|
+
existing_tasks = task_file_content.get("tasks")
|
|
1123
|
+
if not isinstance(existing_tasks, dict):
|
|
1124
|
+
return
|
|
1125
|
+
workspace_file: str = self.__sc.find_file_by_extension(repository_folder, "code-workspace")
|
|
1126
|
+
task_labels_in_workspace_file: set[str] = set(task.label for task in self.__sc.parse_tasks_from_codeworkspace_file(workspace_file))
|
|
1127
|
+
for task_name in existing_tasks.keys():
|
|
1128
|
+
GeneralUtilities.assert_condition(str(task_name) in task_labels_in_workspace_file, f"The task \"{task_name}\" in the taskfile \"{task_file}\" is not defined in the code-workspace-file \"{workspace_file}\" and would be deleted when re-generating the taskfile based on the code-workspace file. Add the task to the code-workspace file to prevent this error.")
|
|
1129
|
+
|
|
1112
1130
|
@GeneralUtilities.check_arguments
|
|
1113
1131
|
def generate_tasksfile_from_workspace_file(self, repository_folder: str, append_cli_args_at_end: bool = False) -> None:
|
|
1114
1132
|
"""This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
|
|
1115
1133
|
if self.__sc.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
|
|
1116
1134
|
self.__sc.log.log("Generate taskfile from code-workspace-file...")
|
|
1135
|
+
self.__check_for_tasks_which_will_be_overwritten(repository_folder)
|
|
1117
1136
|
self.__sc.assert_is_git_repository(repository_folder)
|
|
1118
1137
|
workspace_file: str = self.__sc.find_file_by_extension(repository_folder, "code-workspace")
|
|
1119
1138
|
task_file: str = repository_folder + "/Taskfile.yml"
|
|
@@ -2091,11 +2110,11 @@ class TFCPS_Tools_General:
|
|
|
2091
2110
|
changelog_file = os.path.join(repository_folder, "Other", "Resources", "Changelog", f"v{current_version}.md")
|
|
2092
2111
|
if (not os.path.isfile(changelog_file)):
|
|
2093
2112
|
GeneralUtilities.ensure_file_exists(changelog_file)
|
|
2094
|
-
GeneralUtilities.write_text_to_file(changelog_file, """# Release notes
|
|
2113
|
+
GeneralUtilities.write_text_to_file(changelog_file, f"""# Release notes
|
|
2095
2114
|
|
|
2096
2115
|
## Changes
|
|
2097
2116
|
|
|
2098
|
-
- Updated
|
|
2117
|
+
- Updated submodule {submodule_name}.
|
|
2099
2118
|
""")
|
|
2100
2119
|
|
|
2101
2120
|
@GeneralUtilities.check_arguments
|
|
@@ -2111,6 +2130,19 @@ class TFCPS_Tools_General:
|
|
|
2111
2130
|
if current_version != latest_version:
|
|
2112
2131
|
GeneralUtilities.write_text_to_file(resrepo_commit_id_file, latest_version)
|
|
2113
2132
|
|
|
2133
|
+
@GeneralUtilities.check_arguments
|
|
2134
|
+
def log_newer_major_versions_of_dependency(self, dependency_name: str, available_versions: list[str], used_version: str) -> None:
|
|
2135
|
+
"""Logs if a newer major-version of the given dependency is available than the used one.
|
|
2136
|
+
An update with an echolon like LatestPatchOrLatestMinor does not take over a new major-version, so without this message it would not be visible that the dependency falls behind.
|
|
2137
|
+
Being 1 major-version behind is logged as information, being 2 or more major-versions behind is logged as warning.
|
|
2138
|
+
used_version is the version which is used after the update (if an update was done)."""
|
|
2139
|
+
amount_of_major_versions_behind: int = GeneralUtilities.get_amount_of_major_versions_behind(available_versions, used_version)
|
|
2140
|
+
if amount_of_major_versions_behind == 0:
|
|
2141
|
+
return
|
|
2142
|
+
loglevel: LogLevel = LogLevel.Information if amount_of_major_versions_behind == 1 else LogLevel.Warning
|
|
2143
|
+
latest_version: str = GeneralUtilities.get_latest_version(available_versions)
|
|
2144
|
+
self.__sc.log.log(f"Dependency {dependency_name} is used in version {used_version}, which is {amount_of_major_versions_behind} major-version(s) behind the latest available version {latest_version}. A new major-version has to be taken over deliberately.", loglevel)
|
|
2145
|
+
|
|
2114
2146
|
@GeneralUtilities.check_arguments
|
|
2115
2147
|
def get_dependencies_which_are_ignored_from_updates(self, codeunit_folder: str) -> list[str]:
|
|
2116
2148
|
self.assert_is_codeunit_folder(codeunit_folder)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scriptcollection
|
|
3
|
-
Version: 4.4.
|
|
3
|
+
Version: 4.4.73
|
|
4
4
|
Summary: The ScriptCollection is the place for reusable scripts.
|
|
5
5
|
Author-email: Marius Göcke <marius.goecke@gmail.com>
|
|
6
6
|
License-Expression: GPL-3.0-only
|
|
@@ -35,7 +35,7 @@ Requires-Dist: Pillow>=11.3.0
|
|
|
35
35
|
Requires-Dist: psutil>=7.2.2
|
|
36
36
|
Requires-Dist: pycdlib>=1.22.0
|
|
37
37
|
Requires-Dist: Pygments>=2.21.0
|
|
38
|
-
Requires-Dist: pylint>=4.1.
|
|
38
|
+
Requires-Dist: pylint>=4.1.4
|
|
39
39
|
Requires-Dist: pyOpenSSL>=26.4.0
|
|
40
40
|
Requires-Dist: PyPDF>=6.21.1
|
|
41
41
|
Requires-Dist: pytest>=8.4.2
|
|
@@ -155,7 +155,7 @@ Afterwards `task dl` installs the built wheel locally.
|
|
|
155
155
|
In this repository [pylint](https://pylint.org/) will be used to report linting-issues.
|
|
156
156
|
If you change code in this repository please ensure pylint does not find any issues before creating a pull-request.
|
|
157
157
|
|
|
158
|
-
|
|
158
|
+
Whether linting-issues exist in the current code-base can be checked by running `python ./ScriptCollection/Other/QualityCheck/Linting.py`.
|
|
159
159
|
|
|
160
160
|
## Runtime dependencies
|
|
161
161
|
|