ScriptCollection 4.4.20__py3-none-any.whl → 4.4.22__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -364,13 +364,14 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
364
364
  return self.__use_cache
365
365
 
366
366
  @GeneralUtilities.check_arguments
367
- def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str) -> None:
367
+ def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str, result_folder: str) -> None:
368
368
  """Runs the given program for this codeunit on a remote task-runner that provides the given operating-system (see
369
369
  TFCPS_RemoteBuild and the SCTaskRunnerWindows/SCTaskRunnerMacOS-codeunits). This is used for operating-system-bound
370
370
  build-steps (e.g. flutter-windows- or flutter-ios-builds) which can not run on the current operating-system. After
371
- the runner finished, only the codeunit-folder is mirrored back into the local repository, so it looks as if the
372
- runner had built locally. working_directory must be a folder inside this codeunit."""
373
- TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory)
371
+ the runner finished, the content of result_folder is written back into the local repository, so the build-step can
372
+ continue with the result as if it had been produced here; nothing else of the local repository is touched.
373
+ working_directory and result_folder must be folders inside this codeunit."""
374
+ TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory, result_folder)
374
375
 
375
376
  @GeneralUtilities.check_arguments
376
377
  def get_codeunit_folder(self)->str:
@@ -4,7 +4,6 @@ import re
4
4
  import socket
5
5
  from datetime import datetime, timedelta,timezone
6
6
  import xmlschema
7
- import yaml
8
7
  from packaging.version import Version
9
8
  from ..GeneralUtilities import GeneralUtilities
10
9
  from ..ScriptCollectionCore import ScriptCollectionCore
@@ -35,7 +34,7 @@ class TFCPS_UpdateDependenciesHook(TFCPS_BuildCodeUnitsHook):
35
34
  update_dependencies_script_folder: str = os.path.join(repository, "Other", "Scripts")
36
35
  if os.path.isfile(os.path.join(update_dependencies_script_folder, "UpdateDependencies.py")):
37
36
  self.__sc.log.log("Update dependencies of the repository...")
38
- self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", update_dependencies_script_folder)
37
+ self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", update_dependencies_script_folder,print_live_output=True)
39
38
  self.__ensure_changelog_is_there(repository)
40
39
 
41
40
  @GeneralUtilities.check_arguments
@@ -55,7 +54,7 @@ class TFCPS_UpdateDependenciesHook(TFCPS_BuildCodeUnitsHook):
55
54
  if not self.__tfcps_tools_general.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml")):
56
55
  return
57
56
  self.__sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
58
- self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", os.path.join(codeunit_folder, "Other"))
57
+ self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", os.path.join(codeunit_folder, "Other"),print_live_output=True)
59
58
  self.__ensure_changelog_is_there(repository)
60
59
  if self.__sc.git_repository_has_uncommitted_changes(repository):
61
60
  #the update changed something, so it has to be verified that the codeunit is still buildable with the updated dependencies.
@@ -264,7 +263,7 @@ class TFCPS_CodeUnit_BuildCodeUnits:
264
263
  self.sc.format_json_file(workspace_file)
265
264
 
266
265
  #TODO add option to define exceptions (means: files which should not be normalized).
267
- self.sc.normalize_invisible_characters_of_files_in_folder(self.repository, ["txt", "md", "py", "json", "xml", "svg", "csv", "yml", "yaml", "toml","gitignore", "gitattributes", "code-workspace"])
266
+ self.sc.normalize_invisible_characters_of_files_in_folder(self.repository, ["txt", "md", "py", "json", "xml", "svg", "csv", "yml", "yaml", "toml","gitignore", "gitattributes", "code-workspace", "plantuml"])
268
267
 
269
268
  @GeneralUtilities.check_arguments
270
269
  def is_working_branch(self)->bool:
@@ -7,11 +7,32 @@ import base64
7
7
  import tarfile
8
8
  import urllib.request
9
9
  import urllib.error
10
+ import http.client
10
11
  from enum import Enum
11
12
  from ..GeneralUtilities import GeneralUtilities
12
13
  from ..ScriptCollectionCore import ScriptCollectionCore
13
14
  from ..SCLog import LogLevel
14
15
 
16
+ #Duration for which the client keeps asking a runner which does not answer before it states that the delegated build has
17
+ #no result here. A job runs on the runner independently of the connection to the client, so an interruption of that
18
+ #connection (for example a restart of a reverse-proxy in front of the runner) does not stop the build - giving up
19
+ #immediately would discard a build which is still producing a result. A longer interruption is not bridged, so that a
20
+ #build always ends with a statement instead of with an endless wait.
21
+ _tolerated_interruption_of_the_connection_to_the_runner_in_seconds: int = 300
22
+
23
+ #Timeout of a request whose answer the runner produces without doing anything long-running for it (the state of a job,
24
+ #its log, its deletion, the operating-system a runner provides). It is the timeout of a single operation on the
25
+ #connection and not the duration of the whole request, so it is the time after which a connection which does not
26
+ #transport anything any more counts as interrupted.
27
+ _timeout_of_a_request_in_seconds: int = 300
28
+
29
+ #Timeout of a request whose answer the runner produces only after something long-running which transports nothing while
30
+ #it runs: the result-archive of a job contains the complete build-output of a codeunit and is packed completely before
31
+ #its first byte is sent, which takes minutes for a large build-output. This has to stay above the read-timeout of a
32
+ #reverse-proxy in front of a runner, so that its gateway-timeout - a defined answer - arrives instead of the client
33
+ #giving up on the connection first.
34
+ _timeout_of_a_long_running_request_in_seconds: int = 60*60
35
+
15
36
 
16
37
  class RunnerOperatingSystem(Enum):
17
38
  """Operating-system a remote-build-runner provides. The value is the token exchanged over the wire (see the runner's
@@ -34,12 +55,28 @@ class RunnerEndpoint:
34
55
  self.password = password
35
56
 
36
57
 
58
+ class RunnerRequestFailedError(ValueError):
59
+ """A runner answered a request with an error-status. The request reached the runner, so this is a defined answer of
60
+ the runner and not a problem of the connection to it (see RunnerNotReachableError for that)."""
61
+
62
+ def __init__(self, message: str, status: int):
63
+ super().__init__(message)
64
+ self.status = status
65
+
66
+
67
+ class RunnerNotReachableError(ValueError):
68
+ """A request did not reach a runner, or its answer did not reach the client, for example because the connection was
69
+ reset or because the host is not resolvable. In contrast to RunnerRequestFailedError this states nothing about the
70
+ state of a job on the runner: a job which was started there before keeps running."""
71
+
72
+
37
73
  class TFCPS_RemoteBuild:
38
74
  """Delegates an operating-system-bound build-step to a remote task-runner (see SCTaskRunnerWindows/SCTaskRunnerMacOS):
39
75
  the whole repository (including the .git-folder, uncommitted changes and git-ignored files) is sent as a tar-archive
40
76
  over HTTPS to the runner, the runner runs the given program on its (correct) operating-system, and afterwards only the
41
- codeunit-folder is mirrored back into the local repository (a build-script must by definition not change anything
42
- outside its codeunit-folder, so mirroring just that folder is sufficient and keeps the round-trip small)."""
77
+ folder in which that build-step produces its result is written back into the local repository. Everything else which
78
+ changed in the workspace of the runner stays there: it is state of that machine (its toolchain-paths, its caches) and
79
+ not a result, and a local repository is something a developer works in and which therefore must not be replaced."""
43
80
 
44
81
  __sc: ScriptCollectionCore = None
45
82
 
@@ -52,7 +89,7 @@ class TFCPS_RemoteBuild:
52
89
  return len(self.__load_runner_endpoints()) > 0
53
90
 
54
91
  @GeneralUtilities.check_arguments
55
- def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
92
+ def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
56
93
  endpoints = self.__load_runner_endpoints()
57
94
  if len(endpoints) == 0:
58
95
  raise ValueError("No remote-build-runner is configured. Define runners either in "
@@ -61,7 +98,8 @@ class TFCPS_RemoteBuild:
61
98
  "'Runner_<name>_URL', 'Runner_<name>_Username' and 'Runner_<name>_Password'.")
62
99
  endpoint = self.__select_runner_for_os(endpoints, required_os)
63
100
  working_directory_relative = os.path.relpath(working_directory, repository_folder).replace("\\", "/")
64
- self.__sc.log.log(f"Delegate '{program} {' '.join(arguments)}' (codeunit '{codeunit_name}', folder '{working_directory_relative}') to the {required_os.value}-runner at {endpoint.url}...")
101
+ result_folder_relative = os.path.relpath(result_folder, repository_folder).replace("\\", "/")
102
+ self.__sc.log.log(f"Delegate '{program} {' '.join(arguments)}' (codeunit '{codeunit_name}', folder '{working_directory_relative}', result-folder '{result_folder_relative}') to the {required_os.value}-runner at {endpoint.url}...")
65
103
  archive_file = self.__create_repository_archive(repository_folder)
66
104
  job_id: str = None
67
105
  try:
@@ -71,19 +109,22 @@ class TFCPS_RemoteBuild:
71
109
  "X-Program": program,
72
110
  "X-Arguments": base64.b64encode(json.dumps(arguments).encode("utf-8")).decode("ascii"),
73
111
  "X-Working-Directory": working_directory_relative,
112
+ "X-Result-Folder": result_folder_relative,
74
113
  }
75
114
  with open(archive_file, "rb") as archive_content:
76
115
  archive_bytes = archive_content.read()
77
- _, submit_response = self.__http(endpoint, "POST", "/jobs", metadata_headers, archive_bytes)
116
+ _, submit_response = self.__http(endpoint, "POST", "/jobs", _timeout_of_a_request_in_seconds, metadata_headers, archive_bytes)
78
117
  job_id = json.loads(submit_response.decode("utf-8"))["job_id"]
79
118
  self.__wait_for_job(endpoint, job_id, required_os, poll_interval_in_seconds, timeout_in_seconds)
80
- _, result_bytes = self.__http(endpoint, "GET", f"/jobs/{job_id}/result")
81
- self.__mirror_codeunit_folder_from_result(result_bytes, repository_folder, codeunit_name)
119
+ #The result-archive is packed completely before it is sent, which transports nothing while it runs - hence
120
+ #the timeout of a long-running request here.
121
+ _, result_bytes = self.__http_of_running_job(endpoint, "GET", f"/jobs/{job_id}/result", _timeout_of_a_long_running_request_in_seconds, poll_interval_in_seconds)
122
+ self.__write_result_into_the_repository(result_bytes, result_folder)
82
123
  finally:
83
124
  if job_id is not None:
84
125
  try:
85
126
  # Triggers the runner to delete its (isolated) workspace for this job immediately after the result was fetched.
86
- self.__http(endpoint, "DELETE", f"/jobs/{job_id}")
127
+ self.__http(endpoint, "DELETE", f"/jobs/{job_id}", _timeout_of_a_request_in_seconds)
87
128
  except Exception as exception:
88
129
  self.__sc.log.log_exception(f"Could not delete remote job '{job_id}' on runner '{endpoint.url}'.", exception, LogLevel.Warning)
89
130
  GeneralUtilities.ensure_file_does_not_exist(archive_file)
@@ -93,9 +134,8 @@ class TFCPS_RemoteBuild:
93
134
  start_time = time.time()
94
135
  already_printed_log_length = 0
95
136
  while True:
96
- _, status_bytes = self.__http(endpoint, "GET", f"/jobs/{job_id}")
97
- status = json.loads(status_bytes.decode("utf-8"))
98
- _, log_bytes = self.__http(endpoint, "GET", f"/jobs/{job_id}/logs")
137
+ status = self.__get_job_status(endpoint, job_id, required_os, poll_interval_in_seconds)
138
+ _, log_bytes = self.__http_of_running_job(endpoint, "GET", f"/jobs/{job_id}/logs", _timeout_of_a_request_in_seconds, poll_interval_in_seconds)
99
139
  log_text = log_bytes.decode("utf-8", errors="replace")
100
140
  if len(log_text) > already_printed_log_length:
101
141
  for line in GeneralUtilities.string_to_lines(log_text[already_printed_log_length:]):
@@ -111,6 +151,43 @@ class TFCPS_RemoteBuild:
111
151
  raise ValueError(f"The remote build on the {required_os.value}-runner did not finish within {timeout_in_seconds} seconds.")
112
152
  time.sleep(poll_interval_in_seconds)
113
153
 
154
+ @GeneralUtilities.check_arguments
155
+ def __get_job_status(self, endpoint: RunnerEndpoint, job_id: str, required_os: RunnerOperatingSystem, poll_interval_in_seconds: int) -> dict:
156
+ """Asks the runner for the state of the job. A runner which does not know the job (any more) gets an own message:
157
+ a runner holds its jobs in memory only, so that is what a restart of the runner while a job was running looks
158
+ like from here, and the status-code alone does not state that the delegated build is gone with it."""
159
+ try:
160
+ _, status_bytes = self.__http_of_running_job(endpoint, "GET", f"/jobs/{job_id}", _timeout_of_a_request_in_seconds, poll_interval_in_seconds)
161
+ except RunnerRequestFailedError as request_failed_error:
162
+ if request_failed_error.status == 404:
163
+ raise ValueError(f"The {required_os.value}-runner at '{endpoint.url}' does not know the job '{job_id}' (any more), so the delegated build has no result. A runner holds its jobs in memory only: when it is restarted while a job runs, the job and its build are gone with it.") from None
164
+ raise
165
+ return json.loads(status_bytes.decode("utf-8"))
166
+
167
+ @GeneralUtilities.check_arguments
168
+ def __http_of_running_job(self, endpoint: RunnerEndpoint, method: str, path: str, timeout_in_seconds: int, retry_interval_in_seconds: int) -> tuple[int, bytes]:
169
+ """Does a request which concerns a job that already exists on the runner and which can therefore be repeated
170
+ without side-effects (it neither starts a job twice nor changes one). The job keeps running on the runner while
171
+ the connection to it is interrupted, so an interruption which is over within
172
+ _tolerated_interruption_of_the_connection_to_the_runner_in_seconds is bridged here by repeating the request
173
+ instead of giving up a build which still produces a result. A longer interruption ends in a
174
+ RunnerNotReachableError which states that the result of the build is unknown here. An answer of the runner -
175
+ an error-status included - is not an interruption and is returned respectively raised immediately."""
176
+ moment_of_the_first_unreachable_request: float = None
177
+ while True:
178
+ try:
179
+ result = self.__http(endpoint, method, path, timeout_in_seconds)
180
+ if moment_of_the_first_unreachable_request is not None:
181
+ self.__sc.log.log(f"The runner '{endpoint.url}' is reachable again.", LogLevel.Information)
182
+ return result
183
+ except RunnerNotReachableError as not_reachable_error:
184
+ if moment_of_the_first_unreachable_request is None:
185
+ moment_of_the_first_unreachable_request = time.time()
186
+ self.__sc.log.log(f"{not_reachable_error} The job runs on the runner independently of this connection, so the request is repeated for at most {_tolerated_interruption_of_the_connection_to_the_runner_in_seconds} seconds.", LogLevel.Warning)
187
+ if time.time()-moment_of_the_first_unreachable_request > _tolerated_interruption_of_the_connection_to_the_runner_in_seconds:
188
+ raise RunnerNotReachableError(f"The runner '{endpoint.url}' was not reachable for {_tolerated_interruption_of_the_connection_to_the_runner_in_seconds} seconds while the request '{method} {path}' was repeated (last reason: {not_reachable_error}). The delegated build therefore has no result here; it may still be running on the runner.") from None
189
+ time.sleep(retry_interval_in_seconds)
190
+
114
191
  @GeneralUtilities.check_arguments
115
192
  def __create_repository_archive(self, repository_folder: str) -> str:
116
193
  # Pack the entire repository-working-tree (including .git, uncommitted changes and git-ignored files) so the runner
@@ -122,18 +199,23 @@ class TFCPS_RemoteBuild:
122
199
  return archive_file
123
200
 
124
201
  @GeneralUtilities.check_arguments
125
- def __mirror_codeunit_folder_from_result(self, result_bytes: bytes, repository_folder: str, codeunit_name: str) -> None:
126
- # Mirror the codeunit-folder from the runner-result into the local repository: the local codeunit-folder is replaced
127
- # entirely by the runner's post-build codeunit-folder. This makes it look exactly as if the runner had built locally,
128
- # including files the runner added, changed or deleted (regardless of whether they are git-ignored).
129
- local_codeunit_folder = os.path.join(repository_folder, codeunit_name)
202
+ def __write_result_into_the_repository(self, result_bytes: bytes, result_folder: str) -> None:
203
+ """Writes what the runner produced into the folder of the local repository in which the build-step produces its
204
+ result, so the build can continue with it as if it had been produced here.
205
+
206
+ Nothing is deleted for this. Before, the whole codeunit-folder was replaced by the one of the runner, which
207
+ deleted everything of that codeunit which is not part of the answer of the runner - including files which are
208
+ not in git and can therefore not be restored - and which on Windows can not even complete: the build-script of
209
+ the codeunit runs in a folder below the codeunit, and a folder a process runs in can not be removed there, so
210
+ the deletion failed in the middle and left the codeunit incomplete. It is also not what is wanted: what changes
211
+ in the workspace of the runner besides the result is state of that machine (its toolchain-paths, its caches),
212
+ which has no business in the repository of a developer."""
130
213
  result_archive = os.path.join(GeneralUtilities.get_temp_folder(), f"sc-remotebuild-result-{uuid.uuid4()}.tar.gz")
131
214
  try:
132
215
  GeneralUtilities.write_binary_to_file(result_archive, result_bytes)
133
- GeneralUtilities.ensure_directory_does_not_exist(local_codeunit_folder)
134
- GeneralUtilities.ensure_directory_exists(local_codeunit_folder)
216
+ GeneralUtilities.ensure_directory_exists(result_folder)
135
217
  with tarfile.open(result_archive, "r:gz") as tar:
136
- tar.extractall(local_codeunit_folder, filter="fully_trusted")
218
+ tar.extractall(result_folder, filter="fully_trusted")
137
219
  finally:
138
220
  GeneralUtilities.ensure_file_does_not_exist(result_archive)
139
221
 
@@ -172,17 +254,26 @@ class TFCPS_RemoteBuild:
172
254
  def __select_runner_for_os(self, endpoints: list[RunnerEndpoint], required_os: RunnerOperatingSystem) -> RunnerEndpoint:
173
255
  # The configuration does not state which runner provides which operating-system, so each configured runner is asked
174
256
  # (via its "GET /os"-endpoint) and the first one matching the required operating-system is used.
257
+ runners_which_could_not_be_reached: list[str] = []
175
258
  for endpoint in endpoints:
176
259
  try:
177
- _, os_bytes = self.__http(endpoint, "GET", "/os")
260
+ _, os_bytes = self.__http(endpoint, "GET", "/os", _timeout_of_a_request_in_seconds)
178
261
  if os_bytes.decode("utf-8").strip() == required_os.value:
179
262
  return endpoint
263
+ except RunnerNotReachableError as not_reachable_error:
264
+ #A runner which can not be reached is something else than one which does not provide the required
265
+ #operating-system: it is configured and may well be the right one, which is unknown while it does not
266
+ #answer. It is therefore collected here and stated as the reason if no runner can be selected at all.
267
+ runners_which_could_not_be_reached.append(str(not_reachable_error))
268
+ self.__sc.log.log(str(not_reachable_error), LogLevel.Warning)
180
269
  except Exception as exception:
181
270
  self.__sc.log.log_exception(f"Could not query the operating-system of the runner '{endpoint.url}'.", exception, LogLevel.Warning)
271
+ if len(runners_which_could_not_be_reached) == len(endpoints):
272
+ raise ValueError(f"None of the configured remote-build-runners could be reached, so whether one of them provides the required operating-system '{required_os.value}' is unknown: {' '.join(runners_which_could_not_be_reached)}")
182
273
  raise ValueError(f"No configured remote-build-runner provides the required operating-system '{required_os.value}'.")
183
274
 
184
275
  @GeneralUtilities.check_arguments
185
- def __http(self, endpoint: RunnerEndpoint, method: str, path: str, extra_headers: dict = None, body: bytes = None) -> tuple[int, bytes]:
276
+ def __http(self, endpoint: RunnerEndpoint, method: str, path: str, timeout_in_seconds: int, extra_headers: dict = None, body: bytes = None) -> tuple[int, bytes]:
186
277
  request = urllib.request.Request(endpoint.url + path, data=body, method=method)
187
278
  authorization_token = base64.b64encode(f"{endpoint.username}:{endpoint.password}".encode("utf-8")).decode("ascii")
188
279
  request.add_header("Authorization", f"Basic {authorization_token}")
@@ -190,8 +281,15 @@ class TFCPS_RemoteBuild:
190
281
  for header_name, header_value in extra_headers.items():
191
282
  request.add_header(header_name, header_value)
192
283
  try:
193
- with urllib.request.urlopen(request, timeout=300) as response:
284
+ with urllib.request.urlopen(request, timeout=timeout_in_seconds) as response:
194
285
  return response.status, response.read()
195
286
  except urllib.error.HTTPError as http_error:
196
287
  error_body = http_error.read().decode("utf-8", errors="replace")
197
- raise ValueError(f"The runner-request '{method} {path}' failed with status {http_error.code}: {error_body}") from http_error
288
+ raise RunnerRequestFailedError(f"The runner-request '{method} {path}' failed with status {http_error.code}: {error_body}", http_error.code) from http_error
289
+ except (OSError, http.client.HTTPException) as connection_error:
290
+ #Everything which prevented an answer of the runner: urllib.error.URLError (which is an OSError) covers a
291
+ #connection which was refused, reset or not establishable at all, http.client.HTTPException covers an answer
292
+ #which was cut off while it was read. Both are stated as one error which names what was requested from whom.
293
+ #The cause is deliberately not chained: its message is part of the message here, while its traceback consists
294
+ #of the internals of urllib only and states nothing about the delegated build.
295
+ raise RunnerNotReachableError(f"The runner '{endpoint.url}' could not be reached for the request '{method} {path}': {connection_error}") from None
@@ -45,15 +45,8 @@ class TFCPS_Tools_General:
45
45
  # Font-files (all faces of the diagram-font) bundled with the package and forced via the JVM-font-path when rendering.
46
46
  __bundled_diagram_font_filenames:list[str]=["DejaVuSans.ttf","DejaVuSans-Bold.ttf","DejaVuSans-Oblique.ttf","DejaVuSans-BoldOblique.ttf"]
47
47
  # Relative path (in the source-repository and in the ScriptCollection.Resources-package) of the file that pins the
48
- # default Eclipse-Temurin-JRE-version used to render the PlantUML-diagrams when a repository does not pin its own
49
- # version in '<repo>/Other/Resources/Dependencies/JRE/Version.txt'. The file contains a build like "21.0.6+7" (the
50
- # feature-version is derived from it). Pinning the exact JDK-build (not just the major-version) makes the AWT-font-
51
- # metrics - and thus the computed SVG-geometry - byte-identical across machines: PlantUML computes the SVG purely as
52
- # vector-math from the font-metrics, and different JDK-builds (e.g. a Windows-client vs. the Debian-build-container)
53
- # round these metrics slightly differently. Together with the bundled DejaVu-font (see __default_diagram_font_name)
54
- # this makes the rendered SVG fully deterministic. The PlantUML-rendering uses this pinned JRE instead of the host's
55
- # 'java' on the PATH. When the pinned version is bumped, the committed diagram-SVGs must be regenerated and re-committed
56
- # once so they match the new build's metrics.
48
+ # default Eclipse-Temurin-JRE-version used by ensure_jre_is_available/download_jre when a repository does not pin its
49
+ # own version in '<repo>/Other/Resources/Dependencies/JRE/Version.txt'. The file contains a build like "21.0.6+7".
57
50
  __jre_version_resource_relative_path:str="Dependencies/JRE/Version.txt"
58
51
 
59
52
  def __init__(self,sc:ScriptCollectionCore):
@@ -71,6 +64,8 @@ class TFCPS_Tools_General:
71
64
  local_resource_name="CycloneDXCLI"
72
65
  self.ensure_file_from_github_assets_is_available_with_retry("CycloneDX", "cyclonedx-cli", local_resource_name,"cyclonedx-linux-arm64",lambda latest_version: "cyclonedx-linux-arm64",enforce_update=enforce_update)
73
66
  self.ensure_file_from_github_assets_is_available_with_retry("CycloneDX", "cyclonedx-cli", local_resource_name,"cyclonedx-linux-x64",lambda latest_version: "cyclonedx-linux-x64",enforce_update=enforce_update)
67
+ self.ensure_file_from_github_assets_is_available_with_retry("CycloneDX", "cyclonedx-cli", local_resource_name,"cyclonedx-osx-arm64",lambda latest_version: "cyclonedx-osx-arm64",enforce_update=enforce_update)
68
+ self.ensure_file_from_github_assets_is_available_with_retry("CycloneDX", "cyclonedx-cli", local_resource_name,"cyclonedx-osx-x64",lambda latest_version: "cyclonedx-osx-x64",enforce_update=enforce_update)
74
69
  self.ensure_file_from_github_assets_is_available_with_retry("CycloneDX", "cyclonedx-cli", local_resource_name,"cyclonedx-win-arm64.exe",lambda latest_version: "cyclonedx-win-arm64.exe",enforce_update=enforce_update)
75
70
  self.ensure_file_from_github_assets_is_available_with_retry("CycloneDX", "cyclonedx-cli", local_resource_name, "cyclonedx-win-x64.exe",lambda latest_version: "cyclonedx-win-x64.exe",enforce_update=enforce_update)
76
71
 
@@ -92,6 +87,13 @@ class TFCPS_Tools_General:
92
87
  return os.path.join(resource_folder, "cyclonedx-linux-arm64")
93
88
  else:
94
89
  raise ValueError("Unsupported architecture for cyclonedx-cli on linux.")
90
+ elif GeneralUtilities.current_system_is_macos():
91
+ if is_x64:
92
+ return os.path.join(resource_folder, "cyclonedx-osx-x64")
93
+ elif is_arm:
94
+ return os.path.join(resource_folder, "cyclonedx-osx-arm64")
95
+ else:
96
+ raise ValueError("Unsupported architecture for cyclonedx-cli on macos.")
95
97
  else:
96
98
  raise ValueError("Unsupported operating system for cyclonedx-cli.")
97
99
 
@@ -129,8 +131,11 @@ class TFCPS_Tools_General:
129
131
  @GeneralUtilities.check_arguments
130
132
  def download_jre(self, enforce_update: bool = False) -> None:
131
133
  self.__sc.log.log("Download cachable tool \"JRE\" into global cache...", LogLevel.Debug)
132
- # Only the executing platform is warmed (unlike e.g. MediaMTX): a JDK-archive is large (~200MB), and the
133
- # build-image is built per-platform, so warming foreign platforms would only bloat the image without benefit.
134
+ # No repository-context is available when warming the global cache (e.g. scdownloadcachabletools in a
135
+ # build-image), so download the default version pinned by the ScriptCollection-package. Actual builds
136
+ # pass the version pinned in their own '<repo>/Other/Resources/Dependencies/JRE/Version.txt'. This is
137
+ # independent of the Java used to render PlantUML-diagrams (see __run_plantuml_in_linux_container), which
138
+ # always runs inside a Docker-container instead of this cached JRE.
134
139
  self.ensure_jre_is_available(enforce_update)
135
140
 
136
141
  @GeneralUtilities.check_arguments
@@ -799,23 +804,13 @@ class TFCPS_Tools_General:
799
804
  self.__sc.assert_is_git_repository(repositoryfolder)
800
805
  return self.__sc.get_semver_version(repositoryfolder)
801
806
 
802
- @GeneralUtilities.check_arguments
803
- def __try_calculate_changelog_message(self, repositoryfolder: str):
804
- self.__sc.assert_is_git_repository(repositoryfolder)
805
- message = self.__sc.run_program("git", "log -1 --pretty=%B", repositoryfolder)[1]
806
- message = message.strip()
807
- if len(message) == 0:
808
- raise ValueError("No commit message found.")
809
- return message
807
+
810
808
 
811
809
  @GeneralUtilities.check_arguments
812
810
  def create_changelog_entry(self, repositoryfolder: str, message: str, commit: bool, force: bool):
813
811
  self.__sc.assert_is_git_repository(repositoryfolder)
814
812
  if message is None:
815
- try:
816
- message=self.__try_calculate_changelog_message(repositoryfolder)
817
- except:
818
- message="Update."
813
+ message="Update."
819
814
  random_file = os.path.join(repositoryfolder, str(uuid.uuid4()))
820
815
  try:
821
816
  if force and not self.__sc.git_repository_has_uncommitted_changes(repositoryfolder):
@@ -891,18 +886,26 @@ class TFCPS_Tools_General:
891
886
  def generate_svg_files_from_plantuml_files_for_repository(self, repository_folder: str,use_cache:bool) -> None:
892
887
  self.__sc.log.log("Generate svg-files from plantuml-files...")
893
888
  self.__sc.assert_is_git_repository(repository_folder)
889
+ # If the currently checked-out commit is itself tagged with a release-version ('v<semver>') and the working-tree
890
+ # is clean, the committed diagram-SVGs were generated for exactly this release-state and must not change; a
891
+ # difference would then indicate a rendering-regression (e.g. a font/JRE-mismatch, see
892
+ # __generate_svg_files_from_plantuml) instead of an expected content-change and must fail loudly.
893
+ current_commit_tags = [tag for tag in GeneralUtilities.string_to_lines(self.__sc.run_program_argsasarray("git", ["tag", "--points-at", "HEAD"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)[1]) if GeneralUtilities.string_has_content(tag)]
894
+ current_commit_has_release_tag = any(re.match(r"^v\d+\.\d+\.\d+$", tag) for tag in current_commit_tags)
895
+ assert_remains_unchanged: bool = current_commit_has_release_tag and not self.__sc.git_repository_has_uncommitted_changes(repository_folder)
894
896
  plantuml_jar_file=self.ensure_plantuml_is_available(not use_cache, self.get_plantuml_version(repository_folder))
895
- java_executable=self.ensure_jre_is_available(not use_cache, repository_folder)
896
897
  target_folder = os.path.join(repository_folder, "Other", "Reference")
897
- self.__generate_svg_files_from_plantuml(target_folder, plantuml_jar_file, java_executable)
898
+ self.__generate_svg_files_from_plantuml(target_folder, plantuml_jar_file, repository_folder)
899
+ if assert_remains_unchanged:
900
+ GeneralUtilities.assert_condition(not self.__sc.git_repository_has_uncommitted_changes(repository_folder), f"Generating svg-files from plantuml-files in '{repository_folder}' changed the repository unexpectedly.")
898
901
 
899
902
  @GeneralUtilities.check_arguments
900
903
  def generate_svg_files_from_plantuml_files_for_codeunit(self, codeunit_folder: str,use_cache:bool) -> None:
901
904
  self.assert_is_codeunit_folder(codeunit_folder)
902
- plantuml_jar_file=self.ensure_plantuml_is_available(not use_cache, self.get_plantuml_version(os.path.dirname(codeunit_folder)))
903
- java_executable=self.ensure_jre_is_available(not use_cache, os.path.dirname(codeunit_folder))
905
+ repository_folder = os.path.dirname(codeunit_folder)
906
+ plantuml_jar_file=self.ensure_plantuml_is_available(not use_cache, self.get_plantuml_version(repository_folder))
904
907
  target_folder = os.path.join(codeunit_folder, "Other", "Reference")
905
- self.__generate_svg_files_from_plantuml(target_folder, plantuml_jar_file, java_executable)
908
+ self.__generate_svg_files_from_plantuml(target_folder, plantuml_jar_file, repository_folder)
906
909
 
907
910
  @GeneralUtilities.check_arguments
908
911
  def generate_svg_files_from_vega_files_for_repository(self, repository_folder: str,use_cache:bool) -> None:
@@ -957,26 +960,10 @@ class TFCPS_Tools_General:
957
960
  """Returns the PlantUML-version used when no repository-specific version is pinned (for example when warming
958
961
  the global cache via scdownloadcachabletools). The value is read from the version-file bundled with the
959
962
  ScriptCollection-package, with a fallback to the source-file for an unbuilt source-checkout."""
960
- try:
961
- content = GeneralUtilities._internal_load_resource(TFCPS_Tools_General.__plantuml_version_resource_relative_path)
962
- return content.decode("utf-8").strip()
963
- except (FileNotFoundError, ModuleNotFoundError):
964
- source_version_file = GeneralUtilities.resolve_relative_path(f"../../../Other/Resources/{TFCPS_Tools_General.__plantuml_version_resource_relative_path}", os.path.dirname(__file__))
965
- GeneralUtilities.assert_file_exists(source_version_file)
966
- return GeneralUtilities.read_text_from_file(source_version_file).strip()
967
963
 
968
- @GeneralUtilities.check_arguments
969
- def get_default_jre_version(self) -> str:
970
- """Returns the JRE-version used when a repository does not pin its own version (see
971
- __jre_version_resource_relative_path). The value is read from the version-file bundled with the
972
- ScriptCollection-package, with a fallback to the source-file for an unbuilt source-checkout."""
973
- try:
974
- content = GeneralUtilities._internal_load_resource(TFCPS_Tools_General.__jre_version_resource_relative_path)
975
- return content.decode("utf-8").strip()
976
- except (FileNotFoundError, ModuleNotFoundError):
977
- source_version_file = GeneralUtilities.resolve_relative_path(f"../../../Other/Resources/{TFCPS_Tools_General.__jre_version_resource_relative_path}", os.path.dirname(__file__))
978
- GeneralUtilities.assert_file_exists(source_version_file)
979
- return GeneralUtilities.read_text_from_file(source_version_file).strip()
964
+ content = GeneralUtilities._internal_load_resource(TFCPS_Tools_General.__plantuml_version_resource_relative_path)
965
+ return content.decode("utf-8").strip()
966
+
980
967
 
981
968
  @GeneralUtilities.check_arguments
982
969
  def get_jre_version(self, repository_folder: str = None) -> str:
@@ -988,11 +975,20 @@ class TFCPS_Tools_General:
988
975
  return GeneralUtilities.read_text_from_file(version_file).strip()
989
976
  return self.get_default_jre_version()
990
977
 
978
+ @GeneralUtilities.check_arguments
979
+ def get_default_jre_version(self) -> str:
980
+ """Returns the JRE-version used when no repository-specific version is pinned (for example when warming
981
+ the global cache via scdownloadcachabletools). The value is read from the version-file bundled with the
982
+ ScriptCollection-package."""
983
+ content = GeneralUtilities._internal_load_resource(TFCPS_Tools_General.__jre_version_resource_relative_path)
984
+ return content.decode("utf-8").strip()
985
+
991
986
  @GeneralUtilities.check_arguments
992
987
  def ensure_jre_is_available(self, enforce_update: bool, repository_folder: str = None) -> str:
993
988
  """Ensures the pinned JDK is available in the global cache for the executing platform and returns the absolute path
994
989
  to its 'java'-executable. The version is the repository-specific pin if present, otherwise the bundled default (see
995
- get_jre_version). Pinning the exact build makes the PlantUML-rendering deterministic across machines."""
990
+ get_jre_version). Used by tools which need a locally-runnable, deterministically-versioned JRE (the PlantUML-diagram
991
+ rendering itself now always runs inside a Docker-container instead, see __run_plantuml_in_linux_container)."""
996
992
  local_resource_name = "JRE"
997
993
  jre_version = self.get_jre_version(repository_folder)
998
994
  feature_version = jre_version.split(".")[0] # e.g. "21" from "21.0.6+7"
@@ -1050,7 +1046,7 @@ class TFCPS_Tools_General:
1050
1046
  return java_executable
1051
1047
 
1052
1048
  @GeneralUtilities.check_arguments
1053
- def __generate_svg_files_from_plantuml(self, diagrams_files_folder: str, plantuml_jar_file: str, java_executable: str) -> None:
1049
+ def __generate_svg_files_from_plantuml(self, diagrams_files_folder: str, plantuml_jar_file: str, repository_folder: str) -> None:
1054
1050
  # Pin the font for all rendered diagrams via a PlantUML-config-file so the resulting SVG is identical across
1055
1051
  # machines, without having to add a skinparam to every (also hand-written) .plantuml-file. See the comment at
1056
1052
  # __default_diagram_font_name for the reason.
@@ -1060,17 +1056,22 @@ class TFCPS_Tools_General:
1060
1056
  # font-version happens to be installed on the host. This makes the text-measurement - and thus the resulting
1061
1057
  # SVG-geometry - identical across machines (e.g. a Windows-client and the Debian-build-container), because the
1062
1058
  # exact same font-files are used everywhere.
1063
- java_font_arguments: list[str] = []
1064
1059
  bundled_fonts_folder = self.__extract_bundled_fonts_to_temp_folder()
1065
- if bundled_fonts_folder is not None:
1066
- java_font_arguments.append(f"-Dsun.java2d.fontpath=prepend:{bundled_fonts_folder}")
1060
+ # Pinning the JDK-build and the font is sufficient to get byte-identical SVGs on Windows and Linux, because both
1061
+ # use the same FreeType-based font-rasterizer internally, so the AWT-font-metrics (in particular the vertical
1062
+ # ascent/descent used e.g. for the note-box-positioning) round identically there. macOS-builds of the JDK use
1063
+ # Apple's native CoreText font-manager instead, which rounds those vertical metrics slightly differently even
1064
+ # for the exact same pinned font-file and JDK-build, shifting positions in the rendered SVG by a fraction of a
1065
+ # pixel. To still get an identical result on macOS, the rendering itself does not run on a native macOS-JVM
1066
+ # there but inside a short-lived Docker-container that runs the Java-image the repository itself declares in
1067
+ # '.ScriptCollection/OCIImages/ImageDefinition.csv' (image-name 'Java', see __run_plantuml_in_linux_container) -
1068
+ # i.e. the diagram is always rendered by a Linux-JVM, regardless of host.
1067
1069
  try:
1068
1070
  for file in GeneralUtilities.get_all_files_of_folder(diagrams_files_folder):
1069
1071
  if file.endswith(".plantuml"):
1070
1072
  output_filename = self.get_output_filename_for_plantuml_filename(file)
1071
- argument = java_font_arguments + ['-jar',plantuml_jar_file, '-tsvg', '-config', font_config_file, os.path.basename(file)]
1072
1073
  folder = os.path.dirname(file)
1073
- self.__sc.run_program_argsasarray(java_executable, argument, folder)
1074
+ self.__run_plantuml_in_linux_container(folder, os.path.basename(file), plantuml_jar_file, font_config_file, bundled_fonts_folder, repository_folder)
1074
1075
  result_file = folder+"/" + output_filename
1075
1076
  GeneralUtilities.assert_file_exists(result_file)
1076
1077
  self.__sc.format_xml_file(result_file)
@@ -1079,6 +1080,28 @@ class TFCPS_Tools_General:
1079
1080
  if bundled_fonts_folder is not None:
1080
1081
  GeneralUtilities.ensure_directory_does_not_exist(bundled_fonts_folder)
1081
1082
 
1083
+ @GeneralUtilities.check_arguments
1084
+ def __run_plantuml_in_linux_container(self, folder: str, plantuml_filename: str, plantuml_jar_file: str, font_config_file: str, bundled_fonts_folder: str, repository_folder: str) -> None:
1085
+ """Renders a single .plantuml-file into a svg-file by running PlantUML inside a short-lived, disposable Docker-
1086
+ container. All diagram-rendering happens this way, regardless of the host-platform (see __generate_svg_files_from_plantuml
1087
+ for why this is required to get byte-identical results on Windows/Linux/macOS). The image is not hardcoded here: every
1088
+ repository must declare it itself as the image named 'Java' in '<repository>/.ScriptCollection/OCIImages/ImageDefinition.csv'
1089
+ (the same mechanism used for e.g. 'Betterleaks'/'SCBuilder', see __search_for_secrets_in_repository/build_codeunits_in_container
1090
+ in TFCPS_CodeUnit_BuildCodeUnits), so a repository can pin the exact JDK-build it wants rendered with. The bundled font and
1091
+ the font-config are mounted into the container; no network-access is required inside the container for this."""
1092
+ docker_image = self.oci_image_manager.get_registry_address_for_image_with_default_tag(repository_folder, "Java")
1093
+ container_jar_file = "/opt/plantuml.jar"
1094
+ container_font_config_file = "/opt/font-config.cfg"
1095
+ container_fonts_folder = "/opt/fonts"
1096
+ container_work_folder = "/work"
1097
+ docker_arguments = ["run", "--rm", "-v", f"{plantuml_jar_file}:{container_jar_file}:ro", "-v", f"{font_config_file}:{container_font_config_file}:ro", "-v", f"{folder}:{container_work_folder}", "-w", container_work_folder]
1098
+ java_font_arguments: list[str] = []
1099
+ if bundled_fonts_folder is not None:
1100
+ docker_arguments = docker_arguments+["-v", f"{bundled_fonts_folder}:{container_fonts_folder}:ro"]
1101
+ java_font_arguments = [f"-Dsun.java2d.fontpath=prepend:{container_fonts_folder}"]
1102
+ docker_arguments = docker_arguments+[docker_image, "java"]+java_font_arguments+["-jar", container_jar_file, "-tsvg", "-config", container_font_config_file, plantuml_filename]
1103
+ self.__sc.run_program_argsasarray("docker", docker_arguments)
1104
+
1082
1105
  @GeneralUtilities.check_arguments
1083
1106
  def __extract_bundled_fonts_to_temp_folder(self) -> str:
1084
1107
  # Extracts the DejaVu-fonts bundled with the ScriptCollection-package into a temporary folder and returns its path
@@ -1801,16 +1824,8 @@ class TFCPS_Tools_General:
1801
1824
  (for example when warming the global cache via scdownloadcachabletools). The value is read from
1802
1825
  '<repo>/Other/Resources/Dependencies/OpenAPIGenerator/Version.txt', which the build copies into the
1803
1826
  ScriptCollection.Resources-package so it is available at runtime from the installed wheel."""
1804
- try:
1805
- # Default case: read the version-file that ships inside the installed package (wheel, build-image, ...).
1806
- content = GeneralUtilities._internal_load_resource(TFCPS_Tools_General.__openapigenerator_version_resource_relative_path)
1807
- return content.decode("utf-8").strip()
1808
- except (FileNotFoundError, ModuleNotFoundError):
1809
- # Fallback for running from an unbuilt source-checkout where the file has not been copied into the
1810
- # package-resources yet: read the human-editable source-file directly from the repository.
1811
- source_version_file = GeneralUtilities.resolve_relative_path(f"../../../Other/Resources/{TFCPS_Tools_General.__openapigenerator_version_resource_relative_path}", os.path.dirname(__file__))
1812
- GeneralUtilities.assert_file_exists(source_version_file)
1813
- return GeneralUtilities.read_text_from_file(source_version_file).strip()
1827
+ content = GeneralUtilities._internal_load_resource(TFCPS_Tools_General.__openapigenerator_version_resource_relative_path)
1828
+ return content.decode("utf-8").strip()
1814
1829
 
1815
1830
  @GeneralUtilities.check_arguments
1816
1831
  def update_images_in_example_with_default_excluded(self, codeunit_folder: str,custom_updater:AbstractImageHandler):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptcollection
3
- Version: 4.4.20
3
+ Version: 4.4.22
4
4
  Summary: The ScriptCollection is the place for reusable scripts.
5
5
  Author-email: Marius Göcke <marius.goecke@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/anionDev/ScriptCollection