ScriptCollection 4.4.27__py3-none-any.whl → 4.4.29__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/OCIImages/AbstractImageHandler.py +2 -1
- ScriptCollection/OCIImages/OCIImageManager.py +63 -20
- ScriptCollection/ScriptCollectionCore.py +165 -63
- ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py +3 -1
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +14 -5
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +7 -4
- ScriptCollection/TFCPS/TFCPS_RemoteBuild.py +35 -5
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +8 -4
- {scriptcollection-4.4.27.dist-info → scriptcollection-4.4.29.dist-info}/METADATA +1 -1
- {scriptcollection-4.4.27.dist-info → scriptcollection-4.4.29.dist-info}/RECORD +13 -13
- {scriptcollection-4.4.27.dist-info → scriptcollection-4.4.29.dist-info}/WHEEL +0 -0
- {scriptcollection-4.4.27.dist-info → scriptcollection-4.4.29.dist-info}/entry_points.txt +0 -0
- {scriptcollection-4.4.27.dist-info → scriptcollection-4.4.29.dist-info}/top_level.txt +0 -0
|
@@ -10,7 +10,8 @@ class AbstractImageHandler(ABC):
|
|
|
10
10
|
|
|
11
11
|
def _protected_get_credentials_for_registry(self,registry_address:str,username_str:str)->tuple[str,str]:
|
|
12
12
|
"""return (username, password) for basic auth.
|
|
13
|
-
Data will be taken from "~/.ScriptCollection/GlobalCache/RegistryCredentials.csv"
|
|
13
|
+
Data will be taken from "~/.ScriptCollection/GlobalCache/RegistryCredentials.csv" or from the registries which are declared in the
|
|
14
|
+
environment (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) if available.
|
|
14
15
|
If no credentials are available then None will be returned for the missing values."""
|
|
15
16
|
raise NotImplementedError()
|
|
16
17
|
|
|
@@ -20,12 +20,16 @@ class OCIImageManager:
|
|
|
20
20
|
#image-names for which it was already logged that the fallback-registry is used. The lookup happens for every image of
|
|
21
21
|
#the repository and often several times per build, so without this the same hint would flood the log.
|
|
22
22
|
__images_with_reported_fallback_registry:set[str] = None
|
|
23
|
+
#availability of an image-address with tag in its registry. The address of an image is resolved several times per build and
|
|
24
|
+
#the check costs a request to the registry, so its result is remembered for the runtime of this process.
|
|
25
|
+
__availability_of_image:dict[str,bool] = None
|
|
23
26
|
|
|
24
27
|
def __init__(self,sc:ScriptCollectionCore):
|
|
25
28
|
if sc is None:
|
|
26
29
|
sc=ScriptCollectionCore()
|
|
27
30
|
self.__sc=sc
|
|
28
31
|
self.__images_with_reported_fallback_registry=set()
|
|
32
|
+
self.__availability_of_image=dict()
|
|
29
33
|
self.image_handler=[
|
|
30
34
|
ImageHandlerDebian(),
|
|
31
35
|
ImageHandlerDebianSlim(),
|
|
@@ -105,34 +109,70 @@ class OCIImageManager:
|
|
|
105
109
|
return line[2]
|
|
106
110
|
raise ValueError(f"No tag defined for image \"{image_name}\".")
|
|
107
111
|
|
|
112
|
+
@GeneralUtilities.check_arguments
|
|
113
|
+
def get_custom_registry_address_for_image(self,image_name:str)->str:
|
|
114
|
+
"""Returns the registry-address which is defined for the image in the machine-wide image-registries-file, always without tag.
|
|
115
|
+
Use custom_registry_is_defined to check whether such an address is defined for the image at all."""
|
|
116
|
+
global_docker_image_registries_file=self.get_global_docker_image_registries_file()
|
|
117
|
+
for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(global_docker_image_registries_file)[1:]]:
|
|
118
|
+
if image_name==line[0]:
|
|
119
|
+
return line[1]
|
|
120
|
+
raise ValueError(f"No custom registry is defined for image \"{image_name}\" in \"{global_docker_image_registries_file}\".")
|
|
121
|
+
|
|
122
|
+
@GeneralUtilities.check_arguments
|
|
123
|
+
def get_fallback_registry_address_for_image(self,repository:str,image_name:str)->str:
|
|
124
|
+
"""Returns the upstream-registry-address which the repository defines for the image, always without tag."""
|
|
125
|
+
repository_image_definition_file=self.get_repository_image_definition_file(repository)
|
|
126
|
+
for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(repository_image_definition_file)[1:]]:
|
|
127
|
+
if image_name==line[0]:
|
|
128
|
+
return line[1]
|
|
129
|
+
raise ValueError(f"No registry defined for image \"{image_name}\" in \"{repository_image_definition_file}\".")
|
|
130
|
+
|
|
108
131
|
@GeneralUtilities.check_arguments
|
|
109
132
|
def get_registry_address_for_image(self,repository:str,image_name:str)->str:
|
|
110
|
-
"""
|
|
133
|
+
"""Returns the registry-address the image has to be taken from, always without tag.
|
|
134
|
+
Example: if image_name==Debian this function returns something like "myregistry.example.com/debian".
|
|
135
|
+
This is the custom registry-address which is defined for this machine if the image is really available there, and the
|
|
136
|
+
upstream-registry-address which the repository defines otherwise."""
|
|
137
|
+
fallback_registry_address=self.get_fallback_registry_address_for_image(repository,image_name)
|
|
111
138
|
if self.custom_registry_is_defined(image_name):
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
return line[1]
|
|
139
|
+
custom_registry_address=self.get_custom_registry_address_for_image(image_name)
|
|
140
|
+
if self.__custom_registry_provides_image(custom_registry_address,self.get_tag_for_image(repository,image_name)):
|
|
141
|
+
return custom_registry_address
|
|
142
|
+
self.__report_usage_of_fallback_registry(image_name,fallback_registry_address,f"the image is not available in the custom registry \"{custom_registry_address}\"")
|
|
117
143
|
else:
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
144
|
+
self.__report_usage_of_fallback_registry(image_name,fallback_registry_address,f"no custom registry is defined for the image in \"{self.get_global_docker_image_registries_file()}\"")
|
|
145
|
+
return fallback_registry_address
|
|
146
|
+
|
|
147
|
+
@GeneralUtilities.check_arguments
|
|
148
|
+
def __custom_registry_provides_image(self,custom_registry_address:str,tag:str)->bool:
|
|
149
|
+
"""Returns whether the image can be used with this tag from the custom registry-address, which is the case when it is already
|
|
150
|
+
available in the local image-store of the docker-daemon or when the custom registry provides it.
|
|
151
|
+
The local image-store is checked first because an image which is already available there is not downloaded at all (see
|
|
152
|
+
ScriptCollectionCore.docker_pull), so asking the registry would only cost a request - and would make a build which does not need
|
|
153
|
+
any download depend on a reachable registry.
|
|
154
|
+
A custom registry is typically not publicly readable, so the login to the registries which are defined for this machine is done
|
|
155
|
+
before the registry is asked; without it the answer would be that the image is not available although it only is not readable
|
|
156
|
+
without credentials."""
|
|
157
|
+
image_with_tag=f"{custom_registry_address}:{tag}"
|
|
158
|
+
if image_with_tag not in self.__availability_of_image:
|
|
159
|
+
if self.__sc.local_docker_image_exists(custom_registry_address,tag):
|
|
160
|
+
self.__availability_of_image[image_with_tag]=True
|
|
161
|
+
else:
|
|
162
|
+
self.__sc.login_to_defined_docker_registries()
|
|
163
|
+
self.__availability_of_image[image_with_tag]=self.__sc.image_is_available_in_registry(custom_registry_address,tag)
|
|
164
|
+
return self.__availability_of_image[image_with_tag]
|
|
125
165
|
|
|
126
166
|
@GeneralUtilities.check_arguments
|
|
127
|
-
def __report_usage_of_fallback_registry(self,image_name:str,fallback_registry_address:str)->None:
|
|
128
|
-
"""Logs that the image is taken from the upstream-registry
|
|
129
|
-
The fallback exists so that a freshly cloned product is buildable without further setup
|
|
130
|
-
rate-limited, so using it
|
|
167
|
+
def __report_usage_of_fallback_registry(self,image_name:str,fallback_registry_address:str,reason:str)->None:
|
|
168
|
+
"""Logs that the image is taken from the upstream-registry which the repository defines, together with the reason why the custom
|
|
169
|
+
registry is not used. The fallback exists so that a freshly cloned product is buildable without further setup and so that a custom
|
|
170
|
+
registry which does not provide the image does not break a build, but an upstream-registry is typically rate-limited, so using it
|
|
171
|
+
is worth a hint."""
|
|
131
172
|
if image_name in self.__images_with_reported_fallback_registry:
|
|
132
173
|
return
|
|
133
174
|
self.__images_with_reported_fallback_registry.add(image_name)
|
|
134
|
-
|
|
135
|
-
self.__sc.log.log(f"No custom registry is defined for image \"{image_name}\" in \"{registries_file}\", so the fallback-registry \"{fallback_registry_address}\" is used.",LogLevel.Warning)
|
|
175
|
+
self.__sc.log.log(f"The fallback-registry \"{fallback_registry_address}\" is used for image \"{image_name}\" because {reason}.",LogLevel.Warning)
|
|
136
176
|
|
|
137
177
|
@GeneralUtilities.check_arguments
|
|
138
178
|
def get_registry_address_for_image_with_default_tag(self,repository:str,image_name:str)->str:
|
|
@@ -159,7 +199,10 @@ class OCIImageManager:
|
|
|
159
199
|
try:
|
|
160
200
|
addresses_to_check=[]
|
|
161
201
|
if self.custom_registry_is_defined(image_name):
|
|
162
|
-
|
|
202
|
+
#the custom registry-address is used directly (and not get_registry_address_for_image) because this function does not
|
|
203
|
+
#pull an image but asks the registry which tags it has, so it does not matter whether that registry currently provides
|
|
204
|
+
#the image with the tag which is defined at the moment.
|
|
205
|
+
addresses_to_check.append(self.get_custom_registry_address_for_image(image_name))
|
|
163
206
|
else:
|
|
164
207
|
if search_in_custom_registry_only_if_available:
|
|
165
208
|
raise ValueError(f"No custom registry defined for image {image_name}.")
|
|
@@ -15,7 +15,7 @@ import math
|
|
|
15
15
|
import base64
|
|
16
16
|
import os
|
|
17
17
|
from html.parser import HTMLParser
|
|
18
|
-
from queue import Queue
|
|
18
|
+
from queue import Queue
|
|
19
19
|
from concurrent.futures import ThreadPoolExecutor
|
|
20
20
|
import xml.etree.ElementTree as ET
|
|
21
21
|
from pathlib import Path
|
|
@@ -40,7 +40,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
|
|
|
40
40
|
from .ProgramRunnerPopen import ProgramRunnerPopen
|
|
41
41
|
from .SCLog import SCLog, LogLevel
|
|
42
42
|
|
|
43
|
-
version = "4.4.
|
|
43
|
+
version = "4.4.29"
|
|
44
44
|
__version__ = version
|
|
45
45
|
|
|
46
46
|
class VSCodeWorkspaceShellTask:
|
|
@@ -212,12 +212,20 @@ class ScriptCollectionCore:
|
|
|
212
212
|
# Base-url of the GitHub-REST-API. This is an internal constant on purpose and deliberately not exposed as a parameter of the GitHub-functions.
|
|
213
213
|
__github_api_base_url: str = "https://api.github.com"
|
|
214
214
|
|
|
215
|
+
# The maximal amount of seconds the reader-threads get to transfer the remaining pipe-content into their queue after
|
|
216
|
+
# the process itself terminated. This is only relevant if a grandchild-process inherited the pipes and keeps them
|
|
217
|
+
# open, because in that case the reader-threads never reach the end of their pipe. See __read_popen_pipes.
|
|
218
|
+
__pipe_drain_grace_period_in_seconds: float = 10
|
|
219
|
+
|
|
220
|
+
# Whether the login to the registries which are defined for this machine was already done. See login_to_defined_docker_registries.
|
|
221
|
+
__already_logged_in_to_defined_docker_registries: bool = None
|
|
215
222
|
|
|
216
223
|
def __init__(self):
|
|
217
224
|
self.program_runner = ProgramRunnerPopen()
|
|
218
225
|
self.call_program_runner_directly = None
|
|
219
226
|
self.__mocked_program_calls = list[ScriptCollectionCore.__MockProgramCall]()
|
|
220
227
|
self.log = SCLog(None, LogLevel.Warning, False)
|
|
228
|
+
self.__already_logged_in_to_defined_docker_registries = False
|
|
221
229
|
|
|
222
230
|
@staticmethod
|
|
223
231
|
@GeneralUtilities.check_arguments
|
|
@@ -242,25 +250,26 @@ class ScriptCollectionCore:
|
|
|
242
250
|
return result
|
|
243
251
|
|
|
244
252
|
def __load_credentials_if_required_and_available(self,registry_url:str,registry_username:str,registry_password:str)->tuple[str,str]:
|
|
253
|
+
"""Returns the credentials for the given registry: the ones which were passed by the caller, or - if the caller passed none - the
|
|
254
|
+
ones which are available for this registry on this machine (see __get_docker_registry_credentials). The last matching entry is
|
|
255
|
+
used, because the entries are ordered by increasing precedence."""
|
|
245
256
|
if registry_url.startswith("https://"):
|
|
246
257
|
registry_url=registry_url[len("https://"):]
|
|
247
258
|
if registry_password is None:
|
|
248
|
-
|
|
249
|
-
lines=GeneralUtilities.read_nonempty_lines_from_file(credential_file)[1:]
|
|
250
|
-
for line in lines:
|
|
251
|
-
splitted=line.split(";")
|
|
252
|
-
registry=splitted[0]
|
|
253
|
-
username=splitted[1]
|
|
254
|
-
password=splitted[2]
|
|
259
|
+
for registry,username,password in self.__get_docker_registry_credentials():
|
|
255
260
|
if registry_url==registry and (registry_username is None or username==registry_username):
|
|
256
261
|
registry_username=username
|
|
257
262
|
registry_password=password
|
|
258
|
-
break
|
|
259
263
|
else:
|
|
260
264
|
GeneralUtilities.assert_not_null(registry_username)
|
|
261
265
|
return (registry_username,registry_password)
|
|
262
266
|
|
|
263
267
|
def __get_docker_registry_credentials(self)->list[tuple[str,str,str]]:
|
|
268
|
+
"""Returns all registry-credentials which are available for the current process: the ones of the credentials-file of the
|
|
269
|
+
configuration-folder followed by the ones which are declared in the environment (see
|
|
270
|
+
get_docker_registry_credentials_from_environment_variables). The entries are ordered by increasing precedence: a registry which is
|
|
271
|
+
declared in both places is contained twice and the declaration of the environment is the last one, so it is the one which a login
|
|
272
|
+
executes last and which a lookup of credentials uses."""
|
|
264
273
|
result=[]
|
|
265
274
|
credential_file=self.__get_docker_registry_credentials_file()
|
|
266
275
|
if os.path.isfile(credential_file):
|
|
@@ -271,6 +280,40 @@ class ScriptCollectionCore:
|
|
|
271
280
|
username=splitted[1]
|
|
272
281
|
password=splitted[2]
|
|
273
282
|
result.append((registry,username,password))
|
|
283
|
+
result=result+self.get_docker_registry_credentials_from_environment_variables()
|
|
284
|
+
return result
|
|
285
|
+
|
|
286
|
+
@GeneralUtilities.check_arguments
|
|
287
|
+
def get_docker_registry_credentials_from_environment_variables(self)->list[tuple[str,str,str]]:
|
|
288
|
+
"""Returns the registry-credentials which are declared in the environment of the current process, as tuples of registry-address,
|
|
289
|
+
username and password. The credentials of a registry named '<registryname>' are declared by the environment-variables
|
|
290
|
+
'OCIRegistry_<registryname>_Address', 'OCIRegistry_<registryname>_Username' and 'OCIRegistry_<registryname>_Password'.
|
|
291
|
+
The environment is used in addition to the credentials-file of the configuration-folder because that folder belongs to the user of
|
|
292
|
+
the machine on which a command was started and is therefore not available inside a build-container, while an environment-variable is
|
|
293
|
+
passed into the container (see TFCPS_Tools_General.get_required_environment_variables). It is the same mechanism which makes a
|
|
294
|
+
private package-source available inside a container (see TFCPS_Tools_General.get_declared_package_sources).
|
|
295
|
+
The names of the environment-variables are treated case-insensitively because they are not case-sensitive on all operating-systems.
|
|
296
|
+
A registry-address may be declared with or without the scheme 'https://'; the scheme is removed because docker expects the address
|
|
297
|
+
of a registry without it."""
|
|
298
|
+
address_variable_pattern = re.compile(r"^OCIRegistry_(.+)_Address$", re.IGNORECASE)
|
|
299
|
+
environment_variables: dict[str, str] = {name.lower(): value for name, value in os.environ.items()}
|
|
300
|
+
result: list[tuple[str,str,str]] = []
|
|
301
|
+
for environment_variable_name, environment_variable_value in os.environ.items():
|
|
302
|
+
match = address_variable_pattern.match(environment_variable_name)
|
|
303
|
+
if match is None:
|
|
304
|
+
continue
|
|
305
|
+
registry_name: str = match.group(1).lower()
|
|
306
|
+
if not GeneralUtilities.string_has_content(environment_variable_value):
|
|
307
|
+
continue
|
|
308
|
+
address: str = environment_variable_value.strip()
|
|
309
|
+
if address.startswith("https://"):
|
|
310
|
+
address = address[len("https://"):]
|
|
311
|
+
username: str = environment_variables.get(f"ociregistry_{registry_name}_username")
|
|
312
|
+
password: str = environment_variables.get(f"ociregistry_{registry_name}_password")
|
|
313
|
+
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(username), f"No username is declared for the registry \"{registry_name}\" whose address is declared in the environment.")
|
|
314
|
+
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(password), f"No password is declared for the registry \"{registry_name}\" whose address is declared in the environment.")
|
|
315
|
+
result.append((address, username, password))
|
|
316
|
+
result.sort()#sorted so the order (and therefore the log-output) is deterministic
|
|
274
317
|
return result
|
|
275
318
|
|
|
276
319
|
def registry_contains_image(self,registry_url:str,image:str,registry_username:str,registry_password:str)->bool:
|
|
@@ -348,6 +391,12 @@ class ScriptCollectionCore:
|
|
|
348
391
|
return result
|
|
349
392
|
|
|
350
393
|
def login_to_defined_docker_registries(self)->None:
|
|
394
|
+
"""Logs in to every registry for which credentials are available on this machine (see __get_docker_registry_credentials).
|
|
395
|
+
The login is only executed once per instance: the credentials do not change while a process runs, and the login has to be
|
|
396
|
+
ensured by everything which accesses a registry, so without this the same logins would be executed over and over again."""
|
|
397
|
+
if self.__already_logged_in_to_defined_docker_registries:
|
|
398
|
+
return
|
|
399
|
+
self.__already_logged_in_to_defined_docker_registries=True
|
|
351
400
|
registries=self.__get_docker_registry_credentials()
|
|
352
401
|
if len(registries)==0:
|
|
353
402
|
self.log.log("No docker registry credentials defined. Skipping docker login.",LogLevel.Debug)
|
|
@@ -376,6 +425,18 @@ class ScriptCollectionCore:
|
|
|
376
425
|
program_result = self.run_program("docker", f"image inspect {image}:{tag}", throw_exception_if_exitcode_is_not_zero=False)
|
|
377
426
|
return program_result[0] == 0
|
|
378
427
|
|
|
428
|
+
@GeneralUtilities.check_arguments
|
|
429
|
+
def image_is_available_in_registry(self, image: str, tag: str) -> bool:
|
|
430
|
+
"""Returns True if and only if the image is available with this tag in the registry it addresses, using the credentials the
|
|
431
|
+
docker-client of the current process has (see login_to_defined_docker_registries).
|
|
432
|
+
'image' must be the image-address without tag, for example "myregistry.example.com/debian".
|
|
433
|
+
Only the manifest of the image is inspected, so the check costs one request to the registry and does not download the image.
|
|
434
|
+
The possible reasons why an image is not available (the registry is not reachable, the image does not exist there, the
|
|
435
|
+
credentials are missing or do not permit the access) are not distinguished: in all of these cases the image can not be pulled
|
|
436
|
+
from this address, which is what the caller has to know."""
|
|
437
|
+
program_result = self.run_program("docker", f"buildx imagetools inspect {image}:{tag}", throw_exception_if_exitcode_is_not_zero=False, print_errors_as_information=True, print_live_output=self.log.loglevel == LogLevel.Debug)
|
|
438
|
+
return program_result[0] == 0
|
|
439
|
+
|
|
379
440
|
@GeneralUtilities.check_arguments
|
|
380
441
|
def docker_pull(self, image: str, tag: str, force: bool = False) -> None:
|
|
381
442
|
"""Ensures that the image is available with this tag in the local image-store of the docker-daemon.
|
|
@@ -2282,20 +2343,54 @@ resolving a name requires fontconfig, which does not exist on every system ffmpe
|
|
|
2282
2343
|
file.close()
|
|
2283
2344
|
|
|
2284
2345
|
@staticmethod
|
|
2285
|
-
def
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2346
|
+
def __dequeue_lines(queue: Queue) -> list[str]:
|
|
2347
|
+
"""Removes all lines which are currently available in "queue" and returns them without their line-breaks.
|
|
2348
|
+
Lines without content are discarded."""
|
|
2349
|
+
lines: list[str] = []
|
|
2350
|
+
while not queue.empty():
|
|
2351
|
+
line: str = queue.get_nowait()
|
|
2352
|
+
line = line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
|
|
2353
|
+
if GeneralUtilities.string_has_content(line):
|
|
2354
|
+
lines.append(line)
|
|
2355
|
+
return lines
|
|
2291
2356
|
|
|
2292
|
-
|
|
2293
|
-
|
|
2357
|
+
@staticmethod
|
|
2358
|
+
def __process_stdout_lines(queue: Queue, stdout_result: list[str], print_live_output: bool, log: SCLog) -> None:
|
|
2359
|
+
for out_line in ScriptCollectionCore.__dequeue_lines(queue):
|
|
2360
|
+
stdout_result.append(out_line)
|
|
2361
|
+
if print_live_output:
|
|
2362
|
+
loglevel = LogLevel.Information
|
|
2363
|
+
if out_line.startswith("Debug: "):
|
|
2364
|
+
loglevel = LogLevel.Debug
|
|
2365
|
+
out_line = out_line[len("Debug: "):]
|
|
2366
|
+
if out_line.startswith("Diagnostic: "):
|
|
2367
|
+
loglevel = LogLevel.Diagnostic
|
|
2368
|
+
out_line = out_line[len("Diagnostic: "):]
|
|
2369
|
+
log.log(out_line, loglevel)
|
|
2294
2370
|
|
|
2295
|
-
|
|
2296
|
-
|
|
2371
|
+
@staticmethod
|
|
2372
|
+
def __process_stderr_lines(queue: Queue, stderr_result: list[str], print_live_output: bool, print_errors_as_information: bool, log: SCLog) -> None:
|
|
2373
|
+
for err_line in ScriptCollectionCore.__dequeue_lines(queue):
|
|
2374
|
+
stderr_result.append(err_line)
|
|
2375
|
+
if print_live_output:
|
|
2376
|
+
loglevel = LogLevel.Error
|
|
2377
|
+
if err_line.startswith("Warning: "):
|
|
2378
|
+
loglevel = LogLevel.Warning
|
|
2379
|
+
err_line = err_line[len("Warning: "):]
|
|
2380
|
+
if print_errors_as_information: # "errors" in "print_errors_as_information" means: all what is written to std-err
|
|
2381
|
+
loglevel = LogLevel.Information
|
|
2382
|
+
log.log(err_line, loglevel)
|
|
2297
2383
|
|
|
2298
|
-
|
|
2384
|
+
@staticmethod
|
|
2385
|
+
def __reading_is_finished(readers: list, q_stdout: Queue, q_stderr: Queue, process_end_time: float) -> bool:
|
|
2386
|
+
"""Decides whether all output of an already terminated process was read.
|
|
2387
|
+
Checking the queues alone is not sufficient: a queue can be empty although the according reader-thread did
|
|
2388
|
+
not transfer the content of its pipe into the queue yet, which would silently discard the whole output of
|
|
2389
|
+
short-running processes. Therefore the state of the reader-threads is checked first."""
|
|
2390
|
+
readers_are_finished: bool = all(reader.done() for reader in readers)
|
|
2391
|
+
if not readers_are_finished and (time.monotonic() - process_end_time) < ScriptCollectionCore.__pipe_drain_grace_period_in_seconds:
|
|
2392
|
+
return False
|
|
2393
|
+
return q_stdout.empty() and q_stderr.empty()
|
|
2299
2394
|
|
|
2300
2395
|
@staticmethod
|
|
2301
2396
|
def __read_popen_pipes(p: Popen, print_live_output: bool, print_errors_as_information: bool, log: SCLog, timeoutInSeconds: int = None) -> tuple[list[str], list[str]]:
|
|
@@ -2305,62 +2400,40 @@ resolving a name requires fontconfig, which does not exist on every system ffmpe
|
|
|
2305
2400
|
timeout_enabled: bool = timeoutInSeconds is not None and 0 < timeoutInSeconds
|
|
2306
2401
|
start_time: float = time.monotonic()
|
|
2307
2402
|
timed_out: bool = False
|
|
2403
|
+
process_end_time: float = None # the moment the process terminated; None as long as it is still running.
|
|
2308
2404
|
with ThreadPoolExecutor(2) as pool:
|
|
2309
2405
|
q_stdout = Queue()
|
|
2310
2406
|
q_stderr = Queue()
|
|
2311
2407
|
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2408
|
+
readers: list = [
|
|
2409
|
+
pool.submit(ScriptCollectionCore.__enqueue_output, p.stdout, q_stdout),
|
|
2410
|
+
pool.submit(ScriptCollectionCore.__enqueue_output, p.stderr, q_stderr),
|
|
2411
|
+
]
|
|
2316
2412
|
|
|
2317
2413
|
stdout_result: list[str] = []
|
|
2318
2414
|
stderr_result: list[str] = []
|
|
2319
2415
|
|
|
2320
|
-
while
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
loglevel = LogLevel.Information
|
|
2330
|
-
if out_line.startswith("Debug: "):
|
|
2331
|
-
loglevel = LogLevel.Debug
|
|
2332
|
-
out_line = out_line[len("Debug: "):]
|
|
2333
|
-
if out_line.startswith("Diagnostic: "):
|
|
2334
|
-
loglevel = LogLevel.Diagnostic
|
|
2335
|
-
out_line = out_line[len("Diagnostic: "):]
|
|
2336
|
-
log.log(out_line, loglevel)
|
|
2337
|
-
except Empty:
|
|
2338
|
-
reading_stdout_last_time_resulted_in_exception = True
|
|
2339
|
-
|
|
2340
|
-
try:
|
|
2341
|
-
while not q_stderr.empty():
|
|
2342
|
-
err_line: str = q_stderr.get_nowait()
|
|
2343
|
-
err_line = err_line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
|
|
2344
|
-
if GeneralUtilities.string_has_content(err_line):
|
|
2345
|
-
stderr_result.append(err_line)
|
|
2346
|
-
reading_stderr_last_time_resulted_in_exception = False
|
|
2347
|
-
if print_live_output:
|
|
2348
|
-
loglevel = LogLevel.Error
|
|
2349
|
-
if err_line.startswith("Warning: "):
|
|
2350
|
-
loglevel = LogLevel.Warning
|
|
2351
|
-
err_line = err_line[len("Warning: "):]
|
|
2352
|
-
if print_errors_as_information: # "errors" in "print_errors_as_information" means: all what is written to std-err
|
|
2353
|
-
loglevel = LogLevel.Information
|
|
2354
|
-
log.log(err_line, loglevel)
|
|
2355
|
-
except Empty:
|
|
2356
|
-
reading_stderr_last_time_resulted_in_exception = True
|
|
2416
|
+
while True:
|
|
2417
|
+
ScriptCollectionCore.__process_stdout_lines(q_stdout, stdout_result, print_live_output, log)
|
|
2418
|
+
ScriptCollectionCore.__process_stderr_lines(q_stderr, stderr_result, print_live_output, print_errors_as_information, log)
|
|
2419
|
+
|
|
2420
|
+
if process_end_time is None and p.poll() is not None:
|
|
2421
|
+
process_end_time = time.monotonic()
|
|
2422
|
+
|
|
2423
|
+
if process_end_time is not None and ScriptCollectionCore.__reading_is_finished(readers, q_stdout, q_stderr, process_end_time):
|
|
2424
|
+
break
|
|
2357
2425
|
|
|
2358
2426
|
if timeout_enabled and timeoutInSeconds < (time.monotonic() - start_time):
|
|
2359
2427
|
timed_out = True
|
|
2360
2428
|
p.kill() # the process exceeded its timeout; kill it so the reader-threads get EOF and this loop ends.
|
|
2361
2429
|
break
|
|
2362
2430
|
|
|
2363
|
-
time.sleep(0.01) # this is
|
|
2431
|
+
time.sleep(0.01) # this is the poll-interval; without it the loop would consume a whole cpu-core
|
|
2432
|
+
|
|
2433
|
+
# The reader-threads can have enqueued further lines between the last processing above and the end of the
|
|
2434
|
+
# loop. Without processing them here this output would be lost.
|
|
2435
|
+
ScriptCollectionCore.__process_stdout_lines(q_stdout, stdout_result, print_live_output, log)
|
|
2436
|
+
ScriptCollectionCore.__process_stderr_lines(q_stderr, stderr_result, print_live_output, print_errors_as_information, log)
|
|
2364
2437
|
|
|
2365
2438
|
if timed_out:
|
|
2366
2439
|
raise TimeoutError(f"The process with process-id {p_id} did not finish within the configured timeout of {timeoutInSeconds} second(s) and was killed.")
|
|
@@ -4249,6 +4322,8 @@ OCR-content:
|
|
|
4249
4322
|
"""This function assumes that all files are valid xliff2 files and that the base file is the reference for syncing.
|
|
4250
4323
|
This function adds new entries from the base file to the language files if they do not already exist using the value from base_file.
|
|
4251
4324
|
This function removes entries from the language files if they do not exist in the base file anymore.
|
|
4325
|
+
This function updates the source-text of an entry which exists in a language file and in the base file but whose source-texts differ,
|
|
4326
|
+
see __update_outdated_source_of_unit.
|
|
4252
4327
|
In the end the updated language files are written to the disk. The base file is not changed.
|
|
4253
4328
|
Returns for each language (derived from the filename of the language-file) a dict which contains for each TranslationState
|
|
4254
4329
|
the percentage (as a value between 0.0 and 1.0) of translation-units which are in that state."""
|
|
@@ -4321,6 +4396,10 @@ OCR-content:
|
|
|
4321
4396
|
new_unit = copy.deepcopy(base_units[unit_id])
|
|
4322
4397
|
lang_file_element.append(new_unit)
|
|
4323
4398
|
|
|
4399
|
+
# Update the source of the units which exist in the base file as well as in the language file
|
|
4400
|
+
for unit_id in base_ids & lang_ids:
|
|
4401
|
+
self.__update_outdated_source_of_unit(base_units[unit_id], lang_units[unit_id], NSMAP)
|
|
4402
|
+
|
|
4324
4403
|
# Reorder units to match base order
|
|
4325
4404
|
current_units = {
|
|
4326
4405
|
unit.get("id"): unit
|
|
@@ -4456,6 +4535,29 @@ OCR-content:
|
|
|
4456
4535
|
GeneralUtilities.write_text_to_file(target_file, json.dumps(spec, indent=2, ensure_ascii=False)+"\n")
|
|
4457
4536
|
|
|
4458
4537
|
|
|
4538
|
+
@GeneralUtilities.check_arguments
|
|
4539
|
+
def __update_outdated_source_of_unit(self,base_unit:ET.Element, language_unit:ET.Element, nsmap:dict[str,str])->None:
|
|
4540
|
+
"""Overwrites the source-text of every segment of language_unit by the source-text of the segment at the same position of base_unit,
|
|
4541
|
+
if the two differ, and sets that segment back to the state "initial".
|
|
4542
|
+
The source-text of a unit belongs to the base file: it is the text which gets translated, so a language file which still carries an
|
|
4543
|
+
older source-text claims to translate a text which does not exist anymore - and the target which belongs to that older source-text
|
|
4544
|
+
is not a translation of the current text either, which is why the segment goes back to the state a translation is created from (see
|
|
4545
|
+
translate_xlf_file). The target itself is deliberately kept, so that a translation which was written by a human is still there as
|
|
4546
|
+
long as no new translation was created for the new source-text."""
|
|
4547
|
+
base_segments = base_unit.findall("x:segment", namespaces=nsmap)
|
|
4548
|
+
language_segments = language_unit.findall("x:segment", namespaces=nsmap)
|
|
4549
|
+
for index, language_segment in enumerate(language_segments):
|
|
4550
|
+
if len(base_segments) <= index:
|
|
4551
|
+
continue
|
|
4552
|
+
base_source = base_segments[index].find("x:source", namespaces=nsmap)
|
|
4553
|
+
language_source = language_segment.find("x:source", namespaces=nsmap)
|
|
4554
|
+
if base_source is None or language_source is None:
|
|
4555
|
+
continue
|
|
4556
|
+
if language_source.text == base_source.text:
|
|
4557
|
+
continue
|
|
4558
|
+
language_source.text = base_source.text
|
|
4559
|
+
language_segment.set("state", "initial")
|
|
4560
|
+
|
|
4459
4561
|
@GeneralUtilities.check_arguments
|
|
4460
4562
|
def sync_xlf2_files(self,prefix:str, languages:list[str], folder:str)->dict[str,dict[int, float]]:
|
|
4461
4563
|
#languages=["de", "fr"] for example. the default-language (usually english) must not be included.
|
|
@@ -167,7 +167,9 @@ class TFCPS_CodeUnitSpecific_CPP_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
167
167
|
#The result-folder is the sourcecode-folder itself: where a build-command of a C/C++-codeunit puts its
|
|
168
168
|
#output is defined by that command, so the only folder which is known to contain it is the one the commands
|
|
169
169
|
#run in - which is also the folder __collect_output_files searches afterwards.
|
|
170
|
-
|
|
170
|
+
#Nothing of a C/C++-codeunit is left out of the archive the runner gets: where a build-command puts its
|
|
171
|
+
#output is defined by that command, so there is no folder which is known to contain state of this machine.
|
|
172
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "/bin/sh", ["-c", " && ".join(self.macos_build_commands)], source_folder, source_folder, [])
|
|
171
173
|
self.__collect_output_files(source_folder, self.macos_output_file_patterns, "BuildResult_MacOS")
|
|
172
174
|
|
|
173
175
|
@GeneralUtilities.check_arguments
|
|
@@ -68,6 +68,14 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
68
68
|
else:
|
|
69
69
|
src_folder = GeneralUtilities.resolve_relative_path(package_name, codeunit_folder) # TODO replace packagename
|
|
70
70
|
artifacts_folder = os.path.join(codeunit_folder, "Other", "Artifacts")
|
|
71
|
+
# The targets below which are built on a remote task-runner send the whole working-tree of the repository to
|
|
72
|
+
# it, and this folder is the one part of it which only applies to the machine the build was started on: its
|
|
73
|
+
# "package_config.json" states the folder of every package of the app as an absolute path of this machine,
|
|
74
|
+
# and the flutter-tool of the runner resolves the packages again only when that file is older than
|
|
75
|
+
# "pubspec.yaml". A transferred one therefore makes the runner compile against folders which do not exist
|
|
76
|
+
# there ("Error when reading '/C:/tools/flutter/packages/flutter/lib/material.dart'"), so it is left out of
|
|
77
|
+
# the archive; the runner generates the one of its own machine while it builds.
|
|
78
|
+
folders_which_are_not_transferred: list[str] = [os.path.join(src_folder, ".dart_tool")]
|
|
71
79
|
|
|
72
80
|
target_names: dict[str, str] = {
|
|
73
81
|
"web": "WebApplication",
|
|
@@ -98,7 +106,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
98
106
|
"environment as a runner-built one.", LogLevel.Warning)
|
|
99
107
|
self._protected_sc.run_with_epew("flutter", "build windows", src_folder)
|
|
100
108
|
else:
|
|
101
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder, windows_release_folder)
|
|
109
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder, windows_release_folder, folders_which_are_not_transferred)
|
|
102
110
|
windows_folder = os.path.join(artifacts_folder, "BuildResult_Windows")
|
|
103
111
|
GeneralUtilities.ensure_directory_does_not_exist(windows_folder)
|
|
104
112
|
GeneralUtilities.ensure_directory_exists(windows_folder)
|
|
@@ -120,7 +128,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
120
128
|
# builds), analogous to how ios-builds always run on the iOS-task-runner. See SCTaskRunnerMacOS and
|
|
121
129
|
# the remote-build-article in the reference.
|
|
122
130
|
macos_release_folder = os.path.join(src_folder, "build/macos/Build/Products/Release")
|
|
123
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder, macos_release_folder)
|
|
131
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder, macos_release_folder, folders_which_are_not_transferred)
|
|
124
132
|
macos_folder = os.path.join(artifacts_folder, "BuildResult_MacOS")
|
|
125
133
|
GeneralUtilities.ensure_directory_does_not_exist(macos_folder)
|
|
126
134
|
GeneralUtilities.ensure_directory_exists(macos_folder)
|
|
@@ -129,7 +137,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
129
137
|
# iOS-builds must run on macOS and therefore always run on the dedicated iOS-task-runner (uniform
|
|
130
138
|
# builds). See SCTaskRunnerIOS and the remote-build-article in the reference.
|
|
131
139
|
ios_release_folder = os.path.join(src_folder, "build/ios/iphoneos")
|
|
132
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder, ios_release_folder)
|
|
140
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder, ios_release_folder, folders_which_are_not_transferred)
|
|
133
141
|
ios_folder = os.path.join(artifacts_folder, "BuildResult_IOS")
|
|
134
142
|
GeneralUtilities.ensure_directory_does_not_exist(ios_folder)
|
|
135
143
|
GeneralUtilities.ensure_directory_exists(ios_folder)
|
|
@@ -139,7 +147,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
139
147
|
# remote-build-article in the reference), analogous to how ios-builds always run on a macOS-task-runner:
|
|
140
148
|
# the Android-SDK/NDK-toolchain no longer lives in SCBuilder itself, it moved into SCTaskRunnerAndroid.
|
|
141
149
|
aab_release_folder = os.path.join(src_folder, "build/app/outputs/bundle/release")
|
|
142
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder, aab_release_folder)
|
|
150
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder, aab_release_folder, folders_which_are_not_transferred)
|
|
143
151
|
# The app-bundle which came back from the runner is published as the build-result of this codeunit,
|
|
144
152
|
# analogous to the targets above: it is the artifact of an android-build, and every codeunit has to have
|
|
145
153
|
# an artifact whose name matches "BuildResult_.+" (see TFCPS_CodeUnit_BuildCodeUnit.build_codeunit).
|
|
@@ -168,7 +176,8 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
168
176
|
with zipfile.ZipFile(apks_file, "r") as apks_archive:
|
|
169
177
|
apks_archive.extract("universal.apk", apk_folder)
|
|
170
178
|
GeneralUtilities.ensure_file_does_not_exist(apks_file)
|
|
171
|
-
|
|
179
|
+
codeunit_version: str = self.tfcps_Tools_General.get_version_of_codeunit(self.get_codeunit_file())
|
|
180
|
+
os.rename(os.path.join(apk_folder, "universal.apk"), os.path.join(apk_folder, f"{codeunit_name}_v{codeunit_version}.apk"))
|
|
172
181
|
else:
|
|
173
182
|
raise ValueError(f"Not supported target: {target}")
|
|
174
183
|
self.__generate_bom_for_flutter_package(package_name)
|
|
@@ -188,7 +188,7 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
|
|
|
188
188
|
GeneralUtilities.assert_condition(developmentstate in (developmentstate_active, developmentstate_maintenance, developmentstate_inactive), f"Invalid development-state. Must be '{developmentstate_active}' or '{developmentstate_maintenance}' or '{developmentstate_inactive}' but was '{developmentstate}'.")
|
|
189
189
|
|
|
190
190
|
# check for mandatory files
|
|
191
|
-
files = ["Other/Build/Build.py", "Other/QualityCheck/Linting.py", "Other/Reference/GenerateReference.py"]
|
|
191
|
+
files = ["ReadMe.md", "Other/Build/Build.py", "Other/QualityCheck/Linting.py", "Other/Reference/GenerateReference.py"]
|
|
192
192
|
if self.tfcps_Tools_General.codeunit_has_testable_sourcecode(self.get_codeunit_file()):
|
|
193
193
|
# TODO check if the testsettings-section appears in the codeunit-file
|
|
194
194
|
files.append("Other/QualityCheck/RunTestcases.py")
|
|
@@ -364,14 +364,17 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
|
|
|
364
364
|
return self.__use_cache
|
|
365
365
|
|
|
366
366
|
@GeneralUtilities.check_arguments
|
|
367
|
-
def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str, result_folder: str) -> None:
|
|
367
|
+
def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str]) -> None:
|
|
368
368
|
"""Runs the given program for this codeunit on a remote task-runner that provides the given operating-system (see
|
|
369
369
|
TFCPS_RemoteBuild and the SCTaskRunnerWindows/SCTaskRunnerMacOS-codeunits). This is used for operating-system-bound
|
|
370
370
|
build-steps (e.g. flutter-windows- or flutter-ios-builds) which can not run on the current operating-system. After
|
|
371
371
|
the runner finished, the content of result_folder is written back into the local repository, so the build-step can
|
|
372
372
|
continue with the result as if it had been produced here; nothing else of the local repository is touched.
|
|
373
|
-
working_directory and result_folder must be folders inside this codeunit.
|
|
374
|
-
|
|
373
|
+
working_directory and result_folder must be folders inside this codeunit.
|
|
374
|
+
folders_which_are_not_transferred are the folders of this codeunit whose content only applies to this machine
|
|
375
|
+
and which are therefore left out of the repository-archive the runner gets (see
|
|
376
|
+
TFCPS_RemoteBuild.run_program_on_runner)."""
|
|
377
|
+
TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory, result_folder, folders_which_are_not_transferred)
|
|
375
378
|
|
|
376
379
|
@GeneralUtilities.check_arguments
|
|
377
380
|
def get_codeunit_folder(self)->str:
|
|
@@ -76,7 +76,11 @@ class TFCPS_RemoteBuild:
|
|
|
76
76
|
over HTTPS to the runner, the runner runs the given program on its (correct) operating-system, and afterwards only the
|
|
77
77
|
folder in which that build-step produces its result is written back into the local repository. Everything else which
|
|
78
78
|
changed in the workspace of the runner stays there: it is state of that machine (its toolchain-paths, its caches) and
|
|
79
|
-
not a result, and a local repository is something a developer works in and which therefore must not be replaced.
|
|
79
|
+
not a result, and a local repository is something a developer works in and which therefore must not be replaced.
|
|
80
|
+
|
|
81
|
+
The only part of the working-tree which is not sent are the folders the caller states as
|
|
82
|
+
'folders_which_are_not_transferred' (see run_program_on_runner): a folder which describes the machine it was
|
|
83
|
+
generated on is state of this machine and not state of the repository, so on the runner it describes nothing."""
|
|
80
84
|
|
|
81
85
|
__sc: ScriptCollectionCore = None
|
|
82
86
|
|
|
@@ -89,7 +93,14 @@ class TFCPS_RemoteBuild:
|
|
|
89
93
|
return len(self.__load_runner_endpoints()) > 0
|
|
90
94
|
|
|
91
95
|
@GeneralUtilities.check_arguments
|
|
92
|
-
def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
|
|
96
|
+
def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, folders_which_are_not_transferred: list[str], poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
|
|
97
|
+
"""Runs the given program on a runner which provides the given operating-system, see the docstring of this class.
|
|
98
|
+
|
|
99
|
+
'folders_which_are_not_transferred' are folders (as paths inside the repository) which are left out of the
|
|
100
|
+
archive which is sent to the runner, because their content only applies to this machine (for example the
|
|
101
|
+
'.dart_tool'-folder of a flutter-package, whose 'package_config.json' states the folder of every package of
|
|
102
|
+
the app as an absolute path of this machine). Such a folder is generated again by the toolchain of the
|
|
103
|
+
runner while it builds; transferring it makes the runner use the paths of this machine instead."""
|
|
93
104
|
endpoints = self.__load_runner_endpoints()
|
|
94
105
|
if len(endpoints) == 0:
|
|
95
106
|
raise ValueError("No remote-build-runner is configured. Define runners either in "
|
|
@@ -100,7 +111,8 @@ class TFCPS_RemoteBuild:
|
|
|
100
111
|
working_directory_relative = os.path.relpath(working_directory, repository_folder).replace("\\", "/")
|
|
101
112
|
result_folder_relative = os.path.relpath(result_folder, repository_folder).replace("\\", "/")
|
|
102
113
|
self.__sc.log.log(f"Delegate '{program} {' '.join(arguments)}' (codeunit '{codeunit_name}', folder '{working_directory_relative}', result-folder '{result_folder_relative}') to the {required_os.value}-runner at {endpoint.url}...")
|
|
103
|
-
|
|
114
|
+
folders_which_are_not_transferred_relative = [os.path.relpath(folder, repository_folder).replace("\\", "/") for folder in folders_which_are_not_transferred]
|
|
115
|
+
archive_file = self.__create_repository_archive(repository_folder, folders_which_are_not_transferred_relative)
|
|
104
116
|
job_id: str = None
|
|
105
117
|
try:
|
|
106
118
|
metadata_headers = {
|
|
@@ -189,15 +201,33 @@ class TFCPS_RemoteBuild:
|
|
|
189
201
|
time.sleep(retry_interval_in_seconds)
|
|
190
202
|
|
|
191
203
|
@GeneralUtilities.check_arguments
|
|
192
|
-
def __create_repository_archive(self, repository_folder: str) -> str:
|
|
204
|
+
def __create_repository_archive(self, repository_folder: str, folders_which_are_not_transferred_relative: list[str]) -> str:
|
|
193
205
|
# Pack the entire repository-working-tree (including .git, uncommitted changes and git-ignored files) so the runner
|
|
194
206
|
# has the exact same state - including secrets that are required e.g. for signing windows-builds. Uses tarfile so
|
|
195
207
|
# it works identically on Windows and Linux (scbuildcodeunits runs on both) and preserves symlinks/permissions.
|
|
208
|
+
# The folders which describe this machine (see run_program_on_runner) are left out while the archive is packed
|
|
209
|
+
# and not removed from the working-tree before: the working-tree belongs to the developer, and a tool of theirs
|
|
210
|
+
# (for example the dart-extension of an ide, which resolves the packages of a flutter-package whenever its
|
|
211
|
+
# "pubspec.yaml" changes - which a build does) can generate such a folder again at any moment, so removing it
|
|
212
|
+
# would only be a race against that tool.
|
|
196
213
|
archive_file = os.path.join(GeneralUtilities.get_temp_folder(), f"sc-remotebuild-payload-{uuid.uuid4()}.tar.gz")
|
|
197
214
|
with tarfile.open(archive_file, "w:gz") as tar:
|
|
198
|
-
tar.add(repository_folder, arcname=".")
|
|
215
|
+
tar.add(repository_folder, arcname=".", filter=lambda entry: None if self.__entry_belongs_to_a_folder_which_is_not_transferred(entry.name, folders_which_are_not_transferred_relative) else entry)
|
|
199
216
|
return archive_file
|
|
200
217
|
|
|
218
|
+
@staticmethod
|
|
219
|
+
@GeneralUtilities.check_arguments
|
|
220
|
+
def __entry_belongs_to_a_folder_which_is_not_transferred(entry_name: str, folders_which_are_not_transferred_relative: list[str]) -> bool:
|
|
221
|
+
"""Whether the entry with the given name inside the archive (which is named relative to the repository-folder,
|
|
222
|
+
prefixed with './' because the repository is added under that name) is one of the given folders or lies inside
|
|
223
|
+
one of them."""
|
|
224
|
+
entry_path = entry_name.replace("\\", "/").removeprefix("./")
|
|
225
|
+
for folder in folders_which_are_not_transferred_relative:
|
|
226
|
+
folder_path = folder.replace("\\", "/").strip("/")
|
|
227
|
+
if entry_path == folder_path or entry_path.startswith(f"{folder_path}/"):
|
|
228
|
+
return True
|
|
229
|
+
return False
|
|
230
|
+
|
|
201
231
|
@GeneralUtilities.check_arguments
|
|
202
232
|
def __write_result_into_the_repository(self, result_bytes: bytes, result_folder: str) -> None:
|
|
203
233
|
"""Writes what the runner produced into the folder of the local repository in which the build-step produces its
|
|
@@ -389,10 +389,14 @@ class TFCPS_Tools_General:
|
|
|
389
389
|
return
|
|
390
390
|
version_element.text = current_version
|
|
391
391
|
file_ended_with_newline: bool = GeneralUtilities.read_binary_from_file(codeunit_file).endswith(b"\n")
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
392
|
+
#the file is written through a binary file-object instead of by its name because the serializer opens a file which is given by its name in text-mode, which
|
|
393
|
+
#translates every line-separator it writes to the one of the current operating-system: on windows that converted the codeunit-files to crlf on every build,
|
|
394
|
+
#while the trailing newline below (which was always written in binary-mode) stayed lf, so the files ended up with mixed line-separators. The parser normalizes
|
|
395
|
+
#the line-separators of the file it reads to lf anyway (this is defined by the xml-specification), so a codeunit-file is written with lf on every operating-system.
|
|
396
|
+
with open(codeunit_file, "wb") as file_object:
|
|
397
|
+
tree.write(file_object, xml_declaration=True, encoding="utf-8")
|
|
398
|
+
if file_ended_with_newline:
|
|
399
|
+
#the serializer does not terminate the last line, so without this the trailing newline of the file would get lost on every write.
|
|
396
400
|
file_object.write(b"\n")
|
|
397
401
|
|
|
398
402
|
@GeneralUtilities.check_arguments
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scriptcollection
|
|
3
|
-
Version: 4.4.
|
|
3
|
+
Version: 4.4.29
|
|
4
4
|
Summary: The ScriptCollection is the place for reusable scripts.
|
|
5
5
|
Author-email: Marius Göcke <marius.goecke@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/anionDev/ScriptCollection
|
|
@@ -12,12 +12,12 @@ ScriptCollection/ProgramRunnerSudo.py,sha256=ot9Ib-plQ8wejR9M2XXfnx6GR-va9k8SH1g
|
|
|
12
12
|
ScriptCollection/RepositoryDiagramGenerator.py,sha256=K_u2NW_p4ozdWD50irJEkkcDVa70TcUJpZN63TeXSLw,7092
|
|
13
13
|
ScriptCollection/RepositoryHistoryAnalyzer.py,sha256=rD8eC5lMUe5MHmCBHfYft_G-N9IbvtsFGkU7EMwPjrY,5779
|
|
14
14
|
ScriptCollection/SCLog.py,sha256=Fjf8Pdi63cXHGMxL_aI6A0dwGm7QIfrv3czjvpSUlaM,4705
|
|
15
|
-
ScriptCollection/ScriptCollectionCore.py,sha256=
|
|
15
|
+
ScriptCollection/ScriptCollectionCore.py,sha256=PJysQBH2oGfznko_coqHB-LcQdfV1PshFmYZwHGupNk,285393
|
|
16
16
|
ScriptCollection/TestKind.py,sha256=EBTLR8yBz5VmWA2NZIpDTbQNg6tkfv_kAsIzEJ--mCI,1756
|
|
17
17
|
ScriptCollection/UnassignedVariablesCheck.py,sha256=8xNcp1D5HeOLO1a1NEerq2TsnpFhw7yzYPkaay-oisY,3804
|
|
18
18
|
ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
-
ScriptCollection/OCIImages/AbstractImageHandler.py,sha256=
|
|
20
|
-
ScriptCollection/OCIImages/OCIImageManager.py,sha256=
|
|
19
|
+
ScriptCollection/OCIImages/AbstractImageHandler.py,sha256=W3ZG9IVhRwlWjFvQjIIXq1kz7xCGOOd3g4O0dzDnpkM,2470
|
|
20
|
+
ScriptCollection/OCIImages/OCIImageManager.py,sha256=mVfSeFXraeZIInjMISh_ZZVSvUO0wAqIb7NZRMivMG4,24123
|
|
21
21
|
ScriptCollection/OCIImages/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
22
22
|
ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py,sha256=rty9SnftreYWLIwvdUn4LhiHqCTevb_tWuYWoOnRJuo,732
|
|
23
23
|
ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py,sha256=M-4qxKWBZRtKdYQ-uXgfrxRBoZTXEjgqCxFXyFxTyzc,736
|
|
@@ -35,7 +35,7 @@ ScriptCollection/TFCPS/PackageSource.py,sha256=tQoMbxrIbV3mYHKH2wgOoGzIHY1TvBH8E
|
|
|
35
35
|
ScriptCollection/TFCPS/PlantUMLRenderer.py,sha256=upnafiGKV-SdRI8iRBkbbWHN2u6v9KxoswdKi12AwoY,11060
|
|
36
36
|
ScriptCollection/TFCPS/SCTaskRunnerServer.py,sha256=tub8_1QBu0nWAaGnBWp2nErOW-SOHf_UL6qjOSGbnxU,31026
|
|
37
37
|
ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py,sha256=67_DQwoH5bhxxSmwtuAeoW1NOCMiLMzOy48OqlJX-KY,1744
|
|
38
|
-
ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=
|
|
38
|
+
ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=umUfQDErp7xFLkDnSbQMxzUWnKtcGlrZnIvFcTpfGJs,40499
|
|
39
39
|
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=Mj9t1SyRpkYZE74cLsGWYv4xB89gXL2ld-RX3xvvGdw,9835
|
|
40
40
|
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=l8xfp7n-gDF_cwngox6gJkv5ShHP0oGSBzVXENj-3bM,48546
|
|
41
41
|
ScriptCollection/TFCPS/TFCPS_CreateRelease.py,sha256=kpEmSnfzfgB4OjcCuS_X6jgW_ftcphxEnjjoLoAHTLY,6582
|
|
@@ -44,11 +44,11 @@ ScriptCollection/TFCPS/TFCPS_MergeToMain.py,sha256=Krt5oyGNcRAN3szsgU7LoTh8yVTOd
|
|
|
44
44
|
ScriptCollection/TFCPS/TFCPS_MergeToStable.py,sha256=oYEhz5ZWMnbu97_CcYEqdDxakUFVppyOVQiJzADq59c,22434
|
|
45
45
|
ScriptCollection/TFCPS/TFCPS_OCIImageSecretScan.py,sha256=rhw0glr4gZ5enG9faOD51xE4MMK1cIdNjUdaswiMsrY,14622
|
|
46
46
|
ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py,sha256=XLVsFTA1jvfDbZ7wYJgIdEByZhF_bYVbA5jkisuvK4k,2322
|
|
47
|
-
ScriptCollection/TFCPS/TFCPS_RemoteBuild.py,sha256=
|
|
48
|
-
ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=
|
|
47
|
+
ScriptCollection/TFCPS/TFCPS_RemoteBuild.py,sha256=2Tx5HLk1ZftBWDCTiGZdP2QiiGNyWsEuTCSt33dNQjc,23891
|
|
48
|
+
ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=eGtOfeJEiHV-JoxkbBGw0EP9vDJJAlQWKpJ0hUASwUY,160284
|
|
49
49
|
ScriptCollection/TFCPS/TFCPS_VisualRegressionTests.py,sha256=6_63udxHnPlu_TMk_ftLZTnzriJO8MCxVCVAsweZqHs,37073
|
|
50
50
|
ScriptCollection/TFCPS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
51
|
-
ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py,sha256=
|
|
51
|
+
ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py,sha256=BNZPMjGiWLMBH7QDtP06WEVLMnI38NLX4Dpd13bSbi4,16614
|
|
52
52
|
ScriptCollection/TFCPS/CPP/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
53
53
|
ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=919mNfzVRurTJhWi8WV0oEAFoH-yd4a6m2Rbxcf2oZ4,19927
|
|
54
54
|
ScriptCollection/TFCPS/Docker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -57,7 +57,7 @@ ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py,sha256=
|
|
|
57
57
|
ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py,sha256=i0zEGehj0sttxjjZtoq2KFSKp_ulxVyWp_ZgAhIY_So,241
|
|
58
58
|
ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=GCFzDNAnmm40EeqOYgD_EeuEwDY58ujwJ1-Zn_naamc,61797
|
|
59
59
|
ScriptCollection/TFCPS/DotNet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
60
|
-
ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=
|
|
60
|
+
ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=dW7KoCvpIbjGaN-SRoaKPmF02GVVyT7zJAVR0yIjtag,52842
|
|
61
61
|
ScriptCollection/TFCPS/Flutter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
62
62
|
ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py,sha256=m4Pax-U6o5XuspLnLfnWjyQMtoC7zBDDNJW9EhLWjeE,9012
|
|
63
63
|
ScriptCollection/TFCPS/Go/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -71,8 +71,8 @@ ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=yyjWPMHQjPLkML
|
|
|
71
71
|
ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
72
72
|
ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py,sha256=5z990qyO5durX07xRWRwju1hl6wpjQAy2f-yRwJ-7s0,11856
|
|
73
73
|
ScriptCollection/TFCPS/TypeScript/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
74
|
-
scriptcollection-4.4.
|
|
75
|
-
scriptcollection-4.4.
|
|
76
|
-
scriptcollection-4.4.
|
|
77
|
-
scriptcollection-4.4.
|
|
78
|
-
scriptcollection-4.4.
|
|
74
|
+
scriptcollection-4.4.29.dist-info/METADATA,sha256=x84CpGE0zK9m7D6Dq6XKd0Sp5zaNmAoINt2n99IV4Pc,7690
|
|
75
|
+
scriptcollection-4.4.29.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
76
|
+
scriptcollection-4.4.29.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
|
|
77
|
+
scriptcollection-4.4.29.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
|
|
78
|
+
scriptcollection-4.4.29.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|