ScriptCollection 4.4.27__py3-none-any.whl → 4.4.28__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/OCIImages/AbstractImageHandler.py +2 -1
- ScriptCollection/OCIImages/OCIImageManager.py +63 -20
- ScriptCollection/ScriptCollectionCore.py +165 -63
- {scriptcollection-4.4.27.dist-info → scriptcollection-4.4.28.dist-info}/METADATA +1 -1
- {scriptcollection-4.4.27.dist-info → scriptcollection-4.4.28.dist-info}/RECORD +8 -8
- {scriptcollection-4.4.27.dist-info → scriptcollection-4.4.28.dist-info}/WHEEL +0 -0
- {scriptcollection-4.4.27.dist-info → scriptcollection-4.4.28.dist-info}/entry_points.txt +0 -0
- {scriptcollection-4.4.27.dist-info → scriptcollection-4.4.28.dist-info}/top_level.txt +0 -0
|
@@ -10,7 +10,8 @@ class AbstractImageHandler(ABC):
|
|
|
10
10
|
|
|
11
11
|
def _protected_get_credentials_for_registry(self,registry_address:str,username_str:str)->tuple[str,str]:
|
|
12
12
|
"""return (username, password) for basic auth.
|
|
13
|
-
Data will be taken from "~/.ScriptCollection/GlobalCache/RegistryCredentials.csv"
|
|
13
|
+
Data will be taken from "~/.ScriptCollection/GlobalCache/RegistryCredentials.csv" or from the registries which are declared in the
|
|
14
|
+
environment (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) if available.
|
|
14
15
|
If no credentials are available then None will be returned for the missing values."""
|
|
15
16
|
raise NotImplementedError()
|
|
16
17
|
|
|
@@ -20,12 +20,16 @@ class OCIImageManager:
|
|
|
20
20
|
#image-names for which it was already logged that the fallback-registry is used. The lookup happens for every image of
|
|
21
21
|
#the repository and often several times per build, so without this the same hint would flood the log.
|
|
22
22
|
__images_with_reported_fallback_registry:set[str] = None
|
|
23
|
+
#availability of an image-address with tag in its registry. The address of an image is resolved several times per build and
|
|
24
|
+
#the check costs a request to the registry, so its result is remembered for the runtime of this process.
|
|
25
|
+
__availability_of_image:dict[str,bool] = None
|
|
23
26
|
|
|
24
27
|
def __init__(self,sc:ScriptCollectionCore):
|
|
25
28
|
if sc is None:
|
|
26
29
|
sc=ScriptCollectionCore()
|
|
27
30
|
self.__sc=sc
|
|
28
31
|
self.__images_with_reported_fallback_registry=set()
|
|
32
|
+
self.__availability_of_image=dict()
|
|
29
33
|
self.image_handler=[
|
|
30
34
|
ImageHandlerDebian(),
|
|
31
35
|
ImageHandlerDebianSlim(),
|
|
@@ -105,34 +109,70 @@ class OCIImageManager:
|
|
|
105
109
|
return line[2]
|
|
106
110
|
raise ValueError(f"No tag defined for image \"{image_name}\".")
|
|
107
111
|
|
|
112
|
+
@GeneralUtilities.check_arguments
|
|
113
|
+
def get_custom_registry_address_for_image(self,image_name:str)->str:
|
|
114
|
+
"""Returns the registry-address which is defined for the image in the machine-wide image-registries-file, always without tag.
|
|
115
|
+
Use custom_registry_is_defined to check whether such an address is defined for the image at all."""
|
|
116
|
+
global_docker_image_registries_file=self.get_global_docker_image_registries_file()
|
|
117
|
+
for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(global_docker_image_registries_file)[1:]]:
|
|
118
|
+
if image_name==line[0]:
|
|
119
|
+
return line[1]
|
|
120
|
+
raise ValueError(f"No custom registry is defined for image \"{image_name}\" in \"{global_docker_image_registries_file}\".")
|
|
121
|
+
|
|
122
|
+
@GeneralUtilities.check_arguments
|
|
123
|
+
def get_fallback_registry_address_for_image(self,repository:str,image_name:str)->str:
|
|
124
|
+
"""Returns the upstream-registry-address which the repository defines for the image, always without tag."""
|
|
125
|
+
repository_image_definition_file=self.get_repository_image_definition_file(repository)
|
|
126
|
+
for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(repository_image_definition_file)[1:]]:
|
|
127
|
+
if image_name==line[0]:
|
|
128
|
+
return line[1]
|
|
129
|
+
raise ValueError(f"No registry defined for image \"{image_name}\" in \"{repository_image_definition_file}\".")
|
|
130
|
+
|
|
108
131
|
@GeneralUtilities.check_arguments
|
|
109
132
|
def get_registry_address_for_image(self,repository:str,image_name:str)->str:
|
|
110
|
-
"""
|
|
133
|
+
"""Returns the registry-address the image has to be taken from, always without tag.
|
|
134
|
+
Example: if image_name==Debian this function returns something like "myregistry.example.com/debian".
|
|
135
|
+
This is the custom registry-address which is defined for this machine if the image is really available there, and the
|
|
136
|
+
upstream-registry-address which the repository defines otherwise."""
|
|
137
|
+
fallback_registry_address=self.get_fallback_registry_address_for_image(repository,image_name)
|
|
111
138
|
if self.custom_registry_is_defined(image_name):
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
return line[1]
|
|
139
|
+
custom_registry_address=self.get_custom_registry_address_for_image(image_name)
|
|
140
|
+
if self.__custom_registry_provides_image(custom_registry_address,self.get_tag_for_image(repository,image_name)):
|
|
141
|
+
return custom_registry_address
|
|
142
|
+
self.__report_usage_of_fallback_registry(image_name,fallback_registry_address,f"the image is not available in the custom registry \"{custom_registry_address}\"")
|
|
117
143
|
else:
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
144
|
+
self.__report_usage_of_fallback_registry(image_name,fallback_registry_address,f"no custom registry is defined for the image in \"{self.get_global_docker_image_registries_file()}\"")
|
|
145
|
+
return fallback_registry_address
|
|
146
|
+
|
|
147
|
+
@GeneralUtilities.check_arguments
|
|
148
|
+
def __custom_registry_provides_image(self,custom_registry_address:str,tag:str)->bool:
|
|
149
|
+
"""Returns whether the image can be used with this tag from the custom registry-address, which is the case when it is already
|
|
150
|
+
available in the local image-store of the docker-daemon or when the custom registry provides it.
|
|
151
|
+
The local image-store is checked first because an image which is already available there is not downloaded at all (see
|
|
152
|
+
ScriptCollectionCore.docker_pull), so asking the registry would only cost a request - and would make a build which does not need
|
|
153
|
+
any download depend on a reachable registry.
|
|
154
|
+
A custom registry is typically not publicly readable, so the login to the registries which are defined for this machine is done
|
|
155
|
+
before the registry is asked; without it the answer would be that the image is not available although it only is not readable
|
|
156
|
+
without credentials."""
|
|
157
|
+
image_with_tag=f"{custom_registry_address}:{tag}"
|
|
158
|
+
if image_with_tag not in self.__availability_of_image:
|
|
159
|
+
if self.__sc.local_docker_image_exists(custom_registry_address,tag):
|
|
160
|
+
self.__availability_of_image[image_with_tag]=True
|
|
161
|
+
else:
|
|
162
|
+
self.__sc.login_to_defined_docker_registries()
|
|
163
|
+
self.__availability_of_image[image_with_tag]=self.__sc.image_is_available_in_registry(custom_registry_address,tag)
|
|
164
|
+
return self.__availability_of_image[image_with_tag]
|
|
125
165
|
|
|
126
166
|
@GeneralUtilities.check_arguments
|
|
127
|
-
def __report_usage_of_fallback_registry(self,image_name:str,fallback_registry_address:str)->None:
|
|
128
|
-
"""Logs that the image is taken from the upstream-registry
|
|
129
|
-
The fallback exists so that a freshly cloned product is buildable without further setup
|
|
130
|
-
rate-limited, so using it
|
|
167
|
+
def __report_usage_of_fallback_registry(self,image_name:str,fallback_registry_address:str,reason:str)->None:
|
|
168
|
+
"""Logs that the image is taken from the upstream-registry which the repository defines, together with the reason why the custom
|
|
169
|
+
registry is not used. The fallback exists so that a freshly cloned product is buildable without further setup and so that a custom
|
|
170
|
+
registry which does not provide the image does not break a build, but an upstream-registry is typically rate-limited, so using it
|
|
171
|
+
is worth a hint."""
|
|
131
172
|
if image_name in self.__images_with_reported_fallback_registry:
|
|
132
173
|
return
|
|
133
174
|
self.__images_with_reported_fallback_registry.add(image_name)
|
|
134
|
-
|
|
135
|
-
self.__sc.log.log(f"No custom registry is defined for image \"{image_name}\" in \"{registries_file}\", so the fallback-registry \"{fallback_registry_address}\" is used.",LogLevel.Warning)
|
|
175
|
+
self.__sc.log.log(f"The fallback-registry \"{fallback_registry_address}\" is used for image \"{image_name}\" because {reason}.",LogLevel.Warning)
|
|
136
176
|
|
|
137
177
|
@GeneralUtilities.check_arguments
|
|
138
178
|
def get_registry_address_for_image_with_default_tag(self,repository:str,image_name:str)->str:
|
|
@@ -159,7 +199,10 @@ class OCIImageManager:
|
|
|
159
199
|
try:
|
|
160
200
|
addresses_to_check=[]
|
|
161
201
|
if self.custom_registry_is_defined(image_name):
|
|
162
|
-
|
|
202
|
+
#the custom registry-address is used directly (and not get_registry_address_for_image) because this function does not
|
|
203
|
+
#pull an image but asks the registry which tags it has, so it does not matter whether that registry currently provides
|
|
204
|
+
#the image with the tag which is defined at the moment.
|
|
205
|
+
addresses_to_check.append(self.get_custom_registry_address_for_image(image_name))
|
|
163
206
|
else:
|
|
164
207
|
if search_in_custom_registry_only_if_available:
|
|
165
208
|
raise ValueError(f"No custom registry defined for image {image_name}.")
|
|
@@ -15,7 +15,7 @@ import math
|
|
|
15
15
|
import base64
|
|
16
16
|
import os
|
|
17
17
|
from html.parser import HTMLParser
|
|
18
|
-
from queue import Queue
|
|
18
|
+
from queue import Queue
|
|
19
19
|
from concurrent.futures import ThreadPoolExecutor
|
|
20
20
|
import xml.etree.ElementTree as ET
|
|
21
21
|
from pathlib import Path
|
|
@@ -40,7 +40,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
|
|
|
40
40
|
from .ProgramRunnerPopen import ProgramRunnerPopen
|
|
41
41
|
from .SCLog import SCLog, LogLevel
|
|
42
42
|
|
|
43
|
-
version = "4.4.
|
|
43
|
+
version = "4.4.28"
|
|
44
44
|
__version__ = version
|
|
45
45
|
|
|
46
46
|
class VSCodeWorkspaceShellTask:
|
|
@@ -212,12 +212,20 @@ class ScriptCollectionCore:
|
|
|
212
212
|
# Base-url of the GitHub-REST-API. This is an internal constant on purpose and deliberately not exposed as a parameter of the GitHub-functions.
|
|
213
213
|
__github_api_base_url: str = "https://api.github.com"
|
|
214
214
|
|
|
215
|
+
# The maximal amount of seconds the reader-threads get to transfer the remaining pipe-content into their queue after
|
|
216
|
+
# the process itself terminated. This is only relevant if a grandchild-process inherited the pipes and keeps them
|
|
217
|
+
# open, because in that case the reader-threads never reach the end of their pipe. See __read_popen_pipes.
|
|
218
|
+
__pipe_drain_grace_period_in_seconds: float = 10
|
|
219
|
+
|
|
220
|
+
# Whether the login to the registries which are defined for this machine was already done. See login_to_defined_docker_registries.
|
|
221
|
+
__already_logged_in_to_defined_docker_registries: bool = None
|
|
215
222
|
|
|
216
223
|
def __init__(self):
|
|
217
224
|
self.program_runner = ProgramRunnerPopen()
|
|
218
225
|
self.call_program_runner_directly = None
|
|
219
226
|
self.__mocked_program_calls = list[ScriptCollectionCore.__MockProgramCall]()
|
|
220
227
|
self.log = SCLog(None, LogLevel.Warning, False)
|
|
228
|
+
self.__already_logged_in_to_defined_docker_registries = False
|
|
221
229
|
|
|
222
230
|
@staticmethod
|
|
223
231
|
@GeneralUtilities.check_arguments
|
|
@@ -242,25 +250,26 @@ class ScriptCollectionCore:
|
|
|
242
250
|
return result
|
|
243
251
|
|
|
244
252
|
def __load_credentials_if_required_and_available(self,registry_url:str,registry_username:str,registry_password:str)->tuple[str,str]:
|
|
253
|
+
"""Returns the credentials for the given registry: the ones which were passed by the caller, or - if the caller passed none - the
|
|
254
|
+
ones which are available for this registry on this machine (see __get_docker_registry_credentials). The last matching entry is
|
|
255
|
+
used, because the entries are ordered by increasing precedence."""
|
|
245
256
|
if registry_url.startswith("https://"):
|
|
246
257
|
registry_url=registry_url[len("https://"):]
|
|
247
258
|
if registry_password is None:
|
|
248
|
-
|
|
249
|
-
lines=GeneralUtilities.read_nonempty_lines_from_file(credential_file)[1:]
|
|
250
|
-
for line in lines:
|
|
251
|
-
splitted=line.split(";")
|
|
252
|
-
registry=splitted[0]
|
|
253
|
-
username=splitted[1]
|
|
254
|
-
password=splitted[2]
|
|
259
|
+
for registry,username,password in self.__get_docker_registry_credentials():
|
|
255
260
|
if registry_url==registry and (registry_username is None or username==registry_username):
|
|
256
261
|
registry_username=username
|
|
257
262
|
registry_password=password
|
|
258
|
-
break
|
|
259
263
|
else:
|
|
260
264
|
GeneralUtilities.assert_not_null(registry_username)
|
|
261
265
|
return (registry_username,registry_password)
|
|
262
266
|
|
|
263
267
|
def __get_docker_registry_credentials(self)->list[tuple[str,str,str]]:
|
|
268
|
+
"""Returns all registry-credentials which are available for the current process: the ones of the credentials-file of the
|
|
269
|
+
configuration-folder followed by the ones which are declared in the environment (see
|
|
270
|
+
get_docker_registry_credentials_from_environment_variables). The entries are ordered by increasing precedence: a registry which is
|
|
271
|
+
declared in both places is contained twice and the declaration of the environment is the last one, so it is the one which a login
|
|
272
|
+
executes last and which a lookup of credentials uses."""
|
|
264
273
|
result=[]
|
|
265
274
|
credential_file=self.__get_docker_registry_credentials_file()
|
|
266
275
|
if os.path.isfile(credential_file):
|
|
@@ -271,6 +280,40 @@ class ScriptCollectionCore:
|
|
|
271
280
|
username=splitted[1]
|
|
272
281
|
password=splitted[2]
|
|
273
282
|
result.append((registry,username,password))
|
|
283
|
+
result=result+self.get_docker_registry_credentials_from_environment_variables()
|
|
284
|
+
return result
|
|
285
|
+
|
|
286
|
+
@GeneralUtilities.check_arguments
|
|
287
|
+
def get_docker_registry_credentials_from_environment_variables(self)->list[tuple[str,str,str]]:
|
|
288
|
+
"""Returns the registry-credentials which are declared in the environment of the current process, as tuples of registry-address,
|
|
289
|
+
username and password. The credentials of a registry named '<registryname>' are declared by the environment-variables
|
|
290
|
+
'OCIRegistry_<registryname>_Address', 'OCIRegistry_<registryname>_Username' and 'OCIRegistry_<registryname>_Password'.
|
|
291
|
+
The environment is used in addition to the credentials-file of the configuration-folder because that folder belongs to the user of
|
|
292
|
+
the machine on which a command was started and is therefore not available inside a build-container, while an environment-variable is
|
|
293
|
+
passed into the container (see TFCPS_Tools_General.get_required_environment_variables). It is the same mechanism which makes a
|
|
294
|
+
private package-source available inside a container (see TFCPS_Tools_General.get_declared_package_sources).
|
|
295
|
+
The names of the environment-variables are treated case-insensitively because they are not case-sensitive on all operating-systems.
|
|
296
|
+
A registry-address may be declared with or without the scheme 'https://'; the scheme is removed because docker expects the address
|
|
297
|
+
of a registry without it."""
|
|
298
|
+
address_variable_pattern = re.compile(r"^OCIRegistry_(.+)_Address$", re.IGNORECASE)
|
|
299
|
+
environment_variables: dict[str, str] = {name.lower(): value for name, value in os.environ.items()}
|
|
300
|
+
result: list[tuple[str,str,str]] = []
|
|
301
|
+
for environment_variable_name, environment_variable_value in os.environ.items():
|
|
302
|
+
match = address_variable_pattern.match(environment_variable_name)
|
|
303
|
+
if match is None:
|
|
304
|
+
continue
|
|
305
|
+
registry_name: str = match.group(1).lower()
|
|
306
|
+
if not GeneralUtilities.string_has_content(environment_variable_value):
|
|
307
|
+
continue
|
|
308
|
+
address: str = environment_variable_value.strip()
|
|
309
|
+
if address.startswith("https://"):
|
|
310
|
+
address = address[len("https://"):]
|
|
311
|
+
username: str = environment_variables.get(f"ociregistry_{registry_name}_username")
|
|
312
|
+
password: str = environment_variables.get(f"ociregistry_{registry_name}_password")
|
|
313
|
+
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(username), f"No username is declared for the registry \"{registry_name}\" whose address is declared in the environment.")
|
|
314
|
+
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(password), f"No password is declared for the registry \"{registry_name}\" whose address is declared in the environment.")
|
|
315
|
+
result.append((address, username, password))
|
|
316
|
+
result.sort()#sorted so the order (and therefore the log-output) is deterministic
|
|
274
317
|
return result
|
|
275
318
|
|
|
276
319
|
def registry_contains_image(self,registry_url:str,image:str,registry_username:str,registry_password:str)->bool:
|
|
@@ -348,6 +391,12 @@ class ScriptCollectionCore:
|
|
|
348
391
|
return result
|
|
349
392
|
|
|
350
393
|
def login_to_defined_docker_registries(self)->None:
|
|
394
|
+
"""Logs in to every registry for which credentials are available on this machine (see __get_docker_registry_credentials).
|
|
395
|
+
The login is only executed once per instance: the credentials do not change while a process runs, and the login has to be
|
|
396
|
+
ensured by everything which accesses a registry, so without this the same logins would be executed over and over again."""
|
|
397
|
+
if self.__already_logged_in_to_defined_docker_registries:
|
|
398
|
+
return
|
|
399
|
+
self.__already_logged_in_to_defined_docker_registries=True
|
|
351
400
|
registries=self.__get_docker_registry_credentials()
|
|
352
401
|
if len(registries)==0:
|
|
353
402
|
self.log.log("No docker registry credentials defined. Skipping docker login.",LogLevel.Debug)
|
|
@@ -376,6 +425,18 @@ class ScriptCollectionCore:
|
|
|
376
425
|
program_result = self.run_program("docker", f"image inspect {image}:{tag}", throw_exception_if_exitcode_is_not_zero=False)
|
|
377
426
|
return program_result[0] == 0
|
|
378
427
|
|
|
428
|
+
@GeneralUtilities.check_arguments
|
|
429
|
+
def image_is_available_in_registry(self, image: str, tag: str) -> bool:
|
|
430
|
+
"""Returns True if and only if the image is available with this tag in the registry it addresses, using the credentials the
|
|
431
|
+
docker-client of the current process has (see login_to_defined_docker_registries).
|
|
432
|
+
'image' must be the image-address without tag, for example "myregistry.example.com/debian".
|
|
433
|
+
Only the manifest of the image is inspected, so the check costs one request to the registry and does not download the image.
|
|
434
|
+
The possible reasons why an image is not available (the registry is not reachable, the image does not exist there, the
|
|
435
|
+
credentials are missing or do not permit the access) are not distinguished: in all of these cases the image can not be pulled
|
|
436
|
+
from this address, which is what the caller has to know."""
|
|
437
|
+
program_result = self.run_program("docker", f"buildx imagetools inspect {image}:{tag}", throw_exception_if_exitcode_is_not_zero=False, print_errors_as_information=True, print_live_output=self.log.loglevel == LogLevel.Debug)
|
|
438
|
+
return program_result[0] == 0
|
|
439
|
+
|
|
379
440
|
@GeneralUtilities.check_arguments
|
|
380
441
|
def docker_pull(self, image: str, tag: str, force: bool = False) -> None:
|
|
381
442
|
"""Ensures that the image is available with this tag in the local image-store of the docker-daemon.
|
|
@@ -2282,20 +2343,54 @@ resolving a name requires fontconfig, which does not exist on every system ffmpe
|
|
|
2282
2343
|
file.close()
|
|
2283
2344
|
|
|
2284
2345
|
@staticmethod
|
|
2285
|
-
def
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2346
|
+
def __dequeue_lines(queue: Queue) -> list[str]:
|
|
2347
|
+
"""Removes all lines which are currently available in "queue" and returns them without their line-breaks.
|
|
2348
|
+
Lines without content are discarded."""
|
|
2349
|
+
lines: list[str] = []
|
|
2350
|
+
while not queue.empty():
|
|
2351
|
+
line: str = queue.get_nowait()
|
|
2352
|
+
line = line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
|
|
2353
|
+
if GeneralUtilities.string_has_content(line):
|
|
2354
|
+
lines.append(line)
|
|
2355
|
+
return lines
|
|
2291
2356
|
|
|
2292
|
-
|
|
2293
|
-
|
|
2357
|
+
@staticmethod
|
|
2358
|
+
def __process_stdout_lines(queue: Queue, stdout_result: list[str], print_live_output: bool, log: SCLog) -> None:
|
|
2359
|
+
for out_line in ScriptCollectionCore.__dequeue_lines(queue):
|
|
2360
|
+
stdout_result.append(out_line)
|
|
2361
|
+
if print_live_output:
|
|
2362
|
+
loglevel = LogLevel.Information
|
|
2363
|
+
if out_line.startswith("Debug: "):
|
|
2364
|
+
loglevel = LogLevel.Debug
|
|
2365
|
+
out_line = out_line[len("Debug: "):]
|
|
2366
|
+
if out_line.startswith("Diagnostic: "):
|
|
2367
|
+
loglevel = LogLevel.Diagnostic
|
|
2368
|
+
out_line = out_line[len("Diagnostic: "):]
|
|
2369
|
+
log.log(out_line, loglevel)
|
|
2294
2370
|
|
|
2295
|
-
|
|
2296
|
-
|
|
2371
|
+
@staticmethod
|
|
2372
|
+
def __process_stderr_lines(queue: Queue, stderr_result: list[str], print_live_output: bool, print_errors_as_information: bool, log: SCLog) -> None:
|
|
2373
|
+
for err_line in ScriptCollectionCore.__dequeue_lines(queue):
|
|
2374
|
+
stderr_result.append(err_line)
|
|
2375
|
+
if print_live_output:
|
|
2376
|
+
loglevel = LogLevel.Error
|
|
2377
|
+
if err_line.startswith("Warning: "):
|
|
2378
|
+
loglevel = LogLevel.Warning
|
|
2379
|
+
err_line = err_line[len("Warning: "):]
|
|
2380
|
+
if print_errors_as_information: # "errors" in "print_errors_as_information" means: all what is written to std-err
|
|
2381
|
+
loglevel = LogLevel.Information
|
|
2382
|
+
log.log(err_line, loglevel)
|
|
2297
2383
|
|
|
2298
|
-
|
|
2384
|
+
@staticmethod
|
|
2385
|
+
def __reading_is_finished(readers: list, q_stdout: Queue, q_stderr: Queue, process_end_time: float) -> bool:
|
|
2386
|
+
"""Decides whether all output of an already terminated process was read.
|
|
2387
|
+
Checking the queues alone is not sufficient: a queue can be empty although the according reader-thread did
|
|
2388
|
+
not transfer the content of its pipe into the queue yet, which would silently discard the whole output of
|
|
2389
|
+
short-running processes. Therefore the state of the reader-threads is checked first."""
|
|
2390
|
+
readers_are_finished: bool = all(reader.done() for reader in readers)
|
|
2391
|
+
if not readers_are_finished and (time.monotonic() - process_end_time) < ScriptCollectionCore.__pipe_drain_grace_period_in_seconds:
|
|
2392
|
+
return False
|
|
2393
|
+
return q_stdout.empty() and q_stderr.empty()
|
|
2299
2394
|
|
|
2300
2395
|
@staticmethod
|
|
2301
2396
|
def __read_popen_pipes(p: Popen, print_live_output: bool, print_errors_as_information: bool, log: SCLog, timeoutInSeconds: int = None) -> tuple[list[str], list[str]]:
|
|
@@ -2305,62 +2400,40 @@ resolving a name requires fontconfig, which does not exist on every system ffmpe
|
|
|
2305
2400
|
timeout_enabled: bool = timeoutInSeconds is not None and 0 < timeoutInSeconds
|
|
2306
2401
|
start_time: float = time.monotonic()
|
|
2307
2402
|
timed_out: bool = False
|
|
2403
|
+
process_end_time: float = None # the moment the process terminated; None as long as it is still running.
|
|
2308
2404
|
with ThreadPoolExecutor(2) as pool:
|
|
2309
2405
|
q_stdout = Queue()
|
|
2310
2406
|
q_stderr = Queue()
|
|
2311
2407
|
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2408
|
+
readers: list = [
|
|
2409
|
+
pool.submit(ScriptCollectionCore.__enqueue_output, p.stdout, q_stdout),
|
|
2410
|
+
pool.submit(ScriptCollectionCore.__enqueue_output, p.stderr, q_stderr),
|
|
2411
|
+
]
|
|
2316
2412
|
|
|
2317
2413
|
stdout_result: list[str] = []
|
|
2318
2414
|
stderr_result: list[str] = []
|
|
2319
2415
|
|
|
2320
|
-
while
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
loglevel = LogLevel.Information
|
|
2330
|
-
if out_line.startswith("Debug: "):
|
|
2331
|
-
loglevel = LogLevel.Debug
|
|
2332
|
-
out_line = out_line[len("Debug: "):]
|
|
2333
|
-
if out_line.startswith("Diagnostic: "):
|
|
2334
|
-
loglevel = LogLevel.Diagnostic
|
|
2335
|
-
out_line = out_line[len("Diagnostic: "):]
|
|
2336
|
-
log.log(out_line, loglevel)
|
|
2337
|
-
except Empty:
|
|
2338
|
-
reading_stdout_last_time_resulted_in_exception = True
|
|
2339
|
-
|
|
2340
|
-
try:
|
|
2341
|
-
while not q_stderr.empty():
|
|
2342
|
-
err_line: str = q_stderr.get_nowait()
|
|
2343
|
-
err_line = err_line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
|
|
2344
|
-
if GeneralUtilities.string_has_content(err_line):
|
|
2345
|
-
stderr_result.append(err_line)
|
|
2346
|
-
reading_stderr_last_time_resulted_in_exception = False
|
|
2347
|
-
if print_live_output:
|
|
2348
|
-
loglevel = LogLevel.Error
|
|
2349
|
-
if err_line.startswith("Warning: "):
|
|
2350
|
-
loglevel = LogLevel.Warning
|
|
2351
|
-
err_line = err_line[len("Warning: "):]
|
|
2352
|
-
if print_errors_as_information: # "errors" in "print_errors_as_information" means: all what is written to std-err
|
|
2353
|
-
loglevel = LogLevel.Information
|
|
2354
|
-
log.log(err_line, loglevel)
|
|
2355
|
-
except Empty:
|
|
2356
|
-
reading_stderr_last_time_resulted_in_exception = True
|
|
2416
|
+
while True:
|
|
2417
|
+
ScriptCollectionCore.__process_stdout_lines(q_stdout, stdout_result, print_live_output, log)
|
|
2418
|
+
ScriptCollectionCore.__process_stderr_lines(q_stderr, stderr_result, print_live_output, print_errors_as_information, log)
|
|
2419
|
+
|
|
2420
|
+
if process_end_time is None and p.poll() is not None:
|
|
2421
|
+
process_end_time = time.monotonic()
|
|
2422
|
+
|
|
2423
|
+
if process_end_time is not None and ScriptCollectionCore.__reading_is_finished(readers, q_stdout, q_stderr, process_end_time):
|
|
2424
|
+
break
|
|
2357
2425
|
|
|
2358
2426
|
if timeout_enabled and timeoutInSeconds < (time.monotonic() - start_time):
|
|
2359
2427
|
timed_out = True
|
|
2360
2428
|
p.kill() # the process exceeded its timeout; kill it so the reader-threads get EOF and this loop ends.
|
|
2361
2429
|
break
|
|
2362
2430
|
|
|
2363
|
-
time.sleep(0.01) # this is
|
|
2431
|
+
time.sleep(0.01) # this is the poll-interval; without it the loop would consume a whole cpu-core
|
|
2432
|
+
|
|
2433
|
+
# The reader-threads can have enqueued further lines between the last processing above and the end of the
|
|
2434
|
+
# loop. Without processing them here this output would be lost.
|
|
2435
|
+
ScriptCollectionCore.__process_stdout_lines(q_stdout, stdout_result, print_live_output, log)
|
|
2436
|
+
ScriptCollectionCore.__process_stderr_lines(q_stderr, stderr_result, print_live_output, print_errors_as_information, log)
|
|
2364
2437
|
|
|
2365
2438
|
if timed_out:
|
|
2366
2439
|
raise TimeoutError(f"The process with process-id {p_id} did not finish within the configured timeout of {timeoutInSeconds} second(s) and was killed.")
|
|
@@ -4249,6 +4322,8 @@ OCR-content:
|
|
|
4249
4322
|
"""This function assumes that all files are valid xliff2 files and that the base file is the reference for syncing.
|
|
4250
4323
|
This function adds new entries from the base file to the language files if they do not already exist using the value from base_file.
|
|
4251
4324
|
This function removes entries from the language files if they do not exist in the base file anymore.
|
|
4325
|
+
This function updates the source-text of an entry which exists in a language file and in the base file but whose source-texts differ,
|
|
4326
|
+
see __update_outdated_source_of_unit.
|
|
4252
4327
|
In the end the updated language files are written to the disk. The base file is not changed.
|
|
4253
4328
|
Returns for each language (derived from the filename of the language-file) a dict which contains for each TranslationState
|
|
4254
4329
|
the percentage (as a value between 0.0 and 1.0) of translation-units which are in that state."""
|
|
@@ -4321,6 +4396,10 @@ OCR-content:
|
|
|
4321
4396
|
new_unit = copy.deepcopy(base_units[unit_id])
|
|
4322
4397
|
lang_file_element.append(new_unit)
|
|
4323
4398
|
|
|
4399
|
+
# Update the source of the units which exist in the base file as well as in the language file
|
|
4400
|
+
for unit_id in base_ids & lang_ids:
|
|
4401
|
+
self.__update_outdated_source_of_unit(base_units[unit_id], lang_units[unit_id], NSMAP)
|
|
4402
|
+
|
|
4324
4403
|
# Reorder units to match base order
|
|
4325
4404
|
current_units = {
|
|
4326
4405
|
unit.get("id"): unit
|
|
@@ -4456,6 +4535,29 @@ OCR-content:
|
|
|
4456
4535
|
GeneralUtilities.write_text_to_file(target_file, json.dumps(spec, indent=2, ensure_ascii=False)+"\n")
|
|
4457
4536
|
|
|
4458
4537
|
|
|
4538
|
+
@GeneralUtilities.check_arguments
|
|
4539
|
+
def __update_outdated_source_of_unit(self,base_unit:ET.Element, language_unit:ET.Element, nsmap:dict[str,str])->None:
|
|
4540
|
+
"""Overwrites the source-text of every segment of language_unit by the source-text of the segment at the same position of base_unit,
|
|
4541
|
+
if the two differ, and sets that segment back to the state "initial".
|
|
4542
|
+
The source-text of a unit belongs to the base file: it is the text which gets translated, so a language file which still carries an
|
|
4543
|
+
older source-text claims to translate a text which does not exist anymore - and the target which belongs to that older source-text
|
|
4544
|
+
is not a translation of the current text either, which is why the segment goes back to the state a translation is created from (see
|
|
4545
|
+
translate_xlf_file). The target itself is deliberately kept, so that a translation which was written by a human is still there as
|
|
4546
|
+
long as no new translation was created for the new source-text."""
|
|
4547
|
+
base_segments = base_unit.findall("x:segment", namespaces=nsmap)
|
|
4548
|
+
language_segments = language_unit.findall("x:segment", namespaces=nsmap)
|
|
4549
|
+
for index, language_segment in enumerate(language_segments):
|
|
4550
|
+
if len(base_segments) <= index:
|
|
4551
|
+
continue
|
|
4552
|
+
base_source = base_segments[index].find("x:source", namespaces=nsmap)
|
|
4553
|
+
language_source = language_segment.find("x:source", namespaces=nsmap)
|
|
4554
|
+
if base_source is None or language_source is None:
|
|
4555
|
+
continue
|
|
4556
|
+
if language_source.text == base_source.text:
|
|
4557
|
+
continue
|
|
4558
|
+
language_source.text = base_source.text
|
|
4559
|
+
language_segment.set("state", "initial")
|
|
4560
|
+
|
|
4459
4561
|
@GeneralUtilities.check_arguments
|
|
4460
4562
|
def sync_xlf2_files(self,prefix:str, languages:list[str], folder:str)->dict[str,dict[int, float]]:
|
|
4461
4563
|
#languages=["de", "fr"] for example. the default-language (usually english) must not be included.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scriptcollection
|
|
3
|
-
Version: 4.4.
|
|
3
|
+
Version: 4.4.28
|
|
4
4
|
Summary: The ScriptCollection is the place for reusable scripts.
|
|
5
5
|
Author-email: Marius Göcke <marius.goecke@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/anionDev/ScriptCollection
|
|
@@ -12,12 +12,12 @@ ScriptCollection/ProgramRunnerSudo.py,sha256=ot9Ib-plQ8wejR9M2XXfnx6GR-va9k8SH1g
|
|
|
12
12
|
ScriptCollection/RepositoryDiagramGenerator.py,sha256=K_u2NW_p4ozdWD50irJEkkcDVa70TcUJpZN63TeXSLw,7092
|
|
13
13
|
ScriptCollection/RepositoryHistoryAnalyzer.py,sha256=rD8eC5lMUe5MHmCBHfYft_G-N9IbvtsFGkU7EMwPjrY,5779
|
|
14
14
|
ScriptCollection/SCLog.py,sha256=Fjf8Pdi63cXHGMxL_aI6A0dwGm7QIfrv3czjvpSUlaM,4705
|
|
15
|
-
ScriptCollection/ScriptCollectionCore.py,sha256=
|
|
15
|
+
ScriptCollection/ScriptCollectionCore.py,sha256=nwxFn7tXBV95ixMPgTMQgZ9apII1j_OBdWxZJlXXukA,285393
|
|
16
16
|
ScriptCollection/TestKind.py,sha256=EBTLR8yBz5VmWA2NZIpDTbQNg6tkfv_kAsIzEJ--mCI,1756
|
|
17
17
|
ScriptCollection/UnassignedVariablesCheck.py,sha256=8xNcp1D5HeOLO1a1NEerq2TsnpFhw7yzYPkaay-oisY,3804
|
|
18
18
|
ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
-
ScriptCollection/OCIImages/AbstractImageHandler.py,sha256=
|
|
20
|
-
ScriptCollection/OCIImages/OCIImageManager.py,sha256=
|
|
19
|
+
ScriptCollection/OCIImages/AbstractImageHandler.py,sha256=W3ZG9IVhRwlWjFvQjIIXq1kz7xCGOOd3g4O0dzDnpkM,2470
|
|
20
|
+
ScriptCollection/OCIImages/OCIImageManager.py,sha256=mVfSeFXraeZIInjMISh_ZZVSvUO0wAqIb7NZRMivMG4,24123
|
|
21
21
|
ScriptCollection/OCIImages/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
22
22
|
ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py,sha256=rty9SnftreYWLIwvdUn4LhiHqCTevb_tWuYWoOnRJuo,732
|
|
23
23
|
ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py,sha256=M-4qxKWBZRtKdYQ-uXgfrxRBoZTXEjgqCxFXyFxTyzc,736
|
|
@@ -71,8 +71,8 @@ ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=yyjWPMHQjPLkML
|
|
|
71
71
|
ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
72
72
|
ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py,sha256=5z990qyO5durX07xRWRwju1hl6wpjQAy2f-yRwJ-7s0,11856
|
|
73
73
|
ScriptCollection/TFCPS/TypeScript/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
74
|
-
scriptcollection-4.4.
|
|
75
|
-
scriptcollection-4.4.
|
|
76
|
-
scriptcollection-4.4.
|
|
77
|
-
scriptcollection-4.4.
|
|
78
|
-
scriptcollection-4.4.
|
|
74
|
+
scriptcollection-4.4.28.dist-info/METADATA,sha256=9gAzv0f4JSOokYZOEtxExgHHa9II1AgHMOOe3T0quvo,7690
|
|
75
|
+
scriptcollection-4.4.28.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
76
|
+
scriptcollection-4.4.28.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
|
|
77
|
+
scriptcollection-4.4.28.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
|
|
78
|
+
scriptcollection-4.4.28.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|