ScriptCollection 4.4.27__py3-none-any.whl → 4.4.28__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,7 +10,8 @@ class AbstractImageHandler(ABC):
10
10
 
11
11
  def _protected_get_credentials_for_registry(self,registry_address:str,username_str:str)->tuple[str,str]:
12
12
  """return (username, password) for basic auth.
13
- Data will be taken from "~/.ScriptCollection/GlobalCache/RegistryCredentials.csv" if available.
13
+ Data will be taken from "~/.ScriptCollection/GlobalCache/RegistryCredentials.csv" or from the registries which are declared in the
14
+ environment (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) if available.
14
15
  If no credentials are available then None will be returned for the missing values."""
15
16
  raise NotImplementedError()
16
17
 
@@ -20,12 +20,16 @@ class OCIImageManager:
20
20
  #image-names for which it was already logged that the fallback-registry is used. The lookup happens for every image of
21
21
  #the repository and often several times per build, so without this the same hint would flood the log.
22
22
  __images_with_reported_fallback_registry:set[str] = None
23
+ #availability of an image-address with tag in its registry. The address of an image is resolved several times per build and
24
+ #the check costs a request to the registry, so its result is remembered for the runtime of this process.
25
+ __availability_of_image:dict[str,bool] = None
23
26
 
24
27
  def __init__(self,sc:ScriptCollectionCore):
25
28
  if sc is None:
26
29
  sc=ScriptCollectionCore()
27
30
  self.__sc=sc
28
31
  self.__images_with_reported_fallback_registry=set()
32
+ self.__availability_of_image=dict()
29
33
  self.image_handler=[
30
34
  ImageHandlerDebian(),
31
35
  ImageHandlerDebianSlim(),
@@ -105,34 +109,70 @@ class OCIImageManager:
105
109
  return line[2]
106
110
  raise ValueError(f"No tag defined for image \"{image_name}\".")
107
111
 
112
+ @GeneralUtilities.check_arguments
113
+ def get_custom_registry_address_for_image(self,image_name:str)->str:
114
+ """Returns the registry-address which is defined for the image in the machine-wide image-registries-file, always without tag.
115
+ Use custom_registry_is_defined to check whether such an address is defined for the image at all."""
116
+ global_docker_image_registries_file=self.get_global_docker_image_registries_file()
117
+ for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(global_docker_image_registries_file)[1:]]:
118
+ if image_name==line[0]:
119
+ return line[1]
120
+ raise ValueError(f"No custom registry is defined for image \"{image_name}\" in \"{global_docker_image_registries_file}\".")
121
+
122
+ @GeneralUtilities.check_arguments
123
+ def get_fallback_registry_address_for_image(self,repository:str,image_name:str)->str:
124
+ """Returns the upstream-registry-address which the repository defines for the image, always without tag."""
125
+ repository_image_definition_file=self.get_repository_image_definition_file(repository)
126
+ for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(repository_image_definition_file)[1:]]:
127
+ if image_name==line[0]:
128
+ return line[1]
129
+ raise ValueError(f"No registry defined for image \"{image_name}\" in \"{repository_image_definition_file}\".")
130
+
108
131
  @GeneralUtilities.check_arguments
109
132
  def get_registry_address_for_image(self,repository:str,image_name:str)->str:
110
- """Example: if image_name==Debian this function returns something like "myregistry.example.com/debian", always without tag."""
133
+ """Returns the registry-address the image has to be taken from, always without tag.
134
+ Example: if image_name==Debian this function returns something like "myregistry.example.com/debian".
135
+ This is the custom registry-address which is defined for this machine if the image is really available there, and the
136
+ upstream-registry-address which the repository defines otherwise."""
137
+ fallback_registry_address=self.get_fallback_registry_address_for_image(repository,image_name)
111
138
  if self.custom_registry_is_defined(image_name):
112
- #return image from custom registry-address
113
- global_docker_image_registries_file=self.get_global_docker_image_registries_file()
114
- for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(global_docker_image_registries_file)[1:]]:
115
- if image_name==line[0]:
116
- return line[1]
139
+ custom_registry_address=self.get_custom_registry_address_for_image(image_name)
140
+ if self.__custom_registry_provides_image(custom_registry_address,self.get_tag_for_image(repository,image_name)):
141
+ return custom_registry_address
142
+ self.__report_usage_of_fallback_registry(image_name,fallback_registry_address,f"the image is not available in the custom registry \"{custom_registry_address}\"")
117
143
  else:
118
- #return fallback-registry-address
119
- repository_image_definition_file=self.get_repository_image_definition_file(repository)
120
- for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(repository_image_definition_file)[1:]]:
121
- if image_name==line[0]:
122
- self.__report_usage_of_fallback_registry(image_name,line[1])
123
- return line[1]
124
- raise ValueError(f"No registry defined for image \"{image_name}\".")
144
+ self.__report_usage_of_fallback_registry(image_name,fallback_registry_address,f"no custom registry is defined for the image in \"{self.get_global_docker_image_registries_file()}\"")
145
+ return fallback_registry_address
146
+
147
+ @GeneralUtilities.check_arguments
148
+ def __custom_registry_provides_image(self,custom_registry_address:str,tag:str)->bool:
149
+ """Returns whether the image can be used with this tag from the custom registry-address, which is the case when it is already
150
+ available in the local image-store of the docker-daemon or when the custom registry provides it.
151
+ The local image-store is checked first because an image which is already available there is not downloaded at all (see
152
+ ScriptCollectionCore.docker_pull), so asking the registry would only cost a request - and would make a build which does not need
153
+ any download depend on a reachable registry.
154
+ A custom registry is typically not publicly readable, so the login to the registries which are defined for this machine is done
155
+ before the registry is asked; without it the answer would be that the image is not available although it only is not readable
156
+ without credentials."""
157
+ image_with_tag=f"{custom_registry_address}:{tag}"
158
+ if image_with_tag not in self.__availability_of_image:
159
+ if self.__sc.local_docker_image_exists(custom_registry_address,tag):
160
+ self.__availability_of_image[image_with_tag]=True
161
+ else:
162
+ self.__sc.login_to_defined_docker_registries()
163
+ self.__availability_of_image[image_with_tag]=self.__sc.image_is_available_in_registry(custom_registry_address,tag)
164
+ return self.__availability_of_image[image_with_tag]
125
165
 
126
166
  @GeneralUtilities.check_arguments
127
- def __report_usage_of_fallback_registry(self,image_name:str,fallback_registry_address:str)->None:
128
- """Logs that the image is taken from the upstream-registry defined in the repository because no custom registry is defined for it.
129
- The fallback exists so that a freshly cloned product is buildable without further setup, but an upstream-registry is typically
130
- rate-limited, so using it is worth a hint."""
167
+ def __report_usage_of_fallback_registry(self,image_name:str,fallback_registry_address:str,reason:str)->None:
168
+ """Logs that the image is taken from the upstream-registry which the repository defines, together with the reason why the custom
169
+ registry is not used. The fallback exists so that a freshly cloned product is buildable without further setup and so that a custom
170
+ registry which does not provide the image does not break a build, but an upstream-registry is typically rate-limited, so using it
171
+ is worth a hint."""
131
172
  if image_name in self.__images_with_reported_fallback_registry:
132
173
  return
133
174
  self.__images_with_reported_fallback_registry.add(image_name)
134
- registries_file=self.get_global_docker_image_registries_file()
135
- self.__sc.log.log(f"No custom registry is defined for image \"{image_name}\" in \"{registries_file}\", so the fallback-registry \"{fallback_registry_address}\" is used.",LogLevel.Warning)
175
+ self.__sc.log.log(f"The fallback-registry \"{fallback_registry_address}\" is used for image \"{image_name}\" because {reason}.",LogLevel.Warning)
136
176
 
137
177
  @GeneralUtilities.check_arguments
138
178
  def get_registry_address_for_image_with_default_tag(self,repository:str,image_name:str)->str:
@@ -159,7 +199,10 @@ class OCIImageManager:
159
199
  try:
160
200
  addresses_to_check=[]
161
201
  if self.custom_registry_is_defined(image_name):
162
- addresses_to_check.append(self.get_registry_address_for_image(repository,image_name))
202
+ #the custom registry-address is used directly (and not get_registry_address_for_image) because this function does not
203
+ #pull an image but asks the registry which tags it has, so it does not matter whether that registry currently provides
204
+ #the image with the tag which is defined at the moment.
205
+ addresses_to_check.append(self.get_custom_registry_address_for_image(image_name))
163
206
  else:
164
207
  if search_in_custom_registry_only_if_available:
165
208
  raise ValueError(f"No custom registry defined for image {image_name}.")
@@ -15,7 +15,7 @@ import math
15
15
  import base64
16
16
  import os
17
17
  from html.parser import HTMLParser
18
- from queue import Queue, Empty
18
+ from queue import Queue
19
19
  from concurrent.futures import ThreadPoolExecutor
20
20
  import xml.etree.ElementTree as ET
21
21
  from pathlib import Path
@@ -40,7 +40,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
40
40
  from .ProgramRunnerPopen import ProgramRunnerPopen
41
41
  from .SCLog import SCLog, LogLevel
42
42
 
43
- version = "4.4.27"
43
+ version = "4.4.28"
44
44
  __version__ = version
45
45
 
46
46
  class VSCodeWorkspaceShellTask:
@@ -212,12 +212,20 @@ class ScriptCollectionCore:
212
212
  # Base-url of the GitHub-REST-API. This is an internal constant on purpose and deliberately not exposed as a parameter of the GitHub-functions.
213
213
  __github_api_base_url: str = "https://api.github.com"
214
214
 
215
+ # The maximal amount of seconds the reader-threads get to transfer the remaining pipe-content into their queue after
216
+ # the process itself terminated. This is only relevant if a grandchild-process inherited the pipes and keeps them
217
+ # open, because in that case the reader-threads never reach the end of their pipe. See __read_popen_pipes.
218
+ __pipe_drain_grace_period_in_seconds: float = 10
219
+
220
+ # Whether the login to the registries which are defined for this machine was already done. See login_to_defined_docker_registries.
221
+ __already_logged_in_to_defined_docker_registries: bool = None
215
222
 
216
223
  def __init__(self):
217
224
  self.program_runner = ProgramRunnerPopen()
218
225
  self.call_program_runner_directly = None
219
226
  self.__mocked_program_calls = list[ScriptCollectionCore.__MockProgramCall]()
220
227
  self.log = SCLog(None, LogLevel.Warning, False)
228
+ self.__already_logged_in_to_defined_docker_registries = False
221
229
 
222
230
  @staticmethod
223
231
  @GeneralUtilities.check_arguments
@@ -242,25 +250,26 @@ class ScriptCollectionCore:
242
250
  return result
243
251
 
244
252
  def __load_credentials_if_required_and_available(self,registry_url:str,registry_username:str,registry_password:str)->tuple[str,str]:
253
+ """Returns the credentials for the given registry: the ones which were passed by the caller, or - if the caller passed none - the
254
+ ones which are available for this registry on this machine (see __get_docker_registry_credentials). The last matching entry is
255
+ used, because the entries are ordered by increasing precedence."""
245
256
  if registry_url.startswith("https://"):
246
257
  registry_url=registry_url[len("https://"):]
247
258
  if registry_password is None:
248
- credential_file=self.__get_docker_registry_credentials_file()
249
- lines=GeneralUtilities.read_nonempty_lines_from_file(credential_file)[1:]
250
- for line in lines:
251
- splitted=line.split(";")
252
- registry=splitted[0]
253
- username=splitted[1]
254
- password=splitted[2]
259
+ for registry,username,password in self.__get_docker_registry_credentials():
255
260
  if registry_url==registry and (registry_username is None or username==registry_username):
256
261
  registry_username=username
257
262
  registry_password=password
258
- break
259
263
  else:
260
264
  GeneralUtilities.assert_not_null(registry_username)
261
265
  return (registry_username,registry_password)
262
266
 
263
267
  def __get_docker_registry_credentials(self)->list[tuple[str,str,str]]:
268
+ """Returns all registry-credentials which are available for the current process: the ones of the credentials-file of the
269
+ configuration-folder followed by the ones which are declared in the environment (see
270
+ get_docker_registry_credentials_from_environment_variables). The entries are ordered by increasing precedence: a registry which is
271
+ declared in both places is contained twice and the declaration of the environment is the last one, so it is the one which a login
272
+ executes last and which a lookup of credentials uses."""
264
273
  result=[]
265
274
  credential_file=self.__get_docker_registry_credentials_file()
266
275
  if os.path.isfile(credential_file):
@@ -271,6 +280,40 @@ class ScriptCollectionCore:
271
280
  username=splitted[1]
272
281
  password=splitted[2]
273
282
  result.append((registry,username,password))
283
+ result=result+self.get_docker_registry_credentials_from_environment_variables()
284
+ return result
285
+
286
+ @GeneralUtilities.check_arguments
287
+ def get_docker_registry_credentials_from_environment_variables(self)->list[tuple[str,str,str]]:
288
+ """Returns the registry-credentials which are declared in the environment of the current process, as tuples of registry-address,
289
+ username and password. The credentials of a registry named '<registryname>' are declared by the environment-variables
290
+ 'OCIRegistry_<registryname>_Address', 'OCIRegistry_<registryname>_Username' and 'OCIRegistry_<registryname>_Password'.
291
+ The environment is used in addition to the credentials-file of the configuration-folder because that folder belongs to the user of
292
+ the machine on which a command was started and is therefore not available inside a build-container, while an environment-variable is
293
+ passed into the container (see TFCPS_Tools_General.get_required_environment_variables). It is the same mechanism which makes a
294
+ private package-source available inside a container (see TFCPS_Tools_General.get_declared_package_sources).
295
+ The names of the environment-variables are treated case-insensitively because they are not case-sensitive on all operating-systems.
296
+ A registry-address may be declared with or without the scheme 'https://'; the scheme is removed because docker expects the address
297
+ of a registry without it."""
298
+ address_variable_pattern = re.compile(r"^OCIRegistry_(.+)_Address$", re.IGNORECASE)
299
+ environment_variables: dict[str, str] = {name.lower(): value for name, value in os.environ.items()}
300
+ result: list[tuple[str,str,str]] = []
301
+ for environment_variable_name, environment_variable_value in os.environ.items():
302
+ match = address_variable_pattern.match(environment_variable_name)
303
+ if match is None:
304
+ continue
305
+ registry_name: str = match.group(1).lower()
306
+ if not GeneralUtilities.string_has_content(environment_variable_value):
307
+ continue
308
+ address: str = environment_variable_value.strip()
309
+ if address.startswith("https://"):
310
+ address = address[len("https://"):]
311
+ username: str = environment_variables.get(f"ociregistry_{registry_name}_username")
312
+ password: str = environment_variables.get(f"ociregistry_{registry_name}_password")
313
+ GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(username), f"No username is declared for the registry \"{registry_name}\" whose address is declared in the environment.")
314
+ GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(password), f"No password is declared for the registry \"{registry_name}\" whose address is declared in the environment.")
315
+ result.append((address, username, password))
316
+ result.sort()#sorted so the order (and therefore the log-output) is deterministic
274
317
  return result
275
318
 
276
319
  def registry_contains_image(self,registry_url:str,image:str,registry_username:str,registry_password:str)->bool:
@@ -348,6 +391,12 @@ class ScriptCollectionCore:
348
391
  return result
349
392
 
350
393
  def login_to_defined_docker_registries(self)->None:
394
+ """Logs in to every registry for which credentials are available on this machine (see __get_docker_registry_credentials).
395
+ The login is only executed once per instance: the credentials do not change while a process runs, and the login has to be
396
+ ensured by everything which accesses a registry, so without this the same logins would be executed over and over again."""
397
+ if self.__already_logged_in_to_defined_docker_registries:
398
+ return
399
+ self.__already_logged_in_to_defined_docker_registries=True
351
400
  registries=self.__get_docker_registry_credentials()
352
401
  if len(registries)==0:
353
402
  self.log.log("No docker registry credentials defined. Skipping docker login.",LogLevel.Debug)
@@ -376,6 +425,18 @@ class ScriptCollectionCore:
376
425
  program_result = self.run_program("docker", f"image inspect {image}:{tag}", throw_exception_if_exitcode_is_not_zero=False)
377
426
  return program_result[0] == 0
378
427
 
428
+ @GeneralUtilities.check_arguments
429
+ def image_is_available_in_registry(self, image: str, tag: str) -> bool:
430
+ """Returns True if and only if the image is available with this tag in the registry it addresses, using the credentials the
431
+ docker-client of the current process has (see login_to_defined_docker_registries).
432
+ 'image' must be the image-address without tag, for example "myregistry.example.com/debian".
433
+ Only the manifest of the image is inspected, so the check costs one request to the registry and does not download the image.
434
+ The possible reasons why an image is not available (the registry is not reachable, the image does not exist there, the
435
+ credentials are missing or do not permit the access) are not distinguished: in all of these cases the image can not be pulled
436
+ from this address, which is what the caller has to know."""
437
+ program_result = self.run_program("docker", f"buildx imagetools inspect {image}:{tag}", throw_exception_if_exitcode_is_not_zero=False, print_errors_as_information=True, print_live_output=self.log.loglevel == LogLevel.Debug)
438
+ return program_result[0] == 0
439
+
379
440
  @GeneralUtilities.check_arguments
380
441
  def docker_pull(self, image: str, tag: str, force: bool = False) -> None:
381
442
  """Ensures that the image is available with this tag in the local image-store of the docker-daemon.
@@ -2282,20 +2343,54 @@ resolving a name requires fontconfig, which does not exist on every system ffmpe
2282
2343
  file.close()
2283
2344
 
2284
2345
  @staticmethod
2285
- def __continue_process_reading(pid: int, p: Popen, q_stdout: Queue, q_stderr: Queue, reading_stdout_last_time_resulted_in_exception: bool, reading_stderr_last_time_resulted_in_exception: bool):
2286
- if p.poll() is None:
2287
- return True
2288
-
2289
- # if reading_stdout_last_time_resulted_in_exception and reading_stderr_last_time_resulted_in_exception:
2290
- # return False
2346
+ def __dequeue_lines(queue: Queue) -> list[str]:
2347
+ """Removes all lines which are currently available in "queue" and returns them without their line-breaks.
2348
+ Lines without content are discarded."""
2349
+ lines: list[str] = []
2350
+ while not queue.empty():
2351
+ line: str = queue.get_nowait()
2352
+ line = line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
2353
+ if GeneralUtilities.string_has_content(line):
2354
+ lines.append(line)
2355
+ return lines
2291
2356
 
2292
- if not q_stdout.empty():
2293
- return True
2357
+ @staticmethod
2358
+ def __process_stdout_lines(queue: Queue, stdout_result: list[str], print_live_output: bool, log: SCLog) -> None:
2359
+ for out_line in ScriptCollectionCore.__dequeue_lines(queue):
2360
+ stdout_result.append(out_line)
2361
+ if print_live_output:
2362
+ loglevel = LogLevel.Information
2363
+ if out_line.startswith("Debug: "):
2364
+ loglevel = LogLevel.Debug
2365
+ out_line = out_line[len("Debug: "):]
2366
+ if out_line.startswith("Diagnostic: "):
2367
+ loglevel = LogLevel.Diagnostic
2368
+ out_line = out_line[len("Diagnostic: "):]
2369
+ log.log(out_line, loglevel)
2294
2370
 
2295
- if not q_stderr.empty():
2296
- return True
2371
+ @staticmethod
2372
+ def __process_stderr_lines(queue: Queue, stderr_result: list[str], print_live_output: bool, print_errors_as_information: bool, log: SCLog) -> None:
2373
+ for err_line in ScriptCollectionCore.__dequeue_lines(queue):
2374
+ stderr_result.append(err_line)
2375
+ if print_live_output:
2376
+ loglevel = LogLevel.Error
2377
+ if err_line.startswith("Warning: "):
2378
+ loglevel = LogLevel.Warning
2379
+ err_line = err_line[len("Warning: "):]
2380
+ if print_errors_as_information: # "errors" in "print_errors_as_information" means: all what is written to std-err
2381
+ loglevel = LogLevel.Information
2382
+ log.log(err_line, loglevel)
2297
2383
 
2298
- return False
2384
+ @staticmethod
2385
+ def __reading_is_finished(readers: list, q_stdout: Queue, q_stderr: Queue, process_end_time: float) -> bool:
2386
+ """Decides whether all output of an already terminated process was read.
2387
+ Checking the queues alone is not sufficient: a queue can be empty although the according reader-thread did
2388
+ not transfer the content of its pipe into the queue yet, which would silently discard the whole output of
2389
+ short-running processes. Therefore the state of the reader-threads is checked first."""
2390
+ readers_are_finished: bool = all(reader.done() for reader in readers)
2391
+ if not readers_are_finished and (time.monotonic() - process_end_time) < ScriptCollectionCore.__pipe_drain_grace_period_in_seconds:
2392
+ return False
2393
+ return q_stdout.empty() and q_stderr.empty()
2299
2394
 
2300
2395
  @staticmethod
2301
2396
  def __read_popen_pipes(p: Popen, print_live_output: bool, print_errors_as_information: bool, log: SCLog, timeoutInSeconds: int = None) -> tuple[list[str], list[str]]:
@@ -2305,62 +2400,40 @@ resolving a name requires fontconfig, which does not exist on every system ffmpe
2305
2400
  timeout_enabled: bool = timeoutInSeconds is not None and 0 < timeoutInSeconds
2306
2401
  start_time: float = time.monotonic()
2307
2402
  timed_out: bool = False
2403
+ process_end_time: float = None # the moment the process terminated; None as long as it is still running.
2308
2404
  with ThreadPoolExecutor(2) as pool:
2309
2405
  q_stdout = Queue()
2310
2406
  q_stderr = Queue()
2311
2407
 
2312
- pool.submit(ScriptCollectionCore.__enqueue_output, p.stdout, q_stdout)
2313
- pool.submit(ScriptCollectionCore.__enqueue_output, p.stderr, q_stderr)
2314
- reading_stdout_last_time_resulted_in_exception: bool = False
2315
- reading_stderr_last_time_resulted_in_exception: bool = False
2408
+ readers: list = [
2409
+ pool.submit(ScriptCollectionCore.__enqueue_output, p.stdout, q_stdout),
2410
+ pool.submit(ScriptCollectionCore.__enqueue_output, p.stderr, q_stderr),
2411
+ ]
2316
2412
 
2317
2413
  stdout_result: list[str] = []
2318
2414
  stderr_result: list[str] = []
2319
2415
 
2320
- while (ScriptCollectionCore.__continue_process_reading(p_id, p, q_stdout, q_stderr, reading_stdout_last_time_resulted_in_exception, reading_stderr_last_time_resulted_in_exception)):
2321
- try:
2322
- while not q_stdout.empty():
2323
- out_line: str = q_stdout.get_nowait()
2324
- out_line = out_line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
2325
- if GeneralUtilities.string_has_content(out_line):
2326
- stdout_result.append(out_line)
2327
- reading_stdout_last_time_resulted_in_exception = False
2328
- if print_live_output:
2329
- loglevel = LogLevel.Information
2330
- if out_line.startswith("Debug: "):
2331
- loglevel = LogLevel.Debug
2332
- out_line = out_line[len("Debug: "):]
2333
- if out_line.startswith("Diagnostic: "):
2334
- loglevel = LogLevel.Diagnostic
2335
- out_line = out_line[len("Diagnostic: "):]
2336
- log.log(out_line, loglevel)
2337
- except Empty:
2338
- reading_stdout_last_time_resulted_in_exception = True
2339
-
2340
- try:
2341
- while not q_stderr.empty():
2342
- err_line: str = q_stderr.get_nowait()
2343
- err_line = err_line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
2344
- if GeneralUtilities.string_has_content(err_line):
2345
- stderr_result.append(err_line)
2346
- reading_stderr_last_time_resulted_in_exception = False
2347
- if print_live_output:
2348
- loglevel = LogLevel.Error
2349
- if err_line.startswith("Warning: "):
2350
- loglevel = LogLevel.Warning
2351
- err_line = err_line[len("Warning: "):]
2352
- if print_errors_as_information: # "errors" in "print_errors_as_information" means: all what is written to std-err
2353
- loglevel = LogLevel.Information
2354
- log.log(err_line, loglevel)
2355
- except Empty:
2356
- reading_stderr_last_time_resulted_in_exception = True
2416
+ while True:
2417
+ ScriptCollectionCore.__process_stdout_lines(q_stdout, stdout_result, print_live_output, log)
2418
+ ScriptCollectionCore.__process_stderr_lines(q_stderr, stderr_result, print_live_output, print_errors_as_information, log)
2419
+
2420
+ if process_end_time is None and p.poll() is not None:
2421
+ process_end_time = time.monotonic()
2422
+
2423
+ if process_end_time is not None and ScriptCollectionCore.__reading_is_finished(readers, q_stdout, q_stderr, process_end_time):
2424
+ break
2357
2425
 
2358
2426
  if timeout_enabled and timeoutInSeconds < (time.monotonic() - start_time):
2359
2427
  timed_out = True
2360
2428
  p.kill() # the process exceeded its timeout; kill it so the reader-threads get EOF and this loop ends.
2361
2429
  break
2362
2430
 
2363
- time.sleep(0.01) # this is required to not finish too early
2431
+ time.sleep(0.01) # this is the poll-interval; without it the loop would consume a whole cpu-core
2432
+
2433
+ # The reader-threads can have enqueued further lines between the last processing above and the end of the
2434
+ # loop. Without processing them here this output would be lost.
2435
+ ScriptCollectionCore.__process_stdout_lines(q_stdout, stdout_result, print_live_output, log)
2436
+ ScriptCollectionCore.__process_stderr_lines(q_stderr, stderr_result, print_live_output, print_errors_as_information, log)
2364
2437
 
2365
2438
  if timed_out:
2366
2439
  raise TimeoutError(f"The process with process-id {p_id} did not finish within the configured timeout of {timeoutInSeconds} second(s) and was killed.")
@@ -4249,6 +4322,8 @@ OCR-content:
4249
4322
  """This function assumes that all files are valid xliff2 files and that the base file is the reference for syncing.
4250
4323
  This function adds new entries from the base file to the language files if they do not already exist using the value from base_file.
4251
4324
  This function removes entries from the language files if they do not exist in the base file anymore.
4325
+ This function updates the source-text of an entry which exists in a language file and in the base file but whose source-texts differ,
4326
+ see __update_outdated_source_of_unit.
4252
4327
  In the end the updated language files are written to the disk. The base file is not changed.
4253
4328
  Returns for each language (derived from the filename of the language-file) a dict which contains for each TranslationState
4254
4329
  the percentage (as a value between 0.0 and 1.0) of translation-units which are in that state."""
@@ -4321,6 +4396,10 @@ OCR-content:
4321
4396
  new_unit = copy.deepcopy(base_units[unit_id])
4322
4397
  lang_file_element.append(new_unit)
4323
4398
 
4399
+ # Update the source of the units which exist in the base file as well as in the language file
4400
+ for unit_id in base_ids & lang_ids:
4401
+ self.__update_outdated_source_of_unit(base_units[unit_id], lang_units[unit_id], NSMAP)
4402
+
4324
4403
  # Reorder units to match base order
4325
4404
  current_units = {
4326
4405
  unit.get("id"): unit
@@ -4456,6 +4535,29 @@ OCR-content:
4456
4535
  GeneralUtilities.write_text_to_file(target_file, json.dumps(spec, indent=2, ensure_ascii=False)+"\n")
4457
4536
 
4458
4537
 
4538
+ @GeneralUtilities.check_arguments
4539
+ def __update_outdated_source_of_unit(self,base_unit:ET.Element, language_unit:ET.Element, nsmap:dict[str,str])->None:
4540
+ """Overwrites the source-text of every segment of language_unit by the source-text of the segment at the same position of base_unit,
4541
+ if the two differ, and sets that segment back to the state "initial".
4542
+ The source-text of a unit belongs to the base file: it is the text which gets translated, so a language file which still carries an
4543
+ older source-text claims to translate a text which does not exist anymore - and the target which belongs to that older source-text
4544
+ is not a translation of the current text either, which is why the segment goes back to the state a translation is created from (see
4545
+ translate_xlf_file). The target itself is deliberately kept, so that a translation which was written by a human is still there as
4546
+ long as no new translation was created for the new source-text."""
4547
+ base_segments = base_unit.findall("x:segment", namespaces=nsmap)
4548
+ language_segments = language_unit.findall("x:segment", namespaces=nsmap)
4549
+ for index, language_segment in enumerate(language_segments):
4550
+ if len(base_segments) <= index:
4551
+ continue
4552
+ base_source = base_segments[index].find("x:source", namespaces=nsmap)
4553
+ language_source = language_segment.find("x:source", namespaces=nsmap)
4554
+ if base_source is None or language_source is None:
4555
+ continue
4556
+ if language_source.text == base_source.text:
4557
+ continue
4558
+ language_source.text = base_source.text
4559
+ language_segment.set("state", "initial")
4560
+
4459
4561
  @GeneralUtilities.check_arguments
4460
4562
  def sync_xlf2_files(self,prefix:str, languages:list[str], folder:str)->dict[str,dict[int, float]]:
4461
4563
  #languages=["de", "fr"] for example. the default-language (usually english) must not be included.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptcollection
3
- Version: 4.4.27
3
+ Version: 4.4.28
4
4
  Summary: The ScriptCollection is the place for reusable scripts.
5
5
  Author-email: Marius Göcke <marius.goecke@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/anionDev/ScriptCollection
@@ -12,12 +12,12 @@ ScriptCollection/ProgramRunnerSudo.py,sha256=ot9Ib-plQ8wejR9M2XXfnx6GR-va9k8SH1g
12
12
  ScriptCollection/RepositoryDiagramGenerator.py,sha256=K_u2NW_p4ozdWD50irJEkkcDVa70TcUJpZN63TeXSLw,7092
13
13
  ScriptCollection/RepositoryHistoryAnalyzer.py,sha256=rD8eC5lMUe5MHmCBHfYft_G-N9IbvtsFGkU7EMwPjrY,5779
14
14
  ScriptCollection/SCLog.py,sha256=Fjf8Pdi63cXHGMxL_aI6A0dwGm7QIfrv3czjvpSUlaM,4705
15
- ScriptCollection/ScriptCollectionCore.py,sha256=iWcjDRi8gfu9nt-RTm-JtJggElq-_HDOStDYov6eH-A,276359
15
+ ScriptCollection/ScriptCollectionCore.py,sha256=nwxFn7tXBV95ixMPgTMQgZ9apII1j_OBdWxZJlXXukA,285393
16
16
  ScriptCollection/TestKind.py,sha256=EBTLR8yBz5VmWA2NZIpDTbQNg6tkfv_kAsIzEJ--mCI,1756
17
17
  ScriptCollection/UnassignedVariablesCheck.py,sha256=8xNcp1D5HeOLO1a1NEerq2TsnpFhw7yzYPkaay-oisY,3804
18
18
  ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
19
- ScriptCollection/OCIImages/AbstractImageHandler.py,sha256=WO2M28c2tUwsXFNa2L9zqMhdlyObWm3qpPE0LdIs7SE,2315
20
- ScriptCollection/OCIImages/OCIImageManager.py,sha256=Xfp8ZawYC04_isfeSLm97xW83OhQtdo7j1wyRvmDOqE,20302
19
+ ScriptCollection/OCIImages/AbstractImageHandler.py,sha256=W3ZG9IVhRwlWjFvQjIIXq1kz7xCGOOd3g4O0dzDnpkM,2470
20
+ ScriptCollection/OCIImages/OCIImageManager.py,sha256=mVfSeFXraeZIInjMISh_ZZVSvUO0wAqIb7NZRMivMG4,24123
21
21
  ScriptCollection/OCIImages/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
22
22
  ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py,sha256=rty9SnftreYWLIwvdUn4LhiHqCTevb_tWuYWoOnRJuo,732
23
23
  ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py,sha256=M-4qxKWBZRtKdYQ-uXgfrxRBoZTXEjgqCxFXyFxTyzc,736
@@ -71,8 +71,8 @@ ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=yyjWPMHQjPLkML
71
71
  ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
72
72
  ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py,sha256=5z990qyO5durX07xRWRwju1hl6wpjQAy2f-yRwJ-7s0,11856
73
73
  ScriptCollection/TFCPS/TypeScript/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
74
- scriptcollection-4.4.27.dist-info/METADATA,sha256=Cij77fI-TR6ZbCJ71yzztVtCEE1pH21SzhNglSkXdvQ,7690
75
- scriptcollection-4.4.27.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
76
- scriptcollection-4.4.27.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
77
- scriptcollection-4.4.27.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
78
- scriptcollection-4.4.27.dist-info/RECORD,,
74
+ scriptcollection-4.4.28.dist-info/METADATA,sha256=9gAzv0f4JSOokYZOEtxExgHHa9II1AgHMOOe3T0quvo,7690
75
+ scriptcollection-4.4.28.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
76
+ scriptcollection-4.4.28.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
77
+ scriptcollection-4.4.28.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
78
+ scriptcollection-4.4.28.dist-info/RECORD,,