ScriptCollection 4.4.30__py3-none-any.whl → 4.4.31__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,8 +10,8 @@ class AbstractImageHandler(ABC):
10
10
 
11
11
  def _protected_get_credentials_for_registry(self,registry_address:str,username_str:str)->tuple[str,str]:
12
12
  """return (username, password) for basic auth.
13
- Data will be taken from "~/.ScriptCollection/GlobalCache/RegistryCredentials.csv" or from the registries which are declared in the
14
- environment (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) if available.
13
+ Data will be taken from the registries which are declared in the environment-variables-configuration-file or in the environment
14
+ (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) if available.
15
15
  If no credentials are available then None will be returned for the missing values."""
16
16
  raise NotImplementedError()
17
17
 
@@ -40,7 +40,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
40
40
  from .ProgramRunnerPopen import ProgramRunnerPopen
41
41
  from .SCLog import SCLog, LogLevel
42
42
 
43
- version = "4.4.30"
43
+ version = "4.4.31"
44
44
  __version__ = version
45
45
 
46
46
  class VSCodeWorkspaceShellTask:
@@ -220,6 +220,9 @@ class ScriptCollectionCore:
220
220
  # Whether the login to the registries which are defined for this machine was already done. See login_to_defined_docker_registries.
221
221
  __already_logged_in_to_defined_docker_registries: bool = None
222
222
 
223
+ # Kinds which are allowed in the environment-variables-configuration-file (see get_environment_variables_configuration_file).
224
+ __allowed_environment_variable_kinds: list[str] = ["literal", "hostenvvariable", "file"]
225
+
223
226
  def __init__(self):
224
227
  self.program_runner = ProgramRunnerPopen()
225
228
  self.call_program_runner_directly = None
@@ -243,42 +246,140 @@ class ScriptCollectionCore:
243
246
  return result
244
247
 
245
248
  @GeneralUtilities.check_arguments
246
- def get_registry_credentials_file_in_container(self)->str:
247
- """Returns the file inside a build-container to which the host mounts its registry-credentials-file (see
249
+ def get_tfcps_configuration_folder_in_container(self)->str:
250
+ """Returns the folder inside a build-container to which the host mounts the 'TFCPS'-folder of its configuration-folder (see
248
251
  TFCPS_CodeUnit_BuildCodeUnits.__run_scriptcollection_executable_in_container). It is an own path (and not the
249
252
  configuration-folder of the container-user) because the home-folder inside the container depends on the user the
250
- image runs as, while this path is defined by the mount and is therefore identical for both sides."""
251
- return "/Workspace/ScriptCollectionConfiguration/RegistryCredentials.csv"
252
-
253
- @GeneralUtilities.check_arguments
254
- def get_registry_credentials_file_in_configuration_folder(self)->str:
255
- """Returns the registry-credentials-file in the configuration-folder of the current user, and creates it if it does not exist yet.
256
- This is the file of the machine on which a command was started, so it is also the file which is mounted into a build-container
257
- (in contrast to __get_docker_registry_credentials_file, which inside a container resolves to that mount instead)."""
258
- result=os.path.join(self.get_global_cache_folder(),"RegistryCredentials.csv")
259
- if not os.path.isfile(result):
260
- GeneralUtilities.ensure_file_exists(result)
261
- GeneralUtilities.write_lines_to_file(result,["RegistryName;Username;Password"])
262
- return result
263
-
264
- def __get_docker_registry_credentials_file(self)->str:
265
- """Returns the file which maps a registry to the credentials which are used for it. The file which the host mounted into a
266
- build-container has precedence over the file in the configuration-folder of the current user: in a container that
267
- configuration-folder belongs to the container-user and therefore never contains the configuration of the machine on which the
268
- build was started."""
269
- mounted_file=self.get_registry_credentials_file_in_container()
253
+ image runs as, while this path is defined by the mount and is therefore identical for both sides.
254
+ The whole folder (and not only EnvironmentVariables.csv in it) is the unit which is mounted, because a value of that file can
255
+ point to a secret-file next to it ('file'-kind with a relative path, which is the recommended form): without the surrounding
256
+ folder such a value could not be resolved inside the container."""
257
+ return "/Workspace/ScriptCollectionConfiguration/TFCPS"
258
+
259
+ @GeneralUtilities.check_arguments
260
+ def get_environment_variables_file_in_container(self)->str:
261
+ """Returns the environment-variables-configuration-file inside the folder the host mounts into a build-container
262
+ (see get_tfcps_configuration_folder_in_container)."""
263
+ return os.path.join(self.get_tfcps_configuration_folder_in_container(),"EnvironmentVariables.csv")
264
+
265
+ @GeneralUtilities.check_arguments
266
+ def get_tfcps_configuration_folder(self)->str:
267
+ """Returns the 'TFCPS'-folder of the configuration-folder of the current user. It contains the environment-variables-configuration-file
268
+ and the secret-files and custom scripts which belong to it. This is the folder of the machine on which a command was started, so it
269
+ is also the folder which is mounted into a build-container (see get_tfcps_configuration_folder_in_container)."""
270
+ return os.path.join(self.get_scriptcollection_configuration_folder(),"TFCPS")
271
+
272
+ @GeneralUtilities.check_arguments
273
+ def get_environment_variables_configuration_file(self)->str:
274
+ """Returns the environment-variables-configuration-file in the configuration-folder of the current user. This is the file of the
275
+ machine on which a command was started, so it is also the file which is mounted into a build-container (in contrast to
276
+ get_global_environment_variables_configuration_file, which inside a container resolves to that mount instead). The file defines
277
+ where the values of the required environment-variables of a product (see TFCPS_Tools_General.get_required_environment_variables)
278
+ and of the OCI-registry-credentials (see get_docker_registry_credentials_from_environment_variables) come from."""
279
+ return os.path.join(self.get_tfcps_configuration_folder(),"EnvironmentVariables.csv")
280
+
281
+ def __get_global_environment_variables_configuration_file(self)->str:
282
+ """Returns the machine-wide environment-variables-configuration-file. The file which the host mounted into a build-container has
283
+ precedence over the file in the configuration-folder of the current user: in a container that configuration-folder belongs to the
284
+ container-user and therefore never contains the configuration of the machine on which the build was started."""
285
+ mounted_file=self.get_environment_variables_file_in_container()
270
286
  if os.path.isfile(mounted_file):
271
287
  return mounted_file
272
- return self.get_registry_credentials_file_in_configuration_folder()
288
+ return self.get_environment_variables_configuration_file()
289
+
290
+ def __read_environment_variables_configuration_file(self,configuration_file:str)->dict[str,tuple[str,str]]:
291
+ """Reads the environment-variables-configuration-file (columns 'EnvVariableName;Kind;Value') and returns a mapping of the name of
292
+ the variable to its kind and its value. The file is optional: if it does not exist, nothing is defined by it and all values have
293
+ to come from the environment."""
294
+ result: dict[str, tuple[str, str]] = {}
295
+ if not os.path.isfile(configuration_file):
296
+ return result
297
+ for entry in GeneralUtilities.read_csv_file(configuration_file, True):
298
+ GeneralUtilities.assert_condition(2 < len(entry), f"Invalid line in '{configuration_file}': every line must have the 3 columns 'EnvVariableName;Kind;Value' but '{';'.join(entry)}' has {len(entry)}.")
299
+ #the value itself may contain the separator-character (for example in a literal value), so everything behind the second column belongs to the value.
300
+ result[entry[0]] = (entry[1], ";".join(entry[2:]))
301
+ return result
302
+
303
+ def __resolve_environment_variable_value(self,env_variable_name:str,entry:tuple[str,str],configuration_file:str)->str:
304
+ """Resolves the value of an entry of the environment-variables-configuration-file. 'Kind' is one of 'literal' (the value is used
305
+ as-is), 'hostenvvariable' (the value is the name of an environment-variable which must be set on the current system) or 'file'
306
+ (the value is a path - '~' is expanded, a relative path is resolved against the configuration-folder - to a text-file whose
307
+ content, without surrounding whitespace, is used). A relative path is the recommended form for a secret-file, because it also
308
+ resolves correctly when the configuration-folder is mounted into a build-container."""
309
+ kind, value = entry
310
+ GeneralUtilities.assert_condition(kind in ScriptCollectionCore.__allowed_environment_variable_kinds, f"Unknown kind '{kind}' for environment-variable '{env_variable_name}' defined in '{configuration_file}'. Allowed values are: {', '.join(ScriptCollectionCore.__allowed_environment_variable_kinds)}.")
311
+ if kind == "literal":
312
+ return value
313
+ if kind == "hostenvvariable":
314
+ resolved_value = os.environ.get(value)
315
+ GeneralUtilities.assert_condition(resolved_value is not None, f"The environment-variable '{env_variable_name}' defined in '{configuration_file}' is supposed to be taken from the environment-variable '{value}', but that environment-variable is not set.")
316
+ return resolved_value
317
+ value_file = GeneralUtilities.resolve_relative_path(os.path.expanduser(value), os.path.dirname(configuration_file))
318
+ GeneralUtilities.assert_file_exists(value_file, f"The environment-variable '{env_variable_name}' defined in '{configuration_file}' is supposed to be taken from the file '{value_file}', but that file does not exist.")
319
+ return GeneralUtilities.read_text_from_file(value_file).strip()
320
+
321
+ @GeneralUtilities.check_arguments
322
+ def resolve_environment_variables(self,environment_variable_names:list[str],required_by:str)->dict[str,str]:
323
+ """Resolves the value of every given environment-variable. A value comes from one of two sources:
324
+ - the machine-wide environment-variables-configuration-file (see get_environment_variables_configuration_file /
325
+ get_environment_variables_file_in_container), if the variable is defined there.
326
+ - the environment of the current process otherwise. This is how a build-pipeline provides a value from its own secret-store.
327
+ The configuration-file has precedence, so a resolved value does not depend on what happens to be set in the environment of the
328
+ caller. If a value can not be determined in either way the resolution fails with a message which names both possibilities.
329
+ This function centralizes the resolution so every caller (the required environment-variables of a product as well as the
330
+ OCI-registry-credentials, see get_docker_registry_credentials_from_environment_variables) obtains identical values instead of
331
+ re-implementing the resolution-logic. 'required_by' names the thing which needs the values and is used in the error-message when a
332
+ value can not be determined."""
333
+ result: dict[str, str] = {}
334
+ configuration_file: str = self.__get_global_environment_variables_configuration_file()
335
+ entries: dict[str, tuple[str, str]] = self.__read_environment_variables_configuration_file(configuration_file)
336
+ for environment_variable_name in environment_variable_names:
337
+ if environment_variable_name in entries:
338
+ result[environment_variable_name] = self.__resolve_environment_variable_value(environment_variable_name, entries[environment_variable_name], configuration_file)
339
+ else:
340
+ value_from_environment: str = os.environ.get(environment_variable_name)
341
+ GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(value_from_environment), f"The value of the environment-variable '{environment_variable_name}' which is required by {required_by} is unknown: it is not defined in '{configuration_file}' and it is not set in the environment. Add a line '{environment_variable_name};<kind>;<value>' to that file (allowed kinds are: {', '.join(ScriptCollectionCore.__allowed_environment_variable_kinds)}) or provide the value as an environment-variable, which is how a build-pipeline usually provides it.")
342
+ result[environment_variable_name] = value_from_environment
343
+ return result
344
+
345
+ @GeneralUtilities.check_arguments
346
+ def resolve_environment_variable(self,environment_variable_name:str,required_by:str)->str:
347
+ """Resolves the value of a single environment-variable, see resolve_environment_variables."""
348
+ return self.resolve_environment_variables([environment_variable_name], required_by)[environment_variable_name]
349
+
350
+ def __get_defined_oci_registry_names(self)->list[str]:
351
+ """Returns the names of the OCI-registries for which an 'OCIRegistry_<name>_Address' is declared, either in the
352
+ environment-variables-configuration-file or in the environment of the current process (which is how a build-pipeline typically
353
+ provides it). '<name>' is chosen freely when the registry is declared; it only groups its Address/Username/Password together. Only
354
+ the names are collected here (cheaply, without resolving any value), so that resolving them further down does not read a secret
355
+ which is unrelated to an actually declared registry.
356
+ The configuration-file is scanned before the environment and a name found in both is kept with the casing of the
357
+ configuration-file: on Windows, os.environ normalizes a variable-name to uppercase as soon as it is set, so without this the same
358
+ registry declared in both places would be treated as two different registries (its 'OCIRegistry_<name>_Username/_Password' would
359
+ then be looked up with the mismatching, uppercased name and therefore be resolved from the environment instead of the file, which
360
+ would silently contradict the configuration-file's precedence)."""
361
+ address_variable_pattern = re.compile(r"^OCIRegistry_(.+)_Address$", re.IGNORECASE)
362
+ names_by_lowercase_name: dict[str,str] = {}
363
+ configuration_file = self.__get_global_environment_variables_configuration_file()
364
+ if os.path.isfile(configuration_file):
365
+ for entry in GeneralUtilities.read_csv_file(configuration_file, True):
366
+ match = address_variable_pattern.match(entry[0])
367
+ if match is not None:
368
+ names_by_lowercase_name.setdefault(match.group(1).lower(), match.group(1))
369
+ for environment_variable_name in os.environ.keys():
370
+ match = address_variable_pattern.match(environment_variable_name)
371
+ if match is not None:
372
+ names_by_lowercase_name.setdefault(match.group(1).lower(), match.group(1))
373
+ return sorted(names_by_lowercase_name.values(), key=str.lower)
273
374
 
274
375
  def __load_credentials_if_required_and_available(self,registry_url:str,registry_username:str,registry_password:str)->tuple[str,str]:
275
376
  """Returns the credentials for the given registry: the ones which were passed by the caller, or - if the caller passed none - the
276
- ones which are available for this registry on this machine (see __get_docker_registry_credentials). The last matching entry is
277
- used, because the entries are ordered by increasing precedence."""
377
+ ones which are declared for this registry on this machine (see get_docker_registry_credentials_from_environment_variables). The
378
+ last matching entry is used, because the entries are ordered by increasing precedence."""
278
379
  if registry_url.startswith("https://"):
279
380
  registry_url=registry_url[len("https://"):]
280
381
  if registry_password is None:
281
- for registry,username,password in self.__get_docker_registry_credentials():
382
+ for registry,username,password in self.get_docker_registry_credentials_from_environment_variables():
282
383
  if registry_url==registry and (registry_username is None or username==registry_username):
283
384
  registry_username=username
284
385
  registry_password=password
@@ -286,54 +387,36 @@ class ScriptCollectionCore:
286
387
  GeneralUtilities.assert_not_null(registry_username)
287
388
  return (registry_username,registry_password)
288
389
 
289
- def __get_docker_registry_credentials(self)->list[tuple[str,str,str]]:
290
- """Returns all registry-credentials which are available for the current process: the ones of the credentials-file of the
291
- configuration-folder followed by the ones which are declared in the environment (see
292
- get_docker_registry_credentials_from_environment_variables). The entries are ordered by increasing precedence: a registry which is
293
- declared in both places is contained twice and the declaration of the environment is the last one, so it is the one which a login
294
- executes last and which a lookup of credentials uses."""
295
- result=[]
296
- credential_file=self.__get_docker_registry_credentials_file()
297
- if os.path.isfile(credential_file):
298
- lines=GeneralUtilities.read_nonempty_lines_from_file(credential_file)[1:]
299
- for line in lines:
300
- splitted=line.split(";")
301
- registry=splitted[0]
302
- username=splitted[1]
303
- password=splitted[2]
304
- result.append((registry,username,password))
305
- result=result+self.get_docker_registry_credentials_from_environment_variables()
306
- return result
307
-
308
390
  @GeneralUtilities.check_arguments
309
391
  def get_docker_registry_credentials_from_environment_variables(self)->list[tuple[str,str,str]]:
310
- """Returns the registry-credentials which are declared in the environment of the current process, as tuples of registry-address,
311
- username and password. The credentials of a registry named '<registryname>' are declared by the environment-variables
312
- 'OCIRegistry_<registryname>_Address', 'OCIRegistry_<registryname>_Username' and 'OCIRegistry_<registryname>_Password'.
313
- The environment is used in addition to the credentials-file of the configuration-folder because that folder belongs to the user of
314
- the machine on which a command was started and is therefore not available inside a build-container, while an environment-variable is
315
- passed into the container (see TFCPS_Tools_General.get_required_environment_variables). It is the same mechanism which makes a
316
- private package-source available inside a container (see TFCPS_Tools_General.get_declared_package_sources).
392
+ """Returns the credentials of every OCI-registry which is declared on this machine, as tuples of registry-address, username and
393
+ password. A registry named '<name>' is declared by the three values 'OCIRegistry_<name>_Address',
394
+ 'OCIRegistry_<name>_Username' and 'OCIRegistry_<name>_Password'; '<name>' is chosen freely and only groups the three values
395
+ together. Every value is resolved the same way as a required environment-variable of a product (see
396
+ resolve_environment_variable): from the environment-variables-configuration-file if it is defined there, from the environment of
397
+ the current process otherwise. This is the only place where OCI-registry-credentials are configured - there is deliberately no
398
+ separate credentials-file, so that a build-pipeline which makes the configuration-folder available to the build (see
399
+ BuildRunnerConfiguration.md) does not need a second, registry-specific mount.
317
400
  The names of the environment-variables are treated case-insensitively because they are not case-sensitive on all operating-systems.
318
401
  A registry-address may be declared with or without the scheme 'https://'; the scheme is removed because docker expects the address
319
- of a registry without it."""
320
- address_variable_pattern = re.compile(r"^OCIRegistry_(.+)_Address$", re.IGNORECASE)
321
- environment_variables: dict[str, str] = {name.lower(): value for name, value in os.environ.items()}
402
+ of a registry without it.
403
+ A registry whose values can not be resolved in the current environment (for example because its password is declared as a
404
+ 'hostenvvariable' which is only set on the developer-machine, or because it is declared without credentials at all) is skipped
405
+ with a warning instead of letting the whole build fail: the registries are machine-wide, so a declaration which is unusable here
406
+ typically belongs to a different environment and must not break a build which does not even need that registry. The consequence of
407
+ a skipped registry is the documented one - its images are taken from the fallback-registry - and the warning names the reason."""
322
408
  result: list[tuple[str,str,str]] = []
323
- for environment_variable_name, environment_variable_value in os.environ.items():
324
- match = address_variable_pattern.match(environment_variable_name)
325
- if match is None:
326
- continue
327
- registry_name: str = match.group(1).lower()
328
- if not GeneralUtilities.string_has_content(environment_variable_value):
409
+ for registry_name in self.__get_defined_oci_registry_names():
410
+ required_by = f"the OCI-registry \"{registry_name}\""
411
+ try:
412
+ address: str = self.resolve_environment_variable(f"OCIRegistry_{registry_name}_Address", required_by).strip()
413
+ if address.startswith("https://"):
414
+ address = address[len("https://"):]
415
+ username: str = self.resolve_environment_variable(f"OCIRegistry_{registry_name}_Username", required_by)
416
+ password: str = self.resolve_environment_variable(f"OCIRegistry_{registry_name}_Password", required_by)
417
+ except Exception as exception:
418
+ self.log.log(f"The OCI-registry \"{registry_name}\" is declared but can not be used in this environment, so its images will be taken from the fallback-registry. Reason: {GeneralUtilities.exception_to_str(exception)}", LogLevel.Warning)
329
419
  continue
330
- address: str = environment_variable_value.strip()
331
- if address.startswith("https://"):
332
- address = address[len("https://"):]
333
- username: str = environment_variables.get(f"ociregistry_{registry_name}_username")
334
- password: str = environment_variables.get(f"ociregistry_{registry_name}_password")
335
- GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(username), f"No username is declared for the registry \"{registry_name}\" whose address is declared in the environment.")
336
- GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(password), f"No password is declared for the registry \"{registry_name}\" whose address is declared in the environment.")
337
420
  result.append((address, username, password))
338
421
  result.sort()#sorted so the order (and therefore the log-output) is deterministic
339
422
  return result
@@ -413,13 +496,13 @@ class ScriptCollectionCore:
413
496
  return result
414
497
 
415
498
  def login_to_defined_docker_registries(self)->None:
416
- """Logs in to every registry for which credentials are available on this machine (see __get_docker_registry_credentials).
499
+ """Logs in to every registry for which credentials are available on this machine (see get_docker_registry_credentials_from_environment_variables).
417
500
  The login is only executed once per instance: the credentials do not change while a process runs, and the login has to be
418
501
  ensured by everything which accesses a registry, so without this the same logins would be executed over and over again."""
419
502
  if self.__already_logged_in_to_defined_docker_registries:
420
503
  return
421
504
  self.__already_logged_in_to_defined_docker_registries=True
422
- registries=self.__get_docker_registry_credentials()
505
+ registries=self.get_docker_registry_credentials_from_environment_variables()
423
506
  if len(registries)==0:
424
507
  self.log.log("No docker registry credentials defined. Skipping docker login.",LogLevel.Debug)
425
508
  else:
@@ -411,17 +411,20 @@ class TFCPS_CodeUnit_BuildCodeUnits:
411
411
  if os.path.isfile(image_registries_file):
412
412
  mount_arguments += ["-v", f"{image_registries_file}:{oci_image_manager.get_image_registries_file_in_container()}:ro"]
413
413
 
414
- #mount the machine-wide registry-credentials-file of this host (if it exists), so the build inside the container can log in to the
415
- #registries which the images come from. Without the mount a custom registry which is not publicly readable can not be used inside the
416
- #container at all, because the configuration-folder of the container-user does not contain the credentials of this host. The mount is
417
- #read-only because a build never changes the file. The alternative way to provide the credentials are the environment-variables
418
- #'OCIRegistry_<registryname>_*' (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables), which is the
419
- #way a build-pipeline provides them from its own secret-store; both sources are used together.
420
- #Only the two image-related files are mounted and not the whole configuration-folder, so nothing else of it (in particular not
421
- #EnvironmentVariables.csv) is exposed to the container.
422
- registry_credentials_file: str = self.sc.get_registry_credentials_file_in_configuration_folder()
423
- if os.path.isfile(registry_credentials_file):
424
- mount_arguments += ["-v", f"{registry_credentials_file}:{self.sc.get_registry_credentials_file_in_container()}:ro"]
414
+ #mount the 'TFCPS'-folder of the configuration-folder of this host (if it exists), so the build inside the container resolves the
415
+ #required environment-variables of the product (see TFCPS_Tools_General.get_required_environment_variables) and the
416
+ #OCI-registry-credentials (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) exactly like a
417
+ #build on the host does - including a registry which is not publicly readable, which without this mount could not be used inside
418
+ #the container at all, because the configuration-folder of the container-user does not contain the credentials of this host.
419
+ #The whole folder is mounted and not only EnvironmentVariables.csv in it, because a value of that file can point to a secret-file
420
+ #next to it ('file'-kind with a relative path, which is the recommended form) and such a value is resolved relative to the file
421
+ #itself - so with only the file mounted, every entry of that kind would fail inside the container. Only this folder (and not the
422
+ #whole configuration-folder) is mounted, so nothing else of it is exposed to the container. The mount is read-only because a build
423
+ #never changes it. The alternative way to provide a value is an environment-variable of the build-pipeline itself (the pipeline's
424
+ #own secret-store); both sources are used together, the file taking precedence.
425
+ tfcps_configuration_folder: str = self.sc.get_tfcps_configuration_folder()
426
+ if os.path.isdir(tfcps_configuration_folder):
427
+ mount_arguments += ["-v", f"{tfcps_configuration_folder}:{self.sc.get_tfcps_configuration_folder_in_container()}:ro"]
425
428
 
426
429
  #pass the environment-variables which are declared as required in <repository>/.ScriptCollection/ProductInformation.xml into the container
427
430
  #so they do not have to be specified explicitly on every call. Their values are resolved from the user-specific
@@ -29,9 +29,6 @@ class TFCPS_Tools_General:
29
29
 
30
30
  __sc:ScriptCollectionCore=None
31
31
  oci_image_manager:OCIImageManager=None
32
- # Kinds which are allowed in the configuration-file that defines where the values of the required environment-variables come from
33
- # (see get_environment_variables_configuration_file).
34
- __allowed_environment_variable_kinds:list[str]=["literal","hostenvvariable","file"]
35
32
  # Relative path (inside the source-repository and inside the ScriptCollection.Resources-package) of the
36
33
  # file that pins the OpenAPIGenerator-version used as default when no repository-specific version is given.
37
34
  __openapigenerator_version_resource_relative_path:str="Dependencies/OpenAPIGenerator/Version.txt"
@@ -536,8 +533,11 @@ class TFCPS_Tools_General:
536
533
  def get_environment_variables_configuration_file(self) -> str:
537
534
  """Returns the path of the user-specific file which defines where the values of the required environment-variables come from.
538
535
  The file is looked up in the configuration-folder, so a build-container which gets that folder mounted (which is the recommended
539
- setup for a self-hosted build-runner) resolves the values exactly like a build on a host does."""
540
- return os.path.join(GeneralUtilities.get_scriptcollection_configuration_folder(), "TFCPS", "EnvironmentVariables.csv")
536
+ setup for a self-hosted build-runner) resolves the values exactly like a build on a host does.
537
+ This only returns the path of the file in the configuration-folder of the current user (used to know what to mount into a
538
+ locally-started build-container); resolving a value (resolve_environment_variables) also considers the file mounted into this
539
+ container, if there is one (see ScriptCollectionCore.get_environment_variables_file_in_container)."""
540
+ return self.__sc.get_environment_variables_configuration_file()
541
541
 
542
542
  @GeneralUtilities.check_arguments
543
543
  def get_required_environment_variables(self, repository: str) -> dict[str, str]:
@@ -559,57 +559,16 @@ class TFCPS_Tools_General:
559
559
  """Resolves the values of the given environment-variables (see get_required_environment_variables for the sources and their
560
560
  precedence). This is the same mechanism a codeunit-build uses for the variables a repository declares as required; it is available
561
561
  separately so a tool which needs a value without having such a declaration (for example something which runs inside the
562
- build-container) does not have to read the configuration or the environment on its own.
562
+ build-container) does not have to read the configuration or the environment on its own. Delegates to
563
+ ScriptCollectionCore.resolve_environment_variables, which is also used for the OCI-registry-credentials
564
+ (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables), so both use identical resolution-logic.
563
565
  'required_by' names the thing which needs the values and is used in the error-message when a value can not be determined."""
564
- result: dict[str, str] = {}
565
- configuration_file: str = self.get_environment_variables_configuration_file()
566
- entries: dict[str, tuple[str, str]] = self.__read_environment_variables_configuration_file(configuration_file)
567
- for environment_variable_name in environment_variable_names:
568
- if environment_variable_name in entries:
569
- result[environment_variable_name] = self.__resolve_environment_variable_value(environment_variable_name, entries[environment_variable_name], configuration_file)
570
- else:
571
- value_from_environment: str = os.environ.get(environment_variable_name)
572
- GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(value_from_environment), f"The value of the environment-variable '{environment_variable_name}' which is required by {required_by} is unknown: it is not defined in '{configuration_file}' and it is not set in the environment. Add a line '{environment_variable_name};<kind>;<value>' to that file (allowed kinds are: {', '.join(TFCPS_Tools_General.__allowed_environment_variable_kinds)}) or provide the value as an environment-variable, which is how a build-pipeline usually provides it.")
573
- result[environment_variable_name] = value_from_environment
574
- return result
566
+ return self.__sc.resolve_environment_variables(environment_variable_names, required_by)
575
567
 
576
568
  @GeneralUtilities.check_arguments
577
569
  def resolve_environment_variable(self, environment_variable_name: str, required_by: str) -> str:
578
570
  """Resolves the value of a single environment-variable, see resolve_environment_variables."""
579
- return self.resolve_environment_variables([environment_variable_name], required_by)[environment_variable_name]
580
-
581
- @GeneralUtilities.check_arguments
582
- def __read_environment_variables_configuration_file(self, configuration_file: str) -> dict[str, tuple[str, str]]:
583
- """Reads the configuration-file which defines where the values of environment-variables come from (columns
584
- 'EnvVariableName;Kind;Value') and returns a mapping of the name of the variable to its kind and its value.
585
- The file is optional: if it does not exist, nothing is defined by it and all values have to come from the environment."""
586
- result: dict[str, tuple[str, str]] = {}
587
- if not os.path.isfile(configuration_file):
588
- return result
589
- for entry in GeneralUtilities.read_csv_file(configuration_file, True):
590
- GeneralUtilities.assert_condition(2 < len(entry), f"Invalid line in '{configuration_file}': every line must have the 3 columns 'EnvVariableName;Kind;Value' but '{';'.join(entry)}' has {len(entry)}.")
591
- #the value itself may contain the separator-character (for example in a literal value), so everything behind the second column belongs to the value.
592
- result[entry[0]] = (entry[1], ";".join(entry[2:]))
593
- return result
594
-
595
- @GeneralUtilities.check_arguments
596
- def __resolve_environment_variable_value(self, env_variable_name: str, entry: tuple[str, str], configuration_file: str) -> str:
597
- """Resolves the value of an entry of the configuration-file. 'Kind' is one of 'literal' (the value is used as-is),
598
- 'hostenvvariable' (the value is the name of an environment-variable which must be set on the current system) or 'file' (the value
599
- is a path - '~' is expanded, a relative path is resolved against the configuration-folder - to a text-file whose content, without
600
- surrounding whitespace, is used). A relative path is the recommended form for a secret-file, because it also resolves correctly
601
- when the configuration-folder is mounted into a build-container."""
602
- kind, value = entry
603
- GeneralUtilities.assert_condition(kind in TFCPS_Tools_General.__allowed_environment_variable_kinds, f"Unknown kind '{kind}' for environment-variable '{env_variable_name}' defined in '{configuration_file}'. Allowed values are: {', '.join(TFCPS_Tools_General.__allowed_environment_variable_kinds)}.")
604
- if kind == "literal":
605
- return value
606
- if kind == "hostenvvariable":
607
- resolved_value = os.environ.get(value)
608
- GeneralUtilities.assert_condition(resolved_value is not None, f"The environment-variable '{env_variable_name}' defined in '{configuration_file}' is supposed to be taken from the environment-variable '{value}', but that environment-variable is not set.")
609
- return resolved_value
610
- value_file = GeneralUtilities.resolve_relative_path(os.path.expanduser(value), os.path.dirname(configuration_file))
611
- GeneralUtilities.assert_file_exists(value_file, f"The environment-variable '{env_variable_name}' defined in '{configuration_file}' is supposed to be taken from the file '{value_file}', but that file does not exist.")
612
- return GeneralUtilities.read_text_from_file(value_file).strip()
571
+ return self.__sc.resolve_environment_variable(environment_variable_name, required_by)
613
572
 
614
573
  @GeneralUtilities.check_arguments
615
574
  def ensure_required_environment_variables_are_set(self, repository: str) -> None:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptcollection
3
- Version: 4.4.30
3
+ Version: 4.4.31
4
4
  Summary: The ScriptCollection is the place for reusable scripts.
5
5
  Author-email: Marius Göcke <marius.goecke@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/anionDev/ScriptCollection
@@ -12,11 +12,11 @@ ScriptCollection/ProgramRunnerSudo.py,sha256=ot9Ib-plQ8wejR9M2XXfnx6GR-va9k8SH1g
12
12
  ScriptCollection/RepositoryDiagramGenerator.py,sha256=K_u2NW_p4ozdWD50irJEkkcDVa70TcUJpZN63TeXSLw,7092
13
13
  ScriptCollection/RepositoryHistoryAnalyzer.py,sha256=rD8eC5lMUe5MHmCBHfYft_G-N9IbvtsFGkU7EMwPjrY,5779
14
14
  ScriptCollection/SCLog.py,sha256=Fjf8Pdi63cXHGMxL_aI6A0dwGm7QIfrv3czjvpSUlaM,4705
15
- ScriptCollection/ScriptCollectionCore.py,sha256=BGPQ7NXBPAO2jWEhu9lj-nL4wRMCmDyXSa-61rpa5Y8,288002
15
+ ScriptCollection/ScriptCollectionCore.py,sha256=equQ2c2EpHDlp9-o6vX8ThfiL_0UJKQsAh3vIaQF3yQ,296980
16
16
  ScriptCollection/TestKind.py,sha256=EBTLR8yBz5VmWA2NZIpDTbQNg6tkfv_kAsIzEJ--mCI,1756
17
17
  ScriptCollection/UnassignedVariablesCheck.py,sha256=8xNcp1D5HeOLO1a1NEerq2TsnpFhw7yzYPkaay-oisY,3804
18
18
  ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
19
- ScriptCollection/OCIImages/AbstractImageHandler.py,sha256=W3ZG9IVhRwlWjFvQjIIXq1kz7xCGOOd3g4O0dzDnpkM,2470
19
+ ScriptCollection/OCIImages/AbstractImageHandler.py,sha256=UB0lUYdAjFk8RIgsikKELRWDMZPlR_zNgwvCJS1naVs,2455
20
20
  ScriptCollection/OCIImages/OCIImageManager.py,sha256=mVfSeFXraeZIInjMISh_ZZVSvUO0wAqIb7NZRMivMG4,24123
21
21
  ScriptCollection/OCIImages/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
22
22
  ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py,sha256=rty9SnftreYWLIwvdUn4LhiHqCTevb_tWuYWoOnRJuo,732
@@ -37,7 +37,7 @@ ScriptCollection/TFCPS/SCTaskRunnerServer.py,sha256=tub8_1QBu0nWAaGnBWp2nErOW-SO
37
37
  ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py,sha256=67_DQwoH5bhxxSmwtuAeoW1NOCMiLMzOy48OqlJX-KY,1744
38
38
  ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=umUfQDErp7xFLkDnSbQMxzUWnKtcGlrZnIvFcTpfGJs,40499
39
39
  ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=Mj9t1SyRpkYZE74cLsGWYv4xB89gXL2ld-RX3xvvGdw,9835
40
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=6FmsF2M3GI8-Xjay0oUaLPTQ8yrHVSUiF_6mc01YoRM,49601
40
+ ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=DaKH50q4_QusCdrv9RDR_HNQ1WYZP6NU7qUoP9sZcfk,50023
41
41
  ScriptCollection/TFCPS/TFCPS_CreateRelease.py,sha256=kpEmSnfzfgB4OjcCuS_X6jgW_ftcphxEnjjoLoAHTLY,6582
42
42
  ScriptCollection/TFCPS/TFCPS_Generic.py,sha256=mRBcf7XuNWOSXbZYc3NuJow18AMZ30D6Z-BtyNYfNcM,2072
43
43
  ScriptCollection/TFCPS/TFCPS_MergeToMain.py,sha256=Krt5oyGNcRAN3szsgU7LoTh8yVTOdHW-Kq7R9MDDV-Q,7673
@@ -45,7 +45,7 @@ ScriptCollection/TFCPS/TFCPS_MergeToStable.py,sha256=oYEhz5ZWMnbu97_CcYEqdDxakUF
45
45
  ScriptCollection/TFCPS/TFCPS_OCIImageSecretScan.py,sha256=rhw0glr4gZ5enG9faOD51xE4MMK1cIdNjUdaswiMsrY,14622
46
46
  ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py,sha256=XLVsFTA1jvfDbZ7wYJgIdEByZhF_bYVbA5jkisuvK4k,2322
47
47
  ScriptCollection/TFCPS/TFCPS_RemoteBuild.py,sha256=2Tx5HLk1ZftBWDCTiGZdP2QiiGNyWsEuTCSt33dNQjc,23891
48
- ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=eGtOfeJEiHV-JoxkbBGw0EP9vDJJAlQWKpJ0hUASwUY,160284
48
+ ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=y452sky-RdP__jHqewUvU1rUgJZM68sMZgOQkmSchtI,156175
49
49
  ScriptCollection/TFCPS/TFCPS_VisualRegressionTests.py,sha256=6_63udxHnPlu_TMk_ftLZTnzriJO8MCxVCVAsweZqHs,37073
50
50
  ScriptCollection/TFCPS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
51
51
  ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py,sha256=BNZPMjGiWLMBH7QDtP06WEVLMnI38NLX4Dpd13bSbi4,16614
@@ -71,8 +71,8 @@ ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=yyjWPMHQjPLkML
71
71
  ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
72
72
  ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py,sha256=5z990qyO5durX07xRWRwju1hl6wpjQAy2f-yRwJ-7s0,11856
73
73
  ScriptCollection/TFCPS/TypeScript/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
74
- scriptcollection-4.4.30.dist-info/METADATA,sha256=afxz3vYiDErPCZNzqM6zrv2xVN4yce3S1-MvBo6ebcg,7690
75
- scriptcollection-4.4.30.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
76
- scriptcollection-4.4.30.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
77
- scriptcollection-4.4.30.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
78
- scriptcollection-4.4.30.dist-info/RECORD,,
74
+ scriptcollection-4.4.31.dist-info/METADATA,sha256=rRsrPv4VZbIyRLyv53uQ6jqDWPkDm7BYnWy3zG3cYew,7690
75
+ scriptcollection-4.4.31.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
76
+ scriptcollection-4.4.31.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
77
+ scriptcollection-4.4.31.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
78
+ scriptcollection-4.4.31.dist-info/RECORD,,