ScriptCollection 4.4.29__py3-none-any.whl → 4.4.31__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,8 +10,8 @@ class AbstractImageHandler(ABC):
10
10
 
11
11
  def _protected_get_credentials_for_registry(self,registry_address:str,username_str:str)->tuple[str,str]:
12
12
  """return (username, password) for basic auth.
13
- Data will be taken from "~/.ScriptCollection/GlobalCache/RegistryCredentials.csv" or from the registries which are declared in the
14
- environment (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) if available.
13
+ Data will be taken from the registries which are declared in the environment-variables-configuration-file or in the environment
14
+ (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) if available.
15
15
  If no credentials are available then None will be returned for the missing values."""
16
16
  raise NotImplementedError()
17
17
 
@@ -40,7 +40,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
40
40
  from .ProgramRunnerPopen import ProgramRunnerPopen
41
41
  from .SCLog import SCLog, LogLevel
42
42
 
43
- version = "4.4.29"
43
+ version = "4.4.31"
44
44
  __version__ = version
45
45
 
46
46
  class VSCodeWorkspaceShellTask:
@@ -220,6 +220,9 @@ class ScriptCollectionCore:
220
220
  # Whether the login to the registries which are defined for this machine was already done. See login_to_defined_docker_registries.
221
221
  __already_logged_in_to_defined_docker_registries: bool = None
222
222
 
223
+ # Kinds which are allowed in the environment-variables-configuration-file (see get_environment_variables_configuration_file).
224
+ __allowed_environment_variable_kinds: list[str] = ["literal", "hostenvvariable", "file"]
225
+
223
226
  def __init__(self):
224
227
  self.program_runner = ProgramRunnerPopen()
225
228
  self.call_program_runner_directly = None
@@ -242,21 +245,141 @@ class ScriptCollectionCore:
242
245
  GeneralUtilities.ensure_directory_exists(result)
243
246
  return result
244
247
 
245
- def __get_docker_registry_credentials_file(self)->str:
246
- result=os.path.join(self.get_global_cache_folder(),"RegistryCredentials.csv")
247
- if not os.path.isfile(result):
248
- GeneralUtilities.ensure_file_exists(result)
249
- GeneralUtilities.write_lines_to_file(result,["RegistryName;Username;Password"])
248
+ @GeneralUtilities.check_arguments
249
+ def get_tfcps_configuration_folder_in_container(self)->str:
250
+ """Returns the folder inside a build-container to which the host mounts the 'TFCPS'-folder of its configuration-folder (see
251
+ TFCPS_CodeUnit_BuildCodeUnits.__run_scriptcollection_executable_in_container). It is an own path (and not the
252
+ configuration-folder of the container-user) because the home-folder inside the container depends on the user the
253
+ image runs as, while this path is defined by the mount and is therefore identical for both sides.
254
+ The whole folder (and not only EnvironmentVariables.csv in it) is the unit which is mounted, because a value of that file can
255
+ point to a secret-file next to it ('file'-kind with a relative path, which is the recommended form): without the surrounding
256
+ folder such a value could not be resolved inside the container."""
257
+ return "/Workspace/ScriptCollectionConfiguration/TFCPS"
258
+
259
+ @GeneralUtilities.check_arguments
260
+ def get_environment_variables_file_in_container(self)->str:
261
+ """Returns the environment-variables-configuration-file inside the folder the host mounts into a build-container
262
+ (see get_tfcps_configuration_folder_in_container)."""
263
+ return os.path.join(self.get_tfcps_configuration_folder_in_container(),"EnvironmentVariables.csv")
264
+
265
+ @GeneralUtilities.check_arguments
266
+ def get_tfcps_configuration_folder(self)->str:
267
+ """Returns the 'TFCPS'-folder of the configuration-folder of the current user. It contains the environment-variables-configuration-file
268
+ and the secret-files and custom scripts which belong to it. This is the folder of the machine on which a command was started, so it
269
+ is also the folder which is mounted into a build-container (see get_tfcps_configuration_folder_in_container)."""
270
+ return os.path.join(self.get_scriptcollection_configuration_folder(),"TFCPS")
271
+
272
+ @GeneralUtilities.check_arguments
273
+ def get_environment_variables_configuration_file(self)->str:
274
+ """Returns the environment-variables-configuration-file in the configuration-folder of the current user. This is the file of the
275
+ machine on which a command was started, so it is also the file which is mounted into a build-container (in contrast to
276
+ get_global_environment_variables_configuration_file, which inside a container resolves to that mount instead). The file defines
277
+ where the values of the required environment-variables of a product (see TFCPS_Tools_General.get_required_environment_variables)
278
+ and of the OCI-registry-credentials (see get_docker_registry_credentials_from_environment_variables) come from."""
279
+ return os.path.join(self.get_tfcps_configuration_folder(),"EnvironmentVariables.csv")
280
+
281
+ def __get_global_environment_variables_configuration_file(self)->str:
282
+ """Returns the machine-wide environment-variables-configuration-file. The file which the host mounted into a build-container has
283
+ precedence over the file in the configuration-folder of the current user: in a container that configuration-folder belongs to the
284
+ container-user and therefore never contains the configuration of the machine on which the build was started."""
285
+ mounted_file=self.get_environment_variables_file_in_container()
286
+ if os.path.isfile(mounted_file):
287
+ return mounted_file
288
+ return self.get_environment_variables_configuration_file()
289
+
290
+ def __read_environment_variables_configuration_file(self,configuration_file:str)->dict[str,tuple[str,str]]:
291
+ """Reads the environment-variables-configuration-file (columns 'EnvVariableName;Kind;Value') and returns a mapping of the name of
292
+ the variable to its kind and its value. The file is optional: if it does not exist, nothing is defined by it and all values have
293
+ to come from the environment."""
294
+ result: dict[str, tuple[str, str]] = {}
295
+ if not os.path.isfile(configuration_file):
296
+ return result
297
+ for entry in GeneralUtilities.read_csv_file(configuration_file, True):
298
+ GeneralUtilities.assert_condition(2 < len(entry), f"Invalid line in '{configuration_file}': every line must have the 3 columns 'EnvVariableName;Kind;Value' but '{';'.join(entry)}' has {len(entry)}.")
299
+ #the value itself may contain the separator-character (for example in a literal value), so everything behind the second column belongs to the value.
300
+ result[entry[0]] = (entry[1], ";".join(entry[2:]))
250
301
  return result
251
302
 
303
+ def __resolve_environment_variable_value(self,env_variable_name:str,entry:tuple[str,str],configuration_file:str)->str:
304
+ """Resolves the value of an entry of the environment-variables-configuration-file. 'Kind' is one of 'literal' (the value is used
305
+ as-is), 'hostenvvariable' (the value is the name of an environment-variable which must be set on the current system) or 'file'
306
+ (the value is a path - '~' is expanded, a relative path is resolved against the configuration-folder - to a text-file whose
307
+ content, without surrounding whitespace, is used). A relative path is the recommended form for a secret-file, because it also
308
+ resolves correctly when the configuration-folder is mounted into a build-container."""
309
+ kind, value = entry
310
+ GeneralUtilities.assert_condition(kind in ScriptCollectionCore.__allowed_environment_variable_kinds, f"Unknown kind '{kind}' for environment-variable '{env_variable_name}' defined in '{configuration_file}'. Allowed values are: {', '.join(ScriptCollectionCore.__allowed_environment_variable_kinds)}.")
311
+ if kind == "literal":
312
+ return value
313
+ if kind == "hostenvvariable":
314
+ resolved_value = os.environ.get(value)
315
+ GeneralUtilities.assert_condition(resolved_value is not None, f"The environment-variable '{env_variable_name}' defined in '{configuration_file}' is supposed to be taken from the environment-variable '{value}', but that environment-variable is not set.")
316
+ return resolved_value
317
+ value_file = GeneralUtilities.resolve_relative_path(os.path.expanduser(value), os.path.dirname(configuration_file))
318
+ GeneralUtilities.assert_file_exists(value_file, f"The environment-variable '{env_variable_name}' defined in '{configuration_file}' is supposed to be taken from the file '{value_file}', but that file does not exist.")
319
+ return GeneralUtilities.read_text_from_file(value_file).strip()
320
+
321
+ @GeneralUtilities.check_arguments
322
+ def resolve_environment_variables(self,environment_variable_names:list[str],required_by:str)->dict[str,str]:
323
+ """Resolves the value of every given environment-variable. A value comes from one of two sources:
324
+ - the machine-wide environment-variables-configuration-file (see get_environment_variables_configuration_file /
325
+ get_environment_variables_file_in_container), if the variable is defined there.
326
+ - the environment of the current process otherwise. This is how a build-pipeline provides a value from its own secret-store.
327
+ The configuration-file has precedence, so a resolved value does not depend on what happens to be set in the environment of the
328
+ caller. If a value can not be determined in either way the resolution fails with a message which names both possibilities.
329
+ This function centralizes the resolution so every caller (the required environment-variables of a product as well as the
330
+ OCI-registry-credentials, see get_docker_registry_credentials_from_environment_variables) obtains identical values instead of
331
+ re-implementing the resolution-logic. 'required_by' names the thing which needs the values and is used in the error-message when a
332
+ value can not be determined."""
333
+ result: dict[str, str] = {}
334
+ configuration_file: str = self.__get_global_environment_variables_configuration_file()
335
+ entries: dict[str, tuple[str, str]] = self.__read_environment_variables_configuration_file(configuration_file)
336
+ for environment_variable_name in environment_variable_names:
337
+ if environment_variable_name in entries:
338
+ result[environment_variable_name] = self.__resolve_environment_variable_value(environment_variable_name, entries[environment_variable_name], configuration_file)
339
+ else:
340
+ value_from_environment: str = os.environ.get(environment_variable_name)
341
+ GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(value_from_environment), f"The value of the environment-variable '{environment_variable_name}' which is required by {required_by} is unknown: it is not defined in '{configuration_file}' and it is not set in the environment. Add a line '{environment_variable_name};<kind>;<value>' to that file (allowed kinds are: {', '.join(ScriptCollectionCore.__allowed_environment_variable_kinds)}) or provide the value as an environment-variable, which is how a build-pipeline usually provides it.")
342
+ result[environment_variable_name] = value_from_environment
343
+ return result
344
+
345
+ @GeneralUtilities.check_arguments
346
+ def resolve_environment_variable(self,environment_variable_name:str,required_by:str)->str:
347
+ """Resolves the value of a single environment-variable, see resolve_environment_variables."""
348
+ return self.resolve_environment_variables([environment_variable_name], required_by)[environment_variable_name]
349
+
350
+ def __get_defined_oci_registry_names(self)->list[str]:
351
+ """Returns the names of the OCI-registries for which an 'OCIRegistry_<name>_Address' is declared, either in the
352
+ environment-variables-configuration-file or in the environment of the current process (which is how a build-pipeline typically
353
+ provides it). '<name>' is chosen freely when the registry is declared; it only groups its Address/Username/Password together. Only
354
+ the names are collected here (cheaply, without resolving any value), so that resolving them further down does not read a secret
355
+ which is unrelated to an actually declared registry.
356
+ The configuration-file is scanned before the environment and a name found in both is kept with the casing of the
357
+ configuration-file: on Windows, os.environ normalizes a variable-name to uppercase as soon as it is set, so without this the same
358
+ registry declared in both places would be treated as two different registries (its 'OCIRegistry_<name>_Username/_Password' would
359
+ then be looked up with the mismatching, uppercased name and therefore be resolved from the environment instead of the file, which
360
+ would silently contradict the configuration-file's precedence)."""
361
+ address_variable_pattern = re.compile(r"^OCIRegistry_(.+)_Address$", re.IGNORECASE)
362
+ names_by_lowercase_name: dict[str,str] = {}
363
+ configuration_file = self.__get_global_environment_variables_configuration_file()
364
+ if os.path.isfile(configuration_file):
365
+ for entry in GeneralUtilities.read_csv_file(configuration_file, True):
366
+ match = address_variable_pattern.match(entry[0])
367
+ if match is not None:
368
+ names_by_lowercase_name.setdefault(match.group(1).lower(), match.group(1))
369
+ for environment_variable_name in os.environ.keys():
370
+ match = address_variable_pattern.match(environment_variable_name)
371
+ if match is not None:
372
+ names_by_lowercase_name.setdefault(match.group(1).lower(), match.group(1))
373
+ return sorted(names_by_lowercase_name.values(), key=str.lower)
374
+
252
375
  def __load_credentials_if_required_and_available(self,registry_url:str,registry_username:str,registry_password:str)->tuple[str,str]:
253
376
  """Returns the credentials for the given registry: the ones which were passed by the caller, or - if the caller passed none - the
254
- ones which are available for this registry on this machine (see __get_docker_registry_credentials). The last matching entry is
255
- used, because the entries are ordered by increasing precedence."""
377
+ ones which are declared for this registry on this machine (see get_docker_registry_credentials_from_environment_variables). The
378
+ last matching entry is used, because the entries are ordered by increasing precedence."""
256
379
  if registry_url.startswith("https://"):
257
380
  registry_url=registry_url[len("https://"):]
258
381
  if registry_password is None:
259
- for registry,username,password in self.__get_docker_registry_credentials():
382
+ for registry,username,password in self.get_docker_registry_credentials_from_environment_variables():
260
383
  if registry_url==registry and (registry_username is None or username==registry_username):
261
384
  registry_username=username
262
385
  registry_password=password
@@ -264,54 +387,36 @@ class ScriptCollectionCore:
264
387
  GeneralUtilities.assert_not_null(registry_username)
265
388
  return (registry_username,registry_password)
266
389
 
267
- def __get_docker_registry_credentials(self)->list[tuple[str,str,str]]:
268
- """Returns all registry-credentials which are available for the current process: the ones of the credentials-file of the
269
- configuration-folder followed by the ones which are declared in the environment (see
270
- get_docker_registry_credentials_from_environment_variables). The entries are ordered by increasing precedence: a registry which is
271
- declared in both places is contained twice and the declaration of the environment is the last one, so it is the one which a login
272
- executes last and which a lookup of credentials uses."""
273
- result=[]
274
- credential_file=self.__get_docker_registry_credentials_file()
275
- if os.path.isfile(credential_file):
276
- lines=GeneralUtilities.read_nonempty_lines_from_file(credential_file)[1:]
277
- for line in lines:
278
- splitted=line.split(";")
279
- registry=splitted[0]
280
- username=splitted[1]
281
- password=splitted[2]
282
- result.append((registry,username,password))
283
- result=result+self.get_docker_registry_credentials_from_environment_variables()
284
- return result
285
-
286
390
  @GeneralUtilities.check_arguments
287
391
  def get_docker_registry_credentials_from_environment_variables(self)->list[tuple[str,str,str]]:
288
- """Returns the registry-credentials which are declared in the environment of the current process, as tuples of registry-address,
289
- username and password. The credentials of a registry named '<registryname>' are declared by the environment-variables
290
- 'OCIRegistry_<registryname>_Address', 'OCIRegistry_<registryname>_Username' and 'OCIRegistry_<registryname>_Password'.
291
- The environment is used in addition to the credentials-file of the configuration-folder because that folder belongs to the user of
292
- the machine on which a command was started and is therefore not available inside a build-container, while an environment-variable is
293
- passed into the container (see TFCPS_Tools_General.get_required_environment_variables). It is the same mechanism which makes a
294
- private package-source available inside a container (see TFCPS_Tools_General.get_declared_package_sources).
392
+ """Returns the credentials of every OCI-registry which is declared on this machine, as tuples of registry-address, username and
393
+ password. A registry named '<name>' is declared by the three values 'OCIRegistry_<name>_Address',
394
+ 'OCIRegistry_<name>_Username' and 'OCIRegistry_<name>_Password'; '<name>' is chosen freely and only groups the three values
395
+ together. Every value is resolved the same way as a required environment-variable of a product (see
396
+ resolve_environment_variable): from the environment-variables-configuration-file if it is defined there, from the environment of
397
+ the current process otherwise. This is the only place where OCI-registry-credentials are configured - there is deliberately no
398
+ separate credentials-file, so that a build-pipeline which makes the configuration-folder available to the build (see
399
+ BuildRunnerConfiguration.md) does not need a second, registry-specific mount.
295
400
  The names of the environment-variables are treated case-insensitively because they are not case-sensitive on all operating-systems.
296
401
  A registry-address may be declared with or without the scheme 'https://'; the scheme is removed because docker expects the address
297
- of a registry without it."""
298
- address_variable_pattern = re.compile(r"^OCIRegistry_(.+)_Address$", re.IGNORECASE)
299
- environment_variables: dict[str, str] = {name.lower(): value for name, value in os.environ.items()}
402
+ of a registry without it.
403
+ A registry whose values can not be resolved in the current environment (for example because its password is declared as a
404
+ 'hostenvvariable' which is only set on the developer-machine, or because it is declared without credentials at all) is skipped
405
+ with a warning instead of letting the whole build fail: the registries are machine-wide, so a declaration which is unusable here
406
+ typically belongs to a different environment and must not break a build which does not even need that registry. The consequence of
407
+ a skipped registry is the documented one - its images are taken from the fallback-registry - and the warning names the reason."""
300
408
  result: list[tuple[str,str,str]] = []
301
- for environment_variable_name, environment_variable_value in os.environ.items():
302
- match = address_variable_pattern.match(environment_variable_name)
303
- if match is None:
304
- continue
305
- registry_name: str = match.group(1).lower()
306
- if not GeneralUtilities.string_has_content(environment_variable_value):
409
+ for registry_name in self.__get_defined_oci_registry_names():
410
+ required_by = f"the OCI-registry \"{registry_name}\""
411
+ try:
412
+ address: str = self.resolve_environment_variable(f"OCIRegistry_{registry_name}_Address", required_by).strip()
413
+ if address.startswith("https://"):
414
+ address = address[len("https://"):]
415
+ username: str = self.resolve_environment_variable(f"OCIRegistry_{registry_name}_Username", required_by)
416
+ password: str = self.resolve_environment_variable(f"OCIRegistry_{registry_name}_Password", required_by)
417
+ except Exception as exception:
418
+ self.log.log(f"The OCI-registry \"{registry_name}\" is declared but can not be used in this environment, so its images will be taken from the fallback-registry. Reason: {GeneralUtilities.exception_to_str(exception)}", LogLevel.Warning)
307
419
  continue
308
- address: str = environment_variable_value.strip()
309
- if address.startswith("https://"):
310
- address = address[len("https://"):]
311
- username: str = environment_variables.get(f"ociregistry_{registry_name}_username")
312
- password: str = environment_variables.get(f"ociregistry_{registry_name}_password")
313
- GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(username), f"No username is declared for the registry \"{registry_name}\" whose address is declared in the environment.")
314
- GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(password), f"No password is declared for the registry \"{registry_name}\" whose address is declared in the environment.")
315
420
  result.append((address, username, password))
316
421
  result.sort()#sorted so the order (and therefore the log-output) is deterministic
317
422
  return result
@@ -391,13 +496,13 @@ class ScriptCollectionCore:
391
496
  return result
392
497
 
393
498
  def login_to_defined_docker_registries(self)->None:
394
- """Logs in to every registry for which credentials are available on this machine (see __get_docker_registry_credentials).
499
+ """Logs in to every registry for which credentials are available on this machine (see get_docker_registry_credentials_from_environment_variables).
395
500
  The login is only executed once per instance: the credentials do not change while a process runs, and the login has to be
396
501
  ensured by everything which accesses a registry, so without this the same logins would be executed over and over again."""
397
502
  if self.__already_logged_in_to_defined_docker_registries:
398
503
  return
399
504
  self.__already_logged_in_to_defined_docker_registries=True
400
- registries=self.__get_docker_registry_credentials()
505
+ registries=self.get_docker_registry_credentials_from_environment_variables()
401
506
  if len(registries)==0:
402
507
  self.log.log("No docker registry credentials defined. Skipping docker login.",LogLevel.Debug)
403
508
  else:
@@ -455,6 +560,10 @@ class ScriptCollectionCore:
455
560
  if not force and self.local_docker_image_exists(image, tag):
456
561
  self.log.log(f"Image \"{image_with_tag}\" is already available locally and therefore does not have to be pulled.", LogLevel.Debug)
457
562
  return
563
+ #the image can be located in a registry which is not publicly readable, so the login to the registries which are defined for this
564
+ #machine is done before the pull. Without it the pull of such an image fails with "no basic auth credentials" respectively "401
565
+ #Unauthorized", even though the credentials are configured.
566
+ self.login_to_defined_docker_registries()
458
567
  self.log.log(f"Pull image \"{image_with_tag}\"...")
459
568
  self.run_program_with_retry("docker", f"pull {image_with_tag}", print_errors_as_information=True, print_live_output=self.log.loglevel == LogLevel.Debug)
460
569
 
@@ -4750,24 +4859,23 @@ OCR-content:
4750
4859
 
4751
4860
  def prepare_build_pipeline_for_gitlab(self):
4752
4861
  repository_folder:str=os.getcwd()
4753
- self.__prepare_build_pipeline(repository_folder)
4862
+ self.__prepare_build_pipeline(repository_folder,"CI_COMMIT_REF_NAME")
4754
4863
 
4755
4864
  def prepare_build_pipeline_for_github(self):
4756
4865
  repository_folder:str=os.getcwd()
4757
4866
  self.assert_scbuilder_image_in_github_workflow_matches_image_definition(repository_folder)
4758
- self.__prepare_build_pipeline(repository_folder)
4867
+ self.__prepare_build_pipeline(repository_folder,"GITHUB_REF_NAME")
4759
4868
 
4760
- def __ensure_branch_is_checked_out(self,repository_folder:str) -> None:
4761
- current_branch_result = self.run_program_argsasarray("git", ["symbolic-ref", "--short", "HEAD"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
4762
- repository_is_on_any_branch:bool = current_branch_result[0] == 0
4763
- if not repository_is_on_any_branch:
4764
- timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
4765
- branch_name:str=f"pipeline_{timestamp}"
4766
- self.run_program_argsasarray("git",["checkout","-b",branch_name], repository_folder)
4869
+ def __ensure_branch_is_checked_out(self,repository_folder:str,branchname_environment_variable_name:str) -> None:
4870
+ """Checks out the branch the pipeline was triggered for, because the runner checks out the commit in a detached-head-state while the versioning requires the real branch-name.
4871
+ "checkout -B" is used and not "checkout -b", because the branch already exists when the runner reuses the build-directory of a previous pipeline-run of the same branch."""
4872
+ branch_name:str=os.environ.get(branchname_environment_variable_name)
4873
+ GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(branch_name), f"The environment-variable \"{branchname_environment_variable_name}\", which contains the name of the branch the pipeline runs for, is not set.")
4874
+ self.run_program_argsasarray("git",["checkout","-B",branch_name], repository_folder)
4767
4875
 
4768
- def __prepare_build_pipeline(self,repository_folder:str) -> None:
4876
+ def __prepare_build_pipeline(self,repository_folder:str,branchname_environment_variable_name:str) -> None:
4769
4877
  GeneralUtilities.assert_condition(self.is_running_in_build_container(), "This function should only be run in the build container.")
4770
- self.__ensure_branch_is_checked_out(repository_folder)
4878
+ self.__ensure_branch_is_checked_out(repository_folder,branchname_environment_variable_name)
4771
4879
 
4772
4880
  expected_image = self.__get_scbuilder_image_from_image_definition_file(repository_folder)
4773
4881
  GeneralUtilities.assert_condition("scbuilder:" in expected_image, f"The SCBuilder-image '{expected_image}' defined in the image-definition-file of the repository is not a valid SCBuilder-image. It must contain 'scbuilder:'.")
@@ -25,9 +25,16 @@ class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
25
25
  artifacts_folder = GeneralUtilities.resolve_relative_path("Other/Artifacts", codeunit_folder)
26
26
  app_artifacts_folder = os.path.join(artifacts_folder, "BuildResult_OCIImage")
27
27
  GeneralUtilities.ensure_folder_exists_and_is_empty(app_artifacts_folder)
28
+ image_build_arguments = self.__get_build_arguments_for_images_of_repository()
29
+ #the base-image of the Dockerfile is resolved by buildkit and not by this process, so the login to the registries which are
30
+ #defined for this machine has to be done before the build. Without it buildkit requests the manifest of the base-image
31
+ #anonymously and the build fails with "401 Unauthorized" as soon as the base-image comes from a registry which is not
32
+ #publicly readable.
33
+ self._protected_sc.login_to_defined_docker_registries()
28
34
  for platform in platforms:
29
35
  #builder must be created once before with "docker buildx create --use"
30
36
  args = ["buildx","build", "--platform",GeneralUtilities.platform_to_docker_platform_str(platform), "--pull", "--force-rm", "--progress=plain", "--build-arg", f"TargetEnvironmentType={self.get_target_environment_type()}", "--build-arg", f"CodeUnitName={codeunitname}", "--build-arg", f"CodeUnitVersion={codeunitversion}", "--build-arg", f"CodeUnitOwnerName={self.tfcps_Tools_General.get_codeunit_owner_name(self.get_codeunit_file())}", "--build-arg", f"CodeUnitOwnerEMailAddress={self.tfcps_Tools_General.get_codeunit_owner_emailaddress(self.get_codeunit_file())}", "--build-arg", f"Platform={GeneralUtilities.platform_to_dash_str(platform)}", "--build-arg", f"DotNetRuntime={GeneralUtilities.platform_to_dotnet_runtime_identifier(platform)}", "--build-arg", f"PlatformForGoVersion={GeneralUtilities.platform_to_go_runtime_identifier(platform)}"]
37
+ args = args+image_build_arguments
31
38
  for custom_argument_key, custom_argument_value in custom_arguments.items():
32
39
  args.append("--build-arg")
33
40
  args.append(f"{custom_argument_key}={custom_argument_value}")
@@ -48,6 +55,27 @@ class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
48
55
  self.copy_source_files_to_output_directory()
49
56
 
50
57
 
58
+ @GeneralUtilities.check_arguments
59
+ def __get_build_arguments_for_images_of_repository(self) -> list[str]:
60
+ """Returns a '--build-arg'-pair for every image which the repository declares in
61
+ '.ScriptCollection/OCIImages/ImageDefinition.csv': the argument 'image_<imagename in lowercase>' contains the address with tag
62
+ from which that image has to be taken on this machine.
63
+ A Dockerfile has to declare its base-image through such an argument ('ARG image_debian' followed by 'FROM ${image_debian}')
64
+ instead of writing a registry-address into its 'FROM'-line: the address inside the Dockerfile is resolved by buildkit, so
65
+ neither the custom registry which is preferred on this machine nor the fallback-registry of the repository would apply to it
66
+ (see OCIImageManager.get_registry_address_for_image).
67
+ The arguments are named like the environment-variables which the local test-services get for the same purpose (see
68
+ TFCPS_Tools_General.pull_images_of_test_services), so an image is addressed by the same name everywhere.
69
+ Every declared image is passed and not only the one a Dockerfile uses, because which of them it needs is only known to the
70
+ Dockerfile itself - a multi-stage-build uses several of them."""
71
+ repository_folder = self.get_repository_folder()
72
+ oci_image_manager = self.tfcps_Tools_General.oci_image_manager
73
+ result: list[str] = []
74
+ for image_name in oci_image_manager.get_used_images_in_repository(repository_folder):
75
+ image_address_with_tag = oci_image_manager.get_registry_address_for_image_with_default_tag(repository_folder, image_name)
76
+ result = result+["--build-arg", f"image_{image_name.lower()}={image_address_with_tag}"]
77
+ return result
78
+
51
79
  @GeneralUtilities.check_arguments
52
80
  def __generate_sbom_for_docker_image(self) -> None:
53
81
  codeunitname=self.get_codeunit_name()
@@ -406,13 +406,26 @@ class TFCPS_CodeUnit_BuildCodeUnits:
406
406
  #custom registries as a build on the host. Without the mount the build in the container would find no mapping at all - the
407
407
  #configuration-folder inside the container belongs to the container-user and is empty - and would therefore fall back to the
408
408
  #upstream-registries defined in the repository, which are rate-limited. The mount is read-only because a build never changes the file.
409
- #Only this file is mounted and not the whole configuration-folder, so nothing else of it (in particular not EnvironmentVariables.csv or
410
- #RegistryCredentials.csv, which contain credentials) is exposed to the container.
411
409
  oci_image_manager = self.tfcps_tools_general.oci_image_manager
412
410
  image_registries_file: str = oci_image_manager.get_image_registries_file_in_configuration_folder()
413
411
  if os.path.isfile(image_registries_file):
414
412
  mount_arguments += ["-v", f"{image_registries_file}:{oci_image_manager.get_image_registries_file_in_container()}:ro"]
415
413
 
414
+ #mount the 'TFCPS'-folder of the configuration-folder of this host (if it exists), so the build inside the container resolves the
415
+ #required environment-variables of the product (see TFCPS_Tools_General.get_required_environment_variables) and the
416
+ #OCI-registry-credentials (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) exactly like a
417
+ #build on the host does - including a registry which is not publicly readable, which without this mount could not be used inside
418
+ #the container at all, because the configuration-folder of the container-user does not contain the credentials of this host.
419
+ #The whole folder is mounted and not only EnvironmentVariables.csv in it, because a value of that file can point to a secret-file
420
+ #next to it ('file'-kind with a relative path, which is the recommended form) and such a value is resolved relative to the file
421
+ #itself - so with only the file mounted, every entry of that kind would fail inside the container. Only this folder (and not the
422
+ #whole configuration-folder) is mounted, so nothing else of it is exposed to the container. The mount is read-only because a build
423
+ #never changes it. The alternative way to provide a value is an environment-variable of the build-pipeline itself (the pipeline's
424
+ #own secret-store); both sources are used together, the file taking precedence.
425
+ tfcps_configuration_folder: str = self.sc.get_tfcps_configuration_folder()
426
+ if os.path.isdir(tfcps_configuration_folder):
427
+ mount_arguments += ["-v", f"{tfcps_configuration_folder}:{self.sc.get_tfcps_configuration_folder_in_container()}:ro"]
428
+
416
429
  #pass the environment-variables which are declared as required in <repository>/.ScriptCollection/ProductInformation.xml into the container
417
430
  #so they do not have to be specified explicitly on every call. Their values are resolved from the user-specific
418
431
  #configuration-file (see TFCPS_Tools_General.get_environment_variables_configuration_file), which only exists on the host.
@@ -29,9 +29,6 @@ class TFCPS_Tools_General:
29
29
 
30
30
  __sc:ScriptCollectionCore=None
31
31
  oci_image_manager:OCIImageManager=None
32
- # Kinds which are allowed in the configuration-file that defines where the values of the required environment-variables come from
33
- # (see get_environment_variables_configuration_file).
34
- __allowed_environment_variable_kinds:list[str]=["literal","hostenvvariable","file"]
35
32
  # Relative path (inside the source-repository and inside the ScriptCollection.Resources-package) of the
36
33
  # file that pins the OpenAPIGenerator-version used as default when no repository-specific version is given.
37
34
  __openapigenerator_version_resource_relative_path:str="Dependencies/OpenAPIGenerator/Version.txt"
@@ -536,8 +533,11 @@ class TFCPS_Tools_General:
536
533
  def get_environment_variables_configuration_file(self) -> str:
537
534
  """Returns the path of the user-specific file which defines where the values of the required environment-variables come from.
538
535
  The file is looked up in the configuration-folder, so a build-container which gets that folder mounted (which is the recommended
539
- setup for a self-hosted build-runner) resolves the values exactly like a build on a host does."""
540
- return os.path.join(GeneralUtilities.get_scriptcollection_configuration_folder(), "TFCPS", "EnvironmentVariables.csv")
536
+ setup for a self-hosted build-runner) resolves the values exactly like a build on a host does.
537
+ This only returns the path of the file in the configuration-folder of the current user (used to know what to mount into a
538
+ locally-started build-container); resolving a value (resolve_environment_variables) also considers the file mounted into this
539
+ container, if there is one (see ScriptCollectionCore.get_environment_variables_file_in_container)."""
540
+ return self.__sc.get_environment_variables_configuration_file()
541
541
 
542
542
  @GeneralUtilities.check_arguments
543
543
  def get_required_environment_variables(self, repository: str) -> dict[str, str]:
@@ -559,57 +559,16 @@ class TFCPS_Tools_General:
559
559
  """Resolves the values of the given environment-variables (see get_required_environment_variables for the sources and their
560
560
  precedence). This is the same mechanism a codeunit-build uses for the variables a repository declares as required; it is available
561
561
  separately so a tool which needs a value without having such a declaration (for example something which runs inside the
562
- build-container) does not have to read the configuration or the environment on its own.
562
+ build-container) does not have to read the configuration or the environment on its own. Delegates to
563
+ ScriptCollectionCore.resolve_environment_variables, which is also used for the OCI-registry-credentials
564
+ (see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables), so both use identical resolution-logic.
563
565
  'required_by' names the thing which needs the values and is used in the error-message when a value can not be determined."""
564
- result: dict[str, str] = {}
565
- configuration_file: str = self.get_environment_variables_configuration_file()
566
- entries: dict[str, tuple[str, str]] = self.__read_environment_variables_configuration_file(configuration_file)
567
- for environment_variable_name in environment_variable_names:
568
- if environment_variable_name in entries:
569
- result[environment_variable_name] = self.__resolve_environment_variable_value(environment_variable_name, entries[environment_variable_name], configuration_file)
570
- else:
571
- value_from_environment: str = os.environ.get(environment_variable_name)
572
- GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(value_from_environment), f"The value of the environment-variable '{environment_variable_name}' which is required by {required_by} is unknown: it is not defined in '{configuration_file}' and it is not set in the environment. Add a line '{environment_variable_name};<kind>;<value>' to that file (allowed kinds are: {', '.join(TFCPS_Tools_General.__allowed_environment_variable_kinds)}) or provide the value as an environment-variable, which is how a build-pipeline usually provides it.")
573
- result[environment_variable_name] = value_from_environment
574
- return result
566
+ return self.__sc.resolve_environment_variables(environment_variable_names, required_by)
575
567
 
576
568
  @GeneralUtilities.check_arguments
577
569
  def resolve_environment_variable(self, environment_variable_name: str, required_by: str) -> str:
578
570
  """Resolves the value of a single environment-variable, see resolve_environment_variables."""
579
- return self.resolve_environment_variables([environment_variable_name], required_by)[environment_variable_name]
580
-
581
- @GeneralUtilities.check_arguments
582
- def __read_environment_variables_configuration_file(self, configuration_file: str) -> dict[str, tuple[str, str]]:
583
- """Reads the configuration-file which defines where the values of environment-variables come from (columns
584
- 'EnvVariableName;Kind;Value') and returns a mapping of the name of the variable to its kind and its value.
585
- The file is optional: if it does not exist, nothing is defined by it and all values have to come from the environment."""
586
- result: dict[str, tuple[str, str]] = {}
587
- if not os.path.isfile(configuration_file):
588
- return result
589
- for entry in GeneralUtilities.read_csv_file(configuration_file, True):
590
- GeneralUtilities.assert_condition(2 < len(entry), f"Invalid line in '{configuration_file}': every line must have the 3 columns 'EnvVariableName;Kind;Value' but '{';'.join(entry)}' has {len(entry)}.")
591
- #the value itself may contain the separator-character (for example in a literal value), so everything behind the second column belongs to the value.
592
- result[entry[0]] = (entry[1], ";".join(entry[2:]))
593
- return result
594
-
595
- @GeneralUtilities.check_arguments
596
- def __resolve_environment_variable_value(self, env_variable_name: str, entry: tuple[str, str], configuration_file: str) -> str:
597
- """Resolves the value of an entry of the configuration-file. 'Kind' is one of 'literal' (the value is used as-is),
598
- 'hostenvvariable' (the value is the name of an environment-variable which must be set on the current system) or 'file' (the value
599
- is a path - '~' is expanded, a relative path is resolved against the configuration-folder - to a text-file whose content, without
600
- surrounding whitespace, is used). A relative path is the recommended form for a secret-file, because it also resolves correctly
601
- when the configuration-folder is mounted into a build-container."""
602
- kind, value = entry
603
- GeneralUtilities.assert_condition(kind in TFCPS_Tools_General.__allowed_environment_variable_kinds, f"Unknown kind '{kind}' for environment-variable '{env_variable_name}' defined in '{configuration_file}'. Allowed values are: {', '.join(TFCPS_Tools_General.__allowed_environment_variable_kinds)}.")
604
- if kind == "literal":
605
- return value
606
- if kind == "hostenvvariable":
607
- resolved_value = os.environ.get(value)
608
- GeneralUtilities.assert_condition(resolved_value is not None, f"The environment-variable '{env_variable_name}' defined in '{configuration_file}' is supposed to be taken from the environment-variable '{value}', but that environment-variable is not set.")
609
- return resolved_value
610
- value_file = GeneralUtilities.resolve_relative_path(os.path.expanduser(value), os.path.dirname(configuration_file))
611
- GeneralUtilities.assert_file_exists(value_file, f"The environment-variable '{env_variable_name}' defined in '{configuration_file}' is supposed to be taken from the file '{value_file}', but that file does not exist.")
612
- return GeneralUtilities.read_text_from_file(value_file).strip()
571
+ return self.__sc.resolve_environment_variable(environment_variable_name, required_by)
613
572
 
614
573
  @GeneralUtilities.check_arguments
615
574
  def ensure_required_environment_variables_are_set(self, repository: str) -> None:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptcollection
3
- Version: 4.4.29
3
+ Version: 4.4.31
4
4
  Summary: The ScriptCollection is the place for reusable scripts.
5
5
  Author-email: Marius Göcke <marius.goecke@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/anionDev/ScriptCollection
@@ -12,11 +12,11 @@ ScriptCollection/ProgramRunnerSudo.py,sha256=ot9Ib-plQ8wejR9M2XXfnx6GR-va9k8SH1g
12
12
  ScriptCollection/RepositoryDiagramGenerator.py,sha256=K_u2NW_p4ozdWD50irJEkkcDVa70TcUJpZN63TeXSLw,7092
13
13
  ScriptCollection/RepositoryHistoryAnalyzer.py,sha256=rD8eC5lMUe5MHmCBHfYft_G-N9IbvtsFGkU7EMwPjrY,5779
14
14
  ScriptCollection/SCLog.py,sha256=Fjf8Pdi63cXHGMxL_aI6A0dwGm7QIfrv3czjvpSUlaM,4705
15
- ScriptCollection/ScriptCollectionCore.py,sha256=PJysQBH2oGfznko_coqHB-LcQdfV1PshFmYZwHGupNk,285393
15
+ ScriptCollection/ScriptCollectionCore.py,sha256=equQ2c2EpHDlp9-o6vX8ThfiL_0UJKQsAh3vIaQF3yQ,296980
16
16
  ScriptCollection/TestKind.py,sha256=EBTLR8yBz5VmWA2NZIpDTbQNg6tkfv_kAsIzEJ--mCI,1756
17
17
  ScriptCollection/UnassignedVariablesCheck.py,sha256=8xNcp1D5HeOLO1a1NEerq2TsnpFhw7yzYPkaay-oisY,3804
18
18
  ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
19
- ScriptCollection/OCIImages/AbstractImageHandler.py,sha256=W3ZG9IVhRwlWjFvQjIIXq1kz7xCGOOd3g4O0dzDnpkM,2470
19
+ ScriptCollection/OCIImages/AbstractImageHandler.py,sha256=UB0lUYdAjFk8RIgsikKELRWDMZPlR_zNgwvCJS1naVs,2455
20
20
  ScriptCollection/OCIImages/OCIImageManager.py,sha256=mVfSeFXraeZIInjMISh_ZZVSvUO0wAqIb7NZRMivMG4,24123
21
21
  ScriptCollection/OCIImages/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
22
22
  ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py,sha256=rty9SnftreYWLIwvdUn4LhiHqCTevb_tWuYWoOnRJuo,732
@@ -37,7 +37,7 @@ ScriptCollection/TFCPS/SCTaskRunnerServer.py,sha256=tub8_1QBu0nWAaGnBWp2nErOW-SO
37
37
  ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py,sha256=67_DQwoH5bhxxSmwtuAeoW1NOCMiLMzOy48OqlJX-KY,1744
38
38
  ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=umUfQDErp7xFLkDnSbQMxzUWnKtcGlrZnIvFcTpfGJs,40499
39
39
  ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=Mj9t1SyRpkYZE74cLsGWYv4xB89gXL2ld-RX3xvvGdw,9835
40
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=l8xfp7n-gDF_cwngox6gJkv5ShHP0oGSBzVXENj-3bM,48546
40
+ ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=DaKH50q4_QusCdrv9RDR_HNQ1WYZP6NU7qUoP9sZcfk,50023
41
41
  ScriptCollection/TFCPS/TFCPS_CreateRelease.py,sha256=kpEmSnfzfgB4OjcCuS_X6jgW_ftcphxEnjjoLoAHTLY,6582
42
42
  ScriptCollection/TFCPS/TFCPS_Generic.py,sha256=mRBcf7XuNWOSXbZYc3NuJow18AMZ30D6Z-BtyNYfNcM,2072
43
43
  ScriptCollection/TFCPS/TFCPS_MergeToMain.py,sha256=Krt5oyGNcRAN3szsgU7LoTh8yVTOdHW-Kq7R9MDDV-Q,7673
@@ -45,12 +45,12 @@ ScriptCollection/TFCPS/TFCPS_MergeToStable.py,sha256=oYEhz5ZWMnbu97_CcYEqdDxakUF
45
45
  ScriptCollection/TFCPS/TFCPS_OCIImageSecretScan.py,sha256=rhw0glr4gZ5enG9faOD51xE4MMK1cIdNjUdaswiMsrY,14622
46
46
  ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py,sha256=XLVsFTA1jvfDbZ7wYJgIdEByZhF_bYVbA5jkisuvK4k,2322
47
47
  ScriptCollection/TFCPS/TFCPS_RemoteBuild.py,sha256=2Tx5HLk1ZftBWDCTiGZdP2QiiGNyWsEuTCSt33dNQjc,23891
48
- ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=eGtOfeJEiHV-JoxkbBGw0EP9vDJJAlQWKpJ0hUASwUY,160284
48
+ ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=y452sky-RdP__jHqewUvU1rUgJZM68sMZgOQkmSchtI,156175
49
49
  ScriptCollection/TFCPS/TFCPS_VisualRegressionTests.py,sha256=6_63udxHnPlu_TMk_ftLZTnzriJO8MCxVCVAsweZqHs,37073
50
50
  ScriptCollection/TFCPS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
51
51
  ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py,sha256=BNZPMjGiWLMBH7QDtP06WEVLMnI38NLX4Dpd13bSbi4,16614
52
52
  ScriptCollection/TFCPS/CPP/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
53
- ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=919mNfzVRurTJhWi8WV0oEAFoH-yd4a6m2Rbxcf2oZ4,19927
53
+ ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=zoy1uwgK_9Fxji8mwHoa6X-VQRc9AMKcT1tfJGZMQeI,22343
54
54
  ScriptCollection/TFCPS/Docker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
55
55
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py,sha256=bT6Gd5pQpZCw4OQz6HWkPCSn5z__eUUEisABLDSxd0o,200
56
56
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py,sha256=QyjOfMY22JWCvKjMelHiDWbJiWqotOfebpJpgDUaoO4,237
@@ -71,8 +71,8 @@ ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=yyjWPMHQjPLkML
71
71
  ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
72
72
  ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py,sha256=5z990qyO5durX07xRWRwju1hl6wpjQAy2f-yRwJ-7s0,11856
73
73
  ScriptCollection/TFCPS/TypeScript/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
74
- scriptcollection-4.4.29.dist-info/METADATA,sha256=x84CpGE0zK9m7D6Dq6XKd0Sp5zaNmAoINt2n99IV4Pc,7690
75
- scriptcollection-4.4.29.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
76
- scriptcollection-4.4.29.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
77
- scriptcollection-4.4.29.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
78
- scriptcollection-4.4.29.dist-info/RECORD,,
74
+ scriptcollection-4.4.31.dist-info/METADATA,sha256=rRsrPv4VZbIyRLyv53uQ6jqDWPkDm7BYnWy3zG3cYew,7690
75
+ scriptcollection-4.4.31.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
76
+ scriptcollection-4.4.31.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
77
+ scriptcollection-4.4.31.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
78
+ scriptcollection-4.4.31.dist-info/RECORD,,