ScriptCollection 4.3.56__py3-none-any.whl → 4.3.58__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,9 +9,48 @@ from packaging.version import Version
9
9
  from ..GeneralUtilities import GeneralUtilities
10
10
  from ..ScriptCollectionCore import ScriptCollectionCore
11
11
  from ..SCLog import LogLevel
12
+ from .TFCPS_BuildCodeUnitsHook import TFCPS_BuildCodeUnitsHook
12
13
  from .TFCPS_CodeUnit_BuildCodeUnit import TFCPS_CodeUnit_BuildCodeUnit
14
+ from .TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
13
15
  from .TFCPS_Tools_General import TFCPS_Tools_General
14
16
 
17
+
18
+ class TFCPS_UpdateDependenciesHook(TFCPS_BuildCodeUnitsHook):
19
+ """Updates the dependencies of the repository and of its codeunits while the codeunits are built.
20
+ The dependency-update is implemented as a hook of the regular codeunit-build (and not as an own reduced build-process, which is how
21
+ it was implemented before) so that it runs with exactly the same preparation as a normal build: the required environment-variables
22
+ are set, the custom pre-codeunit-build-script runs and therefore the package-sources which are needed to resolve the dependencies of
23
+ a codeunit are available. Without that preparation an update fails as soon as a codeunit has a dependency which is not available on a
24
+ public package-source - which was the case for an update inside a build-container."""
25
+
26
+ __sc: ScriptCollectionCore = None
27
+ __tfcps_tools_general: TFCPS_Tools_General = None
28
+
29
+ def __init__(self, sc: ScriptCollectionCore, tfcps_tools_general: TFCPS_Tools_General):
30
+ self.__sc = sc
31
+ self.__tfcps_tools_general = tfcps_tools_general
32
+
33
+ @GeneralUtilities.check_arguments
34
+ def run_after_preparation(self, repository: str) -> None:
35
+ update_dependencies_script_folder: str = os.path.join(repository, "Other", "Scripts")
36
+ if os.path.isfile(os.path.join(update_dependencies_script_folder, "UpdateDependencies.py")):
37
+ self.__sc.log.log("Update dependencies of the repository...")
38
+ self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", update_dependencies_script_folder)
39
+
40
+ @GeneralUtilities.check_arguments
41
+ def run_after_codeunit_was_built(self, codeunit_build: TFCPS_CodeUnit_BuildCodeUnit) -> None:
42
+ codeunit_name: str = codeunit_build.codeunit_name
43
+ codeunit_folder: str = codeunit_build.codeunit_folder
44
+ repository: str = codeunit_build.repository_folder
45
+ if not self.__tfcps_tools_general.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml")):
46
+ return
47
+ self.__sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
48
+ self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", os.path.join(codeunit_folder, "Other"))
49
+ if self.__sc.git_repository_has_uncommitted_changes(repository):
50
+ #the update changed something, so it has to be verified that the codeunit is still buildable with the updated dependencies.
51
+ self.__sc.log.log(f"Build codeunit {codeunit_name} again to verify it is still buildable with the updated dependencies...")
52
+ codeunit_build.build_codeunit()
53
+
15
54
  class TFCPS_CodeUnit_BuildCodeUnits:
16
55
  repository:str=None
17
56
  tfcps_tools_general:TFCPS_Tools_General=None
@@ -39,7 +78,11 @@ class TFCPS_CodeUnit_BuildCodeUnits:
39
78
  self.__add_ready_to_merge_flag=add_ready_to_merge_flag
40
79
 
41
80
  @GeneralUtilities.check_arguments
42
- def build_codeunits(self) -> None:
81
+ def build_codeunits(self, hook: TFCPS_BuildCodeUnitsHook = None) -> None:
82
+ """Builds all codeunits of the repository. The optional hook allows a caller inside ScriptCollection to run additional actions
83
+ between the regular build-steps (see TFCPS_BuildCodeUnitsHook); a caller which only wants to build passes nothing."""
84
+ if hook is None:
85
+ hook = TFCPS_BuildCodeUnitsHook()
43
86
  self.sc.log.log(GeneralUtilities.get_line())
44
87
  start_time:datetime=GeneralUtilities.get_now()
45
88
  ready_to_merge_file=os.path.join(self.repository,".ScriptCollection",".IsReadyToMerge")
@@ -72,17 +115,24 @@ class TFCPS_CodeUnit_BuildCodeUnits:
72
115
  self.sc.ensure_scriptcollection_gitignore_is_setup(self.repository)
73
116
 
74
117
  self.sc.log.log(f"Start building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(start_time,False)}. (Target environment-type: {self.target_environment_type})")
118
+
119
+ self.tfcps_tools_general.ensure_required_environment_variables_are_set(self.repository)
120
+
121
+ self.__run_custom_pre_codeunit_build_script()
122
+
75
123
  if self.__assert_no_new_changes:
76
124
  self.sc.assert_no_uncommitted_changes(self.repository,"Can not build codeunit: There are uncommitted changes in the repository.")
77
125
 
78
126
  try:
79
- xmlschema.validate(product_information_file, "https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/raw/main/Conventions/RepositoryStructure/CommonProjectStructure/projectinformation.xsd")
127
+ xmlschema.validate(product_information_file, "https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/raw/main/Conventions/RepositoryStructure/CommonProjectStructure/productinformation.xsd")
80
128
  except Exception as exception:
81
129
  self.sc.log.log_exception(f"'{product_information_file}' could not be validated against the XSD:", exception, LogLevel.Warning)
82
130
 
83
131
  #run prepare-script
84
132
  self.run_prepare_script()
85
133
 
134
+ hook.run_after_preparation(self.repository)
135
+
86
136
  #check if changelog exists
87
137
  changelog_file=os.path.join(self.repository,"Other","Resources","Changelog",f"v{self.tfcps_tools_general.get_version_of_project(self.repository)}.md")
88
138
  GeneralUtilities.assert_file_exists(changelog_file,f"Changelogfile \"{changelog_file}\" does not exist. Try to create it for example using \"sccreatechangelogentry -m ...\".")
@@ -107,6 +157,7 @@ class TFCPS_CodeUnit_BuildCodeUnits:
107
157
  tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(os.path.join(self.repository,codeunit_name),self.sc.log.loglevel,self.target_environment_type,self.additionalargumentsfile,self.use_cache(),self.is_pre_merge())
108
158
  self.sc.log.log(GeneralUtilities.get_line())
109
159
  tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()
160
+ hook.run_after_codeunit_was_built(tFCPS_CodeUnit_BuildCodeUnit)
110
161
 
111
162
  self.sc.log.log(GeneralUtilities.get_line())
112
163
 
@@ -192,20 +243,42 @@ class TFCPS_CodeUnit_BuildCodeUnits:
192
243
  return False
193
244
 
194
245
  @GeneralUtilities.check_arguments
195
- def run_prepare_script(self):
196
- args=["--repository",self.repository,"--targetenvironmenttype",self.target_environment_type,"--verbosity",str(int(self.sc.log.loglevel))]
246
+ def __get_build_arguments(self, repository: str) -> list[str]:
247
+ """Returns the arguments which describe the current build. They are passed to every script which is called by this build
248
+ (the prepare-script of the repository as well as the optional user-specific custom scripts), so such a script can behave
249
+ differently depending on the repository and the target-environment-type it is called for."""
250
+ result: list[str] = ["--repository", repository, "--targetenvironmenttype", self.target_environment_type, "--verbosity", str(int(self.sc.log.loglevel))]
197
251
  if GeneralUtilities.string_has_content(self.additionalargumentsfile):
198
- args=args+["--additionalargumentsfile", self.additionalargumentsfile]
252
+ result = result+["--additionalargumentsfile", self.additionalargumentsfile]
199
253
  if not self.__use_cache:
200
- if self.sc.git_repository_has_uncommitted_changes(self.repository):
201
- self.sc.log.log("No-cache-option can not be applied because there are uncommited changes in the repository.",LogLevel.Warning)
254
+ if self.sc.git_repository_has_uncommitted_changes(repository):
255
+ self.sc.log.log("No-cache-option can not be applied because there are uncommited changes in the repository.", LogLevel.Warning)
202
256
  else:
203
- args=args+["--nocache"]
257
+ result = result+["--nocache"]
258
+ return result
204
259
 
205
- if os.path.isfile( os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS","CustomPreCodeUnitBuildScript.py")):
206
- self.sc.log.log("Run custom pre-codeunitbuild script...")
207
- self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["CustomPreCodeUnitBuildScript.py"]+args, os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS"),print_live_output=True)
260
+ @GeneralUtilities.check_arguments
261
+ def __run_custom_pre_codeunit_build_script(self) -> None:
262
+ """Runs the optional user-specific script which prepares this machine for a codeunit-build. The script is located in
263
+ '~/.ScriptCollection/TFCPS', so it is outside of any repository and is not part of a product's sourcecode.
264
+ Which script is used depends on where the build runs, because both cases need different preparation-steps:
265
+ - build directly on the host ('scbuildcodeunits'): 'CustomPreCodeUnitBuildScript.py'
266
+ - build inside the container ('scbuildcodeunitsc' or a build-pipeline): 'CustomPreCodeUnitBuildScriptInContainer.py'. It is
267
+ searched in the folder into which 'scbuildcodeunitsc' mounts that single script and - if it is not there - in the
268
+ configuration-folder, because a build-runner usually gets the whole configuration-folder mounted instead (see the article
269
+ 'Build-runner-configuration'). The script which the host itself runs before it starts the container is
270
+ 'CustomPreCodeUnitBuildScriptForContainer.py' and is therefore run there and not here."""
271
+ if self.sc.is_runnning_in_container():
272
+ script_file: str = os.path.join(self.tfcps_tools_general.get_folder_of_custom_scripts_in_container(), "CustomPreCodeUnitBuildScriptInContainer.py")
273
+ if not os.path.isfile(script_file):
274
+ script_file = self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScriptInContainer.py")
275
+ else:
276
+ script_file: str = self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScript.py")
277
+ self.tfcps_tools_general.run_custom_script_if_available(script_file, self.__get_build_arguments(self.repository))
208
278
 
279
+ @GeneralUtilities.check_arguments
280
+ def run_prepare_script(self):
281
+ args = self.__get_build_arguments(self.repository)
209
282
  if os.path.isfile( os.path.join(self.repository,"Other","Scripts","PrepareBuildCodeunits.py")):
210
283
  self.sc.log.log("Prepare build codeunits...")
211
284
  self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["PrepareBuildCodeunits.py"]+args, os.path.join(self.repository,"Other","Scripts"),print_live_output=True)
@@ -245,20 +318,26 @@ class TFCPS_CodeUnit_BuildCodeUnits:
245
318
  if test:
246
319
  scbuildcodeunits_arguments=["bash","-c", "pip3 install scriptcollection --upgrade && scshowversion && "+" ".join(scbuildcodeunits_arguments)]
247
320
 
248
- args=["--repository",self.repository,"--targetenvironmenttype",self.target_environment_type,"--verbosity",str(int(self.sc.log.loglevel))]
249
-
250
- if os.path.isfile( os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS","CustomPreCodeUnitBuildScriptForContainer.py")):
251
- self.sc.log.log("Run custom pre-codeunitbuild script for container...")
252
- self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["CustomPreCodeUnitBuildScriptForContainer.py"]+args, os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS"),print_live_output=True)
253
-
254
- if os.path.isfile( os.path.join(self.repository,"Other","Scripts","CustomPreCodeUnitBuildScriptForContainer.py")):
255
- self.sc.log.log("Prepare build codeunits for container...")
256
- self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["CustomPreCodeUnitBuildScriptForContainer.py"]+args, os.path.join(self.repository,"Other","Scripts"),print_live_output=True)
257
-
258
- #pass the env-variables defined in <repository>/.ScriptCollection/RequiredEnvVariables.csv (if any) into the container so they do not have to be specified explicitly on every scbuildcodeunitsc-call.
321
+ #run the optional user-specific script which prepares this host for a container-build (for example to log in to the registry the image is pulled from).
322
+ #it runs before the environment-variables are resolved, so it can also create the files their values are read from.
323
+ self.tfcps_tools_general.run_custom_script_if_available(self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScriptForContainer.py"), self.__get_build_arguments(self.repository))
324
+
325
+ #mount the optional user-specific script which prepares the container itself. it is mounted read-only because the container is not supposed to change the host-configuration.
326
+ #the build inside the container runs it (see __run_custom_pre_codeunit_build_script); if the file does not exist on the host nothing is mounted and nothing is run.
327
+ mount_arguments: list[str] = []
328
+ custom_script_for_inside_the_container: str = self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScriptInContainer.py")
329
+ if os.path.isfile(custom_script_for_inside_the_container):
330
+ mount_arguments += ["-v", f"{custom_script_for_inside_the_container}:{self.tfcps_tools_general.get_folder_of_custom_scripts_in_container()}/CustomPreCodeUnitBuildScriptInContainer.py:ro"]
331
+
332
+ #pass the environment-variables which are declared as required in <repository>/.ScriptCollection/ProductInformation.xml into the container
333
+ #so they do not have to be specified explicitly on every scbuildcodeunits-call. Their values are resolved from the user-specific
334
+ #configuration-file (see TFCPS_Tools_General.get_environment_variables_configuration_file), which only exists on the host.
335
+ #only the names are passed as arguments; the values are given to the docker-client through its own environment, because arguments
336
+ #are written to the log and are visible in the process-list of this host, which must not happen for a value which is typically a secret.
337
+ required_environment_variables: dict[str, str] = self.tfcps_tools_general.get_required_environment_variables(self.repository)
259
338
  env_arguments: list[str] = []
260
- for env_variable_name, env_variable_value in self.tfcps_tools_general.get_required_env_variables(self.repository).items():
261
- env_arguments += ["-e", f"{env_variable_name}={env_variable_value}"]
339
+ for env_variable_name in required_environment_variables:
340
+ env_arguments += ["-e", env_variable_name]
262
341
 
263
342
  #run scbuildcodeunits inside the SCBuilder-image. base_mount_folder is mounted into the container (covering the repository and, for submodules, its real gitdir) and the docker-socket is forwarded because codeunit-builds often start containers (for example local test-services).
264
343
  docker_arguments = [
@@ -266,12 +345,12 @@ class TFCPS_CodeUnit_BuildCodeUnits:
266
345
  "-v", f"{base_mount_folder}:{container_base_mount_folder}",
267
346
  "-v", "/var/run/docker.sock:/var/run/docker.sock",
268
347
  "-w", container_repository_folder,
269
- ] + env_arguments + [
348
+ ] + mount_arguments + env_arguments + [
270
349
  image,
271
350
  ] + scbuildcodeunits_arguments
272
351
  self.sc.log.log(f"Build codeunits in container using image \"{image}\"...")
273
352
  # the exitcode is evaluated by the caller (returned as part of the result-tuple), so the program-runner must not raise on a non-zero exitcode here.
274
- result=self.sc.run_program_argsasarray("docker", docker_arguments, throw_exception_if_exitcode_is_not_zero=False, print_live_output=True)
353
+ result=self.sc.run_program_argsasarray("docker", docker_arguments, throw_exception_if_exitcode_is_not_zero=False, print_live_output=True, env_vars=required_environment_variables)
275
354
  exit_code:int=result[0]
276
355
  stdout:str=result[1] or GeneralUtilities.empty_string
277
356
  stderr:str=result[2] or GeneralUtilities.empty_string
@@ -413,12 +492,47 @@ class TFCPS_CodeUnit_BuildCodeUnits:
413
492
  @GeneralUtilities.check_arguments
414
493
  def search_for_secrets(self) -> None:
415
494
  self.sc.log.log("Search for secrets...")
495
+ self.__search_for_secrets_in_repository()
496
+ self.__search_for_secrets_in_oci_images()
497
+
498
+ @GeneralUtilities.check_arguments
499
+ def __search_for_secrets_in_oci_images(self) -> None:
500
+ #the repository-scan above can not find secrets which are only contained in a built image (build-arguments recorded in the
501
+ #image-history, environment-variables of the image and files which are generated during the image-build), and betterleaks
502
+ #does not look into the "*.tar"-files the image-builds produce. Since these images get published, they are scanned separately.
503
+ scan: TFCPS_OCIImageSecretScan = TFCPS_OCIImageSecretScan(self.sc)
504
+ findings: list[str] = []
505
+ for image in self.__get_oci_images_of_repository():
506
+ findings = findings+scan.search_for_secrets_in_image(image, self.repository)
507
+ if 0 < len(findings):
508
+ for finding in findings:
509
+ self.sc.log.log(finding, LogLevel.Error)
510
+ raise ValueError(f"Found {len(findings)} secret-finding(s) in the built OCI-image(s). A secret which is part of an image is readable by everybody who is allowed to pull that image. See {os.path.join(self.repository, '.betterleaks.toml')} to ignore known false positives.")
511
+
512
+ @GeneralUtilities.check_arguments
513
+ def __get_oci_images_of_repository(self) -> list[str]:
514
+ """Returns the references of the images which were built by the codeunits of this repository. A codeunit builds an image
515
+ exactly if it has an OCI-image-artifacts-folder; the built image is loaded into the local docker-instance by the
516
+ codeunit-build, tagged with the lowercase codeunit-name and the codeunit-version."""
517
+ result: list[str] = []
518
+ for codeunit_name in self.tfcps_tools_general.get_codeunits(self.repository):
519
+ codeunit_folder: str = os.path.join(self.repository, codeunit_name)
520
+ if not os.path.isdir(os.path.join(codeunit_folder, "Other", "Artifacts", "BuildResult_OCIImage")):
521
+ continue
522
+ codeunit_version: str = self.tfcps_tools_general.get_version_of_codeunit(os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml"))
523
+ result.append(f"{codeunit_name}:{codeunit_version}".lower())
524
+ return result
525
+
526
+ @GeneralUtilities.check_arguments
527
+ def __search_for_secrets_in_repository(self) -> None:
416
528
  try:
417
529
  image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "Betterleaks")
418
530
  except Exception:
419
531
  image="ghcr.io/betterleaks/betterleaks:latest"
420
532
  config_file = os.path.join(self.repository, ".betterleaks.toml")
421
- running_in_container = os.path.exists("/.dockerenv") or self.sc.is_running_in_build_container()
533
+ #the filesystem-marker is checked in addition to the convention-based environment-variable (which the rest of this class uses as well),
534
+ #because a wrong result here does not only change a message but makes the scan analyse the wrong folder.
535
+ running_in_container = os.path.exists("/.dockerenv") or self.sc.is_runnning_in_container()
422
536
  if running_in_container:
423
537
  # We run inside the build-container with the docker-socket forwarded to the host-daemon.
424
538
  # A bind-mount of our in-container repository-path (e.g. "/__w/<repo>/<repo>" on a
@@ -453,7 +567,7 @@ class TFCPS_CodeUnit_BuildCodeUnits:
453
567
  if daemon_check[0] != 0:
454
568
  raise ValueError(f"The secret-scan can not be performed because the docker-daemon is not reachable (exit-code {daemon_check[0]}: {daemon_check[2].strip()}). The scan runs betterleaks via 'docker run', which requires a running docker-daemon (inside a build-container its socket must be forwarded to the host-daemon). This is an infrastructure-problem, not a secret-finding in the repository.")
455
569
  args = ["run", "--rm"] + mount_arguments + [image] + scan_args
456
- result = self.sc.run_program_argsasarray("docker", args, throw_exception_if_exitcode_is_not_zero=False, print_live_output=self.sc.log.loglevel==LogLevel.Debug)
570
+ result = self.sc.run_program_argsasarray("docker", args, throw_exception_if_exitcode_is_not_zero=False, print_live_output=self.sc.log.loglevel==LogLevel.Debug,print_errors_as_information=True)
457
571
  if result[0] != 0:
458
572
  for line in GeneralUtilities.string_to_lines(result[1]):
459
573
  self.sc.log.log(line, LogLevel.Information)
@@ -512,29 +626,19 @@ class TFCPS_CodeUnit_BuildCodeUnits:
512
626
  self.update_year_in_license_file()
513
627
  self.sc.assert_is_git_repository(repository)
514
628
  self.sc.assert_no_uncommitted_changes(repository)
515
- self.run_prepare_script()
516
- if os.path.isfile(os.path.join(repository,"Other","Scripts","UpdateDependencies.py")):
517
- self.sc.run_program(GeneralUtilities.get_python_executable(),"UpdateDependencies.py",os.path.join(repository,"Other","Scripts"))
518
- codeunits:list[str]=self.tfcps_tools_general.get_codeunits(repository)
519
- for codeunit_name in codeunits:
520
- self.sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
521
- codeunit_folder=os.path.join(repository,codeunit_name)
522
- tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(codeunit_folder,self.sc.log.loglevel,"QualityCheck",None,True,False)
523
- if self.sc.git_repository_has_uncommitted_changes(repository):
524
- tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()#ensure requirements for updating are there (some programming-languages needs this)
525
- if self.tfcps_tools_general.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder,f"{codeunit_name}.codeunit.xml")):
526
- self.sc.run_program(GeneralUtilities.get_python_executable(),"UpdateDependencies.py",os.path.join(codeunit_folder,"Other"))
527
- if self.sc.git_repository_has_uncommitted_changes(repository):
528
- tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()#check if codeunit is still buildable
629
+ #the update is done while the codeunits are built regularly (see TFCPS_UpdateDependenciesHook): a codeunit is built before its
630
+ #dependencies are updated (some programming-languages need that) and again afterwards if the update changed something. Using the
631
+ #regular build means the update also gets all preparation-steps of a build - especially the package-sources which are required to
632
+ #resolve dependencies which are not available on a public package-source.
633
+ self.build_codeunits(TFCPS_UpdateDependenciesHook(self.sc, self.tfcps_tools_general))
529
634
  if self.sc.git_repository_has_uncommitted_changes(repository):
530
635
  changelog_folder = os.path.join(repository, "Other", "Resources", "Changelog")
531
636
  project_version:str=self.tfcps_tools_general.get_version_of_project(repository)
532
637
  changelog_file = os.path.join(changelog_folder, f"v{project_version}.md")
533
638
  if not os.path.isfile(changelog_file):
534
639
  self.__ensure_changelog_file_is_added(repository, project_version)
535
- t=TFCPS_CodeUnit_BuildCodeUnits(repository,self.sc.log.loglevel,"QualityCheck",None,True,False,False)
536
- t.build_codeunits()#check codeunits are buildable at all
537
- self.sc.git_commit(repository, "Updated dependencies", stage_all_changes=True)
640
+ self.build_codeunits()#check the codeunits are buildable at all with all updates together
641
+ self.sc.git_commit(repository, "Updated dependencies", stage_all_changes=True)
538
642
 
539
643
  @GeneralUtilities.check_arguments
540
644
  def __ensure_changelog_file_is_added(self, repository_folder: str, version_of_project: str):
@@ -0,0 +1,230 @@
1
+ import json
2
+ import os
3
+ import re
4
+ import tarfile
5
+ import tempfile
6
+ import tomllib
7
+ from ..GeneralUtilities import GeneralUtilities
8
+ from ..SCLog import LogLevel
9
+ from ..ScriptCollectionCore import ScriptCollectionCore
10
+
11
+
12
+ class TFCPS_OCIImageSecretScan:
13
+ """Scans the OCI-image-artifacts of a repository for secrets.
14
+
15
+ This is required in addition to the betterleaks-scan of the repository-content because that scan can not find these secrets:
16
+ betterleaks does not look into archives, so the "*.tar"-files produced by the image-builds are invisible to it, and secrets
17
+ which only exist in the image (build-arguments recorded in the image-history, environment-variables of the image and files
18
+ which are generated or copied during the image-build) are not part of the repository-content at all. An image-artifact is
19
+ published to a registry, so a secret contained in it is readable by everybody who is allowed to pull the image."""
20
+
21
+ sc: ScriptCollectionCore = None
22
+
23
+ #only the beginning of a file inside a layer is read: keys, tokens and configuration-entries are located at the beginning of small files, so reading more would only cost runtime.
24
+ __maximum_amount_of_bytes_per_file: int = 1024*1024
25
+
26
+ #files which are bigger than this are only checked by their name, not by their content, because they are program- or data-files which do not contain configuration.
27
+ __maximum_size_of_file_to_read: int = 4*1024*1024
28
+
29
+ #a private key must never be part of an image-artifact, independent of the file it is stored in.
30
+ __private_key_pattern = re.compile(r"-----BEGIN (?:[A-Z0-9 ]+ )?PRIVATE KEY-----")
31
+
32
+ #credentials which are part of an url (for example "https://user:token@my-registry.example.com/simple"), which is the usual way credentials for package-sources leak into an image.
33
+ __credentials_in_url_pattern = re.compile(r"(?P<scheme>[a-zA-Z][a-zA-Z0-9+.\-]*)://(?P<user>[^/\s:@]+):(?P<password>[^/\s@]+)@(?P<host>[^/\s]+)")
34
+
35
+ #an assignment whose name indicates a secret (for example "NuGetPassword=abc123" in a build-argument recorded in the image-history).
36
+ __secret_assignment_pattern = re.compile(r"(?i)\b(?P<name>[A-Za-z0-9_.\-]*(?:password|passwort|token|secret|apikey|api_key|credential|accesskey|privatekey)[A-Za-z0-9_.\-]*)\s*[=:]\s*(?P<value>[^\s\"']+)")
37
+
38
+ #files whose name alone is a reason to look at their content even if the file-extension does not indicate a text-file.
39
+ __sensitive_file_pattern = re.compile(r"(?i)(^|/)(\.npmrc|\.netrc|\.git-credentials|\.pypirc|pip\.conf|nuget\.config|credentials|id_rsa|id_dsa|id_ecdsa|id_ed25519)$|\.(pfx|p12|key|pem|jks|keystore)$")
40
+
41
+ #file-extensions which typically contain configuration and therefore may contain credentials.
42
+ __configuration_file_extensions: list[str] = [".conf", ".config", ".cfg", ".env", ".ini", ".json", ".properties", ".sh", ".toml", ".txt", ".xml", ".yaml", ".yml"]
43
+
44
+ #values which look like a secret-assignment but obviously are none.
45
+ __placeholder_values: list[str] = ["true", "false", "none", "null", "nil", "empty", "changeme", "unset", "todo"]
46
+
47
+ def __init__(self, sc: ScriptCollectionCore):
48
+ self.sc = sc
49
+
50
+ @GeneralUtilities.check_arguments
51
+ def search_for_secrets_in_image(self, image: str, repository_for_allowlist: str = None) -> list[str]:
52
+ """Scans the given OCI-image for secrets and returns a human-readable description for every finding.
53
+ 'image' is the reference of an image which is already available in the local docker-instance (for example
54
+ 'myimage:1.0.0'); it is exported with 'docker save' and the resulting archive is scanned.
55
+ 'repository_for_allowlist' is optional: if it is given, the '[[allowlists]]'-entries of its '.betterleaks.toml' are
56
+ applied, so the same allowlist-file applies to the repository-scan and to the image-scan."""
57
+ self.sc.log.log(f"Search for secrets in image \"{image}\"...", LogLevel.Debug)
58
+ temporary_folder: str = tempfile.mkdtemp()
59
+ try:
60
+ image_file: str = os.path.join(temporary_folder, "image.tar")
61
+ #the image is exported instead of being read from the registry-storage directly, because 'docker save' produces a
62
+ #documented archive-format which is independent of the storage-driver and of the used container-engine.
63
+ result = self.sc.run_program_argsasarray("docker", ["save", image, "-o", image_file], throw_exception_if_exitcode_is_not_zero=False, print_live_output=False)
64
+ if result[0] != 0:
65
+ raise ValueError(f"The image '{image}' could not be exported (exit-code {result[0]}: {result[2].strip()}). The image must be available in the local docker-instance to be scanned.")
66
+ findings: list[str] = self.scan_image_artifact(image_file, repository_for_allowlist)
67
+ finally:
68
+ GeneralUtilities.ensure_directory_does_not_exist(temporary_folder)
69
+ return [f"image \"{image}\": {finding}" for finding in findings]
70
+
71
+ @GeneralUtilities.check_arguments
72
+ def scan_image_artifact(self, image_file: str, repository_for_allowlist: str = None) -> list[str]:
73
+ """Scans a single image-artifact (a tar-file as produced by 'docker save' or 'docker buildx build --output type=docker')."""
74
+ findings: list[str] = []
75
+ with tarfile.open(image_file, "r") as image_archive:
76
+ for member in image_archive.getmembers():
77
+ if not member.isfile() or member.size == 0:
78
+ continue
79
+ findings = findings+self.__scan_member_of_image_archive(image_archive, member)
80
+ if repository_for_allowlist is None:
81
+ return findings
82
+ allowlist_patterns: list[re.Pattern] = self.__get_allowlist_patterns(repository_for_allowlist)
83
+ return [finding for finding in findings if not self.__is_allowlisted(finding, allowlist_patterns)]
84
+
85
+ @GeneralUtilities.check_arguments
86
+ def __scan_member_of_image_archive(self, image_archive: tarfile.TarFile, member: tarfile.TarInfo) -> list[str]:
87
+ #the members of an image-archive are either the image-metadata (json) or a layer (tar). Which one it is, is decided by
88
+ #the content and not by the filename, because the docker-archive-format and the oci-archive-format use different names.
89
+ member_file = image_archive.extractfile(member)
90
+ if member_file is None:
91
+ return []
92
+ first_byte: bytes = member_file.read(1)
93
+ member_file.seek(0)
94
+ if first_byte in (b"{", b"["):
95
+ return self.__scan_image_metadata(member_file)
96
+ return self.__scan_layer(member_file)
97
+
98
+ @GeneralUtilities.check_arguments
99
+ def __scan_image_metadata(self, metadata_file) -> list[str]:
100
+ try:
101
+ metadata = json.loads(metadata_file.read().decode("utf-8", errors="ignore"))
102
+ except ValueError:
103
+ return []
104
+ if not isinstance(metadata, dict):
105
+ return []
106
+ findings: list[str] = []
107
+ #the history contains one entry per instruction of the containerfile. The build-arguments which were in scope are part of
108
+ #the recorded command, which is why a secret passed via "--build-arg" ends up in the metadata of the published image.
109
+ for history_entry in metadata.get("history") or []:
110
+ if isinstance(history_entry, dict):
111
+ findings = findings+[f"image-history: {description}" for description in self.__find_secrets_in_text(str(history_entry.get("created_by") or GeneralUtilities.empty_string), True)]
112
+ configuration = metadata.get("config") or metadata.get("Config") or {}
113
+ if isinstance(configuration, dict):
114
+ for environment_variable in configuration.get("Env") or []:
115
+ findings = findings+[f"image-environment-variable: {description}" for description in self.__find_secrets_in_text(str(environment_variable), True)]
116
+ labels = configuration.get("Labels") or {}
117
+ if isinstance(labels, dict):
118
+ for label_name, label_value in labels.items():
119
+ findings = findings+[f"image-label: {description}" for description in self.__find_secrets_in_text(f"{label_name}={label_value}", True)]
120
+ return findings
121
+
122
+ @GeneralUtilities.check_arguments
123
+ def __scan_layer(self, layer_file) -> list[str]:
124
+ findings: list[str] = []
125
+ try:
126
+ #the layer is read as a stream because a layer can be several gigabytes big and does not have to be kept in memory.
127
+ with tarfile.open(fileobj=layer_file, mode="r|*") as layer:
128
+ for member in layer:
129
+ if not member.isfile() or member.size == 0:
130
+ continue
131
+ if not self.__content_of_member_should_be_checked(member):
132
+ continue
133
+ member_file = layer.extractfile(member)
134
+ if member_file is None:
135
+ continue
136
+ content: str = member_file.read(self.__maximum_amount_of_bytes_per_file).decode("utf-8", errors="ignore")
137
+ findings = findings+[f"file \"{self.__normalize_member_name(member.name)}\" in image-layer: {description}" for description in self.__find_secrets_in_text(content, False)]
138
+ except tarfile.TarError:
139
+ #the member is not a layer (for example a signature- or an index-file). Such members do not contain secrets.
140
+ return []
141
+ return findings
142
+
143
+ @GeneralUtilities.check_arguments
144
+ def __content_of_member_should_be_checked(self, member: tarfile.TarInfo) -> bool:
145
+ name: str = self.__normalize_member_name(member.name)
146
+ if os.path.basename(name).startswith(".wh."):
147
+ return False # a whiteout-file marks a deletion and has no content
148
+ if self.__sensitive_file_pattern.search(name) is not None:
149
+ return True
150
+ if self.__maximum_size_of_file_to_read < member.size:
151
+ return False
152
+ return os.path.splitext(name)[1].lower() in self.__configuration_file_extensions
153
+
154
+ @GeneralUtilities.check_arguments
155
+ def __normalize_member_name(self, member_name: str) -> str:
156
+ result: str = member_name.replace("\\", "/")
157
+ if result.startswith("./"):
158
+ result = result[2:]
159
+ return result
160
+
161
+ @GeneralUtilities.check_arguments
162
+ def __find_secrets_in_text(self, text: str, check_assignments: bool) -> list[str]:
163
+ """Returns a description for every secret found in the given text. The descriptions never contain the secret itself,
164
+ because they are written to the build-log, which is usually kept and often readable by more people than the secret.
165
+
166
+ 'check_assignments' enables the name-based heuristic which reports an assignment whose name indicates a secret (for
167
+ example 'NuGetPassword=abc123'). It is only meaningful for the image-metadata, where the amount of values is small and
168
+ every value was set by the build itself. For the content of the files of a layer it must be disabled: a layer contains
169
+ thousands of files of the base-image (translation-resources, sample-configurations, documentation) in which a word like
170
+ 'Password' is a label and not a secret, which would drown the real findings in false positives."""
171
+ findings: list[str] = []
172
+ if not GeneralUtilities.string_has_content(text):
173
+ return findings
174
+ if self.__private_key_pattern.search(text) is not None:
175
+ findings.append("contains a private key")
176
+ for match in self.__credentials_in_url_pattern.finditer(text):
177
+ findings.append(f"contains credentials in the url \"{match.group('scheme')}://{match.group('user')}:***@{match.group('host')}\"")
178
+ if check_assignments:
179
+ for match in self.__secret_assignment_pattern.finditer(text):
180
+ if self.__value_is_a_secret(match.group("value")):
181
+ findings.append(f"contains a value for \"{match.group('name')}\"")
182
+ return findings
183
+
184
+ @GeneralUtilities.check_arguments
185
+ def __value_is_a_secret(self, value: str) -> bool:
186
+ value = value.strip().strip("\"'")
187
+ if not GeneralUtilities.string_has_content(value):
188
+ return False
189
+ if value.lower() in self.__placeholder_values:
190
+ return False
191
+ if set(value) == {"*"}:
192
+ return False
193
+ #a reference to a variable (for example "$MyToken", "${MyToken}" or "%MyToken%") is the desired way to handle a secret and therefore not a finding.
194
+ if value.startswith("$") or (value.startswith("%") and value.endswith("%")):
195
+ return False
196
+ return True
197
+
198
+ @GeneralUtilities.check_arguments
199
+ def __get_allowlist_patterns(self, repository: str) -> list[re.Pattern]:
200
+ """Reads the allowlisted paths and regexes from '<repository>/.betterleaks.toml' so the image-scan honours the same
201
+ allowlist as the repository-scan instead of requiring a second configuration-file."""
202
+ configuration_file: str = os.path.join(repository, ".betterleaks.toml")
203
+ if not os.path.isfile(configuration_file):
204
+ return []
205
+ try:
206
+ with open(configuration_file, "rb") as file_handle:
207
+ configuration = tomllib.load(file_handle)
208
+ except tomllib.TOMLDecodeError as exception:
209
+ self.sc.log.log_exception(f"'{configuration_file}' could not be parsed, so no allowlist is applied to the image-scan:", exception, LogLevel.Warning)
210
+ return []
211
+ allowlists: list = []
212
+ for key in ("allowlist", "allowlists"):
213
+ value = configuration.get(key)
214
+ if isinstance(value, dict):
215
+ allowlists.append(value)
216
+ elif isinstance(value, list):
217
+ allowlists = allowlists+[entry for entry in value if isinstance(entry, dict)]
218
+ result: list[re.Pattern] = []
219
+ for allowlist in allowlists:
220
+ for key in ("paths", "regexes"):
221
+ for pattern in allowlist.get(key) or []:
222
+ try:
223
+ result.append(re.compile(str(pattern)))
224
+ except re.error as exception:
225
+ self.sc.log.log_exception(f"The allowlist-entry '{pattern}' of '{configuration_file}' is not a valid regular expression and is therefore ignored:", exception, LogLevel.Warning)
226
+ return result
227
+
228
+ @GeneralUtilities.check_arguments
229
+ def __is_allowlisted(self, finding: str, allowlist_patterns: list[re.Pattern]) -> bool:
230
+ return any(pattern.search(finding) is not None for pattern in allowlist_patterns)