ScriptCollection 4.3.57__py3-none-any.whl → 4.3.59__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,9 +9,48 @@ from packaging.version import Version
9
9
  from ..GeneralUtilities import GeneralUtilities
10
10
  from ..ScriptCollectionCore import ScriptCollectionCore
11
11
  from ..SCLog import LogLevel
12
+ from .TFCPS_BuildCodeUnitsHook import TFCPS_BuildCodeUnitsHook
12
13
  from .TFCPS_CodeUnit_BuildCodeUnit import TFCPS_CodeUnit_BuildCodeUnit
14
+ from .TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
13
15
  from .TFCPS_Tools_General import TFCPS_Tools_General
14
16
 
17
+
18
+ class TFCPS_UpdateDependenciesHook(TFCPS_BuildCodeUnitsHook):
19
+ """Updates the dependencies of the repository and of its codeunits while the codeunits are built.
20
+ The dependency-update is implemented as a hook of the regular codeunit-build (and not as an own reduced build-process, which is how
21
+ it was implemented before) so that it runs with exactly the same preparation as a normal build: the required environment-variables
22
+ are set, the custom pre-codeunit-build-script runs and therefore the package-sources which are needed to resolve the dependencies of
23
+ a codeunit are available. Without that preparation an update fails as soon as a codeunit has a dependency which is not available on a
24
+ public package-source - which was the case for an update inside a build-container."""
25
+
26
+ __sc: ScriptCollectionCore = None
27
+ __tfcps_tools_general: TFCPS_Tools_General = None
28
+
29
+ def __init__(self, sc: ScriptCollectionCore, tfcps_tools_general: TFCPS_Tools_General):
30
+ self.__sc = sc
31
+ self.__tfcps_tools_general = tfcps_tools_general
32
+
33
+ @GeneralUtilities.check_arguments
34
+ def run_after_preparation(self, repository: str) -> None:
35
+ update_dependencies_script_folder: str = os.path.join(repository, "Other", "Scripts")
36
+ if os.path.isfile(os.path.join(update_dependencies_script_folder, "UpdateDependencies.py")):
37
+ self.__sc.log.log("Update dependencies of the repository...")
38
+ self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", update_dependencies_script_folder)
39
+
40
+ @GeneralUtilities.check_arguments
41
+ def run_after_codeunit_was_built(self, codeunit_build: TFCPS_CodeUnit_BuildCodeUnit) -> None:
42
+ codeunit_name: str = codeunit_build.codeunit_name
43
+ codeunit_folder: str = codeunit_build.codeunit_folder
44
+ repository: str = codeunit_build.repository_folder
45
+ if not self.__tfcps_tools_general.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml")):
46
+ return
47
+ self.__sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
48
+ self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", os.path.join(codeunit_folder, "Other"))
49
+ if self.__sc.git_repository_has_uncommitted_changes(repository):
50
+ #the update changed something, so it has to be verified that the codeunit is still buildable with the updated dependencies.
51
+ self.__sc.log.log(f"Build codeunit {codeunit_name} again to verify it is still buildable with the updated dependencies...")
52
+ codeunit_build.build_codeunit()
53
+
15
54
  class TFCPS_CodeUnit_BuildCodeUnits:
16
55
  repository:str=None
17
56
  tfcps_tools_general:TFCPS_Tools_General=None
@@ -39,7 +78,11 @@ class TFCPS_CodeUnit_BuildCodeUnits:
39
78
  self.__add_ready_to_merge_flag=add_ready_to_merge_flag
40
79
 
41
80
  @GeneralUtilities.check_arguments
42
- def build_codeunits(self) -> None:
81
+ def build_codeunits(self, hook: TFCPS_BuildCodeUnitsHook = None) -> None:
82
+ """Builds all codeunits of the repository. The optional hook allows a caller inside ScriptCollection to run additional actions
83
+ between the regular build-steps (see TFCPS_BuildCodeUnitsHook); a caller which only wants to build passes nothing."""
84
+ if hook is None:
85
+ hook = TFCPS_BuildCodeUnitsHook()
43
86
  self.sc.log.log(GeneralUtilities.get_line())
44
87
  start_time:datetime=GeneralUtilities.get_now()
45
88
  ready_to_merge_file=os.path.join(self.repository,".ScriptCollection",".IsReadyToMerge")
@@ -72,17 +115,24 @@ class TFCPS_CodeUnit_BuildCodeUnits:
72
115
  self.sc.ensure_scriptcollection_gitignore_is_setup(self.repository)
73
116
 
74
117
  self.sc.log.log(f"Start building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(start_time,False)}. (Target environment-type: {self.target_environment_type})")
118
+
119
+ self.tfcps_tools_general.ensure_required_environment_variables_are_set(self.repository)
120
+
121
+ self.__run_custom_pre_codeunit_build_script()
122
+
75
123
  if self.__assert_no_new_changes:
76
124
  self.sc.assert_no_uncommitted_changes(self.repository,"Can not build codeunit: There are uncommitted changes in the repository.")
77
125
 
78
126
  try:
79
- xmlschema.validate(product_information_file, "https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/raw/main/Conventions/RepositoryStructure/CommonProjectStructure/projectinformation.xsd")
127
+ xmlschema.validate(product_information_file, "https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/raw/main/Conventions/RepositoryStructure/CommonProjectStructure/productinformation.xsd")
80
128
  except Exception as exception:
81
129
  self.sc.log.log_exception(f"'{product_information_file}' could not be validated against the XSD:", exception, LogLevel.Warning)
82
130
 
83
131
  #run prepare-script
84
132
  self.run_prepare_script()
85
133
 
134
+ hook.run_after_preparation(self.repository)
135
+
86
136
  #check if changelog exists
87
137
  changelog_file=os.path.join(self.repository,"Other","Resources","Changelog",f"v{self.tfcps_tools_general.get_version_of_project(self.repository)}.md")
88
138
  GeneralUtilities.assert_file_exists(changelog_file,f"Changelogfile \"{changelog_file}\" does not exist. Try to create it for example using \"sccreatechangelogentry -m ...\".")
@@ -107,6 +157,7 @@ class TFCPS_CodeUnit_BuildCodeUnits:
107
157
  tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(os.path.join(self.repository,codeunit_name),self.sc.log.loglevel,self.target_environment_type,self.additionalargumentsfile,self.use_cache(),self.is_pre_merge())
108
158
  self.sc.log.log(GeneralUtilities.get_line())
109
159
  tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()
160
+ hook.run_after_codeunit_was_built(tFCPS_CodeUnit_BuildCodeUnit)
110
161
 
111
162
  self.sc.log.log(GeneralUtilities.get_line())
112
163
 
@@ -192,30 +243,43 @@ class TFCPS_CodeUnit_BuildCodeUnits:
192
243
  return False
193
244
 
194
245
  @GeneralUtilities.check_arguments
195
- def run_prepare_script(self):
196
- args=["--repository",self.repository,"--targetenvironmenttype",self.target_environment_type,"--verbosity",str(int(self.sc.log.loglevel))]
246
+ def __get_build_arguments(self, repository: str) -> list[str]:
247
+ """Returns the arguments which describe the current build. They are passed to every script which is called by this build
248
+ (the prepare-script of the repository as well as the optional user-specific custom scripts), so such a script can behave
249
+ differently depending on the repository and the target-environment-type it is called for."""
250
+ result: list[str] = ["--repository", repository, "--targetenvironmenttype", self.target_environment_type, "--verbosity", str(int(self.sc.log.loglevel))]
197
251
  if GeneralUtilities.string_has_content(self.additionalargumentsfile):
198
- args=args+["--additionalargumentsfile", self.additionalargumentsfile]
252
+ result = result+["--additionalargumentsfile", self.additionalargumentsfile]
199
253
  if not self.__use_cache:
200
- if self.sc.git_repository_has_uncommitted_changes(self.repository):
201
- self.sc.log.log("No-cache-option can not be applied because there are uncommited changes in the repository.",LogLevel.Warning)
254
+ if self.sc.git_repository_has_uncommitted_changes(repository):
255
+ self.sc.log.log("No-cache-option can not be applied because there are uncommited changes in the repository.", LogLevel.Warning)
202
256
  else:
203
- args=args+["--nocache"]
257
+ result = result+["--nocache"]
258
+ return result
204
259
 
205
- if self.sc.is_running_in_build_container():
206
-
207
- if os.path.isfile( os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS","CustomPreCodeUnitBuildScriptInContainer.py")):
208
- self.sc.log.log("Run custom pre-codeunitbuild script...")
209
- self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["CustomPreCodeUnitBuildScriptInContainer.py"]+args, os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS"),print_live_output=True)
210
-
211
- if os.path.isfile( os.path.join(self.repository,".ScriptCollection","CustomPreCodeUnitBuildScriptInContainer.local.py")):
212
- self.sc.log.log("Prepare build codeunits...")
213
- self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["CustomPreCodeUnitBuildScriptInContainer.local.py"]+args, os.path.join(self.repository,".ScriptCollection"),print_live_output=True)
214
-
215
- if os.path.isfile( os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS","CustomPreCodeUnitBuildScript.py")):
216
- self.sc.log.log("Run custom pre-codeunitbuild script...")
217
- self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["CustomPreCodeUnitBuildScript.py"]+args, os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS"),print_live_output=True)
260
+ @GeneralUtilities.check_arguments
261
+ def __run_custom_pre_codeunit_build_script(self) -> None:
262
+ """Runs the optional user-specific script which prepares this machine for a codeunit-build. The script is located in
263
+ '~/.ScriptCollection/TFCPS', so it is outside of any repository and is not part of a product's sourcecode.
264
+ Which script is used depends on where the build runs, because both cases need different preparation-steps:
265
+ - build directly on the host ('scbuildcodeunits'): 'CustomPreCodeUnitBuildScript.py'
266
+ - build inside the container ('scbuildcodeunitsc' or a build-pipeline): 'CustomPreCodeUnitBuildScriptInContainer.py', located in
267
+ the folder returned by TFCPS_Tools_General.get_custom_scripts_folder_for_container(). It is searched in the folder into which
268
+ 'scbuildcodeunitsc' mounts that whole folder and - if it is not there - directly in the configuration-folder, because a
269
+ build-runner usually gets the whole configuration-folder mounted instead (see the article 'Build-runner-configuration'). The
270
+ script which the host itself runs before it starts the container is 'CustomPreCodeUnitBuildScriptForContainer.py' and is
271
+ therefore run there and not here."""
272
+ if self.sc.is_runnning_in_container():
273
+ script_file: str = os.path.join(self.tfcps_tools_general.get_folder_of_custom_scripts_in_container(), "CustomPreCodeUnitBuildScriptInContainer.py")
274
+ if not os.path.isfile(script_file):
275
+ script_file = os.path.join(self.tfcps_tools_general.get_custom_scripts_folder_for_container(), "CustomPreCodeUnitBuildScriptInContainer.py")
276
+ else:
277
+ script_file: str = self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScript.py")
278
+ self.tfcps_tools_general.run_custom_script_if_available(script_file, self.__get_build_arguments(self.repository))
218
279
 
280
+ @GeneralUtilities.check_arguments
281
+ def run_prepare_script(self):
282
+ args = self.__get_build_arguments(self.repository)
219
283
  if os.path.isfile( os.path.join(self.repository,"Other","Scripts","PrepareBuildCodeunits.py")):
220
284
  self.sc.log.log("Prepare build codeunits...")
221
285
  self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["PrepareBuildCodeunits.py"]+args, os.path.join(self.repository,"Other","Scripts"),print_live_output=True)
@@ -255,25 +319,30 @@ class TFCPS_CodeUnit_BuildCodeUnits:
255
319
  if test:
256
320
  scbuildcodeunits_arguments=["bash","-c", "pip3 install scriptcollection --upgrade && scshowversion && "+" ".join(scbuildcodeunits_arguments)]
257
321
 
258
- args=["--repository",self.repository,"--targetenvironmenttype",self.target_environment_type,"--verbosity",str(int(self.sc.log.loglevel))]
259
-
260
- if os.path.isfile( os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS","CustomPreCodeUnitBuildScriptForContainer.py")):
261
- self.sc.log.log("Run custom pre-codeunitbuild script for container...")
262
- self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["CustomPreCodeUnitBuildScriptForContainer.py"]+args, os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS"),print_live_output=True)
263
-
264
- if os.path.isfile( os.path.join(self.repository,"Other","Scripts","CustomPreCodeUnitBuildScriptForContainer.py")):
265
- self.sc.log.log("Prepare build codeunits for container...")
266
- self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["CustomPreCodeUnitBuildScriptForContainer.py"]+args, os.path.join(self.repository,"Other","Scripts"),print_live_output=True)
267
-
268
- if os.path.isfile( os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS","CustomPreCodeUnitBuildScriptInContainer.py")):
269
- args += ["-v", os.path.join( GeneralUtilities.get_scriptcollection_configuration_folder(),"TFCPS","CustomPreCodeUnitBuildScriptInContainer.py")+f":~/.ScriptCollection/TFCPS/CustomPreCodeUnitBuildScriptInContainer.py" ]
270
-
271
- #pass the env-variables defined in <repository>/.ScriptCollection/RequiredEnvVariables.csv (if any) into the container so they do not have to be specified explicitly on every scbuildcodeunitsc-call.
322
+ #run the optional user-specific script which prepares this host for a container-build (for example to log in to the registry the image is pulled from).
323
+ #it runs before the environment-variables are resolved, so it can also create the files their values are read from.
324
+ self.tfcps_tools_general.run_custom_script_if_available(self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScriptForContainer.py"), self.__get_build_arguments(self.repository))
325
+
326
+ #mount the optional user-specific scripts which prepare the container itself. the whole folder (not only the pre-codeunit-build-hook
327
+ #itself) is mounted, so the hook can start a sibling script placed next to it (for example to keep the hook independent of a
328
+ #package it only installs, while the actual preparation-logic which needs that package lives in the sibling script; see
329
+ #get_custom_scripts_folder_for_container). The mount is writable (not read-only) because the scripts in that folder also use it as
330
+ #their own download-cache-folder. The build inside the container runs the hook (see __run_custom_pre_codeunit_build_script); if the
331
+ #folder does not exist on the host nothing is mounted and nothing is run.
332
+ mount_arguments: list[str] = []
333
+ custom_scripts_folder_for_inside_the_container: str = self.tfcps_tools_general.get_custom_scripts_folder_for_container()
334
+ if os.path.isdir(custom_scripts_folder_for_inside_the_container):
335
+ mount_arguments += ["-v", f"{custom_scripts_folder_for_inside_the_container}:{self.tfcps_tools_general.get_folder_of_custom_scripts_in_container()}"]
336
+
337
+ #pass the environment-variables which are declared as required in <repository>/.ScriptCollection/ProductInformation.xml into the container
338
+ #so they do not have to be specified explicitly on every scbuildcodeunits-call. Their values are resolved from the user-specific
339
+ #configuration-file (see TFCPS_Tools_General.get_environment_variables_configuration_file), which only exists on the host.
340
+ #only the names are passed as arguments; the values are given to the docker-client through its own environment, because arguments
341
+ #are written to the log and are visible in the process-list of this host, which must not happen for a value which is typically a secret.
342
+ required_environment_variables: dict[str, str] = self.tfcps_tools_general.get_required_environment_variables(self.repository)
272
343
  env_arguments: list[str] = []
273
- for env_variable_name, env_variable_value in self.tfcps_tools_general.get_required_env_variables(self.repository).items():
274
- env_arguments += ["-e", f"{env_variable_name}={env_variable_value}"]
275
-
276
-
344
+ for env_variable_name in required_environment_variables:
345
+ env_arguments += ["-e", env_variable_name]
277
346
 
278
347
  #run scbuildcodeunits inside the SCBuilder-image. base_mount_folder is mounted into the container (covering the repository and, for submodules, its real gitdir) and the docker-socket is forwarded because codeunit-builds often start containers (for example local test-services).
279
348
  docker_arguments = [
@@ -281,12 +350,12 @@ class TFCPS_CodeUnit_BuildCodeUnits:
281
350
  "-v", f"{base_mount_folder}:{container_base_mount_folder}",
282
351
  "-v", "/var/run/docker.sock:/var/run/docker.sock",
283
352
  "-w", container_repository_folder,
284
- ] + env_arguments + [
353
+ ] + mount_arguments + env_arguments + [
285
354
  image,
286
355
  ] + scbuildcodeunits_arguments
287
356
  self.sc.log.log(f"Build codeunits in container using image \"{image}\"...")
288
357
  # the exitcode is evaluated by the caller (returned as part of the result-tuple), so the program-runner must not raise on a non-zero exitcode here.
289
- result=self.sc.run_program_argsasarray("docker", docker_arguments, throw_exception_if_exitcode_is_not_zero=False, print_live_output=True)
358
+ result=self.sc.run_program_argsasarray("docker", docker_arguments, throw_exception_if_exitcode_is_not_zero=False, print_live_output=True, env_vars=required_environment_variables)
290
359
  exit_code:int=result[0]
291
360
  stdout:str=result[1] or GeneralUtilities.empty_string
292
361
  stderr:str=result[2] or GeneralUtilities.empty_string
@@ -428,12 +497,47 @@ class TFCPS_CodeUnit_BuildCodeUnits:
428
497
  @GeneralUtilities.check_arguments
429
498
  def search_for_secrets(self) -> None:
430
499
  self.sc.log.log("Search for secrets...")
500
+ self.__search_for_secrets_in_repository()
501
+ self.__search_for_secrets_in_oci_images()
502
+
503
+ @GeneralUtilities.check_arguments
504
+ def __search_for_secrets_in_oci_images(self) -> None:
505
+ #the repository-scan above can not find secrets which are only contained in a built image (build-arguments recorded in the
506
+ #image-history, environment-variables of the image and files which are generated during the image-build), and betterleaks
507
+ #does not look into the "*.tar"-files the image-builds produce. Since these images get published, they are scanned separately.
508
+ scan: TFCPS_OCIImageSecretScan = TFCPS_OCIImageSecretScan(self.sc)
509
+ findings: list[str] = []
510
+ for image in self.__get_oci_images_of_repository():
511
+ findings = findings+scan.search_for_secrets_in_image(image, self.repository)
512
+ if 0 < len(findings):
513
+ for finding in findings:
514
+ self.sc.log.log(finding, LogLevel.Error)
515
+ raise ValueError(f"Found {len(findings)} secret-finding(s) in the built OCI-image(s). A secret which is part of an image is readable by everybody who is allowed to pull that image. See {os.path.join(self.repository, '.betterleaks.toml')} to ignore known false positives.")
516
+
517
+ @GeneralUtilities.check_arguments
518
+ def __get_oci_images_of_repository(self) -> list[str]:
519
+ """Returns the references of the images which were built by the codeunits of this repository. A codeunit builds an image
520
+ exactly if it has an OCI-image-artifacts-folder; the built image is loaded into the local docker-instance by the
521
+ codeunit-build, tagged with the lowercase codeunit-name and the codeunit-version."""
522
+ result: list[str] = []
523
+ for codeunit_name in self.tfcps_tools_general.get_codeunits(self.repository):
524
+ codeunit_folder: str = os.path.join(self.repository, codeunit_name)
525
+ if not os.path.isdir(os.path.join(codeunit_folder, "Other", "Artifacts", "BuildResult_OCIImage")):
526
+ continue
527
+ codeunit_version: str = self.tfcps_tools_general.get_version_of_codeunit(os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml"))
528
+ result.append(f"{codeunit_name}:{codeunit_version}".lower())
529
+ return result
530
+
531
+ @GeneralUtilities.check_arguments
532
+ def __search_for_secrets_in_repository(self) -> None:
431
533
  try:
432
534
  image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "Betterleaks")
433
535
  except Exception:
434
536
  image="ghcr.io/betterleaks/betterleaks:latest"
435
537
  config_file = os.path.join(self.repository, ".betterleaks.toml")
436
- running_in_container = os.path.exists("/.dockerenv") or self.sc.is_running_in_build_container()
538
+ #the filesystem-marker is checked in addition to the convention-based environment-variable (which the rest of this class uses as well),
539
+ #because a wrong result here does not only change a message but makes the scan analyse the wrong folder.
540
+ running_in_container = os.path.exists("/.dockerenv") or self.sc.is_runnning_in_container()
437
541
  if running_in_container:
438
542
  # We run inside the build-container with the docker-socket forwarded to the host-daemon.
439
543
  # A bind-mount of our in-container repository-path (e.g. "/__w/<repo>/<repo>" on a
@@ -468,7 +572,7 @@ class TFCPS_CodeUnit_BuildCodeUnits:
468
572
  if daemon_check[0] != 0:
469
573
  raise ValueError(f"The secret-scan can not be performed because the docker-daemon is not reachable (exit-code {daemon_check[0]}: {daemon_check[2].strip()}). The scan runs betterleaks via 'docker run', which requires a running docker-daemon (inside a build-container its socket must be forwarded to the host-daemon). This is an infrastructure-problem, not a secret-finding in the repository.")
470
574
  args = ["run", "--rm"] + mount_arguments + [image] + scan_args
471
- result = self.sc.run_program_argsasarray("docker", args, throw_exception_if_exitcode_is_not_zero=False, print_live_output=self.sc.log.loglevel==LogLevel.Debug)
575
+ result = self.sc.run_program_argsasarray("docker", args, throw_exception_if_exitcode_is_not_zero=False, print_live_output=self.sc.log.loglevel==LogLevel.Debug,print_errors_as_information=True)
472
576
  if result[0] != 0:
473
577
  for line in GeneralUtilities.string_to_lines(result[1]):
474
578
  self.sc.log.log(line, LogLevel.Information)
@@ -527,29 +631,19 @@ class TFCPS_CodeUnit_BuildCodeUnits:
527
631
  self.update_year_in_license_file()
528
632
  self.sc.assert_is_git_repository(repository)
529
633
  self.sc.assert_no_uncommitted_changes(repository)
530
- self.run_prepare_script()
531
- if os.path.isfile(os.path.join(repository,"Other","Scripts","UpdateDependencies.py")):
532
- self.sc.run_program(GeneralUtilities.get_python_executable(),"UpdateDependencies.py",os.path.join(repository,"Other","Scripts"))
533
- codeunits:list[str]=self.tfcps_tools_general.get_codeunits(repository)
534
- for codeunit_name in codeunits:
535
- self.sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
536
- codeunit_folder=os.path.join(repository,codeunit_name)
537
- tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(codeunit_folder,self.sc.log.loglevel,"QualityCheck",None,True,False)
538
- if self.sc.git_repository_has_uncommitted_changes(repository):
539
- tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()#ensure requirements for updating are there (some programming-languages needs this)
540
- if self.tfcps_tools_general.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder,f"{codeunit_name}.codeunit.xml")):
541
- self.sc.run_program(GeneralUtilities.get_python_executable(),"UpdateDependencies.py",os.path.join(codeunit_folder,"Other"))
542
- if self.sc.git_repository_has_uncommitted_changes(repository):
543
- tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()#check if codeunit is still buildable
634
+ #the update is done while the codeunits are built regularly (see TFCPS_UpdateDependenciesHook): a codeunit is built before its
635
+ #dependencies are updated (some programming-languages need that) and again afterwards if the update changed something. Using the
636
+ #regular build means the update also gets all preparation-steps of a build - especially the package-sources which are required to
637
+ #resolve dependencies which are not available on a public package-source.
638
+ self.build_codeunits(TFCPS_UpdateDependenciesHook(self.sc, self.tfcps_tools_general))
544
639
  if self.sc.git_repository_has_uncommitted_changes(repository):
545
640
  changelog_folder = os.path.join(repository, "Other", "Resources", "Changelog")
546
641
  project_version:str=self.tfcps_tools_general.get_version_of_project(repository)
547
642
  changelog_file = os.path.join(changelog_folder, f"v{project_version}.md")
548
643
  if not os.path.isfile(changelog_file):
549
644
  self.__ensure_changelog_file_is_added(repository, project_version)
550
- t=TFCPS_CodeUnit_BuildCodeUnits(repository,self.sc.log.loglevel,"QualityCheck",None,True,False,False)
551
- t.build_codeunits()#check codeunits are buildable at all
552
- self.sc.git_commit(repository, "Updated dependencies", stage_all_changes=True)
645
+ self.build_codeunits()#check the codeunits are buildable at all with all updates together
646
+ self.sc.git_commit(repository, "Updated dependencies", stage_all_changes=True)
553
647
 
554
648
  @GeneralUtilities.check_arguments
555
649
  def __ensure_changelog_file_is_added(self, repository_folder: str, version_of_project: str):
@@ -0,0 +1,230 @@
1
+ import json
2
+ import os
3
+ import re
4
+ import tarfile
5
+ import tempfile
6
+ import tomllib
7
+ from ..GeneralUtilities import GeneralUtilities
8
+ from ..SCLog import LogLevel
9
+ from ..ScriptCollectionCore import ScriptCollectionCore
10
+
11
+
12
+ class TFCPS_OCIImageSecretScan:
13
+ """Scans the OCI-image-artifacts of a repository for secrets.
14
+
15
+ This is required in addition to the betterleaks-scan of the repository-content because that scan can not find these secrets:
16
+ betterleaks does not look into archives, so the "*.tar"-files produced by the image-builds are invisible to it, and secrets
17
+ which only exist in the image (build-arguments recorded in the image-history, environment-variables of the image and files
18
+ which are generated or copied during the image-build) are not part of the repository-content at all. An image-artifact is
19
+ published to a registry, so a secret contained in it is readable by everybody who is allowed to pull the image."""
20
+
21
+ sc: ScriptCollectionCore = None
22
+
23
+ #only the beginning of a file inside a layer is read: keys, tokens and configuration-entries are located at the beginning of small files, so reading more would only cost runtime.
24
+ __maximum_amount_of_bytes_per_file: int = 1024*1024
25
+
26
+ #files which are bigger than this are only checked by their name, not by their content, because they are program- or data-files which do not contain configuration.
27
+ __maximum_size_of_file_to_read: int = 4*1024*1024
28
+
29
+ #a private key must never be part of an image-artifact, independent of the file it is stored in.
30
+ __private_key_pattern = re.compile(r"-----BEGIN (?:[A-Z0-9 ]+ )?PRIVATE KEY-----")
31
+
32
+ #credentials which are part of an url (for example "https://user:token@my-registry.example.com/simple"), which is the usual way credentials for package-sources leak into an image.
33
+ __credentials_in_url_pattern = re.compile(r"(?P<scheme>[a-zA-Z][a-zA-Z0-9+.\-]*)://(?P<user>[^/\s:@]+):(?P<password>[^/\s@]+)@(?P<host>[^/\s]+)")
34
+
35
+ #an assignment whose name indicates a secret (for example "NuGetPassword=abc123" in a build-argument recorded in the image-history).
36
+ __secret_assignment_pattern = re.compile(r"(?i)\b(?P<name>[A-Za-z0-9_.\-]*(?:password|passwort|token|secret|apikey|api_key|credential|accesskey|privatekey)[A-Za-z0-9_.\-]*)\s*[=:]\s*(?P<value>[^\s\"']+)")
37
+
38
+ #files whose name alone is a reason to look at their content even if the file-extension does not indicate a text-file.
39
+ __sensitive_file_pattern = re.compile(r"(?i)(^|/)(\.npmrc|\.netrc|\.git-credentials|\.pypirc|pip\.conf|nuget\.config|credentials|id_rsa|id_dsa|id_ecdsa|id_ed25519)$|\.(pfx|p12|key|pem|jks|keystore)$")
40
+
41
+ #file-extensions which typically contain configuration and therefore may contain credentials.
42
+ __configuration_file_extensions: list[str] = [".conf", ".config", ".cfg", ".env", ".ini", ".json", ".properties", ".sh", ".toml", ".txt", ".xml", ".yaml", ".yml"]
43
+
44
+ #values which look like a secret-assignment but obviously are none.
45
+ __placeholder_values: list[str] = ["true", "false", "none", "null", "nil", "empty", "changeme", "unset", "todo"]
46
+
47
+ def __init__(self, sc: ScriptCollectionCore):
48
+ self.sc = sc
49
+
50
+ @GeneralUtilities.check_arguments
51
+ def search_for_secrets_in_image(self, image: str, repository_for_allowlist: str = None) -> list[str]:
52
+ """Scans the given OCI-image for secrets and returns a human-readable description for every finding.
53
+ 'image' is the reference of an image which is already available in the local docker-instance (for example
54
+ 'myimage:1.0.0'); it is exported with 'docker save' and the resulting archive is scanned.
55
+ 'repository_for_allowlist' is optional: if it is given, the '[[allowlists]]'-entries of its '.betterleaks.toml' are
56
+ applied, so the same allowlist-file applies to the repository-scan and to the image-scan."""
57
+ self.sc.log.log(f"Search for secrets in image \"{image}\"...", LogLevel.Debug)
58
+ temporary_folder: str = tempfile.mkdtemp()
59
+ try:
60
+ image_file: str = os.path.join(temporary_folder, "image.tar")
61
+ #the image is exported instead of being read from the registry-storage directly, because 'docker save' produces a
62
+ #documented archive-format which is independent of the storage-driver and of the used container-engine.
63
+ result = self.sc.run_program_argsasarray("docker", ["save", image, "-o", image_file], throw_exception_if_exitcode_is_not_zero=False, print_live_output=False)
64
+ if result[0] != 0:
65
+ raise ValueError(f"The image '{image}' could not be exported (exit-code {result[0]}: {result[2].strip()}). The image must be available in the local docker-instance to be scanned.")
66
+ findings: list[str] = self.scan_image_artifact(image_file, repository_for_allowlist)
67
+ finally:
68
+ GeneralUtilities.ensure_directory_does_not_exist(temporary_folder)
69
+ return [f"image \"{image}\": {finding}" for finding in findings]
70
+
71
+ @GeneralUtilities.check_arguments
72
+ def scan_image_artifact(self, image_file: str, repository_for_allowlist: str = None) -> list[str]:
73
+ """Scans a single image-artifact (a tar-file as produced by 'docker save' or 'docker buildx build --output type=docker')."""
74
+ findings: list[str] = []
75
+ with tarfile.open(image_file, "r") as image_archive:
76
+ for member in image_archive.getmembers():
77
+ if not member.isfile() or member.size == 0:
78
+ continue
79
+ findings = findings+self.__scan_member_of_image_archive(image_archive, member)
80
+ if repository_for_allowlist is None:
81
+ return findings
82
+ allowlist_patterns: list[re.Pattern] = self.__get_allowlist_patterns(repository_for_allowlist)
83
+ return [finding for finding in findings if not self.__is_allowlisted(finding, allowlist_patterns)]
84
+
85
+ @GeneralUtilities.check_arguments
86
+ def __scan_member_of_image_archive(self, image_archive: tarfile.TarFile, member: tarfile.TarInfo) -> list[str]:
87
+ #the members of an image-archive are either the image-metadata (json) or a layer (tar). Which one it is, is decided by
88
+ #the content and not by the filename, because the docker-archive-format and the oci-archive-format use different names.
89
+ member_file = image_archive.extractfile(member)
90
+ if member_file is None:
91
+ return []
92
+ first_byte: bytes = member_file.read(1)
93
+ member_file.seek(0)
94
+ if first_byte in (b"{", b"["):
95
+ return self.__scan_image_metadata(member_file)
96
+ return self.__scan_layer(member_file)
97
+
98
+ @GeneralUtilities.check_arguments
99
+ def __scan_image_metadata(self, metadata_file) -> list[str]:
100
+ try:
101
+ metadata = json.loads(metadata_file.read().decode("utf-8", errors="ignore"))
102
+ except ValueError:
103
+ return []
104
+ if not isinstance(metadata, dict):
105
+ return []
106
+ findings: list[str] = []
107
+ #the history contains one entry per instruction of the containerfile. The build-arguments which were in scope are part of
108
+ #the recorded command, which is why a secret passed via "--build-arg" ends up in the metadata of the published image.
109
+ for history_entry in metadata.get("history") or []:
110
+ if isinstance(history_entry, dict):
111
+ findings = findings+[f"image-history: {description}" for description in self.__find_secrets_in_text(str(history_entry.get("created_by") or GeneralUtilities.empty_string), True)]
112
+ configuration = metadata.get("config") or metadata.get("Config") or {}
113
+ if isinstance(configuration, dict):
114
+ for environment_variable in configuration.get("Env") or []:
115
+ findings = findings+[f"image-environment-variable: {description}" for description in self.__find_secrets_in_text(str(environment_variable), True)]
116
+ labels = configuration.get("Labels") or {}
117
+ if isinstance(labels, dict):
118
+ for label_name, label_value in labels.items():
119
+ findings = findings+[f"image-label: {description}" for description in self.__find_secrets_in_text(f"{label_name}={label_value}", True)]
120
+ return findings
121
+
122
+ @GeneralUtilities.check_arguments
123
+ def __scan_layer(self, layer_file) -> list[str]:
124
+ findings: list[str] = []
125
+ try:
126
+ #the layer is read as a stream because a layer can be several gigabytes big and does not have to be kept in memory.
127
+ with tarfile.open(fileobj=layer_file, mode="r|*") as layer:
128
+ for member in layer:
129
+ if not member.isfile() or member.size == 0:
130
+ continue
131
+ if not self.__content_of_member_should_be_checked(member):
132
+ continue
133
+ member_file = layer.extractfile(member)
134
+ if member_file is None:
135
+ continue
136
+ content: str = member_file.read(self.__maximum_amount_of_bytes_per_file).decode("utf-8", errors="ignore")
137
+ findings = findings+[f"file \"{self.__normalize_member_name(member.name)}\" in image-layer: {description}" for description in self.__find_secrets_in_text(content, False)]
138
+ except tarfile.TarError:
139
+ #the member is not a layer (for example a signature- or an index-file). Such members do not contain secrets.
140
+ return []
141
+ return findings
142
+
143
+ @GeneralUtilities.check_arguments
144
+ def __content_of_member_should_be_checked(self, member: tarfile.TarInfo) -> bool:
145
+ name: str = self.__normalize_member_name(member.name)
146
+ if os.path.basename(name).startswith(".wh."):
147
+ return False # a whiteout-file marks a deletion and has no content
148
+ if self.__sensitive_file_pattern.search(name) is not None:
149
+ return True
150
+ if self.__maximum_size_of_file_to_read < member.size:
151
+ return False
152
+ return os.path.splitext(name)[1].lower() in self.__configuration_file_extensions
153
+
154
+ @GeneralUtilities.check_arguments
155
+ def __normalize_member_name(self, member_name: str) -> str:
156
+ result: str = member_name.replace("\\", "/")
157
+ if result.startswith("./"):
158
+ result = result[2:]
159
+ return result
160
+
161
+ @GeneralUtilities.check_arguments
162
+ def __find_secrets_in_text(self, text: str, check_assignments: bool) -> list[str]:
163
+ """Returns a description for every secret found in the given text. The descriptions never contain the secret itself,
164
+ because they are written to the build-log, which is usually kept and often readable by more people than the secret.
165
+
166
+ 'check_assignments' enables the name-based heuristic which reports an assignment whose name indicates a secret (for
167
+ example 'NuGetPassword=abc123'). It is only meaningful for the image-metadata, where the amount of values is small and
168
+ every value was set by the build itself. For the content of the files of a layer it must be disabled: a layer contains
169
+ thousands of files of the base-image (translation-resources, sample-configurations, documentation) in which a word like
170
+ 'Password' is a label and not a secret, which would drown the real findings in false positives."""
171
+ findings: list[str] = []
172
+ if not GeneralUtilities.string_has_content(text):
173
+ return findings
174
+ if self.__private_key_pattern.search(text) is not None:
175
+ findings.append("contains a private key")
176
+ for match in self.__credentials_in_url_pattern.finditer(text):
177
+ findings.append(f"contains credentials in the url \"{match.group('scheme')}://{match.group('user')}:***@{match.group('host')}\"")
178
+ if check_assignments:
179
+ for match in self.__secret_assignment_pattern.finditer(text):
180
+ if self.__value_is_a_secret(match.group("value")):
181
+ findings.append(f"contains a value for \"{match.group('name')}\"")
182
+ return findings
183
+
184
+ @GeneralUtilities.check_arguments
185
+ def __value_is_a_secret(self, value: str) -> bool:
186
+ value = value.strip().strip("\"'")
187
+ if not GeneralUtilities.string_has_content(value):
188
+ return False
189
+ if value.lower() in self.__placeholder_values:
190
+ return False
191
+ if set(value) == {"*"}:
192
+ return False
193
+ #a reference to a variable (for example "$MyToken", "${MyToken}" or "%MyToken%") is the desired way to handle a secret and therefore not a finding.
194
+ if value.startswith("$") or (value.startswith("%") and value.endswith("%")):
195
+ return False
196
+ return True
197
+
198
+ @GeneralUtilities.check_arguments
199
+ def __get_allowlist_patterns(self, repository: str) -> list[re.Pattern]:
200
+ """Reads the allowlisted paths and regexes from '<repository>/.betterleaks.toml' so the image-scan honours the same
201
+ allowlist as the repository-scan instead of requiring a second configuration-file."""
202
+ configuration_file: str = os.path.join(repository, ".betterleaks.toml")
203
+ if not os.path.isfile(configuration_file):
204
+ return []
205
+ try:
206
+ with open(configuration_file, "rb") as file_handle:
207
+ configuration = tomllib.load(file_handle)
208
+ except tomllib.TOMLDecodeError as exception:
209
+ self.sc.log.log_exception(f"'{configuration_file}' could not be parsed, so no allowlist is applied to the image-scan:", exception, LogLevel.Warning)
210
+ return []
211
+ allowlists: list = []
212
+ for key in ("allowlist", "allowlists"):
213
+ value = configuration.get(key)
214
+ if isinstance(value, dict):
215
+ allowlists.append(value)
216
+ elif isinstance(value, list):
217
+ allowlists = allowlists+[entry for entry in value if isinstance(entry, dict)]
218
+ result: list[re.Pattern] = []
219
+ for allowlist in allowlists:
220
+ for key in ("paths", "regexes"):
221
+ for pattern in allowlist.get(key) or []:
222
+ try:
223
+ result.append(re.compile(str(pattern)))
224
+ except re.error as exception:
225
+ self.sc.log.log_exception(f"The allowlist-entry '{pattern}' of '{configuration_file}' is not a valid regular expression and is therefore ignored:", exception, LogLevel.Warning)
226
+ return result
227
+
228
+ @GeneralUtilities.check_arguments
229
+ def __is_allowlisted(self, finding: str, allowlist_patterns: list[re.Pattern]) -> bool:
230
+ return any(pattern.search(finding) is not None for pattern in allowlist_patterns)