ScriptCollection 4.3.57__py3-none-any.whl → 4.3.59__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/Executables.py +27 -0
- ScriptCollection/ScriptCollectionCore.py +7 -32
- ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +1 -4
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +64 -69
- ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py +72 -72
- ScriptCollection/TFCPS/PackageSource.py +26 -0
- ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py +26 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +3 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +154 -60
- ScriptCollection/TFCPS/TFCPS_OCIImageSecretScan.py +230 -0
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +211 -34
- {scriptcollection-4.3.57.dist-info → scriptcollection-4.3.59.dist-info}/METADATA +168 -168
- {scriptcollection-4.3.57.dist-info → scriptcollection-4.3.59.dist-info}/RECORD +16 -13
- {scriptcollection-4.3.57.dist-info → scriptcollection-4.3.59.dist-info}/entry_points.txt +1 -0
- {scriptcollection-4.3.57.dist-info → scriptcollection-4.3.59.dist-info}/WHEEL +0 -0
- {scriptcollection-4.3.57.dist-info → scriptcollection-4.3.59.dist-info}/top_level.txt +0 -0
|
@@ -9,9 +9,48 @@ from packaging.version import Version
|
|
|
9
9
|
from ..GeneralUtilities import GeneralUtilities
|
|
10
10
|
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
11
11
|
from ..SCLog import LogLevel
|
|
12
|
+
from .TFCPS_BuildCodeUnitsHook import TFCPS_BuildCodeUnitsHook
|
|
12
13
|
from .TFCPS_CodeUnit_BuildCodeUnit import TFCPS_CodeUnit_BuildCodeUnit
|
|
14
|
+
from .TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
|
|
13
15
|
from .TFCPS_Tools_General import TFCPS_Tools_General
|
|
14
16
|
|
|
17
|
+
|
|
18
|
+
class TFCPS_UpdateDependenciesHook(TFCPS_BuildCodeUnitsHook):
|
|
19
|
+
"""Updates the dependencies of the repository and of its codeunits while the codeunits are built.
|
|
20
|
+
The dependency-update is implemented as a hook of the regular codeunit-build (and not as an own reduced build-process, which is how
|
|
21
|
+
it was implemented before) so that it runs with exactly the same preparation as a normal build: the required environment-variables
|
|
22
|
+
are set, the custom pre-codeunit-build-script runs and therefore the package-sources which are needed to resolve the dependencies of
|
|
23
|
+
a codeunit are available. Without that preparation an update fails as soon as a codeunit has a dependency which is not available on a
|
|
24
|
+
public package-source - which was the case for an update inside a build-container."""
|
|
25
|
+
|
|
26
|
+
__sc: ScriptCollectionCore = None
|
|
27
|
+
__tfcps_tools_general: TFCPS_Tools_General = None
|
|
28
|
+
|
|
29
|
+
def __init__(self, sc: ScriptCollectionCore, tfcps_tools_general: TFCPS_Tools_General):
|
|
30
|
+
self.__sc = sc
|
|
31
|
+
self.__tfcps_tools_general = tfcps_tools_general
|
|
32
|
+
|
|
33
|
+
@GeneralUtilities.check_arguments
|
|
34
|
+
def run_after_preparation(self, repository: str) -> None:
|
|
35
|
+
update_dependencies_script_folder: str = os.path.join(repository, "Other", "Scripts")
|
|
36
|
+
if os.path.isfile(os.path.join(update_dependencies_script_folder, "UpdateDependencies.py")):
|
|
37
|
+
self.__sc.log.log("Update dependencies of the repository...")
|
|
38
|
+
self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", update_dependencies_script_folder)
|
|
39
|
+
|
|
40
|
+
@GeneralUtilities.check_arguments
|
|
41
|
+
def run_after_codeunit_was_built(self, codeunit_build: TFCPS_CodeUnit_BuildCodeUnit) -> None:
|
|
42
|
+
codeunit_name: str = codeunit_build.codeunit_name
|
|
43
|
+
codeunit_folder: str = codeunit_build.codeunit_folder
|
|
44
|
+
repository: str = codeunit_build.repository_folder
|
|
45
|
+
if not self.__tfcps_tools_general.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml")):
|
|
46
|
+
return
|
|
47
|
+
self.__sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
|
|
48
|
+
self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", os.path.join(codeunit_folder, "Other"))
|
|
49
|
+
if self.__sc.git_repository_has_uncommitted_changes(repository):
|
|
50
|
+
#the update changed something, so it has to be verified that the codeunit is still buildable with the updated dependencies.
|
|
51
|
+
self.__sc.log.log(f"Build codeunit {codeunit_name} again to verify it is still buildable with the updated dependencies...")
|
|
52
|
+
codeunit_build.build_codeunit()
|
|
53
|
+
|
|
15
54
|
class TFCPS_CodeUnit_BuildCodeUnits:
|
|
16
55
|
repository:str=None
|
|
17
56
|
tfcps_tools_general:TFCPS_Tools_General=None
|
|
@@ -39,7 +78,11 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
39
78
|
self.__add_ready_to_merge_flag=add_ready_to_merge_flag
|
|
40
79
|
|
|
41
80
|
@GeneralUtilities.check_arguments
|
|
42
|
-
def build_codeunits(self) -> None:
|
|
81
|
+
def build_codeunits(self, hook: TFCPS_BuildCodeUnitsHook = None) -> None:
|
|
82
|
+
"""Builds all codeunits of the repository. The optional hook allows a caller inside ScriptCollection to run additional actions
|
|
83
|
+
between the regular build-steps (see TFCPS_BuildCodeUnitsHook); a caller which only wants to build passes nothing."""
|
|
84
|
+
if hook is None:
|
|
85
|
+
hook = TFCPS_BuildCodeUnitsHook()
|
|
43
86
|
self.sc.log.log(GeneralUtilities.get_line())
|
|
44
87
|
start_time:datetime=GeneralUtilities.get_now()
|
|
45
88
|
ready_to_merge_file=os.path.join(self.repository,".ScriptCollection",".IsReadyToMerge")
|
|
@@ -72,17 +115,24 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
72
115
|
self.sc.ensure_scriptcollection_gitignore_is_setup(self.repository)
|
|
73
116
|
|
|
74
117
|
self.sc.log.log(f"Start building codeunits at {GeneralUtilities.datetime_to_string_for_readable_entry(start_time,False)}. (Target environment-type: {self.target_environment_type})")
|
|
118
|
+
|
|
119
|
+
self.tfcps_tools_general.ensure_required_environment_variables_are_set(self.repository)
|
|
120
|
+
|
|
121
|
+
self.__run_custom_pre_codeunit_build_script()
|
|
122
|
+
|
|
75
123
|
if self.__assert_no_new_changes:
|
|
76
124
|
self.sc.assert_no_uncommitted_changes(self.repository,"Can not build codeunit: There are uncommitted changes in the repository.")
|
|
77
125
|
|
|
78
126
|
try:
|
|
79
|
-
xmlschema.validate(product_information_file, "https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/raw/main/Conventions/RepositoryStructure/CommonProjectStructure/
|
|
127
|
+
xmlschema.validate(product_information_file, "https://projects.aniondev.de/PublicProjects/Common/ProjectTemplates/-/raw/main/Conventions/RepositoryStructure/CommonProjectStructure/productinformation.xsd")
|
|
80
128
|
except Exception as exception:
|
|
81
129
|
self.sc.log.log_exception(f"'{product_information_file}' could not be validated against the XSD:", exception, LogLevel.Warning)
|
|
82
130
|
|
|
83
131
|
#run prepare-script
|
|
84
132
|
self.run_prepare_script()
|
|
85
133
|
|
|
134
|
+
hook.run_after_preparation(self.repository)
|
|
135
|
+
|
|
86
136
|
#check if changelog exists
|
|
87
137
|
changelog_file=os.path.join(self.repository,"Other","Resources","Changelog",f"v{self.tfcps_tools_general.get_version_of_project(self.repository)}.md")
|
|
88
138
|
GeneralUtilities.assert_file_exists(changelog_file,f"Changelogfile \"{changelog_file}\" does not exist. Try to create it for example using \"sccreatechangelogentry -m ...\".")
|
|
@@ -107,6 +157,7 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
107
157
|
tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(os.path.join(self.repository,codeunit_name),self.sc.log.loglevel,self.target_environment_type,self.additionalargumentsfile,self.use_cache(),self.is_pre_merge())
|
|
108
158
|
self.sc.log.log(GeneralUtilities.get_line())
|
|
109
159
|
tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()
|
|
160
|
+
hook.run_after_codeunit_was_built(tFCPS_CodeUnit_BuildCodeUnit)
|
|
110
161
|
|
|
111
162
|
self.sc.log.log(GeneralUtilities.get_line())
|
|
112
163
|
|
|
@@ -192,30 +243,43 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
192
243
|
return False
|
|
193
244
|
|
|
194
245
|
@GeneralUtilities.check_arguments
|
|
195
|
-
def
|
|
196
|
-
|
|
246
|
+
def __get_build_arguments(self, repository: str) -> list[str]:
|
|
247
|
+
"""Returns the arguments which describe the current build. They are passed to every script which is called by this build
|
|
248
|
+
(the prepare-script of the repository as well as the optional user-specific custom scripts), so such a script can behave
|
|
249
|
+
differently depending on the repository and the target-environment-type it is called for."""
|
|
250
|
+
result: list[str] = ["--repository", repository, "--targetenvironmenttype", self.target_environment_type, "--verbosity", str(int(self.sc.log.loglevel))]
|
|
197
251
|
if GeneralUtilities.string_has_content(self.additionalargumentsfile):
|
|
198
|
-
|
|
252
|
+
result = result+["--additionalargumentsfile", self.additionalargumentsfile]
|
|
199
253
|
if not self.__use_cache:
|
|
200
|
-
if self.sc.git_repository_has_uncommitted_changes(
|
|
201
|
-
self.sc.log.log("No-cache-option can not be applied because there are uncommited changes in the repository.",LogLevel.Warning)
|
|
254
|
+
if self.sc.git_repository_has_uncommitted_changes(repository):
|
|
255
|
+
self.sc.log.log("No-cache-option can not be applied because there are uncommited changes in the repository.", LogLevel.Warning)
|
|
202
256
|
else:
|
|
203
|
-
|
|
257
|
+
result = result+["--nocache"]
|
|
258
|
+
return result
|
|
204
259
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
260
|
+
@GeneralUtilities.check_arguments
|
|
261
|
+
def __run_custom_pre_codeunit_build_script(self) -> None:
|
|
262
|
+
"""Runs the optional user-specific script which prepares this machine for a codeunit-build. The script is located in
|
|
263
|
+
'~/.ScriptCollection/TFCPS', so it is outside of any repository and is not part of a product's sourcecode.
|
|
264
|
+
Which script is used depends on where the build runs, because both cases need different preparation-steps:
|
|
265
|
+
- build directly on the host ('scbuildcodeunits'): 'CustomPreCodeUnitBuildScript.py'
|
|
266
|
+
- build inside the container ('scbuildcodeunitsc' or a build-pipeline): 'CustomPreCodeUnitBuildScriptInContainer.py', located in
|
|
267
|
+
the folder returned by TFCPS_Tools_General.get_custom_scripts_folder_for_container(). It is searched in the folder into which
|
|
268
|
+
'scbuildcodeunitsc' mounts that whole folder and - if it is not there - directly in the configuration-folder, because a
|
|
269
|
+
build-runner usually gets the whole configuration-folder mounted instead (see the article 'Build-runner-configuration'). The
|
|
270
|
+
script which the host itself runs before it starts the container is 'CustomPreCodeUnitBuildScriptForContainer.py' and is
|
|
271
|
+
therefore run there and not here."""
|
|
272
|
+
if self.sc.is_runnning_in_container():
|
|
273
|
+
script_file: str = os.path.join(self.tfcps_tools_general.get_folder_of_custom_scripts_in_container(), "CustomPreCodeUnitBuildScriptInContainer.py")
|
|
274
|
+
if not os.path.isfile(script_file):
|
|
275
|
+
script_file = os.path.join(self.tfcps_tools_general.get_custom_scripts_folder_for_container(), "CustomPreCodeUnitBuildScriptInContainer.py")
|
|
276
|
+
else:
|
|
277
|
+
script_file: str = self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScript.py")
|
|
278
|
+
self.tfcps_tools_general.run_custom_script_if_available(script_file, self.__get_build_arguments(self.repository))
|
|
218
279
|
|
|
280
|
+
@GeneralUtilities.check_arguments
|
|
281
|
+
def run_prepare_script(self):
|
|
282
|
+
args = self.__get_build_arguments(self.repository)
|
|
219
283
|
if os.path.isfile( os.path.join(self.repository,"Other","Scripts","PrepareBuildCodeunits.py")):
|
|
220
284
|
self.sc.log.log("Prepare build codeunits...")
|
|
221
285
|
self.sc.run_program_argsasarray(GeneralUtilities.get_python_executable(),["PrepareBuildCodeunits.py"]+args, os.path.join(self.repository,"Other","Scripts"),print_live_output=True)
|
|
@@ -255,25 +319,30 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
255
319
|
if test:
|
|
256
320
|
scbuildcodeunits_arguments=["bash","-c", "pip3 install scriptcollection --upgrade && scshowversion && "+" ".join(scbuildcodeunits_arguments)]
|
|
257
321
|
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
322
|
+
#run the optional user-specific script which prepares this host for a container-build (for example to log in to the registry the image is pulled from).
|
|
323
|
+
#it runs before the environment-variables are resolved, so it can also create the files their values are read from.
|
|
324
|
+
self.tfcps_tools_general.run_custom_script_if_available(self.tfcps_tools_general.get_custom_script_file("CustomPreCodeUnitBuildScriptForContainer.py"), self.__get_build_arguments(self.repository))
|
|
325
|
+
|
|
326
|
+
#mount the optional user-specific scripts which prepare the container itself. the whole folder (not only the pre-codeunit-build-hook
|
|
327
|
+
#itself) is mounted, so the hook can start a sibling script placed next to it (for example to keep the hook independent of a
|
|
328
|
+
#package it only installs, while the actual preparation-logic which needs that package lives in the sibling script; see
|
|
329
|
+
#get_custom_scripts_folder_for_container). The mount is writable (not read-only) because the scripts in that folder also use it as
|
|
330
|
+
#their own download-cache-folder. The build inside the container runs the hook (see __run_custom_pre_codeunit_build_script); if the
|
|
331
|
+
#folder does not exist on the host nothing is mounted and nothing is run.
|
|
332
|
+
mount_arguments: list[str] = []
|
|
333
|
+
custom_scripts_folder_for_inside_the_container: str = self.tfcps_tools_general.get_custom_scripts_folder_for_container()
|
|
334
|
+
if os.path.isdir(custom_scripts_folder_for_inside_the_container):
|
|
335
|
+
mount_arguments += ["-v", f"{custom_scripts_folder_for_inside_the_container}:{self.tfcps_tools_general.get_folder_of_custom_scripts_in_container()}"]
|
|
336
|
+
|
|
337
|
+
#pass the environment-variables which are declared as required in <repository>/.ScriptCollection/ProductInformation.xml into the container
|
|
338
|
+
#so they do not have to be specified explicitly on every scbuildcodeunits-call. Their values are resolved from the user-specific
|
|
339
|
+
#configuration-file (see TFCPS_Tools_General.get_environment_variables_configuration_file), which only exists on the host.
|
|
340
|
+
#only the names are passed as arguments; the values are given to the docker-client through its own environment, because arguments
|
|
341
|
+
#are written to the log and are visible in the process-list of this host, which must not happen for a value which is typically a secret.
|
|
342
|
+
required_environment_variables: dict[str, str] = self.tfcps_tools_general.get_required_environment_variables(self.repository)
|
|
272
343
|
env_arguments: list[str] = []
|
|
273
|
-
for env_variable_name
|
|
274
|
-
env_arguments += ["-e",
|
|
275
|
-
|
|
276
|
-
|
|
344
|
+
for env_variable_name in required_environment_variables:
|
|
345
|
+
env_arguments += ["-e", env_variable_name]
|
|
277
346
|
|
|
278
347
|
#run scbuildcodeunits inside the SCBuilder-image. base_mount_folder is mounted into the container (covering the repository and, for submodules, its real gitdir) and the docker-socket is forwarded because codeunit-builds often start containers (for example local test-services).
|
|
279
348
|
docker_arguments = [
|
|
@@ -281,12 +350,12 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
281
350
|
"-v", f"{base_mount_folder}:{container_base_mount_folder}",
|
|
282
351
|
"-v", "/var/run/docker.sock:/var/run/docker.sock",
|
|
283
352
|
"-w", container_repository_folder,
|
|
284
|
-
] + env_arguments + [
|
|
353
|
+
] + mount_arguments + env_arguments + [
|
|
285
354
|
image,
|
|
286
355
|
] + scbuildcodeunits_arguments
|
|
287
356
|
self.sc.log.log(f"Build codeunits in container using image \"{image}\"...")
|
|
288
357
|
# the exitcode is evaluated by the caller (returned as part of the result-tuple), so the program-runner must not raise on a non-zero exitcode here.
|
|
289
|
-
result=self.sc.run_program_argsasarray("docker", docker_arguments, throw_exception_if_exitcode_is_not_zero=False, print_live_output=True)
|
|
358
|
+
result=self.sc.run_program_argsasarray("docker", docker_arguments, throw_exception_if_exitcode_is_not_zero=False, print_live_output=True, env_vars=required_environment_variables)
|
|
290
359
|
exit_code:int=result[0]
|
|
291
360
|
stdout:str=result[1] or GeneralUtilities.empty_string
|
|
292
361
|
stderr:str=result[2] or GeneralUtilities.empty_string
|
|
@@ -428,12 +497,47 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
428
497
|
@GeneralUtilities.check_arguments
|
|
429
498
|
def search_for_secrets(self) -> None:
|
|
430
499
|
self.sc.log.log("Search for secrets...")
|
|
500
|
+
self.__search_for_secrets_in_repository()
|
|
501
|
+
self.__search_for_secrets_in_oci_images()
|
|
502
|
+
|
|
503
|
+
@GeneralUtilities.check_arguments
|
|
504
|
+
def __search_for_secrets_in_oci_images(self) -> None:
|
|
505
|
+
#the repository-scan above can not find secrets which are only contained in a built image (build-arguments recorded in the
|
|
506
|
+
#image-history, environment-variables of the image and files which are generated during the image-build), and betterleaks
|
|
507
|
+
#does not look into the "*.tar"-files the image-builds produce. Since these images get published, they are scanned separately.
|
|
508
|
+
scan: TFCPS_OCIImageSecretScan = TFCPS_OCIImageSecretScan(self.sc)
|
|
509
|
+
findings: list[str] = []
|
|
510
|
+
for image in self.__get_oci_images_of_repository():
|
|
511
|
+
findings = findings+scan.search_for_secrets_in_image(image, self.repository)
|
|
512
|
+
if 0 < len(findings):
|
|
513
|
+
for finding in findings:
|
|
514
|
+
self.sc.log.log(finding, LogLevel.Error)
|
|
515
|
+
raise ValueError(f"Found {len(findings)} secret-finding(s) in the built OCI-image(s). A secret which is part of an image is readable by everybody who is allowed to pull that image. See {os.path.join(self.repository, '.betterleaks.toml')} to ignore known false positives.")
|
|
516
|
+
|
|
517
|
+
@GeneralUtilities.check_arguments
|
|
518
|
+
def __get_oci_images_of_repository(self) -> list[str]:
|
|
519
|
+
"""Returns the references of the images which were built by the codeunits of this repository. A codeunit builds an image
|
|
520
|
+
exactly if it has an OCI-image-artifacts-folder; the built image is loaded into the local docker-instance by the
|
|
521
|
+
codeunit-build, tagged with the lowercase codeunit-name and the codeunit-version."""
|
|
522
|
+
result: list[str] = []
|
|
523
|
+
for codeunit_name in self.tfcps_tools_general.get_codeunits(self.repository):
|
|
524
|
+
codeunit_folder: str = os.path.join(self.repository, codeunit_name)
|
|
525
|
+
if not os.path.isdir(os.path.join(codeunit_folder, "Other", "Artifacts", "BuildResult_OCIImage")):
|
|
526
|
+
continue
|
|
527
|
+
codeunit_version: str = self.tfcps_tools_general.get_version_of_codeunit(os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml"))
|
|
528
|
+
result.append(f"{codeunit_name}:{codeunit_version}".lower())
|
|
529
|
+
return result
|
|
530
|
+
|
|
531
|
+
@GeneralUtilities.check_arguments
|
|
532
|
+
def __search_for_secrets_in_repository(self) -> None:
|
|
431
533
|
try:
|
|
432
534
|
image = self.tfcps_tools_general.oci_image_manager.get_registry_address_for_image_with_default_tag(self.repository, "Betterleaks")
|
|
433
535
|
except Exception:
|
|
434
536
|
image="ghcr.io/betterleaks/betterleaks:latest"
|
|
435
537
|
config_file = os.path.join(self.repository, ".betterleaks.toml")
|
|
436
|
-
|
|
538
|
+
#the filesystem-marker is checked in addition to the convention-based environment-variable (which the rest of this class uses as well),
|
|
539
|
+
#because a wrong result here does not only change a message but makes the scan analyse the wrong folder.
|
|
540
|
+
running_in_container = os.path.exists("/.dockerenv") or self.sc.is_runnning_in_container()
|
|
437
541
|
if running_in_container:
|
|
438
542
|
# We run inside the build-container with the docker-socket forwarded to the host-daemon.
|
|
439
543
|
# A bind-mount of our in-container repository-path (e.g. "/__w/<repo>/<repo>" on a
|
|
@@ -468,7 +572,7 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
468
572
|
if daemon_check[0] != 0:
|
|
469
573
|
raise ValueError(f"The secret-scan can not be performed because the docker-daemon is not reachable (exit-code {daemon_check[0]}: {daemon_check[2].strip()}). The scan runs betterleaks via 'docker run', which requires a running docker-daemon (inside a build-container its socket must be forwarded to the host-daemon). This is an infrastructure-problem, not a secret-finding in the repository.")
|
|
470
574
|
args = ["run", "--rm"] + mount_arguments + [image] + scan_args
|
|
471
|
-
result = self.sc.run_program_argsasarray("docker", args, throw_exception_if_exitcode_is_not_zero=False, print_live_output=self.sc.log.loglevel==LogLevel.Debug)
|
|
575
|
+
result = self.sc.run_program_argsasarray("docker", args, throw_exception_if_exitcode_is_not_zero=False, print_live_output=self.sc.log.loglevel==LogLevel.Debug,print_errors_as_information=True)
|
|
472
576
|
if result[0] != 0:
|
|
473
577
|
for line in GeneralUtilities.string_to_lines(result[1]):
|
|
474
578
|
self.sc.log.log(line, LogLevel.Information)
|
|
@@ -527,29 +631,19 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
527
631
|
self.update_year_in_license_file()
|
|
528
632
|
self.sc.assert_is_git_repository(repository)
|
|
529
633
|
self.sc.assert_no_uncommitted_changes(repository)
|
|
530
|
-
|
|
531
|
-
if
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
self.sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
|
|
536
|
-
codeunit_folder=os.path.join(repository,codeunit_name)
|
|
537
|
-
tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(codeunit_folder,self.sc.log.loglevel,"QualityCheck",None,True,False)
|
|
538
|
-
if self.sc.git_repository_has_uncommitted_changes(repository):
|
|
539
|
-
tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()#ensure requirements for updating are there (some programming-languages needs this)
|
|
540
|
-
if self.tfcps_tools_general.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder,f"{codeunit_name}.codeunit.xml")):
|
|
541
|
-
self.sc.run_program(GeneralUtilities.get_python_executable(),"UpdateDependencies.py",os.path.join(codeunit_folder,"Other"))
|
|
542
|
-
if self.sc.git_repository_has_uncommitted_changes(repository):
|
|
543
|
-
tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()#check if codeunit is still buildable
|
|
634
|
+
#the update is done while the codeunits are built regularly (see TFCPS_UpdateDependenciesHook): a codeunit is built before its
|
|
635
|
+
#dependencies are updated (some programming-languages need that) and again afterwards if the update changed something. Using the
|
|
636
|
+
#regular build means the update also gets all preparation-steps of a build - especially the package-sources which are required to
|
|
637
|
+
#resolve dependencies which are not available on a public package-source.
|
|
638
|
+
self.build_codeunits(TFCPS_UpdateDependenciesHook(self.sc, self.tfcps_tools_general))
|
|
544
639
|
if self.sc.git_repository_has_uncommitted_changes(repository):
|
|
545
640
|
changelog_folder = os.path.join(repository, "Other", "Resources", "Changelog")
|
|
546
641
|
project_version:str=self.tfcps_tools_general.get_version_of_project(repository)
|
|
547
642
|
changelog_file = os.path.join(changelog_folder, f"v{project_version}.md")
|
|
548
643
|
if not os.path.isfile(changelog_file):
|
|
549
644
|
self.__ensure_changelog_file_is_added(repository, project_version)
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
self.sc.git_commit(repository, "Updated dependencies", stage_all_changes=True)
|
|
645
|
+
self.build_codeunits()#check the codeunits are buildable at all with all updates together
|
|
646
|
+
self.sc.git_commit(repository, "Updated dependencies", stage_all_changes=True)
|
|
553
647
|
|
|
554
648
|
@GeneralUtilities.check_arguments
|
|
555
649
|
def __ensure_changelog_file_is_added(self, repository_folder: str, version_of_project: str):
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
import re
|
|
4
|
+
import tarfile
|
|
5
|
+
import tempfile
|
|
6
|
+
import tomllib
|
|
7
|
+
from ..GeneralUtilities import GeneralUtilities
|
|
8
|
+
from ..SCLog import LogLevel
|
|
9
|
+
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class TFCPS_OCIImageSecretScan:
|
|
13
|
+
"""Scans the OCI-image-artifacts of a repository for secrets.
|
|
14
|
+
|
|
15
|
+
This is required in addition to the betterleaks-scan of the repository-content because that scan can not find these secrets:
|
|
16
|
+
betterleaks does not look into archives, so the "*.tar"-files produced by the image-builds are invisible to it, and secrets
|
|
17
|
+
which only exist in the image (build-arguments recorded in the image-history, environment-variables of the image and files
|
|
18
|
+
which are generated or copied during the image-build) are not part of the repository-content at all. An image-artifact is
|
|
19
|
+
published to a registry, so a secret contained in it is readable by everybody who is allowed to pull the image."""
|
|
20
|
+
|
|
21
|
+
sc: ScriptCollectionCore = None
|
|
22
|
+
|
|
23
|
+
#only the beginning of a file inside a layer is read: keys, tokens and configuration-entries are located at the beginning of small files, so reading more would only cost runtime.
|
|
24
|
+
__maximum_amount_of_bytes_per_file: int = 1024*1024
|
|
25
|
+
|
|
26
|
+
#files which are bigger than this are only checked by their name, not by their content, because they are program- or data-files which do not contain configuration.
|
|
27
|
+
__maximum_size_of_file_to_read: int = 4*1024*1024
|
|
28
|
+
|
|
29
|
+
#a private key must never be part of an image-artifact, independent of the file it is stored in.
|
|
30
|
+
__private_key_pattern = re.compile(r"-----BEGIN (?:[A-Z0-9 ]+ )?PRIVATE KEY-----")
|
|
31
|
+
|
|
32
|
+
#credentials which are part of an url (for example "https://user:token@my-registry.example.com/simple"), which is the usual way credentials for package-sources leak into an image.
|
|
33
|
+
__credentials_in_url_pattern = re.compile(r"(?P<scheme>[a-zA-Z][a-zA-Z0-9+.\-]*)://(?P<user>[^/\s:@]+):(?P<password>[^/\s@]+)@(?P<host>[^/\s]+)")
|
|
34
|
+
|
|
35
|
+
#an assignment whose name indicates a secret (for example "NuGetPassword=abc123" in a build-argument recorded in the image-history).
|
|
36
|
+
__secret_assignment_pattern = re.compile(r"(?i)\b(?P<name>[A-Za-z0-9_.\-]*(?:password|passwort|token|secret|apikey|api_key|credential|accesskey|privatekey)[A-Za-z0-9_.\-]*)\s*[=:]\s*(?P<value>[^\s\"']+)")
|
|
37
|
+
|
|
38
|
+
#files whose name alone is a reason to look at their content even if the file-extension does not indicate a text-file.
|
|
39
|
+
__sensitive_file_pattern = re.compile(r"(?i)(^|/)(\.npmrc|\.netrc|\.git-credentials|\.pypirc|pip\.conf|nuget\.config|credentials|id_rsa|id_dsa|id_ecdsa|id_ed25519)$|\.(pfx|p12|key|pem|jks|keystore)$")
|
|
40
|
+
|
|
41
|
+
#file-extensions which typically contain configuration and therefore may contain credentials.
|
|
42
|
+
__configuration_file_extensions: list[str] = [".conf", ".config", ".cfg", ".env", ".ini", ".json", ".properties", ".sh", ".toml", ".txt", ".xml", ".yaml", ".yml"]
|
|
43
|
+
|
|
44
|
+
#values which look like a secret-assignment but obviously are none.
|
|
45
|
+
__placeholder_values: list[str] = ["true", "false", "none", "null", "nil", "empty", "changeme", "unset", "todo"]
|
|
46
|
+
|
|
47
|
+
def __init__(self, sc: ScriptCollectionCore):
|
|
48
|
+
self.sc = sc
|
|
49
|
+
|
|
50
|
+
@GeneralUtilities.check_arguments
|
|
51
|
+
def search_for_secrets_in_image(self, image: str, repository_for_allowlist: str = None) -> list[str]:
|
|
52
|
+
"""Scans the given OCI-image for secrets and returns a human-readable description for every finding.
|
|
53
|
+
'image' is the reference of an image which is already available in the local docker-instance (for example
|
|
54
|
+
'myimage:1.0.0'); it is exported with 'docker save' and the resulting archive is scanned.
|
|
55
|
+
'repository_for_allowlist' is optional: if it is given, the '[[allowlists]]'-entries of its '.betterleaks.toml' are
|
|
56
|
+
applied, so the same allowlist-file applies to the repository-scan and to the image-scan."""
|
|
57
|
+
self.sc.log.log(f"Search for secrets in image \"{image}\"...", LogLevel.Debug)
|
|
58
|
+
temporary_folder: str = tempfile.mkdtemp()
|
|
59
|
+
try:
|
|
60
|
+
image_file: str = os.path.join(temporary_folder, "image.tar")
|
|
61
|
+
#the image is exported instead of being read from the registry-storage directly, because 'docker save' produces a
|
|
62
|
+
#documented archive-format which is independent of the storage-driver and of the used container-engine.
|
|
63
|
+
result = self.sc.run_program_argsasarray("docker", ["save", image, "-o", image_file], throw_exception_if_exitcode_is_not_zero=False, print_live_output=False)
|
|
64
|
+
if result[0] != 0:
|
|
65
|
+
raise ValueError(f"The image '{image}' could not be exported (exit-code {result[0]}: {result[2].strip()}). The image must be available in the local docker-instance to be scanned.")
|
|
66
|
+
findings: list[str] = self.scan_image_artifact(image_file, repository_for_allowlist)
|
|
67
|
+
finally:
|
|
68
|
+
GeneralUtilities.ensure_directory_does_not_exist(temporary_folder)
|
|
69
|
+
return [f"image \"{image}\": {finding}" for finding in findings]
|
|
70
|
+
|
|
71
|
+
@GeneralUtilities.check_arguments
|
|
72
|
+
def scan_image_artifact(self, image_file: str, repository_for_allowlist: str = None) -> list[str]:
|
|
73
|
+
"""Scans a single image-artifact (a tar-file as produced by 'docker save' or 'docker buildx build --output type=docker')."""
|
|
74
|
+
findings: list[str] = []
|
|
75
|
+
with tarfile.open(image_file, "r") as image_archive:
|
|
76
|
+
for member in image_archive.getmembers():
|
|
77
|
+
if not member.isfile() or member.size == 0:
|
|
78
|
+
continue
|
|
79
|
+
findings = findings+self.__scan_member_of_image_archive(image_archive, member)
|
|
80
|
+
if repository_for_allowlist is None:
|
|
81
|
+
return findings
|
|
82
|
+
allowlist_patterns: list[re.Pattern] = self.__get_allowlist_patterns(repository_for_allowlist)
|
|
83
|
+
return [finding for finding in findings if not self.__is_allowlisted(finding, allowlist_patterns)]
|
|
84
|
+
|
|
85
|
+
@GeneralUtilities.check_arguments
|
|
86
|
+
def __scan_member_of_image_archive(self, image_archive: tarfile.TarFile, member: tarfile.TarInfo) -> list[str]:
|
|
87
|
+
#the members of an image-archive are either the image-metadata (json) or a layer (tar). Which one it is, is decided by
|
|
88
|
+
#the content and not by the filename, because the docker-archive-format and the oci-archive-format use different names.
|
|
89
|
+
member_file = image_archive.extractfile(member)
|
|
90
|
+
if member_file is None:
|
|
91
|
+
return []
|
|
92
|
+
first_byte: bytes = member_file.read(1)
|
|
93
|
+
member_file.seek(0)
|
|
94
|
+
if first_byte in (b"{", b"["):
|
|
95
|
+
return self.__scan_image_metadata(member_file)
|
|
96
|
+
return self.__scan_layer(member_file)
|
|
97
|
+
|
|
98
|
+
@GeneralUtilities.check_arguments
|
|
99
|
+
def __scan_image_metadata(self, metadata_file) -> list[str]:
|
|
100
|
+
try:
|
|
101
|
+
metadata = json.loads(metadata_file.read().decode("utf-8", errors="ignore"))
|
|
102
|
+
except ValueError:
|
|
103
|
+
return []
|
|
104
|
+
if not isinstance(metadata, dict):
|
|
105
|
+
return []
|
|
106
|
+
findings: list[str] = []
|
|
107
|
+
#the history contains one entry per instruction of the containerfile. The build-arguments which were in scope are part of
|
|
108
|
+
#the recorded command, which is why a secret passed via "--build-arg" ends up in the metadata of the published image.
|
|
109
|
+
for history_entry in metadata.get("history") or []:
|
|
110
|
+
if isinstance(history_entry, dict):
|
|
111
|
+
findings = findings+[f"image-history: {description}" for description in self.__find_secrets_in_text(str(history_entry.get("created_by") or GeneralUtilities.empty_string), True)]
|
|
112
|
+
configuration = metadata.get("config") or metadata.get("Config") or {}
|
|
113
|
+
if isinstance(configuration, dict):
|
|
114
|
+
for environment_variable in configuration.get("Env") or []:
|
|
115
|
+
findings = findings+[f"image-environment-variable: {description}" for description in self.__find_secrets_in_text(str(environment_variable), True)]
|
|
116
|
+
labels = configuration.get("Labels") or {}
|
|
117
|
+
if isinstance(labels, dict):
|
|
118
|
+
for label_name, label_value in labels.items():
|
|
119
|
+
findings = findings+[f"image-label: {description}" for description in self.__find_secrets_in_text(f"{label_name}={label_value}", True)]
|
|
120
|
+
return findings
|
|
121
|
+
|
|
122
|
+
@GeneralUtilities.check_arguments
|
|
123
|
+
def __scan_layer(self, layer_file) -> list[str]:
|
|
124
|
+
findings: list[str] = []
|
|
125
|
+
try:
|
|
126
|
+
#the layer is read as a stream because a layer can be several gigabytes big and does not have to be kept in memory.
|
|
127
|
+
with tarfile.open(fileobj=layer_file, mode="r|*") as layer:
|
|
128
|
+
for member in layer:
|
|
129
|
+
if not member.isfile() or member.size == 0:
|
|
130
|
+
continue
|
|
131
|
+
if not self.__content_of_member_should_be_checked(member):
|
|
132
|
+
continue
|
|
133
|
+
member_file = layer.extractfile(member)
|
|
134
|
+
if member_file is None:
|
|
135
|
+
continue
|
|
136
|
+
content: str = member_file.read(self.__maximum_amount_of_bytes_per_file).decode("utf-8", errors="ignore")
|
|
137
|
+
findings = findings+[f"file \"{self.__normalize_member_name(member.name)}\" in image-layer: {description}" for description in self.__find_secrets_in_text(content, False)]
|
|
138
|
+
except tarfile.TarError:
|
|
139
|
+
#the member is not a layer (for example a signature- or an index-file). Such members do not contain secrets.
|
|
140
|
+
return []
|
|
141
|
+
return findings
|
|
142
|
+
|
|
143
|
+
@GeneralUtilities.check_arguments
|
|
144
|
+
def __content_of_member_should_be_checked(self, member: tarfile.TarInfo) -> bool:
|
|
145
|
+
name: str = self.__normalize_member_name(member.name)
|
|
146
|
+
if os.path.basename(name).startswith(".wh."):
|
|
147
|
+
return False # a whiteout-file marks a deletion and has no content
|
|
148
|
+
if self.__sensitive_file_pattern.search(name) is not None:
|
|
149
|
+
return True
|
|
150
|
+
if self.__maximum_size_of_file_to_read < member.size:
|
|
151
|
+
return False
|
|
152
|
+
return os.path.splitext(name)[1].lower() in self.__configuration_file_extensions
|
|
153
|
+
|
|
154
|
+
@GeneralUtilities.check_arguments
|
|
155
|
+
def __normalize_member_name(self, member_name: str) -> str:
|
|
156
|
+
result: str = member_name.replace("\\", "/")
|
|
157
|
+
if result.startswith("./"):
|
|
158
|
+
result = result[2:]
|
|
159
|
+
return result
|
|
160
|
+
|
|
161
|
+
@GeneralUtilities.check_arguments
|
|
162
|
+
def __find_secrets_in_text(self, text: str, check_assignments: bool) -> list[str]:
|
|
163
|
+
"""Returns a description for every secret found in the given text. The descriptions never contain the secret itself,
|
|
164
|
+
because they are written to the build-log, which is usually kept and often readable by more people than the secret.
|
|
165
|
+
|
|
166
|
+
'check_assignments' enables the name-based heuristic which reports an assignment whose name indicates a secret (for
|
|
167
|
+
example 'NuGetPassword=abc123'). It is only meaningful for the image-metadata, where the amount of values is small and
|
|
168
|
+
every value was set by the build itself. For the content of the files of a layer it must be disabled: a layer contains
|
|
169
|
+
thousands of files of the base-image (translation-resources, sample-configurations, documentation) in which a word like
|
|
170
|
+
'Password' is a label and not a secret, which would drown the real findings in false positives."""
|
|
171
|
+
findings: list[str] = []
|
|
172
|
+
if not GeneralUtilities.string_has_content(text):
|
|
173
|
+
return findings
|
|
174
|
+
if self.__private_key_pattern.search(text) is not None:
|
|
175
|
+
findings.append("contains a private key")
|
|
176
|
+
for match in self.__credentials_in_url_pattern.finditer(text):
|
|
177
|
+
findings.append(f"contains credentials in the url \"{match.group('scheme')}://{match.group('user')}:***@{match.group('host')}\"")
|
|
178
|
+
if check_assignments:
|
|
179
|
+
for match in self.__secret_assignment_pattern.finditer(text):
|
|
180
|
+
if self.__value_is_a_secret(match.group("value")):
|
|
181
|
+
findings.append(f"contains a value for \"{match.group('name')}\"")
|
|
182
|
+
return findings
|
|
183
|
+
|
|
184
|
+
@GeneralUtilities.check_arguments
|
|
185
|
+
def __value_is_a_secret(self, value: str) -> bool:
|
|
186
|
+
value = value.strip().strip("\"'")
|
|
187
|
+
if not GeneralUtilities.string_has_content(value):
|
|
188
|
+
return False
|
|
189
|
+
if value.lower() in self.__placeholder_values:
|
|
190
|
+
return False
|
|
191
|
+
if set(value) == {"*"}:
|
|
192
|
+
return False
|
|
193
|
+
#a reference to a variable (for example "$MyToken", "${MyToken}" or "%MyToken%") is the desired way to handle a secret and therefore not a finding.
|
|
194
|
+
if value.startswith("$") or (value.startswith("%") and value.endswith("%")):
|
|
195
|
+
return False
|
|
196
|
+
return True
|
|
197
|
+
|
|
198
|
+
@GeneralUtilities.check_arguments
|
|
199
|
+
def __get_allowlist_patterns(self, repository: str) -> list[re.Pattern]:
|
|
200
|
+
"""Reads the allowlisted paths and regexes from '<repository>/.betterleaks.toml' so the image-scan honours the same
|
|
201
|
+
allowlist as the repository-scan instead of requiring a second configuration-file."""
|
|
202
|
+
configuration_file: str = os.path.join(repository, ".betterleaks.toml")
|
|
203
|
+
if not os.path.isfile(configuration_file):
|
|
204
|
+
return []
|
|
205
|
+
try:
|
|
206
|
+
with open(configuration_file, "rb") as file_handle:
|
|
207
|
+
configuration = tomllib.load(file_handle)
|
|
208
|
+
except tomllib.TOMLDecodeError as exception:
|
|
209
|
+
self.sc.log.log_exception(f"'{configuration_file}' could not be parsed, so no allowlist is applied to the image-scan:", exception, LogLevel.Warning)
|
|
210
|
+
return []
|
|
211
|
+
allowlists: list = []
|
|
212
|
+
for key in ("allowlist", "allowlists"):
|
|
213
|
+
value = configuration.get(key)
|
|
214
|
+
if isinstance(value, dict):
|
|
215
|
+
allowlists.append(value)
|
|
216
|
+
elif isinstance(value, list):
|
|
217
|
+
allowlists = allowlists+[entry for entry in value if isinstance(entry, dict)]
|
|
218
|
+
result: list[re.Pattern] = []
|
|
219
|
+
for allowlist in allowlists:
|
|
220
|
+
for key in ("paths", "regexes"):
|
|
221
|
+
for pattern in allowlist.get(key) or []:
|
|
222
|
+
try:
|
|
223
|
+
result.append(re.compile(str(pattern)))
|
|
224
|
+
except re.error as exception:
|
|
225
|
+
self.sc.log.log_exception(f"The allowlist-entry '{pattern}' of '{configuration_file}' is not a valid regular expression and is therefore ignored:", exception, LogLevel.Warning)
|
|
226
|
+
return result
|
|
227
|
+
|
|
228
|
+
@GeneralUtilities.check_arguments
|
|
229
|
+
def __is_allowlisted(self, finding: str, allowlist_patterns: list[re.Pattern]) -> bool:
|
|
230
|
+
return any(pattern.search(finding) is not None for pattern in allowlist_patterns)
|