ScriptCollection 4.3.56__py3-none-any.whl → 4.3.58__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/Executables.py +35 -2
- ScriptCollection/GeneralUtilities.py +1 -0
- ScriptCollection/ScriptCollectionCore.py +7 -31
- ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +1 -4
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +64 -69
- ScriptCollection/TFCPS/PackageSource.py +26 -0
- ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py +26 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +3 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +149 -45
- ScriptCollection/TFCPS/TFCPS_OCIImageSecretScan.py +230 -0
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +169 -34
- {scriptcollection-4.3.56.dist-info → scriptcollection-4.3.58.dist-info}/METADATA +1 -1
- {scriptcollection-4.3.56.dist-info → scriptcollection-4.3.58.dist-info}/RECORD +16 -13
- {scriptcollection-4.3.56.dist-info → scriptcollection-4.3.58.dist-info}/entry_points.txt +1 -0
- {scriptcollection-4.3.56.dist-info → scriptcollection-4.3.58.dist-info}/WHEEL +0 -0
- {scriptcollection-4.3.56.dist-info → scriptcollection-4.3.58.dist-info}/top_level.txt +0 -0
ScriptCollection/Executables.py
CHANGED
|
@@ -12,6 +12,7 @@ from .ScriptCollectionCore import ScriptCollectionCore
|
|
|
12
12
|
from .GeneralUtilities import GeneralUtilities, VersionEcholon
|
|
13
13
|
from .SCLog import LogLevel, SCLog
|
|
14
14
|
from .TFCPS.TFCPS_CodeUnit_BuildCodeUnits import TFCPS_CodeUnit_BuildCodeUnits
|
|
15
|
+
from .TFCPS.TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
|
|
15
16
|
from .TFCPS.TFCPS_Tools_General import TFCPS_Tools_General
|
|
16
17
|
from .OCIImages.OCIImageManager import OCIImageManager
|
|
17
18
|
|
|
@@ -258,16 +259,23 @@ def BuildCodeUnits() -> int:
|
|
|
258
259
|
parser.add_argument('-u','--assertnonewchanges', required=False, default=False, action='store_true')
|
|
259
260
|
parser.add_argument('-m','--addreadytomergeflag', required=False, default=False, action='store_true')
|
|
260
261
|
parser.add_argument('-c','--runincontainer', required=False, default=False, action='store_true')
|
|
262
|
+
parser.add_argument('-b','--basemountfolder', required=False, default=None, help="Only used if -c is set.")
|
|
261
263
|
args = parser.parse_args()
|
|
262
264
|
|
|
263
265
|
verbosity=LogLevel(int(args.verbosity))
|
|
264
266
|
repo:str=GeneralUtilities.resolve_relative_path(args.repositoryfolder,os.getcwd())
|
|
267
|
+
|
|
265
268
|
t:TFCPS_CodeUnit_BuildCodeUnits=TFCPS_CodeUnit_BuildCodeUnits(repo,verbosity,args.targetenvironment,args.additionalargumentsfile,not args.nocache,args.ispremerge,args.assertnonewchanges,args.addreadytomergeflag)
|
|
266
269
|
if args.runincontainer:
|
|
267
|
-
|
|
270
|
+
base_mount_folder:str=args.basemountfolder
|
|
271
|
+
if base_mount_folder is None:
|
|
272
|
+
base_mount_folder=repo
|
|
273
|
+
base_mount_folder:str=GeneralUtilities.resolve_relative_path(base_mount_folder,os.getcwd())
|
|
274
|
+
success, _ = t.build_codeunits_in_container(base_mount_folder)
|
|
275
|
+
return 0 if success else 1
|
|
268
276
|
else:
|
|
269
277
|
t.build_codeunits()
|
|
270
|
-
|
|
278
|
+
return 0
|
|
271
279
|
|
|
272
280
|
def BuildCodeUnitsC() -> int:
|
|
273
281
|
parser = argparse.ArgumentParser()
|
|
@@ -290,6 +298,7 @@ def BuildCodeUnitsC() -> int:
|
|
|
290
298
|
base_mount_folder:str=args.basemountfolder
|
|
291
299
|
if base_mount_folder is None:
|
|
292
300
|
base_mount_folder=repo
|
|
301
|
+
base_mount_folder:str=GeneralUtilities.resolve_relative_path(base_mount_folder,os.getcwd())
|
|
293
302
|
t:TFCPS_CodeUnit_BuildCodeUnits=TFCPS_CodeUnit_BuildCodeUnits(repo,verbosity,args.targetenvironment,args.additionalargumentsfile,not args.nocache,args.ispremerge,args.assertnonewchanges,args.addreadytomergeflag)
|
|
294
303
|
success, _ = t.build_codeunits_in_container(base_mount_folder)
|
|
295
304
|
return 0 if success else 1
|
|
@@ -1180,6 +1189,30 @@ def SearchForSecrets() -> int:
|
|
|
1180
1189
|
t.search_for_secrets()
|
|
1181
1190
|
return 0
|
|
1182
1191
|
|
|
1192
|
+
def SearchForSecretsInImage() -> int:
|
|
1193
|
+
parser = argparse.ArgumentParser(description="Scans an OCI-image which is available in the local docker-instance for secrets. Exit-code 0: no secrets found. Exit-code 1: secrets found. Exit-code 2: an error occurred.")
|
|
1194
|
+
parser.add_argument('-i', '--image', required=True, help="Reference of the image which should be scanned, for example \"myimage:1.0.0\". The image must already be available in the local docker-instance.")
|
|
1195
|
+
parser.add_argument('-r', '--repository', required=False, default=None, help="Optional path to a repository whose \".betterleaks.toml\" is used to allowlist known false positives.")
|
|
1196
|
+
verbosity_values = ", ".join(f"{lvl.value}={lvl.name}" for lvl in LogLevel)
|
|
1197
|
+
parser.add_argument('-v', '--verbosity', required=False, default=3, help=f"Sets the loglevel. Possible values: {verbosity_values}")
|
|
1198
|
+
args = parser.parse_args()
|
|
1199
|
+
sc: ScriptCollectionCore = ScriptCollectionCore()
|
|
1200
|
+
try:
|
|
1201
|
+
sc.log.loglevel = LogLevel(int(args.verbosity))
|
|
1202
|
+
repository = GeneralUtilities.resolve_relative_path(args.repository, os.getcwd()) if args.repository is not None else None
|
|
1203
|
+
findings: list[str] = TFCPS_OCIImageSecretScan(sc).search_for_secrets_in_image(args.image, repository)
|
|
1204
|
+
if len(findings) == 0:
|
|
1205
|
+
sc.log.log(f"No secrets found in image \"{args.image}\".")
|
|
1206
|
+
return 0
|
|
1207
|
+
for finding in findings:
|
|
1208
|
+
sc.log.log(finding, LogLevel.Error)
|
|
1209
|
+
sc.log.log(f"Found {len(findings)} secret-finding(s) in image \"{args.image}\". A secret which is part of an image is readable by everybody who is allowed to pull that image.", LogLevel.Error)
|
|
1210
|
+
return 1
|
|
1211
|
+
except Exception as exception: # pylint:disable=broad-exception-caught
|
|
1212
|
+
#an error must be distinguishable from a found secret, because "no secrets found" and "the scan could not be done" have opposite meanings for the caller.
|
|
1213
|
+
sc.log.log_exception(f"The image \"{args.image}\" could not be scanned:", exception, LogLevel.Error)
|
|
1214
|
+
return 2
|
|
1215
|
+
|
|
1183
1216
|
def PrepareBuildPipelineForGitlab() -> int:
|
|
1184
1217
|
parser = argparse.ArgumentParser(description="Prepares the build-pipeline-configuration for GitLab.")
|
|
1185
1218
|
verbosity_values = ", ".join(f"{lvl.value}={lvl.name}" for lvl in LogLevel)
|
|
@@ -39,7 +39,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
|
|
|
39
39
|
from .ProgramRunnerPopen import ProgramRunnerPopen
|
|
40
40
|
from .SCLog import SCLog, LogLevel
|
|
41
41
|
|
|
42
|
-
version = "4.3.
|
|
42
|
+
version = "4.3.58"
|
|
43
43
|
__version__ = version
|
|
44
44
|
|
|
45
45
|
class VSCodeWorkspaceShellTask:
|
|
@@ -2547,7 +2547,6 @@ class ScriptCollectionCore:
|
|
|
2547
2547
|
gitignore_file = os.path.join(scriptcollection_folder, ".gitignore")
|
|
2548
2548
|
lines = [
|
|
2549
2549
|
"/Cache/",
|
|
2550
|
-
"RequiredEnvVariables.local.csv",
|
|
2551
2550
|
]
|
|
2552
2551
|
GeneralUtilities.write_lines_to_file(gitignore_file, lines)
|
|
2553
2552
|
|
|
@@ -2693,16 +2692,15 @@ DNS = {domain}
|
|
|
2693
2692
|
|
|
2694
2693
|
@GeneralUtilities.check_arguments
|
|
2695
2694
|
def update_dependencies_of_python_in_requirementstxt_file(self, file: str, ignored_dependencies: list[str]):
|
|
2696
|
-
# TODO consider ignored_dependencies
|
|
2697
2695
|
lines = GeneralUtilities.read_lines_from_file(file)
|
|
2698
2696
|
new_lines = []
|
|
2699
2697
|
for line in lines:
|
|
2700
2698
|
if GeneralUtilities.string_has_content(line):
|
|
2701
|
-
new_lines.append(self.__get_updated_line_for_python_requirements(line.strip()))
|
|
2699
|
+
new_lines.append(self.__get_updated_line_for_python_requirements(ignored_dependencies,line.strip()))
|
|
2702
2700
|
GeneralUtilities.write_lines_to_file(file, new_lines)
|
|
2703
2701
|
|
|
2704
2702
|
@GeneralUtilities.check_arguments
|
|
2705
|
-
def __get_updated_line_for_python_requirements(self, line: str) -> str:
|
|
2703
|
+
def __get_updated_line_for_python_requirements(self, ignored_dependencies: list[str],line: str) -> str:
|
|
2706
2704
|
if "==" in line or "<" in line:
|
|
2707
2705
|
return line
|
|
2708
2706
|
elif ">" in line:
|
|
@@ -2710,6 +2708,8 @@ DNS = {domain}
|
|
|
2710
2708
|
# line is something like "cyclonedx-bom>=2.0.2" and the function must return with the updated version
|
|
2711
2709
|
# (something like "cyclonedx-bom>=2.11.0" for example)
|
|
2712
2710
|
package = line.split(">")[0]
|
|
2711
|
+
if package in ignored_dependencies:
|
|
2712
|
+
return line
|
|
2713
2713
|
operator = ">=" if ">=" in line else ">"
|
|
2714
2714
|
headers = {'Cache-Control': 'no-cache'}
|
|
2715
2715
|
response = requests.get(f'https://pypi.org/pypi/{package}/json', timeout=5, headers=headers)
|
|
@@ -2724,7 +2724,6 @@ DNS = {domain}
|
|
|
2724
2724
|
|
|
2725
2725
|
@GeneralUtilities.check_arguments
|
|
2726
2726
|
def update_dependencies_of_python_in_setupcfg_file(self, setup_cfg_file: str, ignored_dependencies: list[str]):
|
|
2727
|
-
# TODO consider ignored_dependencies
|
|
2728
2727
|
lines = GeneralUtilities.read_lines_from_file(setup_cfg_file)
|
|
2729
2728
|
new_lines = []
|
|
2730
2729
|
requirement_parsing_mode = False
|
|
@@ -2732,7 +2731,7 @@ DNS = {domain}
|
|
|
2732
2731
|
new_line = line
|
|
2733
2732
|
if (requirement_parsing_mode):
|
|
2734
2733
|
if ("<" in line or "=" in line or ">" in line):
|
|
2735
|
-
updated_line = f" {self.__get_updated_line_for_python_requirements(line.strip())}"
|
|
2734
|
+
updated_line = f" {self.__get_updated_line_for_python_requirements(ignored_dependencies,line.strip())}"
|
|
2736
2735
|
new_line = updated_line
|
|
2737
2736
|
else:
|
|
2738
2737
|
requirement_parsing_mode = False
|
|
@@ -3219,35 +3218,12 @@ TXDX
|
|
|
3219
3218
|
result = "<!DOCTYPE html>\n" + result
|
|
3220
3219
|
return result
|
|
3221
3220
|
|
|
3222
|
-
@GeneralUtilities.check_arguments
|
|
3223
|
-
def get_pip_index_url_arguments_from_local_cache(self)->list[str]:
|
|
3224
|
-
arguments=[]
|
|
3225
|
-
pip_folder=GeneralUtilities.normalize_path(self.get_global_cache_folder()+"/Pip")
|
|
3226
|
-
if os.path.isdir(pip_folder):
|
|
3227
|
-
main_index_file=GeneralUtilities.normalize_path(os.path.join(pip_folder, "MainIndex.txt"))
|
|
3228
|
-
if os.path.isfile(main_index_file):
|
|
3229
|
-
lines=GeneralUtilities.read_nonempty_lines_from_file(main_index_file)
|
|
3230
|
-
url=[line for line in lines if line.startswith("IndexURL: ")][0].split(":", 1)[1].strip()
|
|
3231
|
-
arguments.append("--index-url")
|
|
3232
|
-
arguments.append(url)
|
|
3233
|
-
extra_index_folder=GeneralUtilities.normalize_path(os.path.join(pip_folder, "ExtraIndexURLs"))
|
|
3234
|
-
if os.path.isdir(extra_index_folder):
|
|
3235
|
-
index_files=GeneralUtilities.get_direct_files_of_folder(extra_index_folder)
|
|
3236
|
-
if len(index_files) > 0:
|
|
3237
|
-
for indexurl_file in index_files:
|
|
3238
|
-
lines=GeneralUtilities.read_nonempty_lines_from_file(indexurl_file)
|
|
3239
|
-
url=[line for line in lines if line.startswith("IndexURL: ")][0].split(":", 1)[1].strip()
|
|
3240
|
-
arguments.append("--extra-index-url")
|
|
3241
|
-
arguments.append(url)
|
|
3242
|
-
return arguments
|
|
3243
|
-
|
|
3244
3221
|
@GeneralUtilities.check_arguments
|
|
3245
3222
|
def install_requirementstxt_file(self, requirements_txt_file: str):
|
|
3223
|
+
#TODO add the package-sources (including their credentials) which are required to install the dependencies
|
|
3246
3224
|
folder: str = os.path.dirname(requirements_txt_file)
|
|
3247
3225
|
filename: str = os.path.basename(requirements_txt_file)
|
|
3248
3226
|
arguments= ["install", "-r", filename]
|
|
3249
|
-
for argument in self.get_pip_index_url_arguments_from_local_cache():
|
|
3250
|
-
arguments.append(argument)
|
|
3251
3227
|
self.run_program_argsasarray("pip", arguments, folder,print_live_output=self.log.loglevel==LogLevel.Debug)
|
|
3252
3228
|
|
|
3253
3229
|
@GeneralUtilities.check_arguments
|
|
@@ -31,10 +31,7 @@ class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
31
31
|
for custom_argument_key, custom_argument_value in custom_arguments.items():
|
|
32
32
|
args.append("--build-arg")
|
|
33
33
|
args.append(f"{custom_argument_key}={custom_argument_value}")
|
|
34
|
-
|
|
35
|
-
if len(pip_args)>0:
|
|
36
|
-
args.append("--build-arg")
|
|
37
|
-
args.append(f"PipIndexUrlArguments={' '.join(pip_args)}")
|
|
34
|
+
#TODO pass the package-sources (including their credentials) which are required to install the dependencies inside the image
|
|
38
35
|
args = args+["--tag", f"{codeunitname_lower}:latest", "--tag", f"{codeunitname_lower}:{codeunitversion}", "--file", f"{codeunitname}/Dockerfile"]
|
|
39
36
|
if not self.use_cache():
|
|
40
37
|
args.append("--no-cache")
|
|
@@ -3,12 +3,12 @@ import re
|
|
|
3
3
|
import shutil
|
|
4
4
|
import uuid
|
|
5
5
|
import json
|
|
6
|
-
import csv
|
|
7
6
|
from lxml import etree
|
|
8
7
|
import yaml
|
|
9
8
|
from .CertificateGeneratorInformationBase import CertificateGeneratorInformationBase
|
|
10
9
|
from ...GeneralUtilities import GeneralUtilities
|
|
11
10
|
from ...SCLog import LogLevel
|
|
11
|
+
from ..PackageSource import PackageSource
|
|
12
12
|
from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base,TFCPS_CodeUnitSpecific_Base_CLI
|
|
13
13
|
|
|
14
14
|
class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
@@ -20,10 +20,72 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
20
20
|
super().__init__(current_file, verbosity,targetenvironmenttype,use_cache,is_pre_merge)
|
|
21
21
|
self.csproj_file=os.path.join(self.get_codeunit_folder(), self.get_codeunit_name(), self.get_codeunit_name() + ".csproj")
|
|
22
22
|
self.is_library="<OutputType>Library</OutputType>" in GeneralUtilities.read_text_from_file(self.csproj_file)#TODO do a real check by checking this property using xpath
|
|
23
|
+
#every dotnet-operation of a codeunit (build, test, linting, ...) restores the dependencies of the codeunit, so the sources
|
|
24
|
+
#they are downloaded from have to be available for all of them and not only for the build.
|
|
25
|
+
self.__ensure_declared_package_sources_are_available()
|
|
26
|
+
|
|
27
|
+
@GeneralUtilities.check_arguments
|
|
28
|
+
def __ensure_declared_package_sources_are_available(self) -> None:
|
|
29
|
+
"""Ensures that every NuGet-source which is declared for this machine (see TFCPS_Tools_General.get_declared_package_sources) is
|
|
30
|
+
registered with its current credentials, so a codeunit can also use dependencies which are not available on nuget.org.
|
|
31
|
+
Already existing sources are updated instead of being replaced, and sources which are not declared are not touched at all: the
|
|
32
|
+
registration is stored in the NuGet-configuration of the current user, which - when the build runs directly on the host - is
|
|
33
|
+
the configuration the user also uses for their own work and which must not be destroyed by a build."""
|
|
34
|
+
package_sources: list[PackageSource] = self.tfcps_Tools_General.get_declared_package_sources("CSharp")
|
|
35
|
+
if len(package_sources) == 0:
|
|
36
|
+
return
|
|
37
|
+
codeunit_folder: str = self.get_codeunit_folder()
|
|
38
|
+
registered_sources: dict[str, str] = self.__get_registered_nuget_sources(codeunit_folder)
|
|
39
|
+
for package_source in package_sources:
|
|
40
|
+
#a source which is already registered is updated instead of being added a second time. It is searched by its url and not only
|
|
41
|
+
#by its name, because the same feed can already be registered under a different name (which is the usual case on a machine on
|
|
42
|
+
#which the user configured the feed manually) and a second registration of the same url would let every restore query it twice.
|
|
43
|
+
existing_name: str = None
|
|
44
|
+
for registered_name, registered_url in registered_sources.items():
|
|
45
|
+
if self.__nuget_sources_are_equal(registered_url, package_source.url) or registered_name.lower() == package_source.name:
|
|
46
|
+
existing_name = registered_name
|
|
47
|
+
break
|
|
48
|
+
if existing_name is None:
|
|
49
|
+
arguments = ["nuget", "add", "source", package_source.url, "--name", package_source.name]
|
|
50
|
+
else:
|
|
51
|
+
#the credentials are set again even when the source already exists, because a token can have been rotated since it was registered.
|
|
52
|
+
arguments = ["nuget", "update", "source", existing_name, "--source", package_source.url]
|
|
53
|
+
if package_source.has_credentials():
|
|
54
|
+
arguments = arguments+["--username", package_source.username, "--password", package_source.password]
|
|
55
|
+
if not GeneralUtilities.current_system_is_windows():
|
|
56
|
+
#without this the password can not be encrypted, which lets the command fail on the operating-systems which do not support the encryption.
|
|
57
|
+
arguments.append("--store-password-in-clear-text")
|
|
58
|
+
arguments_for_log = list(arguments)
|
|
59
|
+
if package_source.has_credentials():
|
|
60
|
+
arguments_for_log[arguments_for_log.index(package_source.password)] = "***"#the password must not be written to the build-log
|
|
61
|
+
self._protected_sc.log.log(f"Add NuGet-source \"{package_source.name}\" ({package_source.url})..." if existing_name is None else f"Update the already registered NuGet-source \"{existing_name}\" ({package_source.url})...", LogLevel.Debug)
|
|
62
|
+
self._protected_sc.run_program_argsasarray("dotnet", arguments, codeunit_folder, arguments_for_log=arguments_for_log, print_live_output=False)
|
|
63
|
+
|
|
64
|
+
@GeneralUtilities.check_arguments
|
|
65
|
+
def __get_registered_nuget_sources(self, folder: str) -> dict[str, str]:
|
|
66
|
+
"""Returns the NuGet-sources which are registered for the given folder as a mapping of their name to their url. The sources are
|
|
67
|
+
read from the output of "dotnet nuget list source", which prints the name (with its state) and the url of a source in two
|
|
68
|
+
consecutive lines."""
|
|
69
|
+
result: dict[str, str] = {}
|
|
70
|
+
list_result = self._protected_sc.run_program_argsasarray("dotnet", ["nuget", "list", "source"], folder, throw_exception_if_exitcode_is_not_zero=False, print_live_output=False)
|
|
71
|
+
name_pattern = re.compile(r"^\s*\d+\.\s+(.+?)\s+\[(?:Enabled|Disabled)\]\s*$")
|
|
72
|
+
current_name: str = None
|
|
73
|
+
for line in list_result[1].splitlines():
|
|
74
|
+
match = name_pattern.match(line)
|
|
75
|
+
if match is not None:
|
|
76
|
+
current_name = match.group(1).strip()
|
|
77
|
+
elif current_name is not None and GeneralUtilities.string_has_content(line):
|
|
78
|
+
result[current_name] = line.strip()
|
|
79
|
+
current_name = None
|
|
80
|
+
return result
|
|
81
|
+
|
|
82
|
+
@GeneralUtilities.check_arguments
|
|
83
|
+
def __nuget_sources_are_equal(self, url: str, other_url: str) -> bool:
|
|
84
|
+
"""Compares two urls of package-sources. A trailing slash and the case are ignored because they do not address a different feed."""
|
|
85
|
+
return url.strip().rstrip("/").lower() == other_url.strip().rstrip("/").lower()
|
|
23
86
|
|
|
24
87
|
@GeneralUtilities.check_arguments
|
|
25
88
|
def build(self,runtimes:list[str],generate_open_api_spec:bool) -> None:
|
|
26
|
-
self.__reset_nuget_sources()
|
|
27
89
|
if self.is_library:
|
|
28
90
|
self.standardized_tasks_build_for_dotnet_library_project(runtimes)
|
|
29
91
|
GeneralUtilities.assert_condition(not generate_open_api_spec,"OpenAPI-Specification can not be generated for a library.")
|
|
@@ -32,73 +94,6 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
32
94
|
if generate_open_api_spec:
|
|
33
95
|
self.generate_openapi_file(runtimes[0])
|
|
34
96
|
|
|
35
|
-
@GeneralUtilities.check_arguments
|
|
36
|
-
def __reset_nuget_sources(self) -> None:
|
|
37
|
-
if self._protected_sc.is_running_in_build_container():
|
|
38
|
-
|
|
39
|
-
self._protected_sc.log.log("Remove all existing NuGet-sources...", LogLevel.Debug)
|
|
40
|
-
list_result = self._protected_sc.run_program_argsasarray("dotnet", ["nuget", "list", "source"], throw_exception_if_exitcode_is_not_zero=False,print_live_output=False)
|
|
41
|
-
existing_source_name_pattern = re.compile(r"^\s*\d+\.\s+(.+?)\s+\[(?:Enabled|Disabled)\]\s*$")
|
|
42
|
-
for line in list_result[1].splitlines():
|
|
43
|
-
m = existing_source_name_pattern.match(line)
|
|
44
|
-
if not m:
|
|
45
|
-
continue
|
|
46
|
-
existing_source_name = m.group(1).strip()
|
|
47
|
-
self._protected_sc.log.log(f"Remove existing NuGet-source '{existing_source_name}'", LogLevel.Debug)
|
|
48
|
-
self._protected_sc.run_program_argsasarray("dotnet", ["nuget", "remove", "source", existing_source_name], throw_exception_if_exitcode_is_not_zero=False)
|
|
49
|
-
|
|
50
|
-
sources:list[tuple[str,str,str,str]] = []
|
|
51
|
-
|
|
52
|
-
csv_file = os.path.join(self._protected_sc.get_scriptcollection_configuration_folder(), "TFCPS", "CustomC#Dependencies.csv")
|
|
53
|
-
if os.path.isfile(csv_file):
|
|
54
|
-
self._protected_sc.log.log(f"Add custom NuGet-sources from '{csv_file}'...",LogLevel.Debug)
|
|
55
|
-
with open(csv_file, encoding="utf-8-sig", newline="") as f:
|
|
56
|
-
reader = csv.DictReader(f)
|
|
57
|
-
for row in reader:
|
|
58
|
-
name = (row.get("Name") or "").strip()
|
|
59
|
-
url = (row.get("Url") or "").strip()
|
|
60
|
-
username = (row.get("Username") or "").strip()
|
|
61
|
-
password = (row.get("Password") or "").strip()
|
|
62
|
-
if not name or not url:
|
|
63
|
-
continue
|
|
64
|
-
sources.append((name, url, username, password))
|
|
65
|
-
|
|
66
|
-
if len(os.environ.items()) > 0:
|
|
67
|
-
self._protected_sc.log.log(f"Add custom NuGet-sources from environment variables...",LogLevel.Debug)
|
|
68
|
-
env_name_pattern = re.compile(r"^Dependency_CSharp_(.+?)_Name$")
|
|
69
|
-
for env_var_name, env_var_value in os.environ.items():
|
|
70
|
-
m = env_name_pattern.match(env_var_name)
|
|
71
|
-
if not m:
|
|
72
|
-
continue
|
|
73
|
-
dependency_name = m.group(1)
|
|
74
|
-
name = (env_var_value or "").strip()
|
|
75
|
-
url = (os.environ.get(f"Dependency_CSharp_{dependency_name}_URL") or "").strip()
|
|
76
|
-
username = (os.environ.get(f"Dependency_CSharp_{dependency_name}_Username") or "").strip()
|
|
77
|
-
password = (os.environ.get(f"Dependency_CSharp_{dependency_name}_Passwort") or "").strip()
|
|
78
|
-
if not name or not url:
|
|
79
|
-
continue
|
|
80
|
-
sources.append((name, url, username, password))
|
|
81
|
-
|
|
82
|
-
self._protected_sc.run_program_argsasarray("dotnet", ["nuget", "add", "source", "https://api.nuget.org/v3/index.json", "--name", "nuget.org"], print_live_output=True)
|
|
83
|
-
for name, url, username, password in sources:
|
|
84
|
-
#TODO add only the sources which are needed by the project
|
|
85
|
-
self._protected_sc.log.log(f"Add NuGet-source '{name}' with url '{url}'",LogLevel.Debug)
|
|
86
|
-
args = ["nuget", "add", "source", url, "--name", name]
|
|
87
|
-
if username:
|
|
88
|
-
args += ["--username", username]
|
|
89
|
-
if password:
|
|
90
|
-
args += ["--password", password, "--store-password-in-clear-text"]
|
|
91
|
-
args_for_log = list(args)
|
|
92
|
-
if password:
|
|
93
|
-
args_for_log[args_for_log.index(password)] = "***"
|
|
94
|
-
self._protected_sc.run_program_argsasarray("dotnet", args, arguments_for_log=args_for_log, throw_exception_if_exitcode_is_not_zero=False, print_live_output=self.get_verbosity()==LogLevel.Debug)
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
if self._protected_sc.log.loglevel==LogLevel.Debug:
|
|
98
|
-
self._protected_sc.run_program_argsasarray("dotnet", ["nuget","list","source","--format","detailed"], print_live_output=True)
|
|
99
|
-
else:
|
|
100
|
-
self._protected_sc.run_program_argsasarray("dotnet", ["nuget","list","source"], print_live_output=True)
|
|
101
|
-
|
|
102
97
|
@GeneralUtilities.check_arguments
|
|
103
98
|
def generate_openapi_file(self, runtime: str) -> None:
|
|
104
99
|
swagger_document_name: str = "APISpecification"
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from ..GeneralUtilities import GeneralUtilities
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class PackageSource:
|
|
5
|
+
"""A source from which packages of a dependency of a codeunit are downloaded (for example a NuGet-feed).
|
|
6
|
+
A package-source is not part of a repository: which sources a machine may use and which credentials it uses for them is
|
|
7
|
+
machine-specific information. It is therefore transported via environment-variables (see
|
|
8
|
+
TFCPS_Tools_General.get_declared_package_sources), whose values are typically defined in the user-specific
|
|
9
|
+
configuration-file of the required environment-variables."""
|
|
10
|
+
|
|
11
|
+
name: str = None
|
|
12
|
+
url: str = None
|
|
13
|
+
username: str = None
|
|
14
|
+
password: str = None
|
|
15
|
+
|
|
16
|
+
def __init__(self, name: str, url: str, username: str, password: str):
|
|
17
|
+
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(name), "A package-source must have a name.")
|
|
18
|
+
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(url), f"The package-source '{name}' must have an url.")
|
|
19
|
+
self.name = name
|
|
20
|
+
self.url = url
|
|
21
|
+
self.username = username
|
|
22
|
+
self.password = password
|
|
23
|
+
|
|
24
|
+
@GeneralUtilities.check_arguments
|
|
25
|
+
def has_credentials(self) -> bool:
|
|
26
|
+
return GeneralUtilities.string_has_content(self.username) and GeneralUtilities.string_has_content(self.password)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from ..GeneralUtilities import GeneralUtilities
|
|
2
|
+
from .TFCPS_CodeUnit_BuildCodeUnit import TFCPS_CodeUnit_BuildCodeUnit
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class TFCPS_BuildCodeUnitsHook:
|
|
6
|
+
"""Allows to run additional actions at defined moments of the build of the codeunits of a repository.
|
|
7
|
+
|
|
8
|
+
This is an internal extension-point of ScriptCollection: it exists so a workflow which has to do something between the regular
|
|
9
|
+
build-steps (for example the dependency-update) can use the regular build-process instead of reimplementing a reduced variant of it.
|
|
10
|
+
Reimplementing it is what makes such a workflow miss preparation-steps like the required environment-variables, the custom
|
|
11
|
+
pre-codeunit-build-script or the package-sources which are needed to resolve the dependencies of a codeunit.
|
|
12
|
+
|
|
13
|
+
The extension-point for the user of ScriptCollection are the custom scripts in the configuration-folder (see
|
|
14
|
+
TFCPS_Tools_General.get_custom_script_file), not this class.
|
|
15
|
+
|
|
16
|
+
All operations have an empty default-implementation, so an implementation only has to define the moments it is interested in."""
|
|
17
|
+
|
|
18
|
+
@GeneralUtilities.check_arguments
|
|
19
|
+
def run_after_preparation(self, repository: str) -> None:
|
|
20
|
+
"""Is called after the build was prepared (environment-variables, custom pre-codeunit-build-script and prepare-script of the
|
|
21
|
+
repository) and before the first codeunit is built."""
|
|
22
|
+
|
|
23
|
+
@GeneralUtilities.check_arguments
|
|
24
|
+
def run_after_codeunit_was_built(self, codeunit_build: TFCPS_CodeUnit_BuildCodeUnit) -> None:
|
|
25
|
+
"""Is called after a codeunit was built. The codeunit-build is passed so an implementation can build the codeunit again, for
|
|
26
|
+
example when it changed the codeunit and wants to verify that it is still buildable."""
|
|
@@ -45,6 +45,9 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
|
|
|
45
45
|
self.tfcps_Tools_General.assert_is_codeunit_folder(self.__codeunit_folder)
|
|
46
46
|
self.__repository_folder=GeneralUtilities.resolve_relative_path("..",self.__codeunit_folder)
|
|
47
47
|
self._protected_sc.assert_is_git_repository(self.__repository_folder)
|
|
48
|
+
#ensure the environment-variables which the product declares as required are also available when this codeunit-script is started
|
|
49
|
+
#directly (for example "<codeunit>/Other/Build/Build.py" or "task bb") and not by "scbuildcodeunits".
|
|
50
|
+
self.tfcps_Tools_General.ensure_required_environment_variables_are_set(self.__repository_folder)
|
|
48
51
|
|
|
49
52
|
def __search_codeunit_folder(self)->str:
|
|
50
53
|
current_path:str=os.path.dirname(self.__current_file)
|