ScriptCollection 4.3.62__py3-none-any.whl → 4.3.64__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/CertificateUpdater.py +2 -0
- ScriptCollection/ScriptCollectionCore.py +91 -17
- ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +8 -2
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +10 -5
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +1 -1
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +14 -5
- ScriptCollection/TFCPS/TFCPS_MergeToMain.py +2 -1
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +8 -4
- {scriptcollection-4.3.62.dist-info → scriptcollection-4.3.64.dist-info}/METADATA +1 -1
- {scriptcollection-4.3.62.dist-info → scriptcollection-4.3.64.dist-info}/RECORD +13 -13
- {scriptcollection-4.3.62.dist-info → scriptcollection-4.3.64.dist-info}/WHEEL +0 -0
- {scriptcollection-4.3.62.dist-info → scriptcollection-4.3.64.dist-info}/entry_points.txt +0 -0
- {scriptcollection-4.3.62.dist-info → scriptcollection-4.3.64.dist-info}/top_level.txt +0 -0
|
@@ -93,6 +93,8 @@ class CertificateUpdater:
|
|
|
93
93
|
@GeneralUtilities.check_arguments
|
|
94
94
|
def __update_certificates(self) -> None:
|
|
95
95
|
self.__sc.git_commit(self.__repository_folder, "Saved current changes")
|
|
96
|
+
certbot_image_address, certbot_image_tag = ScriptCollectionCore.split_image_address_and_tag(self.__certbot_image_address_with_tag)
|
|
97
|
+
self.__sc.docker_pull(certbot_image_address, certbot_image_tag)
|
|
96
98
|
error_occurred = False
|
|
97
99
|
for domain in self.__domains:
|
|
98
100
|
certbot_container_name = "certificate_updater"
|
|
@@ -39,7 +39,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
|
|
|
39
39
|
from .ProgramRunnerPopen import ProgramRunnerPopen
|
|
40
40
|
from .SCLog import SCLog, LogLevel
|
|
41
41
|
|
|
42
|
-
version = "4.3.
|
|
42
|
+
version = "4.3.64"
|
|
43
43
|
__version__ = version
|
|
44
44
|
|
|
45
45
|
class VSCodeWorkspaceShellTask:
|
|
@@ -308,7 +308,7 @@ class ScriptCollectionCore:
|
|
|
308
308
|
registry_username, registry_password = self.__load_credentials_if_required_and_available(remote_hub, registry_username, registry_password)
|
|
309
309
|
source_address = f"{remote_hub}/{imagename_on_remote_hub}:{tag}"
|
|
310
310
|
target_address = f"{own_registry_address}/{imagename_on_own_registry}:{tag}"
|
|
311
|
-
self.run_program("docker", f"buildx imagetools create --tag {target_address} {source_address}")#this does pull and push for each platform
|
|
311
|
+
self.run_program("docker", f"buildx imagetools create --tag {target_address} {source_address}", print_errors_as_information=True)#this does pull and push for each platform. docker writes the progress-output of the pull to std-err, so it must not be treated as error.
|
|
312
312
|
|
|
313
313
|
|
|
314
314
|
def get_tags_of_images_from_registry(self,registry_base_url:str,image:str,registry_username:str,registry_password:str)->list[str]:
|
|
@@ -348,7 +348,60 @@ class ScriptCollectionCore:
|
|
|
348
348
|
arg=f"login {registry} -u {username} -p {password}"
|
|
349
349
|
arg_for_log=f"login {registry} -u {username} -p ***"
|
|
350
350
|
self.run_program("docker",arg,arguments_for_log=arg_for_log,print_live_output=self.log.loglevel==LogLevel.Debug)
|
|
351
|
-
|
|
351
|
+
|
|
352
|
+
@staticmethod
|
|
353
|
+
@GeneralUtilities.check_arguments
|
|
354
|
+
def split_image_address_and_tag(image_with_tag: str) -> tuple[str, str]:
|
|
355
|
+
"""Splits an image-address which contains a tag into the address and the tag.
|
|
356
|
+
Example: "myregistry.example.com:5000/debian:12" is splitted into ("myregistry.example.com:5000/debian", "12").
|
|
357
|
+
If the given address does not contain a tag then "latest" is returned as tag, which is the tag docker assumes in this case too."""
|
|
358
|
+
last_colon_index = image_with_tag.rfind(":")
|
|
359
|
+
last_slash_index = image_with_tag.rfind("/")
|
|
360
|
+
if last_colon_index == -1 or last_colon_index < last_slash_index:
|
|
361
|
+
# There is no tag at all. (A colon which occurs before the last slash belongs to the port of the registry-address and is not a tag-separator.)
|
|
362
|
+
return (image_with_tag, "latest")
|
|
363
|
+
return (image_with_tag[:last_colon_index], image_with_tag[last_colon_index+1:])
|
|
364
|
+
|
|
365
|
+
@GeneralUtilities.check_arguments
|
|
366
|
+
def local_docker_image_exists(self, image: str, tag: str) -> bool:
|
|
367
|
+
"""Returns True if and only if the image is already available with this tag in the local image-store of the docker-daemon."""
|
|
368
|
+
program_result = self.run_program("docker", f"image inspect {image}:{tag}", throw_exception_if_exitcode_is_not_zero=False)
|
|
369
|
+
return program_result[0] == 0
|
|
370
|
+
|
|
371
|
+
@GeneralUtilities.check_arguments
|
|
372
|
+
def docker_pull(self, image: str, tag: str, force: bool = False) -> None:
|
|
373
|
+
"""Ensures that the image is available with this tag in the local image-store of the docker-daemon.
|
|
374
|
+
'image' must be the image-address without tag, for example "myregistry.example.com/debian".
|
|
375
|
+
Commands like "docker run" or "docker compose up" download a missing image implicitly, but docker writes the
|
|
376
|
+
progress-output of such a download to std-err, which would then be logged as error although nothing failed.
|
|
377
|
+
Therefore the download is done explicitly by this function, which treats the output of the pull as information,
|
|
378
|
+
so the surrounding commands can be executed with the usual error-handling.
|
|
379
|
+
By default the image is only pulled if it is not already available locally with this tag. This is exactly the
|
|
380
|
+
behavior of the implicit download (docker only downloads a missing image), so an unavailable registry does not
|
|
381
|
+
break a process which can also be executed with the already available image.
|
|
382
|
+
If 'force' is True then the image is pulled in any case, which is required to update a locally available image
|
|
383
|
+
whose tag was moved in the registry (for example "latest") and which therefore lets the pull fail if the
|
|
384
|
+
registry is not available."""
|
|
385
|
+
image_with_tag = f"{image}:{tag}"
|
|
386
|
+
if not force and self.local_docker_image_exists(image, tag):
|
|
387
|
+
self.log.log(f"Image \"{image_with_tag}\" is already available locally and therefore does not have to be pulled.", LogLevel.Debug)
|
|
388
|
+
return
|
|
389
|
+
self.log.log(f"Pull image \"{image_with_tag}\"...")
|
|
390
|
+
self.run_program_with_retry("docker", f"pull {image_with_tag}", print_errors_as_information=True, print_live_output=self.log.loglevel == LogLevel.Debug)
|
|
391
|
+
|
|
392
|
+
@GeneralUtilities.check_arguments
|
|
393
|
+
def docker_compose_pull(self, folder: str, compose_arguments: str, ignore_pull_failures: bool) -> None:
|
|
394
|
+
"""Pulls the images which are used by a docker-compose-file explicitly, so that the commands which use them
|
|
395
|
+
(for example "compose up") do not have to download them implicitly. See docker_pull for the reason.
|
|
396
|
+
'compose_arguments' must contain all arguments which have to be passed before the "pull"-verb,
|
|
397
|
+
for example "compose -f docker-compose.yml -p myproject --env-file Parameters.env".
|
|
398
|
+
'ignore_pull_failures' must be True if the compose-file also contains images which are built locally
|
|
399
|
+
(and therefore do not exist in a registry), because pulling such an image always fails."""
|
|
400
|
+
argument = f"{compose_arguments} pull --quiet"
|
|
401
|
+
if ignore_pull_failures:
|
|
402
|
+
argument = argument+" --ignore-pull-failures"
|
|
403
|
+
self.run_program_with_retry("docker", argument, folder, print_errors_as_information=True, print_live_output=self.log.loglevel == LogLevel.Debug)
|
|
404
|
+
|
|
352
405
|
|
|
353
406
|
@GeneralUtilities.check_arguments
|
|
354
407
|
def get_issues_of_github_repository(self, owner: str, repository: str, github_token: str = None) -> list[ProjectServerIssueSummary]:
|
|
@@ -458,7 +511,9 @@ class ScriptCollectionCore:
|
|
|
458
511
|
versiononlyregex = f"^{versionregex}$"
|
|
459
512
|
pattern = re.compile(versiononlyregex)
|
|
460
513
|
if pattern.match(new_version):
|
|
461
|
-
|
|
514
|
+
#the tag can be prefixed (for example "<ns0:version>") because the nuspec-namespace can be declared with a prefix instead of being declared as default-namespace.
|
|
515
|
+
version_tag_regex = f"(<(?:[\\w.-]+:)?version>){versionregex}(<\\/(?:[\\w.-]+:)?version>)"
|
|
516
|
+
GeneralUtilities.write_text_to_file(nuspec_file, re.sub(version_tag_regex, lambda match: match.group(1)+new_version+match.group(2), GeneralUtilities.read_text_from_file(nuspec_file)))
|
|
462
517
|
else:
|
|
463
518
|
raise ValueError(f"Version '{new_version}' does not match version-regex '{versiononlyregex}'")
|
|
464
519
|
|
|
@@ -1950,6 +2005,8 @@ class ScriptCollectionCore:
|
|
|
1950
2005
|
@GeneralUtilities.check_arguments
|
|
1951
2006
|
def get_latest_apt_package_version_in_debian(self, image: str,package:str) -> str:
|
|
1952
2007
|
#docker run --rm -it debian bash -c "apt update && apt list -a tor"
|
|
2008
|
+
image_address, image_tag = ScriptCollectionCore.split_image_address_and_tag(image)
|
|
2009
|
+
self.docker_pull(image_address, image_tag)
|
|
1953
2010
|
output=self.run_with_epew("docker", f"run --rm -it {image} bash -c \"apt --color=false update && apt --color=false list -a tor\"",os.getcwd(),encode_argument_in_base64=True)
|
|
1954
2011
|
stdout=output[1]
|
|
1955
2012
|
version_lines=[line.strip() for line in GeneralUtilities.string_to_lines(stdout) if GeneralUtilities.string_has_nonwhitespace_content(line) and line.startswith(package+"/")]
|
|
@@ -2835,7 +2892,8 @@ chmod {permission} {link_file}
|
|
|
2835
2892
|
lines = GeneralUtilities.read_lines_from_file(file)
|
|
2836
2893
|
lines_result = []
|
|
2837
2894
|
for line in lines:
|
|
2838
|
-
|
|
2895
|
+
#the tag can be prefixed (for example "<ns0:copyright>") because in xml-files like nuspec-files the namespace can be declared with a prefix instead of being declared as default-namespace.
|
|
2896
|
+
if match := re.search("(.*<(?:[\\w.-]+:)?[Cc]opyright>.*)\\d\\d\\d\\d(.*<\\/(?:[\\w.-]+:)?[Cc]opyright>.*)", line):
|
|
2839
2897
|
part1 = match.group(1)
|
|
2840
2898
|
part2 = match.group(2)
|
|
2841
2899
|
adapted = part1+current_year+part2
|
|
@@ -3080,21 +3138,33 @@ TXDX
|
|
|
3080
3138
|
trim_texts(child)
|
|
3081
3139
|
trim_texts(element)
|
|
3082
3140
|
ET.indent(element)
|
|
3083
|
-
|
|
3084
|
-
if element.tag.startswith("{"):
|
|
3085
|
-
#keep the namespace of the root-element as default-namespace. without this the whole document would be rewritten with a
|
|
3086
|
-
#generated prefix (for example "<ns0:package xmlns:ns0=...>" instead of "<package xmlns=...>"), which is semantically
|
|
3087
|
-
#equivalent but unusual and therefore an unnecessary risk for tools which process the formatted file.
|
|
3088
|
-
default_namespace = element.tag[1:element.tag.index("}")]
|
|
3089
|
-
try:
|
|
3090
|
-
content = ET.tostring(element, xml_declaration=add_xml_declaration, encoding="unicode", default_namespace=default_namespace)
|
|
3091
|
-
except ValueError:
|
|
3092
|
-
#documents which contain elements without a namespace besides the namespace of the root-element can not be serialized
|
|
3093
|
-
#with a default-namespace at all. in that case the generated prefixes are the only possible representation.
|
|
3094
|
-
content = ET.tostring(element, xml_declaration=add_xml_declaration, encoding="unicode")
|
|
3141
|
+
content = self.__serialize_xml_element(element, add_xml_declaration)
|
|
3095
3142
|
GeneralUtilities.write_text_to_file(file, content.rstrip("\n") + "\n", encoding)
|
|
3096
3143
|
self.normalize_line_endings(file)
|
|
3097
3144
|
|
|
3145
|
+
@GeneralUtilities.check_arguments
|
|
3146
|
+
def __serialize_xml_element(self, element: ET.Element, add_xml_declaration: bool) -> str:
|
|
3147
|
+
if not element.tag.startswith("{"):
|
|
3148
|
+
return ET.tostring(element, xml_declaration=add_xml_declaration, encoding="unicode")
|
|
3149
|
+
#keep the namespace of the root-element as default-namespace. without this the whole document would be rewritten with a
|
|
3150
|
+
#generated prefix (for example "<ns0:package xmlns:ns0=...>" instead of "<package xmlns=...>"), which is semantically
|
|
3151
|
+
#equivalent but unusual and therefore an unnecessary risk for tools which process the formatted file.
|
|
3152
|
+
#the "default_namespace"-argument of "tostring" can not be used for this because it rejects documents which contain
|
|
3153
|
+
#attributes without a namespace (which is the usual case, for example in nuspec-files), so the namespace is registered
|
|
3154
|
+
#as default-namespace instead. that registration is global, therefore the previous state gets restored afterwards.
|
|
3155
|
+
namespace = element.tag[1:element.tag.index("}")]
|
|
3156
|
+
namespace_map: dict = getattr(ET, "_namespace_map", None)
|
|
3157
|
+
previous_prefix = namespace_map.get(namespace) if namespace_map is not None else None
|
|
3158
|
+
ET.register_namespace(GeneralUtilities.empty_string, namespace)
|
|
3159
|
+
try:
|
|
3160
|
+
return ET.tostring(element, xml_declaration=add_xml_declaration, encoding="unicode")
|
|
3161
|
+
finally:
|
|
3162
|
+
if namespace_map is not None:
|
|
3163
|
+
if previous_prefix is None:
|
|
3164
|
+
namespace_map.pop(namespace, None)
|
|
3165
|
+
else:
|
|
3166
|
+
namespace_map[namespace] = previous_prefix
|
|
3167
|
+
|
|
3098
3168
|
@GeneralUtilities.check_arguments
|
|
3099
3169
|
def format_html_file(self, file: str, add_html_declaration: bool = False) -> None:
|
|
3100
3170
|
encoding = "utf-8"
|
|
@@ -3474,7 +3544,11 @@ OCR-content:
|
|
|
3474
3544
|
argument = argument+f" -p {title.lower()}"
|
|
3475
3545
|
if os.path.isfile(os.path.join(example_folder,"Parameters.env")):
|
|
3476
3546
|
argument=argument+" --env-file Parameters.env"
|
|
3547
|
+
compose_arguments_without_verb=argument
|
|
3477
3548
|
argument=argument+" up --detach"
|
|
3549
|
+
# The images used by the compose-file are pulled explicitly here so that the following commands do not have to download them implicitly (see docker_compose_pull).
|
|
3550
|
+
# Pull-failures are ignored because a compose-file of an example typically also contains the image of the codeunit itself, which is built locally and does not exist in a registry.
|
|
3551
|
+
self.docker_compose_pull(example_folder, compose_arguments_without_verb, True)
|
|
3478
3552
|
if in_build_container:
|
|
3479
3553
|
# In a DooD-setup the test-service-containers are siblings on the docker-daemon and resolve their bind-mount-sources against the DAEMON's
|
|
3480
3554
|
# filesystem, not against this build-container's filesystem. Clearing the source-folders from within this build-container (as the *Start.py-
|
|
@@ -64,7 +64,10 @@ class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
64
64
|
# rather than into this job's BOM folder. stdout flows back through the docker client,
|
|
65
65
|
# so it works regardless of where the daemon runs. format_xml_file re-parses and
|
|
66
66
|
# rewrites the document, so the resulting file is byte-stable.
|
|
67
|
-
|
|
67
|
+
syft_image = self.tfcps_Tools_General.oci_image_manager.get_registry_address_for_image(self.get_repository_folder(), "Syft")
|
|
68
|
+
syft_image_tag = self.tfcps_Tools_General.oci_image_manager.get_tag_for_image(self.get_repository_folder(), "Syft")
|
|
69
|
+
self._protected_sc.docker_pull(syft_image, syft_image_tag)
|
|
70
|
+
syft_result = self._protected_sc.run_program_argsasarray("docker", ["run","--rm","-v","/var/run/docker.sock:/var/run/docker.sock",f"{syft_image}:{syft_image_tag}",f"{codeunitname_lower}:{codeunitversion}","-o","cyclonedx-xml"], artifacts_folder)
|
|
68
71
|
GeneralUtilities.write_text_to_file(sbom_file, syft_result[1])
|
|
69
72
|
self._protected_sc.format_xml_file(sbom_file)
|
|
70
73
|
|
|
@@ -230,7 +233,10 @@ class TFCPS_CodeUnitSpecific_Docker_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
230
233
|
# so it does not depend on the image under test providing curl. It is resolved via
|
|
231
234
|
# the image-manager (like Syft), so it is pulled from the configured registry and
|
|
232
235
|
# does not hit registry-rate-limits.
|
|
233
|
-
|
|
236
|
+
curl_image_address=self.tfcps_Tools_General.oci_image_manager.get_registry_address_for_image(self.get_repository_folder(),"Curl")
|
|
237
|
+
curl_image_tag=self.tfcps_Tools_General.oci_image_manager.get_tag_for_image(self.get_repository_folder(),"Curl")
|
|
238
|
+
self._protected_sc.docker_pull(curl_image_address,curl_image_tag)
|
|
239
|
+
curl_image=f"{curl_image_address}:{curl_image_tag}"
|
|
234
240
|
try:
|
|
235
241
|
last_exception:Exception=None
|
|
236
242
|
self._protected_sc.run_program("docker",argument)
|
|
@@ -257,13 +257,18 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
257
257
|
repository_folder: str =self.get_repository_folder()
|
|
258
258
|
build_folder = os.path.join(repository_folder, codeunitname, "Other", "Build")
|
|
259
259
|
outputfolder = GeneralUtilities.resolve_relative_path("./Other/Artifacts/BuildResult_NuGet",self.get_codeunit_folder())
|
|
260
|
-
|
|
261
|
-
|
|
260
|
+
nuspec_file = os.path.join(build_folder, f"{codeunitname}.nuspec")
|
|
261
|
+
root: etree._ElementTree = etree.parse(nuspec_file)
|
|
262
|
+
# the elements are matched by their local name because the nuspec-namespace can be declared as default-namespace
|
|
263
|
+
# (<package xmlns="...">) as well as with a prefix (<ns0:package xmlns:ns0="...">) and both variants are valid.
|
|
264
|
+
versions = root.xpath("//*[local-name() = 'package']/*[local-name() = 'metadata']/*[local-name() = 'version']/text()")
|
|
265
|
+
if len(versions) == 0:
|
|
266
|
+
raise ValueError(f"Can not determine the version because there is no version-element in \"{nuspec_file}\".")
|
|
267
|
+
current_version = versions[0]
|
|
262
268
|
nupkg_filename = f"{codeunitname}.{current_version}.nupkg"
|
|
263
269
|
nupkg_file = f"{build_folder}/{nupkg_filename}"
|
|
264
270
|
GeneralUtilities.ensure_file_does_not_exist(nupkg_file)
|
|
265
271
|
commit_id = self._protected_sc.git_get_commit_id(repository_folder)
|
|
266
|
-
nuspec_file = os.path.join(build_folder, f"{codeunitname}.nuspec")
|
|
267
272
|
# Pack the hand-written nuspec using "dotnet pack" (the classic "nuget pack" is not available in the dotnet-only build-container).
|
|
268
273
|
# With NuspecFile the package-content is taken entirely from the nuspec's <files>-list, so no rebuild happens here (the library was already built before).
|
|
269
274
|
self._protected_sc.run_program_argsasarray("dotnet", [
|
|
@@ -587,7 +592,7 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
587
592
|
self._protected_sc.update_year_in_copyright_tags(nuspec_file)
|
|
588
593
|
|
|
589
594
|
@GeneralUtilities.check_arguments
|
|
590
|
-
def run_testcases(self) -> None:
|
|
595
|
+
def run_testcases(self, timeoutInSeconds:int=60*10) -> None:
|
|
591
596
|
self._protected_sc.log.log("Run testcases...")
|
|
592
597
|
dotnet_build_configuration: str = self.get_target_environment_type()
|
|
593
598
|
codeunit_name: str = self.get_codeunit_name()
|
|
@@ -620,7 +625,7 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
620
625
|
test_working_directory = os.path.join(temp_folder, "WorkingDirectory")
|
|
621
626
|
GeneralUtilities.ensure_directory_exists(test_working_directory)
|
|
622
627
|
try:
|
|
623
|
-
program_output=self._protected_sc.run_program_argsasarray("dotnet", args, test_working_directory, print_live_output=self.get_verbosity()==LogLevel.Debug, timeoutInSeconds=
|
|
628
|
+
program_output=self._protected_sc.run_program_argsasarray("dotnet", args, test_working_directory, print_live_output=self.get_verbosity()==LogLevel.Debug, timeoutInSeconds=timeoutInSeconds, env_vars={"MSBUILDDISABLENODEREUSE": "1"})
|
|
624
629
|
test_output:str=program_output[1]
|
|
625
630
|
output_lines=program_output[1].split("\n")
|
|
626
631
|
output_lines=[line for line in output_lines if GeneralUtilities.string_has_content(line)]
|
|
@@ -10,7 +10,7 @@ class TFCPS_CodeUnit_BuildCodeUnit:
|
|
|
10
10
|
|
|
11
11
|
codeunit_folder: str = None
|
|
12
12
|
repository_folder: str = None
|
|
13
|
-
sc: ScriptCollectionCore =
|
|
13
|
+
sc: ScriptCollectionCore = None
|
|
14
14
|
codeunit_name: str = None
|
|
15
15
|
tFCPS_Tools: TFCPS_Tools_General = None
|
|
16
16
|
target_environment_type: str = None
|
|
@@ -158,17 +158,19 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
158
158
|
self.sc.log.log(GeneralUtilities.get_line())
|
|
159
159
|
tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()
|
|
160
160
|
hook.run_after_codeunit_was_built(tFCPS_CodeUnit_BuildCodeUnit)
|
|
161
|
-
|
|
162
161
|
self.sc.log.log(GeneralUtilities.get_line())
|
|
163
162
|
|
|
164
163
|
self.search_for_secrets()
|
|
165
|
-
self.__normalize_line_endings_of_common_files()
|
|
166
164
|
self.tfcps_tools_general.generate_svg_files_from_plantuml_files_for_repository(self.repository, self.use_cache())
|
|
165
|
+
self.__normalize_line_endings_of_common_files()
|
|
166
|
+
|
|
167
|
+
self.sc.log.log(GeneralUtilities.get_line())
|
|
167
168
|
|
|
168
169
|
if self.is_pre_merge():
|
|
169
170
|
self.__translate()
|
|
170
171
|
self.__collect_metrics()
|
|
171
172
|
self.__generate_loc_diagram()
|
|
173
|
+
self.sc.log.log(GeneralUtilities.get_line())
|
|
172
174
|
|
|
173
175
|
except Exception:
|
|
174
176
|
error_occurred=True
|
|
@@ -231,14 +233,17 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
231
233
|
|
|
232
234
|
@GeneralUtilities.check_arguments
|
|
233
235
|
def __normalize_line_endings_of_common_files(self) -> None:
|
|
234
|
-
#TODO add option to define exceptions (means: files which should not be normalized).
|
|
235
236
|
self.sc.format_xml_file(os.path.join(self.repository, ".ScriptCollection", "ProductInformation.xml"))
|
|
237
|
+
|
|
236
238
|
for codeunit in self.tfcps_tools_general.get_codeunits(self.repository):
|
|
237
239
|
self.sc.format_xml_file(os.path.join(self.repository, codeunit, f"{codeunit}.codeunit.xml"))
|
|
240
|
+
|
|
238
241
|
workspace_file = os.path.join(self.repository, f"{self.tfcps_tools_general.get_product_name(self.repository)}.code-workspace")
|
|
239
242
|
if os.path.isfile(workspace_file):
|
|
240
243
|
self.sc.format_json_file(workspace_file)
|
|
241
|
-
|
|
244
|
+
|
|
245
|
+
#TODO add option to define exceptions (means: files which should not be normalized).
|
|
246
|
+
self.sc.normalize_line_endings_of_files_in_folder(self.repository, ["txt", "md", "py", "json", "xml", "svg", "csv", "yml", "yaml", "toml","gitignore", "gitattributes", "code-workspace"])
|
|
242
247
|
|
|
243
248
|
@GeneralUtilities.check_arguments
|
|
244
249
|
def is_working_branch(self)->bool:
|
|
@@ -360,6 +365,8 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
360
365
|
image,
|
|
361
366
|
] + scbuildcodeunits_arguments
|
|
362
367
|
self.sc.log.log(f"Build codeunits in container using image \"{image}\"...")
|
|
368
|
+
image_address, image_tag = ScriptCollectionCore.split_image_address_and_tag(image)
|
|
369
|
+
self.sc.docker_pull(image_address, image_tag)
|
|
363
370
|
# the exitcode is evaluated by the caller (returned as part of the result-tuple), so the program-runner must not raise on a non-zero exitcode here.
|
|
364
371
|
result=self.sc.run_program_argsasarray("docker", docker_arguments, throw_exception_if_exitcode_is_not_zero=False, print_live_output=True, env_vars=required_environment_variables)
|
|
365
372
|
exit_code:int=result[0]
|
|
@@ -582,7 +589,9 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
582
589
|
if daemon_check[0] != 0:
|
|
583
590
|
raise ValueError(f"The secret-scan can not be performed because the docker-daemon is not reachable (exit-code {daemon_check[0]}: {daemon_check[2].strip()}). The scan runs betterleaks via 'docker run', which requires a running docker-daemon (inside a build-container its socket must be forwarded to the host-daemon). This is an infrastructure-problem, not a secret-finding in the repository.")
|
|
584
591
|
args = ["run", "--rm"] + mount_arguments + [image] + scan_args
|
|
585
|
-
|
|
592
|
+
image_address, image_tag = ScriptCollectionCore.split_image_address_and_tag(image)
|
|
593
|
+
self.sc.docker_pull(image_address, image_tag)
|
|
594
|
+
result = self.sc.run_program_argsasarray("docker", args, throw_exception_if_exitcode_is_not_zero=False, print_live_output=self.sc.log.loglevel==LogLevel.Debug)
|
|
586
595
|
if result[0] != 0:
|
|
587
596
|
for line in GeneralUtilities.string_to_lines(result[1]):
|
|
588
597
|
self.sc.log.log(line, LogLevel.Information)
|
|
@@ -18,9 +18,10 @@ class MergeToMainConfiguration:
|
|
|
18
18
|
tFCPS_Generic_Functions:TFCPS_Generic_Functions = None
|
|
19
19
|
common_remote_name:str = None
|
|
20
20
|
build_repo:str = None
|
|
21
|
-
sc:ScriptCollectionCore=
|
|
21
|
+
sc:ScriptCollectionCore=None
|
|
22
22
|
|
|
23
23
|
def __init__(self, current_file: str,repository:str, product_name: str,merge_source_branch:str,log_level:LogLevel,additional_arguments_file:str,main_branch:str,common_remote_name:str,build_repo:str):
|
|
24
|
+
self.sc = ScriptCollectionCore()
|
|
24
25
|
self.sc.log.loglevel=log_level
|
|
25
26
|
self.repository_folder = repository
|
|
26
27
|
self.product_name = product_name
|
|
@@ -1839,7 +1839,7 @@ class TFCPS_Tools_General:
|
|
|
1839
1839
|
self.__sc.run_program_argsasarray("docker",[ "push", arch_tag])
|
|
1840
1840
|
# Create multi-arch manifest
|
|
1841
1841
|
final_tag = f"{image_address}:{tag}"
|
|
1842
|
-
self.__sc.run_program_argsasarray("docker", [ "buildx", "imagetools", "create", "--tag", final_tag] + arch_tags)
|
|
1842
|
+
self.__sc.run_program_argsasarray("docker", [ "buildx", "imagetools", "create", "--tag", final_tag] + arch_tags, print_errors_as_information=True)#imagetools create pulls and pushes the manifests; docker writes the progress-output of the pull to std-err, so it must not be treated as error.
|
|
1843
1843
|
|
|
1844
1844
|
@GeneralUtilities.check_arguments
|
|
1845
1845
|
def platform_from_filename(self,filename: str) -> Platform:
|
|
@@ -1917,7 +1917,11 @@ class TFCPS_Tools_General:
|
|
|
1917
1917
|
argument = f"compose --project-name {docker_project_name}"
|
|
1918
1918
|
if env_file is not None:
|
|
1919
1919
|
argument = f"{argument} --env-file {env_file}"
|
|
1920
|
+
compose_arguments_without_verb = argument
|
|
1920
1921
|
argument = f"{argument} up --detach"
|
|
1922
|
+
# The images used by the compose-file are pulled explicitly here so that "compose up" does not have to download them implicitly (see docker_compose_pull).
|
|
1923
|
+
# Pull-failures are ignored because the compose-file also contains the image of the codeunit itself, which was loaded from the build-artifact above and does not exist in a registry.
|
|
1924
|
+
self.__sc.docker_compose_pull(folder_of_current_file, compose_arguments_without_verb, True)
|
|
1921
1925
|
self.__sc.run_program("docker", argument, folder_of_current_file)
|
|
1922
1926
|
|
|
1923
1927
|
@GeneralUtilities.check_arguments
|
|
@@ -2053,9 +2057,9 @@ class TFCPS_Tools_General:
|
|
|
2053
2057
|
for k,v in env_variables.items():
|
|
2054
2058
|
lines=lines+[f"{k}={v}"]
|
|
2055
2059
|
GeneralUtilities.write_lines_to_file(env_variables_file,lines)
|
|
2056
|
-
arguments=arguments + " --env-file Parameters.env
|
|
2057
|
-
|
|
2058
|
-
self.__sc.
|
|
2060
|
+
arguments=arguments + " --env-file Parameters.env"
|
|
2061
|
+
# Pull-failures are not ignored here because a test-service only uses images which are available in a registry.
|
|
2062
|
+
self.__sc.docker_compose_pull(test_service_folder,arguments,False)
|
|
2059
2063
|
|
|
2060
2064
|
@GeneralUtilities.check_arguments
|
|
2061
2065
|
def load_deb_control_file_content(self, file: str, codeunitname: str, codeunitversion: str, installedsize: int, maintainername: str, maintaineremail: str, description: str) -> str:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scriptcollection
|
|
3
|
-
Version: 4.3.
|
|
3
|
+
Version: 4.3.64
|
|
4
4
|
Summary: The ScriptCollection is the place for reusable scripts.
|
|
5
5
|
Author-email: Marius Göcke <marius.goecke@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/anionDev/ScriptCollection
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
ScriptCollection/AnionBuildPlatform.py,sha256=GP-jnb1GbvT8jzlH-cqnxgOzBzeUdiaWnDXtKNjsxiA,12153
|
|
2
|
-
ScriptCollection/CertificateUpdater.py,sha256=
|
|
2
|
+
ScriptCollection/CertificateUpdater.py,sha256=K-yo0WCG246bGsP-9gcNOjSKTGIh4vKYZ2JN_F2B_w0,9906
|
|
3
3
|
ScriptCollection/Executables.py,sha256=qnL7sUN_yHnQ3jlS2_OjC4Blp9mcxC5q0VXFUSfqZLU,70211
|
|
4
4
|
ScriptCollection/GeneralUtilities.py,sha256=SmN5PV0vI5TZZCjVulIrYRzdogxNAeTqQ27ZKb6gul0,72184
|
|
5
5
|
ScriptCollection/HTTPMaintenanceOverheadHelper.py,sha256=TToNtyO1XzsMbBsTBf3o0xgOK0v4Jf03qw2Z0xb2nCk,2007
|
|
@@ -9,7 +9,7 @@ ScriptCollection/ProgramRunnerMock.py,sha256=uTu-aFle1W_oKjeQEmuPsFPQpvo0kRf2FrR
|
|
|
9
9
|
ScriptCollection/ProgramRunnerPopen.py,sha256=zVD-gZY85o8wiGFcpYAGNDl-tXaKzipl1Bkhqy2ois4,4681
|
|
10
10
|
ScriptCollection/ProgramRunnerSudo.py,sha256=RXsiceQ8W6a8YDk_2BPscVbIAmDIpIxqe9WHBy2r4CM,5233
|
|
11
11
|
ScriptCollection/SCLog.py,sha256=LdzSJy3NnnwYCCefz2h4tiTy4BjT8lOz3IBG_bMOg9E,4821
|
|
12
|
-
ScriptCollection/ScriptCollectionCore.py,sha256=
|
|
12
|
+
ScriptCollection/ScriptCollectionCore.py,sha256=gXpicqm4lPjl8WCFQKs0znYWumyOFvxnswgS7rNBMzo,257238
|
|
13
13
|
ScriptCollection/UnassignedVariablesCheck.py,sha256=v59qLQ5RRvx31ouuCW7NWZj3McWENvGBPkV5ZGBmqJE,4344
|
|
14
14
|
ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
15
15
|
ScriptCollection/OCIImages/AbstractImageHandler.py,sha256=WO2M28c2tUwsXFNa2L9zqMhdlyObWm3qpPE0LdIs7SE,2315
|
|
@@ -37,23 +37,23 @@ ScriptCollection/TFCPS/PackageSource.py,sha256=tQoMbxrIbV3mYHKH2wgOoGzIHY1TvBH8E
|
|
|
37
37
|
ScriptCollection/TFCPS/SCTaskRunnerServer.py,sha256=rpuwSsQ6YQfZS2LGnvOH5ql8zs_-cHU4LQttS7XNT60,11272
|
|
38
38
|
ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py,sha256=67_DQwoH5bhxxSmwtuAeoW1NOCMiLMzOy48OqlJX-KY,1744
|
|
39
39
|
ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=N-Wvxzwv_R8EVybjQjE7wXX-VpFLa_62LinWa57C_Wo,35345
|
|
40
|
-
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=
|
|
41
|
-
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=
|
|
40
|
+
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=bgtLLUcB3gE9uXlyEuf4KNE-MqX4n1oSk0C1AOiCLhk,9886
|
|
41
|
+
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=PHbsWtlGuq7NwOR-oZmtK8zTZuOTmA_erZIU9JhgNP8,46541
|
|
42
42
|
ScriptCollection/TFCPS/TFCPS_CreateRelease.py,sha256=X2NgZgbOkJsdZ6wMHB_RuS_rQCuz5oQCqzYrxyXBN9Q,6680
|
|
43
43
|
ScriptCollection/TFCPS/TFCPS_Generic.py,sha256=5vCFfDiU8bvfcnoKNs2ATS8nsl6mYXcIuX7XfYE0tvg,2116
|
|
44
|
-
ScriptCollection/TFCPS/TFCPS_MergeToMain.py,sha256=
|
|
44
|
+
ScriptCollection/TFCPS/TFCPS_MergeToMain.py,sha256=LZwlK5QcSXWUhRaBcFLNZeEC6rysJoZNzr5T6-ISE2o,7805
|
|
45
45
|
ScriptCollection/TFCPS/TFCPS_MergeToStable.py,sha256=jQ0B3yq45we-8a0KuNFMfXCk9MfzPpTv2QgDWMN34x4,22792
|
|
46
46
|
ScriptCollection/TFCPS/TFCPS_OCIImageSecretScan.py,sha256=rhw0glr4gZ5enG9faOD51xE4MMK1cIdNjUdaswiMsrY,14622
|
|
47
47
|
ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py,sha256=JYiDqDsfVJQWS9xWMi_UzfXnXsJRnPRxAzFr82bUEvI,2370
|
|
48
48
|
ScriptCollection/TFCPS/TFCPS_RemoteBuild.py,sha256=Z8Gw-j5I7VPafHEsVeQJ2gTTKVLAnnhwZhY8IVTi2Qk,11867
|
|
49
|
-
ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=
|
|
49
|
+
ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=hhzY9rl2PpcywN04m8VuQwBRai75h0xLQbIQnrnUqiI,151007
|
|
50
50
|
ScriptCollection/TFCPS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
51
|
-
ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=
|
|
51
|
+
ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=919mNfzVRurTJhWi8WV0oEAFoH-yd4a6m2Rbxcf2oZ4,19927
|
|
52
52
|
ScriptCollection/TFCPS/Docker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
53
53
|
ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py,sha256=bT6Gd5pQpZCw4OQz6HWkPCSn5z__eUUEisABLDSxd0o,200
|
|
54
54
|
ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py,sha256=QyjOfMY22JWCvKjMelHiDWbJiWqotOfebpJpgDUaoO4,237
|
|
55
55
|
ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py,sha256=i0zEGehj0sttxjjZtoq2KFSKp_ulxVyWp_ZgAhIY_So,241
|
|
56
|
-
ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=
|
|
56
|
+
ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=iLK6hYd_NFzS5FxML8NXcfrBicHUCyjMGO6uTLOMt8E,49144
|
|
57
57
|
ScriptCollection/TFCPS/DotNet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
58
58
|
ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=CwDBogG7Lg3IyywpCGta9gcaqWpgjENNXaupS19ZwTM,8751
|
|
59
59
|
ScriptCollection/TFCPS/Flutter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -67,8 +67,8 @@ ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py,sha256=Bd1gxCk9Fb
|
|
|
67
67
|
ScriptCollection/TFCPS/Python/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
68
68
|
ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=WhOsMpJxNGqnQGA3nJ99ruyqymxBeJxCcpK98zJduTo,1680
|
|
69
69
|
ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
70
|
-
scriptcollection-4.3.
|
|
71
|
-
scriptcollection-4.3.
|
|
72
|
-
scriptcollection-4.3.
|
|
73
|
-
scriptcollection-4.3.
|
|
74
|
-
scriptcollection-4.3.
|
|
70
|
+
scriptcollection-4.3.64.dist-info/METADATA,sha256=VrBE86FyQRoV6mtB4_YVzr815joB2EWDzYZJh0zhuvU,7648
|
|
71
|
+
scriptcollection-4.3.64.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
|
72
|
+
scriptcollection-4.3.64.dist-info/entry_points.txt,sha256=52v0Kt6uaeQeDFjvc0Xcsd4YqQw2qKcs6GNgPO71pkU,5996
|
|
73
|
+
scriptcollection-4.3.64.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
|
|
74
|
+
scriptcollection-4.3.64.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|