ScriptCollection 4.4.21__py3-none-any.whl → 4.4.23__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/Executables.py +1 -1
- ScriptCollection/ScriptCollectionCore.py +52 -5
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +2 -3
- ScriptCollection/TFCPS/PlantUMLRenderer.py +162 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +4 -32
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +52 -147
- {scriptcollection-4.4.21.dist-info → scriptcollection-4.4.23.dist-info}/METADATA +169 -169
- {scriptcollection-4.4.21.dist-info → scriptcollection-4.4.23.dist-info}/RECORD +11 -16
- ScriptCollection/Resources/Dependencies/PlantUML/Version.txt +0 -1
- ScriptCollection/Resources/Fonts/DejaVuSans-Bold.ttf +0 -0
- ScriptCollection/Resources/Fonts/DejaVuSans-BoldOblique.ttf +0 -0
- ScriptCollection/Resources/Fonts/DejaVuSans-Oblique.ttf +0 -0
- ScriptCollection/Resources/Fonts/DejaVuSans.ttf +0 -0
- ScriptCollection/Resources/Fonts/LICENSE.txt +0 -187
- {scriptcollection-4.4.21.dist-info → scriptcollection-4.4.23.dist-info}/WHEEL +0 -0
- {scriptcollection-4.4.21.dist-info → scriptcollection-4.4.23.dist-info}/entry_points.txt +0 -0
- {scriptcollection-4.4.21.dist-info → scriptcollection-4.4.23.dist-info}/top_level.txt +0 -0
ScriptCollection/Executables.py
CHANGED
|
@@ -1147,7 +1147,7 @@ def CleanToolsCache()->int:
|
|
|
1147
1147
|
|
|
1148
1148
|
|
|
1149
1149
|
def DownloadCachableTools()->int:
|
|
1150
|
-
parser = argparse.ArgumentParser(description="Downloads all tools that are stored in the global ScriptCollection-cache (for example CycloneDX-CLI,
|
|
1150
|
+
parser = argparse.ArgumentParser(description="Downloads all tools that are stored in the global ScriptCollection-cache (for example CycloneDX-CLI, MediaMTX, TruffleHog, OpenAPIGenerator and AndroidAppBundleTool). Running this - for example in a build-image - lets repeated pipeline-runs avoid rate-limits and run faster because the tools are already present.")
|
|
1151
1151
|
parser.add_argument('-v', '--verbose', action='store_true', required=False, default=False, help="Enables verbose (debug) output.")
|
|
1152
1152
|
parser.add_argument('-e', '--enforceupdate', action='store_true', required=False, default=False, help="Re-download the tools even if they are already present in the cache.")
|
|
1153
1153
|
#TODO for each tool which will be installed allow to specify a specific version number
|
|
@@ -22,6 +22,7 @@ from pathlib import Path
|
|
|
22
22
|
from subprocess import Popen
|
|
23
23
|
import re
|
|
24
24
|
import shutil
|
|
25
|
+
import socket
|
|
25
26
|
from typing import IO
|
|
26
27
|
import fnmatch
|
|
27
28
|
import uuid
|
|
@@ -39,7 +40,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
|
|
|
39
40
|
from .ProgramRunnerPopen import ProgramRunnerPopen
|
|
40
41
|
from .SCLog import SCLog, LogLevel
|
|
41
42
|
|
|
42
|
-
version = "4.4.
|
|
43
|
+
version = "4.4.23"
|
|
43
44
|
__version__ = version
|
|
44
45
|
|
|
45
46
|
class VSCodeWorkspaceShellTask:
|
|
@@ -2442,15 +2443,23 @@ resolving a name requires fontconfig, which does not exist on every system ffmpe
|
|
|
2442
2443
|
relative entries of 'excluded_folders' are resolved against 'base_folder'."""
|
|
2443
2444
|
resolved_base_folder = GeneralUtilities.resolve_relative_path(base_folder, os.getcwd())
|
|
2444
2445
|
resolved_path = GeneralUtilities.resolve_relative_path(path, resolved_base_folder)
|
|
2445
|
-
|
|
2446
|
-
|
|
2446
|
+
# GeneralUtilities.resolve_relative_path only resolves symlinks (via Path.resolve()) for relative inputs; an
|
|
2447
|
+
# already-absolute input is returned unchanged. On macOS the system temp-folder lies under "/var", which
|
|
2448
|
+
# itself is a symlink to "/private/var" (Linux/Windows temp-folders normally have no such symlink-indirection).
|
|
2449
|
+
# Without canonicalizing all three values the same way here, an absolute base_folder/path (unresolved, still
|
|
2450
|
+
# containing "/var/...") could fail to share a prefix with a relative excluded_folder entry like ".git" (which
|
|
2451
|
+
# got symlink-resolved to "/private/var/..." further above), so the exclusion-check would silently never match.
|
|
2452
|
+
# Resolving symlinks consistently for all three values fixes this and is also the safer choice from a security
|
|
2453
|
+
# standpoint, since it prevents a symlink from being used to bypass the excluded-folders check.
|
|
2454
|
+
normalized_base_folder = os.path.normcase(os.path.normpath(os.path.realpath(resolved_base_folder)))
|
|
2455
|
+
normalized_path = os.path.normcase(os.path.normpath(os.path.realpath(resolved_path)))
|
|
2447
2456
|
# 1. path must be equal to or located inside base_folder
|
|
2448
2457
|
if not (normalized_path == normalized_base_folder or normalized_path.startswith(normalized_base_folder + os.sep)):
|
|
2449
2458
|
return False
|
|
2450
2459
|
# 2. path must not be equal to or located inside any excluded folder
|
|
2451
2460
|
for excluded_folder in excluded_folders:
|
|
2452
2461
|
resolved_excluded_folder = GeneralUtilities.resolve_relative_path(excluded_folder, resolved_base_folder)
|
|
2453
|
-
normalized_excluded_folder = os.path.normcase(os.path.normpath(resolved_excluded_folder))
|
|
2462
|
+
normalized_excluded_folder = os.path.normcase(os.path.normpath(os.path.realpath(resolved_excluded_folder)))
|
|
2454
2463
|
if normalized_path == normalized_excluded_folder or normalized_path.startswith(normalized_excluded_folder + os.sep):
|
|
2455
2464
|
return False
|
|
2456
2465
|
# 3. path is inside base_folder and not inside any excluded folder
|
|
@@ -4567,6 +4576,35 @@ OCR-content:
|
|
|
4567
4576
|
def is_running_in_build_container(self) ->bool:
|
|
4568
4577
|
"""this function is based on a convention and does not do a real check."""
|
|
4569
4578
|
return os.environ.get("ISRUNNINGINBUILDCONTAINER") == "true"
|
|
4579
|
+
|
|
4580
|
+
@GeneralUtilities.check_arguments
|
|
4581
|
+
def get_own_container_id(self) -> str:
|
|
4582
|
+
"""Returns the id of the container this process runs in, so its volumes can be shared with sibling-containers
|
|
4583
|
+
via "docker run --volumes-from". That is required whenever this process runs in a container whose docker-socket
|
|
4584
|
+
is forwarded to the daemon of the host: a bind-mount of a path of this process would be resolved by that
|
|
4585
|
+
daemon, where the path does not exist."""
|
|
4586
|
+
# In mountinfo the own container-id only appears reliably in the source-path of the
|
|
4587
|
+
# "/etc/hostname"/"/etc/hosts"/"/etc/resolv.conf"-mounts (".../containers/<id>/..."). A plain
|
|
4588
|
+
# 64-hex-match there would also hit overlay-layer-hashes (which are not containers), so the
|
|
4589
|
+
# "containers/"-prefix must be matched explicitly.
|
|
4590
|
+
try:
|
|
4591
|
+
with open("/proc/self/mountinfo", "r", encoding="utf-8") as file_handle:
|
|
4592
|
+
match = re.search(r"/containers/([0-9a-f]{64})/", file_handle.read())
|
|
4593
|
+
if match is not None:
|
|
4594
|
+
return match.group(1)
|
|
4595
|
+
except Exception:
|
|
4596
|
+
pass
|
|
4597
|
+
# cgroup (v1): the container-id is part of the cgroup-path; here a plain 64-hex-match is safe.
|
|
4598
|
+
try:
|
|
4599
|
+
with open("/proc/self/cgroup", "r", encoding="utf-8") as file_handle:
|
|
4600
|
+
match = re.search(r"[0-9a-f]{64}", file_handle.read())
|
|
4601
|
+
if match is not None:
|
|
4602
|
+
return match.group(0)
|
|
4603
|
+
except Exception:
|
|
4604
|
+
pass
|
|
4605
|
+
# Fallback: the hostname equals the short container-id for containers started without an
|
|
4606
|
+
# explicit hostname.
|
|
4607
|
+
return socket.gethostname()
|
|
4570
4608
|
|
|
4571
4609
|
def prepare_build_pipeline_for_gitlab(self):
|
|
4572
4610
|
repository_folder:str=os.getcwd()
|
|
@@ -4577,9 +4615,18 @@ OCR-content:
|
|
|
4577
4615
|
self.assert_scbuilder_image_in_github_workflow_matches_image_definition(repository_folder)
|
|
4578
4616
|
self.__prepare_build_pipeline(repository_folder)
|
|
4579
4617
|
|
|
4618
|
+
def __ensure_branch_is_checked_out(self,repository_folder:str) -> None:
|
|
4619
|
+
current_branch_result = self.run_program_argsasarray("git", ["symbolic-ref", "--short", "HEAD"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
|
|
4620
|
+
repository_is_on_any_branch:bool = current_branch_result[0] == 0
|
|
4621
|
+
if not repository_is_on_any_branch:
|
|
4622
|
+
timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
|
|
4623
|
+
branch_name:str=f"pipeline_{timestamp}"
|
|
4624
|
+
self.run_program_argsasarray("git",["checkout","-b",branch_name], repository_folder)
|
|
4625
|
+
|
|
4580
4626
|
def __prepare_build_pipeline(self,repository_folder:str) -> None:
|
|
4581
4627
|
GeneralUtilities.assert_condition(self.is_running_in_build_container(), "This function should only be run in the build container.")
|
|
4582
|
-
|
|
4628
|
+
self.__ensure_branch_is_checked_out(repository_folder)
|
|
4629
|
+
|
|
4583
4630
|
expected_image = self.__get_scbuilder_image_from_image_definition_file(repository_folder)
|
|
4584
4631
|
GeneralUtilities.assert_condition("scbuilder:" in expected_image, f"The SCBuilder-image '{expected_image}' defined in the image-definition-file of the repository is not a valid SCBuilder-image. It must contain 'scbuilder:'.")
|
|
4585
4632
|
if not expected_image.endswith(":latest"):
|
|
@@ -152,9 +152,8 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
152
152
|
# which can be installed directly, is generated from it with Google's bundletool.
|
|
153
153
|
bundletool = self.tfcps_Tools_General.ensure_androidappbundletool_is_available(None, not self.use_cache())
|
|
154
154
|
# The pinned JRE from the global cache is used instead of a "java" of the machine, because a
|
|
155
|
-
# java-runtime is not part of every machine which builds a codeunit.
|
|
156
|
-
#
|
|
157
|
-
# anyway.
|
|
155
|
+
# java-runtime is not part of every machine which builds a codeunit. Pinning it also keeps this step
|
|
156
|
+
# independent of which java-version a machine happens to have.
|
|
158
157
|
java_executable = self.tfcps_Tools_General.ensure_jre_is_available(not self.use_cache(), self.get_repository_folder())
|
|
159
158
|
apk_folder = os.path.join(artifacts_folder, "BuildResult_APK")
|
|
160
159
|
GeneralUtilities.ensure_directory_does_not_exist(apk_folder)
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from ..GeneralUtilities import GeneralUtilities
|
|
4
|
+
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
5
|
+
from ..OCIImages.OCIImageManager import OCIImageManager
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class PlantUMLContainerCall:
|
|
9
|
+
"""Describes how the container which renders a diagram has to be started: which mount-arguments it gets and under
|
|
10
|
+
which path the folder with the diagram-sources is visible inside it. Which of the two possible shapes is used is
|
|
11
|
+
decided by PlantUMLRenderer.get_container_call_configuration."""
|
|
12
|
+
|
|
13
|
+
mount_arguments: list[str] = None
|
|
14
|
+
work_folder: str = None
|
|
15
|
+
|
|
16
|
+
def __init__(self, mount_arguments: list[str], work_folder: str):
|
|
17
|
+
self.mount_arguments = mount_arguments
|
|
18
|
+
self.work_folder = work_folder
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class PlantUMLRenderer:
|
|
22
|
+
"""Renders '*.plantuml'-files into svg-files.
|
|
23
|
+
|
|
24
|
+
The rendering always happens inside a short-lived, disposable container which runs the image the repository
|
|
25
|
+
declares as 'PlantUML' in '<repository>/.ScriptCollection/OCIImages/ImageDefinition.csv'. Nothing of the rendering
|
|
26
|
+
comes from the executing machine: plantuml itself, the java-runtime, Graphviz (which plantuml needs to lay out
|
|
27
|
+
component- and class-diagrams) and the fonts are all part of that image, so the pinned image-tag alone determines
|
|
28
|
+
what a diagram looks like.
|
|
29
|
+
|
|
30
|
+
That is what makes the generated svg byte-identical on every machine. It is required because plantuml computes the
|
|
31
|
+
whole svg-geometry from the font-metrics of the jvm: Windows- and Linux-builds of a jdk use the same FreeType-based
|
|
32
|
+
rasterizer and round identically, but a macOS-build uses Apple's CoreText and rounds the vertical metrics slightly
|
|
33
|
+
differently, which shifts positions in the resulting svg. Rendering with one pinned linux-image everywhere avoids
|
|
34
|
+
that, so a regenerated diagram does not show up as a spurious change. When the image-tag is bumped, the committed
|
|
35
|
+
diagram-svgs have to be regenerated and re-committed once."""
|
|
36
|
+
|
|
37
|
+
# Name under which the image used for the rendering has to be declared by every repository.
|
|
38
|
+
__image_name: str = "PlantUML"
|
|
39
|
+
# Font all diagrams are rendered with. It is passed on the commandline (and not written into every single
|
|
40
|
+
# diagram-source) so that a hand-written diagram is rendered with the same font as a generated one. Pinning it
|
|
41
|
+
# keeps the diagrams independent of which font plantuml's default 'sans-serif' happens to resolve to.
|
|
42
|
+
__diagram_font_name: str = "DejaVu Sans"
|
|
43
|
+
# Folder inside the rendering-container to which the folder with the diagram-sources is mounted when the rendering
|
|
44
|
+
# runs on a host. This is the working-directory the image itself declares.
|
|
45
|
+
__container_work_folder: str = "/data"
|
|
46
|
+
|
|
47
|
+
__sc: ScriptCollectionCore = None
|
|
48
|
+
__oci_image_manager: OCIImageManager = None
|
|
49
|
+
__repository_folder: str = None
|
|
50
|
+
|
|
51
|
+
def __init__(self, sc: ScriptCollectionCore, oci_image_manager: OCIImageManager, repository_folder: str):
|
|
52
|
+
self.__sc = sc
|
|
53
|
+
self.__oci_image_manager = oci_image_manager
|
|
54
|
+
self.__repository_folder = repository_folder
|
|
55
|
+
|
|
56
|
+
@staticmethod
|
|
57
|
+
@GeneralUtilities.check_arguments
|
|
58
|
+
def get_diagram_font_name() -> str:
|
|
59
|
+
"""Returns the font all diagrams are rendered with. Generated diagram-sources state it as 'skinparam
|
|
60
|
+
defaultFontName' themselves so their committed content already says which font they belong to."""
|
|
61
|
+
return PlantUMLRenderer.__diagram_font_name
|
|
62
|
+
|
|
63
|
+
@staticmethod
|
|
64
|
+
@GeneralUtilities.check_arguments
|
|
65
|
+
def get_image_name() -> str:
|
|
66
|
+
"""Returns the name under which the image used for the rendering has to be declared in
|
|
67
|
+
'<repository>/.ScriptCollection/OCIImages/ImageDefinition.csv'."""
|
|
68
|
+
return PlantUMLRenderer.__image_name
|
|
69
|
+
|
|
70
|
+
@staticmethod
|
|
71
|
+
@GeneralUtilities.check_arguments
|
|
72
|
+
def get_output_filename_for_plantuml_filename(plantuml_file: str) -> str:
|
|
73
|
+
"""Returns the name of the svg-file plantuml writes for the given diagram-source. Plantuml derives it from the
|
|
74
|
+
title given behind '@startuml' and falls back to the name of the source-file when there is none."""
|
|
75
|
+
for line in GeneralUtilities.read_lines_from_file(plantuml_file):
|
|
76
|
+
prefix = "@startuml "
|
|
77
|
+
if line.startswith(prefix):
|
|
78
|
+
title = line[len(prefix):]
|
|
79
|
+
return title+".svg"
|
|
80
|
+
return Path(plantuml_file).stem+".svg"
|
|
81
|
+
|
|
82
|
+
@GeneralUtilities.check_arguments
|
|
83
|
+
def render_folder(self, diagrams_files_folder: str) -> None:
|
|
84
|
+
"""Renders every '*.plantuml'-file which is located somewhere below the given folder into a svg-file next to it."""
|
|
85
|
+
plantuml_files: list[str] = [file for file in GeneralUtilities.get_all_files_of_folder(diagrams_files_folder) if file.endswith(".plantuml")]
|
|
86
|
+
if len(plantuml_files) == 0:
|
|
87
|
+
return
|
|
88
|
+
docker_image: str = self.__oci_image_manager.get_registry_address_for_image_with_default_tag(self.__repository_folder, PlantUMLRenderer.__image_name)
|
|
89
|
+
self.__assert_docker_daemon_is_reachable()
|
|
90
|
+
image_address, image_tag = ScriptCollectionCore.split_image_address_and_tag(docker_image)
|
|
91
|
+
# Pull explicitly (once for all diagrams) instead of letting the first "docker run" download the image
|
|
92
|
+
# implicitly, because docker writes the progress of an implicit download to std-err, which would be logged as
|
|
93
|
+
# an error although nothing failed.
|
|
94
|
+
self.__sc.docker_pull(image_address, image_tag)
|
|
95
|
+
# Inside a build-container the forwarded docker-socket belongs to the daemon of the host, so the container
|
|
96
|
+
# started here is a sibling-container and not a child. See get_container_call_configuration.
|
|
97
|
+
own_container_id: str = self.__sc.get_own_container_id() if self.__sc.is_runnning_in_container() else None
|
|
98
|
+
for plantuml_file in plantuml_files:
|
|
99
|
+
folder: str = os.path.dirname(plantuml_file)
|
|
100
|
+
container_call: PlantUMLContainerCall = PlantUMLRenderer.get_container_call_configuration(folder, own_container_id)
|
|
101
|
+
self.__run_plantuml_in_container(container_call, docker_image, os.path.basename(plantuml_file))
|
|
102
|
+
result_file: str = os.path.join(folder, PlantUMLRenderer.get_output_filename_for_plantuml_filename(plantuml_file))
|
|
103
|
+
GeneralUtilities.assert_file_exists(result_file)
|
|
104
|
+
self.__sc.format_xml_file(result_file)
|
|
105
|
+
|
|
106
|
+
@staticmethod
|
|
107
|
+
@GeneralUtilities.check_arguments
|
|
108
|
+
def get_container_call_configuration(diagrams_files_folder: str, own_container_id: str) -> PlantUMLContainerCall:
|
|
109
|
+
"""Returns how the rendering-container has to be started for the given folder. There are two cases:
|
|
110
|
+
|
|
111
|
+
- This process runs directly on a host: the docker-daemon resolves a bind-mount on the same filesystem this
|
|
112
|
+
process sees, so the folder with the diagram-sources is bind-mounted into the container. 'own_container_id'
|
|
113
|
+
is None in this case.
|
|
114
|
+
- This process itself runs in a container whose docker-socket is forwarded to the daemon of the host (for
|
|
115
|
+
example the build-container): a bind-mount would then be resolved by that daemon, where the paths of this
|
|
116
|
+
process do not exist. Docker silently creates an empty directory for a missing bind-mount-source, so the
|
|
117
|
+
rendering would run in an empty working-directory and would not find any diagram. The volumes of this
|
|
118
|
+
container are therefore shared with the sibling-container instead, which makes the repository visible there
|
|
119
|
+
under exactly the path this process uses.
|
|
120
|
+
|
|
121
|
+
The same rendering-command is used in both cases; only the mount-arguments and the working-directory differ."""
|
|
122
|
+
if own_container_id is None:
|
|
123
|
+
return PlantUMLContainerCall(["-v", f"{diagrams_files_folder}:{PlantUMLRenderer.__container_work_folder}"], PlantUMLRenderer.__container_work_folder)
|
|
124
|
+
return PlantUMLContainerCall(["--volumes-from", own_container_id], PlantUMLRenderer.__to_container_path(diagrams_files_folder))
|
|
125
|
+
|
|
126
|
+
@staticmethod
|
|
127
|
+
@GeneralUtilities.check_arguments
|
|
128
|
+
def get_user_arguments() -> list[str]:
|
|
129
|
+
"""Returns the arguments which make the rendering-container write the generated svg as the user this process
|
|
130
|
+
runs as. The image runs as an own unprivileged user by default, which can not write into a folder of the
|
|
131
|
+
machine when the ownership does not match - the generated svg has to belong to the user who runs the build.
|
|
132
|
+
On windows there is no such ownership, so nothing is passed there."""
|
|
133
|
+
if GeneralUtilities.current_system_is_windows():
|
|
134
|
+
return []
|
|
135
|
+
# "getuid"/"getgid" do not exist on windows, which is why they are only reached below that check; pylint
|
|
136
|
+
# analyses this file on windows as well and does not see that.
|
|
137
|
+
return ["--user", f"{os.getuid()}:{os.getgid()}"] # pylint: disable=no-member
|
|
138
|
+
|
|
139
|
+
@GeneralUtilities.check_arguments
|
|
140
|
+
def __run_plantuml_in_container(self, container_call: PlantUMLContainerCall, docker_image: str, plantuml_filename: str) -> None:
|
|
141
|
+
# The image declares "java -jar <plantuml.jar>" as its entrypoint, so only the arguments of plantuml itself are
|
|
142
|
+
# passed here.
|
|
143
|
+
plantuml_arguments: list[str] = ["-tsvg", f"-SdefaultFontName={PlantUMLRenderer.__diagram_font_name}", plantuml_filename]
|
|
144
|
+
docker_arguments: list[str] = ["run", "--rm"]+container_call.mount_arguments+PlantUMLRenderer.get_user_arguments()+["-w", container_call.work_folder, docker_image]+plantuml_arguments
|
|
145
|
+
self.__sc.run_program_argsasarray("docker", docker_arguments)
|
|
146
|
+
|
|
147
|
+
@GeneralUtilities.check_arguments
|
|
148
|
+
def __assert_docker_daemon_is_reachable(self) -> None:
|
|
149
|
+
# The rendering runs via "docker run", so an unreachable daemon makes it fail with an error which looks like a
|
|
150
|
+
# rendering-problem. "docker version --format {{.Server.Version}}" queries the daemon itself and exits
|
|
151
|
+
# non-zero exactly when it can not be reached, which distinguishes the infrastructure-problem from a real
|
|
152
|
+
# rendering-error.
|
|
153
|
+
daemon_check = self.__sc.run_program_argsasarray("docker", ["version", "--format", "{{.Server.Version}}"], throw_exception_if_exitcode_is_not_zero=False, print_live_output=False)
|
|
154
|
+
if daemon_check[0] != 0:
|
|
155
|
+
raise ValueError(f"The plantuml-diagrams can not be rendered because the docker-daemon is not reachable (exit-code {daemon_check[0]}: {daemon_check[2].strip()}). The rendering runs plantuml via 'docker run', which requires a running docker-daemon (inside a build-container its socket must be forwarded to the daemon of the host). This is an infrastructure-problem, not a problem of a diagram.")
|
|
156
|
+
|
|
157
|
+
@staticmethod
|
|
158
|
+
@GeneralUtilities.check_arguments
|
|
159
|
+
def __to_container_path(path: str) -> str:
|
|
160
|
+
# A path which is used inside the container has to be written in the linux-notation. This only has an effect
|
|
161
|
+
# when this function is called with a path of a windows-host, which happens in the tests.
|
|
162
|
+
return path.replace("\\", "/").rstrip("/")
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import json
|
|
3
3
|
import re
|
|
4
|
-
import socket
|
|
5
4
|
from datetime import datetime, timedelta,timezone
|
|
6
5
|
import xmlschema
|
|
7
6
|
from packaging.version import Version
|
|
@@ -34,7 +33,7 @@ class TFCPS_UpdateDependenciesHook(TFCPS_BuildCodeUnitsHook):
|
|
|
34
33
|
update_dependencies_script_folder: str = os.path.join(repository, "Other", "Scripts")
|
|
35
34
|
if os.path.isfile(os.path.join(update_dependencies_script_folder, "UpdateDependencies.py")):
|
|
36
35
|
self.__sc.log.log("Update dependencies of the repository...")
|
|
37
|
-
self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", update_dependencies_script_folder)
|
|
36
|
+
self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", update_dependencies_script_folder,print_live_output=True)
|
|
38
37
|
self.__ensure_changelog_is_there(repository)
|
|
39
38
|
|
|
40
39
|
@GeneralUtilities.check_arguments
|
|
@@ -54,7 +53,7 @@ class TFCPS_UpdateDependenciesHook(TFCPS_BuildCodeUnitsHook):
|
|
|
54
53
|
if not self.__tfcps_tools_general.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder, f"{codeunit_name}.codeunit.xml")):
|
|
55
54
|
return
|
|
56
55
|
self.__sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
|
|
57
|
-
self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", os.path.join(codeunit_folder, "Other"))
|
|
56
|
+
self.__sc.run_program(GeneralUtilities.get_python_executable(), "UpdateDependencies.py", os.path.join(codeunit_folder, "Other"),print_live_output=True)
|
|
58
57
|
self.__ensure_changelog_is_there(repository)
|
|
59
58
|
if self.__sc.git_repository_has_uncommitted_changes(repository):
|
|
60
59
|
#the update changed something, so it has to be verified that the codeunit is still buildable with the updated dependencies.
|
|
@@ -263,7 +262,7 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
263
262
|
self.sc.format_json_file(workspace_file)
|
|
264
263
|
|
|
265
264
|
#TODO add option to define exceptions (means: files which should not be normalized).
|
|
266
|
-
self.sc.normalize_invisible_characters_of_files_in_folder(self.repository, ["txt", "md", "py", "json", "xml", "svg", "csv", "yml", "yaml", "toml","gitignore", "gitattributes", "code-workspace"])
|
|
265
|
+
self.sc.normalize_invisible_characters_of_files_in_folder(self.repository, ["txt", "md", "py", "json", "xml", "svg", "csv", "yml", "yaml", "toml","gitignore", "gitattributes", "code-workspace", "plantuml"])
|
|
267
266
|
|
|
268
267
|
@GeneralUtilities.check_arguments
|
|
269
268
|
def is_working_branch(self)->bool:
|
|
@@ -609,7 +608,7 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
609
608
|
# The sibling betterleaks-container would then scan the wrong directory - without the
|
|
610
609
|
# repository-content and without ".betterleaks.toml", which causes false positives.
|
|
611
610
|
# Sharing our own volumes instead exposes the repository to betterleaks at the same path.
|
|
612
|
-
mount_arguments = ["--volumes-from", self.
|
|
611
|
+
mount_arguments = ["--volumes-from", self.sc.get_own_container_id()]
|
|
613
612
|
repository_in_scan_container = self.repository
|
|
614
613
|
else:
|
|
615
614
|
# Running directly on the host: a normal bind-mount works because the path is resolved
|
|
@@ -644,33 +643,6 @@ class TFCPS_CodeUnit_BuildCodeUnits:
|
|
|
644
643
|
self.sc.log.log(line, LogLevel.Error)
|
|
645
644
|
raise ValueError(f"Found unignored secret findings (exit code {result[0]}). See {os.path.join(self.repository, '.betterleaks.toml')} to ignore known false positives.")
|
|
646
645
|
|
|
647
|
-
@GeneralUtilities.check_arguments
|
|
648
|
-
def __get_own_container_id(self) -> str:
|
|
649
|
-
# Determine the id of the container this process runs in so its volumes can be shared with
|
|
650
|
-
# sibling-containers via "docker run --volumes-from".
|
|
651
|
-
# In mountinfo the own container-id only appears reliably in the source-path of the
|
|
652
|
-
# "/etc/hostname"/"/etc/hosts"/"/etc/resolv.conf"-mounts (".../containers/<id>/..."). A plain
|
|
653
|
-
# 64-hex-match there would also hit overlay-layer-hashes (which are not containers), so the
|
|
654
|
-
# "containers/"-prefix must be matched explicitly.
|
|
655
|
-
try:
|
|
656
|
-
with open("/proc/self/mountinfo", "r", encoding="utf-8") as file_handle:
|
|
657
|
-
match = re.search(r"/containers/([0-9a-f]{64})/", file_handle.read())
|
|
658
|
-
if match is not None:
|
|
659
|
-
return match.group(1)
|
|
660
|
-
except Exception:
|
|
661
|
-
pass
|
|
662
|
-
# cgroup (v1): the container-id is part of the cgroup-path; here a plain 64-hex-match is safe.
|
|
663
|
-
try:
|
|
664
|
-
with open("/proc/self/cgroup", "r", encoding="utf-8") as file_handle:
|
|
665
|
-
match = re.search(r"[0-9a-f]{64}", file_handle.read())
|
|
666
|
-
if match is not None:
|
|
667
|
-
return match.group(0)
|
|
668
|
-
except Exception:
|
|
669
|
-
pass
|
|
670
|
-
# Fallback: the hostname equals the short container-id for containers started without an
|
|
671
|
-
# explicit hostname.
|
|
672
|
-
return socket.gethostname()
|
|
673
|
-
|
|
674
646
|
@GeneralUtilities.check_arguments
|
|
675
647
|
def use_cache(self) -> bool:
|
|
676
648
|
return self.__use_cache
|