ScriptCollection 4.4.19__py3-none-any.whl → 4.4.21__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/ScriptCollectionCore.py +12 -7
- ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py +5 -2
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +45 -26
- ScriptCollection/TFCPS/SCTaskRunnerServer.py +283 -28
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +5 -4
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +0 -1
- ScriptCollection/TFCPS/TFCPS_RemoteBuild.py +121 -23
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +2 -12
- {scriptcollection-4.4.19.dist-info → scriptcollection-4.4.21.dist-info}/METADATA +1 -1
- {scriptcollection-4.4.19.dist-info → scriptcollection-4.4.21.dist-info}/RECORD +13 -13
- {scriptcollection-4.4.19.dist-info → scriptcollection-4.4.21.dist-info}/WHEEL +0 -0
- {scriptcollection-4.4.19.dist-info → scriptcollection-4.4.21.dist-info}/entry_points.txt +0 -0
- {scriptcollection-4.4.19.dist-info → scriptcollection-4.4.21.dist-info}/top_level.txt +0 -0
|
@@ -39,7 +39,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
|
|
|
39
39
|
from .ProgramRunnerPopen import ProgramRunnerPopen
|
|
40
40
|
from .SCLog import SCLog, LogLevel
|
|
41
41
|
|
|
42
|
-
version = "4.4.
|
|
42
|
+
version = "4.4.21"
|
|
43
43
|
__version__ = version
|
|
44
44
|
|
|
45
45
|
class VSCodeWorkspaceShellTask:
|
|
@@ -2644,8 +2644,11 @@ resolving a name requires fontconfig, which does not exist on every system ffmpe
|
|
|
2644
2644
|
patch = int(splitted[2])
|
|
2645
2645
|
if increment_major:
|
|
2646
2646
|
major = major+1
|
|
2647
|
+
minor=0
|
|
2648
|
+
patch=0
|
|
2647
2649
|
if increment_minor:
|
|
2648
2650
|
minor = minor+1
|
|
2651
|
+
patch=0
|
|
2649
2652
|
if increment_patch:
|
|
2650
2653
|
patch = patch+1
|
|
2651
2654
|
return f"{major}.{minor}.{patch}"
|
|
@@ -2700,16 +2703,18 @@ resolving a name requires fontconfig, which does not exist on every system ffmpe
|
|
|
2700
2703
|
or self.is_fix_branch(current_branch_name)
|
|
2701
2704
|
or self.is_other_branch(current_branch_name),
|
|
2702
2705
|
f"Repository '{repository_folder}' is on branch '{current_branch_name}' which is not a major-, minor-, fix- or other-branch. This is not allowed.")
|
|
2703
|
-
if self.
|
|
2704
|
-
|
|
2705
|
-
major = incremented.split(".")[0]
|
|
2706
|
-
result = f"{major}.0.0"
|
|
2706
|
+
if self.is_other_branch(current_branch_name) or self.is_fix_branch(current_branch_name):
|
|
2707
|
+
result = self.increment_version(current_version, False, False, True)
|
|
2707
2708
|
elif self.is_minor_branch(current_branch_name):
|
|
2708
2709
|
incremented = self.increment_version(current_version, False, True, False)
|
|
2709
2710
|
splitted = incremented.split(".")
|
|
2710
2711
|
result = f"{splitted[0]}.{splitted[1]}.0"
|
|
2712
|
+
elif self.is_major_branch(current_branch_name):
|
|
2713
|
+
incremented = self.increment_version(current_version, True, False, False)
|
|
2714
|
+
major = incremented.split(".")[0]
|
|
2715
|
+
result = f"{major}.0.0"
|
|
2711
2716
|
else:
|
|
2712
|
-
|
|
2717
|
+
raise ValueError(f"Repository '{repository_folder}' is on branch '{current_branch_name}' which is not a major-, minor-, fix- or other-branch. This is not allowed.")
|
|
2713
2718
|
else:
|
|
2714
2719
|
result = "0.1.0"
|
|
2715
2720
|
else:
|
|
@@ -4578,7 +4583,7 @@ OCR-content:
|
|
|
4578
4583
|
expected_image = self.__get_scbuilder_image_from_image_definition_file(repository_folder)
|
|
4579
4584
|
GeneralUtilities.assert_condition("scbuilder:" in expected_image, f"The SCBuilder-image '{expected_image}' defined in the image-definition-file of the repository is not a valid SCBuilder-image. It must contain 'scbuilder:'.")
|
|
4580
4585
|
if not expected_image.endswith(":latest"):
|
|
4581
|
-
minimal_required_scbuilder_version = Version("1.2.
|
|
4586
|
+
minimal_required_scbuilder_version = Version("1.2.11")#the preparation-steps below require at least this SCBuilder-version
|
|
4582
4587
|
defined_scbuilder_version = self.__get_version_of_scbuilder_image(expected_image)#TODO remove this check after 2026-12-31
|
|
4583
4588
|
GeneralUtilities.assert_condition(defined_scbuilder_version >= minimal_required_scbuilder_version, f"The SCBuilder-version {defined_scbuilder_version} defined in the image-definition-file of the repository is older than the minimal required version {minimal_required_scbuilder_version}. Please update the SCBuilder-image.")
|
|
4584
4589
|
|
|
@@ -6,7 +6,7 @@ import shutil
|
|
|
6
6
|
from ...GeneralUtilities import GeneralUtilities
|
|
7
7
|
from ...SCLog import LogLevel
|
|
8
8
|
from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base, TFCPS_CodeUnitSpecific_Base_CLI
|
|
9
|
-
from ..TFCPS_RemoteBuild import
|
|
9
|
+
from ..TFCPS_RemoteBuild import RunnerOperatingSystem
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class TFCPS_CodeUnitSpecific_CPP_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
@@ -164,7 +164,10 @@ class TFCPS_CodeUnitSpecific_CPP_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
164
164
|
else:
|
|
165
165
|
# macOS-binaries can only be produced on macOS, so this always delegates to a macOS-task-runner when not
|
|
166
166
|
# already running on macOS (see the remote-build-article in the reference).
|
|
167
|
-
|
|
167
|
+
#The result-folder is the sourcecode-folder itself: where a build-command of a C/C++-codeunit puts its
|
|
168
|
+
#output is defined by that command, so the only folder which is known to contain it is the one the commands
|
|
169
|
+
#run in - which is also the folder __collect_output_files searches afterwards.
|
|
170
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "/bin/sh", ["-c", " && ".join(self.macos_build_commands)], source_folder, source_folder)
|
|
168
171
|
self.__collect_output_files(source_folder, self.macos_output_file_patterns, "BuildResult_MacOS")
|
|
169
172
|
|
|
170
173
|
@GeneralUtilities.check_arguments
|
|
@@ -91,14 +91,14 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
91
91
|
# are produced uniformly in the same defined environment. See the remote-build-article in the reference. If
|
|
92
92
|
# no runner is configured, fall back to building locally (only possible when already running on Windows)
|
|
93
93
|
# instead of failing, so a development machine without a configured runner is not blocked.
|
|
94
|
+
windows_release_folder = os.path.join(src_folder, "build/windows/x64/runner/Release")
|
|
94
95
|
if platform.system() == "Windows" and not TFCPS_RemoteBuild(self._protected_sc).has_any_runner_configured():
|
|
95
96
|
self._protected_sc.log.log("No remote-build-runner is configured; building the windows-target locally "
|
|
96
97
|
"instead. This build is not guaranteed to be produced in the same uniform "
|
|
97
98
|
"environment as a runner-built one.", LogLevel.Warning)
|
|
98
99
|
self._protected_sc.run_with_epew("flutter", "build windows", src_folder)
|
|
99
100
|
else:
|
|
100
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder)
|
|
101
|
-
windows_release_folder = os.path.join(src_folder, "build/windows/x64/runner/Release")
|
|
101
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder, windows_release_folder)
|
|
102
102
|
windows_folder = os.path.join(artifacts_folder, "BuildResult_Windows")
|
|
103
103
|
GeneralUtilities.ensure_directory_does_not_exist(windows_folder)
|
|
104
104
|
GeneralUtilities.ensure_directory_exists(windows_folder)
|
|
@@ -119,8 +119,8 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
119
119
|
# macOS-desktop-builds must run on macOS and therefore always run on the macOS-task-runner (uniform
|
|
120
120
|
# builds), analogous to how ios-builds always run on the iOS-task-runner. See SCTaskRunnerMacOS and
|
|
121
121
|
# the remote-build-article in the reference.
|
|
122
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder)
|
|
123
122
|
macos_release_folder = os.path.join(src_folder, "build/macos/Build/Products/Release")
|
|
123
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder, macos_release_folder)
|
|
124
124
|
macos_folder = os.path.join(artifacts_folder, "BuildResult_MacOS")
|
|
125
125
|
GeneralUtilities.ensure_directory_does_not_exist(macos_folder)
|
|
126
126
|
GeneralUtilities.ensure_directory_exists(macos_folder)
|
|
@@ -128,8 +128,8 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
128
128
|
elif target == "ios":
|
|
129
129
|
# iOS-builds must run on macOS and therefore always run on the dedicated iOS-task-runner (uniform
|
|
130
130
|
# builds). See SCTaskRunnerIOS and the remote-build-article in the reference.
|
|
131
|
-
self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder)
|
|
132
131
|
ios_release_folder = os.path.join(src_folder, "build/ios/iphoneos")
|
|
132
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder, ios_release_folder)
|
|
133
133
|
ios_folder = os.path.join(artifacts_folder, "BuildResult_IOS")
|
|
134
134
|
GeneralUtilities.ensure_directory_does_not_exist(ios_folder)
|
|
135
135
|
GeneralUtilities.ensure_directory_exists(ios_folder)
|
|
@@ -138,27 +138,38 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
138
138
|
# Android-app-builds always run on an Android-task-runner (see SCTaskRunnerAndroid and the
|
|
139
139
|
# remote-build-article in the reference), analogous to how ios-builds always run on a macOS-task-runner:
|
|
140
140
|
# the Android-SDK/NDK-toolchain no longer lives in SCBuilder itself, it moved into SCTaskRunnerAndroid.
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
141
|
+
aab_release_folder = os.path.join(src_folder, "build/app/outputs/bundle/release")
|
|
142
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder, aab_release_folder)
|
|
143
|
+
# The app-bundle which came back from the runner is published as the build-result of this codeunit,
|
|
144
|
+
# analogous to the targets above: it is the artifact of an android-build, and every codeunit has to have
|
|
145
|
+
# an artifact whose name matches "BuildResult_.+" (see TFCPS_CodeUnit_BuildCodeUnit.build_codeunit).
|
|
146
|
+
aab_folder = os.path.join(artifacts_folder, "BuildResult_AAB")
|
|
147
|
+
GeneralUtilities.ensure_folder_exists_and_is_empty(aab_folder)
|
|
148
|
+
aab_file_original = self._protected_sc.find_file_by_extension(aab_release_folder, "aab")
|
|
149
|
+
aab_file = os.path.join(aab_folder, f"{codeunit_name}.aab")
|
|
150
|
+
shutil.copyfile(aab_file_original, aab_file)
|
|
151
|
+
# An app-bundle is not installable itself - it is what is uploaded to the store - so the universal apk,
|
|
152
|
+
# which can be installed directly, is generated from it with Google's bundletool.
|
|
153
|
+
bundletool = self.tfcps_Tools_General.ensure_androidappbundletool_is_available(None, not self.use_cache())
|
|
154
|
+
# The pinned JRE from the global cache is used instead of a "java" of the machine, because a
|
|
155
|
+
# java-runtime is not part of every machine which builds a codeunit. It is the same JRE ScriptCollection
|
|
156
|
+
# already uses to render the plantuml-diagrams, so this step needs nothing which a build does not need
|
|
157
|
+
# anyway.
|
|
158
|
+
java_executable = self.tfcps_Tools_General.ensure_jre_is_available(not self.use_cache(), self.get_repository_folder())
|
|
159
|
+
apk_folder = os.path.join(artifacts_folder, "BuildResult_APK")
|
|
160
|
+
GeneralUtilities.ensure_directory_does_not_exist(apk_folder)
|
|
161
|
+
GeneralUtilities.ensure_directory_exists(apk_folder)
|
|
162
|
+
apks_file = os.path.join(apk_folder, f"{codeunit_name}.apks")
|
|
163
|
+
# The arguments are passed as an array and not as one string, because they contain absolute paths and a
|
|
164
|
+
# path which contains a space would otherwise be split into several arguments (see
|
|
165
|
+
# GeneralUtilities.arguments_to_array).
|
|
166
|
+
self._protected_sc.run_program_argsasarray(java_executable, ["-jar", bundletool, "build-apks", f"--bundle={aab_file}", f"--output={apks_file}", "--mode=universal"], aab_release_folder)
|
|
167
|
+
# "--mode=universal" produces exactly one apk for all architectures, which the archive contains under
|
|
168
|
+
# this name; it is renamed to the codeunit so the artifact states which codeunit it belongs to.
|
|
169
|
+
with zipfile.ZipFile(apks_file, "r") as apks_archive:
|
|
170
|
+
apks_archive.extract("universal.apk", apk_folder)
|
|
171
|
+
GeneralUtilities.ensure_file_does_not_exist(apks_file)
|
|
172
|
+
os.rename(os.path.join(apk_folder, "universal.apk"), os.path.join(apk_folder, f"{codeunit_name}.apk"))
|
|
162
173
|
else:
|
|
163
174
|
raise ValueError(f"Not supported target: {target}")
|
|
164
175
|
self.__generate_bom_for_flutter_package(package_name)
|
|
@@ -546,7 +557,15 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
546
557
|
codeunit_name = os.path.basename(codeunit_folder)
|
|
547
558
|
src_folder = GeneralUtilities.resolve_relative_path(package_name, codeunit_folder)
|
|
548
559
|
|
|
549
|
-
|
|
560
|
+
# The testcases are executed one after the other ("--concurrency=1") although that is slower than the
|
|
561
|
+
# default (one test-file per processor-core): the coverage which is collected while several test-files run
|
|
562
|
+
# at the same time is not reproducible. Flutter attaches to the vm-service of every test-isolate to read
|
|
563
|
+
# its coverage, and under load a library which was loaded by another isolate is sometimes counted and
|
|
564
|
+
# sometimes not - the very same code produced 91.14, 91.14 and 90.58 percent in three runs, while it
|
|
565
|
+
# produces exactly 90.30 percent in every serial run. A coverage-value which changes without the code
|
|
566
|
+
# changing makes the coverage-badge of a codeunit differ after every build (so the build is not idempotent
|
|
567
|
+
# anymore) and it can push a codeunit below its minimal-coverage-threshold by chance.
|
|
568
|
+
self._protected_sc.run_with_epew("flutter", "test --coverage --concurrency=1", src_folder)
|
|
550
569
|
test_coverage_folder_relative = "Other/Artifacts/TestCoverage"
|
|
551
570
|
test_coverage_folder = GeneralUtilities.resolve_relative_path(test_coverage_folder_relative, codeunit_folder)
|
|
552
571
|
GeneralUtilities.ensure_directory_exists(test_coverage_folder)
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import json
|
|
3
3
|
import base64
|
|
4
|
+
import shutil
|
|
4
5
|
import ssl
|
|
6
|
+
import time
|
|
5
7
|
import uuid
|
|
6
8
|
import threading
|
|
7
9
|
import subprocess
|
|
@@ -11,29 +13,74 @@ from ..GeneralUtilities import GeneralUtilities
|
|
|
11
13
|
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
12
14
|
from ..SCLog import LogLevel
|
|
13
15
|
|
|
16
|
+
#Size of the chunks in which the archives are copied between the network-connection and the disk. The archives are
|
|
17
|
+
#transferred chunk-wise instead of being held in memory as a whole because both of them are regularly large (the payload
|
|
18
|
+
#contains the whole repository of the client, the result contains the complete build-output of a codeunit),
|
|
19
|
+
#while a runner often runs in a container with a small memory-limit and needs that memory for the build itself.
|
|
20
|
+
_transfer_chunk_size_in_bytes: int = 1024*1024
|
|
21
|
+
|
|
22
|
+
#Interval in which a job which is still running states that in the log of the runner. The output of a job belongs to the
|
|
23
|
+
#job and is fetched by the client, so without such a line the log of the runner would state nothing at all between the
|
|
24
|
+
#start of a job and its result - which is an hour or more for an app-build.
|
|
25
|
+
_interval_between_two_progress_logs_in_seconds: int = 60
|
|
26
|
+
|
|
27
|
+
#Amount of lines of the output of a job which did not succeed that the runner logs. The complete output belongs to the
|
|
28
|
+
#job and is fetched by the client; this is what makes the log of the runner state why a build failed even when no client
|
|
29
|
+
#is connected any more when it ends.
|
|
30
|
+
_amount_of_logged_lines_of_a_failed_job: int = 25
|
|
31
|
+
|
|
14
32
|
|
|
15
33
|
class _RunnerJob:
|
|
16
|
-
def __init__(self, workspace_folder: str,
|
|
34
|
+
def __init__(self, job_id: str, workspace_folder: str, result_folder: str):
|
|
35
|
+
self.job_id = job_id
|
|
17
36
|
self.workspace_folder = workspace_folder
|
|
18
|
-
|
|
37
|
+
#Folder whose content is the result of the job, relative to the workspace (which is the repository the client
|
|
38
|
+
#sent). Only this folder is transferred back, because only it contains what the client asked the runner to
|
|
39
|
+
#produce: everything else in the workspace is either the unchanged content the client sent or state of this
|
|
40
|
+
#machine (its paths, its caches, its toolchain-configuration), which has no business in the repository of a
|
|
41
|
+
#client.
|
|
42
|
+
self.result_folder = result_folder
|
|
19
43
|
self.state = "running" # "running" | "completed" | "failed"
|
|
20
44
|
self.exitcode = None
|
|
21
45
|
self.log = GeneralUtilities.empty_string
|
|
22
46
|
self.lock = threading.Lock()
|
|
47
|
+
#Serializes the transfer of the result-archive against the deletion of the workspace. A client deletes the job as
|
|
48
|
+
#soon as it stops waiting for the result - which also happens when it gives up on a still-running result-request,
|
|
49
|
+
#for example because a reverse-proxy in front of the runner ran into its timeout. Without this lock that deletion
|
|
50
|
+
#removes the workspace while it is being packed, which makes the transfer fail with a FileNotFoundError naming an
|
|
51
|
+
#arbitrary file of the build-output and can leave a partially deleted workspace behind.
|
|
52
|
+
self.transfer_lock = threading.Lock()
|
|
53
|
+
|
|
54
|
+
def append_to_log(self, message: str) -> None:
|
|
55
|
+
"""Adds a line of the runner itself - as opposed to a line of the program which the job runs - to the log of the
|
|
56
|
+
job. The client prints the log of a job as the output of the delegated build, so this is the place where the
|
|
57
|
+
runner can state something about a job to the one who is waiting for it."""
|
|
58
|
+
with self.lock:
|
|
59
|
+
self.log = self.log+message+"\n"
|
|
23
60
|
|
|
24
61
|
|
|
25
62
|
class SCTaskRunnerServer:
|
|
26
63
|
"""HTTP-server that compiles operating-system-bound build-steps on behalf of a (remote) client (see TFCPS_RemoteBuild).
|
|
27
64
|
Per job it extracts the received repository-archive into a fresh, empty workspace, runs the requested program on this
|
|
28
|
-
machine's operating-system, and returns the
|
|
29
|
-
job (immediately after fetching the result), so no repository-content remains on the
|
|
65
|
+
machine's operating-system, and returns the folder the client stated its result is in. The workspace is deleted as
|
|
66
|
+
soon as the client deletes the job (immediately after fetching the result), so no repository-content remains on the
|
|
67
|
+
runner."""
|
|
30
68
|
|
|
31
69
|
def __init__(self, operating_system_name: str, username: str, password: str, work_folder: str = None, sc: ScriptCollectionCore = None):
|
|
32
70
|
self.operating_system_name = operating_system_name
|
|
33
71
|
self.__username = username # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
34
72
|
self.__password = password # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
35
73
|
self.work_folder = work_folder if work_folder is not None else os.path.join(GeneralUtilities.get_temp_folder(), "SCTaskRunner")
|
|
36
|
-
|
|
74
|
+
if sc is None:
|
|
75
|
+
#The default-loglevel of a ScriptCollectionCore is Warning, while everything this server logs about its
|
|
76
|
+
#lifecycle and its jobs is logged as Information. A runner which uses that default therefore produces no
|
|
77
|
+
#output at all - neither its start nor the jobs it ran become visible in the console-output of the process,
|
|
78
|
+
#which is the only log a permanently running server has. Hence the loglevel is raised here. A caller which
|
|
79
|
+
#passes its own ScriptCollectionCore keeps the loglevel it configured there.
|
|
80
|
+
self.__sc = ScriptCollectionCore()
|
|
81
|
+
self.__sc.log.loglevel = LogLevel.Information
|
|
82
|
+
else:
|
|
83
|
+
self.__sc = sc
|
|
37
84
|
self.__jobs: dict[str, _RunnerJob] = {}
|
|
38
85
|
self.__jobs_lock = threading.Lock()
|
|
39
86
|
|
|
@@ -41,6 +88,9 @@ class SCTaskRunnerServer:
|
|
|
41
88
|
def run(self, host: str = "0.0.0.0", port: int = 8080, certificate_file: str = None, certificate_key_file: str = None) -> None:
|
|
42
89
|
"""Starts the HTTP-server. When both certificate_file and certificate_key_file are given the server is served over
|
|
43
90
|
TLS (https); otherwise it is served over plain http (e.g. when TLS is terminated by a reverse-proxy in front of it)."""
|
|
91
|
+
#Logged before anything else happens, so that the log of the runner also shows a start which did not get as far
|
|
92
|
+
#as listening (for example because the port is occupied or because the certificate can not be loaded).
|
|
93
|
+
self.__sc.log.log(f"Start the SCTaskRunner for operating-system '{self.operating_system_name}' (work-folder: '{self.work_folder}').")
|
|
44
94
|
GeneralUtilities.ensure_directory_exists(self.work_folder)
|
|
45
95
|
server = ThreadingHTTPServer((host, port), self.__create_request_handler())
|
|
46
96
|
protocol = "http"
|
|
@@ -53,71 +103,233 @@ class SCTaskRunnerServer:
|
|
|
53
103
|
server.serve_forever()
|
|
54
104
|
|
|
55
105
|
@GeneralUtilities.check_arguments
|
|
56
|
-
def __start_job(self,
|
|
106
|
+
def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
107
|
+
"""Starts a job for the repository-archive which can be read from archive_stream (the body of the request which
|
|
108
|
+
submitted the job; archive_size is its announced length). The archive is taken as a stream instead of as bytes
|
|
109
|
+
because it contains the complete repository of the client and is therefore regularly several hundred megabytes,
|
|
110
|
+
which must not be held in memory in addition to the build which runs afterwards. Only the transfer of the archive
|
|
111
|
+
happens here (it can only be read from the request which brought it); everything which follows belongs to the
|
|
112
|
+
job-thread, see __extract_payload_and_run_job."""
|
|
57
113
|
job_id = str(uuid.uuid4())
|
|
58
114
|
workspace_folder = os.path.join(self.work_folder, job_id)
|
|
59
115
|
# Isolation: each job gets a fresh, empty workspace-folder.
|
|
60
116
|
GeneralUtilities.ensure_directory_does_not_exist(workspace_folder)
|
|
61
117
|
GeneralUtilities.ensure_directory_exists(workspace_folder)
|
|
62
118
|
archive_file = os.path.join(self.work_folder, f"{job_id}.payload.tar.gz")
|
|
63
|
-
GeneralUtilities.write_binary_to_file(archive_file, archive_bytes)
|
|
64
119
|
try:
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
120
|
+
self.__copy_stream_to_file(archive_stream, archive_size, archive_file)
|
|
121
|
+
except Exception:
|
|
122
|
+
#A transfer which did not complete leaves no job behind, so the workspace and the incomplete archive are
|
|
123
|
+
#removed here - the job-thread which would do that is not started in this case.
|
|
68
124
|
GeneralUtilities.ensure_file_does_not_exist(archive_file)
|
|
69
|
-
|
|
125
|
+
GeneralUtilities.ensure_directory_does_not_exist(workspace_folder)
|
|
126
|
+
raise
|
|
127
|
+
job = _RunnerJob(job_id, workspace_folder, result_folder)
|
|
70
128
|
with self.__jobs_lock:
|
|
71
129
|
self.__jobs[job_id] = job
|
|
72
|
-
thread = threading.Thread(target=self.
|
|
130
|
+
thread = threading.Thread(target=self.__extract_payload_and_run_job, args=(job, archive_file, program, arguments, working_directory), daemon=True)
|
|
73
131
|
thread.start()
|
|
132
|
+
self.__sc.log.log(f"Accepted job '{job_id}': run '{program} {' '.join(arguments)}' in '{working_directory}' of codeunit '{codeunit_name}', result-folder '{result_folder}' ({archive_size} transferred bytes of repository-content).")
|
|
74
133
|
return job_id
|
|
75
134
|
|
|
135
|
+
@GeneralUtilities.check_arguments
|
|
136
|
+
def __extract_payload_and_run_job(self, job: _RunnerJob, archive_file: str, program: str, arguments: list[str], working_directory: str) -> None:
|
|
137
|
+
"""Extracts the transferred repository-archive into the workspace of the job and runs the requested program
|
|
138
|
+
afterwards. The extraction happens here - in the thread of the job - and not in the request which submitted the
|
|
139
|
+
job, because extracting the repository of a large codeunit takes minutes during which no data flows: the client
|
|
140
|
+
which waits for the answer of its submit, and a reverse-proxy between it and the runner, would run into their
|
|
141
|
+
timeouts although the job is being prepared correctly. Because the job already exists while this runs, the state
|
|
142
|
+
of that preparation is what a client sees and what a failure of it is reported as."""
|
|
143
|
+
try:
|
|
144
|
+
job.append_to_log("Extract the transferred repository into the workspace of the job...")
|
|
145
|
+
with tarfile.open(archive_file, "r:gz") as tar:
|
|
146
|
+
tar.extractall(job.workspace_folder, filter="fully_trusted")
|
|
147
|
+
except Exception as exception:
|
|
148
|
+
self.__end_job_as_failed_because_of_an_internal_error(job, "the transferred repository could not be extracted", exception)
|
|
149
|
+
return
|
|
150
|
+
finally:
|
|
151
|
+
GeneralUtilities.ensure_file_does_not_exist(archive_file)
|
|
152
|
+
self.__sc.log.log(f"Job '{job.job_id}' works on the repository '{self.__get_origin_of_the_transferred_repository(job.workspace_folder)}'.")
|
|
153
|
+
self.__run_job(job, program, arguments, working_directory)
|
|
154
|
+
|
|
155
|
+
@GeneralUtilities.check_arguments
|
|
156
|
+
def __get_origin_of_the_transferred_repository(self, workspace_folder: str) -> str:
|
|
157
|
+
"""Returns the address of the git-remote 'origin' of the transferred repository. This states which repository a
|
|
158
|
+
job builds without requiring the client to send anything in addition: the repository is transferred including
|
|
159
|
+
its .git-folder anyway. The address is not always available (a repository does not have to have a remote), which
|
|
160
|
+
is stated instead of being treated as an error - this is information for the log and nothing a job depends on."""
|
|
161
|
+
configuration_file = os.path.join(workspace_folder, ".git", "config")
|
|
162
|
+
if not os.path.isfile(configuration_file):
|
|
163
|
+
return "unknown: the transferred content has no git-configuration"
|
|
164
|
+
#Read line-wise and not with configparser, because git indents the entries of a section, which configparser reads
|
|
165
|
+
#as continuations of the value of the previous entry.
|
|
166
|
+
section_is_the_origin_remote = False
|
|
167
|
+
for line in GeneralUtilities.read_lines_from_file(configuration_file):
|
|
168
|
+
stripped_line = line.strip()
|
|
169
|
+
if stripped_line.startswith("["):
|
|
170
|
+
section_is_the_origin_remote = stripped_line.replace(" ", GeneralUtilities.empty_string) == '[remote"origin"]'
|
|
171
|
+
elif section_is_the_origin_remote and stripped_line.startswith("url"):
|
|
172
|
+
return self.__remove_credentials_from_address(stripped_line.split("=", 1)[1].strip())
|
|
173
|
+
return "unknown: the transferred content has no git-remote 'origin'"
|
|
174
|
+
|
|
175
|
+
@GeneralUtilities.check_arguments
|
|
176
|
+
def __remove_credentials_from_address(self, address: str) -> str:
|
|
177
|
+
"""Removes the user-information of an address like 'https://user:token@host/repository.git'. An address which is
|
|
178
|
+
written to a log must not contain a credential. An address without a scheme is returned unchanged: it is the
|
|
179
|
+
ssh-notation 'user@host:path', whose user is part of the address and not a credential."""
|
|
180
|
+
scheme_separator = "://"
|
|
181
|
+
if scheme_separator not in address:
|
|
182
|
+
return address
|
|
183
|
+
host_and_path = address.split(scheme_separator, 1)[1]
|
|
184
|
+
end_of_the_host = host_and_path.find("/")
|
|
185
|
+
host = host_and_path if end_of_the_host < 0 else host_and_path[:end_of_the_host]
|
|
186
|
+
if "@" not in host:
|
|
187
|
+
return address
|
|
188
|
+
return address.replace(host, host.split("@", 1)[1], 1)
|
|
189
|
+
|
|
190
|
+
@GeneralUtilities.check_arguments
|
|
191
|
+
def __copy_stream_to_file(self, stream, amount_of_bytes: int, target_file: str) -> None:
|
|
192
|
+
"""Copies exactly amount_of_bytes bytes from stream into target_file, chunk-wise (see
|
|
193
|
+
_transfer_chunk_size_in_bytes for why the content is not read into memory as a whole). A stream which ends early
|
|
194
|
+
is reported as an error instead of resulting in a truncated file which would fail later as an unspecific
|
|
195
|
+
archive-error."""
|
|
196
|
+
remaining_amount_of_bytes = amount_of_bytes
|
|
197
|
+
with open(target_file, "wb") as target_file_object:
|
|
198
|
+
while 0 < remaining_amount_of_bytes:
|
|
199
|
+
chunk = stream.read(min(_transfer_chunk_size_in_bytes, remaining_amount_of_bytes))
|
|
200
|
+
if not chunk:
|
|
201
|
+
raise ValueError(f"The transferred content ended after {amount_of_bytes-remaining_amount_of_bytes} bytes although {amount_of_bytes} bytes were announced.")
|
|
202
|
+
target_file_object.write(chunk)
|
|
203
|
+
remaining_amount_of_bytes = remaining_amount_of_bytes-len(chunk)
|
|
204
|
+
|
|
76
205
|
@GeneralUtilities.check_arguments
|
|
77
206
|
def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str) -> None:
|
|
78
207
|
command_folder = os.path.join(job.workspace_folder, working_directory)
|
|
79
208
|
try:
|
|
80
|
-
self.__sc.log.log(f"
|
|
209
|
+
self.__sc.log.log(f"Job '{job.job_id}' runs '{program} {' '.join(arguments)}' in '{command_folder}'...")
|
|
210
|
+
moment_of_the_start = time.time()
|
|
211
|
+
moment_of_the_last_progress_log = moment_of_the_start
|
|
212
|
+
amount_of_lines = 0
|
|
81
213
|
with subprocess.Popen([program] + arguments, cwd=command_folder, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace") as process:
|
|
82
214
|
for line in process.stdout:
|
|
83
215
|
with job.lock:
|
|
84
216
|
job.log = job.log + line
|
|
217
|
+
amount_of_lines = amount_of_lines+1
|
|
218
|
+
if time.time()-moment_of_the_last_progress_log > _interval_between_two_progress_logs_in_seconds:
|
|
219
|
+
moment_of_the_last_progress_log = time.time()
|
|
220
|
+
self.__sc.log.log(f"Job '{job.job_id}' is still running (since {round(time.time()-moment_of_the_start)} seconds, {amount_of_lines} lines of output so far).")
|
|
85
221
|
process.wait()
|
|
86
222
|
with job.lock:
|
|
87
223
|
job.exitcode = process.returncode
|
|
88
224
|
job.state = "completed" if process.returncode == 0 else "failed"
|
|
225
|
+
state_of_the_job = job.state
|
|
226
|
+
exitcode_of_the_job = job.exitcode
|
|
227
|
+
#Logged so that the log of the runner states the result of every job it ran, also when no client is
|
|
228
|
+
#waiting for it any more: a connection to the client can break while a job runs, but the job keeps
|
|
229
|
+
#running and therefore has to end with a stated result on this side too.
|
|
230
|
+
self.__sc.log.log(f"Job '{job.job_id}' is {state_of_the_job} (exitcode: {exitcode_of_the_job}) after {round(time.time()-moment_of_the_start)} seconds.")
|
|
231
|
+
if exitcode_of_the_job != 0:
|
|
232
|
+
self.__log_the_end_of_the_output_of_the_job(job)
|
|
89
233
|
except Exception as exception:
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
234
|
+
self.__end_job_as_failed_because_of_an_internal_error(job, "the program could not be run", exception)
|
|
235
|
+
|
|
236
|
+
@GeneralUtilities.check_arguments
|
|
237
|
+
def __log_the_end_of_the_output_of_the_job(self, job: _RunnerJob) -> None:
|
|
238
|
+
"""Logs the last lines of the output of a job which did not succeed. The complete output belongs to the job and
|
|
239
|
+
is fetched by the client, but the log of the runner has to state why a build failed too: a client is not
|
|
240
|
+
necessarily still connected when a job ends, and its output is gone with the job."""
|
|
241
|
+
with job.lock:
|
|
242
|
+
lines = GeneralUtilities.string_to_lines(job.log)
|
|
243
|
+
last_lines = lines[-_amount_of_logged_lines_of_a_failed_job:]
|
|
244
|
+
self.__sc.log.log(f"The last {len(last_lines)} of the {len(lines)} lines of the output of job '{job.job_id}' are:")
|
|
245
|
+
for line in last_lines:
|
|
246
|
+
self.__sc.log.log(f" {line}")
|
|
247
|
+
|
|
248
|
+
@GeneralUtilities.check_arguments
|
|
249
|
+
def __end_job_as_failed_because_of_an_internal_error(self, job: _RunnerJob, what_did_not_work: str, exception: Exception) -> None:
|
|
250
|
+
"""Ends a job which failed because of the runner itself and not because of what it was asked to build. The
|
|
251
|
+
details of such a failure - stacktraces and paths inside this runner - go to the log of the runner only: they
|
|
252
|
+
state something about the internals of the runner, and the one who waits for the build can not act on them. The
|
|
253
|
+
client gets the statement that the runner failed plus the error-id under which the details are logged, so that
|
|
254
|
+
both sides can be brought together. The output of the program of a job is something else: it is what the client
|
|
255
|
+
asked for and is transferred in full, so that a build which does not compile states why."""
|
|
256
|
+
error_id = str(uuid.uuid4())
|
|
257
|
+
self.__sc.log.log_exception(f"Internal error '{error_id}': job '{job.job_id}' failed because {what_did_not_work}.", exception)
|
|
258
|
+
self.__end_job_as_failed(job, f"The runner could not run this job because {what_did_not_work}. This is a problem of the runner itself; its details are in the log of the runner under the error-id '{error_id}'.")
|
|
259
|
+
|
|
260
|
+
@GeneralUtilities.check_arguments
|
|
261
|
+
def __end_job_as_failed(self, job: _RunnerJob, message: str) -> None:
|
|
262
|
+
"""Ends a job which could not be run at all with the same shape of result a program which returned a non-zero
|
|
263
|
+
exitcode produces, so that a client always gets a state and an exitcode and does not have to distinguish the
|
|
264
|
+
two cases."""
|
|
265
|
+
job.append_to_log(message)
|
|
266
|
+
with job.lock:
|
|
267
|
+
job.exitcode = -1
|
|
268
|
+
job.state = "failed"
|
|
94
269
|
|
|
95
270
|
@GeneralUtilities.check_arguments
|
|
96
|
-
def
|
|
97
|
-
|
|
271
|
+
def __create_result_archive(self, job: _RunnerJob) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
272
|
+
"""Packs the result-folder of the job and returns the file the archive was written to. The archive is returned
|
|
273
|
+
as a file (and not as bytes) because the build-output of a codeunit regularly has a size of several gigabytes,
|
|
274
|
+
which must not be held in memory. The caller is responsible for deleting the returned file after it was
|
|
275
|
+
transferred."""
|
|
98
276
|
archive_file = os.path.join(self.work_folder, f"{uuid.uuid4()}.result.tar.gz")
|
|
99
277
|
try:
|
|
100
278
|
with tarfile.open(archive_file, "w:gz") as tar:
|
|
101
|
-
tar.add(
|
|
102
|
-
|
|
103
|
-
finally:
|
|
279
|
+
tar.add(self.__get_result_folder_of_the_job(job), arcname=".")
|
|
280
|
+
except Exception:
|
|
104
281
|
GeneralUtilities.ensure_file_does_not_exist(archive_file)
|
|
282
|
+
raise
|
|
283
|
+
return archive_file
|
|
284
|
+
|
|
285
|
+
@GeneralUtilities.check_arguments
|
|
286
|
+
def __get_result_folder_of_the_job(self, job: _RunnerJob) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
287
|
+
"""Returns the folder whose content is the result of the job. A folder outside of the workspace of the job is
|
|
288
|
+
refused: what a runner returns is what the job produced and never something else of this machine."""
|
|
289
|
+
result_folder = os.path.realpath(os.path.join(job.workspace_folder, job.result_folder))
|
|
290
|
+
workspace_folder = os.path.realpath(job.workspace_folder)
|
|
291
|
+
if result_folder != workspace_folder and not result_folder.startswith(workspace_folder+os.sep):
|
|
292
|
+
raise ValueError(f"The result-folder '{job.result_folder}' of job '{job.job_id}' is not inside the workspace of that job.")
|
|
293
|
+
return result_folder
|
|
105
294
|
|
|
106
295
|
@GeneralUtilities.check_arguments
|
|
107
296
|
def __delete_job(self, job_id: str) -> None: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
108
297
|
with self.__jobs_lock:
|
|
109
298
|
job = self.__jobs.pop(job_id, None)
|
|
110
299
|
if job is not None:
|
|
111
|
-
|
|
300
|
+
# The job was already removed from the job-list above, so no further result-request can start for it. This
|
|
301
|
+
# waits for one which is still running (see _RunnerJob.transfer_lock) before the workspace is removed.
|
|
302
|
+
with job.transfer_lock:
|
|
303
|
+
GeneralUtilities.ensure_directory_does_not_exist(job.workspace_folder)
|
|
112
304
|
|
|
113
305
|
def __create_request_handler(self):
|
|
114
306
|
outer = self
|
|
115
307
|
|
|
116
308
|
class Handler(BaseHTTPRequestHandler):
|
|
117
309
|
|
|
310
|
+
#Whether the answer of the request which is currently handled was already begun. After that point the
|
|
311
|
+
#status-line is already on the connection, so a failure can not be answered with a status-code any more.
|
|
312
|
+
__answer_was_begun: bool = False
|
|
313
|
+
|
|
118
314
|
def log_message(self, format, *args): # pylint:disable=redefined-builtin
|
|
119
315
|
outer._SCTaskRunnerServer__sc.log.log((format % args), LogLevel.Debug)
|
|
120
316
|
|
|
317
|
+
def __handle(self, handle_request) -> None:
|
|
318
|
+
"""Runs the handling of a request and turns everything which it did not handle itself into a defined
|
|
319
|
+
answer. An unexpected exception here is a problem of the runner itself, so its details go to the log of
|
|
320
|
+
the runner only - they state something about the internals of this runner, which the one who waits for a
|
|
321
|
+
build can not act on - while the client gets a 500 naming the error-id under which they are logged.
|
|
322
|
+
Without this the request would stay unanswered completely, which a client sees as a broken connection
|
|
323
|
+
and not as a statement about what happened."""
|
|
324
|
+
self.__answer_was_begun = False
|
|
325
|
+
try:
|
|
326
|
+
handle_request()
|
|
327
|
+
except Exception as exception:
|
|
328
|
+
error_id = str(uuid.uuid4())
|
|
329
|
+
outer._SCTaskRunnerServer__sc.log.log_exception(f"Internal error '{error_id}' while handling '{self.command} {self.path}'.", exception)
|
|
330
|
+
if not self.__answer_was_begun:
|
|
331
|
+
self.__send_text(500, f"Internal error of the runner. Its details are in the log of the runner under the error-id '{error_id}'.")
|
|
332
|
+
|
|
121
333
|
def __is_authorized(self) -> bool:
|
|
122
334
|
header = self.headers.get("Authorization", GeneralUtilities.empty_string)
|
|
123
335
|
if not header.startswith("Basic "):
|
|
@@ -130,12 +342,24 @@ class SCTaskRunnerServer:
|
|
|
130
342
|
return decoded == expected
|
|
131
343
|
|
|
132
344
|
def __send(self, status: int, body: bytes, content_type: str = "application/octet-stream") -> None:
|
|
345
|
+
self.__answer_was_begun = True
|
|
133
346
|
self.send_response(status)
|
|
134
347
|
self.send_header("Content-Type", content_type)
|
|
135
348
|
self.send_header("Content-Length", str(len(body)))
|
|
136
349
|
self.end_headers()
|
|
137
350
|
self.wfile.write(body)
|
|
138
351
|
|
|
352
|
+
def __send_file(self, status: int, file: str) -> None:
|
|
353
|
+
"""Sends the content of a file without reading it into memory as a whole (see
|
|
354
|
+
_transfer_chunk_size_in_bytes)."""
|
|
355
|
+
self.__answer_was_begun = True
|
|
356
|
+
self.send_response(status)
|
|
357
|
+
self.send_header("Content-Type", "application/octet-stream")
|
|
358
|
+
self.send_header("Content-Length", str(os.path.getsize(file)))
|
|
359
|
+
self.end_headers()
|
|
360
|
+
with open(file, "rb") as file_object:
|
|
361
|
+
shutil.copyfileobj(file_object, self.wfile, _transfer_chunk_size_in_bytes)
|
|
362
|
+
|
|
139
363
|
def __send_text(self, status: int, text: str) -> None:
|
|
140
364
|
self.__send(status, text.encode("utf-8"), "text/plain; charset=utf-8")
|
|
141
365
|
|
|
@@ -147,6 +371,9 @@ class SCTaskRunnerServer:
|
|
|
147
371
|
return outer._SCTaskRunnerServer__jobs.get(job_id, None)
|
|
148
372
|
|
|
149
373
|
def do_GET(self): # pylint:disable=invalid-name
|
|
374
|
+
self.__handle(self.__get)
|
|
375
|
+
|
|
376
|
+
def __get(self) -> None:
|
|
150
377
|
if not self.__is_authorized():
|
|
151
378
|
self.__send_text(401, "Unauthorized")
|
|
152
379
|
return
|
|
@@ -178,27 +405,55 @@ class SCTaskRunnerServer:
|
|
|
178
405
|
if not ready:
|
|
179
406
|
self.__send_text(409, "Job is not completed")
|
|
180
407
|
return
|
|
181
|
-
self.
|
|
408
|
+
self.__send_result_archive(job)
|
|
182
409
|
return
|
|
183
410
|
self.__send_text(404, "Not found")
|
|
184
411
|
|
|
412
|
+
def __send_result_archive(self, job: _RunnerJob) -> None:
|
|
413
|
+
# The transfer-lock is held while the archive is created and sent, so a DELETE for this job waits instead
|
|
414
|
+
# of removing the workspace while it is being packed (see _RunnerJob.transfer_lock).
|
|
415
|
+
with job.transfer_lock:
|
|
416
|
+
if not os.path.isdir(outer._SCTaskRunnerServer__get_result_folder_of_the_job(job)):
|
|
417
|
+
#A job which ran successfully but produced no result-folder is a defined outcome and not an
|
|
418
|
+
#error of the runner: what a build-step produces is stated by the client which submitted it.
|
|
419
|
+
self.__send_text(409, f"The job did not produce its result-folder '{job.result_folder}'.")
|
|
420
|
+
return
|
|
421
|
+
archive_file = outer._SCTaskRunnerServer__create_result_archive(job)
|
|
422
|
+
try:
|
|
423
|
+
self.__send_file(200, archive_file)
|
|
424
|
+
finally:
|
|
425
|
+
GeneralUtilities.ensure_file_does_not_exist(archive_file)
|
|
426
|
+
|
|
185
427
|
def do_POST(self): # pylint:disable=invalid-name
|
|
428
|
+
self.__handle(self.__post)
|
|
429
|
+
|
|
430
|
+
def __post(self) -> None:
|
|
186
431
|
if not self.__is_authorized():
|
|
187
432
|
self.__send_text(401, "Unauthorized")
|
|
188
433
|
return
|
|
189
434
|
if self.path != "/jobs":
|
|
190
435
|
self.__send_text(404, "Not found")
|
|
191
436
|
return
|
|
192
|
-
|
|
193
|
-
|
|
437
|
+
#A request which does not state what is to be run is a defect of the client and not of the runner, so it
|
|
438
|
+
#is answered with what is missing instead of with the 500 of an unexpected failure.
|
|
439
|
+
missing_headers = [header for header in ("Content-Length", "X-Codeunit-Name", "X-Program", "X-Arguments", "X-Result-Folder") if self.headers.get(header) is None]
|
|
440
|
+
if len(missing_headers) > 0:
|
|
441
|
+
self.__send_text(400, f"The request does not contain the required header(s) {', '.join(missing_headers)}.")
|
|
442
|
+
return
|
|
443
|
+
content_length = int(self.headers.get("Content-Length"))
|
|
194
444
|
codeunit_name = self.headers.get("X-Codeunit-Name")
|
|
195
445
|
program = self.headers.get("X-Program")
|
|
196
|
-
arguments = json.loads(base64.b64decode(self.headers.get("X-Arguments"
|
|
446
|
+
arguments = json.loads(base64.b64decode(self.headers.get("X-Arguments")).decode("utf-8"))
|
|
197
447
|
working_directory = self.headers.get("X-Working-Directory", ".")
|
|
198
|
-
|
|
448
|
+
result_folder = self.headers.get("X-Result-Folder")
|
|
449
|
+
#the body is handed over as a stream (and not read here) so the archive is written to disk chunk-wise, see __start_job.
|
|
450
|
+
job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory, result_folder)
|
|
199
451
|
self.__send_json(200, {"job_id": job_id})
|
|
200
452
|
|
|
201
453
|
def do_DELETE(self): # pylint:disable=invalid-name
|
|
454
|
+
self.__handle(self.__delete)
|
|
455
|
+
|
|
456
|
+
def __delete(self) -> None:
|
|
202
457
|
if not self.__is_authorized():
|
|
203
458
|
self.__send_text(401, "Unauthorized")
|
|
204
459
|
return
|
|
@@ -364,13 +364,14 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
|
|
|
364
364
|
return self.__use_cache
|
|
365
365
|
|
|
366
366
|
@GeneralUtilities.check_arguments
|
|
367
|
-
def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str) -> None:
|
|
367
|
+
def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str, result_folder: str) -> None:
|
|
368
368
|
"""Runs the given program for this codeunit on a remote task-runner that provides the given operating-system (see
|
|
369
369
|
TFCPS_RemoteBuild and the SCTaskRunnerWindows/SCTaskRunnerMacOS-codeunits). This is used for operating-system-bound
|
|
370
370
|
build-steps (e.g. flutter-windows- or flutter-ios-builds) which can not run on the current operating-system. After
|
|
371
|
-
the runner finished,
|
|
372
|
-
|
|
373
|
-
|
|
371
|
+
the runner finished, the content of result_folder is written back into the local repository, so the build-step can
|
|
372
|
+
continue with the result as if it had been produced here; nothing else of the local repository is touched.
|
|
373
|
+
working_directory and result_folder must be folders inside this codeunit."""
|
|
374
|
+
TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory, result_folder)
|
|
374
375
|
|
|
375
376
|
@GeneralUtilities.check_arguments
|
|
376
377
|
def get_codeunit_folder(self)->str:
|
|
@@ -7,11 +7,32 @@ import base64
|
|
|
7
7
|
import tarfile
|
|
8
8
|
import urllib.request
|
|
9
9
|
import urllib.error
|
|
10
|
+
import http.client
|
|
10
11
|
from enum import Enum
|
|
11
12
|
from ..GeneralUtilities import GeneralUtilities
|
|
12
13
|
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
13
14
|
from ..SCLog import LogLevel
|
|
14
15
|
|
|
16
|
+
#Duration for which the client keeps asking a runner which does not answer before it states that the delegated build has
|
|
17
|
+
#no result here. A job runs on the runner independently of the connection to the client, so an interruption of that
|
|
18
|
+
#connection (for example a restart of a reverse-proxy in front of the runner) does not stop the build - giving up
|
|
19
|
+
#immediately would discard a build which is still producing a result. A longer interruption is not bridged, so that a
|
|
20
|
+
#build always ends with a statement instead of with an endless wait.
|
|
21
|
+
_tolerated_interruption_of_the_connection_to_the_runner_in_seconds: int = 300
|
|
22
|
+
|
|
23
|
+
#Timeout of a request whose answer the runner produces without doing anything long-running for it (the state of a job,
|
|
24
|
+
#its log, its deletion, the operating-system a runner provides). It is the timeout of a single operation on the
|
|
25
|
+
#connection and not the duration of the whole request, so it is the time after which a connection which does not
|
|
26
|
+
#transport anything any more counts as interrupted.
|
|
27
|
+
_timeout_of_a_request_in_seconds: int = 300
|
|
28
|
+
|
|
29
|
+
#Timeout of a request whose answer the runner produces only after something long-running which transports nothing while
|
|
30
|
+
#it runs: the result-archive of a job contains the complete build-output of a codeunit and is packed completely before
|
|
31
|
+
#its first byte is sent, which takes minutes for a large build-output. This has to stay above the read-timeout of a
|
|
32
|
+
#reverse-proxy in front of a runner, so that its gateway-timeout - a defined answer - arrives instead of the client
|
|
33
|
+
#giving up on the connection first.
|
|
34
|
+
_timeout_of_a_long_running_request_in_seconds: int = 60*60
|
|
35
|
+
|
|
15
36
|
|
|
16
37
|
class RunnerOperatingSystem(Enum):
|
|
17
38
|
"""Operating-system a remote-build-runner provides. The value is the token exchanged over the wire (see the runner's
|
|
@@ -34,12 +55,28 @@ class RunnerEndpoint:
|
|
|
34
55
|
self.password = password
|
|
35
56
|
|
|
36
57
|
|
|
58
|
+
class RunnerRequestFailedError(ValueError):
|
|
59
|
+
"""A runner answered a request with an error-status. The request reached the runner, so this is a defined answer of
|
|
60
|
+
the runner and not a problem of the connection to it (see RunnerNotReachableError for that)."""
|
|
61
|
+
|
|
62
|
+
def __init__(self, message: str, status: int):
|
|
63
|
+
super().__init__(message)
|
|
64
|
+
self.status = status
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class RunnerNotReachableError(ValueError):
|
|
68
|
+
"""A request did not reach a runner, or its answer did not reach the client, for example because the connection was
|
|
69
|
+
reset or because the host is not resolvable. In contrast to RunnerRequestFailedError this states nothing about the
|
|
70
|
+
state of a job on the runner: a job which was started there before keeps running."""
|
|
71
|
+
|
|
72
|
+
|
|
37
73
|
class TFCPS_RemoteBuild:
|
|
38
74
|
"""Delegates an operating-system-bound build-step to a remote task-runner (see SCTaskRunnerWindows/SCTaskRunnerMacOS):
|
|
39
75
|
the whole repository (including the .git-folder, uncommitted changes and git-ignored files) is sent as a tar-archive
|
|
40
76
|
over HTTPS to the runner, the runner runs the given program on its (correct) operating-system, and afterwards only the
|
|
41
|
-
|
|
42
|
-
|
|
77
|
+
folder in which that build-step produces its result is written back into the local repository. Everything else which
|
|
78
|
+
changed in the workspace of the runner stays there: it is state of that machine (its toolchain-paths, its caches) and
|
|
79
|
+
not a result, and a local repository is something a developer works in and which therefore must not be replaced."""
|
|
43
80
|
|
|
44
81
|
__sc: ScriptCollectionCore = None
|
|
45
82
|
|
|
@@ -52,7 +89,7 @@ class TFCPS_RemoteBuild:
|
|
|
52
89
|
return len(self.__load_runner_endpoints()) > 0
|
|
53
90
|
|
|
54
91
|
@GeneralUtilities.check_arguments
|
|
55
|
-
def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
|
|
92
|
+
def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str, poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
|
|
56
93
|
endpoints = self.__load_runner_endpoints()
|
|
57
94
|
if len(endpoints) == 0:
|
|
58
95
|
raise ValueError("No remote-build-runner is configured. Define runners either in "
|
|
@@ -61,7 +98,8 @@ class TFCPS_RemoteBuild:
|
|
|
61
98
|
"'Runner_<name>_URL', 'Runner_<name>_Username' and 'Runner_<name>_Password'.")
|
|
62
99
|
endpoint = self.__select_runner_for_os(endpoints, required_os)
|
|
63
100
|
working_directory_relative = os.path.relpath(working_directory, repository_folder).replace("\\", "/")
|
|
64
|
-
|
|
101
|
+
result_folder_relative = os.path.relpath(result_folder, repository_folder).replace("\\", "/")
|
|
102
|
+
self.__sc.log.log(f"Delegate '{program} {' '.join(arguments)}' (codeunit '{codeunit_name}', folder '{working_directory_relative}', result-folder '{result_folder_relative}') to the {required_os.value}-runner at {endpoint.url}...")
|
|
65
103
|
archive_file = self.__create_repository_archive(repository_folder)
|
|
66
104
|
job_id: str = None
|
|
67
105
|
try:
|
|
@@ -71,19 +109,22 @@ class TFCPS_RemoteBuild:
|
|
|
71
109
|
"X-Program": program,
|
|
72
110
|
"X-Arguments": base64.b64encode(json.dumps(arguments).encode("utf-8")).decode("ascii"),
|
|
73
111
|
"X-Working-Directory": working_directory_relative,
|
|
112
|
+
"X-Result-Folder": result_folder_relative,
|
|
74
113
|
}
|
|
75
114
|
with open(archive_file, "rb") as archive_content:
|
|
76
115
|
archive_bytes = archive_content.read()
|
|
77
|
-
_, submit_response = self.__http(endpoint, "POST", "/jobs", metadata_headers, archive_bytes)
|
|
116
|
+
_, submit_response = self.__http(endpoint, "POST", "/jobs", _timeout_of_a_request_in_seconds, metadata_headers, archive_bytes)
|
|
78
117
|
job_id = json.loads(submit_response.decode("utf-8"))["job_id"]
|
|
79
118
|
self.__wait_for_job(endpoint, job_id, required_os, poll_interval_in_seconds, timeout_in_seconds)
|
|
80
|
-
|
|
81
|
-
|
|
119
|
+
#The result-archive is packed completely before it is sent, which transports nothing while it runs - hence
|
|
120
|
+
#the timeout of a long-running request here.
|
|
121
|
+
_, result_bytes = self.__http_of_running_job(endpoint, "GET", f"/jobs/{job_id}/result", _timeout_of_a_long_running_request_in_seconds, poll_interval_in_seconds)
|
|
122
|
+
self.__write_result_into_the_repository(result_bytes, result_folder)
|
|
82
123
|
finally:
|
|
83
124
|
if job_id is not None:
|
|
84
125
|
try:
|
|
85
126
|
# Triggers the runner to delete its (isolated) workspace for this job immediately after the result was fetched.
|
|
86
|
-
self.__http(endpoint, "DELETE", f"/jobs/{job_id}")
|
|
127
|
+
self.__http(endpoint, "DELETE", f"/jobs/{job_id}", _timeout_of_a_request_in_seconds)
|
|
87
128
|
except Exception as exception:
|
|
88
129
|
self.__sc.log.log_exception(f"Could not delete remote job '{job_id}' on runner '{endpoint.url}'.", exception, LogLevel.Warning)
|
|
89
130
|
GeneralUtilities.ensure_file_does_not_exist(archive_file)
|
|
@@ -93,9 +134,8 @@ class TFCPS_RemoteBuild:
|
|
|
93
134
|
start_time = time.time()
|
|
94
135
|
already_printed_log_length = 0
|
|
95
136
|
while True:
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
_, log_bytes = self.__http(endpoint, "GET", f"/jobs/{job_id}/logs")
|
|
137
|
+
status = self.__get_job_status(endpoint, job_id, required_os, poll_interval_in_seconds)
|
|
138
|
+
_, log_bytes = self.__http_of_running_job(endpoint, "GET", f"/jobs/{job_id}/logs", _timeout_of_a_request_in_seconds, poll_interval_in_seconds)
|
|
99
139
|
log_text = log_bytes.decode("utf-8", errors="replace")
|
|
100
140
|
if len(log_text) > already_printed_log_length:
|
|
101
141
|
for line in GeneralUtilities.string_to_lines(log_text[already_printed_log_length:]):
|
|
@@ -111,6 +151,43 @@ class TFCPS_RemoteBuild:
|
|
|
111
151
|
raise ValueError(f"The remote build on the {required_os.value}-runner did not finish within {timeout_in_seconds} seconds.")
|
|
112
152
|
time.sleep(poll_interval_in_seconds)
|
|
113
153
|
|
|
154
|
+
@GeneralUtilities.check_arguments
|
|
155
|
+
def __get_job_status(self, endpoint: RunnerEndpoint, job_id: str, required_os: RunnerOperatingSystem, poll_interval_in_seconds: int) -> dict:
|
|
156
|
+
"""Asks the runner for the state of the job. A runner which does not know the job (any more) gets an own message:
|
|
157
|
+
a runner holds its jobs in memory only, so that is what a restart of the runner while a job was running looks
|
|
158
|
+
like from here, and the status-code alone does not state that the delegated build is gone with it."""
|
|
159
|
+
try:
|
|
160
|
+
_, status_bytes = self.__http_of_running_job(endpoint, "GET", f"/jobs/{job_id}", _timeout_of_a_request_in_seconds, poll_interval_in_seconds)
|
|
161
|
+
except RunnerRequestFailedError as request_failed_error:
|
|
162
|
+
if request_failed_error.status == 404:
|
|
163
|
+
raise ValueError(f"The {required_os.value}-runner at '{endpoint.url}' does not know the job '{job_id}' (any more), so the delegated build has no result. A runner holds its jobs in memory only: when it is restarted while a job runs, the job and its build are gone with it.") from None
|
|
164
|
+
raise
|
|
165
|
+
return json.loads(status_bytes.decode("utf-8"))
|
|
166
|
+
|
|
167
|
+
@GeneralUtilities.check_arguments
|
|
168
|
+
def __http_of_running_job(self, endpoint: RunnerEndpoint, method: str, path: str, timeout_in_seconds: int, retry_interval_in_seconds: int) -> tuple[int, bytes]:
|
|
169
|
+
"""Does a request which concerns a job that already exists on the runner and which can therefore be repeated
|
|
170
|
+
without side-effects (it neither starts a job twice nor changes one). The job keeps running on the runner while
|
|
171
|
+
the connection to it is interrupted, so an interruption which is over within
|
|
172
|
+
_tolerated_interruption_of_the_connection_to_the_runner_in_seconds is bridged here by repeating the request
|
|
173
|
+
instead of giving up a build which still produces a result. A longer interruption ends in a
|
|
174
|
+
RunnerNotReachableError which states that the result of the build is unknown here. An answer of the runner -
|
|
175
|
+
an error-status included - is not an interruption and is returned respectively raised immediately."""
|
|
176
|
+
moment_of_the_first_unreachable_request: float = None
|
|
177
|
+
while True:
|
|
178
|
+
try:
|
|
179
|
+
result = self.__http(endpoint, method, path, timeout_in_seconds)
|
|
180
|
+
if moment_of_the_first_unreachable_request is not None:
|
|
181
|
+
self.__sc.log.log(f"The runner '{endpoint.url}' is reachable again.", LogLevel.Information)
|
|
182
|
+
return result
|
|
183
|
+
except RunnerNotReachableError as not_reachable_error:
|
|
184
|
+
if moment_of_the_first_unreachable_request is None:
|
|
185
|
+
moment_of_the_first_unreachable_request = time.time()
|
|
186
|
+
self.__sc.log.log(f"{not_reachable_error} The job runs on the runner independently of this connection, so the request is repeated for at most {_tolerated_interruption_of_the_connection_to_the_runner_in_seconds} seconds.", LogLevel.Warning)
|
|
187
|
+
if time.time()-moment_of_the_first_unreachable_request > _tolerated_interruption_of_the_connection_to_the_runner_in_seconds:
|
|
188
|
+
raise RunnerNotReachableError(f"The runner '{endpoint.url}' was not reachable for {_tolerated_interruption_of_the_connection_to_the_runner_in_seconds} seconds while the request '{method} {path}' was repeated (last reason: {not_reachable_error}). The delegated build therefore has no result here; it may still be running on the runner.") from None
|
|
189
|
+
time.sleep(retry_interval_in_seconds)
|
|
190
|
+
|
|
114
191
|
@GeneralUtilities.check_arguments
|
|
115
192
|
def __create_repository_archive(self, repository_folder: str) -> str:
|
|
116
193
|
# Pack the entire repository-working-tree (including .git, uncommitted changes and git-ignored files) so the runner
|
|
@@ -122,18 +199,23 @@ class TFCPS_RemoteBuild:
|
|
|
122
199
|
return archive_file
|
|
123
200
|
|
|
124
201
|
@GeneralUtilities.check_arguments
|
|
125
|
-
def
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
202
|
+
def __write_result_into_the_repository(self, result_bytes: bytes, result_folder: str) -> None:
|
|
203
|
+
"""Writes what the runner produced into the folder of the local repository in which the build-step produces its
|
|
204
|
+
result, so the build can continue with it as if it had been produced here.
|
|
205
|
+
|
|
206
|
+
Nothing is deleted for this. Before, the whole codeunit-folder was replaced by the one of the runner, which
|
|
207
|
+
deleted everything of that codeunit which is not part of the answer of the runner - including files which are
|
|
208
|
+
not in git and can therefore not be restored - and which on Windows can not even complete: the build-script of
|
|
209
|
+
the codeunit runs in a folder below the codeunit, and a folder a process runs in can not be removed there, so
|
|
210
|
+
the deletion failed in the middle and left the codeunit incomplete. It is also not what is wanted: what changes
|
|
211
|
+
in the workspace of the runner besides the result is state of that machine (its toolchain-paths, its caches),
|
|
212
|
+
which has no business in the repository of a developer."""
|
|
130
213
|
result_archive = os.path.join(GeneralUtilities.get_temp_folder(), f"sc-remotebuild-result-{uuid.uuid4()}.tar.gz")
|
|
131
214
|
try:
|
|
132
215
|
GeneralUtilities.write_binary_to_file(result_archive, result_bytes)
|
|
133
|
-
GeneralUtilities.
|
|
134
|
-
GeneralUtilities.ensure_directory_exists(local_codeunit_folder)
|
|
216
|
+
GeneralUtilities.ensure_directory_exists(result_folder)
|
|
135
217
|
with tarfile.open(result_archive, "r:gz") as tar:
|
|
136
|
-
tar.extractall(
|
|
218
|
+
tar.extractall(result_folder, filter="fully_trusted")
|
|
137
219
|
finally:
|
|
138
220
|
GeneralUtilities.ensure_file_does_not_exist(result_archive)
|
|
139
221
|
|
|
@@ -172,17 +254,26 @@ class TFCPS_RemoteBuild:
|
|
|
172
254
|
def __select_runner_for_os(self, endpoints: list[RunnerEndpoint], required_os: RunnerOperatingSystem) -> RunnerEndpoint:
|
|
173
255
|
# The configuration does not state which runner provides which operating-system, so each configured runner is asked
|
|
174
256
|
# (via its "GET /os"-endpoint) and the first one matching the required operating-system is used.
|
|
257
|
+
runners_which_could_not_be_reached: list[str] = []
|
|
175
258
|
for endpoint in endpoints:
|
|
176
259
|
try:
|
|
177
|
-
_, os_bytes = self.__http(endpoint, "GET", "/os")
|
|
260
|
+
_, os_bytes = self.__http(endpoint, "GET", "/os", _timeout_of_a_request_in_seconds)
|
|
178
261
|
if os_bytes.decode("utf-8").strip() == required_os.value:
|
|
179
262
|
return endpoint
|
|
263
|
+
except RunnerNotReachableError as not_reachable_error:
|
|
264
|
+
#A runner which can not be reached is something else than one which does not provide the required
|
|
265
|
+
#operating-system: it is configured and may well be the right one, which is unknown while it does not
|
|
266
|
+
#answer. It is therefore collected here and stated as the reason if no runner can be selected at all.
|
|
267
|
+
runners_which_could_not_be_reached.append(str(not_reachable_error))
|
|
268
|
+
self.__sc.log.log(str(not_reachable_error), LogLevel.Warning)
|
|
180
269
|
except Exception as exception:
|
|
181
270
|
self.__sc.log.log_exception(f"Could not query the operating-system of the runner '{endpoint.url}'.", exception, LogLevel.Warning)
|
|
271
|
+
if len(runners_which_could_not_be_reached) == len(endpoints):
|
|
272
|
+
raise ValueError(f"None of the configured remote-build-runners could be reached, so whether one of them provides the required operating-system '{required_os.value}' is unknown: {' '.join(runners_which_could_not_be_reached)}")
|
|
182
273
|
raise ValueError(f"No configured remote-build-runner provides the required operating-system '{required_os.value}'.")
|
|
183
274
|
|
|
184
275
|
@GeneralUtilities.check_arguments
|
|
185
|
-
def __http(self, endpoint: RunnerEndpoint, method: str, path: str, extra_headers: dict = None, body: bytes = None) -> tuple[int, bytes]:
|
|
276
|
+
def __http(self, endpoint: RunnerEndpoint, method: str, path: str, timeout_in_seconds: int, extra_headers: dict = None, body: bytes = None) -> tuple[int, bytes]:
|
|
186
277
|
request = urllib.request.Request(endpoint.url + path, data=body, method=method)
|
|
187
278
|
authorization_token = base64.b64encode(f"{endpoint.username}:{endpoint.password}".encode("utf-8")).decode("ascii")
|
|
188
279
|
request.add_header("Authorization", f"Basic {authorization_token}")
|
|
@@ -190,8 +281,15 @@ class TFCPS_RemoteBuild:
|
|
|
190
281
|
for header_name, header_value in extra_headers.items():
|
|
191
282
|
request.add_header(header_name, header_value)
|
|
192
283
|
try:
|
|
193
|
-
with urllib.request.urlopen(request, timeout=
|
|
284
|
+
with urllib.request.urlopen(request, timeout=timeout_in_seconds) as response:
|
|
194
285
|
return response.status, response.read()
|
|
195
286
|
except urllib.error.HTTPError as http_error:
|
|
196
287
|
error_body = http_error.read().decode("utf-8", errors="replace")
|
|
197
|
-
raise
|
|
288
|
+
raise RunnerRequestFailedError(f"The runner-request '{method} {path}' failed with status {http_error.code}: {error_body}", http_error.code) from http_error
|
|
289
|
+
except (OSError, http.client.HTTPException) as connection_error:
|
|
290
|
+
#Everything which prevented an answer of the runner: urllib.error.URLError (which is an OSError) covers a
|
|
291
|
+
#connection which was refused, reset or not establishable at all, http.client.HTTPException covers an answer
|
|
292
|
+
#which was cut off while it was read. Both are stated as one error which names what was requested from whom.
|
|
293
|
+
#The cause is deliberately not chained: its message is part of the message here, while its traceback consists
|
|
294
|
+
#of the internals of urllib only and states nothing about the delegated build.
|
|
295
|
+
raise RunnerNotReachableError(f"The runner '{endpoint.url}' could not be reached for the request '{method} {path}': {connection_error}") from None
|
|
@@ -799,23 +799,13 @@ class TFCPS_Tools_General:
|
|
|
799
799
|
self.__sc.assert_is_git_repository(repositoryfolder)
|
|
800
800
|
return self.__sc.get_semver_version(repositoryfolder)
|
|
801
801
|
|
|
802
|
-
|
|
803
|
-
def __try_calculate_changelog_message(self, repositoryfolder: str):
|
|
804
|
-
self.__sc.assert_is_git_repository(repositoryfolder)
|
|
805
|
-
message = self.__sc.run_program("git", "log -1 --pretty=%B", repositoryfolder)[1]
|
|
806
|
-
message = message.strip()
|
|
807
|
-
if len(message) == 0:
|
|
808
|
-
raise ValueError("No commit message found.")
|
|
809
|
-
return message
|
|
802
|
+
|
|
810
803
|
|
|
811
804
|
@GeneralUtilities.check_arguments
|
|
812
805
|
def create_changelog_entry(self, repositoryfolder: str, message: str, commit: bool, force: bool):
|
|
813
806
|
self.__sc.assert_is_git_repository(repositoryfolder)
|
|
814
807
|
if message is None:
|
|
815
|
-
|
|
816
|
-
message=self.__try_calculate_changelog_message(repositoryfolder)
|
|
817
|
-
except:
|
|
818
|
-
message="Update."
|
|
808
|
+
message="Update."
|
|
819
809
|
random_file = os.path.join(repositoryfolder, str(uuid.uuid4()))
|
|
820
810
|
try:
|
|
821
811
|
if force and not self.__sc.git_repository_has_uncommitted_changes(repositoryfolder):
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scriptcollection
|
|
3
|
-
Version: 4.4.
|
|
3
|
+
Version: 4.4.21
|
|
4
4
|
Summary: The ScriptCollection is the place for reusable scripts.
|
|
5
5
|
Author-email: Marius Göcke <marius.goecke@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/anionDev/ScriptCollection
|
|
@@ -12,7 +12,7 @@ ScriptCollection/ProgramRunnerSudo.py,sha256=ot9Ib-plQ8wejR9M2XXfnx6GR-va9k8SH1g
|
|
|
12
12
|
ScriptCollection/RepositoryDiagramGenerator.py,sha256=K_u2NW_p4ozdWD50irJEkkcDVa70TcUJpZN63TeXSLw,7092
|
|
13
13
|
ScriptCollection/RepositoryHistoryAnalyzer.py,sha256=rD8eC5lMUe5MHmCBHfYft_G-N9IbvtsFGkU7EMwPjrY,5779
|
|
14
14
|
ScriptCollection/SCLog.py,sha256=Fjf8Pdi63cXHGMxL_aI6A0dwGm7QIfrv3czjvpSUlaM,4705
|
|
15
|
-
ScriptCollection/ScriptCollectionCore.py,sha256=
|
|
15
|
+
ScriptCollection/ScriptCollectionCore.py,sha256=ytzdfn0MVfEuAyh4CPKYQop8Kxc5364lLxyDE4a6SK0,270437
|
|
16
16
|
ScriptCollection/TestKind.py,sha256=EBTLR8yBz5VmWA2NZIpDTbQNg6tkfv_kAsIzEJ--mCI,1756
|
|
17
17
|
ScriptCollection/UnassignedVariablesCheck.py,sha256=8xNcp1D5HeOLO1a1NEerq2TsnpFhw7yzYPkaay-oisY,3804
|
|
18
18
|
ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -38,22 +38,22 @@ ScriptCollection/Resources/Fonts/DejaVuSans.ttf,sha256=faGVp0xVvvmI0NSPlQi9XYSUJ
|
|
|
38
38
|
ScriptCollection/Resources/Fonts/LICENSE.txt,sha256=egg7E25k0GR5TDQZdR5cfdENL2TBCP5boWHq5eWVipM,8816
|
|
39
39
|
ScriptCollection/Resources/MaintenanceSite/MaintenanceSite.html,sha256=CX9S1bdOz6xU2aGfr03tJRGczQrlpn-IeODc15d5kc0,232
|
|
40
40
|
ScriptCollection/TFCPS/PackageSource.py,sha256=tQoMbxrIbV3mYHKH2wgOoGzIHY1TvBH8EAFpWGhryrQ,1311
|
|
41
|
-
ScriptCollection/TFCPS/SCTaskRunnerServer.py,sha256=
|
|
41
|
+
ScriptCollection/TFCPS/SCTaskRunnerServer.py,sha256=tub8_1QBu0nWAaGnBWp2nErOW-SOHf_UL6qjOSGbnxU,31026
|
|
42
42
|
ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py,sha256=67_DQwoH5bhxxSmwtuAeoW1NOCMiLMzOy48OqlJX-KY,1744
|
|
43
|
-
ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=
|
|
43
|
+
ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=aCfGCbevCOV1IJ444dhvwJkGDKQNH3mqIucPAuSUq-o,40149
|
|
44
44
|
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=Mj9t1SyRpkYZE74cLsGWYv4xB89gXL2ld-RX3xvvGdw,9835
|
|
45
|
-
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=
|
|
45
|
+
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=9nibE3xSaBkMCLUq2qRl5qdeZBL017rzI6fYCYVxzeY,48812
|
|
46
46
|
ScriptCollection/TFCPS/TFCPS_CreateRelease.py,sha256=kpEmSnfzfgB4OjcCuS_X6jgW_ftcphxEnjjoLoAHTLY,6582
|
|
47
47
|
ScriptCollection/TFCPS/TFCPS_Generic.py,sha256=mRBcf7XuNWOSXbZYc3NuJow18AMZ30D6Z-BtyNYfNcM,2072
|
|
48
48
|
ScriptCollection/TFCPS/TFCPS_MergeToMain.py,sha256=Krt5oyGNcRAN3szsgU7LoTh8yVTOdHW-Kq7R9MDDV-Q,7673
|
|
49
49
|
ScriptCollection/TFCPS/TFCPS_MergeToStable.py,sha256=oYEhz5ZWMnbu97_CcYEqdDxakUFVppyOVQiJzADq59c,22434
|
|
50
50
|
ScriptCollection/TFCPS/TFCPS_OCIImageSecretScan.py,sha256=rhw0glr4gZ5enG9faOD51xE4MMK1cIdNjUdaswiMsrY,14622
|
|
51
51
|
ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py,sha256=XLVsFTA1jvfDbZ7wYJgIdEByZhF_bYVbA5jkisuvK4k,2322
|
|
52
|
-
ScriptCollection/TFCPS/TFCPS_RemoteBuild.py,sha256=
|
|
53
|
-
ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=
|
|
52
|
+
ScriptCollection/TFCPS/TFCPS_RemoteBuild.py,sha256=gQQbk7Va7c_0VL2pu_UYM-GPkbQxe0vvs2D_gJdZulI,21103
|
|
53
|
+
ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=SGskMzZwJ3M17v5TqP5v-OIhQuPPWj9CvPkIfMJ95_U,166483
|
|
54
54
|
ScriptCollection/TFCPS/TFCPS_VisualRegressionTests.py,sha256=6_63udxHnPlu_TMk_ftLZTnzriJO8MCxVCVAsweZqHs,37073
|
|
55
55
|
ScriptCollection/TFCPS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
56
|
-
ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py,sha256=
|
|
56
|
+
ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py,sha256=4skj2I73VOo93HMXcBuWmdyuoE9zjqr2H_DtsYrcvNo,16375
|
|
57
57
|
ScriptCollection/TFCPS/CPP/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
58
58
|
ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=919mNfzVRurTJhWi8WV0oEAFoH-yd4a6m2Rbxcf2oZ4,19927
|
|
59
59
|
ScriptCollection/TFCPS/Docker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -62,7 +62,7 @@ ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py,sha256=
|
|
|
62
62
|
ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py,sha256=i0zEGehj0sttxjjZtoq2KFSKp_ulxVyWp_ZgAhIY_So,241
|
|
63
63
|
ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=GCFzDNAnmm40EeqOYgD_EeuEwDY58ujwJ1-Zn_naamc,61797
|
|
64
64
|
ScriptCollection/TFCPS/DotNet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
65
|
-
ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=
|
|
65
|
+
ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=USyCVpAtuvOr2snQA_AM-YcSJzQ3vnUQWrVgXX9Wgyk,51768
|
|
66
66
|
ScriptCollection/TFCPS/Flutter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
67
67
|
ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py,sha256=m4Pax-U6o5XuspLnLfnWjyQMtoC7zBDDNJW9EhLWjeE,9012
|
|
68
68
|
ScriptCollection/TFCPS/Go/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -76,8 +76,8 @@ ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=yyjWPMHQjPLkML
|
|
|
76
76
|
ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
77
77
|
ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py,sha256=5z990qyO5durX07xRWRwju1hl6wpjQAy2f-yRwJ-7s0,11856
|
|
78
78
|
ScriptCollection/TFCPS/TypeScript/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
79
|
-
scriptcollection-4.4.
|
|
80
|
-
scriptcollection-4.4.
|
|
81
|
-
scriptcollection-4.4.
|
|
82
|
-
scriptcollection-4.4.
|
|
83
|
-
scriptcollection-4.4.
|
|
79
|
+
scriptcollection-4.4.21.dist-info/METADATA,sha256=edAZRh_aRm-vGhvbfcbkqpGa3TVL_26e0e5IHdG0_Q0,7690
|
|
80
|
+
scriptcollection-4.4.21.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
81
|
+
scriptcollection-4.4.21.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
|
|
82
|
+
scriptcollection-4.4.21.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
|
|
83
|
+
scriptcollection-4.4.21.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|