ScriptCollection 4.2.150__py3-none-any.whl → 4.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/Executables.py +43 -6
- ScriptCollection/GeneralUtilities.py +4 -1
- ScriptCollection/ScriptCollectionCore.py +81 -1
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +19 -11
- ScriptCollection/TFCPS/SCTaskRunnerServer.py +207 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +10 -0
- ScriptCollection/TFCPS/TFCPS_RemoteBuild.py +190 -0
- {scriptcollection-4.2.150.dist-info → scriptcollection-4.3.0.dist-info}/METADATA +1 -1
- {scriptcollection-4.2.150.dist-info → scriptcollection-4.3.0.dist-info}/RECORD +12 -10
- {scriptcollection-4.2.150.dist-info → scriptcollection-4.3.0.dist-info}/entry_points.txt +3 -0
- {scriptcollection-4.2.150.dist-info → scriptcollection-4.3.0.dist-info}/WHEEL +0 -0
- {scriptcollection-4.2.150.dist-info → scriptcollection-4.3.0.dist-info}/top_level.txt +0 -0
ScriptCollection/Executables.py
CHANGED
|
@@ -1074,6 +1074,26 @@ def EnsureDockerNetworkIsAvailable()->int:
|
|
|
1074
1074
|
return 0
|
|
1075
1075
|
|
|
1076
1076
|
|
|
1077
|
+
def ShowExternalDockerNetworks() -> int:
|
|
1078
|
+
parser = argparse.ArgumentParser(description="Prints all external networks declared in a Docker Compose file.")
|
|
1079
|
+
parser.add_argument('-f', '--file', required=False, default="docker-compose.yml", help="Path to the Docker Compose file. Defaults to docker-compose.yml in the current working directory.")
|
|
1080
|
+
args = parser.parse_args()
|
|
1081
|
+
sc: ScriptCollectionCore = ScriptCollectionCore()
|
|
1082
|
+
file = GeneralUtilities.resolve_relative_path(args.file, os.getcwd())
|
|
1083
|
+
sc.show_external_docker_networks_from_compose_file(file)
|
|
1084
|
+
return 0
|
|
1085
|
+
|
|
1086
|
+
|
|
1087
|
+
def EnsureExternalDockerNetworksExist() -> int:
|
|
1088
|
+
parser = argparse.ArgumentParser(description="Ensures all external networks declared in a Docker Compose file exist.")
|
|
1089
|
+
parser.add_argument('-f', '--file', required=False, default="docker-compose.yml", help="Path to the Docker Compose file. Defaults to docker-compose.yml in the current working directory.")
|
|
1090
|
+
args = parser.parse_args()
|
|
1091
|
+
sc: ScriptCollectionCore = ScriptCollectionCore()
|
|
1092
|
+
file = GeneralUtilities.resolve_relative_path(args.file, os.getcwd())
|
|
1093
|
+
sc.ensure_external_docker_networks_exist_from_compose_file(file)
|
|
1094
|
+
return 0
|
|
1095
|
+
|
|
1096
|
+
|
|
1077
1097
|
def ReclaimSpaceFromDocker()->int:
|
|
1078
1098
|
sc = ScriptCollectionCore()
|
|
1079
1099
|
parser = argparse.ArgumentParser()
|
|
@@ -1122,17 +1142,34 @@ def SearchForSecrets() -> int:
|
|
|
1122
1142
|
return 0
|
|
1123
1143
|
|
|
1124
1144
|
|
|
1125
|
-
def
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1145
|
+
def __prepare_build_pipeline(verbosity: int) -> None:
|
|
1146
|
+
# Common preparation for the build-pipeline. The actual buildx-build (see TFCPS_CodeUnitSpecific_Docker.build)
|
|
1147
|
+
# exports the image via "--output type=docker,dest=...tar". The default "docker"-driver can not do that unless
|
|
1148
|
+
# the containerd-image-store is enabled on the daemon, so a "docker-container"-builder is created and selected
|
|
1149
|
+
# here. This is required for GitLab and GitHub (and any other CI) alike, so both entry-points share this logic.
|
|
1130
1150
|
sc: ScriptCollectionCore = ScriptCollectionCore()
|
|
1131
|
-
sc.log.loglevel = LogLevel(
|
|
1151
|
+
sc.log.loglevel = LogLevel(verbosity)
|
|
1132
1152
|
GeneralUtilities.assert_condition(sc.is_running_in_build_container(), "This function should only be run in the build container.")
|
|
1133
1153
|
sc.run_program("update-ca-certificates")
|
|
1134
1154
|
sc.run_program_argsasarray("sh",["-c","docker buildx create --name ci-builder --driver docker-container --use 2>/dev/null || docker buildx use ci-builder"])
|
|
1135
1155
|
sc.run_program("docker","buildx inspect --bootstrap")
|
|
1156
|
+
|
|
1157
|
+
|
|
1158
|
+
def PrepareBuildPipelineForGitlab() -> int:
|
|
1159
|
+
parser = argparse.ArgumentParser(description="Prepares the build-pipeline-configuration for GitLab.")
|
|
1160
|
+
verbosity_values = ", ".join(f"{lvl.value}={lvl.name}" for lvl in LogLevel)
|
|
1161
|
+
parser.add_argument('-v', '--verbosity', required=False, default=3, help=f"Sets the loglevel. Possible values: {verbosity_values}")
|
|
1162
|
+
args = parser.parse_args()
|
|
1163
|
+
__prepare_build_pipeline(int(args.verbosity))
|
|
1164
|
+
return 0
|
|
1165
|
+
|
|
1166
|
+
|
|
1167
|
+
def PrepareBuildPipelineForGithub() -> int:
|
|
1168
|
+
parser = argparse.ArgumentParser(description="Prepares the build-pipeline-configuration for GitHub.")
|
|
1169
|
+
verbosity_values = ", ".join(f"{lvl.value}={lvl.name}" for lvl in LogLevel)
|
|
1170
|
+
parser.add_argument('-v', '--verbosity', required=False, default=3, help=f"Sets the loglevel. Possible values: {verbosity_values}")
|
|
1171
|
+
args = parser.parse_args()
|
|
1172
|
+
__prepare_build_pipeline(int(args.verbosity))
|
|
1136
1173
|
return 0
|
|
1137
1174
|
|
|
1138
1175
|
|
|
@@ -86,7 +86,7 @@ class GeneralUtilities:
|
|
|
86
86
|
raise TypeError(f"Argument with index {index} for function {function.__name__} ('{str(argument)}') is not of type {function.__annotations__[parameters[index]]} but has type "+str(type(argument)))
|
|
87
87
|
for index, named_argument in enumerate(named_args):
|
|
88
88
|
if named_args[named_argument] is not None:
|
|
89
|
-
if
|
|
89
|
+
if named_argument in function.__annotations__:
|
|
90
90
|
if not GeneralUtilities.is_generic(function.__annotations__.get(named_argument)):
|
|
91
91
|
if not isinstance(named_args[named_argument], function.__annotations__.get(named_argument)):
|
|
92
92
|
raise TypeError(f"Argument with name {named_argument} for function {function.__name__} ('{str(named_args[named_argument])}') is not of type {function.__annotations__.get(named_argument)}")
|
|
@@ -1163,6 +1163,9 @@ class GeneralUtilities:
|
|
|
1163
1163
|
return platform.system() == 'Linux'
|
|
1164
1164
|
|
|
1165
1165
|
@staticmethod
|
|
1166
|
+
def current_system_is_macos():
|
|
1167
|
+
return platform.system() == 'Darwin'
|
|
1168
|
+
|
|
1166
1169
|
def get_temp_folder() -> str:
|
|
1167
1170
|
"""Returns an absolute path to the system-temp-folder. Use this instead of tempfile.gettempdir() directly:
|
|
1168
1171
|
in some Linux-build-containers TMPDIR/TEMP/TMP is set to a relative value, which makes tempfile.gettempdir()
|
|
@@ -37,7 +37,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
|
|
|
37
37
|
from .ProgramRunnerPopen import ProgramRunnerPopen
|
|
38
38
|
from .SCLog import SCLog, LogLevel
|
|
39
39
|
|
|
40
|
-
version = "4.
|
|
40
|
+
version = "4.3.0"
|
|
41
41
|
__version__ = version
|
|
42
42
|
|
|
43
43
|
class VSCodeWorkspaceShellTask:
|
|
@@ -3325,6 +3325,25 @@ OCR-content:
|
|
|
3325
3325
|
# or "network not found") instead of being swallowed.
|
|
3326
3326
|
self.run_program_argsasarray("docker", ["network", "connect", network_name, own_container_id], throw_exception_if_exitcode_is_not_zero=True)
|
|
3327
3327
|
|
|
3328
|
+
@GeneralUtilities.check_arguments
|
|
3329
|
+
def __disconnect_build_container_from_compose_networks(self, docker_compose_file: str) -> None:
|
|
3330
|
+
# Detaches THIS build-container again from every (external) network used by the compose-file. This is the inverse of
|
|
3331
|
+
# __connect_build_container_to_compose_networks and is run before "compose down" so the network can be removed without leaving a stale/null
|
|
3332
|
+
# network-reference on the build-container (which would otherwise break the GitHub-runner "Stop containers"-post-step). Failures are ignored on
|
|
3333
|
+
# purpose: the network might already be gone, or the build-container might never have been attached (e.g. the attach failed or did not run).
|
|
3334
|
+
own_container_id = self.__get_own_container_id()
|
|
3335
|
+
if not GeneralUtilities.string_has_content(own_container_id):
|
|
3336
|
+
return
|
|
3337
|
+
if not os.path.isfile(docker_compose_file):
|
|
3338
|
+
return
|
|
3339
|
+
with open(docker_compose_file, encoding="utf-8") as stream:
|
|
3340
|
+
loaded = yaml.safe_load(stream)
|
|
3341
|
+
networks = loaded.get("networks", dict())
|
|
3342
|
+
for network_key, network_definition in networks.items():
|
|
3343
|
+
network_definition = network_definition or dict()
|
|
3344
|
+
network_name = network_definition.get("name", network_key)
|
|
3345
|
+
self.run_program_argsasarray("docker", ["network", "disconnect", "-f", network_name, own_container_id], throw_exception_if_exitcode_is_not_zero=False)
|
|
3346
|
+
|
|
3328
3347
|
@GeneralUtilities.check_arguments
|
|
3329
3348
|
def __register_test_service_containers_in_etc_hosts(self, docker_compose_file: str) -> None:
|
|
3330
3349
|
# Maps the container-names of the test-services to their current container-IPs in the /etc/hosts of THIS build-container (neither the runner-host
|
|
@@ -3361,6 +3380,14 @@ OCR-content:
|
|
|
3361
3380
|
example_folder = os.path.dirname(file)
|
|
3362
3381
|
example_name = os.path.basename(example_folder)
|
|
3363
3382
|
title = f"Test{example_name}"
|
|
3383
|
+
if self.is_running_in_build_container():
|
|
3384
|
+
# Inverse of __connect_build_container_to_compose_networks (see start_local_test_service): detach THIS build-container from the compose-networks
|
|
3385
|
+
# BEFORE "compose down" removes them. Otherwise "compose down" removes a network this build-container is still attached to, which leaves a
|
|
3386
|
+
# stale/null network-reference on the build-container. The GitHub-runner adds an automatic post-job "Stop containers"-step (present whenever a
|
|
3387
|
+
# job uses "container:") that enumerates the build-container's networks to disconnect them; that stale entry then makes it fail with
|
|
3388
|
+
# "Value cannot be null. (Parameter 'network')". Disconnecting here first leaves the build-container attached only to the runner-managed
|
|
3389
|
+
# job-network, so the cleanup finds a consistent state.
|
|
3390
|
+
self.__disconnect_build_container_from_compose_networks(os.path.join(example_folder, "docker-compose.yml"))
|
|
3364
3391
|
self.run_program("docker", f"compose -p {title.lower()} down", example_folder, title=title,print_live_output=True)
|
|
3365
3392
|
|
|
3366
3393
|
@GeneralUtilities.check_arguments
|
|
@@ -3482,6 +3509,59 @@ OCR-content:
|
|
|
3482
3509
|
if network_name in self.get_docker_networks():
|
|
3483
3510
|
self.run_program("docker",f"network rm {network_name}")
|
|
3484
3511
|
|
|
3512
|
+
def get_external_docker_networks_from_compose_file(self,compose_file_path: str) -> list[str]:
|
|
3513
|
+
"""
|
|
3514
|
+
Parses a docker-compose.yml file and returns all network names
|
|
3515
|
+
that are marked as external: true.
|
|
3516
|
+
"""
|
|
3517
|
+
|
|
3518
|
+
with open(compose_file_path, "r", encoding="utf-8") as f:
|
|
3519
|
+
data = yaml.safe_load(f)
|
|
3520
|
+
|
|
3521
|
+
external_networks = []
|
|
3522
|
+
|
|
3523
|
+
networks = data.get("networks", {})
|
|
3524
|
+
if not isinstance(networks, dict):
|
|
3525
|
+
return external_networks
|
|
3526
|
+
|
|
3527
|
+
for name, config in networks.items():
|
|
3528
|
+
if isinstance(config, dict):
|
|
3529
|
+
if config.get("external") is True:
|
|
3530
|
+
# case: external: true
|
|
3531
|
+
external_networks.append(name)
|
|
3532
|
+
|
|
3533
|
+
# optional advanced case: external with explicit name
|
|
3534
|
+
elif "external" in config and isinstance(config["external"], dict):
|
|
3535
|
+
# e.g. external: { name: "real_network_name" }
|
|
3536
|
+
real_name = config["external"].get("name")
|
|
3537
|
+
if real_name:
|
|
3538
|
+
external_networks.append(real_name)
|
|
3539
|
+
|
|
3540
|
+
return external_networks
|
|
3541
|
+
|
|
3542
|
+
@GeneralUtilities.check_arguments
|
|
3543
|
+
def get_external_docker_networks_from_compose_file(self, compose_file: str) -> list[str]:
|
|
3544
|
+
with open(compose_file, encoding="utf-8") as stream:
|
|
3545
|
+
loaded = yaml.safe_load(stream)
|
|
3546
|
+
networks = loaded.get("networks", dict())
|
|
3547
|
+
result = []
|
|
3548
|
+
for network_key, network_definition in networks.items():
|
|
3549
|
+
network_definition = network_definition or dict()
|
|
3550
|
+
if network_definition.get("external", False):
|
|
3551
|
+
network_name = network_definition.get("name", network_key)
|
|
3552
|
+
result.append(network_name)
|
|
3553
|
+
return result
|
|
3554
|
+
|
|
3555
|
+
@GeneralUtilities.check_arguments
|
|
3556
|
+
def show_external_docker_networks_from_compose_file(self, compose_file: str) -> None:
|
|
3557
|
+
for network_name in self.get_external_docker_networks_from_compose_file(compose_file):
|
|
3558
|
+
GeneralUtilities.write_message_to_stdout(network_name)
|
|
3559
|
+
|
|
3560
|
+
@GeneralUtilities.check_arguments
|
|
3561
|
+
def ensure_external_docker_networks_exist_from_compose_file(self, compose_file: str) -> None:
|
|
3562
|
+
for network_name in self.get_external_docker_networks_from_compose_file(compose_file):
|
|
3563
|
+
self.ensure_docker_network_is_available(network_name)
|
|
3564
|
+
|
|
3485
3565
|
@GeneralUtilities.check_arguments
|
|
3486
3566
|
def get_available_cultures_for_angular_app(self,angular_json_file:str)->list[str]:
|
|
3487
3567
|
languages = ["en"]
|
|
@@ -5,6 +5,7 @@ import zipfile
|
|
|
5
5
|
from ...GeneralUtilities import GeneralUtilities
|
|
6
6
|
from ...SCLog import LogLevel
|
|
7
7
|
from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base,TFCPS_CodeUnitSpecific_Base_CLI
|
|
8
|
+
from ..TFCPS_RemoteBuild import RunnerOperatingSystem
|
|
8
9
|
|
|
9
10
|
class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
10
11
|
|
|
@@ -31,26 +32,33 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
31
32
|
}
|
|
32
33
|
for target in targets:
|
|
33
34
|
self._protected_sc.log.log(f"Build flutter-codeunit {codeunit_name} for target {target_names[target]}...")
|
|
34
|
-
self._protected_sc.run_with_epew("flutter", f"build {target}", src_folder)
|
|
35
35
|
if target == "web":
|
|
36
|
+
self._protected_sc.run_with_epew("flutter", "build web", src_folder)
|
|
36
37
|
web_relase_folder = os.path.join(src_folder, "build/web")
|
|
37
38
|
web_folder = os.path.join(artifacts_folder, "BuildResult_WebApplication")
|
|
38
39
|
GeneralUtilities.ensure_directory_does_not_exist(web_folder)
|
|
39
40
|
GeneralUtilities.ensure_directory_exists(web_folder)
|
|
40
41
|
GeneralUtilities.copy_content_of_folder(web_relase_folder, web_folder)
|
|
41
42
|
elif target == "windows":
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
43
|
+
# Windows-builds always run on a Windows-task-runner - even when building on a Windows-client - so that all
|
|
44
|
+
# builds are produced uniformly in the same defined environment. See the remote-build-article in the reference.
|
|
45
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder)
|
|
46
|
+
windows_release_folder = os.path.join(src_folder, "build/windows/x64/runner/Release")
|
|
47
|
+
windows_folder = os.path.join(artifacts_folder, "BuildResult_Windows")
|
|
48
|
+
GeneralUtilities.ensure_directory_does_not_exist(windows_folder)
|
|
49
|
+
GeneralUtilities.ensure_directory_exists(windows_folder)
|
|
50
|
+
GeneralUtilities.copy_content_of_folder(windows_release_folder, windows_folder)
|
|
49
51
|
elif target == "ios":
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
52
|
+
# iOS-builds must run on macOS and therefore always run on a macOS-task-runner (uniform builds). See the
|
|
53
|
+
# remote-build-article in the reference.
|
|
54
|
+
self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "ios"], src_folder)
|
|
55
|
+
ios_release_folder = os.path.join(src_folder, "build/ios/iphoneos")
|
|
56
|
+
ios_folder = os.path.join(artifacts_folder, "BuildResult_IOS")
|
|
57
|
+
GeneralUtilities.ensure_directory_does_not_exist(ios_folder)
|
|
58
|
+
GeneralUtilities.ensure_directory_exists(ios_folder)
|
|
59
|
+
GeneralUtilities.copy_content_of_folder(ios_release_folder, ios_folder)
|
|
53
60
|
elif target == "appbundle":
|
|
61
|
+
self._protected_sc.run_with_epew("flutter", "build appbundle", src_folder)
|
|
54
62
|
enabled=False
|
|
55
63
|
if enabled:#TODO move to external because this is not platform indepent
|
|
56
64
|
aab_folder = os.path.join(artifacts_folder, "BuildResult_AAB")
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import json
|
|
3
|
+
import base64
|
|
4
|
+
import ssl
|
|
5
|
+
import uuid
|
|
6
|
+
import threading
|
|
7
|
+
import subprocess
|
|
8
|
+
import tarfile
|
|
9
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
10
|
+
from ..GeneralUtilities import GeneralUtilities
|
|
11
|
+
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
12
|
+
from ..SCLog import LogLevel
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class _RunnerJob:
|
|
16
|
+
def __init__(self, workspace_folder: str, codeunit_name: str):
|
|
17
|
+
self.workspace_folder = workspace_folder
|
|
18
|
+
self.codeunit_name = codeunit_name
|
|
19
|
+
self.state = "running" # "running" | "completed" | "failed"
|
|
20
|
+
self.exitcode = None
|
|
21
|
+
self.log = GeneralUtilities.empty_string
|
|
22
|
+
self.lock = threading.Lock()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class SCTaskRunnerServer:
|
|
26
|
+
"""HTTP-server that compiles operating-system-bound build-steps on behalf of a (remote) client (see TFCPS_RemoteBuild).
|
|
27
|
+
Per job it extracts the received repository-archive into a fresh, empty workspace, runs the requested program on this
|
|
28
|
+
machine's operating-system, and returns the codeunit-folder. The workspace is deleted as soon as the client deletes the
|
|
29
|
+
job (immediately after fetching the result), so no repository-content remains on the runner."""
|
|
30
|
+
|
|
31
|
+
def __init__(self, operating_system_name: str, username: str, password: str, work_folder: str = None, sc: ScriptCollectionCore = None):
|
|
32
|
+
self.operating_system_name = operating_system_name
|
|
33
|
+
self.__username = username
|
|
34
|
+
self.__password = password
|
|
35
|
+
self.work_folder = work_folder if work_folder is not None else os.path.join(GeneralUtilities.get_temp_folder(), "SCTaskRunner")
|
|
36
|
+
self.__sc = sc if sc is not None else ScriptCollectionCore()
|
|
37
|
+
self.__jobs: dict[str, _RunnerJob] = {}
|
|
38
|
+
self.__jobs_lock = threading.Lock()
|
|
39
|
+
|
|
40
|
+
@GeneralUtilities.check_arguments
|
|
41
|
+
def run(self, host: str = "0.0.0.0", port: int = 8080, certificate_file: str = None, certificate_key_file: str = None) -> None:
|
|
42
|
+
"""Starts the HTTP-server. When both certificate_file and certificate_key_file are given the server is served over
|
|
43
|
+
TLS (https); otherwise it is served over plain http (e.g. when TLS is terminated by a reverse-proxy in front of it)."""
|
|
44
|
+
GeneralUtilities.ensure_directory_exists(self.work_folder)
|
|
45
|
+
server = ThreadingHTTPServer((host, port), self.__create_request_handler())
|
|
46
|
+
protocol = "http"
|
|
47
|
+
if certificate_file is not None and certificate_key_file is not None:
|
|
48
|
+
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
|
49
|
+
ssl_context.load_cert_chain(certfile=certificate_file, keyfile=certificate_key_file)
|
|
50
|
+
server.socket = ssl_context.wrap_socket(server.socket, server_side=True)
|
|
51
|
+
protocol = "https"
|
|
52
|
+
self.__sc.log.log(f"SCTaskRunner for operating-system '{self.operating_system_name}' is listening on {protocol}://{host}:{port}.")
|
|
53
|
+
server.serve_forever()
|
|
54
|
+
|
|
55
|
+
@GeneralUtilities.check_arguments
|
|
56
|
+
def __start_job(self, archive_bytes: bytes, codeunit_name: str, program: str, arguments: list[str], working_directory: str) -> str:
|
|
57
|
+
job_id = str(uuid.uuid4())
|
|
58
|
+
workspace_folder = os.path.join(self.work_folder, job_id)
|
|
59
|
+
# Isolation: each job gets a fresh, empty workspace-folder.
|
|
60
|
+
GeneralUtilities.ensure_directory_does_not_exist(workspace_folder)
|
|
61
|
+
GeneralUtilities.ensure_directory_exists(workspace_folder)
|
|
62
|
+
archive_file = os.path.join(self.work_folder, f"{job_id}.payload.tar.gz")
|
|
63
|
+
GeneralUtilities.write_binary_to_file(archive_file, archive_bytes)
|
|
64
|
+
try:
|
|
65
|
+
with tarfile.open(archive_file, "r:gz") as tar:
|
|
66
|
+
tar.extractall(workspace_folder, filter="fully_trusted")
|
|
67
|
+
finally:
|
|
68
|
+
GeneralUtilities.ensure_file_does_not_exist(archive_file)
|
|
69
|
+
job = _RunnerJob(workspace_folder, codeunit_name)
|
|
70
|
+
with self.__jobs_lock:
|
|
71
|
+
self.__jobs[job_id] = job
|
|
72
|
+
thread = threading.Thread(target=self.__run_job, args=(job, program, arguments, working_directory), daemon=True)
|
|
73
|
+
thread.start()
|
|
74
|
+
return job_id
|
|
75
|
+
|
|
76
|
+
@GeneralUtilities.check_arguments
|
|
77
|
+
def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str) -> None:
|
|
78
|
+
command_folder = os.path.join(job.workspace_folder, working_directory)
|
|
79
|
+
try:
|
|
80
|
+
self.__sc.log.log(f"Run '{program} {' '.join(arguments)}' in '{command_folder}'...")
|
|
81
|
+
process = subprocess.Popen([program] + arguments, cwd=command_folder, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace")
|
|
82
|
+
for line in process.stdout:
|
|
83
|
+
with job.lock:
|
|
84
|
+
job.log = job.log + line
|
|
85
|
+
process.wait()
|
|
86
|
+
with job.lock:
|
|
87
|
+
job.exitcode = process.returncode
|
|
88
|
+
job.state = "completed" if process.returncode == 0 else "failed"
|
|
89
|
+
except Exception as exception:
|
|
90
|
+
with job.lock:
|
|
91
|
+
job.log = job.log + f"\nException while running the program: {exception}\n"
|
|
92
|
+
job.exitcode = -1
|
|
93
|
+
job.state = "failed"
|
|
94
|
+
|
|
95
|
+
@GeneralUtilities.check_arguments
|
|
96
|
+
def __get_result_archive(self, job: _RunnerJob) -> bytes:
|
|
97
|
+
codeunit_folder = os.path.join(job.workspace_folder, job.codeunit_name)
|
|
98
|
+
archive_file = os.path.join(self.work_folder, f"{uuid.uuid4()}.result.tar.gz")
|
|
99
|
+
try:
|
|
100
|
+
with tarfile.open(archive_file, "w:gz") as tar:
|
|
101
|
+
tar.add(codeunit_folder, arcname=".")
|
|
102
|
+
return GeneralUtilities.read_binary_from_file(archive_file)
|
|
103
|
+
finally:
|
|
104
|
+
GeneralUtilities.ensure_file_does_not_exist(archive_file)
|
|
105
|
+
|
|
106
|
+
@GeneralUtilities.check_arguments
|
|
107
|
+
def __delete_job(self, job_id: str) -> None:
|
|
108
|
+
with self.__jobs_lock:
|
|
109
|
+
job = self.__jobs.pop(job_id, None)
|
|
110
|
+
if job is not None:
|
|
111
|
+
GeneralUtilities.ensure_directory_does_not_exist(job.workspace_folder)
|
|
112
|
+
|
|
113
|
+
def __create_request_handler(self):
|
|
114
|
+
outer = self
|
|
115
|
+
|
|
116
|
+
class Handler(BaseHTTPRequestHandler):
|
|
117
|
+
|
|
118
|
+
def log_message(self, format, *args): # pylint:disable=redefined-builtin
|
|
119
|
+
outer._SCTaskRunnerServer__sc.log.log((format % args), LogLevel.Debug)
|
|
120
|
+
|
|
121
|
+
def __is_authorized(self) -> bool:
|
|
122
|
+
header = self.headers.get("Authorization", GeneralUtilities.empty_string)
|
|
123
|
+
if not header.startswith("Basic "):
|
|
124
|
+
return False
|
|
125
|
+
try:
|
|
126
|
+
decoded = base64.b64decode(header[len("Basic "):]).decode("utf-8")
|
|
127
|
+
except Exception:
|
|
128
|
+
return False
|
|
129
|
+
expected = f"{outer._SCTaskRunnerServer__username}:{outer._SCTaskRunnerServer__password}"
|
|
130
|
+
return decoded == expected
|
|
131
|
+
|
|
132
|
+
def __send(self, status: int, body: bytes, content_type: str = "application/octet-stream") -> None:
|
|
133
|
+
self.send_response(status)
|
|
134
|
+
self.send_header("Content-Type", content_type)
|
|
135
|
+
self.send_header("Content-Length", str(len(body)))
|
|
136
|
+
self.end_headers()
|
|
137
|
+
self.wfile.write(body)
|
|
138
|
+
|
|
139
|
+
def __send_text(self, status: int, text: str) -> None:
|
|
140
|
+
self.__send(status, text.encode("utf-8"), "text/plain; charset=utf-8")
|
|
141
|
+
|
|
142
|
+
def __send_json(self, status: int, obj) -> None:
|
|
143
|
+
self.__send(status, json.dumps(obj).encode("utf-8"), "application/json")
|
|
144
|
+
|
|
145
|
+
def __get_job(self, job_id: str):
|
|
146
|
+
with outer._SCTaskRunnerServer__jobs_lock:
|
|
147
|
+
return outer._SCTaskRunnerServer__jobs.get(job_id, None)
|
|
148
|
+
|
|
149
|
+
def do_GET(self): # pylint:disable=invalid-name
|
|
150
|
+
if not self.__is_authorized():
|
|
151
|
+
self.__send_text(401, "Unauthorized")
|
|
152
|
+
return
|
|
153
|
+
if self.path == "/os":
|
|
154
|
+
self.__send_text(200, outer.operating_system_name)
|
|
155
|
+
return
|
|
156
|
+
parts = self.path.strip("/").split("/")
|
|
157
|
+
if len(parts) >= 2 and parts[0] == "jobs":
|
|
158
|
+
job = self.__get_job(parts[1])
|
|
159
|
+
if job is None:
|
|
160
|
+
self.__send_text(404, "Unknown job")
|
|
161
|
+
return
|
|
162
|
+
if len(parts) == 2:
|
|
163
|
+
with job.lock:
|
|
164
|
+
self.__send_json(200, {"state": job.state, "exitcode": job.exitcode})
|
|
165
|
+
return
|
|
166
|
+
if len(parts) == 3 and parts[2] == "logs":
|
|
167
|
+
with job.lock:
|
|
168
|
+
self.__send_text(200, job.log)
|
|
169
|
+
return
|
|
170
|
+
if len(parts) == 3 and parts[2] == "result":
|
|
171
|
+
with job.lock:
|
|
172
|
+
ready = job.state == "completed"
|
|
173
|
+
if not ready:
|
|
174
|
+
self.__send_text(409, "Job is not completed")
|
|
175
|
+
return
|
|
176
|
+
self.__send(200, outer._SCTaskRunnerServer__get_result_archive(job))
|
|
177
|
+
return
|
|
178
|
+
self.__send_text(404, "Not found")
|
|
179
|
+
|
|
180
|
+
def do_POST(self): # pylint:disable=invalid-name
|
|
181
|
+
if not self.__is_authorized():
|
|
182
|
+
self.__send_text(401, "Unauthorized")
|
|
183
|
+
return
|
|
184
|
+
if self.path != "/jobs":
|
|
185
|
+
self.__send_text(404, "Not found")
|
|
186
|
+
return
|
|
187
|
+
content_length = int(self.headers.get("Content-Length", "0"))
|
|
188
|
+
archive_bytes = self.rfile.read(content_length)
|
|
189
|
+
codeunit_name = self.headers.get("X-Codeunit-Name")
|
|
190
|
+
program = self.headers.get("X-Program")
|
|
191
|
+
arguments = json.loads(base64.b64decode(self.headers.get("X-Arguments", "")).decode("utf-8"))
|
|
192
|
+
working_directory = self.headers.get("X-Working-Directory", ".")
|
|
193
|
+
job_id = outer._SCTaskRunnerServer__start_job(archive_bytes, codeunit_name, program, arguments, working_directory)
|
|
194
|
+
self.__send_json(200, {"job_id": job_id})
|
|
195
|
+
|
|
196
|
+
def do_DELETE(self): # pylint:disable=invalid-name
|
|
197
|
+
if not self.__is_authorized():
|
|
198
|
+
self.__send_text(401, "Unauthorized")
|
|
199
|
+
return
|
|
200
|
+
parts = self.path.strip("/").split("/")
|
|
201
|
+
if len(parts) == 2 and parts[0] == "jobs":
|
|
202
|
+
outer._SCTaskRunnerServer__delete_job(parts[1])
|
|
203
|
+
self.__send_text(200, "Deleted")
|
|
204
|
+
return
|
|
205
|
+
self.__send_text(404, "Not found")
|
|
206
|
+
|
|
207
|
+
return Handler
|
|
@@ -12,6 +12,7 @@ from ..GeneralUtilities import GeneralUtilities, VersionEcholon
|
|
|
12
12
|
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
13
13
|
from ..SCLog import LogLevel
|
|
14
14
|
from .TFCPS_Tools_General import TFCPS_Tools_General
|
|
15
|
+
from .TFCPS_RemoteBuild import TFCPS_RemoteBuild, RunnerOperatingSystem
|
|
15
16
|
|
|
16
17
|
|
|
17
18
|
|
|
@@ -252,6 +253,15 @@ class TFCPS_CodeUnitSpecific_Base(ABC):
|
|
|
252
253
|
def use_cache(self)->bool:
|
|
253
254
|
return self.__use_cache
|
|
254
255
|
|
|
256
|
+
@GeneralUtilities.check_arguments
|
|
257
|
+
def run_program_on_remote_runner(self, required_os: RunnerOperatingSystem, program: str, arguments: list[str], working_directory: str) -> None:
|
|
258
|
+
"""Runs the given program for this codeunit on a remote task-runner that provides the given operating-system (see
|
|
259
|
+
TFCPS_RemoteBuild and the SCTaskRunnerWindows/SCTaskRunnerMacOS-codeunits). This is used for operating-system-bound
|
|
260
|
+
build-steps (e.g. flutter-windows- or flutter-ios-builds) which can not run on the current operating-system. After
|
|
261
|
+
the runner finished, only the codeunit-folder is mirrored back into the local repository, so it looks as if the
|
|
262
|
+
runner had built locally. working_directory must be a folder inside this codeunit."""
|
|
263
|
+
TFCPS_RemoteBuild(self._protected_sc).run_program_on_runner(required_os, self.get_repository_folder(), self.get_codeunit_name(), program, arguments, working_directory)
|
|
264
|
+
|
|
255
265
|
@GeneralUtilities.check_arguments
|
|
256
266
|
def get_codeunit_folder(self)->str:
|
|
257
267
|
return self.__codeunit_folder
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
import json
|
|
4
|
+
import time
|
|
5
|
+
import uuid
|
|
6
|
+
import base64
|
|
7
|
+
import tarfile
|
|
8
|
+
import urllib.request
|
|
9
|
+
import urllib.error
|
|
10
|
+
from enum import Enum
|
|
11
|
+
from ..GeneralUtilities import GeneralUtilities
|
|
12
|
+
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
13
|
+
from ..SCLog import LogLevel
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class RunnerOperatingSystem(Enum):
|
|
17
|
+
"""Operating-system a remote-build-runner provides. The value is the token exchanged over the wire (see the runner's
|
|
18
|
+
'GET /os'-endpoint)."""
|
|
19
|
+
Windows = "Windows"
|
|
20
|
+
MacOS = "MacOS"
|
|
21
|
+
Linux = "Linux"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class RunnerEndpoint:
|
|
25
|
+
url: str = None
|
|
26
|
+
username: str = None
|
|
27
|
+
password: str = None
|
|
28
|
+
|
|
29
|
+
def __init__(self, url: str, username: str, password: str):
|
|
30
|
+
self.url = url.rstrip("/")
|
|
31
|
+
self.username = username
|
|
32
|
+
self.password = password
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class TFCPS_RemoteBuild:
|
|
36
|
+
"""Delegates an operating-system-bound build-step to a remote task-runner (see SCTaskRunnerWindows/SCTaskRunnerMacOS):
|
|
37
|
+
the whole repository (including the .git-folder, uncommitted changes and git-ignored files) is sent as a tar-archive
|
|
38
|
+
over HTTPS to the runner, the runner runs the given program on its (correct) operating-system, and afterwards only the
|
|
39
|
+
codeunit-folder is mirrored back into the local repository (a build-script must by definition not change anything
|
|
40
|
+
outside its codeunit-folder, so mirroring just that folder is sufficient and keeps the round-trip small)."""
|
|
41
|
+
|
|
42
|
+
__sc: ScriptCollectionCore = None
|
|
43
|
+
|
|
44
|
+
def __init__(self, sc: ScriptCollectionCore):
|
|
45
|
+
self.__sc = sc
|
|
46
|
+
|
|
47
|
+
@GeneralUtilities.check_arguments
|
|
48
|
+
def run_program_on_runner(self, required_os: RunnerOperatingSystem, repository_folder: str, codeunit_name: str, program: str, arguments: list[str], working_directory: str, poll_interval_in_seconds: int = 3, timeout_in_seconds: int = 60*60) -> None:
|
|
49
|
+
endpoints = self.__load_runner_endpoints()
|
|
50
|
+
if len(endpoints) == 0:
|
|
51
|
+
raise ValueError("No remote-build-runner is configured. Define runners either in "
|
|
52
|
+
f"'{os.path.join(self.__sc.get_scriptcollection_configuration_folder(), 'TFCPS', 'Runner.csv')}' "
|
|
53
|
+
"(one line per runner in the format 'url;user;password') or via environment-variables "
|
|
54
|
+
"'Runner_<name>_URL', 'Runner_<name>_Username' and 'Runner_<name>_Password'.")
|
|
55
|
+
endpoint = self.__select_runner_for_os(endpoints, required_os)
|
|
56
|
+
working_directory_relative = os.path.relpath(working_directory, repository_folder).replace("\\", "/")
|
|
57
|
+
self.__sc.log.log(f"Delegate '{program} {' '.join(arguments)}' (codeunit '{codeunit_name}', folder '{working_directory_relative}') to the {required_os.value}-runner at {endpoint.url}...")
|
|
58
|
+
archive_file = self.__create_repository_archive(repository_folder)
|
|
59
|
+
job_id: str = None
|
|
60
|
+
try:
|
|
61
|
+
metadata_headers = {
|
|
62
|
+
"Content-Type": "application/octet-stream",
|
|
63
|
+
"X-Codeunit-Name": codeunit_name,
|
|
64
|
+
"X-Program": program,
|
|
65
|
+
"X-Arguments": base64.b64encode(json.dumps(arguments).encode("utf-8")).decode("ascii"),
|
|
66
|
+
"X-Working-Directory": working_directory_relative,
|
|
67
|
+
}
|
|
68
|
+
with open(archive_file, "rb") as archive_content:
|
|
69
|
+
archive_bytes = archive_content.read()
|
|
70
|
+
_, submit_response = self.__http(endpoint, "POST", "/jobs", metadata_headers, archive_bytes)
|
|
71
|
+
job_id = json.loads(submit_response.decode("utf-8"))["job_id"]
|
|
72
|
+
self.__wait_for_job(endpoint, job_id, required_os, poll_interval_in_seconds, timeout_in_seconds)
|
|
73
|
+
_, result_bytes = self.__http(endpoint, "GET", f"/jobs/{job_id}/result")
|
|
74
|
+
self.__mirror_codeunit_folder_from_result(result_bytes, repository_folder, codeunit_name)
|
|
75
|
+
finally:
|
|
76
|
+
if job_id is not None:
|
|
77
|
+
try:
|
|
78
|
+
# Triggers the runner to delete its (isolated) workspace for this job immediately after the result was fetched.
|
|
79
|
+
self.__http(endpoint, "DELETE", f"/jobs/{job_id}")
|
|
80
|
+
except Exception as exception:
|
|
81
|
+
self.__sc.log.log_exception(f"Could not delete remote job '{job_id}' on runner '{endpoint.url}'.", exception, LogLevel.Warning)
|
|
82
|
+
GeneralUtilities.ensure_file_does_not_exist(archive_file)
|
|
83
|
+
|
|
84
|
+
@GeneralUtilities.check_arguments
|
|
85
|
+
def __wait_for_job(self, endpoint: RunnerEndpoint, job_id: str, required_os: RunnerOperatingSystem, poll_interval_in_seconds: int, timeout_in_seconds: int) -> None:
|
|
86
|
+
start_time = time.time()
|
|
87
|
+
already_printed_log_length = 0
|
|
88
|
+
while True:
|
|
89
|
+
_, status_bytes = self.__http(endpoint, "GET", f"/jobs/{job_id}")
|
|
90
|
+
status = json.loads(status_bytes.decode("utf-8"))
|
|
91
|
+
_, log_bytes = self.__http(endpoint, "GET", f"/jobs/{job_id}/logs")
|
|
92
|
+
log_text = log_bytes.decode("utf-8", errors="replace")
|
|
93
|
+
if len(log_text) > already_printed_log_length:
|
|
94
|
+
for line in GeneralUtilities.string_to_lines(log_text[already_printed_log_length:]):
|
|
95
|
+
self.__sc.log.log(line, LogLevel.Information)
|
|
96
|
+
already_printed_log_length = len(log_text)
|
|
97
|
+
state = status["state"]
|
|
98
|
+
if state in ("completed", "failed"):
|
|
99
|
+
exitcode = status.get("exitcode")
|
|
100
|
+
if state == "failed" or (exitcode is not None and exitcode != 0):
|
|
101
|
+
raise ValueError(f"The remote build on the {required_os.value}-runner failed (exitcode {exitcode}). See the runner-log above for details.")
|
|
102
|
+
return
|
|
103
|
+
if time.time() - start_time > timeout_in_seconds:
|
|
104
|
+
raise ValueError(f"The remote build on the {required_os.value}-runner did not finish within {timeout_in_seconds} seconds.")
|
|
105
|
+
time.sleep(poll_interval_in_seconds)
|
|
106
|
+
|
|
107
|
+
@GeneralUtilities.check_arguments
|
|
108
|
+
def __create_repository_archive(self, repository_folder: str) -> str:
|
|
109
|
+
# Pack the entire repository-working-tree (including .git, uncommitted changes and git-ignored files) so the runner
|
|
110
|
+
# has the exact same state - including secrets that are required e.g. for signing windows-builds. Uses tarfile so
|
|
111
|
+
# it works identically on Windows and Linux (scbuildcodeunits runs on both) and preserves symlinks/permissions.
|
|
112
|
+
archive_file = os.path.join(GeneralUtilities.get_temp_folder(), f"sc-remotebuild-payload-{uuid.uuid4()}.tar.gz")
|
|
113
|
+
with tarfile.open(archive_file, "w:gz") as tar:
|
|
114
|
+
tar.add(repository_folder, arcname=".")
|
|
115
|
+
return archive_file
|
|
116
|
+
|
|
117
|
+
@GeneralUtilities.check_arguments
|
|
118
|
+
def __mirror_codeunit_folder_from_result(self, result_bytes: bytes, repository_folder: str, codeunit_name: str) -> None:
|
|
119
|
+
# Mirror the codeunit-folder from the runner-result into the local repository: the local codeunit-folder is replaced
|
|
120
|
+
# entirely by the runner's post-build codeunit-folder. This makes it look exactly as if the runner had built locally,
|
|
121
|
+
# including files the runner added, changed or deleted (regardless of whether they are git-ignored).
|
|
122
|
+
local_codeunit_folder = os.path.join(repository_folder, codeunit_name)
|
|
123
|
+
result_archive = os.path.join(GeneralUtilities.get_temp_folder(), f"sc-remotebuild-result-{uuid.uuid4()}.tar.gz")
|
|
124
|
+
try:
|
|
125
|
+
GeneralUtilities.write_binary_to_file(result_archive, result_bytes)
|
|
126
|
+
GeneralUtilities.ensure_directory_does_not_exist(local_codeunit_folder)
|
|
127
|
+
GeneralUtilities.ensure_directory_exists(local_codeunit_folder)
|
|
128
|
+
with tarfile.open(result_archive, "r:gz") as tar:
|
|
129
|
+
tar.extractall(local_codeunit_folder, filter="fully_trusted")
|
|
130
|
+
finally:
|
|
131
|
+
GeneralUtilities.ensure_file_does_not_exist(result_archive)
|
|
132
|
+
|
|
133
|
+
@GeneralUtilities.check_arguments
|
|
134
|
+
def __load_runner_endpoints(self) -> list[RunnerEndpoint]:
|
|
135
|
+
endpoints: list[RunnerEndpoint] = []
|
|
136
|
+
# Source 1 (primarily for developer-clients): a csv-file with one line per runner in the format "url;user;password".
|
|
137
|
+
csv_file = os.path.join(self.__sc.get_scriptcollection_configuration_folder(), "TFCPS", "Runner.csv")
|
|
138
|
+
if os.path.isfile(csv_file):
|
|
139
|
+
for line in GeneralUtilities.read_lines_from_file(csv_file):
|
|
140
|
+
stripped_line = line.strip()
|
|
141
|
+
if stripped_line == GeneralUtilities.empty_string or stripped_line.startswith("#"):
|
|
142
|
+
continue
|
|
143
|
+
parts = stripped_line.split(";")
|
|
144
|
+
if len(parts) < 3 or parts[0].strip() == GeneralUtilities.empty_string:
|
|
145
|
+
continue
|
|
146
|
+
endpoints.append(RunnerEndpoint(parts[0].strip(), parts[1].strip(), parts[2].strip()))
|
|
147
|
+
# Source 2 (primarily for the build-pipeline): environment-variables "Runner_<name>_URL/_Username/_Password".
|
|
148
|
+
# The lookup is case-insensitive because Windows exposes environment-variable-names upper-cased via os.environ.
|
|
149
|
+
environment_variables_upper = {name.upper(): value for name, value in os.environ.items()}
|
|
150
|
+
url_env_pattern = re.compile(r"^RUNNER_(.+?)_URL$")
|
|
151
|
+
for env_var_name, env_var_value in environment_variables_upper.items():
|
|
152
|
+
match = url_env_pattern.match(env_var_name)
|
|
153
|
+
if match is None:
|
|
154
|
+
continue
|
|
155
|
+
runner_name = match.group(1)
|
|
156
|
+
url = (env_var_value or GeneralUtilities.empty_string).strip()
|
|
157
|
+
if url == GeneralUtilities.empty_string:
|
|
158
|
+
continue
|
|
159
|
+
username = (environment_variables_upper.get(f"RUNNER_{runner_name}_USERNAME") or GeneralUtilities.empty_string).strip()
|
|
160
|
+
password = (environment_variables_upper.get(f"RUNNER_{runner_name}_PASSWORD") or GeneralUtilities.empty_string).strip()
|
|
161
|
+
endpoints.append(RunnerEndpoint(url, username, password))
|
|
162
|
+
return endpoints
|
|
163
|
+
|
|
164
|
+
@GeneralUtilities.check_arguments
|
|
165
|
+
def __select_runner_for_os(self, endpoints: list[RunnerEndpoint], required_os: RunnerOperatingSystem) -> RunnerEndpoint:
|
|
166
|
+
# The configuration does not state which runner provides which operating-system, so each configured runner is asked
|
|
167
|
+
# (via its "GET /os"-endpoint) and the first one matching the required operating-system is used.
|
|
168
|
+
for endpoint in endpoints:
|
|
169
|
+
try:
|
|
170
|
+
_, os_bytes = self.__http(endpoint, "GET", "/os")
|
|
171
|
+
if os_bytes.decode("utf-8").strip() == required_os.value:
|
|
172
|
+
return endpoint
|
|
173
|
+
except Exception as exception:
|
|
174
|
+
self.__sc.log.log_exception(f"Could not query the operating-system of the runner '{endpoint.url}'.", exception, LogLevel.Warning)
|
|
175
|
+
raise ValueError(f"No configured remote-build-runner provides the required operating-system '{required_os.value}'.")
|
|
176
|
+
|
|
177
|
+
@GeneralUtilities.check_arguments
|
|
178
|
+
def __http(self, endpoint: RunnerEndpoint, method: str, path: str, extra_headers: dict = None, body: bytes = None) -> tuple[int, bytes]:
|
|
179
|
+
request = urllib.request.Request(endpoint.url + path, data=body, method=method)
|
|
180
|
+
authorization_token = base64.b64encode(f"{endpoint.username}:{endpoint.password}".encode("utf-8")).decode("ascii")
|
|
181
|
+
request.add_header("Authorization", f"Basic {authorization_token}")
|
|
182
|
+
if extra_headers is not None:
|
|
183
|
+
for header_name, header_value in extra_headers.items():
|
|
184
|
+
request.add_header(header_name, header_value)
|
|
185
|
+
try:
|
|
186
|
+
with urllib.request.urlopen(request, timeout=300) as response:
|
|
187
|
+
return response.status, response.read()
|
|
188
|
+
except urllib.error.HTTPError as http_error:
|
|
189
|
+
error_body = http_error.read().decode("utf-8", errors="replace")
|
|
190
|
+
raise ValueError(f"The runner-request '{method} {path}' failed with status {http_error.code}: {error_body}") from http_error
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scriptcollection
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.3.0
|
|
4
4
|
Summary: The ScriptCollection is the place for reusable scripts.
|
|
5
5
|
Author-email: Marius Göcke <marius.goecke@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/anionDev/ScriptCollection
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
ScriptCollection/AnionBuildPlatform.py,sha256=vg1lZ6D1nqq_mO7iYTDSktSImXd34scfPaXiIrbvrg0,11848
|
|
2
2
|
ScriptCollection/CertificateUpdater.py,sha256=dR5JpJDlZdjD4_pgMz7zCrt1IM7VfatmK6pULSczxQw,9693
|
|
3
|
-
ScriptCollection/Executables.py,sha256=
|
|
4
|
-
ScriptCollection/GeneralUtilities.py,sha256=
|
|
3
|
+
ScriptCollection/Executables.py,sha256=l_HLG1oIcmIWh9QKhcQdgocSADQbqxGBChs3D-r-SSk,66070
|
|
4
|
+
ScriptCollection/GeneralUtilities.py,sha256=4RKClFNriMiiNPMQ4yK9eiBhF_z_9TCd9jumhcwJJeI,69142
|
|
5
5
|
ScriptCollection/HTTPMaintenanceOverheadHelper.py,sha256=TToNtyO1XzsMbBsTBf3o0xgOK0v4Jf03qw2Z0xb2nCk,2007
|
|
6
6
|
ScriptCollection/ProcessesRunner.py,sha256=yYaC95h40uszuZPggn2kK3qXIN2ZjGMCavo-NQ53ElQ,1537
|
|
7
7
|
ScriptCollection/ProgramRunnerBase.py,sha256=eZenpiIFW1dhvSVCG7cfBs5hO4_KLlOxR7oXoSxUDmk,2286
|
|
@@ -9,7 +9,7 @@ ScriptCollection/ProgramRunnerMock.py,sha256=uTu-aFle1W_oKjeQEmuPsFPQpvo0kRf2FrR
|
|
|
9
9
|
ScriptCollection/ProgramRunnerPopen.py,sha256=8tP0xWKeRYoGlaNB3iUh4o61O-tKtcxrS7IUqtU85C4,4091
|
|
10
10
|
ScriptCollection/ProgramRunnerSudo.py,sha256=RXsiceQ8W6a8YDk_2BPscVbIAmDIpIxqe9WHBy2r4CM,5233
|
|
11
11
|
ScriptCollection/SCLog.py,sha256=Z-HhC_CRtU3HdQflNsabFh9IJOucHcXmxMDcf58fli0,4671
|
|
12
|
-
ScriptCollection/ScriptCollectionCore.py,sha256=
|
|
12
|
+
ScriptCollection/ScriptCollectionCore.py,sha256=NNxEzJgIRwJBadChFmPtmNEcL9bQwClSBG94zDUlf90,224854
|
|
13
13
|
ScriptCollection/UnassignedVariablesCheck.py,sha256=v59qLQ5RRvx31ouuCW7NWZj3McWENvGBPkV5ZGBmqJE,4344
|
|
14
14
|
ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
15
15
|
ScriptCollection/OCIImages/AbstractImageHandler.py,sha256=o129UYDtLymCRjMZBP9Puaj19ueMHoNt6UTLspfnaRc,1758
|
|
@@ -32,7 +32,8 @@ ScriptCollection/Resources/Fonts/DejaVuSans-Oblique.ttf,sha256=SvdfoW7m061D4ezsQ
|
|
|
32
32
|
ScriptCollection/Resources/Fonts/DejaVuSans.ttf,sha256=faGVp0xVvvmI0NSPlQi9XYSUJcF3Dbpde_xs6e2EiVQ,757076
|
|
33
33
|
ScriptCollection/Resources/Fonts/LICENSE.txt,sha256=egg7E25k0GR5TDQZdR5cfdENL2TBCP5boWHq5eWVipM,8816
|
|
34
34
|
ScriptCollection/Resources/MaintenanceSite/MaintenanceSite.html,sha256=CX9S1bdOz6xU2aGfr03tJRGczQrlpn-IeODc15d5kc0,232
|
|
35
|
-
ScriptCollection/TFCPS/
|
|
35
|
+
ScriptCollection/TFCPS/SCTaskRunnerServer.py,sha256=-jdoajgECUYrYIk0zmcOcGYMw2NkahEFp3sy8htGE08,10603
|
|
36
|
+
ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=UNVUq2PwrrrxFkn4v_IusGm-RSC5jB3fZStCIfXlPqo,30784
|
|
36
37
|
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=ulxU6CykkqyE6QWeX8PEyW0rntuK3qfBEbPSIplekfI,9677
|
|
37
38
|
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=cScWO7n89Gbs3au8U_EczJO53jZx6NFzpm7c46IqO_Q,23733
|
|
38
39
|
ScriptCollection/TFCPS/TFCPS_CreateRelease.py,sha256=X2NgZgbOkJsdZ6wMHB_RuS_rQCuz5oQCqzYrxyXBN9Q,6680
|
|
@@ -40,6 +41,7 @@ ScriptCollection/TFCPS/TFCPS_Generic.py,sha256=5vCFfDiU8bvfcnoKNs2ATS8nsl6mYXcIu
|
|
|
40
41
|
ScriptCollection/TFCPS/TFCPS_MergeToMain.py,sha256=BsSV4a-NZY86MZ20d-_rYxNVAI4UzAfmEwXlRZ8qWQc,7685
|
|
41
42
|
ScriptCollection/TFCPS/TFCPS_MergeToStable.py,sha256=QRm8n1qH8H_XbJhGCbzwdNr9KSEy0WkbG7ulpc4Md3U,22696
|
|
42
43
|
ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py,sha256=JYiDqDsfVJQWS9xWMi_UzfXnXsJRnPRxAzFr82bUEvI,2370
|
|
44
|
+
ScriptCollection/TFCPS/TFCPS_RemoteBuild.py,sha256=Z8Gw-j5I7VPafHEsVeQJ2gTTKVLAnnhwZhY8IVTi2Qk,11867
|
|
43
45
|
ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=HtIpLK9AaIzvLB_nOC8vPKosuwgi_7XV0xoZxut9AFs,126278
|
|
44
46
|
ScriptCollection/TFCPS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
45
47
|
ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=wS2yzYUmJ4m0cf6cBYVTHsiXSXfYhHoLFFy8x6yEQvY,19573
|
|
@@ -49,7 +51,7 @@ ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py,sha256=
|
|
|
49
51
|
ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py,sha256=i0zEGehj0sttxjjZtoq2KFSKp_ulxVyWp_ZgAhIY_So,241
|
|
50
52
|
ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=7grKSK9qtYbpyvk3y7RsygBjmKQw1JlANSwk6GGmbBU,41731
|
|
51
53
|
ScriptCollection/TFCPS/DotNet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
52
|
-
ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=
|
|
54
|
+
ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=d8U-sIhsneMplYkG73BJRQQZh4uLoX7QiOhbnZNuOM8,8540
|
|
53
55
|
ScriptCollection/TFCPS/Flutter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
54
56
|
ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py,sha256=kyx26AnT1-LySFA46wfJ9yZUKYdMWTD0U2XZfSQbuB0,3497
|
|
55
57
|
ScriptCollection/TFCPS/Go/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -61,8 +63,8 @@ ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py,sha256=Bd1gxCk9Fb
|
|
|
61
63
|
ScriptCollection/TFCPS/Python/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
62
64
|
ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=WhOsMpJxNGqnQGA3nJ99ruyqymxBeJxCcpK98zJduTo,1680
|
|
63
65
|
ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
64
|
-
scriptcollection-4.
|
|
65
|
-
scriptcollection-4.
|
|
66
|
-
scriptcollection-4.
|
|
67
|
-
scriptcollection-4.
|
|
68
|
-
scriptcollection-4.
|
|
66
|
+
scriptcollection-4.3.0.dist-info/METADATA,sha256=OwhkFvglNgQNX3I3sOGZ5nEzEFvYd0mEAxxLI4Dv9WY,7815
|
|
67
|
+
scriptcollection-4.3.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
68
|
+
scriptcollection-4.3.0.dist-info/entry_points.txt,sha256=cvkJt0PR9co3xeHlUDNufX7A37gyZB5WAu3waWzi3Cg,5711
|
|
69
|
+
scriptcollection-4.3.0.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
|
|
70
|
+
scriptcollection-4.3.0.dist-info/RECORD,,
|
|
@@ -21,6 +21,7 @@ sccreateskill = ScriptCollection.Executables:CreateSkill
|
|
|
21
21
|
sccurrentuserhaselevatedprivileges = ScriptCollection.Executables:CurrentUserHasElevatedPrivileges
|
|
22
22
|
scdownloadcachabletools = ScriptCollection.Executables:DownloadCachableTools
|
|
23
23
|
scensuredockernetworkisavailable = ScriptCollection.Executables:EnsureDockerNetworkIsAvailable
|
|
24
|
+
scensureexternaldockernetworksexist = ScriptCollection.Executables:EnsureExternalDockerNetworksExist
|
|
24
25
|
scespoc = ScriptCollection.Executables:Espoc
|
|
25
26
|
scextractpdfpages = ScriptCollection.Executables:ExtractPDFPages
|
|
26
27
|
scfilecontainscontent = ScriptCollection.Executables:FileContainsContent
|
|
@@ -48,6 +49,7 @@ scocranalysisoffile = ScriptCollection.Executables:OCRAnalysisOfFile
|
|
|
48
49
|
scocranalysisoffolder = ScriptCollection.Executables:OCRAnalysisOfFolder
|
|
49
50
|
scocranalysisofrepository = ScriptCollection.Executables:OCRAnalysisOfRepository
|
|
50
51
|
scorganizelinesinfile = ScriptCollection.Executables:OrganizeLinesInFile
|
|
52
|
+
scpreparebuildpipelineforgithub = ScriptCollection.Executables:PrepareBuildPipelineForGithub
|
|
51
53
|
scpreparebuildpipelineforgitlab = ScriptCollection.Executables:PrepareBuildPipelineForGitlab
|
|
52
54
|
scprintcurrentworkingdirectory = ScriptCollection.Executables:PrintCurrecntWorkingDirectory
|
|
53
55
|
scprintfilecontent = ScriptCollection.Executables:PrintFileContent
|
|
@@ -65,6 +67,7 @@ scsearchforsecrets = ScriptCollection.Executables:SearchForSecrets
|
|
|
65
67
|
scsearchinfiles = ScriptCollection.Executables:SearchInFiles
|
|
66
68
|
scsetcontentoffile = ScriptCollection.Executables:SetContentOfFile
|
|
67
69
|
scshow2faasqrcode = ScriptCollection.Executables:Show2FAAsQRCode
|
|
70
|
+
scshowexternaldockernetworks = ScriptCollection.Executables:ShowExternalDockerNetworks
|
|
68
71
|
scshowmissingfiles = ScriptCollection.Executables:ShowMissingFiles
|
|
69
72
|
scshowprojectversion = ScriptCollection.Executables:ShowProjectVersion
|
|
70
73
|
scshowversion = ScriptCollection.Executables:ShowVersion
|
|
File without changes
|
|
File without changes
|