ScriptCollection 4.4.20__py3-none-any.whl → 4.4.22__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -39,7 +39,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
39
39
  from .ProgramRunnerPopen import ProgramRunnerPopen
40
40
  from .SCLog import SCLog, LogLevel
41
41
 
42
- version = "4.4.20"
42
+ version = "4.4.22"
43
43
  __version__ = version
44
44
 
45
45
  class VSCodeWorkspaceShellTask:
@@ -2442,15 +2442,23 @@ resolving a name requires fontconfig, which does not exist on every system ffmpe
2442
2442
  relative entries of 'excluded_folders' are resolved against 'base_folder'."""
2443
2443
  resolved_base_folder = GeneralUtilities.resolve_relative_path(base_folder, os.getcwd())
2444
2444
  resolved_path = GeneralUtilities.resolve_relative_path(path, resolved_base_folder)
2445
- normalized_base_folder = os.path.normcase(os.path.normpath(resolved_base_folder))
2446
- normalized_path = os.path.normcase(os.path.normpath(resolved_path))
2445
+ # GeneralUtilities.resolve_relative_path only resolves symlinks (via Path.resolve()) for relative inputs; an
2446
+ # already-absolute input is returned unchanged. On macOS the system temp-folder lies under "/var", which
2447
+ # itself is a symlink to "/private/var" (Linux/Windows temp-folders normally have no such symlink-indirection).
2448
+ # Without canonicalizing all three values the same way here, an absolute base_folder/path (unresolved, still
2449
+ # containing "/var/...") could fail to share a prefix with a relative excluded_folder entry like ".git" (which
2450
+ # got symlink-resolved to "/private/var/..." further above), so the exclusion-check would silently never match.
2451
+ # Resolving symlinks consistently for all three values fixes this and is also the safer choice from a security
2452
+ # standpoint, since it prevents a symlink from being used to bypass the excluded-folders check.
2453
+ normalized_base_folder = os.path.normcase(os.path.normpath(os.path.realpath(resolved_base_folder)))
2454
+ normalized_path = os.path.normcase(os.path.normpath(os.path.realpath(resolved_path)))
2447
2455
  # 1. path must be equal to or located inside base_folder
2448
2456
  if not (normalized_path == normalized_base_folder or normalized_path.startswith(normalized_base_folder + os.sep)):
2449
2457
  return False
2450
2458
  # 2. path must not be equal to or located inside any excluded folder
2451
2459
  for excluded_folder in excluded_folders:
2452
2460
  resolved_excluded_folder = GeneralUtilities.resolve_relative_path(excluded_folder, resolved_base_folder)
2453
- normalized_excluded_folder = os.path.normcase(os.path.normpath(resolved_excluded_folder))
2461
+ normalized_excluded_folder = os.path.normcase(os.path.normpath(os.path.realpath(resolved_excluded_folder)))
2454
2462
  if normalized_path == normalized_excluded_folder or normalized_path.startswith(normalized_excluded_folder + os.sep):
2455
2463
  return False
2456
2464
  # 3. path is inside base_folder and not inside any excluded folder
@@ -2644,8 +2652,11 @@ resolving a name requires fontconfig, which does not exist on every system ffmpe
2644
2652
  patch = int(splitted[2])
2645
2653
  if increment_major:
2646
2654
  major = major+1
2655
+ minor=0
2656
+ patch=0
2647
2657
  if increment_minor:
2648
2658
  minor = minor+1
2659
+ patch=0
2649
2660
  if increment_patch:
2650
2661
  patch = patch+1
2651
2662
  return f"{major}.{minor}.{patch}"
@@ -2700,16 +2711,18 @@ resolving a name requires fontconfig, which does not exist on every system ffmpe
2700
2711
  or self.is_fix_branch(current_branch_name)
2701
2712
  or self.is_other_branch(current_branch_name),
2702
2713
  f"Repository '{repository_folder}' is on branch '{current_branch_name}' which is not a major-, minor-, fix- or other-branch. This is not allowed.")
2703
- if self.is_major_branch(current_branch_name):
2704
- incremented = self.increment_version(current_version, True, False, False)
2705
- major = incremented.split(".")[0]
2706
- result = f"{major}.0.0"
2714
+ if self.is_other_branch(current_branch_name) or self.is_fix_branch(current_branch_name):
2715
+ result = self.increment_version(current_version, False, False, True)
2707
2716
  elif self.is_minor_branch(current_branch_name):
2708
2717
  incremented = self.increment_version(current_version, False, True, False)
2709
2718
  splitted = incremented.split(".")
2710
2719
  result = f"{splitted[0]}.{splitted[1]}.0"
2720
+ elif self.is_major_branch(current_branch_name):
2721
+ incremented = self.increment_version(current_version, True, False, False)
2722
+ major = incremented.split(".")[0]
2723
+ result = f"{major}.0.0"
2711
2724
  else:
2712
- result = self.increment_version(current_version, False, False, True)
2725
+ raise ValueError(f"Repository '{repository_folder}' is on branch '{current_branch_name}' which is not a major-, minor-, fix- or other-branch. This is not allowed.")
2713
2726
  else:
2714
2727
  result = "0.1.0"
2715
2728
  else:
@@ -4572,13 +4585,22 @@ OCR-content:
4572
4585
  self.assert_scbuilder_image_in_github_workflow_matches_image_definition(repository_folder)
4573
4586
  self.__prepare_build_pipeline(repository_folder)
4574
4587
 
4588
+ def __ensure_branch_is_checked_out(self,repository_folder:str) -> None:
4589
+ current_branch_result = self.run_program_argsasarray("git", ["symbolic-ref", "--short", "HEAD"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
4590
+ repository_is_on_any_branch:bool = current_branch_result[0] == 0
4591
+ if not repository_is_on_any_branch:
4592
+ timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
4593
+ branch_name:str=f"pipeline_{timestamp}"
4594
+ self.run_program_argsasarray("git",["checkout","-b",branch_name], repository_folder)
4595
+
4575
4596
  def __prepare_build_pipeline(self,repository_folder:str) -> None:
4576
4597
  GeneralUtilities.assert_condition(self.is_running_in_build_container(), "This function should only be run in the build container.")
4577
-
4598
+ self.__ensure_branch_is_checked_out(repository_folder)
4599
+
4578
4600
  expected_image = self.__get_scbuilder_image_from_image_definition_file(repository_folder)
4579
4601
  GeneralUtilities.assert_condition("scbuilder:" in expected_image, f"The SCBuilder-image '{expected_image}' defined in the image-definition-file of the repository is not a valid SCBuilder-image. It must contain 'scbuilder:'.")
4580
4602
  if not expected_image.endswith(":latest"):
4581
- minimal_required_scbuilder_version = Version("1.2.9")#the preparation-steps below require at least this SCBuilder-version
4603
+ minimal_required_scbuilder_version = Version("1.2.11")#the preparation-steps below require at least this SCBuilder-version
4582
4604
  defined_scbuilder_version = self.__get_version_of_scbuilder_image(expected_image)#TODO remove this check after 2026-12-31
4583
4605
  GeneralUtilities.assert_condition(defined_scbuilder_version >= minimal_required_scbuilder_version, f"The SCBuilder-version {defined_scbuilder_version} defined in the image-definition-file of the repository is older than the minimal required version {minimal_required_scbuilder_version}. Please update the SCBuilder-image.")
4584
4606
 
@@ -6,7 +6,7 @@ import shutil
6
6
  from ...GeneralUtilities import GeneralUtilities
7
7
  from ...SCLog import LogLevel
8
8
  from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base, TFCPS_CodeUnitSpecific_Base_CLI
9
- from ..TFCPS_RemoteBuild import TFCPS_RemoteBuild, RunnerOperatingSystem
9
+ from ..TFCPS_RemoteBuild import RunnerOperatingSystem
10
10
 
11
11
 
12
12
  class TFCPS_CodeUnitSpecific_CPP_Functions(TFCPS_CodeUnitSpecific_Base):
@@ -164,7 +164,10 @@ class TFCPS_CodeUnitSpecific_CPP_Functions(TFCPS_CodeUnitSpecific_Base):
164
164
  else:
165
165
  # macOS-binaries can only be produced on macOS, so this always delegates to a macOS-task-runner when not
166
166
  # already running on macOS (see the remote-build-article in the reference).
167
- self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "/bin/sh", ["-c", " && ".join(self.macos_build_commands)], source_folder)
167
+ #The result-folder is the sourcecode-folder itself: where a build-command of a C/C++-codeunit puts its
168
+ #output is defined by that command, so the only folder which is known to contain it is the one the commands
169
+ #run in - which is also the folder __collect_output_files searches afterwards.
170
+ self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "/bin/sh", ["-c", " && ".join(self.macos_build_commands)], source_folder, source_folder)
168
171
  self.__collect_output_files(source_folder, self.macos_output_file_patterns, "BuildResult_MacOS")
169
172
 
170
173
  @GeneralUtilities.check_arguments
@@ -91,14 +91,14 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
91
91
  # are produced uniformly in the same defined environment. See the remote-build-article in the reference. If
92
92
  # no runner is configured, fall back to building locally (only possible when already running on Windows)
93
93
  # instead of failing, so a development machine without a configured runner is not blocked.
94
+ windows_release_folder = os.path.join(src_folder, "build/windows/x64/runner/Release")
94
95
  if platform.system() == "Windows" and not TFCPS_RemoteBuild(self._protected_sc).has_any_runner_configured():
95
96
  self._protected_sc.log.log("No remote-build-runner is configured; building the windows-target locally "
96
97
  "instead. This build is not guaranteed to be produced in the same uniform "
97
98
  "environment as a runner-built one.", LogLevel.Warning)
98
99
  self._protected_sc.run_with_epew("flutter", "build windows", src_folder)
99
100
  else:
100
- self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder)
101
- windows_release_folder = os.path.join(src_folder, "build/windows/x64/runner/Release")
101
+ self.run_program_on_remote_runner(RunnerOperatingSystem.Windows, "flutter", ["build", "windows"], src_folder, windows_release_folder)
102
102
  windows_folder = os.path.join(artifacts_folder, "BuildResult_Windows")
103
103
  GeneralUtilities.ensure_directory_does_not_exist(windows_folder)
104
104
  GeneralUtilities.ensure_directory_exists(windows_folder)
@@ -119,8 +119,8 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
119
119
  # macOS-desktop-builds must run on macOS and therefore always run on the macOS-task-runner (uniform
120
120
  # builds), analogous to how ios-builds always run on the iOS-task-runner. See SCTaskRunnerMacOS and
121
121
  # the remote-build-article in the reference.
122
- self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder)
123
122
  macos_release_folder = os.path.join(src_folder, "build/macos/Build/Products/Release")
123
+ self.run_program_on_remote_runner(RunnerOperatingSystem.MacOS, "flutter", ["build", "macos"], src_folder, macos_release_folder)
124
124
  macos_folder = os.path.join(artifacts_folder, "BuildResult_MacOS")
125
125
  GeneralUtilities.ensure_directory_does_not_exist(macos_folder)
126
126
  GeneralUtilities.ensure_directory_exists(macos_folder)
@@ -128,8 +128,8 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
128
128
  elif target == "ios":
129
129
  # iOS-builds must run on macOS and therefore always run on the dedicated iOS-task-runner (uniform
130
130
  # builds). See SCTaskRunnerIOS and the remote-build-article in the reference.
131
- self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder)
132
131
  ios_release_folder = os.path.join(src_folder, "build/ios/iphoneos")
132
+ self.run_program_on_remote_runner(RunnerOperatingSystem.IOS, "flutter", ["build", "ios"], src_folder, ios_release_folder)
133
133
  ios_folder = os.path.join(artifacts_folder, "BuildResult_IOS")
134
134
  GeneralUtilities.ensure_directory_does_not_exist(ios_folder)
135
135
  GeneralUtilities.ensure_directory_exists(ios_folder)
@@ -138,27 +138,38 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
138
138
  # Android-app-builds always run on an Android-task-runner (see SCTaskRunnerAndroid and the
139
139
  # remote-build-article in the reference), analogous to how ios-builds always run on a macOS-task-runner:
140
140
  # the Android-SDK/NDK-toolchain no longer lives in SCBuilder itself, it moved into SCTaskRunnerAndroid.
141
- self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder)
142
- enabled=False
143
- if enabled:#TODO move to external because this is not platform indepent
144
- aab_folder = os.path.join(artifacts_folder, "BuildResult_AAB")
145
- GeneralUtilities.ensure_directory_does_not_exist(aab_folder)
146
- GeneralUtilities.ensure_directory_exists(aab_folder)
147
- aab_relase_folder = os.path.join(src_folder, "build/app/outputs/bundle/release")
148
- aab_file_original = self._protected_sc.find_file_by_extension(aab_relase_folder, "aab")
149
- aab_file = os.path.join(aab_folder, f"{codeunit_name}.aab")
150
- shutil.copyfile(aab_file_original, aab_file)
151
-
152
- bundletool = self.tfcps_Tools_General.ensure_androidappbundletool_is_available(None,self.use_cache())
153
- apk_folder = os.path.join(artifacts_folder, "BuildResult_APK")
154
- GeneralUtilities.ensure_directory_does_not_exist(apk_folder)
155
- GeneralUtilities.ensure_directory_exists(apk_folder)
156
- apks_file = f"{apk_folder}/{codeunit_name}.apks"
157
- self._protected_sc.run_program("java", f"-jar {bundletool} build-apks --bundle={aab_file} --output={apks_file} --mode=universal", aab_relase_folder)
158
- with zipfile.ZipFile(apks_file, "r") as zip_ref:
159
- zip_ref.extract("universal.apk", apk_folder)
160
- GeneralUtilities.ensure_file_does_not_exist(apks_file)
161
- os.rename(f"{apk_folder}/universal.apk", f"{apk_folder}/{codeunit_name}.apk")
141
+ aab_release_folder = os.path.join(src_folder, "build/app/outputs/bundle/release")
142
+ self.run_program_on_remote_runner(RunnerOperatingSystem.Android, "flutter", ["build", "appbundle"], src_folder, aab_release_folder)
143
+ # The app-bundle which came back from the runner is published as the build-result of this codeunit,
144
+ # analogous to the targets above: it is the artifact of an android-build, and every codeunit has to have
145
+ # an artifact whose name matches "BuildResult_.+" (see TFCPS_CodeUnit_BuildCodeUnit.build_codeunit).
146
+ aab_folder = os.path.join(artifacts_folder, "BuildResult_AAB")
147
+ GeneralUtilities.ensure_folder_exists_and_is_empty(aab_folder)
148
+ aab_file_original = self._protected_sc.find_file_by_extension(aab_release_folder, "aab")
149
+ aab_file = os.path.join(aab_folder, f"{codeunit_name}.aab")
150
+ shutil.copyfile(aab_file_original, aab_file)
151
+ # An app-bundle is not installable itself - it is what is uploaded to the store - so the universal apk,
152
+ # which can be installed directly, is generated from it with Google's bundletool.
153
+ bundletool = self.tfcps_Tools_General.ensure_androidappbundletool_is_available(None, not self.use_cache())
154
+ # The pinned JRE from the global cache is used instead of a "java" of the machine, because a
155
+ # java-runtime is not part of every machine which builds a codeunit. It is the same JRE ScriptCollection
156
+ # already uses to render the plantuml-diagrams, so this step needs nothing which a build does not need
157
+ # anyway.
158
+ java_executable = self.tfcps_Tools_General.ensure_jre_is_available(not self.use_cache(), self.get_repository_folder())
159
+ apk_folder = os.path.join(artifacts_folder, "BuildResult_APK")
160
+ GeneralUtilities.ensure_directory_does_not_exist(apk_folder)
161
+ GeneralUtilities.ensure_directory_exists(apk_folder)
162
+ apks_file = os.path.join(apk_folder, f"{codeunit_name}.apks")
163
+ # The arguments are passed as an array and not as one string, because they contain absolute paths and a
164
+ # path which contains a space would otherwise be split into several arguments (see
165
+ # GeneralUtilities.arguments_to_array).
166
+ self._protected_sc.run_program_argsasarray(java_executable, ["-jar", bundletool, "build-apks", f"--bundle={aab_file}", f"--output={apks_file}", "--mode=universal"], aab_release_folder)
167
+ # "--mode=universal" produces exactly one apk for all architectures, which the archive contains under
168
+ # this name; it is renamed to the codeunit so the artifact states which codeunit it belongs to.
169
+ with zipfile.ZipFile(apks_file, "r") as apks_archive:
170
+ apks_archive.extract("universal.apk", apk_folder)
171
+ GeneralUtilities.ensure_file_does_not_exist(apks_file)
172
+ os.rename(os.path.join(apk_folder, "universal.apk"), os.path.join(apk_folder, f"{codeunit_name}.apk"))
162
173
  else:
163
174
  raise ValueError(f"Not supported target: {target}")
164
175
  self.__generate_bom_for_flutter_package(package_name)
@@ -3,6 +3,7 @@ import json
3
3
  import base64
4
4
  import shutil
5
5
  import ssl
6
+ import time
6
7
  import uuid
7
8
  import threading
8
9
  import subprocess
@@ -14,15 +15,31 @@ from ..SCLog import LogLevel
14
15
 
15
16
  #Size of the chunks in which the archives are copied between the network-connection and the disk. The archives are
16
17
  #transferred chunk-wise instead of being held in memory as a whole because both of them are regularly large (the payload
17
- #contains the whole repository of the client, the result contains the whole codeunit-folder including its build-output),
18
+ #contains the whole repository of the client, the result contains the complete build-output of a codeunit),
18
19
  #while a runner often runs in a container with a small memory-limit and needs that memory for the build itself.
19
20
  _transfer_chunk_size_in_bytes: int = 1024*1024
20
21
 
22
+ #Interval in which a job which is still running states that in the log of the runner. The output of a job belongs to the
23
+ #job and is fetched by the client, so without such a line the log of the runner would state nothing at all between the
24
+ #start of a job and its result - which is an hour or more for an app-build.
25
+ _interval_between_two_progress_logs_in_seconds: int = 60
26
+
27
+ #Amount of lines of the output of a job which did not succeed that the runner logs. The complete output belongs to the
28
+ #job and is fetched by the client; this is what makes the log of the runner state why a build failed even when no client
29
+ #is connected any more when it ends.
30
+ _amount_of_logged_lines_of_a_failed_job: int = 25
31
+
21
32
 
22
33
  class _RunnerJob:
23
- def __init__(self, workspace_folder: str, codeunit_name: str):
34
+ def __init__(self, job_id: str, workspace_folder: str, result_folder: str):
35
+ self.job_id = job_id
24
36
  self.workspace_folder = workspace_folder
25
- self.codeunit_name = codeunit_name
37
+ #Folder whose content is the result of the job, relative to the workspace (which is the repository the client
38
+ #sent). Only this folder is transferred back, because only it contains what the client asked the runner to
39
+ #produce: everything else in the workspace is either the unchanged content the client sent or state of this
40
+ #machine (its paths, its caches, its toolchain-configuration), which has no business in the repository of a
41
+ #client.
42
+ self.result_folder = result_folder
26
43
  self.state = "running" # "running" | "completed" | "failed"
27
44
  self.exitcode = None
28
45
  self.log = GeneralUtilities.empty_string
@@ -34,19 +51,36 @@ class _RunnerJob:
34
51
  #arbitrary file of the build-output and can leave a partially deleted workspace behind.
35
52
  self.transfer_lock = threading.Lock()
36
53
 
54
+ def append_to_log(self, message: str) -> None:
55
+ """Adds a line of the runner itself - as opposed to a line of the program which the job runs - to the log of the
56
+ job. The client prints the log of a job as the output of the delegated build, so this is the place where the
57
+ runner can state something about a job to the one who is waiting for it."""
58
+ with self.lock:
59
+ self.log = self.log+message+"\n"
60
+
37
61
 
38
62
  class SCTaskRunnerServer:
39
63
  """HTTP-server that compiles operating-system-bound build-steps on behalf of a (remote) client (see TFCPS_RemoteBuild).
40
64
  Per job it extracts the received repository-archive into a fresh, empty workspace, runs the requested program on this
41
- machine's operating-system, and returns the codeunit-folder. The workspace is deleted as soon as the client deletes the
42
- job (immediately after fetching the result), so no repository-content remains on the runner."""
65
+ machine's operating-system, and returns the folder the client stated its result is in. The workspace is deleted as
66
+ soon as the client deletes the job (immediately after fetching the result), so no repository-content remains on the
67
+ runner."""
43
68
 
44
69
  def __init__(self, operating_system_name: str, username: str, password: str, work_folder: str = None, sc: ScriptCollectionCore = None):
45
70
  self.operating_system_name = operating_system_name
46
71
  self.__username = username # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
47
72
  self.__password = password # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
48
73
  self.work_folder = work_folder if work_folder is not None else os.path.join(GeneralUtilities.get_temp_folder(), "SCTaskRunner")
49
- self.__sc = sc if sc is not None else ScriptCollectionCore()
74
+ if sc is None:
75
+ #The default-loglevel of a ScriptCollectionCore is Warning, while everything this server logs about its
76
+ #lifecycle and its jobs is logged as Information. A runner which uses that default therefore produces no
77
+ #output at all - neither its start nor the jobs it ran become visible in the console-output of the process,
78
+ #which is the only log a permanently running server has. Hence the loglevel is raised here. A caller which
79
+ #passes its own ScriptCollectionCore keeps the loglevel it configured there.
80
+ self.__sc = ScriptCollectionCore()
81
+ self.__sc.log.loglevel = LogLevel.Information
82
+ else:
83
+ self.__sc = sc
50
84
  self.__jobs: dict[str, _RunnerJob] = {}
51
85
  self.__jobs_lock = threading.Lock()
52
86
 
@@ -54,6 +88,9 @@ class SCTaskRunnerServer:
54
88
  def run(self, host: str = "0.0.0.0", port: int = 8080, certificate_file: str = None, certificate_key_file: str = None) -> None:
55
89
  """Starts the HTTP-server. When both certificate_file and certificate_key_file are given the server is served over
56
90
  TLS (https); otherwise it is served over plain http (e.g. when TLS is terminated by a reverse-proxy in front of it)."""
91
+ #Logged before anything else happens, so that the log of the runner also shows a start which did not get as far
92
+ #as listening (for example because the port is occupied or because the certificate can not be loaded).
93
+ self.__sc.log.log(f"Start the SCTaskRunner for operating-system '{self.operating_system_name}' (work-folder: '{self.work_folder}').")
57
94
  GeneralUtilities.ensure_directory_exists(self.work_folder)
58
95
  server = ThreadingHTTPServer((host, port), self.__create_request_handler())
59
96
  protocol = "http"
@@ -66,11 +103,13 @@ class SCTaskRunnerServer:
66
103
  server.serve_forever()
67
104
 
68
105
  @GeneralUtilities.check_arguments
69
- def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
106
+ def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str, result_folder: str) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
70
107
  """Starts a job for the repository-archive which can be read from archive_stream (the body of the request which
71
108
  submitted the job; archive_size is its announced length). The archive is taken as a stream instead of as bytes
72
109
  because it contains the complete repository of the client and is therefore regularly several hundred megabytes,
73
- which must not be held in memory in addition to the build which runs afterwards."""
110
+ which must not be held in memory in addition to the build which runs afterwards. Only the transfer of the archive
111
+ happens here (it can only be read from the request which brought it); everything which follows belongs to the
112
+ job-thread, see __extract_payload_and_run_job."""
74
113
  job_id = str(uuid.uuid4())
75
114
  workspace_folder = os.path.join(self.work_folder, job_id)
76
115
  # Isolation: each job gets a fresh, empty workspace-folder.
@@ -79,17 +118,75 @@ class SCTaskRunnerServer:
79
118
  archive_file = os.path.join(self.work_folder, f"{job_id}.payload.tar.gz")
80
119
  try:
81
120
  self.__copy_stream_to_file(archive_stream, archive_size, archive_file)
82
- with tarfile.open(archive_file, "r:gz") as tar:
83
- tar.extractall(workspace_folder, filter="fully_trusted")
84
- finally:
121
+ except Exception:
122
+ #A transfer which did not complete leaves no job behind, so the workspace and the incomplete archive are
123
+ #removed here - the job-thread which would do that is not started in this case.
85
124
  GeneralUtilities.ensure_file_does_not_exist(archive_file)
86
- job = _RunnerJob(workspace_folder, codeunit_name)
125
+ GeneralUtilities.ensure_directory_does_not_exist(workspace_folder)
126
+ raise
127
+ job = _RunnerJob(job_id, workspace_folder, result_folder)
87
128
  with self.__jobs_lock:
88
129
  self.__jobs[job_id] = job
89
- thread = threading.Thread(target=self.__run_job, args=(job, program, arguments, working_directory), daemon=True)
130
+ thread = threading.Thread(target=self.__extract_payload_and_run_job, args=(job, archive_file, program, arguments, working_directory), daemon=True)
90
131
  thread.start()
132
+ self.__sc.log.log(f"Accepted job '{job_id}': run '{program} {' '.join(arguments)}' in '{working_directory}' of codeunit '{codeunit_name}', result-folder '{result_folder}' ({archive_size} transferred bytes of repository-content).")
91
133
  return job_id
92
134
 
135
+ @GeneralUtilities.check_arguments
136
+ def __extract_payload_and_run_job(self, job: _RunnerJob, archive_file: str, program: str, arguments: list[str], working_directory: str) -> None:
137
+ """Extracts the transferred repository-archive into the workspace of the job and runs the requested program
138
+ afterwards. The extraction happens here - in the thread of the job - and not in the request which submitted the
139
+ job, because extracting the repository of a large codeunit takes minutes during which no data flows: the client
140
+ which waits for the answer of its submit, and a reverse-proxy between it and the runner, would run into their
141
+ timeouts although the job is being prepared correctly. Because the job already exists while this runs, the state
142
+ of that preparation is what a client sees and what a failure of it is reported as."""
143
+ try:
144
+ job.append_to_log("Extract the transferred repository into the workspace of the job...")
145
+ with tarfile.open(archive_file, "r:gz") as tar:
146
+ tar.extractall(job.workspace_folder, filter="fully_trusted")
147
+ except Exception as exception:
148
+ self.__end_job_as_failed_because_of_an_internal_error(job, "the transferred repository could not be extracted", exception)
149
+ return
150
+ finally:
151
+ GeneralUtilities.ensure_file_does_not_exist(archive_file)
152
+ self.__sc.log.log(f"Job '{job.job_id}' works on the repository '{self.__get_origin_of_the_transferred_repository(job.workspace_folder)}'.")
153
+ self.__run_job(job, program, arguments, working_directory)
154
+
155
+ @GeneralUtilities.check_arguments
156
+ def __get_origin_of_the_transferred_repository(self, workspace_folder: str) -> str:
157
+ """Returns the address of the git-remote 'origin' of the transferred repository. This states which repository a
158
+ job builds without requiring the client to send anything in addition: the repository is transferred including
159
+ its .git-folder anyway. The address is not always available (a repository does not have to have a remote), which
160
+ is stated instead of being treated as an error - this is information for the log and nothing a job depends on."""
161
+ configuration_file = os.path.join(workspace_folder, ".git", "config")
162
+ if not os.path.isfile(configuration_file):
163
+ return "unknown: the transferred content has no git-configuration"
164
+ #Read line-wise and not with configparser, because git indents the entries of a section, which configparser reads
165
+ #as continuations of the value of the previous entry.
166
+ section_is_the_origin_remote = False
167
+ for line in GeneralUtilities.read_lines_from_file(configuration_file):
168
+ stripped_line = line.strip()
169
+ if stripped_line.startswith("["):
170
+ section_is_the_origin_remote = stripped_line.replace(" ", GeneralUtilities.empty_string) == '[remote"origin"]'
171
+ elif section_is_the_origin_remote and stripped_line.startswith("url"):
172
+ return self.__remove_credentials_from_address(stripped_line.split("=", 1)[1].strip())
173
+ return "unknown: the transferred content has no git-remote 'origin'"
174
+
175
+ @GeneralUtilities.check_arguments
176
+ def __remove_credentials_from_address(self, address: str) -> str:
177
+ """Removes the user-information of an address like 'https://user:token@host/repository.git'. An address which is
178
+ written to a log must not contain a credential. An address without a scheme is returned unchanged: it is the
179
+ ssh-notation 'user@host:path', whose user is part of the address and not a credential."""
180
+ scheme_separator = "://"
181
+ if scheme_separator not in address:
182
+ return address
183
+ host_and_path = address.split(scheme_separator, 1)[1]
184
+ end_of_the_host = host_and_path.find("/")
185
+ host = host_and_path if end_of_the_host < 0 else host_and_path[:end_of_the_host]
186
+ if "@" not in host:
187
+ return address
188
+ return address.replace(host, host.split("@", 1)[1], 1)
189
+
93
190
  @GeneralUtilities.check_arguments
94
191
  def __copy_stream_to_file(self, stream, amount_of_bytes: int, target_file: str) -> None:
95
192
  """Copies exactly amount_of_bytes bytes from stream into target_file, chunk-wise (see
@@ -109,37 +206,92 @@ class SCTaskRunnerServer:
109
206
  def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str) -> None:
110
207
  command_folder = os.path.join(job.workspace_folder, working_directory)
111
208
  try:
112
- self.__sc.log.log(f"Run '{program} {' '.join(arguments)}' in '{command_folder}'...")
209
+ self.__sc.log.log(f"Job '{job.job_id}' runs '{program} {' '.join(arguments)}' in '{command_folder}'...")
210
+ moment_of_the_start = time.time()
211
+ moment_of_the_last_progress_log = moment_of_the_start
212
+ amount_of_lines = 0
113
213
  with subprocess.Popen([program] + arguments, cwd=command_folder, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace") as process:
114
214
  for line in process.stdout:
115
215
  with job.lock:
116
216
  job.log = job.log + line
217
+ amount_of_lines = amount_of_lines+1
218
+ if time.time()-moment_of_the_last_progress_log > _interval_between_two_progress_logs_in_seconds:
219
+ moment_of_the_last_progress_log = time.time()
220
+ self.__sc.log.log(f"Job '{job.job_id}' is still running (since {round(time.time()-moment_of_the_start)} seconds, {amount_of_lines} lines of output so far).")
117
221
  process.wait()
118
222
  with job.lock:
119
223
  job.exitcode = process.returncode
120
224
  job.state = "completed" if process.returncode == 0 else "failed"
225
+ state_of_the_job = job.state
226
+ exitcode_of_the_job = job.exitcode
227
+ #Logged so that the log of the runner states the result of every job it ran, also when no client is
228
+ #waiting for it any more: a connection to the client can break while a job runs, but the job keeps
229
+ #running and therefore has to end with a stated result on this side too.
230
+ self.__sc.log.log(f"Job '{job.job_id}' is {state_of_the_job} (exitcode: {exitcode_of_the_job}) after {round(time.time()-moment_of_the_start)} seconds.")
231
+ if exitcode_of_the_job != 0:
232
+ self.__log_the_end_of_the_output_of_the_job(job)
121
233
  except Exception as exception:
122
- with job.lock:
123
- job.log = job.log + f"\nException while running the program: {exception}\n"
124
- job.exitcode = -1
125
- job.state = "failed"
234
+ self.__end_job_as_failed_because_of_an_internal_error(job, "the program could not be run", exception)
235
+
236
+ @GeneralUtilities.check_arguments
237
+ def __log_the_end_of_the_output_of_the_job(self, job: _RunnerJob) -> None:
238
+ """Logs the last lines of the output of a job which did not succeed. The complete output belongs to the job and
239
+ is fetched by the client, but the log of the runner has to state why a build failed too: a client is not
240
+ necessarily still connected when a job ends, and its output is gone with the job."""
241
+ with job.lock:
242
+ lines = GeneralUtilities.string_to_lines(job.log)
243
+ last_lines = lines[-_amount_of_logged_lines_of_a_failed_job:]
244
+ self.__sc.log.log(f"The last {len(last_lines)} of the {len(lines)} lines of the output of job '{job.job_id}' are:")
245
+ for line in last_lines:
246
+ self.__sc.log.log(f" {line}")
247
+
248
+ @GeneralUtilities.check_arguments
249
+ def __end_job_as_failed_because_of_an_internal_error(self, job: _RunnerJob, what_did_not_work: str, exception: Exception) -> None:
250
+ """Ends a job which failed because of the runner itself and not because of what it was asked to build. The
251
+ details of such a failure - stacktraces and paths inside this runner - go to the log of the runner only: they
252
+ state something about the internals of the runner, and the one who waits for the build can not act on them. The
253
+ client gets the statement that the runner failed plus the error-id under which the details are logged, so that
254
+ both sides can be brought together. The output of the program of a job is something else: it is what the client
255
+ asked for and is transferred in full, so that a build which does not compile states why."""
256
+ error_id = str(uuid.uuid4())
257
+ self.__sc.log.log_exception(f"Internal error '{error_id}': job '{job.job_id}' failed because {what_did_not_work}.", exception)
258
+ self.__end_job_as_failed(job, f"The runner could not run this job because {what_did_not_work}. This is a problem of the runner itself; its details are in the log of the runner under the error-id '{error_id}'.")
259
+
260
+ @GeneralUtilities.check_arguments
261
+ def __end_job_as_failed(self, job: _RunnerJob, message: str) -> None:
262
+ """Ends a job which could not be run at all with the same shape of result a program which returned a non-zero
263
+ exitcode produces, so that a client always gets a state and an exitcode and does not have to distinguish the
264
+ two cases."""
265
+ job.append_to_log(message)
266
+ with job.lock:
267
+ job.exitcode = -1
268
+ job.state = "failed"
126
269
 
127
270
  @GeneralUtilities.check_arguments
128
271
  def __create_result_archive(self, job: _RunnerJob) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
129
- """Packs the codeunit-folder of the job and returns the file the archive was written to. The archive is returned
130
- as a file (and not as bytes) because it contains the whole build-output of the codeunit and therefore regularly
131
- has a size of several gigabytes, which must not be held in memory. The caller is responsible for deleting the
132
- returned file after it was transferred."""
133
- codeunit_folder = os.path.join(job.workspace_folder, job.codeunit_name)
272
+ """Packs the result-folder of the job and returns the file the archive was written to. The archive is returned
273
+ as a file (and not as bytes) because the build-output of a codeunit regularly has a size of several gigabytes,
274
+ which must not be held in memory. The caller is responsible for deleting the returned file after it was
275
+ transferred."""
134
276
  archive_file = os.path.join(self.work_folder, f"{uuid.uuid4()}.result.tar.gz")
135
277
  try:
136
278
  with tarfile.open(archive_file, "w:gz") as tar:
137
- tar.add(codeunit_folder, arcname=".")
279
+ tar.add(self.__get_result_folder_of_the_job(job), arcname=".")
138
280
  except Exception:
139
281
  GeneralUtilities.ensure_file_does_not_exist(archive_file)
140
282
  raise
141
283
  return archive_file
142
284
 
285
+ @GeneralUtilities.check_arguments
286
+ def __get_result_folder_of_the_job(self, job: _RunnerJob) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
287
+ """Returns the folder whose content is the result of the job. A folder outside of the workspace of the job is
288
+ refused: what a runner returns is what the job produced and never something else of this machine."""
289
+ result_folder = os.path.realpath(os.path.join(job.workspace_folder, job.result_folder))
290
+ workspace_folder = os.path.realpath(job.workspace_folder)
291
+ if result_folder != workspace_folder and not result_folder.startswith(workspace_folder+os.sep):
292
+ raise ValueError(f"The result-folder '{job.result_folder}' of job '{job.job_id}' is not inside the workspace of that job.")
293
+ return result_folder
294
+
143
295
  @GeneralUtilities.check_arguments
144
296
  def __delete_job(self, job_id: str) -> None: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
145
297
  with self.__jobs_lock:
@@ -155,9 +307,29 @@ class SCTaskRunnerServer:
155
307
 
156
308
  class Handler(BaseHTTPRequestHandler):
157
309
 
310
+ #Whether the answer of the request which is currently handled was already begun. After that point the
311
+ #status-line is already on the connection, so a failure can not be answered with a status-code any more.
312
+ __answer_was_begun: bool = False
313
+
158
314
  def log_message(self, format, *args): # pylint:disable=redefined-builtin
159
315
  outer._SCTaskRunnerServer__sc.log.log((format % args), LogLevel.Debug)
160
316
 
317
+ def __handle(self, handle_request) -> None:
318
+ """Runs the handling of a request and turns everything which it did not handle itself into a defined
319
+ answer. An unexpected exception here is a problem of the runner itself, so its details go to the log of
320
+ the runner only - they state something about the internals of this runner, which the one who waits for a
321
+ build can not act on - while the client gets a 500 naming the error-id under which they are logged.
322
+ Without this the request would stay unanswered completely, which a client sees as a broken connection
323
+ and not as a statement about what happened."""
324
+ self.__answer_was_begun = False
325
+ try:
326
+ handle_request()
327
+ except Exception as exception:
328
+ error_id = str(uuid.uuid4())
329
+ outer._SCTaskRunnerServer__sc.log.log_exception(f"Internal error '{error_id}' while handling '{self.command} {self.path}'.", exception)
330
+ if not self.__answer_was_begun:
331
+ self.__send_text(500, f"Internal error of the runner. Its details are in the log of the runner under the error-id '{error_id}'.")
332
+
161
333
  def __is_authorized(self) -> bool:
162
334
  header = self.headers.get("Authorization", GeneralUtilities.empty_string)
163
335
  if not header.startswith("Basic "):
@@ -170,6 +342,7 @@ class SCTaskRunnerServer:
170
342
  return decoded == expected
171
343
 
172
344
  def __send(self, status: int, body: bytes, content_type: str = "application/octet-stream") -> None:
345
+ self.__answer_was_begun = True
173
346
  self.send_response(status)
174
347
  self.send_header("Content-Type", content_type)
175
348
  self.send_header("Content-Length", str(len(body)))
@@ -179,6 +352,7 @@ class SCTaskRunnerServer:
179
352
  def __send_file(self, status: int, file: str) -> None:
180
353
  """Sends the content of a file without reading it into memory as a whole (see
181
354
  _transfer_chunk_size_in_bytes)."""
355
+ self.__answer_was_begun = True
182
356
  self.send_response(status)
183
357
  self.send_header("Content-Type", "application/octet-stream")
184
358
  self.send_header("Content-Length", str(os.path.getsize(file)))
@@ -197,6 +371,9 @@ class SCTaskRunnerServer:
197
371
  return outer._SCTaskRunnerServer__jobs.get(job_id, None)
198
372
 
199
373
  def do_GET(self): # pylint:disable=invalid-name
374
+ self.__handle(self.__get)
375
+
376
+ def __get(self) -> None:
200
377
  if not self.__is_authorized():
201
378
  self.__send_text(401, "Unauthorized")
202
379
  return
@@ -236,6 +413,11 @@ class SCTaskRunnerServer:
236
413
  # The transfer-lock is held while the archive is created and sent, so a DELETE for this job waits instead
237
414
  # of removing the workspace while it is being packed (see _RunnerJob.transfer_lock).
238
415
  with job.transfer_lock:
416
+ if not os.path.isdir(outer._SCTaskRunnerServer__get_result_folder_of_the_job(job)):
417
+ #A job which ran successfully but produced no result-folder is a defined outcome and not an
418
+ #error of the runner: what a build-step produces is stated by the client which submitted it.
419
+ self.__send_text(409, f"The job did not produce its result-folder '{job.result_folder}'.")
420
+ return
239
421
  archive_file = outer._SCTaskRunnerServer__create_result_archive(job)
240
422
  try:
241
423
  self.__send_file(200, archive_file)
@@ -243,22 +425,35 @@ class SCTaskRunnerServer:
243
425
  GeneralUtilities.ensure_file_does_not_exist(archive_file)
244
426
 
245
427
  def do_POST(self): # pylint:disable=invalid-name
428
+ self.__handle(self.__post)
429
+
430
+ def __post(self) -> None:
246
431
  if not self.__is_authorized():
247
432
  self.__send_text(401, "Unauthorized")
248
433
  return
249
434
  if self.path != "/jobs":
250
435
  self.__send_text(404, "Not found")
251
436
  return
252
- content_length = int(self.headers.get("Content-Length", "0"))
437
+ #A request which does not state what is to be run is a defect of the client and not of the runner, so it
438
+ #is answered with what is missing instead of with the 500 of an unexpected failure.
439
+ missing_headers = [header for header in ("Content-Length", "X-Codeunit-Name", "X-Program", "X-Arguments", "X-Result-Folder") if self.headers.get(header) is None]
440
+ if len(missing_headers) > 0:
441
+ self.__send_text(400, f"The request does not contain the required header(s) {', '.join(missing_headers)}.")
442
+ return
443
+ content_length = int(self.headers.get("Content-Length"))
253
444
  codeunit_name = self.headers.get("X-Codeunit-Name")
254
445
  program = self.headers.get("X-Program")
255
- arguments = json.loads(base64.b64decode(self.headers.get("X-Arguments", "")).decode("utf-8"))
446
+ arguments = json.loads(base64.b64decode(self.headers.get("X-Arguments")).decode("utf-8"))
256
447
  working_directory = self.headers.get("X-Working-Directory", ".")
448
+ result_folder = self.headers.get("X-Result-Folder")
257
449
  #the body is handed over as a stream (and not read here) so the archive is written to disk chunk-wise, see __start_job.
258
- job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory)
450
+ job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory, result_folder)
259
451
  self.__send_json(200, {"job_id": job_id})
260
452
 
261
453
  def do_DELETE(self): # pylint:disable=invalid-name
454
+ self.__handle(self.__delete)
455
+
456
+ def __delete(self) -> None:
262
457
  if not self.__is_authorized():
263
458
  self.__send_text(401, "Unauthorized")
264
459
  return