ScriptCollection 4.4.19__py3-none-any.whl → 4.4.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/ScriptCollectionCore.py +1 -1
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +9 -1
- ScriptCollection/TFCPS/SCTaskRunnerServer.py +69 -9
- {scriptcollection-4.4.19.dist-info → scriptcollection-4.4.20.dist-info}/METADATA +1 -1
- {scriptcollection-4.4.19.dist-info → scriptcollection-4.4.20.dist-info}/RECORD +8 -8
- {scriptcollection-4.4.19.dist-info → scriptcollection-4.4.20.dist-info}/WHEEL +0 -0
- {scriptcollection-4.4.19.dist-info → scriptcollection-4.4.20.dist-info}/entry_points.txt +0 -0
- {scriptcollection-4.4.19.dist-info → scriptcollection-4.4.20.dist-info}/top_level.txt +0 -0
|
@@ -546,7 +546,15 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
|
|
|
546
546
|
codeunit_name = os.path.basename(codeunit_folder)
|
|
547
547
|
src_folder = GeneralUtilities.resolve_relative_path(package_name, codeunit_folder)
|
|
548
548
|
|
|
549
|
-
|
|
549
|
+
# The testcases are executed one after the other ("--concurrency=1") although that is slower than the
|
|
550
|
+
# default (one test-file per processor-core): the coverage which is collected while several test-files run
|
|
551
|
+
# at the same time is not reproducible. Flutter attaches to the vm-service of every test-isolate to read
|
|
552
|
+
# its coverage, and under load a library which was loaded by another isolate is sometimes counted and
|
|
553
|
+
# sometimes not - the very same code produced 91.14, 91.14 and 90.58 percent in three runs, while it
|
|
554
|
+
# produces exactly 90.30 percent in every serial run. A coverage-value which changes without the code
|
|
555
|
+
# changing makes the coverage-badge of a codeunit differ after every build (so the build is not idempotent
|
|
556
|
+
# anymore) and it can push a codeunit below its minimal-coverage-threshold by chance.
|
|
557
|
+
self._protected_sc.run_with_epew("flutter", "test --coverage --concurrency=1", src_folder)
|
|
550
558
|
test_coverage_folder_relative = "Other/Artifacts/TestCoverage"
|
|
551
559
|
test_coverage_folder = GeneralUtilities.resolve_relative_path(test_coverage_folder_relative, codeunit_folder)
|
|
552
560
|
GeneralUtilities.ensure_directory_exists(test_coverage_folder)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import json
|
|
3
3
|
import base64
|
|
4
|
+
import shutil
|
|
4
5
|
import ssl
|
|
5
6
|
import uuid
|
|
6
7
|
import threading
|
|
@@ -11,6 +12,12 @@ from ..GeneralUtilities import GeneralUtilities
|
|
|
11
12
|
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
12
13
|
from ..SCLog import LogLevel
|
|
13
14
|
|
|
15
|
+
#Size of the chunks in which the archives are copied between the network-connection and the disk. The archives are
|
|
16
|
+
#transferred chunk-wise instead of being held in memory as a whole because both of them are regularly large (the payload
|
|
17
|
+
#contains the whole repository of the client, the result contains the whole codeunit-folder including its build-output),
|
|
18
|
+
#while a runner often runs in a container with a small memory-limit and needs that memory for the build itself.
|
|
19
|
+
_transfer_chunk_size_in_bytes: int = 1024*1024
|
|
20
|
+
|
|
14
21
|
|
|
15
22
|
class _RunnerJob:
|
|
16
23
|
def __init__(self, workspace_folder: str, codeunit_name: str):
|
|
@@ -20,6 +27,12 @@ class _RunnerJob:
|
|
|
20
27
|
self.exitcode = None
|
|
21
28
|
self.log = GeneralUtilities.empty_string
|
|
22
29
|
self.lock = threading.Lock()
|
|
30
|
+
#Serializes the transfer of the result-archive against the deletion of the workspace. A client deletes the job as
|
|
31
|
+
#soon as it stops waiting for the result - which also happens when it gives up on a still-running result-request,
|
|
32
|
+
#for example because a reverse-proxy in front of the runner ran into its timeout. Without this lock that deletion
|
|
33
|
+
#removes the workspace while it is being packed, which makes the transfer fail with a FileNotFoundError naming an
|
|
34
|
+
#arbitrary file of the build-output and can leave a partially deleted workspace behind.
|
|
35
|
+
self.transfer_lock = threading.Lock()
|
|
23
36
|
|
|
24
37
|
|
|
25
38
|
class SCTaskRunnerServer:
|
|
@@ -53,15 +66,19 @@ class SCTaskRunnerServer:
|
|
|
53
66
|
server.serve_forever()
|
|
54
67
|
|
|
55
68
|
@GeneralUtilities.check_arguments
|
|
56
|
-
def __start_job(self,
|
|
69
|
+
def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
70
|
+
"""Starts a job for the repository-archive which can be read from archive_stream (the body of the request which
|
|
71
|
+
submitted the job; archive_size is its announced length). The archive is taken as a stream instead of as bytes
|
|
72
|
+
because it contains the complete repository of the client and is therefore regularly several hundred megabytes,
|
|
73
|
+
which must not be held in memory in addition to the build which runs afterwards."""
|
|
57
74
|
job_id = str(uuid.uuid4())
|
|
58
75
|
workspace_folder = os.path.join(self.work_folder, job_id)
|
|
59
76
|
# Isolation: each job gets a fresh, empty workspace-folder.
|
|
60
77
|
GeneralUtilities.ensure_directory_does_not_exist(workspace_folder)
|
|
61
78
|
GeneralUtilities.ensure_directory_exists(workspace_folder)
|
|
62
79
|
archive_file = os.path.join(self.work_folder, f"{job_id}.payload.tar.gz")
|
|
63
|
-
GeneralUtilities.write_binary_to_file(archive_file, archive_bytes)
|
|
64
80
|
try:
|
|
81
|
+
self.__copy_stream_to_file(archive_stream, archive_size, archive_file)
|
|
65
82
|
with tarfile.open(archive_file, "r:gz") as tar:
|
|
66
83
|
tar.extractall(workspace_folder, filter="fully_trusted")
|
|
67
84
|
finally:
|
|
@@ -73,6 +90,21 @@ class SCTaskRunnerServer:
|
|
|
73
90
|
thread.start()
|
|
74
91
|
return job_id
|
|
75
92
|
|
|
93
|
+
@GeneralUtilities.check_arguments
|
|
94
|
+
def __copy_stream_to_file(self, stream, amount_of_bytes: int, target_file: str) -> None:
|
|
95
|
+
"""Copies exactly amount_of_bytes bytes from stream into target_file, chunk-wise (see
|
|
96
|
+
_transfer_chunk_size_in_bytes for why the content is not read into memory as a whole). A stream which ends early
|
|
97
|
+
is reported as an error instead of resulting in a truncated file which would fail later as an unspecific
|
|
98
|
+
archive-error."""
|
|
99
|
+
remaining_amount_of_bytes = amount_of_bytes
|
|
100
|
+
with open(target_file, "wb") as target_file_object:
|
|
101
|
+
while 0 < remaining_amount_of_bytes:
|
|
102
|
+
chunk = stream.read(min(_transfer_chunk_size_in_bytes, remaining_amount_of_bytes))
|
|
103
|
+
if not chunk:
|
|
104
|
+
raise ValueError(f"The transferred content ended after {amount_of_bytes-remaining_amount_of_bytes} bytes although {amount_of_bytes} bytes were announced.")
|
|
105
|
+
target_file_object.write(chunk)
|
|
106
|
+
remaining_amount_of_bytes = remaining_amount_of_bytes-len(chunk)
|
|
107
|
+
|
|
76
108
|
@GeneralUtilities.check_arguments
|
|
77
109
|
def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str) -> None:
|
|
78
110
|
command_folder = os.path.join(job.workspace_folder, working_directory)
|
|
@@ -93,22 +125,30 @@ class SCTaskRunnerServer:
|
|
|
93
125
|
job.state = "failed"
|
|
94
126
|
|
|
95
127
|
@GeneralUtilities.check_arguments
|
|
96
|
-
def
|
|
128
|
+
def __create_result_archive(self, job: _RunnerJob) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
129
|
+
"""Packs the codeunit-folder of the job and returns the file the archive was written to. The archive is returned
|
|
130
|
+
as a file (and not as bytes) because it contains the whole build-output of the codeunit and therefore regularly
|
|
131
|
+
has a size of several gigabytes, which must not be held in memory. The caller is responsible for deleting the
|
|
132
|
+
returned file after it was transferred."""
|
|
97
133
|
codeunit_folder = os.path.join(job.workspace_folder, job.codeunit_name)
|
|
98
134
|
archive_file = os.path.join(self.work_folder, f"{uuid.uuid4()}.result.tar.gz")
|
|
99
135
|
try:
|
|
100
136
|
with tarfile.open(archive_file, "w:gz") as tar:
|
|
101
137
|
tar.add(codeunit_folder, arcname=".")
|
|
102
|
-
|
|
103
|
-
finally:
|
|
138
|
+
except Exception:
|
|
104
139
|
GeneralUtilities.ensure_file_does_not_exist(archive_file)
|
|
140
|
+
raise
|
|
141
|
+
return archive_file
|
|
105
142
|
|
|
106
143
|
@GeneralUtilities.check_arguments
|
|
107
144
|
def __delete_job(self, job_id: str) -> None: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
|
|
108
145
|
with self.__jobs_lock:
|
|
109
146
|
job = self.__jobs.pop(job_id, None)
|
|
110
147
|
if job is not None:
|
|
111
|
-
|
|
148
|
+
# The job was already removed from the job-list above, so no further result-request can start for it. This
|
|
149
|
+
# waits for one which is still running (see _RunnerJob.transfer_lock) before the workspace is removed.
|
|
150
|
+
with job.transfer_lock:
|
|
151
|
+
GeneralUtilities.ensure_directory_does_not_exist(job.workspace_folder)
|
|
112
152
|
|
|
113
153
|
def __create_request_handler(self):
|
|
114
154
|
outer = self
|
|
@@ -136,6 +176,16 @@ class SCTaskRunnerServer:
|
|
|
136
176
|
self.end_headers()
|
|
137
177
|
self.wfile.write(body)
|
|
138
178
|
|
|
179
|
+
def __send_file(self, status: int, file: str) -> None:
|
|
180
|
+
"""Sends the content of a file without reading it into memory as a whole (see
|
|
181
|
+
_transfer_chunk_size_in_bytes)."""
|
|
182
|
+
self.send_response(status)
|
|
183
|
+
self.send_header("Content-Type", "application/octet-stream")
|
|
184
|
+
self.send_header("Content-Length", str(os.path.getsize(file)))
|
|
185
|
+
self.end_headers()
|
|
186
|
+
with open(file, "rb") as file_object:
|
|
187
|
+
shutil.copyfileobj(file_object, self.wfile, _transfer_chunk_size_in_bytes)
|
|
188
|
+
|
|
139
189
|
def __send_text(self, status: int, text: str) -> None:
|
|
140
190
|
self.__send(status, text.encode("utf-8"), "text/plain; charset=utf-8")
|
|
141
191
|
|
|
@@ -178,10 +228,20 @@ class SCTaskRunnerServer:
|
|
|
178
228
|
if not ready:
|
|
179
229
|
self.__send_text(409, "Job is not completed")
|
|
180
230
|
return
|
|
181
|
-
self.
|
|
231
|
+
self.__send_result_archive(job)
|
|
182
232
|
return
|
|
183
233
|
self.__send_text(404, "Not found")
|
|
184
234
|
|
|
235
|
+
def __send_result_archive(self, job: _RunnerJob) -> None:
|
|
236
|
+
# The transfer-lock is held while the archive is created and sent, so a DELETE for this job waits instead
|
|
237
|
+
# of removing the workspace while it is being packed (see _RunnerJob.transfer_lock).
|
|
238
|
+
with job.transfer_lock:
|
|
239
|
+
archive_file = outer._SCTaskRunnerServer__create_result_archive(job)
|
|
240
|
+
try:
|
|
241
|
+
self.__send_file(200, archive_file)
|
|
242
|
+
finally:
|
|
243
|
+
GeneralUtilities.ensure_file_does_not_exist(archive_file)
|
|
244
|
+
|
|
185
245
|
def do_POST(self): # pylint:disable=invalid-name
|
|
186
246
|
if not self.__is_authorized():
|
|
187
247
|
self.__send_text(401, "Unauthorized")
|
|
@@ -190,12 +250,12 @@ class SCTaskRunnerServer:
|
|
|
190
250
|
self.__send_text(404, "Not found")
|
|
191
251
|
return
|
|
192
252
|
content_length = int(self.headers.get("Content-Length", "0"))
|
|
193
|
-
archive_bytes = self.rfile.read(content_length)
|
|
194
253
|
codeunit_name = self.headers.get("X-Codeunit-Name")
|
|
195
254
|
program = self.headers.get("X-Program")
|
|
196
255
|
arguments = json.loads(base64.b64decode(self.headers.get("X-Arguments", "")).decode("utf-8"))
|
|
197
256
|
working_directory = self.headers.get("X-Working-Directory", ".")
|
|
198
|
-
|
|
257
|
+
#the body is handed over as a stream (and not read here) so the archive is written to disk chunk-wise, see __start_job.
|
|
258
|
+
job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory)
|
|
199
259
|
self.__send_json(200, {"job_id": job_id})
|
|
200
260
|
|
|
201
261
|
def do_DELETE(self): # pylint:disable=invalid-name
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scriptcollection
|
|
3
|
-
Version: 4.4.
|
|
3
|
+
Version: 4.4.20
|
|
4
4
|
Summary: The ScriptCollection is the place for reusable scripts.
|
|
5
5
|
Author-email: Marius Göcke <marius.goecke@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/anionDev/ScriptCollection
|
|
@@ -12,7 +12,7 @@ ScriptCollection/ProgramRunnerSudo.py,sha256=ot9Ib-plQ8wejR9M2XXfnx6GR-va9k8SH1g
|
|
|
12
12
|
ScriptCollection/RepositoryDiagramGenerator.py,sha256=K_u2NW_p4ozdWD50irJEkkcDVa70TcUJpZN63TeXSLw,7092
|
|
13
13
|
ScriptCollection/RepositoryHistoryAnalyzer.py,sha256=rD8eC5lMUe5MHmCBHfYft_G-N9IbvtsFGkU7EMwPjrY,5779
|
|
14
14
|
ScriptCollection/SCLog.py,sha256=Fjf8Pdi63cXHGMxL_aI6A0dwGm7QIfrv3czjvpSUlaM,4705
|
|
15
|
-
ScriptCollection/ScriptCollectionCore.py,sha256=
|
|
15
|
+
ScriptCollection/ScriptCollectionCore.py,sha256=2iA-A2d_dldeefMRdI4mtW1RsC-fTvY27CKOynJhGgw,270068
|
|
16
16
|
ScriptCollection/TestKind.py,sha256=EBTLR8yBz5VmWA2NZIpDTbQNg6tkfv_kAsIzEJ--mCI,1756
|
|
17
17
|
ScriptCollection/UnassignedVariablesCheck.py,sha256=8xNcp1D5HeOLO1a1NEerq2TsnpFhw7yzYPkaay-oisY,3804
|
|
18
18
|
ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -38,7 +38,7 @@ ScriptCollection/Resources/Fonts/DejaVuSans.ttf,sha256=faGVp0xVvvmI0NSPlQi9XYSUJ
|
|
|
38
38
|
ScriptCollection/Resources/Fonts/LICENSE.txt,sha256=egg7E25k0GR5TDQZdR5cfdENL2TBCP5boWHq5eWVipM,8816
|
|
39
39
|
ScriptCollection/Resources/MaintenanceSite/MaintenanceSite.html,sha256=CX9S1bdOz6xU2aGfr03tJRGczQrlpn-IeODc15d5kc0,232
|
|
40
40
|
ScriptCollection/TFCPS/PackageSource.py,sha256=tQoMbxrIbV3mYHKH2wgOoGzIHY1TvBH8EAFpWGhryrQ,1311
|
|
41
|
-
ScriptCollection/TFCPS/SCTaskRunnerServer.py,sha256
|
|
41
|
+
ScriptCollection/TFCPS/SCTaskRunnerServer.py,sha256=-pxCbqK742EP5waQj9k7y0JwOv_RWPxDWCjrCX_gcVc,15917
|
|
42
42
|
ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py,sha256=67_DQwoH5bhxxSmwtuAeoW1NOCMiLMzOy48OqlJX-KY,1744
|
|
43
43
|
ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=ngbtRfZiu6RDeV9RuxmGz7SiACQ-t1O9S6HZte-FxK8,40005
|
|
44
44
|
ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=Mj9t1SyRpkYZE74cLsGWYv4xB89gXL2ld-RX3xvvGdw,9835
|
|
@@ -62,7 +62,7 @@ ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py,sha256=
|
|
|
62
62
|
ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py,sha256=i0zEGehj0sttxjjZtoq2KFSKp_ulxVyWp_ZgAhIY_So,241
|
|
63
63
|
ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=GCFzDNAnmm40EeqOYgD_EeuEwDY58ujwJ1-Zn_naamc,61797
|
|
64
64
|
ScriptCollection/TFCPS/DotNet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
65
|
-
ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=
|
|
65
|
+
ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=U23Y_lNdgTsjhT2edVfqS8V2tbZqsNsQ7DRyJxM3wWY,50278
|
|
66
66
|
ScriptCollection/TFCPS/Flutter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
67
67
|
ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py,sha256=m4Pax-U6o5XuspLnLfnWjyQMtoC7zBDDNJW9EhLWjeE,9012
|
|
68
68
|
ScriptCollection/TFCPS/Go/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -76,8 +76,8 @@ ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=yyjWPMHQjPLkML
|
|
|
76
76
|
ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
77
77
|
ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py,sha256=5z990qyO5durX07xRWRwju1hl6wpjQAy2f-yRwJ-7s0,11856
|
|
78
78
|
ScriptCollection/TFCPS/TypeScript/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
79
|
-
scriptcollection-4.4.
|
|
80
|
-
scriptcollection-4.4.
|
|
81
|
-
scriptcollection-4.4.
|
|
82
|
-
scriptcollection-4.4.
|
|
83
|
-
scriptcollection-4.4.
|
|
79
|
+
scriptcollection-4.4.20.dist-info/METADATA,sha256=BA4KYDdeFjskuRd1BQCI_yeeyYNPJqdhb7wKL25YKH8,7690
|
|
80
|
+
scriptcollection-4.4.20.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
81
|
+
scriptcollection-4.4.20.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
|
|
82
|
+
scriptcollection-4.4.20.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
|
|
83
|
+
scriptcollection-4.4.20.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|