ScriptCollection 4.4.19__py3-none-any.whl → 4.4.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -39,7 +39,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
39
39
  from .ProgramRunnerPopen import ProgramRunnerPopen
40
40
  from .SCLog import SCLog, LogLevel
41
41
 
42
- version = "4.4.19"
42
+ version = "4.4.20"
43
43
  __version__ = version
44
44
 
45
45
  class VSCodeWorkspaceShellTask:
@@ -546,7 +546,15 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
546
546
  codeunit_name = os.path.basename(codeunit_folder)
547
547
  src_folder = GeneralUtilities.resolve_relative_path(package_name, codeunit_folder)
548
548
 
549
- self._protected_sc.run_with_epew("flutter", "test --coverage", src_folder)
549
+ # The testcases are executed one after the other ("--concurrency=1") although that is slower than the
550
+ # default (one test-file per processor-core): the coverage which is collected while several test-files run
551
+ # at the same time is not reproducible. Flutter attaches to the vm-service of every test-isolate to read
552
+ # its coverage, and under load a library which was loaded by another isolate is sometimes counted and
553
+ # sometimes not - the very same code produced 91.14, 91.14 and 90.58 percent in three runs, while it
554
+ # produces exactly 90.30 percent in every serial run. A coverage-value which changes without the code
555
+ # changing makes the coverage-badge of a codeunit differ after every build (so the build is not idempotent
556
+ # anymore) and it can push a codeunit below its minimal-coverage-threshold by chance.
557
+ self._protected_sc.run_with_epew("flutter", "test --coverage --concurrency=1", src_folder)
550
558
  test_coverage_folder_relative = "Other/Artifacts/TestCoverage"
551
559
  test_coverage_folder = GeneralUtilities.resolve_relative_path(test_coverage_folder_relative, codeunit_folder)
552
560
  GeneralUtilities.ensure_directory_exists(test_coverage_folder)
@@ -1,6 +1,7 @@
1
1
  import os
2
2
  import json
3
3
  import base64
4
+ import shutil
4
5
  import ssl
5
6
  import uuid
6
7
  import threading
@@ -11,6 +12,12 @@ from ..GeneralUtilities import GeneralUtilities
11
12
  from ..ScriptCollectionCore import ScriptCollectionCore
12
13
  from ..SCLog import LogLevel
13
14
 
15
+ #Size of the chunks in which the archives are copied between the network-connection and the disk. The archives are
16
+ #transferred chunk-wise instead of being held in memory as a whole because both of them are regularly large (the payload
17
+ #contains the whole repository of the client, the result contains the whole codeunit-folder including its build-output),
18
+ #while a runner often runs in a container with a small memory-limit and needs that memory for the build itself.
19
+ _transfer_chunk_size_in_bytes: int = 1024*1024
20
+
14
21
 
15
22
  class _RunnerJob:
16
23
  def __init__(self, workspace_folder: str, codeunit_name: str):
@@ -20,6 +27,12 @@ class _RunnerJob:
20
27
  self.exitcode = None
21
28
  self.log = GeneralUtilities.empty_string
22
29
  self.lock = threading.Lock()
30
+ #Serializes the transfer of the result-archive against the deletion of the workspace. A client deletes the job as
31
+ #soon as it stops waiting for the result - which also happens when it gives up on a still-running result-request,
32
+ #for example because a reverse-proxy in front of the runner ran into its timeout. Without this lock that deletion
33
+ #removes the workspace while it is being packed, which makes the transfer fail with a FileNotFoundError naming an
34
+ #arbitrary file of the build-output and can leave a partially deleted workspace behind.
35
+ self.transfer_lock = threading.Lock()
23
36
 
24
37
 
25
38
  class SCTaskRunnerServer:
@@ -53,15 +66,19 @@ class SCTaskRunnerServer:
53
66
  server.serve_forever()
54
67
 
55
68
  @GeneralUtilities.check_arguments
56
- def __start_job(self, archive_bytes: bytes, codeunit_name: str, program: str, arguments: list[str], working_directory: str) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
69
+ def __start_job(self, archive_stream, archive_size: int, codeunit_name: str, program: str, arguments: list[str], working_directory: str) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
70
+ """Starts a job for the repository-archive which can be read from archive_stream (the body of the request which
71
+ submitted the job; archive_size is its announced length). The archive is taken as a stream instead of as bytes
72
+ because it contains the complete repository of the client and is therefore regularly several hundred megabytes,
73
+ which must not be held in memory in addition to the build which runs afterwards."""
57
74
  job_id = str(uuid.uuid4())
58
75
  workspace_folder = os.path.join(self.work_folder, job_id)
59
76
  # Isolation: each job gets a fresh, empty workspace-folder.
60
77
  GeneralUtilities.ensure_directory_does_not_exist(workspace_folder)
61
78
  GeneralUtilities.ensure_directory_exists(workspace_folder)
62
79
  archive_file = os.path.join(self.work_folder, f"{job_id}.payload.tar.gz")
63
- GeneralUtilities.write_binary_to_file(archive_file, archive_bytes)
64
80
  try:
81
+ self.__copy_stream_to_file(archive_stream, archive_size, archive_file)
65
82
  with tarfile.open(archive_file, "r:gz") as tar:
66
83
  tar.extractall(workspace_folder, filter="fully_trusted")
67
84
  finally:
@@ -73,6 +90,21 @@ class SCTaskRunnerServer:
73
90
  thread.start()
74
91
  return job_id
75
92
 
93
+ @GeneralUtilities.check_arguments
94
+ def __copy_stream_to_file(self, stream, amount_of_bytes: int, target_file: str) -> None:
95
+ """Copies exactly amount_of_bytes bytes from stream into target_file, chunk-wise (see
96
+ _transfer_chunk_size_in_bytes for why the content is not read into memory as a whole). A stream which ends early
97
+ is reported as an error instead of resulting in a truncated file which would fail later as an unspecific
98
+ archive-error."""
99
+ remaining_amount_of_bytes = amount_of_bytes
100
+ with open(target_file, "wb") as target_file_object:
101
+ while 0 < remaining_amount_of_bytes:
102
+ chunk = stream.read(min(_transfer_chunk_size_in_bytes, remaining_amount_of_bytes))
103
+ if not chunk:
104
+ raise ValueError(f"The transferred content ended after {amount_of_bytes-remaining_amount_of_bytes} bytes although {amount_of_bytes} bytes were announced.")
105
+ target_file_object.write(chunk)
106
+ remaining_amount_of_bytes = remaining_amount_of_bytes-len(chunk)
107
+
76
108
  @GeneralUtilities.check_arguments
77
109
  def __run_job(self, job: _RunnerJob, program: str, arguments: list[str], working_directory: str) -> None:
78
110
  command_folder = os.path.join(job.workspace_folder, working_directory)
@@ -93,22 +125,30 @@ class SCTaskRunnerServer:
93
125
  job.state = "failed"
94
126
 
95
127
  @GeneralUtilities.check_arguments
96
- def __get_result_archive(self, job: _RunnerJob) -> bytes: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
128
+ def __create_result_archive(self, job: _RunnerJob) -> str: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
129
+ """Packs the codeunit-folder of the job and returns the file the archive was written to. The archive is returned
130
+ as a file (and not as bytes) because it contains the whole build-output of the codeunit and therefore regularly
131
+ has a size of several gigabytes, which must not be held in memory. The caller is responsible for deleting the
132
+ returned file after it was transferred."""
97
133
  codeunit_folder = os.path.join(job.workspace_folder, job.codeunit_name)
98
134
  archive_file = os.path.join(self.work_folder, f"{uuid.uuid4()}.result.tar.gz")
99
135
  try:
100
136
  with tarfile.open(archive_file, "w:gz") as tar:
101
137
  tar.add(codeunit_folder, arcname=".")
102
- return GeneralUtilities.read_binary_from_file(archive_file)
103
- finally:
138
+ except Exception:
104
139
  GeneralUtilities.ensure_file_does_not_exist(archive_file)
140
+ raise
141
+ return archive_file
105
142
 
106
143
  @GeneralUtilities.check_arguments
107
144
  def __delete_job(self, job_id: str) -> None: # pylint:disable=unused-private-member # accessed via name-mangling inside the nested request-handler
108
145
  with self.__jobs_lock:
109
146
  job = self.__jobs.pop(job_id, None)
110
147
  if job is not None:
111
- GeneralUtilities.ensure_directory_does_not_exist(job.workspace_folder)
148
+ # The job was already removed from the job-list above, so no further result-request can start for it. This
149
+ # waits for one which is still running (see _RunnerJob.transfer_lock) before the workspace is removed.
150
+ with job.transfer_lock:
151
+ GeneralUtilities.ensure_directory_does_not_exist(job.workspace_folder)
112
152
 
113
153
  def __create_request_handler(self):
114
154
  outer = self
@@ -136,6 +176,16 @@ class SCTaskRunnerServer:
136
176
  self.end_headers()
137
177
  self.wfile.write(body)
138
178
 
179
+ def __send_file(self, status: int, file: str) -> None:
180
+ """Sends the content of a file without reading it into memory as a whole (see
181
+ _transfer_chunk_size_in_bytes)."""
182
+ self.send_response(status)
183
+ self.send_header("Content-Type", "application/octet-stream")
184
+ self.send_header("Content-Length", str(os.path.getsize(file)))
185
+ self.end_headers()
186
+ with open(file, "rb") as file_object:
187
+ shutil.copyfileobj(file_object, self.wfile, _transfer_chunk_size_in_bytes)
188
+
139
189
  def __send_text(self, status: int, text: str) -> None:
140
190
  self.__send(status, text.encode("utf-8"), "text/plain; charset=utf-8")
141
191
 
@@ -178,10 +228,20 @@ class SCTaskRunnerServer:
178
228
  if not ready:
179
229
  self.__send_text(409, "Job is not completed")
180
230
  return
181
- self.__send(200, outer._SCTaskRunnerServer__get_result_archive(job))
231
+ self.__send_result_archive(job)
182
232
  return
183
233
  self.__send_text(404, "Not found")
184
234
 
235
+ def __send_result_archive(self, job: _RunnerJob) -> None:
236
+ # The transfer-lock is held while the archive is created and sent, so a DELETE for this job waits instead
237
+ # of removing the workspace while it is being packed (see _RunnerJob.transfer_lock).
238
+ with job.transfer_lock:
239
+ archive_file = outer._SCTaskRunnerServer__create_result_archive(job)
240
+ try:
241
+ self.__send_file(200, archive_file)
242
+ finally:
243
+ GeneralUtilities.ensure_file_does_not_exist(archive_file)
244
+
185
245
  def do_POST(self): # pylint:disable=invalid-name
186
246
  if not self.__is_authorized():
187
247
  self.__send_text(401, "Unauthorized")
@@ -190,12 +250,12 @@ class SCTaskRunnerServer:
190
250
  self.__send_text(404, "Not found")
191
251
  return
192
252
  content_length = int(self.headers.get("Content-Length", "0"))
193
- archive_bytes = self.rfile.read(content_length)
194
253
  codeunit_name = self.headers.get("X-Codeunit-Name")
195
254
  program = self.headers.get("X-Program")
196
255
  arguments = json.loads(base64.b64decode(self.headers.get("X-Arguments", "")).decode("utf-8"))
197
256
  working_directory = self.headers.get("X-Working-Directory", ".")
198
- job_id = outer._SCTaskRunnerServer__start_job(archive_bytes, codeunit_name, program, arguments, working_directory)
257
+ #the body is handed over as a stream (and not read here) so the archive is written to disk chunk-wise, see __start_job.
258
+ job_id = outer._SCTaskRunnerServer__start_job(self.rfile, content_length, codeunit_name, program, arguments, working_directory)
199
259
  self.__send_json(200, {"job_id": job_id})
200
260
 
201
261
  def do_DELETE(self): # pylint:disable=invalid-name
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptcollection
3
- Version: 4.4.19
3
+ Version: 4.4.20
4
4
  Summary: The ScriptCollection is the place for reusable scripts.
5
5
  Author-email: Marius Göcke <marius.goecke@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/anionDev/ScriptCollection
@@ -12,7 +12,7 @@ ScriptCollection/ProgramRunnerSudo.py,sha256=ot9Ib-plQ8wejR9M2XXfnx6GR-va9k8SH1g
12
12
  ScriptCollection/RepositoryDiagramGenerator.py,sha256=K_u2NW_p4ozdWD50irJEkkcDVa70TcUJpZN63TeXSLw,7092
13
13
  ScriptCollection/RepositoryHistoryAnalyzer.py,sha256=rD8eC5lMUe5MHmCBHfYft_G-N9IbvtsFGkU7EMwPjrY,5779
14
14
  ScriptCollection/SCLog.py,sha256=Fjf8Pdi63cXHGMxL_aI6A0dwGm7QIfrv3czjvpSUlaM,4705
15
- ScriptCollection/ScriptCollectionCore.py,sha256=SwQlqDCXwL-O1jcOBdOyb7KEWlZDMccNaaijvC-jzmE,270068
15
+ ScriptCollection/ScriptCollectionCore.py,sha256=2iA-A2d_dldeefMRdI4mtW1RsC-fTvY27CKOynJhGgw,270068
16
16
  ScriptCollection/TestKind.py,sha256=EBTLR8yBz5VmWA2NZIpDTbQNg6tkfv_kAsIzEJ--mCI,1756
17
17
  ScriptCollection/UnassignedVariablesCheck.py,sha256=8xNcp1D5HeOLO1a1NEerq2TsnpFhw7yzYPkaay-oisY,3804
18
18
  ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -38,7 +38,7 @@ ScriptCollection/Resources/Fonts/DejaVuSans.ttf,sha256=faGVp0xVvvmI0NSPlQi9XYSUJ
38
38
  ScriptCollection/Resources/Fonts/LICENSE.txt,sha256=egg7E25k0GR5TDQZdR5cfdENL2TBCP5boWHq5eWVipM,8816
39
39
  ScriptCollection/Resources/MaintenanceSite/MaintenanceSite.html,sha256=CX9S1bdOz6xU2aGfr03tJRGczQrlpn-IeODc15d5kc0,232
40
40
  ScriptCollection/TFCPS/PackageSource.py,sha256=tQoMbxrIbV3mYHKH2wgOoGzIHY1TvBH8EAFpWGhryrQ,1311
41
- ScriptCollection/TFCPS/SCTaskRunnerServer.py,sha256=rpuwSsQ6YQfZS2LGnvOH5ql8zs_-cHU4LQttS7XNT60,11272
41
+ ScriptCollection/TFCPS/SCTaskRunnerServer.py,sha256=-pxCbqK742EP5waQj9k7y0JwOv_RWPxDWCjrCX_gcVc,15917
42
42
  ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py,sha256=67_DQwoH5bhxxSmwtuAeoW1NOCMiLMzOy48OqlJX-KY,1744
43
43
  ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=ngbtRfZiu6RDeV9RuxmGz7SiACQ-t1O9S6HZte-FxK8,40005
44
44
  ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=Mj9t1SyRpkYZE74cLsGWYv4xB89gXL2ld-RX3xvvGdw,9835
@@ -62,7 +62,7 @@ ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py,sha256=
62
62
  ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py,sha256=i0zEGehj0sttxjjZtoq2KFSKp_ulxVyWp_ZgAhIY_So,241
63
63
  ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=GCFzDNAnmm40EeqOYgD_EeuEwDY58ujwJ1-Zn_naamc,61797
64
64
  ScriptCollection/TFCPS/DotNet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
65
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=8u7VriF6HGQxVNF1npojjcFyZ1ufVUot9LlJjUJuziI,49386
65
+ ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=U23Y_lNdgTsjhT2edVfqS8V2tbZqsNsQ7DRyJxM3wWY,50278
66
66
  ScriptCollection/TFCPS/Flutter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
67
67
  ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py,sha256=m4Pax-U6o5XuspLnLfnWjyQMtoC7zBDDNJW9EhLWjeE,9012
68
68
  ScriptCollection/TFCPS/Go/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -76,8 +76,8 @@ ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=yyjWPMHQjPLkML
76
76
  ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
77
77
  ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py,sha256=5z990qyO5durX07xRWRwju1hl6wpjQAy2f-yRwJ-7s0,11856
78
78
  ScriptCollection/TFCPS/TypeScript/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
79
- scriptcollection-4.4.19.dist-info/METADATA,sha256=EQOL_andRQ3Km02Zed7SToJuv5qC91KAseyDNlz1Q-0,7690
80
- scriptcollection-4.4.19.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
81
- scriptcollection-4.4.19.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
82
- scriptcollection-4.4.19.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
83
- scriptcollection-4.4.19.dist-info/RECORD,,
79
+ scriptcollection-4.4.20.dist-info/METADATA,sha256=BA4KYDdeFjskuRd1BQCI_yeeyYNPJqdhb7wKL25YKH8,7690
80
+ scriptcollection-4.4.20.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
81
+ scriptcollection-4.4.20.dist-info/entry_points.txt,sha256=GNma7tN0J59EvK71su-G1w22RnZ05bXcPrELyLx3AL0,6151
82
+ scriptcollection-4.4.20.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
83
+ scriptcollection-4.4.20.dist-info/RECORD,,