pybuilder-integration 107__tar.gz → 108__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pybuilder_integration-107 → pybuilder_integration-108}/PKG-INFO +1 -1
- {pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration/tasks.py +472 -39
- {pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration.egg-info/PKG-INFO +1 -1
- {pybuilder_integration-107 → pybuilder_integration-108}/setup.py +1 -1
- {pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration/__init__.py +0 -0
- {pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration/artifact_manager.py +0 -0
- {pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration/cloudwatchlogs_utility.py +0 -0
- {pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration/directory_utility.py +0 -0
- {pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration/exec_utility.py +0 -0
- {pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration/properties.py +0 -0
- {pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration/tool_utility.py +0 -0
- {pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration.egg-info/SOURCES.txt +0 -0
- {pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration.egg-info/dependency_links.txt +0 -0
- {pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration.egg-info/namespace_packages.txt +0 -0
- {pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration.egg-info/requires.txt +0 -0
- {pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration.egg-info/top_level.txt +0 -0
- {pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration.egg-info/zip-safe +0 -0
- {pybuilder_integration-107 → pybuilder_integration-108}/setup.cfg +0 -0
|
@@ -1,9 +1,12 @@
|
|
|
1
|
+
import atexit
|
|
2
|
+
import errno
|
|
1
3
|
import json
|
|
2
4
|
import os
|
|
3
5
|
import re
|
|
4
6
|
import shutil
|
|
5
7
|
import subprocess
|
|
6
8
|
import tempfile
|
|
9
|
+
import threading
|
|
7
10
|
|
|
8
11
|
from defusedxml import ElementTree as ET
|
|
9
12
|
from defusedxml.common import DefusedXmlException
|
|
@@ -124,15 +127,21 @@ def _run_verify_environment_passes_in_subprocesses(project: Project, logger: Log
|
|
|
124
127
|
process, log_file = _start_verify_environment_subprocess(project, logger, pass_config, properties_file)
|
|
125
128
|
processes.append((pass_config, process, log_file))
|
|
126
129
|
|
|
127
|
-
|
|
130
|
+
results = []
|
|
128
131
|
for pass_config, process, log_file in processes:
|
|
129
132
|
exit_code = process.wait()
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
133
|
+
_close_subprocess_log(process)
|
|
134
|
+
results.append({
|
|
135
|
+
"name": pass_config["name"],
|
|
136
|
+
"exit_code": exit_code,
|
|
137
|
+
"log_file": log_file,
|
|
138
|
+
})
|
|
139
|
+
|
|
140
|
+
_log_parallel_verify_environment_summary(logger, results)
|
|
141
|
+
failures = [
|
|
142
|
+
f"{result['name']} exit_code={result['exit_code']} log={result['log_file']}"
|
|
143
|
+
for result in results if result["exit_code"] != 0
|
|
144
|
+
]
|
|
136
145
|
if failures:
|
|
137
146
|
raise BuildFailedException("Parallel verify_environment failed: " + "; ".join(failures))
|
|
138
147
|
finally:
|
|
@@ -141,24 +150,422 @@ def _run_verify_environment_passes_in_subprocesses(project: Project, logger: Log
|
|
|
141
150
|
if callable(poll) and poll() is None:
|
|
142
151
|
process.terminate()
|
|
143
152
|
process.wait()
|
|
144
|
-
|
|
145
|
-
if log_handle and not log_handle.closed:
|
|
146
|
-
log_handle.close()
|
|
153
|
+
_close_subprocess_log(process)
|
|
147
154
|
_merge_parallel_verify_environment_outputs(project, passes)
|
|
155
|
+
_write_cypress_summary(project, logger)
|
|
148
156
|
if os.path.exists(properties_file):
|
|
149
157
|
os.remove(properties_file)
|
|
150
158
|
|
|
151
159
|
|
|
160
|
+
_ANSI_ESCAPE = re.compile(r"\x1b\[[0-9;]*m")
|
|
161
|
+
_SUITE_RUNNING_LINE = re.compile(
|
|
162
|
+
r"^\[INFO\]\s+Running (?!tavern tests\b|against target\b|cypress on host\b|args:).+"
|
|
163
|
+
)
|
|
164
|
+
_SECRET_QUOTE = r"(?:['\"]|"|')?"
|
|
165
|
+
_SECRET_VALUE = rf"{_SECRET_QUOTE}[^'\"\s<&>,}}]+"
|
|
166
|
+
_SECRET_LITERAL_VALUE = rf"(?!(?:true|false|yes|no|on|off|null|none)\b){_SECRET_VALUE}"
|
|
167
|
+
_SECRET_KEY_NAME = (
|
|
168
|
+
r"(?:[A-Za-z][A-Za-z0-9]*[_-])*"
|
|
169
|
+
r"(?:"
|
|
170
|
+
r"x-api-key|x-amz-security-token|"
|
|
171
|
+
r"api[_-]?key|secret[_-]?access[_-]?key|access[_-]?key(?:[_-]?id)?|"
|
|
172
|
+
r"client[_-]?secret|password|secret|"
|
|
173
|
+
r"(?:id|access|refresh|session)?[_-]?token"
|
|
174
|
+
r")"
|
|
175
|
+
)
|
|
176
|
+
_SECRET_HINT_LINE = re.compile(
|
|
177
|
+
r"("
|
|
178
|
+
rf"\bauthorization\b{_SECRET_QUOTE}\s*:\s*{_SECRET_QUOTE}(?:(?:bearer|basic)\s+)?{_SECRET_VALUE}"
|
|
179
|
+
rf"|\b(?:set-cookie|cookie)\b{_SECRET_QUOTE}\s*:\s*{_SECRET_QUOTE}{_SECRET_VALUE}(?:\s*;\s*{_SECRET_VALUE})*"
|
|
180
|
+
rf"|{_SECRET_KEY_NAME}{_SECRET_QUOTE}\s*[:=]\s*{_SECRET_LITERAL_VALUE}"
|
|
181
|
+
r"|\bbearer\s+[A-Za-z0-9._\-]+"
|
|
182
|
+
r"|eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}(?:\.[A-Za-z0-9_-]{10,})?"
|
|
183
|
+
r"|https?://[^/\s]+:[^/\s]+@"
|
|
184
|
+
r"|\b(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})"
|
|
185
|
+
r")",
|
|
186
|
+
re.IGNORECASE,
|
|
187
|
+
)
|
|
188
|
+
_REDACTED_SECRET = "[redacted]"
|
|
189
|
+
_REDACTED_COPY_EXTENSIONS = {".html", ".json", ".log", ".md", ".out", ".txt", ".xml"}
|
|
190
|
+
_REDACT_STAGING_DIRNAME = ".pybuilder-integration-redact"
|
|
191
|
+
_REDACT_TEMP_FILES = set()
|
|
192
|
+
_SUBPROCESS_LOG_JOIN_SECONDS = 30
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _log_parallel_verify_environment_summary(logger: Logger, results):
|
|
196
|
+
summary = _build_parallel_verify_environment_summary(results)
|
|
197
|
+
print(summary)
|
|
198
|
+
github_summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
|
|
199
|
+
if not github_summary_path:
|
|
200
|
+
return
|
|
201
|
+
try:
|
|
202
|
+
with os.fdopen(
|
|
203
|
+
os.open(github_summary_path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600),
|
|
204
|
+
"a",
|
|
205
|
+
encoding="utf-8",
|
|
206
|
+
) as summary_file:
|
|
207
|
+
summary_file.write(summary)
|
|
208
|
+
if not summary.endswith("\n"):
|
|
209
|
+
summary_file.write("\n")
|
|
210
|
+
except OSError:
|
|
211
|
+
logger.warn("Could not append parallel verify_environment summary to GITHUB_STEP_SUMMARY")
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _build_parallel_verify_environment_summary(results):
|
|
215
|
+
lines = ["Parallel verify_environment summary"]
|
|
216
|
+
for result in results:
|
|
217
|
+
status = "PASSED" if result["exit_code"] == 0 else "FAILED"
|
|
218
|
+
lines.append(f" {result['name']}: {status} (exit_code={result['exit_code']})")
|
|
219
|
+
for result in results:
|
|
220
|
+
if result["exit_code"] == 0:
|
|
221
|
+
continue
|
|
222
|
+
lines.append(f" {result['name']} log: {result['log_file']}")
|
|
223
|
+
for hint in _extract_parallel_verify_failure_hints(result["log_file"]):
|
|
224
|
+
lines.append(f" {hint}")
|
|
225
|
+
return "\n".join(lines) + "\n"
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _extract_parallel_verify_failure_hints(log_file, max_hints=12):
|
|
229
|
+
if not log_file or not os.path.exists(log_file):
|
|
230
|
+
return ["child log was not found"]
|
|
231
|
+
try:
|
|
232
|
+
with open(log_file, encoding="utf-8", errors="replace") as log_handle:
|
|
233
|
+
raw = log_handle.read()
|
|
234
|
+
except OSError:
|
|
235
|
+
return ["child log could not be read"]
|
|
236
|
+
text = _ANSI_ESCAPE.sub("", raw)
|
|
237
|
+
lines = [line.rstrip() for line in text.splitlines()]
|
|
238
|
+
last_running = None
|
|
239
|
+
last_build_failed = None
|
|
240
|
+
failed_tests = []
|
|
241
|
+
session_summaries = []
|
|
242
|
+
extra_failures = []
|
|
243
|
+
for line in lines:
|
|
244
|
+
stripped = line.strip()
|
|
245
|
+
if not stripped:
|
|
246
|
+
continue
|
|
247
|
+
if _SUITE_RUNNING_LINE.match(stripped):
|
|
248
|
+
last_running = stripped
|
|
249
|
+
elif stripped.startswith("BUILD FAILED"):
|
|
250
|
+
last_build_failed = stripped
|
|
251
|
+
elif stripped.startswith("FAILED ") and "::" in stripped:
|
|
252
|
+
failed_tests.append(stripped)
|
|
253
|
+
elif stripped.startswith("===") and "failed" in stripped.lower():
|
|
254
|
+
session_summaries.append(stripped)
|
|
255
|
+
elif stripped.startswith("### FAIL") or "Failed to execute cypress" in stripped:
|
|
256
|
+
extra_failures.append(stripped)
|
|
257
|
+
hints = []
|
|
258
|
+
if last_running:
|
|
259
|
+
hints.append(last_running)
|
|
260
|
+
if last_build_failed:
|
|
261
|
+
hints.append(last_build_failed)
|
|
262
|
+
hints.extend(extra_failures[-3:])
|
|
263
|
+
hints.extend(failed_tests[-8:])
|
|
264
|
+
hints.extend(session_summaries[-3:])
|
|
265
|
+
if hints:
|
|
266
|
+
return _without_secret_hint_lines(hints)[:max_hints]
|
|
267
|
+
tail = [line.strip() for line in lines[-20:] if line.strip()]
|
|
268
|
+
if not tail:
|
|
269
|
+
return ["child log was empty"]
|
|
270
|
+
return _without_secret_hint_lines(tail)[-max_hints:]
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _start_subprocess_output_tee(process, log_handle, logger: Logger, pass_name):
|
|
274
|
+
stdout = getattr(process, "stdout", None)
|
|
275
|
+
if stdout is None or isinstance(stdout, (int, str)):
|
|
276
|
+
process._pybuilder_integration_log_thread = None
|
|
277
|
+
return
|
|
278
|
+
|
|
279
|
+
def tee():
|
|
280
|
+
prefix = f"[{pass_name}] "
|
|
281
|
+
try:
|
|
282
|
+
for line in stdout:
|
|
283
|
+
redacted = _redact_secret_log_line(_decode_subprocess_stdout_line(line))
|
|
284
|
+
try:
|
|
285
|
+
log_handle.write(redacted + "\n")
|
|
286
|
+
log_handle.flush()
|
|
287
|
+
except Exception:
|
|
288
|
+
pass
|
|
289
|
+
try:
|
|
290
|
+
logger.info(prefix + redacted)
|
|
291
|
+
except Exception:
|
|
292
|
+
pass
|
|
293
|
+
finally:
|
|
294
|
+
try:
|
|
295
|
+
stdout.close()
|
|
296
|
+
except Exception:
|
|
297
|
+
pass
|
|
298
|
+
|
|
299
|
+
thread = threading.Thread(target=tee, name=f"verify-environment-{pass_name}-log", daemon=True)
|
|
300
|
+
thread.start()
|
|
301
|
+
process._pybuilder_integration_log_thread = thread
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _close_subprocess_stdout(process):
|
|
305
|
+
stdout = getattr(process, "stdout", None)
|
|
306
|
+
if stdout is None or isinstance(stdout, (int, str)):
|
|
307
|
+
return
|
|
308
|
+
try:
|
|
309
|
+
stdout.close()
|
|
310
|
+
except Exception:
|
|
311
|
+
pass
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _close_subprocess_log(process):
|
|
315
|
+
thread = getattr(process, "_pybuilder_integration_log_thread", None)
|
|
316
|
+
if thread is not None:
|
|
317
|
+
thread.join(timeout=_SUBPROCESS_LOG_JOIN_SECONDS)
|
|
318
|
+
if thread.is_alive():
|
|
319
|
+
_close_subprocess_stdout(process)
|
|
320
|
+
thread.join(timeout=_SUBPROCESS_LOG_JOIN_SECONDS)
|
|
321
|
+
if thread.is_alive():
|
|
322
|
+
return
|
|
323
|
+
process._pybuilder_integration_log_thread = None
|
|
324
|
+
log_handle = getattr(process, "_pybuilder_integration_log_handle", None)
|
|
325
|
+
if log_handle and not getattr(log_handle, "closed", True):
|
|
326
|
+
log_handle.close()
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _decode_subprocess_stdout_line(line):
|
|
330
|
+
if isinstance(line, bytes):
|
|
331
|
+
line = line.decode("utf-8", "replace")
|
|
332
|
+
return line.rstrip("\n\r")
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _redact_secret_log_line(line):
|
|
336
|
+
return _SECRET_HINT_LINE.sub(_REDACTED_SECRET, line)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _redact_secret_log_text(text):
|
|
340
|
+
if not text:
|
|
341
|
+
return text
|
|
342
|
+
return "\n".join(_redact_secret_log_line(line) for line in text.splitlines()) + ("\n" if text.endswith("\n") else "")
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _without_secret_hint_lines(lines):
|
|
346
|
+
return [_redact_secret_log_line(line) for line in lines]
|
|
347
|
+
|
|
348
|
+
|
|
152
349
|
_CHILD_OUTPUT_SOURCES = (
|
|
153
350
|
("logs", "logs"),
|
|
154
351
|
("reports", "reports"),
|
|
155
352
|
("reports", "test-reports"),
|
|
156
353
|
)
|
|
157
354
|
_SUBPROCESS_WORKDIR_SKIP = {".pybuilder", "target", ".git", ".nudge-ci-venv"}
|
|
355
|
+
_CHILD_TARGET_LINK_SKIP = {"verify_environment", "logs", "reports", "test-reports"}
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _child_output_source_roots(stage_target):
|
|
359
|
+
return (
|
|
360
|
+
stage_target,
|
|
361
|
+
os.path.join(stage_target, "project", "target"),
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _first_child_cypress_log(stage_target):
|
|
366
|
+
for source_root in _child_output_source_roots(stage_target):
|
|
367
|
+
candidate = os.path.join(source_root, "logs", "integration", "cypress_run.log")
|
|
368
|
+
if os.path.isfile(candidate):
|
|
369
|
+
return candidate
|
|
370
|
+
return None
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _copy_maybe_redacted_file(src, dst, *, follow_symlinks=True):
|
|
374
|
+
if os.path.splitext(src)[1].lower() in _REDACTED_COPY_EXTENSIONS:
|
|
375
|
+
_copy_redacted_text_file(src, dst)
|
|
376
|
+
return dst
|
|
377
|
+
return shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _redact_text_file_inplace(path):
|
|
381
|
+
if os.path.islink(path) or not os.path.isfile(path):
|
|
382
|
+
return
|
|
383
|
+
try:
|
|
384
|
+
with open(path, encoding="utf-8", errors="replace") as source_file:
|
|
385
|
+
text = source_file.read()
|
|
386
|
+
except OSError:
|
|
387
|
+
return
|
|
388
|
+
redacted = _redact_secret_log_text(text)
|
|
389
|
+
if redacted == text:
|
|
390
|
+
return
|
|
391
|
+
fd, temporary_copy = tempfile.mkstemp(
|
|
392
|
+
prefix=f".{os.path.basename(path)}.",
|
|
393
|
+
suffix=".redact",
|
|
394
|
+
dir=tempfile.gettempdir(),
|
|
395
|
+
)
|
|
396
|
+
_register_redact_temp(temporary_copy)
|
|
397
|
+
try:
|
|
398
|
+
with os.fdopen(fd, "w", encoding="utf-8") as dest_file:
|
|
399
|
+
dest_file.write(redacted)
|
|
400
|
+
dest_file.flush()
|
|
401
|
+
os.fchmod(dest_file.fileno(), 0o600)
|
|
402
|
+
_replace_redacted_tempfile(temporary_copy, path)
|
|
403
|
+
except OSError:
|
|
404
|
+
_unlink_redact_temp(temporary_copy)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _register_redact_temp(path):
|
|
408
|
+
_REDACT_TEMP_FILES.add(path)
|
|
409
|
+
if not getattr(_register_redact_temp, "_atexit_registered", False):
|
|
410
|
+
atexit.register(_cleanup_redact_temps)
|
|
411
|
+
_register_redact_temp._atexit_registered = True
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _unlink_redact_temp(path):
|
|
415
|
+
try:
|
|
416
|
+
os.unlink(path)
|
|
417
|
+
except OSError:
|
|
418
|
+
pass
|
|
419
|
+
_REDACT_TEMP_FILES.discard(path)
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def _cleanup_redact_temps():
|
|
423
|
+
for path in list(_REDACT_TEMP_FILES):
|
|
424
|
+
_unlink_redact_temp(path)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _same_device_staging_base(path):
|
|
428
|
+
dest = os.path.abspath(path)
|
|
429
|
+
dest_dev = os.stat(dest).st_dev
|
|
430
|
+
current = dest if os.path.isdir(dest) else os.path.dirname(dest)
|
|
431
|
+
last_same = current
|
|
432
|
+
outside_target = None
|
|
433
|
+
while True:
|
|
434
|
+
try:
|
|
435
|
+
if os.stat(current).st_dev != dest_dev:
|
|
436
|
+
break
|
|
437
|
+
except OSError:
|
|
438
|
+
break
|
|
439
|
+
last_same = current
|
|
440
|
+
if "target" not in os.path.abspath(current).split(os.sep):
|
|
441
|
+
outside_target = current
|
|
442
|
+
break
|
|
443
|
+
parent = os.path.dirname(current)
|
|
444
|
+
if parent == current:
|
|
445
|
+
break
|
|
446
|
+
current = parent
|
|
447
|
+
return outside_target or last_same
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def _same_device_staging_dir(path):
|
|
451
|
+
staging = os.path.join(_same_device_staging_base(path), _REDACT_STAGING_DIRNAME)
|
|
452
|
+
os.makedirs(staging, mode=0o700, exist_ok=True)
|
|
453
|
+
try:
|
|
454
|
+
os.chmod(staging, 0o700)
|
|
455
|
+
except OSError:
|
|
456
|
+
pass
|
|
457
|
+
return staging
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def _remove_empty_redact_staging_dir(staging):
|
|
461
|
+
try:
|
|
462
|
+
os.rmdir(staging)
|
|
463
|
+
except OSError:
|
|
464
|
+
pass
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def _purge_redact_staging_dir(staging):
|
|
468
|
+
if os.path.basename(os.path.abspath(staging)) != _REDACT_STAGING_DIRNAME:
|
|
469
|
+
return
|
|
470
|
+
if os.path.islink(staging) or not os.path.isdir(staging):
|
|
471
|
+
return
|
|
472
|
+
for name in os.listdir(staging):
|
|
473
|
+
path = os.path.join(staging, name)
|
|
474
|
+
try:
|
|
475
|
+
if os.path.isdir(path) and not os.path.islink(path):
|
|
476
|
+
shutil.rmtree(path)
|
|
477
|
+
else:
|
|
478
|
+
os.unlink(path)
|
|
479
|
+
except OSError:
|
|
480
|
+
pass
|
|
481
|
+
_REDACT_TEMP_FILES.discard(path)
|
|
482
|
+
_remove_empty_redact_staging_dir(staging)
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def _redact_staging_candidates(path):
|
|
486
|
+
candidates = []
|
|
487
|
+
try:
|
|
488
|
+
candidates.append(os.path.join(_same_device_staging_base(path), _REDACT_STAGING_DIRNAME))
|
|
489
|
+
except OSError:
|
|
490
|
+
pass
|
|
491
|
+
abs_path = os.path.abspath(path)
|
|
492
|
+
if os.path.isdir(abs_path):
|
|
493
|
+
candidates.append(os.path.join(abs_path, _REDACT_STAGING_DIRNAME))
|
|
494
|
+
return candidates
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _purge_known_redact_staging_dirs(*paths):
|
|
498
|
+
seen = set()
|
|
499
|
+
for path in paths:
|
|
500
|
+
if not path or not os.path.exists(path):
|
|
501
|
+
continue
|
|
502
|
+
for staging in _redact_staging_candidates(path):
|
|
503
|
+
if staging in seen:
|
|
504
|
+
continue
|
|
505
|
+
seen.add(staging)
|
|
506
|
+
_purge_redact_staging_dir(staging)
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _replace_redacted_tempfile(temporary_copy, path):
|
|
510
|
+
try:
|
|
511
|
+
os.replace(temporary_copy, path)
|
|
512
|
+
_REDACT_TEMP_FILES.discard(temporary_copy)
|
|
513
|
+
return
|
|
514
|
+
except OSError as error:
|
|
515
|
+
if getattr(error, "errno", None) != errno.EXDEV:
|
|
516
|
+
raise
|
|
517
|
+
staging = _same_device_staging_dir(path)
|
|
518
|
+
fd, local_copy = tempfile.mkstemp(
|
|
519
|
+
prefix=f".{os.path.basename(path)}.",
|
|
520
|
+
suffix=".redact",
|
|
521
|
+
dir=staging,
|
|
522
|
+
)
|
|
523
|
+
_register_redact_temp(local_copy)
|
|
524
|
+
try:
|
|
525
|
+
with os.fdopen(fd, "wb") as destination, open(temporary_copy, "rb") as source:
|
|
526
|
+
shutil.copyfileobj(source, destination)
|
|
527
|
+
destination.flush()
|
|
528
|
+
os.fchmod(destination.fileno(), 0o600)
|
|
529
|
+
os.replace(local_copy, path)
|
|
530
|
+
_REDACT_TEMP_FILES.discard(local_copy)
|
|
531
|
+
except OSError:
|
|
532
|
+
_unlink_redact_temp(local_copy)
|
|
533
|
+
raise
|
|
534
|
+
finally:
|
|
535
|
+
_unlink_redact_temp(temporary_copy)
|
|
536
|
+
_remove_empty_redact_staging_dir(staging)
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def _redact_directory_text_files(root):
|
|
540
|
+
if not os.path.isdir(root):
|
|
541
|
+
return
|
|
542
|
+
_purge_known_redact_staging_dirs(root)
|
|
543
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
544
|
+
if _REDACT_STAGING_DIRNAME in dirnames:
|
|
545
|
+
dirnames.remove(_REDACT_STAGING_DIRNAME)
|
|
546
|
+
for name in filenames:
|
|
547
|
+
if os.path.splitext(name)[1].lower() not in _REDACTED_COPY_EXTENSIONS:
|
|
548
|
+
continue
|
|
549
|
+
_redact_text_file_inplace(os.path.join(dirpath, name))
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def _copy_redacted_text_file(src, dst):
|
|
553
|
+
parent = os.path.dirname(dst)
|
|
554
|
+
if parent:
|
|
555
|
+
os.makedirs(parent, exist_ok=True)
|
|
556
|
+
try:
|
|
557
|
+
with open(src, encoding="utf-8", errors="replace") as source_file:
|
|
558
|
+
text = _redact_secret_log_text(source_file.read())
|
|
559
|
+
except OSError:
|
|
560
|
+
shutil.copy2(src, dst)
|
|
561
|
+
return
|
|
562
|
+
with os.fdopen(os.open(dst, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "w", encoding="utf-8") as dest_file:
|
|
563
|
+
dest_file.write(text)
|
|
158
564
|
|
|
159
565
|
|
|
160
566
|
def _merge_parallel_verify_environment_outputs(project: Project, passes):
|
|
161
567
|
parent_target = project.expand_path("$dir_target")
|
|
568
|
+
_purge_known_redact_staging_dirs(parent_target, project.basedir)
|
|
162
569
|
parent_logs_directory = prepare_logs_directory(project)
|
|
163
570
|
parent_destinations = {
|
|
164
571
|
"logs": parent_logs_directory,
|
|
@@ -168,29 +575,36 @@ def _merge_parallel_verify_environment_outputs(project: Project, passes):
|
|
|
168
575
|
has_parent_cypress_log = False
|
|
169
576
|
for pass_config in passes:
|
|
170
577
|
stage_target = os.path.join(parent_target, "verify_environment", pass_config["name"])
|
|
171
|
-
for
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
578
|
+
for source_root in _child_output_source_roots(stage_target):
|
|
579
|
+
for destination_kind, source_dirname in _CHILD_OUTPUT_SOURCES:
|
|
580
|
+
source = os.path.join(source_root, source_dirname, "integration")
|
|
581
|
+
if not os.path.exists(source):
|
|
582
|
+
continue
|
|
583
|
+
_redact_directory_text_files(source)
|
|
584
|
+
destination = os.path.join(
|
|
585
|
+
parent_destinations[destination_kind],
|
|
586
|
+
"parallel",
|
|
587
|
+
pass_config["name"],
|
|
588
|
+
)
|
|
589
|
+
shutil.copytree(source, destination, dirs_exist_ok=True, copy_function=_copy_maybe_redacted_file)
|
|
590
|
+
cypress_log = _first_child_cypress_log(stage_target)
|
|
591
|
+
if cypress_log:
|
|
592
|
+
_copy_cypress_log(cypress_log, parent_cypress_log, has_parent_cypress_log)
|
|
593
|
+
has_parent_cypress_log = True
|
|
186
594
|
|
|
187
595
|
|
|
188
596
|
def _copy_cypress_log(source, destination, append):
|
|
597
|
+
try:
|
|
598
|
+
with open(source, encoding="utf-8", errors="replace") as source_file:
|
|
599
|
+
text = _redact_secret_log_text(source_file.read())
|
|
600
|
+
except OSError:
|
|
601
|
+
return
|
|
602
|
+
if text and not text.endswith("\n"):
|
|
603
|
+
text += "\n"
|
|
189
604
|
flags = os.O_WRONLY | os.O_CREAT | (os.O_APPEND if append else os.O_TRUNC)
|
|
190
|
-
mode = "
|
|
191
|
-
with open(
|
|
192
|
-
|
|
193
|
-
shutil.copyfileobj(source_file, destination_file)
|
|
605
|
+
mode = "a" if append else "w"
|
|
606
|
+
with os.fdopen(os.open(destination, flags, 0o600), mode, encoding="utf-8") as destination_file:
|
|
607
|
+
destination_file.write(text)
|
|
194
608
|
|
|
195
609
|
|
|
196
610
|
def _verify_environment_subprocess_stage_dir(project: Project, pass_config):
|
|
@@ -231,7 +645,7 @@ def _link_parent_target_tree(project: Project, workdir):
|
|
|
231
645
|
if not os.path.isdir(parent_target):
|
|
232
646
|
return
|
|
233
647
|
for name in os.listdir(parent_target):
|
|
234
|
-
if name
|
|
648
|
+
if name in _CHILD_TARGET_LINK_SKIP:
|
|
235
649
|
continue
|
|
236
650
|
os.symlink(os.path.join(parent_target, name), os.path.join(child_target, name))
|
|
237
651
|
|
|
@@ -317,21 +731,27 @@ _PYBUILDER_CLI_MAIN = "import sys; from pybuilder.cli import main; raise SystemE
|
|
|
317
731
|
def _isolated_plugin_python_command(plugin_python, args):
|
|
318
732
|
pyb_script = os.path.join(os.path.dirname(plugin_python), "pyb")
|
|
319
733
|
if os.path.isfile(pyb_script):
|
|
320
|
-
return [plugin_python, pyb_script] + args
|
|
321
|
-
return [plugin_python, "-c", _PYBUILDER_CLI_MAIN] + args
|
|
734
|
+
return [plugin_python, "-u", pyb_script] + args
|
|
735
|
+
return [plugin_python, "-u", "-c", _PYBUILDER_CLI_MAIN] + args
|
|
322
736
|
|
|
323
737
|
|
|
324
738
|
def _uses_isolated_plugin_python(command):
|
|
325
739
|
if len(command) < 2:
|
|
326
740
|
return False
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
741
|
+
args = command[1:]
|
|
742
|
+
if args and args[0] == "-u":
|
|
743
|
+
args = args[1:]
|
|
744
|
+
if not args:
|
|
745
|
+
return False
|
|
746
|
+
if args[0] == "-c":
|
|
747
|
+
return len(args) >= 2 and args[1] == _PYBUILDER_CLI_MAIN
|
|
748
|
+
return os.path.basename(args[0]) == "pyb"
|
|
330
749
|
|
|
331
750
|
|
|
332
751
|
def _verify_environment_subprocess_env(workdir, command, properties_file):
|
|
333
752
|
env = os.environ.copy()
|
|
334
753
|
env[SUBPROCESS_PROPERTIES_FILE_ENV] = properties_file
|
|
754
|
+
env["PYTHONUNBUFFERED"] = "1"
|
|
335
755
|
env.pop("GITHUB_STEP_SUMMARY", None)
|
|
336
756
|
env.pop("PYTHONHOME", None)
|
|
337
757
|
env.pop("VIRTUAL_ENV", None)
|
|
@@ -354,15 +774,16 @@ def _start_verify_environment_subprocess(project: Project, logger: Logger, pass_
|
|
|
354
774
|
)
|
|
355
775
|
try:
|
|
356
776
|
process = subprocess.Popen(command,
|
|
357
|
-
stdout=
|
|
777
|
+
stdout=subprocess.PIPE,
|
|
358
778
|
stderr=subprocess.STDOUT,
|
|
359
779
|
cwd=workdir,
|
|
360
780
|
env=env,
|
|
361
781
|
shell=False)
|
|
782
|
+
process._pybuilder_integration_log_handle = log_handle
|
|
783
|
+
_start_subprocess_output_tee(process, log_handle, logger, pass_config["name"])
|
|
362
784
|
except Exception:
|
|
363
785
|
log_handle.close()
|
|
364
786
|
raise
|
|
365
|
-
process._pybuilder_integration_log_handle = log_handle
|
|
366
787
|
return process, log_file
|
|
367
788
|
|
|
368
789
|
|
|
@@ -540,11 +961,23 @@ def _run_cypress_tests_in_directory(work_dir, logger, project, reactor: Reactor)
|
|
|
540
961
|
|
|
541
962
|
|
|
542
963
|
def _write_cypress_summary(project, logger):
|
|
543
|
-
"""Parse cypress_run.log and write a portable Markdown summary artifact.
|
|
964
|
+
"""Parse cypress_run.log and write a portable Markdown summary artifact.
|
|
965
|
+
|
|
966
|
+
No-op when Cypress did not run (missing or empty cypress_run.log), so
|
|
967
|
+
verify_environment jobs without Cypress do not emit a false warning.
|
|
968
|
+
Parallel child subprocesses also skip it; the parent writes one summary
|
|
969
|
+
after merging child Cypress logs.
|
|
970
|
+
"""
|
|
544
971
|
log_path = os.path.join(prepare_logs_directory(project), "cypress_run.log")
|
|
972
|
+
if os.environ.get(SUBPROCESS_PROPERTIES_FILE_ENV):
|
|
973
|
+
logger.debug("Skipping Cypress summary in verify_environment subprocess")
|
|
974
|
+
return None
|
|
975
|
+
if not os.path.isfile(log_path) or os.path.getsize(log_path) == 0:
|
|
976
|
+
logger.debug("Skipping Cypress summary: cypress_run.log not found")
|
|
977
|
+
return None
|
|
545
978
|
summary_path = os.path.join(prepare_reports_directory(project), "cypress-summary.md")
|
|
546
979
|
try:
|
|
547
|
-
output = _build_cypress_summary(log_path)
|
|
980
|
+
output = _redact_secret_log_text(_build_cypress_summary(log_path))
|
|
548
981
|
with os.fdopen(
|
|
549
982
|
os.open(summary_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600),
|
|
550
983
|
"w",
|
|
@@ -21,7 +21,7 @@ class install(_install):
|
|
|
21
21
|
if __name__ == '__main__':
|
|
22
22
|
setup(
|
|
23
23
|
name = 'pybuilder-integration',
|
|
24
|
-
version = '
|
|
24
|
+
version = '108',
|
|
25
25
|
description = 'A pybuilder plugin that runs integration tests (Tavern & Cypress) against a target.',
|
|
26
26
|
long_description = 'A pybuilder plugin that runs integration tests against a target. This is intended to be a broader scope than unit-tests encompassing dependant functionality.',
|
|
27
27
|
long_description_content_type = None,
|
|
File without changes
|
{pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration/artifact_manager.py
RENAMED
|
File without changes
|
|
File without changes
|
{pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration/directory_utility.py
RENAMED
|
File without changes
|
{pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration/exec_utility.py
RENAMED
|
File without changes
|
|
File without changes
|
{pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration/tool_utility.py
RENAMED
|
File without changes
|
{pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration.egg-info/SOURCES.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration.egg-info/requires.txt
RENAMED
|
File without changes
|
{pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration.egg-info/top_level.txt
RENAMED
|
File without changes
|
{pybuilder_integration-107 → pybuilder_integration-108}/pybuilder_integration.egg-info/zip-safe
RENAMED
|
File without changes
|
|
File without changes
|