docker-harbormaster 0.4.0__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {docker_harbormaster-0.4.0 → docker_harbormaster-0.4.1}/PKG-INFO +1 -1
- {docker_harbormaster-0.4.0 → docker_harbormaster-0.4.1}/docker_harbormaster/cli.py +43 -8
- {docker_harbormaster-0.4.0 → docker_harbormaster-0.4.1}/pyproject.toml +1 -1
- {docker_harbormaster-0.4.0 → docker_harbormaster-0.4.1}/LICENSE +0 -0
- {docker_harbormaster-0.4.0 → docker_harbormaster-0.4.1}/README.md +0 -0
- {docker_harbormaster-0.4.0 → docker_harbormaster-0.4.1}/docker_harbormaster/__init__.py +0 -0
- {docker_harbormaster-0.4.0 → docker_harbormaster-0.4.1}/docker_harbormaster/conftest.py +0 -0
- {docker_harbormaster-0.4.0 → docker_harbormaster-0.4.1}/docker_harbormaster/utils.py +0 -0
|
@@ -12,6 +12,7 @@ import subprocess
|
|
|
12
12
|
import sys
|
|
13
13
|
import tempfile
|
|
14
14
|
import time
|
|
15
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
15
16
|
from pathlib import Path
|
|
16
17
|
from time import strftime
|
|
17
18
|
from typing import Any
|
|
@@ -37,6 +38,10 @@ DEBUG: bool = False
|
|
|
37
38
|
|
|
38
39
|
MAX_GIT_NETWORK_ATTEMPTS = 3
|
|
39
40
|
RETRY_WAIT_SECONDS = 10
|
|
41
|
+
# All app repos are on GitHub, which throttles many parallel SSH connections
|
|
42
|
+
# coming from a single IP, so the git fetches run through a bounded pool rather
|
|
43
|
+
# than one worker per app.
|
|
44
|
+
MAX_PARALLEL_FETCHES = 10
|
|
40
45
|
|
|
41
46
|
# The filenames Compose v2 checks, in order, when discovering the Compose file
|
|
42
47
|
# of a project. Harbormaster uses the same order when an app does not set
|
|
@@ -410,8 +415,11 @@ def _run_command_full(
|
|
|
410
415
|
if environment:
|
|
411
416
|
env.update(environment)
|
|
412
417
|
|
|
413
|
-
|
|
414
|
-
|
|
418
|
+
# We pass the working directory to Popen instead of calling os.chdir(),
|
|
419
|
+
# because the process-wide cwd is shared across the threads that run the
|
|
420
|
+
# git fetches, and changing it there would race. The shell still starts
|
|
421
|
+
# with `cd <chdir>;` below, which the container symlink trick needs.
|
|
422
|
+
#
|
|
415
423
|
# We concatenate the command here instead of just passing it to Popen, because the
|
|
416
424
|
# Harbormaster container (the way to deploy HM) uses a symlink inside with the same
|
|
417
425
|
# name as the host directory (to make the paths inside the container match up with
|
|
@@ -437,6 +445,7 @@ def _run_command_full(
|
|
|
437
445
|
stderr=subprocess.STDOUT,
|
|
438
446
|
env=env,
|
|
439
447
|
shell=True,
|
|
448
|
+
cwd=chdir,
|
|
440
449
|
)
|
|
441
450
|
|
|
442
451
|
stdout_list: List[bytes] = []
|
|
@@ -453,7 +462,6 @@ def _run_command_full(
|
|
|
453
462
|
returncode = process.wait()
|
|
454
463
|
stdout = b"".join(stdout_list)
|
|
455
464
|
debug(f"Return code: {returncode}")
|
|
456
|
-
os.chdir(wd)
|
|
457
465
|
return (returncode, stdout)
|
|
458
466
|
|
|
459
467
|
|
|
@@ -1003,15 +1011,21 @@ class App:
|
|
|
1003
1011
|
for _ in range(MAX_GIT_NETWORK_ATTEMPTS):
|
|
1004
1012
|
try:
|
|
1005
1013
|
if self.is_repo():
|
|
1006
|
-
click.echo(
|
|
1014
|
+
click.echo(
|
|
1015
|
+
f"{self.id}: Pulling {self.url} to {self.paths.repo_dir}..."
|
|
1016
|
+
)
|
|
1007
1017
|
updated = self.pull()
|
|
1008
1018
|
else:
|
|
1009
|
-
click.echo(
|
|
1019
|
+
click.echo(
|
|
1020
|
+
f"{self.id}: Cloning {self.url} to {self.paths.repo_dir}..."
|
|
1021
|
+
)
|
|
1010
1022
|
updated = self.clone()
|
|
1011
1023
|
except Exception as e:
|
|
1012
1024
|
last_exception = e
|
|
1013
|
-
click.echo(
|
|
1014
|
-
|
|
1025
|
+
click.echo(
|
|
1026
|
+
f"{self.id}: Error with git clone/pull request: {last_exception}"
|
|
1027
|
+
)
|
|
1028
|
+
click.echo(f"{self.id}: Will retry after {RETRY_WAIT_SECONDS} seconds.")
|
|
1015
1029
|
time.sleep(RETRY_WAIT_SECONDS)
|
|
1016
1030
|
else:
|
|
1017
1031
|
# Rendering (and the Compose file resolution it triggers) is
|
|
@@ -1064,6 +1078,22 @@ def process_config(configuration: Configuration, force_restart: bool = False) ->
|
|
|
1064
1078
|
This is the main function that loads the configuration the file and starts/stops
|
|
1065
1079
|
apps as needed.
|
|
1066
1080
|
"""
|
|
1081
|
+
# The git clones/pulls are network-bound, so they all run first, in a
|
|
1082
|
+
# bounded pool. Only enabled apps are fetched, as before. The Docker steps
|
|
1083
|
+
# remain serial, in config order, because they are heavy and mostly local.
|
|
1084
|
+
fetch_results: Dict[App, Union[bool, Exception]] = {}
|
|
1085
|
+
enabled_apps = [app for app in configuration.apps if app.enabled]
|
|
1086
|
+
if enabled_apps:
|
|
1087
|
+
with ThreadPoolExecutor(max_workers=MAX_PARALLEL_FETCHES) as executor:
|
|
1088
|
+
futures = {app: executor.submit(app.clone_or_pull) for app in enabled_apps}
|
|
1089
|
+
for app, future in futures.items():
|
|
1090
|
+
try:
|
|
1091
|
+
fetch_results[app] = future.result()
|
|
1092
|
+
except Exception as e:
|
|
1093
|
+
# The failure is reported in the per-app loop below, so one
|
|
1094
|
+
# app's failure never stops the others from being fetched.
|
|
1095
|
+
fetch_results[app] = e
|
|
1096
|
+
|
|
1067
1097
|
successes = []
|
|
1068
1098
|
cache = {"version": 1}
|
|
1069
1099
|
for app in configuration.apps:
|
|
@@ -1071,7 +1101,12 @@ def process_config(configuration: Configuration, force_restart: bool = False) ->
|
|
|
1071
1101
|
click.echo(f"Updating {app.id} ({app.branch})...")
|
|
1072
1102
|
try:
|
|
1073
1103
|
if app.enabled:
|
|
1074
|
-
|
|
1104
|
+
fetch_result = fetch_results[app]
|
|
1105
|
+
if isinstance(fetch_result, Exception):
|
|
1106
|
+
# This app's fetch failed in phase 1; report it like any
|
|
1107
|
+
# other per-app error and keep going with the others.
|
|
1108
|
+
raise fetch_result
|
|
1109
|
+
updated_repo = fetch_result
|
|
1075
1110
|
if updated_repo:
|
|
1076
1111
|
click.echo(f"{app.id}: Repo was updated.")
|
|
1077
1112
|
else:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|