docker-harbormaster 0.4.0__tar.gz → 0.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: docker-harbormaster
3
- Version: 0.4.0
3
+ Version: 0.4.1
4
4
  Summary: A supervisor for docker-compose apps.
5
5
  License: AGPL-3.0-or-later
6
6
  License-File: LICENSE
@@ -12,6 +12,7 @@ import subprocess
12
12
  import sys
13
13
  import tempfile
14
14
  import time
15
+ from concurrent.futures import ThreadPoolExecutor
15
16
  from pathlib import Path
16
17
  from time import strftime
17
18
  from typing import Any
@@ -37,6 +38,10 @@ DEBUG: bool = False
37
38
 
38
39
  MAX_GIT_NETWORK_ATTEMPTS = 3
39
40
  RETRY_WAIT_SECONDS = 10
41
+ # All app repos are on GitHub, which throttles many parallel SSH connections
42
+ # coming from a single IP, so the git fetches run through a bounded pool rather
43
+ # than one worker per app.
44
+ MAX_PARALLEL_FETCHES = 10
40
45
 
41
46
  # The filenames Compose v2 checks, in order, when discovering the Compose file
42
47
  # of a project. Harbormaster uses the same order when an app does not set
@@ -410,8 +415,11 @@ def _run_command_full(
410
415
  if environment:
411
416
  env.update(environment)
412
417
 
413
- wd = os.getcwd()
414
- os.chdir(chdir)
418
+ # We pass the working directory to Popen instead of calling os.chdir(),
419
+ # because the process-wide cwd is shared across the threads that run the
420
+ # git fetches, and changing it there would race. The shell still starts
421
+ # with `cd <chdir>;` below, which the container symlink trick needs.
422
+ #
415
423
  # We concatenate the command here instead of just passing it to Popen, because the
416
424
  # Harbormaster container (the way to deploy HM) uses a symlink inside with the same
417
425
  # name as the host directory (to make the paths inside the container match up with
@@ -437,6 +445,7 @@ def _run_command_full(
437
445
  stderr=subprocess.STDOUT,
438
446
  env=env,
439
447
  shell=True,
448
+ cwd=chdir,
440
449
  )
441
450
 
442
451
  stdout_list: List[bytes] = []
@@ -453,7 +462,6 @@ def _run_command_full(
453
462
  returncode = process.wait()
454
463
  stdout = b"".join(stdout_list)
455
464
  debug(f"Return code: {returncode}")
456
- os.chdir(wd)
457
465
  return (returncode, stdout)
458
466
 
459
467
 
@@ -1003,15 +1011,21 @@ class App:
1003
1011
  for _ in range(MAX_GIT_NETWORK_ATTEMPTS):
1004
1012
  try:
1005
1013
  if self.is_repo():
1006
- click.echo(f"Pulling {self.url} to {self.paths.repo_dir}...")
1014
+ click.echo(
1015
+ f"{self.id}: Pulling {self.url} to {self.paths.repo_dir}..."
1016
+ )
1007
1017
  updated = self.pull()
1008
1018
  else:
1009
- click.echo(f"Cloning {self.url} to {self.paths.repo_dir}...")
1019
+ click.echo(
1020
+ f"{self.id}: Cloning {self.url} to {self.paths.repo_dir}..."
1021
+ )
1010
1022
  updated = self.clone()
1011
1023
  except Exception as e:
1012
1024
  last_exception = e
1013
- click.echo(f"Error with git clone/pull request: {last_exception}")
1014
- click.echo(f"Will retry after {RETRY_WAIT_SECONDS} seconds.")
1025
+ click.echo(
1026
+ f"{self.id}: Error with git clone/pull request: {last_exception}"
1027
+ )
1028
+ click.echo(f"{self.id}: Will retry after {RETRY_WAIT_SECONDS} seconds.")
1015
1029
  time.sleep(RETRY_WAIT_SECONDS)
1016
1030
  else:
1017
1031
  # Rendering (and the Compose file resolution it triggers) is
@@ -1064,6 +1078,22 @@ def process_config(configuration: Configuration, force_restart: bool = False) ->
1064
1078
  This is the main function that loads the configuration the file and starts/stops
1065
1079
  apps as needed.
1066
1080
  """
1081
+ # The git clones/pulls are network-bound, so they all run first, in a
1082
+ # bounded pool. Only enabled apps are fetched, as before. The Docker steps
1083
+ # remain serial, in config order, because they are heavy and mostly local.
1084
+ fetch_results: Dict[App, Union[bool, Exception]] = {}
1085
+ enabled_apps = [app for app in configuration.apps if app.enabled]
1086
+ if enabled_apps:
1087
+ with ThreadPoolExecutor(max_workers=MAX_PARALLEL_FETCHES) as executor:
1088
+ futures = {app: executor.submit(app.clone_or_pull) for app in enabled_apps}
1089
+ for app, future in futures.items():
1090
+ try:
1091
+ fetch_results[app] = future.result()
1092
+ except Exception as e:
1093
+ # The failure is reported in the per-app loop below, so one
1094
+ # app's failure never stops the others from being fetched.
1095
+ fetch_results[app] = e
1096
+
1067
1097
  successes = []
1068
1098
  cache = {"version": 1}
1069
1099
  for app in configuration.apps:
@@ -1071,7 +1101,12 @@ def process_config(configuration: Configuration, force_restart: bool = False) ->
1071
1101
  click.echo(f"Updating {app.id} ({app.branch})...")
1072
1102
  try:
1073
1103
  if app.enabled:
1074
- updated_repo = app.clone_or_pull()
1104
+ fetch_result = fetch_results[app]
1105
+ if isinstance(fetch_result, Exception):
1106
+ # This app's fetch failed in phase 1; report it like any
1107
+ # other per-app error and keep going with the others.
1108
+ raise fetch_result
1109
+ updated_repo = fetch_result
1075
1110
  if updated_repo:
1076
1111
  click.echo(f"{app.id}: Repo was updated.")
1077
1112
  else:
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "docker-harbormaster"
3
- version = "0.4.0"
3
+ version = "0.4.1"
4
4
  description = "A supervisor for docker-compose apps."
5
5
  authors = ["Stavros Korokithakis <hi@stavros.io>"]
6
6
  repository = "https://github.com/skorokithakis/harbormaster"