pybuilder-docker-too 47__tar.gz → 48__tar.gz

{pybuilder_docker_too-47 → pybuilder_docker_too-48}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pybuilder-docker-too
-Version: 47
+Version: 48
 Summary: A pybuilder plugin that stages a python package into a docker container and optionally publishes it to a registry.
 Home-page: https://github.com/rspitler/pybuilder-docker
 Author: 

{pybuilder_docker_too-47 → pybuilder_docker_too-48}/pybuilder_docker_too/__init__.py

@@ -41,10 +41,15 @@ RUN ${package_cmd}
 """)
 
 DEFAULT_DOCKER_COMPOSE_READY_TIMEOUT_SECONDS = 180
-DEFAULT_DOCKER_COMPOSE_READY_POLL_INTERVAL_SECONDS = 2
+DEFAULT_DOCKER_COMPOSE_READY_POLL_INTERVAL_SECONDS = 1
+DEFAULT_DOCKER_COMPOSE_DOWN_TIMEOUT_SECONDS = 1
 COMPOSE_DIAGNOSTIC_LOG_TAIL = 200
 COMPOSE_DIAGNOSTIC_MAX_BYTES = 1024 * 1024
 _DEFAULT_SERVICES = object()
+_PRESTARTED_COMPOSE_SERVICES_PROPERTY = (
+    "_pybuilder_docker_prestarted_compose_services")
+_INFERRED_PRETEST_SERVICES_PROPERTY = (
+    "_pybuilder_docker_inferred_pretest_services")
 
 
 @init
@@ -56,6 +61,12 @@ def init_plugin(project: Project, logger: Logger, reactor: Reactor):
     project.set_property_if_unset(
         'docker_compose_ready_poll_interval',
         DEFAULT_DOCKER_COMPOSE_READY_POLL_INTERVAL_SECONDS)
+    project.set_property_if_unset(
+        'docker_compose_down_timeout',
+        DEFAULT_DOCKER_COMPOSE_DOWN_TIMEOUT_SECONDS)
+    project.set_property_if_unset(
+        'docker_compose_prestart_during_package',
+        False)
 
 
 @task(description="Package artifact into a docker container with the tag "
@@ -109,6 +120,7 @@ def do_docker_package(project, logger, reactor):
     if ecr_login_before_build:
         registry = project.get_mandatory_property("docker_push_registry")
         _ecr_login(project, registry, logger, reactor)
+    _prestart_compose_services_during_package(project, logger)
     dist_dir = prepare_dist_directory(project)
     reactor.pybuilder_venv.verify_can_execute(["docker", "--version"], prerequisite="docker", caller="docker_package")
     # is true if user set verbose in build.py or from command line
@@ -661,6 +673,62 @@ def _run_compose_command(
     return output
 
 
+def _prestart_compose_services_during_package(project, logger):
+    if not project.get_property(
+            "docker_compose_prestart_during_package", False):
+        return []
+    if not project.get_property("run_docker_compose_on_verify", False):
+        return []
+    if not (
+            project.get_property("run_pretest_executable", False)
+            and project.get_property("pretest_executable", None)):
+        return []
+    if _normalize_required_services(
+            project.get_property("docker_compose_pretest_services", None)):
+        logger.debug(
+            "Skipping early Compose prestart because pretest services "
+            "are explicitly configured")
+        return []
+
+    compose_file = _get_docker_compose_file(project)
+    if not compose_file:
+        logger.debug(
+            "Skipping early Compose prestart because the Compose file "
+            "is not prepared yet")
+        return []
+
+    timeout_seconds = _to_number(
+        project.get_property(
+            "docker_compose_ready_timeout",
+            DEFAULT_DOCKER_COMPOSE_READY_TIMEOUT_SECONDS),
+        DEFAULT_DOCKER_COMPOSE_READY_TIMEOUT_SECONDS)
+    if not math.isfinite(timeout_seconds) or timeout_seconds < 0:
+        raise ValueError(
+            "docker_compose_ready_timeout must be finite and >= 0")
+    deadline = time.monotonic() + timeout_seconds
+    compose_env = _compose_environment(project)
+    inferred_services = _list_compose_healthy_dependencies(
+        compose_file, compose_env, deadline)
+    if not inferred_services:
+        return []
+
+    logger.info(
+        "Starting Compose infrastructure during image packaging ({})".format(
+            ", ".join(inferred_services)))
+    _run_compose_command(
+        compose_file,
+        ["up", "--detach", "--no-deps"] + inferred_services,
+        compose_env,
+        _remaining_timeout(deadline, time.monotonic))
+    project.set_property(
+        _PRESTARTED_COMPOSE_SERVICES_PROPERTY,
+        list(inferred_services))
+    project.set_property(
+        _INFERRED_PRETEST_SERVICES_PROPERTY,
+        list(inferred_services))
+    return inferred_services
+
+
 def _capture_compose_service_logs(
         compose_file,
         pending,
@@ -830,6 +898,8 @@ def _wait_for_compose_services(
     last_services = []
     last_inspect_error = None
     last_service_logs = ""
+    logs_captured = False
+    last_log_capture_pending = None
     while True:
         if compose_process is not None:
             exit_code = compose_process.poll()
@@ -860,24 +930,33 @@ def _wait_for_compose_services(
                 logger.info("Required compose services are ready")
                 return last_services
 
-        try:
-            captured_service_logs = logs_fn(
-                compose_file,
-                last_pending,
-                compose_env,
-                deadline,
-                time_fn)
-            if (
-                    captured_service_logs
-                    and not captured_service_logs.startswith(
-                        "<unable to collect service logs:")):
-                last_service_logs = captured_service_logs
-            elif not last_service_logs:
-                last_service_logs = captured_service_logs
-        except Exception as exc:
-            if not last_service_logs:
-                last_service_logs = (
-                    "<unable to collect service logs: {}>".format(exc))
+        remaining_before_logs = deadline - time_fn()
+        pending_key = tuple(sorted(last_pending))
+        should_capture_logs = (
+            not logs_captured
+            or pending_key != last_log_capture_pending
+            or 0 < remaining_before_logs <= poll_interval)
+        if should_capture_logs:
+            try:
+                captured_service_logs = logs_fn(
+                    compose_file,
+                    last_pending,
+                    compose_env,
+                    deadline,
+                    time_fn)
+                if (
+                        captured_service_logs
+                        and not captured_service_logs.startswith(
+                            "<unable to collect service logs:")):
+                    last_service_logs = captured_service_logs
+                elif not last_service_logs:
+                    last_service_logs = captured_service_logs
+            except Exception as exc:
+                if not last_service_logs:
+                    last_service_logs = (
+                        "<unable to collect service logs: {}>".format(exc))
+            logs_captured = True
+            last_log_capture_pending = pending_key
 
         remaining = deadline - time_fn()
         if remaining <= 0:
@@ -951,10 +1030,12 @@ def _run_docker_compose(project, logger, reactor: Reactor):
                 "up"]
 
         expected_services = None
+        application_services = None
         pretest_enabled = (
             project.get_property("run_pretest_executable", False)
             and project.get_property("pretest_executable", None))
         if pretest_enabled:
+            pretest_services_started_at = time.monotonic()
             timeout_seconds = _to_number(
                 project.get_property(
                     "docker_compose_ready_timeout",
@@ -966,10 +1047,15 @@ def _run_docker_compose(project, logger, reactor: Reactor):
             setup_deadline = time.monotonic() + timeout_seconds
             expected_services = _list_compose_services(
                 compose_file, compose_env, setup_deadline)
-            inferred_pretest_services = None
-            if not _normalize_required_services(
-                    project.get_property(
-                        "docker_compose_pretest_services", None)):
+            inferred_pretest_services = _normalize_required_services(
+                project.get_property(
+                    _INFERRED_PRETEST_SERVICES_PROPERTY, None))
+            configured_pretest_services = _normalize_required_services(
+                project.get_property(
+                    "docker_compose_pretest_services", None))
+            if (
+                    not configured_pretest_services
+                    and not inferred_pretest_services):
                 inferred_pretest_services = (
                     _list_compose_healthy_dependencies(
                         compose_file, compose_env, setup_deadline))
@@ -987,14 +1073,27 @@ def _run_docker_compose(project, logger, reactor: Reactor):
                 project,
                 created_services,
                 inferred_services=inferred_pretest_services)
-            start_output = _run_compose_command(
-                compose_file,
-                ["up", "--detach", "--no-deps"] + pretest_services,
-                compose_env,
-                _remaining_timeout(setup_deadline, time.monotonic))
-            if start_output:
-                fp.write(start_output)
-                fp.flush()
+            pretest_service_names = set(pretest_services)
+            application_services = [
+                service for service in expected_services
+                if service not in pretest_service_names]
+            prestarted_services = set(
+                _normalize_required_services(
+                    project.get_property(
+                        _PRESTARTED_COMPOSE_SERVICES_PROPERTY, None))
+                or [])
+            services_to_start = [
+                service for service in pretest_services
+                if service not in prestarted_services]
+            if services_to_start:
+                start_output = _run_compose_command(
+                    compose_file,
+                    ["up", "--detach", "--no-deps"] + services_to_start,
+                    compose_env,
+                    _remaining_timeout(setup_deadline, time.monotonic))
+                if start_output:
+                    fp.write(start_output)
+                    fp.flush()
             _wait_for_compose_services(
                 project,
                 logger,
@@ -1002,8 +1101,19 @@ def _run_docker_compose(project, logger, reactor: Reactor):
                 compose_env=compose_env,
                 required_services_override=pretest_services,
                 deadline_override=setup_deadline)
+            logger.info(
+                "Pretest compose services ready after {:.1f}s".format(
+                    time.monotonic() - pretest_services_started_at))
+            pretest_started_at = time.monotonic()
             _run_pretest_executable(project, logger, reactor)
-
+            logger.info(
+                "Pretest executable completed after {:.1f}s".format(
+                    time.monotonic() - pretest_started_at))
+
+        if application_services:
+            args += (
+                ["--no-deps", "--no-recreate"]
+                + application_services)
         logger.debug(f"Running docker compose with {args}")
         application_timeout = _to_number(
             project.get_property(
@@ -1014,6 +1124,7 @@ def _run_docker_compose(project, logger, reactor: Reactor):
             raise ValueError(
                 "docker_compose_ready_timeout must be finite and >= 0")
         application_deadline = time.monotonic() + application_timeout
+        application_started_at = time.monotonic()
         compose_process = subprocess.Popen(
             args, stderr=fp_err, stdout=fp, env=compose_env)
         running_services = _wait_for_compose_services(
@@ -1024,10 +1135,17 @@ def _run_docker_compose(project, logger, reactor: Reactor):
             compose_env=compose_env,
             expected_services_override=expected_services,
             deadline_override=application_deadline)
+        logger.info(
+            "Compose application services ready after {:.1f}s".format(
+                time.monotonic() - application_started_at))
+        target_started_at = time.monotonic()
         _wait_for_target_url(
             project,
             logger,
             compose_services=running_services)
+        logger.info(
+            "Integration target ready after {:.1f}s".format(
+                time.monotonic() - target_started_at))
         if not pretest_enabled:
             _run_pretest_executable(project, logger, reactor)
 
@@ -1063,12 +1181,22 @@ def _docker_compose_down(project, logger, reactor: Reactor):
     compose_file = _get_docker_compose_file(project)
     if compose_file:
         logger.info(f"Docker Compose Down: {compose_file}")
+        timeout_seconds = _to_number(
+            project.get_property(
+                "docker_compose_down_timeout",
+                DEFAULT_DOCKER_COMPOSE_DOWN_TIMEOUT_SECONDS),
+            DEFAULT_DOCKER_COMPOSE_DOWN_TIMEOUT_SECONDS)
+        if not math.isfinite(timeout_seconds) or timeout_seconds < 0:
+            raise ValueError(
+                "docker_compose_down_timeout must be finite and >= 0")
         # clean up our test run
         exec_command("docker", [
             "compose",
             "-f",
             compose_file,
-            "down"
+            "down",
+            "--timeout",
+            str(int(math.ceil(timeout_seconds)))
         ], output_file_name="docker_compose_run", project=project, logger=logger,
             reactor=reactor, env=_compose_environment(project))
 

{pybuilder_docker_too-47 → pybuilder_docker_too-48}/pybuilder_docker_too.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pybuilder-docker-too
-Version: 47
+Version: 48
 Summary: A pybuilder plugin that stages a python package into a docker container and optionally publishes it to a registry.
 Home-page: https://github.com/rspitler/pybuilder-docker
 Author: 

{pybuilder_docker_too-47 → pybuilder_docker_too-48}/setup.py

@@ -21,7 +21,7 @@ class install(_install):
 if __name__ == '__main__':
     setup(
         name = 'pybuilder-docker-too',
-        version = '47',
+        version = '48',
         description = 'A pybuilder plugin that stages a python package into a docker container and optionally publishes it to a registry.',
         long_description = 'A pybuilder plugin that stages a python package into a docker container and optionally publishes it to a registry.',
         long_description_content_type = None,