devs-webhook 0.1.2__tar.gz → 0.1.4__tar.gz

{devs_webhook-0.1.2 → devs_webhook-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devs-webhook
-Version: 0.1.2
+Version: 0.1.4
 Summary: GitHub webhook handler for automated devcontainer operations with Claude Code
 Author: Dan Lester
 License-Expression: MIT

{devs_webhook-0.1.2 → devs_webhook-0.1.4}/devs_webhook/config.py

@@ -89,6 +89,16 @@ class WebhookConfig(BaseSettings, BaseConfig):
     log_level: str = Field(default="INFO", description="Logging level")
     log_format: str = Field(default="json", description="Logging format (json|console)")
 
+    # Container output logging (CloudWatch-friendly)
+    container_logs_dir: Path = Field(
+        default_factory=lambda: Path("/var/log/devs-webhook/containers"),
+        description="Directory for container output logs (CloudWatch agent compatible)"
+    )
+    container_logs_enabled: bool = Field(
+        default=False,
+        description="Enable writing container output to log files"
+    )
+
     # Task source configuration
     task_source: str = Field(
         default="webhook",
@@ -166,6 +176,9 @@ class WebhookConfig(BaseSettings, BaseConfig):
         self.repo_cache_dir.mkdir(parents=True, exist_ok=True)
         # Claude config directory for container mounts
         self.claude_config_dir.mkdir(parents=True, exist_ok=True)
+        # Container logs directory (if enabled)
+        if self.container_logs_enabled:
+            self.container_logs_dir.mkdir(parents=True, exist_ok=True)
     
     def validate_required_settings(self) -> None:
         """Validate that required settings are present."""

{devs_webhook-0.1.2 → devs_webhook-0.1.4}/devs_webhook/core/claude_dispatcher.py

@@ -1,5 +1,6 @@
 """Claude Code CLI integration for executing tasks in containers."""
 
+import uuid
 from typing import Optional
 import structlog
 from pathlib import Path
@@ -10,6 +11,7 @@ from devs_common.core.workspace import WorkspaceManager
 from ..github.models import WebhookEvent, IssueEvent, PullRequestEvent, CommentEvent
 from devs_common.devs_config import DevsOptions
 from .base_dispatcher import BaseDispatcher, TaskResult
+from ..utils.container_logs import create_container_log_writer
 
 logger = structlog.get_logger()
 
@@ -29,36 +31,44 @@ class ClaudeDispatcher(BaseDispatcher):
         repo_path: Path,
         event: WebhookEvent,
         devs_options: Optional[DevsOptions] = None,
-        task_description: Optional[str] = None
+        task_description: Optional[str] = None,
+        task_id: Optional[str] = None
     ) -> TaskResult:
         """Execute a task using Claude Code CLI in a container.
-        
+
         Args:
             dev_name: Name of dev container (e.g., eamonn)
             repo_path: Path to repository on host (already calculated by container_pool)
             task_description: Task description for Claude
             event: Original webhook event
             devs_options: Options from DEVS.yml file
-            
+            task_id: Optional task identifier for logging
+
         Returns:
             Task execution result
         """
+        # Generate task_id if not provided
+        if not task_id:
+            task_id = str(uuid.uuid4())[:8]
+
         try:
             logger.info("Starting Claude Code CLI task",
                        container=dev_name,
                        repo=event.repository.full_name,
-                       repo_path=str(repo_path))
-            
+                       repo_path=str(repo_path),
+                       task_id=task_id)
+
             # Execute Claude directly - prompt building, workspace setup, container startup, Claude execution
             # Use task_description if provided, otherwise extract from event
             task_desc = task_description or "Process webhook event"
-            
+
             success, output, error = self._execute_claude_sync(
                 repo_path,
                 dev_name,
                 task_desc,
                 event,
-                devs_options
+                devs_options,
+                task_id=task_id
             )
             
             # Build result - ensure we have meaningful error messages
@@ -107,26 +117,37 @@ class ClaudeDispatcher(BaseDispatcher):
         dev_name: str,
         task_description: str,
         event: WebhookEvent,
-        devs_options: Optional[DevsOptions] = None
+        devs_options: Optional[DevsOptions] = None,
+        task_id: Optional[str] = None
     ) -> tuple[bool, str, str]:
         """Execute complete Claude workflow synchronously.
-        
+
         This mirrors the CLI approach exactly:
         1. Create project, workspace manager, and container manager
         2. Create/reset workspace (force=True for webhook)
         3. Build prompt
         4. Execute Claude (which handles container startup)
-        
+
         Args:
             repo_path: Path to repository
             dev_name: Development environment name
             task_description: Task description for Claude
             event: Webhook event
             devs_options: Options from DEVS.yml
-            
+            task_id: Optional task identifier for logging
+
         Returns:
             Tuple of (success, stdout, stderr)
         """
+        # Create container log writer if enabled
+        container_log = create_container_log_writer(
+            config=self.config,
+            container_name=dev_name,
+            task_id=task_id or str(uuid.uuid4())[:8],
+            repo_name=event.repository.full_name,
+            task_type="claude"
+        )
+
         try:
             # 1. Create project, workspace manager, and container manager like CLI
             project = Project(repo_path)
@@ -240,11 +261,15 @@ Always remember to PUSH your work to origin!
             # 4. Execute Claude (like CLI pattern) with environment variables from DEVS.yml
             logger.info("Executing Claude via ContainerManager (like CLI)",
                        container=dev_name)
-            
+
             extra_env = None
             if devs_options:
                 extra_env = devs_options.get_env_vars(dev_name)
-            
+
+            # Start container logging if enabled
+            if container_log:
+                container_log.start(prompt=prompt, workspace_dir=str(workspace_dir))
+
             success, stdout, stderr = container_manager.exec_claude(
                 dev_name=dev_name,
                 workspace_dir=workspace_dir,
@@ -253,7 +278,15 @@ Always remember to PUSH your work to origin!
                 stream=False,  # Don't stream in webhook mode
                 extra_env=extra_env
             )
-            
+
+            # Write container output to log file if enabled
+            if container_log:
+                container_log.write_output(stdout, stderr)
+                container_log.end(
+                    success=success,
+                    error=stderr if not success else None
+                )
+
             # Log the actual output for debugging
             if not success:
                 logger.error("Claude execution failed",
@@ -261,20 +294,25 @@ Always remember to PUSH your work to origin!
                            stdout=stdout[:1000] if stdout else "",
                            stderr=stderr[:1000] if stderr else "",
                            success=success)
-            
+
             # If failed and no stderr, check stdout for error messages
             # (Claude sometimes outputs errors to stdout)
             if not success and not stderr:
                 stderr = stdout
-            
+
             return success, stdout, stderr
-            
+
         except Exception as e:
             error_msg = f"Claude execution failed: {str(e)}"
             logger.error("Claude execution error",
                         container=dev_name,
                         error=error_msg,
                         exc_info=True)
+
+            # Log the error to container log if enabled
+            if container_log:
+                container_log.end(success=False, error=error_msg)
+
             return False, "", error_msg
     
     

{devs_webhook-0.1.2 → devs_webhook-0.1.4}/devs_webhook/core/container_pool.py

@@ -929,6 +929,55 @@ Please check the webhook handler logs for more details, or try mentioning me aga
                 "cached_repo_configs": list(self.repo_configs.keys()),
             }
 
+    def get_total_queued_tasks(self) -> int:
+        """Get the total number of tasks queued across all containers.
+
+        Returns:
+            Total number of tasks waiting in all queues
+        """
+        return sum(queue.qsize() for queue in self.container_queues.values())
+
+    async def wait_for_all_tasks_complete(self, timeout: Optional[float] = None) -> bool:
+        """Wait for all queued tasks to be processed.
+
+        This waits for all container queues to be fully drained, meaning
+        all tasks have been picked up by workers AND task_done() has been
+        called for each (i.e., processing is complete, not just started).
+
+        Args:
+            timeout: Optional timeout in seconds. If None, waits indefinitely.
+
+        Returns:
+            True if all tasks completed, False if timeout occurred.
+        """
+        logger.info("Waiting for all container queues to drain",
+                   queues={name: q.qsize() for name, q in self.container_queues.items()})
+
+        async def wait_all_queues():
+            # Wait for each queue to be fully processed
+            # asyncio.Queue.join() waits until all items have had task_done() called
+            wait_tasks = [
+                queue.join()
+                for queue in self.container_queues.values()
+            ]
+            await asyncio.gather(*wait_tasks)
+
+        try:
+            if timeout is not None:
+                await asyncio.wait_for(wait_all_queues(), timeout=timeout)
+            else:
+                await wait_all_queues()
+
+            logger.info("All container queues drained successfully")
+            return True
+
+        except asyncio.TimeoutError:
+            remaining = {name: q.qsize() for name, q in self.container_queues.items()}
+            logger.warning("Timeout waiting for queues to drain",
+                          remaining_tasks=remaining,
+                          timeout_seconds=timeout)
+            return False
+
     async def _idle_cleanup_worker(self) -> None:
         """Periodically clean up idle containers."""
         while True:

{devs_webhook-0.1.2 → devs_webhook-0.1.4}/devs_webhook/core/test_dispatcher.py

@@ -1,5 +1,6 @@
 """Test runner dispatcher for executing CI tests in containers."""
 
+import uuid
 from typing import Optional
 import structlog
 from pathlib import Path
@@ -10,6 +11,7 @@ from devs_common.core.workspace import WorkspaceManager
 from ..github.models import WebhookEvent, PushEvent, PullRequestEvent
 from devs_common.devs_config import DevsOptions
 from .base_dispatcher import BaseDispatcher, TaskResult
+from ..utils.container_logs import create_container_log_writer
 
 logger = structlog.get_logger()
 
@@ -29,27 +31,34 @@ class TestDispatcher(BaseDispatcher):
         repo_path: Path,
         event: WebhookEvent,
         devs_options: Optional[DevsOptions] = None,
-        task_description: Optional[str] = None
+        task_description: Optional[str] = None,
+        task_id: Optional[str] = None
     ) -> TaskResult:
         """Execute tests using container and report results via GitHub Checks API.
-        
+
         Args:
             dev_name: Name of dev container (e.g., eamonn)
             repo_path: Path to repository on host (already calculated by container_pool)
             event: Original webhook event
             devs_options: Options from DEVS.yml file
             task_description: Task description (ignored by test dispatcher)
-            
+            task_id: Optional task identifier for logging
+
         Returns:
             Test execution result
         """
+        # Generate task_id if not provided
+        if not task_id:
+            task_id = str(uuid.uuid4())[:8]
+
         check_run_id = None
-        
+
         try:
             logger.info("Starting test execution",
                        container=dev_name,
                        repo=event.repository.full_name,
-                       repo_path=str(repo_path))
+                       repo_path=str(repo_path),
+                       task_id=task_id)
             
             # Determine the commit SHA to test
             commit_sha = self._get_commit_sha(event)
@@ -106,7 +115,8 @@ class TestDispatcher(BaseDispatcher):
                 repo_path,
                 dev_name,
                 event,
-                devs_options
+                devs_options,
+                task_id=task_id
             )
             
             # Build result
@@ -186,19 +196,30 @@ class TestDispatcher(BaseDispatcher):
         repo_path: Path,
         dev_name: str,
         event: WebhookEvent,
-        devs_options: Optional[DevsOptions] = None
+        devs_options: Optional[DevsOptions] = None,
+        task_id: Optional[str] = None
     ) -> tuple[bool, str, str, int]:
         """Execute tests synchronously in container.
-        
+
         Args:
             repo_path: Path to repository
             dev_name: Development environment name
             event: Webhook event
             devs_options: Options from DEVS.yml
-            
+            task_id: Optional task identifier for logging
+
         Returns:
             Tuple of (success, stdout, stderr, exit_code)
         """
+        # Create container log writer if enabled
+        container_log = create_container_log_writer(
+            config=self.config,
+            container_name=dev_name,
+            task_id=task_id or str(uuid.uuid4())[:8],
+            repo_name=event.repository.full_name,
+            task_type="tests"
+        )
+
         try:
             # 1. Create project, workspace manager, and container manager
             project = Project(repo_path)
@@ -256,11 +277,15 @@ class TestDispatcher(BaseDispatcher):
             test_command = "./runtests.sh"  # Default
             if devs_options and devs_options.ci_test_command:
                 test_command = devs_options.ci_test_command
-            
+
             logger.info("Executing test command",
                        container=dev_name,
                        test_command=test_command)
-            
+
+            # Start container logging if enabled
+            if container_log:
+                container_log.start(test_command=test_command, workspace_dir=str(workspace_dir))
+
             # 6. Execute tests
             success, stdout, stderr, exit_code = self._exec_command_in_container(
                 project=project,
@@ -269,22 +294,36 @@ class TestDispatcher(BaseDispatcher):
                 command=test_command,
                 debug=self.config.dev_mode
             )
-            
+
+            # Write container output to log file if enabled
+            if container_log:
+                container_log.write_output(stdout, stderr)
+                container_log.end(
+                    success=success,
+                    exit_code=exit_code,
+                    error=stderr if not success else None
+                )
+
             logger.info("Test command completed",
                        container=dev_name,
                        success=success,
                        exit_code=exit_code,
                        output_length=len(stdout) if stdout else 0,
                        error_length=len(stderr) if stderr else 0)
-            
+
             return success, stdout, stderr, exit_code
-            
+
         except Exception as e:
             error_msg = f"Test execution failed: {str(e)}"
             logger.error("Test execution error",
                         container=dev_name,
                         error=error_msg,
                         exc_info=True)
+
+            # Log the error to container log if enabled
+            if container_log:
+                container_log.end(success=False, exit_code=1, error=error_msg)
+
             return False, "", error_msg, 1
     
     def _get_commit_sha(self, event: WebhookEvent) -> Optional[str]:

{devs_webhook-0.1.2 → devs_webhook-0.1.4}/devs_webhook/main_cli.py

@@ -30,7 +30,11 @@ cli.add_command(worker)
 @click.option('--dev', is_flag=True, help='Development mode (auto-loads .env, enables reload, console logs)')
 @click.option('--source', type=click.Choice(['webhook', 'sqs'], case_sensitive=False), help='Task source override')
 @click.option('--burst', is_flag=True, help='Burst mode: process all available SQS messages then exit (SQS mode only)')
-def serve(host: str, port: int, reload: bool, env_file: Path, dev: bool, source: str, burst: bool):
+@click.option('--no-wait', is_flag=True, help='In burst mode, exit immediately after draining SQS queue without waiting for tasks to complete')
+@click.option('--timeout', type=int, default=None, help='Timeout in seconds for waiting on task completion in burst mode (default: wait indefinitely)')
+@click.option('--container-logs', is_flag=True, help='Enable container output logging to files (CloudWatch compatible)')
+@click.option('--container-logs-dir', type=click.Path(path_type=Path), default=None, help='Directory for container log files (default: /var/log/devs-webhook/containers)')
+def serve(host: str, port: int, reload: bool, env_file: Path, dev: bool, source: str, burst: bool, no_wait: bool, timeout: int, container_logs: bool, container_logs_dir: Path):
     """Start the webhook handler server.
 
     The server can run in two modes:
@@ -40,14 +44,21 @@ def serve(host: str, port: int, reload: bool, env_file: Path, dev: bool, source:
     SQS mode supports --burst flag to process all available messages then exit:
     - Exit code 0: Processed one or more messages successfully
     - Exit code 42: Queue was empty (no messages to process)
+    - Exit code 43: Timeout waiting for tasks to complete
     - Other codes: Error occurred
 
+    By default, burst mode waits for all container tasks (Docker jobs) to complete
+    before exiting. Use --no-wait to exit immediately after draining the SQS queue,
+    or --timeout to set a maximum wait time.
+
     Examples:
         devs-webhook serve --dev                    # Development mode with .env loading
         devs-webhook serve --env-file /path/.env    # Load specific .env file
         devs-webhook serve --host 127.0.0.1        # Override host from config
         devs-webhook serve --source sqs            # Use SQS polling mode
-        devs-webhook serve --source sqs --burst    # Process all SQS messages then exit
+        devs-webhook serve --source sqs --burst    # Process all SQS messages, wait for completion
+        devs-webhook serve --source sqs --burst --no-wait  # Drain SQS and exit immediately
+        devs-webhook serve --source sqs --burst --timeout 3600  # Wait up to 1 hour for tasks
     """
     # Handle development mode
     if dev:
@@ -87,6 +98,12 @@ def serve(host: str, port: int, reload: bool, env_file: Path, dev: bool, source:
     if source:
         os.environ["TASK_SOURCE"] = source
 
+    # Configure container logs if specified via CLI
+    if container_logs:
+        os.environ["CONTAINER_LOGS_ENABLED"] = "true"
+    if container_logs_dir:
+        os.environ["CONTAINER_LOGS_DIR"] = str(container_logs_dir)
+
     # Now setup logging after environment is configured
     setup_logging()
 
@@ -97,6 +114,8 @@ def serve(host: str, port: int, reload: bool, env_file: Path, dev: bool, source:
     click.echo(f"Task source: {config.task_source}")
     click.echo(f"Watching for @{config.github_mentioned_user} mentions")
     click.echo(f"Container pool: {', '.join(config.get_container_pool_list())}")
+    if config.container_logs_enabled:
+        click.echo(f"Container logs: {config.container_logs_dir}")
 
     # Validate burst mode is only used with SQS
     if burst and config.task_source != "sqs":
@@ -128,13 +147,24 @@ def serve(host: str, port: int, reload: bool, env_file: Path, dev: bool, source:
             click.echo(f"DLQ configured: {config.aws_sqs_dlq_url}")
         if burst:
             click.echo("Burst mode: will process all messages then exit")
+            if no_wait:
+                click.echo("  --no-wait: will NOT wait for container tasks to complete")
+            else:
+                if timeout:
+                    click.echo(f"  Will wait up to {timeout}s for container tasks to complete")
+                else:
+                    click.echo("  Will wait for all container tasks to complete before exit")
 
         # Import and run SQS source
         import asyncio
         from .sources.sqs_source import SQSTaskSource
 
         async def run_sqs():
-            sqs_source = SQSTaskSource(burst_mode=burst)
+            sqs_source = SQSTaskSource(
+                burst_mode=burst,
+                wait_for_tasks=not no_wait,
+                task_timeout=float(timeout) if timeout else None,
+            )
             try:
                 return await sqs_source.start()
             except KeyboardInterrupt:
@@ -149,6 +179,22 @@ def serve(host: str, port: int, reload: bool, env_file: Path, dev: bool, source:
                 if result.messages_processed == 0:
                     click.echo("Queue was empty, no messages processed")
                     exit(42)
+                elif no_wait:
+                    # Not waiting for tasks - just report messages processed
+                    click.echo(f"Burst complete: queued {result.messages_processed} message(s)")
+                    click.echo("  (not waiting for container tasks to complete)")
+                    exit(0)
+                elif result.tasks_completed == result.messages_processed:
+                    # All tasks completed successfully
+                    click.echo(f"Burst complete: processed {result.messages_processed} message(s), "
+                              f"all {result.tasks_completed} task(s) completed")
+                    exit(0)
+                elif result.tasks_completed < result.messages_processed:
+                    # Timeout - some tasks didn't complete
+                    remaining = result.messages_processed - result.tasks_completed
+                    click.echo(f"Burst timeout: processed {result.messages_processed} message(s), "
+                              f"but {remaining} task(s) still running")
+                    exit(43)
                 else:
                     click.echo(f"Burst complete: processed {result.messages_processed} message(s)")
                     exit(0)

{devs_webhook-0.1.2 → devs_webhook-0.1.4}/devs_webhook/sources/sqs_source.py

@@ -23,6 +23,7 @@ logger = structlog.get_logger()
 class BurstResult:
     """Result of a burst mode SQS run."""
     messages_processed: int
+    tasks_completed: int = 0
     errors: int = 0
 
 
@@ -44,7 +45,13 @@ class SQSTaskSource(TaskSource):
     }
     """
 
-    def __init__(self, task_processor: Optional[TaskProcessor] = None, burst_mode: bool = False):
+    def __init__(
+        self,
+        task_processor: Optional[TaskProcessor] = None,
+        burst_mode: bool = False,
+        wait_for_tasks: bool = True,
+        task_timeout: Optional[float] = None,
+    ):
         """Initialize SQS task source.
 
         Args:
@@ -52,13 +59,21 @@ class SQSTaskSource(TaskSource):
                           a new one will be created.
             burst_mode: If True, process all available messages and exit instead
                        of polling indefinitely.
+            wait_for_tasks: If True (default), burst mode will wait for all
+                          queued tasks to complete before exiting. If False,
+                          exits as soon as SQS queue is drained.
+            task_timeout: Optional timeout in seconds for waiting on task completion
+                         in burst mode. If None, waits indefinitely.
         """
         self.task_processor = task_processor or TaskProcessor()
         self.config = get_config()
         self._running = False
         self._poll_task: Optional[asyncio.Task] = None
         self._burst_mode = burst_mode
+        self._wait_for_tasks = wait_for_tasks
+        self._task_timeout = task_timeout
         self._messages_processed = 0
+        self._tasks_completed = 0
         self._errors = 0
 
         # Import boto3 lazily to avoid requiring it for webhook-only deployments
@@ -80,6 +95,8 @@ class SQSTaskSource(TaskSource):
             queue_url=self.config.aws_sqs_queue_url,
             region=self.config.aws_region,
             burst_mode=self._burst_mode,
+            wait_for_tasks=self._wait_for_tasks,
+            task_timeout=self._task_timeout,
         )
 
     async def start(self) -> Optional[BurstResult]:
@@ -116,10 +133,16 @@ class SQSTaskSource(TaskSource):
     async def _run_burst_mode(self) -> BurstResult:
         """Run in burst mode: process all available messages then exit.
 
+        If wait_for_tasks is True (default), this will wait for all queued
+        tasks to complete before returning. This ensures that Docker jobs
+        (e.g., Claude executions) have finished, not just been queued.
+
         Returns:
-            BurstResult with count of messages processed.
+            BurstResult with count of messages processed and tasks completed.
         """
-        logger.info("Running in burst mode - will drain queue and exit")
+        logger.info("Running in burst mode - will drain queue and exit",
+                   wait_for_tasks=self._wait_for_tasks,
+                   task_timeout=self._task_timeout)
 
         # Track if we found any messages on the first poll
         first_poll = True
@@ -135,7 +158,7 @@ class SQSTaskSource(TaskSource):
                 else:
                     # We've drained the queue
                     logger.info(
-                        "Queue drained",
+                        "SQS queue drained",
                         messages_processed=self._messages_processed,
                         errors=self._errors,
                     )
@@ -156,8 +179,37 @@ class SQSTaskSource(TaskSource):
                         exc_info=True,
                     )
 
+        # Now wait for all queued tasks to complete (if enabled)
+        if self._wait_for_tasks and self._messages_processed > 0:
+            container_pool = self.task_processor.container_pool
+            queued_count = container_pool.get_total_queued_tasks()
+
+            logger.info("SQS queue drained, waiting for container tasks to complete",
+                       queued_tasks=queued_count,
+                       timeout=self._task_timeout)
+
+            all_completed = await container_pool.wait_for_all_tasks_complete(
+                timeout=self._task_timeout
+            )
+
+            if all_completed:
+                self._tasks_completed = self._messages_processed
+                logger.info("All container tasks completed successfully",
+                           tasks_completed=self._tasks_completed)
+            else:
+                # Timeout occurred - some tasks may still be running
+                remaining = container_pool.get_total_queued_tasks()
+                self._tasks_completed = self._messages_processed - remaining
+                logger.warning("Timeout waiting for container tasks",
+                              tasks_completed=self._tasks_completed,
+                              tasks_remaining=remaining)
+        else:
+            # Not waiting for tasks, or no messages processed
+            self._tasks_completed = 0
+
         return BurstResult(
             messages_processed=self._messages_processed,
+            tasks_completed=self._tasks_completed,
             errors=self._errors,
         )