diagram-to-iac 1.4.0__py3-none-any.whl → 1.6.0__py3-none-any.whl

diagram_to_iac/core/test_config.py

@@ -0,0 +1,123 @@
+"""
+Configuration loader for diagram-to-iac tests.
+
+This module provides utilities to load test configuration from the main config.yaml file,
+making test repository URLs and other test settings configurable and centralized.
+"""
+
+import os
+import yaml
+from pathlib import Path
+from typing import Dict, Any
+
+
+def load_test_config() -> Dict[str, Any]:
+    """
+    Load test configuration from the main config.yaml file.
+    
+    Returns:
+        Dict containing test configuration settings
+    """
+    # Find the config file relative to this module
+    # The config.yaml is at src/diagram_to_iac/config.yaml
+    # This file is at src/diagram_to_iac/core/test_config.py
+    # So we need to go up one directory
+    current_dir = Path(__file__).parent
+    config_path = current_dir.parent / "config.yaml"
+    
+    if not config_path.exists():
+        raise FileNotFoundError(f"Config file not found at {config_path}")
+    
+    with open(config_path, 'r') as f:
+        config = yaml.safe_load(f)
+    
+    return config.get('test', {})
+
+
+def get_test_repo_url() -> str:
+    """
+    Get the test repository URL from configuration.
+    
+    Returns:
+        The configured test repository URL
+    """
+    config = load_test_config()
+    return config.get('github', {}).get('test_repo_url', 'https://github.com/amartyamandal/test_iac_agent_private.git')
+
+
+def get_test_repo_owner() -> str:
+    """
+    Get the test repository owner from configuration.
+    
+    Returns:
+        The configured test repository owner
+    """
+    config = load_test_config()
+    return config.get('github', {}).get('test_repo_owner', 'amartyamandal')
+
+
+def get_test_repo_name() -> str:
+    """
+    Get the test repository name from configuration.
+    
+    Returns:
+        The configured test repository name
+    """
+    config = load_test_config()
+    return config.get('github', {}).get('test_repo_name', 'test_iac_agent_private')
+
+
+def get_public_test_repo_url() -> str:
+    """
+    Get the public test repository URL from configuration.
+    
+    Returns:
+        The configured public test repository URL
+    """
+    config = load_test_config()
+    return config.get('github', {}).get('public_test_repo_url', 'https://github.com/amartyamandal/test_iac_agent_public.git')
+
+
+def should_skip_integration_tests() -> bool:
+    """
+    Check if integration tests should be skipped when no GitHub token is available.
+    
+    Returns:
+        True if integration tests should be skipped without a token
+    """
+    config = load_test_config()
+    return config.get('settings', {}).get('skip_integration_tests_without_token', True)
+
+
+def should_use_real_github_api() -> bool:
+    """
+    Check if tests should use real GitHub API calls.
+    
+    Returns:
+        True if real GitHub API calls should be used
+    """
+    config = load_test_config()
+    return config.get('settings', {}).get('use_real_github_api', False)
+
+
+def should_mock_network_calls() -> bool:
+    """
+    Check if network calls should be mocked by default.
+    
+    Returns:
+        True if network calls should be mocked
+    """
+    config = load_test_config()
+    return config.get('settings', {}).get('mock_network_calls', True)
+
+
+# Convenience function for backwards compatibility
+def get_test_github_repo() -> str:
+    """
+    Get the test GitHub repository URL.
+    Alias for get_test_repo_url() for backwards compatibility.
+    
+    Returns:
+        The configured test repository URL
+    """
+    return get_test_repo_url()

diagram_to_iac/services/observability.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+import os
 from datetime import datetime, timezone
 from pathlib import Path
 import threading
@@ -11,7 +12,17 @@ class LogBus:
     """Simple JSONL logging service."""
 
     def __init__(self, log_dir: str | Path | None = None) -> None:
-        self.log_dir = Path(log_dir) if log_dir else Path(__file__).resolve().parents[3] / "logs"
+        if log_dir:
+            self.log_dir = Path(log_dir)
+        else:
+            # Check for workspace-based path first (for containers)
+            workspace_base = os.environ.get('WORKSPACE_BASE', '/workspace')
+            if os.path.exists(workspace_base):
+                self.log_dir = Path(workspace_base) / "logs"
+            else:
+                # Fallback to package-relative path
+                self.log_dir = Path(__file__).resolve().parents[3] / "logs"
+        
         self.log_dir.mkdir(parents=True, exist_ok=True)
         timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
         self.log_path = self.log_dir / f"run-{timestamp}.jsonl"

diagram_to_iac/templates/issue_frontmatter.yml

@@ -0,0 +1,240 @@
+# Issue Frontmatter Template for R2D Umbrella Issues
+# =====================================================
+# This template provides the structured metadata format for GitHub issues
+# created by the DevOps-in-a-Box R2D Action.
+
+# Standard R2D Issue Metadata
+r2d_metadata:
+  # Core identifiers
+  run_key: "{run_key}"
+  repo_url: "{repo_url}"
+  commit_sha: "{commit_sha}"
+  job_name: "{job_name}"
+  
+  # Timestamps
+  created_at: "{created_at}"
+  updated_at: "{updated_at}"
+  
+  # Status tracking
+  status: "{status}"
+  wait_reason: "{wait_reason}"
+  
+  # Associated resources
+  umbrella_issue_id: "{umbrella_issue_id}"
+  linked_pr: "{linked_pr}"
+  branch_name: "{branch_name}"
+  thread_id: "{thread_id}"
+  
+  # Artifacts and outputs
+  artifacts_path: "{artifacts_path}"
+  terraform_summary: "{terraform_summary}"
+  
+  # Retry information
+  retry_count: {retry_count}
+  predecessor_run: "{predecessor_run}"
+  
+  # Agent status summary
+  agent_statuses:
+    supervisor:
+      status: "{supervisor_status}"
+      last_updated: "{supervisor_last_updated}"
+    git_agent:
+      status: "{git_agent_status}"
+      last_updated: "{git_agent_last_updated}"
+    terraform_agent:
+      status: "{terraform_agent_status}"
+      last_updated: "{terraform_agent_last_updated}"
+    shell_agent:
+      status: "{shell_agent_status}"
+      last_updated: "{shell_agent_last_updated}"
+
+# Issue Template Configuration
+issue_template:
+  title: "🚀 R2D Deployment: {repo_name} ({short_sha})"
+  labels:
+    - "r2d-deployment"
+    - "automated"
+    - "infrastructure"
+  assignees:
+    - "github-copilot"  # Auto-assign to GitHub Copilot for AI assistance
+  
+  # Issue body sections
+  body_sections:
+    header:
+      title: "## 🤖 DevOps-in-a-Box: Automated R2D Deployment"
+      content: |
+        This is an automated deployment issue created by the DevOps-in-a-Box R2D Action.
+        
+        **Repository:** {repo_url}
+        **Commit:** `{commit_sha}`
+        **Job:** {job_name}
+        **Started:** {created_at}
+    
+    status:
+      title: "## 📊 Deployment Status"
+      content: |
+        - **Overall Status:** {status}
+        - **Current Phase:** {current_phase}
+        - **Progress:** {progress_percentage}%
+        
+        {status_details}
+    
+    agents:
+      title: "## 🤖 Agent Status"
+      content: |
+        | Agent | Status | Last Updated | Notes |
+        |-------|--------|--------------|-------|
+        | Supervisor | {supervisor_status} | {supervisor_last_updated} | {supervisor_notes} |
+        | Git Agent | {git_agent_status} | {git_agent_last_updated} | {git_agent_notes} |
+        | Terraform Agent | {terraform_agent_status} | {terraform_agent_last_updated} | {terraform_agent_notes} |
+        | Shell Agent | {shell_agent_status} | {shell_agent_last_updated} | {shell_agent_notes} |
+    
+    resources:
+      title: "## 🔗 Related Resources"
+      content: |
+        - **Branch:** {branch_name}
+        - **Thread ID:** {thread_id}
+        - **Artifacts:** {artifacts_path}
+        {linked_resources}
+    
+    terraform:
+      title: "## ⚡ Terraform Operations"
+      content: |
+        ```
+        {terraform_summary}
+        ```
+    
+    logs:
+      title: "## 📋 Deployment Logs"
+      content: |
+        <details>
+        <summary>Click to view detailed logs</summary>
+        
+        ```
+        {deployment_logs}
+        ```
+        </details>
+    
+    commands:
+      title: "## 🔧 Available Commands"
+      content: |
+        You can control this deployment by commenting on this issue:
+        
+        - `retry` - Retry the current step
+        - `cancel` - Cancel the deployment
+        - `status` - Get current status
+        - `logs` - Show recent logs
+        - `help` - Show all available commands
+    
+    footer:
+      title: "## 🤖 Automation Info"
+      content: |
+        This issue is managed by the DevOps-in-a-Box R2D Action.
+        
+        **Run Key:** `{run_key}`
+        **Retry Count:** {retry_count}
+        **Predecessor:** {predecessor_run}
+        
+        ---
+        *Powered by [DevOps-in-a-Box](https://github.com/amartyamandal/diagram-to-iac)*
+
+# Status messages for different phases
+status_messages:
+  created: "🆕 Deployment created and queued"
+  in_progress: "🚀 Deployment in progress"
+  waiting_for_pat: "⏳ Waiting for Personal Access Token configuration"
+  waiting_for_pr: "🔄 Waiting for Pull Request review and merge"
+  completed: "✅ Deployment completed successfully"
+  failed: "❌ Deployment failed"
+  cancelled: "🛑 Deployment cancelled"
+
+# Progress indicators
+progress_indicators:
+  created: 10
+  in_progress: 50
+  waiting_for_pat: 30
+  waiting_for_pr: 80
+  completed: 100
+  failed: 0
+  cancelled: 0
+
+# Command responses
+command_responses:
+  retry: |
+    🔄 **Retry Requested**
+    
+    The deployment will be retried from the current step.
+    This may take a few minutes to start.
+    
+  cancel: |
+    🛑 **Cancellation Requested**
+    
+    The deployment has been marked for cancellation.
+    Any running operations will be stopped gracefully.
+    
+  status: |
+    📊 **Current Status**
+    
+    - **Phase:** {current_phase}
+    - **Progress:** {progress_percentage}%
+    - **Last Updated:** {last_updated}
+    
+    {detailed_status}
+    
+  help: |
+    🔧 **Available Commands**
+    
+    Comment on this issue with any of these commands:
+    
+    - `retry` - Retry the current step
+    - `cancel` - Cancel the deployment  
+    - `status` - Get current status
+    - `logs` - Show recent logs
+    - `help` - Show this help message
+    
+    **Note:** Only repository members can execute commands.
+
+# Error templates
+error_templates:
+  terraform_auth_error: |
+    ❌ **Terraform Authentication Error**
+    
+    The deployment failed because Terraform Cloud authentication is not properly configured.
+    
+    **Required Action:**
+    1. Ensure your `TFE_TOKEN` secret is set in repository settings
+    2. Verify the token has access to the required workspace
+    3. Comment `retry` to restart the deployment
+    
+  git_auth_error: |
+    ❌ **Git Authentication Error**
+    
+    The deployment failed because Git operations require authentication.
+    
+    **Required Action:**
+    1. Ensure your `GITHUB_TOKEN` has sufficient permissions
+    2. Verify repository access settings
+    3. Comment `retry` to restart the deployment
+    
+  missing_terraform_files: |
+    ❌ **Missing Terraform Files**
+    
+    The deployment failed because no Terraform files were found in the repository.
+    
+    **Required Action:**
+    1. Ensure your repository contains `.tf` files
+    2. Check that files are in the expected directory structure
+    3. Update the repository and create a new issue
+    
+  policy_violation: |
+    ❌ **Security Policy Violation**
+    
+    The deployment was blocked by security policies.
+    
+    **Details:**
+    {policy_details}
+    
+    **Required Action:**
+    1. Review and fix the security issues listed above
+    2. Update your Terraform code
+    3. Create a new deployment issue

diagram_to_iac/tools/api_utils.py

@@ -3,8 +3,14 @@ import os
 from openai import OpenAI
 from anthropic import Anthropic
 import requests
-import google.generativeai as genai
-import googleapiclient.discovery
+try:
+    import google.generativeai as genai
+except ImportError:
+    genai = None
+try:
+    import googleapiclient.discovery
+except ImportError:
+    googleapiclient = None
 from concurrent.futures import ThreadPoolExecutor, TimeoutError
 
 # Import centralized configuration
@@ -45,6 +51,10 @@ def test_openai_api():
 
 def test_gemini_api():
     try:
+        if genai is None:
+            print("❌ Gemini API error: google-generativeai package not installed.")
+            return False
+            
         google_api_key = os.environ.get("GOOGLE_API_KEY")
         if not google_api_key:
             print("❌ Gemini API error: GOOGLE_API_KEY environment variable not set.")

diagram_to_iac/tools/git/git_config.yaml

@@ -0,0 +1,102 @@
+# Git Tools specific configuration
+# Inherits common settings from src/diagram_to_iac/config.yaml
+# Only tool-specific settings and overrides are defined here
+
+git_executor:
+  # Default clone settings
+  default_clone_depth: 1
+  
+  # Authentication failure detection patterns
+  auth_failure_patterns:
+    - "Authentication failed"
+    - "Permission denied"
+    - "Could not read from remote repository"
+    - "fatal: unable to access"
+    - "403 Forbidden"
+    - "401 Unauthorized"
+    - "Please make sure you have the correct access rights"
+  
+  # Repository path generation
+  repo_path_template: "{workspace}/{repo_name}"
+  sanitize_repo_names: true
+  
+  # Workspace cleanup configuration
+  auto_cleanup_existing_repos: true  # Automatically remove existing repos before cloning
+  cleanup_safety_check: true         # Verify path is within workspace before removal
+  backup_existing_repos: false       # Create backup before removal (future feature)
+  
+  # Git-specific logging configuration
+  enable_detailed_logging: true
+  log_git_commands: true
+  log_auth_failures: true
+  
+  # Memory integration
+  store_operations_in_memory: true
+  store_command_output: true
+  
+  # GitHub CLI configuration
+  github_cli:
+    default_timeout: 60  # GitHub CLI operations timeout
+    require_auth_check: true
+    auth_failure_patterns:
+      - "authentication failed"
+      - "bad credentials"
+      - "token does not have permission"
+      - "Must have admin rights to Repository"
+      - "HTTP 401"
+      - "HTTP 403"
+    issue_creation_patterns:
+      - "title is required"
+      - "body is required"
+      - "repository not found"
+    success_patterns:
+      - "https://github.com/"
+      - "/issues/"
+
+# Error messages following our pattern
+error_messages:
+  invalid_repo_url: "Git executor: Invalid repository URL '{repo_url}'"
+  workspace_not_accessible: "Git executor: Workspace directory '{workspace}' is not accessible"
+  clone_failed: "Git executor: Failed to clone repository '{repo_url}'"
+  auth_failed: "Git executor: Authentication failed for repository '{repo_url}'"
+  timeout_exceeded: "Git executor: Git operation timed out after {timeout} seconds"
+  shell_executor_error: "Git executor: Shell executor error: {error}"
+  repo_already_exists: "Git executor: Repository '{repo_name}' already exists in workspace"
+  repo_cleanup_started: "Git executor: Cleaning up existing repository '{repo_name}' at '{repo_path}'"
+  repo_cleanup_completed: "Git executor: Successfully cleaned up existing repository '{repo_name}'"
+  repo_cleanup_failed: "Git executor: Failed to clean up existing repository '{repo_name}': {error}"
+  cleanup_safety_violation: "Git executor: Repository path '{repo_path}' is outside workspace - cleanup denied for security"
+  
+  # GitHub CLI error messages
+  gh_auth_failed: "GitHub CLI: Authentication failed - please check GITHUB_TOKEN"
+  gh_repo_not_found: "GitHub CLI: Repository '{repo_url}' not found or access denied"
+  gh_issue_creation_failed: "GitHub CLI: Failed to create issue in '{repo_url}'"
+  gh_invalid_repo_format: "GitHub CLI: Invalid repository format '{repo_url}' - expected owner/repo"
+  gh_command_failed: "GitHub CLI: Command failed with exit code {exit_code}"
+
+# Success messages following our pattern
+success_messages:
+  clone_started: "Git executor: Starting clone of '{repo_url}'"
+  clone_completed: "Git executor: Successfully cloned '{repo_url}' to '{repo_path}' in {duration:.2f}s"
+  repo_path_resolved: "Git executor: Repository path resolved to '{repo_path}'"
+  repo_cleanup_success: "Git executor: Successfully cleaned up existing repository before cloning"
+  
+  # GitHub CLI success messages
+  gh_issue_created: "GitHub CLI: Successfully created issue '{title}' in '{repo_url}'"
+  gh_auth_verified: "GitHub CLI: Authentication verified for GitHub operations"
+  gh_repo_accessible: "GitHub CLI: Repository '{repo_url}' is accessible"
+
+# Status codes for structured responses
+status_codes:
+  success: "SUCCESS"
+  auth_failed: "AUTH_FAILED"
+  error: "ERROR"
+  timeout: "TIMEOUT"
+  already_exists: "ALREADY_EXISTS"
+  
+  # GitHub CLI status codes
+  gh_success: "GH_SUCCESS"
+  gh_auth_failed: "GH_AUTH_FAILED"
+  gh_repo_not_found: "GH_REPO_NOT_FOUND"
+  gh_permission_denied: "GH_PERMISSION_DENIED"
+  gh_error: "GH_ERROR"

diagram_to_iac/tools/sec_utils.py

@@ -37,44 +37,77 @@ except ImportError:
 # Path inside container where the encoded YAML is mounted (dev only)
 _YAML_PATH = pathlib.Path("/run/secrets.yaml")
 
-# Expected secrets based on secrets_example.yaml
-EXPECTED_SECRETS = [
-    "DOCKERHUB_API_KEY",
-    "DOCKERHUB_USERNAME", 
-    "TF_API_KEY",
-    "PYPI_API_KEY",
-    "OPENAI_API_KEY",
-    "GOOGLE_API_KEY",
-    "ANTHROPIC_API_KEY", 
-    "GROK_API_KEY",
-    "REPO_API_KEY"
-]
-
-# Required secrets that must be present (others are optional)
-REQUIRED_SECRETS = [
-    "REPO_API_KEY"  # GITHUB_TOKEN is required for repo operations
-]
-
-# Optional AI API secrets (at least one should be present for AI functionality)
-AI_API_SECRETS = [
-    "OPENAI_API_KEY",
-    "GOOGLE_API_KEY", 
-    "ANTHROPIC_API_KEY",
-    "GROK_API_KEY"
-]
+def _load_config_secrets():
+    """Load secret configuration from config.yaml."""
+    try:
+        # Try to load from config.yaml
+        config_path = pathlib.Path(__file__).parent.parent / "config.yaml"
+        if config_path.exists() and yaml:
+            config_data = yaml.safe_load(config_path.read_text())
+            security_config = config_data.get("security", {})
+            
+            required_secrets = security_config.get("required_secrets", ["REPO_API_KEY"])
+            optional_secrets = security_config.get("optional_secrets", [])
+            secret_mappings = security_config.get("secret_mappings", {})
+            
+            # Combine required and optional secrets
+            expected_secrets = required_secrets + optional_secrets
+            
+            # Create full mapping (defaults to same name if not mapped)
+            full_mapping = {}
+            for secret in expected_secrets:
+                full_mapping[secret] = secret_mappings.get(secret, secret)
+            
+            # AI API secrets (hardcoded as they're specific to AI functionality)
+            ai_secrets = [
+                "OPENAI_API_KEY",
+                "GOOGLE_API_KEY", 
+                "ANTHROPIC_API_KEY",
+                "GROK_API_KEY"
+            ]
+            
+            return expected_secrets, required_secrets, ai_secrets, full_mapping
+    except Exception as e:
+        print(f"⚠️ Warning: Could not load secret config from config.yaml: {e}")
+    
+    # Fallback to hardcoded values
+    expected_secrets = [
+        "DOCKERHUB_API_KEY",
+        "DOCKERHUB_USERNAME", 
+        "TF_API_KEY",
+        "PYPI_API_KEY",
+        "OPENAI_API_KEY",
+        "GOOGLE_API_KEY",
+        "ANTHROPIC_API_KEY", 
+        "GROK_API_KEY",
+        "REPO_API_KEY"
+    ]
+    
+    required_secrets = ["REPO_API_KEY"]
+    
+    ai_secrets = [
+        "OPENAI_API_KEY",
+        "GOOGLE_API_KEY", 
+        "ANTHROPIC_API_KEY",
+        "GROK_API_KEY"
+    ]
+    
+    secret_mapping = {
+        "REPO_API_KEY": "GITHUB_TOKEN",
+        "TF_API_KEY": "TFE_TOKEN",
+        "DOCKERHUB_API_KEY": "DOCKERHUB_API_KEY",
+        "DOCKERHUB_USERNAME": "DOCKERHUB_USERNAME",
+        "PYPI_API_KEY": "PYPI_API_KEY", 
+        "OPENAI_API_KEY": "OPENAI_API_KEY",
+        "GOOGLE_API_KEY": "GOOGLE_API_KEY",
+        "ANTHROPIC_API_KEY": "ANTHROPIC_API_KEY",
+        "GROK_API_KEY": "GROK_API_KEY"
+    }
+    
+    return expected_secrets, required_secrets, ai_secrets, secret_mapping
 
-# Map internal secret names to environment variable names
-SECRET_ENV_MAPPING = {
-    "REPO_API_KEY": "GITHUB_TOKEN",
-    "TF_API_KEY": "TFE_TOKEN",
-    "DOCKERHUB_API_KEY": "DOCKERHUB_API_KEY",
-    "DOCKERHUB_USERNAME": "DOCKERHUB_USERNAME",
-    "PYPI_API_KEY": "PYPI_API_KEY", 
-    "OPENAI_API_KEY": "OPENAI_API_KEY",
-    "GOOGLE_API_KEY": "GOOGLE_API_KEY",
-    "ANTHROPIC_API_KEY": "ANTHROPIC_API_KEY",
-    "GROK_API_KEY": "GROK_API_KEY"
-}
+# Load configuration at module level
+EXPECTED_SECRETS, REQUIRED_SECRETS, AI_API_SECRETS, SECRET_ENV_MAPPING = _load_config_secrets()
 
 
 def _decode_b64(enc: str) -> str:
@@ -107,12 +140,35 @@ def _is_dev_environment() -> bool:
     )
 
 
+def _is_ci_environment() -> bool:
+    """Check if running in CI environment."""
+    return os.environ.get("CI") == "true" or os.environ.get("GITHUB_ACTIONS") == "true"
+
+
 def _get_env_secrets() -> Dict[str, Optional[str]]:
     """Get secrets from environment variables."""
     env_secrets = {}
+    is_ci = _is_ci_environment()
+    
     for secret_key in EXPECTED_SECRETS:
         env_name = SECRET_ENV_MAPPING.get(secret_key, secret_key)
-        raw_value = os.environ.get(env_name)
+        raw_value = None
+        
+        if is_ci:
+            # In CI, check for _ENCODED variant first (since that's how they're provided)
+            encoded_env_name = f"{secret_key}_ENCODED"  # Use secret_key directly for CI
+            raw_value = os.environ.get(encoded_env_name)
+            
+            # If not found with secret_key, try with env_name
+            if raw_value is None:
+                encoded_env_name = f"{env_name}_ENCODED"
+                raw_value = os.environ.get(encoded_env_name)
+        
+        # If still not found, try the direct environment variable name
+        if raw_value is None:
+            raw_value = os.environ.get(env_name)
+        
+        # If we found a value, process it
         if raw_value:
             # Check if value is already decoded, use as-is; otherwise decode it
             if (secret_key == "TF_API_KEY" and ".atlasv1." in raw_value) or \
@@ -126,6 +182,7 @@ def _get_env_secrets() -> Dict[str, Optional[str]]:
                 env_secrets[secret_key] = _decode_b64(raw_value)
         else:
             env_secrets[secret_key] = None
+    
     return env_secrets