diagram-to-iac 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

diagram_to_iac/agents/terraform_langgraph/agent.py

@@ -42,6 +42,7 @@ from diagram_to_iac.tools.llm_utils.router import get_llm, LLMRouter
 from diagram_to_iac.core.agent_base import AgentBase
 from diagram_to_iac.core.memory import create_memory, LangGraphMemoryAdapter
 from diagram_to_iac.services.observability import log_event
+from diagram_to_iac.core.config_loader import get_config, get_config_value
 from .parser import classify_terraform_error
 
 
@@ -125,36 +126,54 @@ class TerraformAgent(AgentBase):
         # Store memory type for tool initialization
         self.memory_type = memory_type
 
-        # Load configuration
+        # Load configuration using centralized system
         if config_path is None:
             base_dir = os.path.dirname(os.path.abspath(__file__))
             config_path = os.path.join(base_dir, "config.yaml")
             self.logger.debug(f"Default config path set to: {config_path}")
 
         try:
-            with open(config_path, "r") as f:
-                self.config = yaml.safe_load(f)
-            if self.config is None:
+            # Use centralized configuration loading with hierarchical merging
+            base_config = get_config()
+            
+            # Load agent-specific config if provided
+            agent_config = {}
+            if config_path and os.path.exists(config_path):
+                with open(config_path, 'r') as f:
+                    agent_config = yaml.safe_load(f) or {}
+            
+            # Deep merge base config with agent-specific overrides
+            self.config = self._deep_merge(base_config, agent_config)
+            self.logger.info(f"Configuration loaded successfully from centralized system")
+            if "terraform_token_env" not in self.config:
+                self.config["terraform_token_env"] = "TFE_TOKEN"
+        except Exception as e:
+            self.logger.warning(f"Failed to load configuration via centralized system: {e}. Using fallback.")
+            # Fallback to direct YAML loading for backward compatibility
+            try:
+                with open(config_path, "r") as f:
+                    self.config = yaml.safe_load(f)
+                if self.config is None:
+                    self.logger.warning(
+                        f"Configuration file at {config_path} is empty. Using defaults."
+                    )
+                    self._set_default_config()
+                else:
+                    self.logger.info(
+                        f"Configuration loaded successfully from {config_path}"
+                    )
+                    if "terraform_token_env" not in self.config:
+                        self.config["terraform_token_env"] = "TFE_TOKEN"
+            except FileNotFoundError:
                 self.logger.warning(
-                    f"Configuration file at {config_path} is empty. Using defaults."
+                    f"Configuration file not found at {config_path}. Using defaults."
                 )
                 self._set_default_config()
-            else:
-                self.logger.info(
-                    f"Configuration loaded successfully from {config_path}"
+            except yaml.YAMLError as e:
+                self.logger.error(
+                    f"Error parsing YAML configuration: {e}. Using defaults.", exc_info=True
                 )
-                if "terraform_token_env" not in self.config:
-                    self.config["terraform_token_env"] = "TFE_TOKEN"
-        except FileNotFoundError:
-            self.logger.warning(
-                f"Configuration file not found at {config_path}. Using defaults."
-            )
-            self._set_default_config()
-        except yaml.YAMLError as e:
-            self.logger.error(
-                f"Error parsing YAML configuration: {e}. Using defaults.", exc_info=True
-            )
-            self._set_default_config()
+                self._set_default_config()
 
         # Initialize enhanced LLM router
         self.llm_router = LLMRouter()
@@ -178,20 +197,49 @@ class TerraformAgent(AgentBase):
         self.logger.info("TerraformAgent initialized successfully")
 
     def _set_default_config(self):
-        """Set default configuration values."""
+        """Set default configuration values using centralized system."""
         self.logger.info("Setting default configuration for TerraformAgent")
         self.config = {
-            "llm": {"model_name": "gpt-4o-mini", "temperature": 0.1},
+            "llm": {
+                "model_name": get_config_value("ai.default_model", "gpt-4o-mini"),
+                "temperature": get_config_value("ai.default_temperature", 0.1)
+            },
             "routing_keys": {
-                "terraform_init": "ROUTE_TO_TF_INIT",
-                "terraform_plan": "ROUTE_TO_TF_PLAN",
-                "terraform_apply": "ROUTE_TO_TF_APPLY",
-                "open_issue": "ROUTE_TO_OPEN_ISSUE",
-                "end": "ROUTE_TO_END",
+                "terraform_init": get_config_value("routing.tokens.terraform_init", "ROUTE_TO_TF_INIT"),
+                "terraform_plan": get_config_value("routing.tokens.terraform_plan", "ROUTE_TO_TF_PLAN"),
+                "terraform_apply": get_config_value("routing.tokens.terraform_apply", "ROUTE_TO_TF_APPLY"),
+                "open_issue": get_config_value("routing.tokens.open_issue", "ROUTE_TO_OPEN_ISSUE"),
+                "end": get_config_value("routing.tokens.end", "ROUTE_TO_END"),
             },
             "terraform_token_env": "TFE_TOKEN",
+            "tools": {
+                "terraform": {
+                    "timeout": get_config_value("network.terraform_timeout", 300),
+                    "default_auto_approve": get_config_value("tools.terraform.default_auto_approve", True),
+                    "default_plan_file": get_config_value("tools.terraform.default_plan_file", "plan.tfplan")
+                }
+            }
         }
 
+    def _deep_merge(self, base: dict, overlay: dict) -> dict:
+        """
+        Deep merge two dictionaries, with overlay taking precedence.
+        
+        Args:
+            base: Base dictionary
+            overlay: Dictionary to overlay on base
+            
+        Returns:
+            Merged dictionary
+        """
+        result = base.copy()
+        for key, value in overlay.items():
+            if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+                result[key] = self._deep_merge(result[key], value)
+            else:
+                result[key] = value
+        return result
+
     def _initialize_tools(self):
         """Initialize the Terraform tools following the established pattern."""
         try:

diagram_to_iac/core/config_loader.py

@@ -0,0 +1,281 @@
+# src/diagram_to_iac/core/config_loader.py
+"""
+Central configuration loader for diagram-to-iac project.
+Handles loading and merging configuration from multiple sources with environment variable overrides.
+"""
+
+import os
+import yaml
+import logging
+from pathlib import Path
+from typing import Dict, Any, Optional, Union
+from functools import lru_cache
+
+logger = logging.getLogger(__name__)
+
+class ConfigLoader:
+    """
+    Central configuration management for the diagram-to-iac project.
+    Loads configuration from YAML files and provides environment variable override capability.
+    """
+    
+    def __init__(self, app_config_path: Optional[str] = None, system_config_path: Optional[str] = None):
+        """
+        Initialize ConfigLoader with optional custom config paths.
+        
+        Args:
+            app_config_path: Path to application config file (default: src/diagram_to_iac/config.yaml)
+            system_config_path: Path to system config file (default: config/system.yaml)
+        """
+        self.logger = logging.getLogger(self.__class__.__name__)
+        
+        # Set default paths
+        self.base_path = Path(__file__).parent.parent.parent.parent  # diagram-to-iac root
+        self.app_config_path = Path(app_config_path) if app_config_path else self.base_path / "src" / "diagram_to_iac" / "config.yaml"
+        self.system_config_path = Path(system_config_path) if system_config_path else self.base_path / "config" / "system.yaml"
+        
+        # Cache for loaded configs
+        self._app_config = None
+        self._system_config = None
+        self._merged_config = None
+    
+    @lru_cache(maxsize=1)
+    def get_config(self) -> Dict[str, Any]:
+        """
+        Get the complete merged configuration with environment variable overrides.
+        
+        Returns:
+            Merged configuration dictionary
+        """
+        if self._merged_config is None:
+            self._merged_config = self._load_and_merge_configs()
+        return self._merged_config
+    
+    def _load_and_merge_configs(self) -> Dict[str, Any]:
+        """
+        Load and merge all configuration sources.
+        
+        Returns:
+            Merged configuration dictionary
+        """
+        # Load base configs
+        app_config = self._load_app_config()
+        system_config = self._load_system_config()
+        
+        # Start with system config as base, overlay app config
+        merged = self._deep_merge(system_config, app_config)
+        
+        # Apply environment variable overrides
+        merged = self._apply_env_overrides(merged)
+        
+        self.logger.debug("Configuration loaded and merged successfully")
+        return merged
+    
+    def _load_app_config(self) -> Dict[str, Any]:
+        """Load application configuration from YAML file."""
+        if self._app_config is None:
+            try:
+                if self.app_config_path.exists():
+                    with open(self.app_config_path, 'r') as f:
+                        self._app_config = yaml.safe_load(f) or {}
+                    self.logger.debug(f"Loaded app config from {self.app_config_path}")
+                else:
+                    self.logger.warning(f"App config file not found: {self.app_config_path}")
+                    self._app_config = {}
+            except Exception as e:
+                self.logger.error(f"Failed to load app config: {e}")
+                self._app_config = {}
+        return self._app_config
+    
+    def _load_system_config(self) -> Dict[str, Any]:
+        """Load system configuration from YAML file."""
+        if self._system_config is None:
+            try:
+                if self.system_config_path.exists():
+                    with open(self.system_config_path, 'r') as f:
+                        self._system_config = yaml.safe_load(f) or {}
+                    self.logger.debug(f"Loaded system config from {self.system_config_path}")
+                else:
+                    self.logger.warning(f"System config file not found: {self.system_config_path}")
+                    self._system_config = {}
+            except Exception as e:
+                self.logger.error(f"Failed to load system config: {e}")
+                self._system_config = {}
+        return self._system_config
+    
+    def _deep_merge(self, base: Dict[str, Any], overlay: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Deep merge two dictionaries, with overlay taking precedence.
+        
+        Args:
+            base: Base dictionary
+            overlay: Dictionary to overlay on base
+            
+        Returns:
+            Merged dictionary
+        """
+        result = base.copy()
+        
+        for key, value in overlay.items():
+            if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+                result[key] = self._deep_merge(result[key], value)
+            else:
+                result[key] = value
+        
+        return result
+    
+    def _apply_env_overrides(self, config: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Apply environment variable overrides to configuration.
+        Environment variables should be in the format: DIAGRAM_TO_IAC_<SECTION>_<KEY>
+        
+        Args:
+            config: Base configuration dictionary
+            
+        Returns:
+            Configuration with environment overrides applied
+        """
+        result = config.copy()
+        env_prefix = "DIAGRAM_TO_IAC_"
+        
+        # Get list of allowed overrides from config
+        allowed_overrides = config.get("environment_overrides", {}).get("allowed_overrides", [])
+        
+        for env_var, env_value in os.environ.items():
+            if not env_var.startswith(env_prefix):
+                continue
+                
+            # Parse environment variable name
+            # e.g., DIAGRAM_TO_IAC_NETWORK_API_TIMEOUT -> network.api_timeout
+            var_path = env_var[len(env_prefix):].lower().replace('_', '.')
+            
+            # Check if override is allowed
+            if var_path not in allowed_overrides:
+                self.logger.debug(f"Environment override not allowed: {var_path}")
+                continue
+            
+            # Convert string value to appropriate type
+            converted_value = self._convert_env_value(env_value)
+            
+            # Apply override to config
+            self._set_nested_value(result, var_path, converted_value)
+            self.logger.debug(f"Applied environment override: {var_path} = {converted_value}")
+        
+        return result
+    
+    def _convert_env_value(self, value: str) -> Union[str, int, float, bool]:
+        """
+        Convert environment variable string value to appropriate type.
+        
+        Args:
+            value: String value from environment variable
+            
+        Returns:
+            Converted value
+        """
+        # Handle boolean values
+        if value.lower() in ("true", "yes", "1", "on"):
+            return True
+        elif value.lower() in ("false", "no", "0", "off"):
+            return False
+        
+        # Handle numeric values
+        try:
+            if '.' in value:
+                return float(value)
+            else:
+                return int(value)
+        except ValueError:
+            pass
+        
+        # Return as string
+        return value
+    
+    def _set_nested_value(self, config: Dict[str, Any], path: str, value: Any) -> None:
+        """
+        Set a nested value in configuration using dot notation.
+        
+        Args:
+            config: Configuration dictionary to modify
+            path: Dot-separated path (e.g., "network.api_timeout")
+            value: Value to set
+        """
+        keys = path.split('.')
+        current = config
+        
+        # Navigate to the parent of the target key
+        for key in keys[:-1]:
+            if key not in current:
+                current[key] = {}
+            current = current[key]
+        
+        # Set the final value
+        current[keys[-1]] = value
+    
+    def get_section(self, section: str) -> Dict[str, Any]:
+        """
+        Get a specific configuration section.
+        
+        Args:
+            section: Section name (e.g., "network", "ai", "routing")
+            
+        Returns:
+            Configuration section dictionary
+        """
+        return self.get_config().get(section, {})
+    
+    def get_value(self, path: str, default: Any = None) -> Any:
+        """
+        Get a configuration value using dot notation.
+        
+        Args:
+            path: Dot-separated path (e.g., "network.api_timeout")
+            default: Default value if path not found
+            
+        Returns:
+            Configuration value or default
+        """
+        keys = path.split('.')
+        current = self.get_config()
+        
+        try:
+            for key in keys:
+                current = current[key]
+            return current
+        except (KeyError, TypeError):
+            return default
+    
+    def reload(self) -> None:
+        """Reload configuration from files (clears cache)."""
+        self._app_config = None
+        self._system_config = None
+        self._merged_config = None
+        self.get_config.cache_clear()
+        self.logger.debug("Configuration cache cleared, will reload on next access")
+
+
+# Global configuration loader instance
+_config_loader = None
+
+def get_config_loader() -> ConfigLoader:
+    """Get the global configuration loader instance."""
+    global _config_loader
+    if _config_loader is None:
+        _config_loader = ConfigLoader()
+    return _config_loader
+
+def get_config() -> Dict[str, Any]:
+    """Get the complete merged configuration."""
+    return get_config_loader().get_config()
+
+def get_config_section(section: str) -> Dict[str, Any]:
+    """Get a specific configuration section."""
+    return get_config_loader().get_section(section)
+
+def get_config_value(path: str, default: Any = None) -> Any:
+    """Get a configuration value using dot notation."""
+    return get_config_loader().get_value(path, default)
+
+def reload_config() -> None:
+    """Reload configuration from files."""
+    get_config_loader().reload()

diagram_to_iac/core/memory.py

@@ -2,7 +2,7 @@ from typing import Any, Dict, List, Optional
 import json
 import os
 from pathlib import Path
-from subprocess import check_output, CalledProcessError
+from subprocess import check_output, CalledProcessError, TimeoutExpired
 # Intended to be a thin wrapper over LangGraph's StateGraph node store.
 # For now, a simple dictionary-based state.
 
@@ -126,7 +126,12 @@ def save_agent_state(state: Dict[str, Any]) -> None:
 def current_git_sha() -> Optional[str]:
     """Return the current git commit SHA, or ``None`` if unavailable."""
     try:
-        return check_output(["git", "rev-parse", "HEAD"], text=True).strip()
+        return check_output(
+            ["git", "rev-parse", "HEAD"], 
+            text=True, 
+            timeout=5,
+            env={**os.environ, "GIT_TERMINAL_PROMPT": "0"}
+        ).strip()
     except Exception:
         return None
 

diagram_to_iac/tools/api_utils.py

@@ -7,6 +7,14 @@ import google.generativeai as genai
 import googleapiclient.discovery
 from concurrent.futures import ThreadPoolExecutor, TimeoutError
 
+# Import centralized configuration
+try:
+    from diagram_to_iac.core.config_loader import get_config_value
+except ImportError:
+    # Fallback if config system not available
+    def get_config_value(path: str, default=None):
+        return default
+
 def test_openai_api():
     try:
         if not os.environ.get("OPENAI_API_KEY"):
@@ -14,7 +22,10 @@ def test_openai_api():
             return False
         client = OpenAI()
         
-        # Run the API call with a 10-second timeout
+        # Get timeout from configuration
+        api_timeout = get_config_value("network.api_timeout", 10)
+        
+        # Run the API call with configured timeout
         with ThreadPoolExecutor(max_workers=1) as executor:
             future = executor.submit(
                 client.chat.completions.create,
@@ -23,9 +34,9 @@ def test_openai_api():
                 max_tokens=10
             )
             try:
-                response = future.result(timeout=10)
+                response = future.result(timeout=api_timeout)
             except TimeoutError:
-                print("❌ OpenAI API error: request timed out.")
+                print(f"❌ OpenAI API error: request timed out after {api_timeout}s.")
                 return False
         return True
     except Exception as e:
@@ -41,13 +52,16 @@ def test_gemini_api():
         genai.configure(api_key=google_api_key)
         model = genai.GenerativeModel('gemini-2.0-flash') # Corrected model name
         
-        # Run the API call with a 10-second timeout
+        # Get timeout from configuration
+        api_timeout = get_config_value("network.api_timeout", 10)
+        
+        # Run the API call with configured timeout
         with ThreadPoolExecutor(max_workers=1) as executor:
             future = executor.submit(model.generate_content, "Hello, are you working?")
             try:
-                response = future.result(timeout=10)
+                response = future.result(timeout=api_timeout)
             except TimeoutError:
-                print("❌ Gemini API error: request timed out.")
+                print(f"❌ Gemini API error: request timed out after {api_timeout}s.")
                 return False
         return True
     except Exception as e:
@@ -61,7 +75,10 @@ def test_anthropic_api():
             return False
         client = Anthropic()
         
-        # Run the API call with a 10-second timeout
+        # Get timeout from configuration
+        api_timeout = get_config_value("network.api_timeout", 10)
+        
+        # Run the API call with configured timeout
         with ThreadPoolExecutor(max_workers=1) as executor:
             future = executor.submit(
                 client.messages.create,
@@ -70,9 +87,9 @@ def test_anthropic_api():
                 messages=[{"role": "user", "content": "Hello, are you working?"}]
             )
             try:
-                response = future.result(timeout=10)
+                response = future.result(timeout=api_timeout)
             except TimeoutError:
-                print("❌ Anthropic API error: request timed out.")
+                print(f"❌ Anthropic API error: request timed out after {api_timeout}s.")
                 return False
         return True
     except Exception as e:
@@ -92,7 +109,10 @@ def test_github_api():
             "Accept": "application/vnd.github.v3+json"
         }
         
-        # Run the API call with a 10-second timeout
+        # Get timeout from configuration
+        github_timeout = get_config_value("network.github_timeout", 15)
+        
+        # Run the API call with configured timeout
         with ThreadPoolExecutor(max_workers=1) as executor:
             future = executor.submit(
                 requests.get,
@@ -100,9 +120,9 @@ def test_github_api():
                 headers=headers
             )
             try:
-                response = future.result(timeout=10)
+                response = future.result(timeout=github_timeout)
             except TimeoutError:
-                print("❌ GitHub API error: request timed out.")
+                print(f"❌ GitHub API error: request timed out after {github_timeout}s.")
                 return False
         
         if response.status_code == 200:
@@ -119,33 +139,53 @@ def test_github_api():
 
 def test_Terraform_API():
     try:
-        if not os.environ.get("TFE_TOKEN"):
+        tfe_token = os.environ.get("TFE_TOKEN")
+        if not tfe_token:
             print("❌ Terraform API error: TFE_TOKEN environment variable not set.")
             return False
+        
         headers = {
-            "Authorization": f"Bearer {os.environ.get('TFE_TOKEN')}",
+            "Authorization": f"Bearer {tfe_token}",
             "Content-Type": "application/vnd.api+json"
         }
-        # Run the API call with a 10-second timeout
+        
+        # Get timeout from configuration
+        terraform_timeout = get_config_value("network.terraform_timeout", 30)
+        
+        # Test account details endpoint (more reliable than organizations)
+        # Run the API call with configured timeout
         with ThreadPoolExecutor(max_workers=1) as executor:
             future = executor.submit(
                 requests.get,
-                "https://app.terraform.io/api/v2/organizations",
+                "https://app.terraform.io/api/v2/account/details",
                 headers=headers
             )
             try:
-                response = future.result(timeout=10)
+                response = future.result(timeout=terraform_timeout)
             except TimeoutError:
-                print("❌ Terraform API error: request timed out.")
+                print(f"❌ Terraform API error: request timed out after {terraform_timeout}s.")
                 return False
+        
         if response.status_code == 200:
-            org_data = response.json()
-            # print(f"✅ Terraform API works! Organizations: {org_data}")
+            print("✅ Terraform API works! Account access verified.")
             return True
         else:
             print(f"❌ Terraform API error: Status code {response.status_code}")
             if hasattr(response, 'text'):
                 print(f"   Response body: {response.text}")
+            
+            # Provide specific guidance for common error codes
+            if response.status_code == 401:
+                print("   This is an authentication error. Common causes:")
+                print("   - TFE_TOKEN is invalid or expired")
+                print("   - Token format is incorrect (should be a Terraform Cloud API token)")
+                print("   - Token doesn't have sufficient permissions")
+                print("   - Please verify your token at: https://app.terraform.io/app/settings/tokens")
+            elif response.status_code == 403:
+                print("   This is a permission error. The token is valid but lacks access rights.")
+            elif response.status_code == 404:
+                print("   The API endpoint was not found. Check if the URL is correct.")
+            
             return False
     except requests.exceptions.ConnectionError as e:
         print(f"❌ Terraform API error: Connection failed - {str(e)}")
@@ -167,6 +207,7 @@ def test_Terraform_API():
         print(f"❌ Terraform API error: Unexpected error - {str(e)}")
         return False
 
+
 def test_all_apis():
     print("Hello from the test workflow!")