diagram-to-iac 1.0.1__py3-none-any.whl → 1.0.2__py3-none-any.whl

diagram_to_iac/__init__.py

@@ -1,10 +1,36 @@
+# Initialize diagram-to-iac package with centralized configuration and secret management
+
+# Initialize configuration system first
+try:
+    from diagram_to_iac.core.config_loader import get_config_loader
+    # Initialize the global configuration loader
+    config_loader = get_config_loader()
+    # Pre-load configuration to catch any issues early
+    config = config_loader.get_config()
+except ImportError as e:
+    # If config_loader can't be imported, log but continue
+    print(f"⚠️ Warning: Could not import configuration system: {e}")
+except Exception as e:
+    # If configuration loading fails, log but continue
+    print(f"⚠️ Warning: Configuration system initialization failed: {e}")
+
+# Initialize secret management system
 try:
-    from diagram_to_iac.tools.sec_utils import load_yaml_secrets
-except Exception:  # noqa: BLE001
-    load_yaml_secrets = None
+    from diagram_to_iac.tools.sec_utils import load_secrets
+    load_secrets()
+except ImportError:
+    # If sec_utils can't be imported, that's fine during development
+    pass
+except SystemExit:
+    # load_secrets() calls sys.exit() on critical errors
+    # In development, we don't want to crash the entire package import
+    pass
+except Exception as e:
+    print(f"⚠️ Warning: Secret management initialization failed: {e}")
 
-if load_yaml_secrets:
-    try:
-        load_yaml_secrets()
-    except Exception:  # noqa: BLE001
-        pass
+# Make configuration easily accessible
+try:
+    from diagram_to_iac.core.config_loader import get_config, get_config_section, get_config_value
+    __all__ = ["get_config", "get_config_section", "get_config_value"]
+except ImportError:
+    __all__ = []

diagram_to_iac/agents/demonstrator_langgraph/agent.py

@@ -22,6 +22,7 @@ from langgraph.checkpoint.memory import MemorySaver
 
 from diagram_to_iac.core.agent_base import AgentBase
 from diagram_to_iac.core.memory import create_memory
+from diagram_to_iac.core.config_loader import get_config, get_config_value
 from diagram_to_iac.tools.llm_utils.router import LLMRouter, get_llm
 from diagram_to_iac.tools.text_utils import (
     generate_organic_issue_title,
@@ -151,16 +152,19 @@ class DemonstratorAgent(AgentBase):
         self.logger.info("DemonstratorAgent initialized successfully with organic LangGraph architecture")
 
     def _set_default_config(self):
-        """Set default configuration values."""
+        """Set default configuration values using centralized system."""
         self.config = {
-            "llm": {"model_name": "gpt-4o-mini", "temperature": 0.1},
+            "llm": {
+                "model_name": get_config_value("ai.default_model", "gpt-4o-mini"), 
+                "temperature": get_config_value("ai.default_temperature", 0.1)
+            },
             "routing_keys": {
-                "analyze": "ROUTE_TO_ANALYZE",
-                "demonstrate": "ROUTE_TO_DEMONSTRATE", 
-                "collect_inputs": "ROUTE_TO_COLLECT_INPUTS",
-                "retry": "ROUTE_TO_RETRY",
-                "create_issue": "ROUTE_TO_CREATE_ISSUE",
-                "end": "ROUTE_TO_END",
+                "analyze": get_config_value("routing.tokens.analyze", "ROUTE_TO_ANALYZE"),
+                "demonstrate": get_config_value("routing.tokens.demonstrate", "ROUTE_TO_DEMONSTRATE"), 
+                "collect_inputs": get_config_value("routing.tokens.collect_inputs", "ROUTE_TO_COLLECT_INPUTS"),
+                "retry": get_config_value("routing.tokens.retry", "ROUTE_TO_RETRY"),
+                "create_issue": get_config_value("routing.tokens.create_issue", "ROUTE_TO_CREATE_ISSUE"),
+                "end": get_config_value("routing.tokens.end", "ROUTE_TO_END"),
             },
             "prompts": {
                 "planner_prompt": """User request: "{user_input}"

diagram_to_iac/agents/git_langgraph/agent.py

@@ -46,6 +46,7 @@ from diagram_to_iac.tools.llm_utils.router import get_llm, LLMRouter
 from diagram_to_iac.core.agent_base import AgentBase
 from diagram_to_iac.core.memory import create_memory, LangGraphMemoryAdapter
 from diagram_to_iac.services.observability import log_event
+from diagram_to_iac.core.config_loader import get_config, get_config_value
 
 from diagram_to_iac.services.observability import log_event
 
@@ -131,26 +132,42 @@ class GitAgent(AgentBase):
         # Store memory type for tool initialization
         self.memory_type = memory_type
         
-        # Load configuration
+        # Load configuration using centralized system
         if config_path is None:
             base_dir = os.path.dirname(os.path.abspath(__file__))
             config_path = os.path.join(base_dir, 'config.yaml')
             self.logger.debug(f"Default config path set to: {config_path}")
         
         try:
-            with open(config_path, 'r') as f:
-                self.config = yaml.safe_load(f)
-            if self.config is None:
-                self.logger.warning(f"Configuration file at {config_path} is empty. Using defaults.")
+            # Use centralized configuration loading with hierarchical merging
+            base_config = get_config()
+            
+            # Load agent-specific config if provided
+            agent_config = {}
+            if config_path and os.path.exists(config_path):
+                with open(config_path, 'r') as f:
+                    agent_config = yaml.safe_load(f) or {}
+            
+            # Deep merge base config with agent-specific overrides
+            self.config = self._deep_merge(base_config, agent_config)
+            self.logger.info(f"Configuration loaded successfully from centralized system")
+        except Exception as e:
+            self.logger.warning(f"Failed to load configuration via centralized system: {e}. Using fallback.")
+            # Fallback to direct YAML loading for backward compatibility
+            try:
+                with open(config_path, 'r') as f:
+                    self.config = yaml.safe_load(f)
+                if self.config is None:
+                    self.logger.warning(f"Configuration file at {config_path} is empty. Using defaults.")
+                    self._set_default_config()
+                else:
+                    self.logger.info(f"Configuration loaded successfully from {config_path}")
+            except FileNotFoundError:
+                self.logger.warning(f"Configuration file not found at {config_path}. Using defaults.")
+                self._set_default_config()
+            except yaml.YAMLError as e:
+                self.logger.error(f"Error parsing YAML configuration: {e}. Using defaults.", exc_info=True)
                 self._set_default_config()
-            else:
-                self.logger.info(f"Configuration loaded successfully from {config_path}")
-        except FileNotFoundError:
-            self.logger.warning(f"Configuration file not found at {config_path}. Using defaults.")
-            self._set_default_config()
-        except yaml.YAMLError as e:
-            self.logger.error(f"Error parsing YAML configuration: {e}. Using defaults.", exc_info=True)
-            self._set_default_config()
         
         # Initialize enhanced LLM router
         self.llm_router = LLMRouter()
@@ -172,27 +189,60 @@ class GitAgent(AgentBase):
         self.logger.info("GitAgent initialized successfully")
     
     def _set_default_config(self):
-        """Set default configuration values."""
+        """Set default configuration values using centralized system."""
         self.logger.info("Setting default configuration for GitAgent")
         self.config = {
             'llm': {
-                'model_name': 'gpt-4o-mini',
-                'temperature': 0.1
+                'model_name': get_config_value("ai.default_model", "gpt-4o-mini"),
+                'temperature': get_config_value("ai.git_agent_temperature", 0.1)
             },
             'routing_keys': {
-                'git_clone': 'ROUTE_TO_GIT_CLONE', # Changed from 'clone'
-                'github_cli': 'ROUTE_TO_GITHUB_CLI', # Changed from 'issue'
-                'shell_exec': 'ROUTE_TO_SHELL_EXEC', # Changed from 'shell'
-                'create_pr': 'ROUTE_TO_CREATE_PR',
-                'end': 'ROUTE_TO_END'
+                'git_clone': get_config_value("routing.tokens.git_clone", "ROUTE_TO_GIT_CLONE"),
+                'github_cli': get_config_value("routing.tokens.github_cli", "ROUTE_TO_GITHUB_CLI"),
+                'shell_exec': get_config_value("routing.tokens.shell_exec", "ROUTE_TO_SHELL_EXEC"),
+                'create_pr': get_config_value("routing.tokens.create_pr", "ROUTE_TO_CREATE_PR"),
+                'end': get_config_value("routing.tokens.end", "ROUTE_TO_END")
             },
             'git_pr_creator': {
-                'copilot_assignee': "CopilotUser", # Placeholder
-                'default_assignees': ["team-infra"], # Placeholder
-                'remote_name': "origin"
+                'copilot_assignee': get_config_value("github.copilot_assignee", "CopilotUser"),
+                'default_assignees': get_config_value("github.default_assignees", ["team-infra"]),
+                'remote_name': get_config_value("tools.git.remote_name", "origin")
+            },
+            'tools': {
+                'git_clone': {
+                    'enabled': True,
+                    'timeout': get_config_value("network.github_timeout", 300)
+                },
+                'gh_open_issue': {
+                    'enabled': True,
+                    'timeout': get_config_value("network.github_timeout", 15)
+                },
+                'shell_exec': {
+                    'enabled': True,
+                    'timeout': get_config_value("network.shell_timeout", 30)
+                }
             }
         }
     
+    def _deep_merge(self, base: dict, overlay: dict) -> dict:
+        """
+        Deep merge two dictionaries, with overlay taking precedence.
+        
+        Args:
+            base: Base dictionary
+            overlay: Dictionary to overlay on base
+            
+        Returns:
+            Merged dictionary
+        """
+        result = base.copy()
+        for key, value in overlay.items():
+            if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+                result[key] = self._deep_merge(result[key], value)
+            else:
+                result[key] = value
+        return result
+    
     def _initialize_tools(self):
         """Initialize the DevOps tools."""
         try:

diagram_to_iac/agents/hello_langgraph/agent.py

@@ -24,6 +24,7 @@ from diagram_to_iac.core.memory import (
     save_agent_state,
     current_git_sha,
 )
+from diagram_to_iac.core.config_loader import get_config, get_config_value
 
 
 # --- Pydantic Schemas for Agent I/O ---
@@ -81,25 +82,55 @@ class HelloAgent(AgentBase):
                 datefmt='%Y-%m-%d %H:%M:%S'
             )
 
+        # Load configuration using centralized system
         if config_path is None:
             base_dir = os.path.dirname(os.path.abspath(__file__))
             config_path = os.path.join(base_dir, 'config.yaml')
             self.logger.debug(f"Default config path set to: {config_path}")
 
-        try:
-            with open(config_path, 'r') as f:
-                self.config = yaml.safe_load(f)
-            if self.config is None:
-                self.logger.warning(f"Configuration file at {config_path} is empty. Using default values.")
+        # Handle configuration loading with proper fallback and logging for tests
+        if config_path:
+            # First try to load agent-specific config directly for test compatibility
+            if not os.path.exists(config_path):
+                self.logger.warning(f"Configuration file not found at {config_path}. Using default values.")
                 self._set_default_config()
             else:
-                self.logger.info(f"Configuration loaded successfully from {config_path}")
-        except FileNotFoundError:
-            self.logger.warning(f"Configuration file not found at {config_path}. Using default values.")
-            self._set_default_config()
-        except yaml.YAMLError as e:
-            self.logger.error(f"Error parsing YAML configuration from {config_path}: {e}. Using default values.", exc_info=True)
-            self._set_default_config()
+                try:
+                    with open(config_path, 'r') as f:
+                        content = f.read().strip()
+                        if not content:
+                            self.logger.warning(f"Configuration file at {config_path} is empty. Using default values.")
+                            self._set_default_config()
+                        else:
+                            f.seek(0)  # Reset file pointer
+                            agent_config = yaml.safe_load(f)
+                            if agent_config is None:
+                                self.logger.warning(f"Configuration file at {config_path} is empty. Using default values.")
+                                self._set_default_config()
+                            else:
+                                # Use centralized config as base and merge with agent-specific overrides
+                                try:
+                                    base_config = get_config()
+                                    self.config = self._deep_merge(base_config, agent_config)
+                                    self.logger.info(f"Configuration loaded successfully from {config_path}")
+                                except Exception as e:
+                                    # Fallback to agent-specific config only
+                                    self.config = agent_config
+                                    self.logger.info(f"Configuration loaded successfully from {config_path}")
+                except yaml.YAMLError as e:
+                    self.logger.error(f"Error parsing YAML configuration from {config_path}: {e}. Using default values.", exc_info=True)
+                    self._set_default_config()
+                except Exception as e:
+                    self.logger.error(f"Unexpected error loading configuration from {config_path}: {e}. Using default values.", exc_info=True)
+                    self._set_default_config()
+        else:
+            # No config path provided, use centralized config only
+            try:
+                self.config = get_config()
+                self.logger.info(f"Configuration loaded successfully from centralized system")
+            except Exception as e:
+                self.logger.warning(f"Failed to load configuration via centralized system: {e}. Using default values.")
+                self._set_default_config()
 
         # Ensure a dummy API key is set so tests can initialize the router without real credentials
         if not os.getenv("OPENAI_API_KEY"):
@@ -137,13 +168,38 @@ class HelloAgent(AgentBase):
 
     def _set_default_config(self):
         self.logger.info("Setting default configuration for HelloAgent.")
+        # Use hardcoded defaults when agent-specific config fails
+        # This ensures test compatibility with expected DEFAULT_TEMP and DEFAULT_MODEL
         self.config = {
             'llm': {
-                'model_name': 'gpt-4o-mini', # Default fallback
+                'model_name': "gpt-4o-mini",
                 'temperature': 0.0
+            },
+            'routing_keys': {
+                'addition': "ROUTE_TO_ADDITION",
+                'multiplication': "ROUTE_TO_MULTIPLICATION"
             }
         }
 
+    def _deep_merge(self, base: dict, overlay: dict) -> dict:
+        """
+        Deep merge two dictionaries, with overlay taking precedence.
+        
+        Args:
+            base: Base dictionary
+            overlay: Dictionary to overlay on base
+            
+        Returns:
+            Merged dictionary
+        """
+        result = base.copy()
+        for key, value in overlay.items():
+            if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+                result[key] = self._deep_merge(result[key], value)
+            else:
+                result[key] = value
+        return result
+
     def _planner_llm_node(self, state: HelloAgentState):
         """LLM decides if tool is needed, and which specific tool to route to."""
         # Config values for the LLM

diagram_to_iac/agents/policy_agent/agent.py

@@ -37,6 +37,7 @@ from pydantic import BaseModel, Field
 # Import core infrastructure
 from diagram_to_iac.core.agent_base import AgentBase
 from diagram_to_iac.core.memory import create_memory, LangGraphMemoryAdapter
+from diagram_to_iac.core.config_loader import get_config, get_config_value
 from .tools.tfsec_tool import TfSecTool, TfSecScanInput, TfSecScanOutput
 
 
@@ -89,12 +90,12 @@ class PolicyAgent(AgentBase):
         """Initialize PolicyAgent with configuration and tools."""
         self.config_path = config_path
         
+        # Initialize logging first
+        self.logger = logging.getLogger(__name__)
+        
         # Load configuration
         self.config = self._load_config()
         
-        # Initialize logging
-        self.logger = logging.getLogger(__name__)
-        
         # Initialize memory
         self.memory = create_memory(memory_type=memory_type)
         self.memory_adapter = LangGraphMemoryAdapter(self.memory)
@@ -106,24 +107,46 @@ class PolicyAgent(AgentBase):
         self.workflow = self._build_workflow()
         
     def _load_config(self) -> Dict[str, Any]:
-        """Load agent configuration from YAML file."""
-        if self.config_path and os.path.exists(self.config_path):
-            with open(self.config_path, 'r') as f:
-                return yaml.safe_load(f)
-        
-        # Default configuration
-        config_file = os.path.join(os.path.dirname(__file__), "config.yaml")
-        if os.path.exists(config_file):
-            with open(config_file, 'r') as f:
-                return yaml.safe_load(f)
-                
-        # Fallback minimal configuration
+        """Load agent configuration using centralized system with fallback to direct file loading."""
+        try:
+            config = get_config()
+            policy_config = config.get('agents', {}).get('policy_agent', {})
+            
+            if policy_config:
+                self.logger.info("Configuration loaded from centralized system")
+                return policy_config
+            else:
+                self.logger.warning("No policy agent configuration found in centralized system. Using defaults.")
+                return self._get_fallback_config()
+        except Exception as e:
+            self.logger.warning(f"Failed to load from centralized config: {e}. Falling back to direct file loading.")
+            # Fallback to direct file loading for backward compatibility
+            if self.config_path and os.path.exists(self.config_path):
+                with open(self.config_path, 'r') as f:
+                    return yaml.safe_load(f)
+            
+            # Default configuration
+            config_file = os.path.join(os.path.dirname(__file__), "config.yaml")
+            if os.path.exists(config_file):
+                with open(config_file, 'r') as f:
+                    return yaml.safe_load(f)
+                    
+            # Fallback minimal configuration
+            return self._get_fallback_config()
+    
+    def _get_fallback_config(self) -> Dict[str, Any]:
+        """Get fallback configuration using centralized values."""
         return {
+            "llm": {
+                "model_name": get_config_value("ai.default_model", "gpt-4o-mini"),
+                "temperature": get_config_value("ai.default_temperature", 0.1)
+            },
             "policy": {
                 "tfsec": {
-                    "enabled": True,
-                    "block_on_severity": ["CRITICAL", "HIGH"],
-                    "artifact_on_severity": ["CRITICAL", "HIGH", "MEDIUM"]
+                    "enabled": get_config_value("tools.policy.tfsec_enabled", True),
+                    "timeout_seconds": get_config_value("network.shell_timeout", 120),
+                    "block_on_severity": get_config_value("tools.policy.block_on_severity", ["CRITICAL", "HIGH"]),
+                    "artifact_on_severity": get_config_value("tools.policy.artifact_on_severity", ["CRITICAL", "HIGH", "MEDIUM"])
                 }
             }
         }

diagram_to_iac/agents/supervisor_langgraph/agent.py

@@ -40,6 +40,7 @@ from diagram_to_iac.core.agent_base import AgentBase
 from diagram_to_iac.core.memory import create_memory, LangGraphMemoryAdapter
 from diagram_to_iac.core import IssueTracker, MissingSecretError
 from diagram_to_iac.services.observability import log_event
+from diagram_to_iac.core.config_loader import get_config, get_config_value
 from .guards import check_required_secrets
 from diagram_to_iac.tools.llm_utils.router import get_llm, LLMRouter
 from diagram_to_iac.agents.git_langgraph import GitAgent, GitAgentInput, GitAgentOutput
@@ -147,34 +148,50 @@ class SupervisorAgent(AgentBase):
                 datefmt="%Y-%m-%d %H:%M:%S",
             )
 
-        # Load configuration
+        # Load configuration using centralized system
         if config_path is None:
             base_dir = os.path.dirname(os.path.abspath(__file__))
             config_path = os.path.join(base_dir, "config.yaml")
             self.logger.debug(f"Default config path set to: {config_path}")
 
         try:
-            with open(config_path, "r") as f:
-                self.config = yaml.safe_load(f)
-            if self.config is None:
+            # Use centralized configuration loading with hierarchical merging
+            base_config = get_config()
+            
+            # Load agent-specific config if provided
+            agent_config = {}
+            if config_path and os.path.exists(config_path):
+                with open(config_path, 'r') as f:
+                    agent_config = yaml.safe_load(f) or {}
+            
+            # Deep merge base config with agent-specific overrides
+            self.config = self._deep_merge(base_config, agent_config)
+            self.logger.info(f"Configuration loaded successfully from centralized system")
+        except Exception as e:
+            self.logger.warning(f"Failed to load configuration via centralized system: {e}. Using fallback.")
+            # Fallback to direct YAML loading for backward compatibility
+            try:
+                with open(config_path, "r") as f:
+                    self.config = yaml.safe_load(f)
+                if self.config is None:
+                    self.logger.warning(
+                        f"Configuration file at {config_path} is empty. Using defaults."
+                    )
+                    self._set_default_config()
+                else:
+                    self.logger.info(
+                        f"Configuration loaded successfully from {config_path}"
+                    )
+            except FileNotFoundError:
                 self.logger.warning(
-                    f"Configuration file at {config_path} is empty. Using defaults."
+                    f"Configuration file not found at {config_path}. Using defaults."
                 )
                 self._set_default_config()
-            else:
-                self.logger.info(
-                    f"Configuration loaded successfully from {config_path}"
+            except yaml.YAMLError as e:
+                self.logger.error(
+                    f"Error parsing YAML configuration: {e}. Using defaults.", exc_info=True
                 )
-        except FileNotFoundError:
-            self.logger.warning(
-                f"Configuration file not found at {config_path}. Using defaults."
-            )
-            self._set_default_config()
-        except yaml.YAMLError as e:
-            self.logger.error(
-                f"Error parsing YAML configuration: {e}. Using defaults.", exc_info=True
-            )
-            self._set_default_config()
+                self._set_default_config()
 
         # Initialize enhanced LLM router
         self.llm_router = LLMRouter()
@@ -236,15 +253,18 @@ class SupervisorAgent(AgentBase):
 
 
     def _set_default_config(self):
-        """Set default configuration values."""
+        """Set default configuration values using centralized system."""
         self.config = {
-            "llm": {"model_name": "gpt-4o-mini", "temperature": 0.1},
+            "llm": {
+                "model_name": get_config_value("ai.default_model", "gpt-4o-mini"),
+                "temperature": get_config_value("ai.default_temperature", 0.1)
+            },
             "routing_keys": {
-                "clone": "ROUTE_TO_CLONE",
-                "stack_detect": "ROUTE_TO_STACK_DETECT",
-                "terraform": "ROUTE_TO_TERRAFORM",
-                "issue": "ROUTE_TO_ISSUE",
-                "end": "ROUTE_TO_END",
+                "clone": get_config_value("routing.tokens.git_clone", "ROUTE_TO_CLONE"),
+                "stack_detect": get_config_value("routing.tokens.analyze", "ROUTE_TO_STACK_DETECT"),
+                "terraform": get_config_value("routing.tokens.terraform_init", "ROUTE_TO_TERRAFORM"),
+                "issue": get_config_value("routing.tokens.open_issue", "ROUTE_TO_ISSUE"),
+                "end": get_config_value("routing.tokens.end", "ROUTE_TO_END"),
             },
             "prompts": {
                 "planner_prompt": """User input: "{user_input}"
@@ -258,9 +278,34 @@ Analyze this R2D (Repo-to-Deployment) request and determine the appropriate acti
 
 Important: Only use routing tokens if the input contains actionable R2D workflow requests."""
             },
+            "workflow": {
+                "timeout_seconds": get_config_value("network.terraform_timeout", 600),
+                "working_directory": get_config_value("system.workspace_base", "/workspace"),
+                "auto_branch_naming": True,
+                "enhanced_terraform": True
+            }
         }
         self.logger.info("Default configuration set")
 
+    def _deep_merge(self, base: dict, overlay: dict) -> dict:
+        """
+        Deep merge two dictionaries, with overlay taking precedence.
+        
+        Args:
+            base: Base dictionary
+            overlay: Dictionary to overlay on base
+            
+        Returns:
+            Merged dictionary
+        """
+        result = base.copy()
+        for key, value in overlay.items():
+            if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+                result[key] = self._deep_merge(result[key], value)
+            else:
+                result[key] = value
+        return result
+
     # --- AgentBase interface -------------------------------------------------
     def plan(self, query: str, **kwargs):
         """Generate a plan for the R2D workflow (required by AgentBase)."""