nl2sql-engine 0.1.0__py3-none-any.whl

nl2sql/cli/types.py

@@ -0,0 +1,13 @@
+from pydantic import BaseModel
+from typing import Optional
+
+class RunConfig(BaseModel):
+    """Configuration for running the pipeline."""
+    query: str
+    ds_id: Optional[str] = None
+    role: str = "admin"
+    no_exec: bool = False
+    verbose: bool = False
+    show_perf: bool = False
+
+

nl2sql/common/__init__.py

@@ -0,0 +1 @@
+"""Common utilities and shared models for the NL2SQL package."""

nl2sql/common/cancellation.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import threading
+from typing import Optional
+
+
+class CancellationToken:
+    """Per-run cancellation flag.
+
+    Each run owns its own token, so cancelling one run never affects another.
+    Runs pass their token to the graph via
+    ``config={"configurable": {"cancellation_token": token}}``.
+    """
+
+    def __init__(self) -> None:
+        self._event = threading.Event()
+
+    def cancel(self) -> None:
+        self._event.set()
+
+    def is_cancelled(self) -> bool:
+        return self._event.is_set()
+
+    def wait(self, timeout: Optional[float] = None) -> bool:
+        return self._event.wait(timeout=timeout)

nl2sql/common/context.py

@@ -0,0 +1,5 @@
+"""Global context management for the NL2SQL pipeline."""
+from contextvars import ContextVar
+from typing import Optional
+
+current_datasource_id: ContextVar[Optional[str]] = ContextVar("current_datasource_id", default=None)

nl2sql/common/errors.py

@@ -0,0 +1,109 @@
+from enum import Enum, auto
+from typing import Optional, Any
+from pydantic import BaseModel, ConfigDict
+
+
+class ErrorSeverity(str, Enum):
+    """Severity levels for pipeline errors."""
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    CRITICAL = "CRITICAL"
+
+
+class ErrorCode(str, Enum):
+    """Standardized error codes for the pipeline."""
+    MISSING_LLM = "MISSING_LLM"
+    MISSING_SQL = "MISSING_SQL"
+    MISSING_DATASOURCE_ID = "MISSING_DATASOURCE_ID"
+    MISSING_PLAN = "MISSING_PLAN"
+    INVALID_STATE = "INVALID_STATE"
+    INVALID_PLAN_STRUCTURE = "INVALID_PLAN_STRUCTURE"
+    SCHEMA_RETRIEVAL_FAILED = "SCHEMA_RETRIEVAL_FAILED"
+    SQL_GEN_FAILED = "SQL_GEN_FAILED"
+    TABLE_NOT_FOUND = "TABLE_NOT_FOUND"
+    COLUMN_NOT_FOUND = "COLUMN_NOT_FOUND"
+    INVALID_ALIAS_USAGE = "INVALID_ALIAS_USAGE"
+    MISSING_GROUP_BY = "MISSING_GROUP_BY"
+    INVALID_DATE_FORMAT = "INVALID_DATE_FORMAT"
+    INVALID_NUMERIC_VALUE = "INVALID_NUMERIC_VALUE"
+    JOIN_TABLE_NOT_IN_PLAN = "JOIN_TABLE_NOT_IN_PLAN"
+    JOIN_MISSING_ON_CLAUSE = "JOIN_MISSING_ON_CLAUSE"
+    SECURITY_VIOLATION = "SECURITY_VIOLATION"
+    SAFEGUARD_VIOLATION = "SAFEGUARD_VIOLATION"
+    DB_EXECUTION_ERROR = "DB_EXECUTION_ERROR"
+    EXECUTOR_CRASH = "EXECUTOR_CRASH"
+    PLANNING_FAILURE = "PLANNING_FAILURE"
+    VALIDATOR_CRASH = "VALIDATOR_CRASH"
+    REFINER_FAILED = "REFINER_FAILED"
+    PHYSICAL_VALIDATOR_FAILED = "PHYSICAL_VALIDATOR_FAILED"
+    PLAN_FEEDBACK = "PLAN_FEEDBACK"
+    UNKNOWN_ERROR = "UNKNOWN_ERROR"
+    AGGREGATOR_FAILED = "AGGREGATOR_FAILED"
+    PERFORMANCE_WARNING = "PERFORMANCE_WARNING"
+    EXECUTION_ERROR = "EXECUTION_ERROR"
+    ORCHESTRATOR_CRASH = "ORCHESTRATOR_CRASH"
+    INTENT_VIOLATION = "INTENT_VIOLATION"
+    PIPELINE_TIMEOUT = "PIPELINE_TIMEOUT"
+    SERVICE_UNAVAILABLE = "SERVICE_UNAVAILABLE"
+    EXECUTION_TIMEOUT = "EXECUTION_TIMEOUT"
+    CANCELLED = "CANCELLED",
+    EXECUTION_FAILED = "EXECUTION_FAILED"
+
+
+
+FATAL_ERRORS = {
+    ErrorCode.SECURITY_VIOLATION,
+    ErrorCode.INTENT_VIOLATION,
+    ErrorCode.SAFEGUARD_VIOLATION,
+    ErrorCode.MISSING_DATASOURCE_ID,
+    ErrorCode.MISSING_LLM,
+    ErrorCode.INVALID_STATE
+}
+
+SAFE_ERROR_MESSAGES = {
+    ErrorCode.DB_EXECUTION_ERROR: "An internal database error occurred while executing the query.",
+    ErrorCode.SAFEGUARD_VIOLATION: "The query result was blocked by data protection safeguards.",
+    ErrorCode.EXECUTOR_CRASH: "The query execution service encountered an unexpected error.",
+    ErrorCode.VALIDATOR_CRASH: "The validation service encountered an unexpected error.",
+    ErrorCode.MISSING_DATASOURCE_ID: "Datasource configuration error."
+}
+
+class PipelineError(BaseModel):
+    """Represents a structured error within the pipeline.
+
+    Attributes:
+        node (str): The node where the error occurred.
+        message (str): A human-readable error message.
+        severity (ErrorSeverity): The severity of the error.
+        error_code (ErrorCode): The standardized error code.
+        stack_trace (Optional[str]): Stack trace if applicable.
+        details (Optional[Any]): Additional context or metadata.
+    """
+    model_config = ConfigDict(extra="ignore")
+    
+    node: str
+    message: str
+    severity: ErrorSeverity
+    error_code: ErrorCode
+    stack_trace: Optional[str] = None
+    details: Optional[Any] = None
+
+    @property
+    def is_retryable(self) -> bool:
+        """Determines if this error should trigger a retry/refinement loop."""
+        if self.severity == ErrorSeverity.CRITICAL:
+            return False
+        return self.error_code not in FATAL_ERRORS
+
+    def get_safe_message(self) -> str:
+        """Returns a sanitized error message safe for exposure to LLMs or users.
+
+        If a safe mapping exists for the error code, it is returned.
+        Otherwise, the original message is used (assuming it's safe).
+
+        Returns:
+            str: The sanitized error message.
+        """
+        return SAFE_ERROR_MESSAGES.get(self.error_code, self.message)
+

nl2sql/common/event_logger.py

@@ -0,0 +1,88 @@
+import logging
+import json
+import os
+from logging.handlers import RotatingFileHandler
+from typing import Any, Dict, Optional
+from datetime import datetime
+
+class EventLogger:
+    """Persistent audit logger for high-value AI events.
+    
+    Writes structured JSON events to a dedicated log file, separate from
+    application debug logs. Used for forensic analysis and "Time Travel" debugging.
+    """
+    
+    def __init__(self):
+        self.logger = logging.getLogger("nl2sql.audit")
+        self.logger.setLevel(logging.INFO)
+        self.logger.propagate = False  # Do not bubble up to root logger (avoid stdout spam)
+        
+        # Ensure handlers are set up (singleton-ish check)
+        if not self.logger.handlers:
+            log_path = "logs/audit_events.log"
+            
+            # Ensure directory exists
+            os.makedirs(os.path.dirname(log_path), exist_ok=True)
+            
+            # 10MB per file, max 5 backup files
+            handler = RotatingFileHandler(
+                log_path, maxBytes=10*1024*1024, backupCount=5, encoding="utf-8"
+            )
+            
+            # Use specific JSON formatter for audit events
+            formatter = logging.Formatter("%(message)s")
+            handler.setFormatter(formatter)
+            
+            self.logger.addHandler(handler)
+
+    def log_event(
+        self, 
+        event_type: str, 
+        payload: Dict[str, Any], 
+        trace_id: Optional[str] = None, 
+        tenant_id: Optional[str] = None
+    ):
+        """Logs a structured event to the audit log.
+        
+        Args:
+            event_type: Category of event (e.g., 'llm_interaction', 'security_violation')
+            payload: The event data dictionary.
+            trace_id: Correlation ID.
+            tenant_id: Tenant/Customer ID.
+        """
+        
+        sensitive_keys = {"api_key", "password", "secret", "authorization"}
+        cleaned_payload = self._redact(payload, sensitive_keys)
+        
+        event = {
+            "timestamp": datetime.utcnow().isoformat(),
+            "event_type": event_type,
+            "trace_id": trace_id,
+            "tenant_id": tenant_id,
+            "data": cleaned_payload
+        }
+        
+        self.logger.info(json.dumps(event))
+
+    def _redact(self, data: Any, keys_to_redact: set) -> Any:
+        """Recursively redact sensitive keys from dictionary.
+
+        Args:
+            data: Input data (dict, list, or primitive).
+            keys_to_redact: Set of lowercase keys to match and redact.
+
+        Returns:
+            The sanitized data structure with sensitive values replaced by '***REDACTED***'.
+        """
+        if isinstance(data, dict):
+            return {
+                k: ("***REDACTED***" if k.lower() in keys_to_redact else self._redact(v, keys_to_redact))
+                for k, v in data.items()
+            }
+        elif isinstance(data, list):
+            return [self._redact(item, keys_to_redact) for item in data]
+        else:
+            return data
+
+# Global instance
+event_logger = EventLogger()

nl2sql/common/exceptions.py

@@ -0,0 +1,3 @@
+class NL2SQLError(Exception):
+    """Base exception for all NL2SQL errors."""
+    pass

nl2sql/common/logger.py

@@ -0,0 +1,119 @@
+import logging
+import json
+import time
+import contextvars
+from contextlib import contextmanager
+from typing import Any, Dict, Optional
+
+_trace_id_ctx = contextvars.ContextVar("trace_id", default=None)
+_tenant_id_ctx = contextvars.ContextVar("tenant_id", default=None)
+
+class TraceContextFilter(logging.Filter):
+    """Injects trace_id and tenant_id from contextvars into the log record."""
+    def filter(self, record):
+        record.trace_id = _trace_id_ctx.get()
+        record.tenant_id = _tenant_id_ctx.get()
+        return True
+
+@contextmanager
+def trace_context(trace_id: str):
+    """Context manager to set the trace_id for the current context."""
+    token = _trace_id_ctx.set(trace_id)
+    try:
+        yield
+    finally:
+        _trace_id_ctx.reset(token)
+
+@contextmanager
+def tenant_context(tenant_id: Optional[str]):
+    """Context manager to set the tenant_id for the current context."""
+    token = _tenant_id_ctx.set(tenant_id)
+    try:
+        yield
+    finally:
+        _tenant_id_ctx.reset(token)
+
+class JsonFormatter(logging.Formatter):
+    """Formatter that outputs JSON strings after parsing the LogRecord."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Formats the log record as a JSON string.
+
+        Args:
+           record (logging.LogRecord): The log record to format.
+
+        Returns:
+            str: The JSON-formatted log string.
+        """
+        log_record = {
+            "timestamp": self.formatTime(record, self.datefmt),
+            "level": record.levelname,
+            "name": record.name,
+            "message": record.getMessage(),
+        }
+        
+        if getattr(record, "trace_id", None):
+            log_record["trace_id"] = record.trace_id
+            
+        if getattr(record, "tenant_id", None):
+            log_record["tenant_id"] = record.tenant_id
+
+        # Standard LogRecord attributes to ignore
+        standard_attrs = {
+            "args", "asctime", "created", "exc_info", "exc_text", "filename",
+            "funcName", "levelname", "levelno", "lineno", "module",
+            "msecs", "message", "msg", "name", "pathname", "process",
+            "processName", "relativeCreated", "stack_info", "thread", "threadName",
+            "taskName", "trace_id", "tenant_id"
+        }
+
+        for key, value in record.__dict__.items():
+            if key not in standard_attrs and not key.startswith("_"):
+                log_record[key] = value
+            
+        return json.dumps(log_record)
+
+
+def configure_logging(level: str = "INFO", json_format: bool = False):
+    """Configures the root logger.
+
+    Args:
+        level (str): The logging level (default: INFO).
+        json_format (bool): Whether to use JSON formatting (default: False).
+    """
+    root_logger = logging.getLogger()
+    root_logger.setLevel(level)
+    
+    # Remove existing handlers
+    for handler in root_logger.handlers[:]:
+        root_logger.removeHandler(handler)
+        
+    handler = logging.StreamHandler()
+    handler.addFilter(TraceContextFilter())
+    
+    if json_format:
+        handler.setFormatter(JsonFormatter())
+    else:
+        # Standard text format
+        formatter = logging.Formatter(
+            "%(asctime)s - [%(trace_id)s] - %(name)s - %(levelname)s - %(message)s"
+        )
+        handler.setFormatter(formatter)
+        
+    root_logger.addHandler(handler)
+    
+    # Silence noisy libraries
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    logging.getLogger("httpcore").setLevel(logging.WARNING)
+
+
+def get_logger(name: str) -> logging.Logger:
+    """Gets a named logger.
+
+    Args:
+        name (str): The name of the logger.
+
+    Returns:
+        logging.Logger: The logger instance.
+    """
+    return logging.getLogger(name)

nl2sql/common/metrics.py

@@ -0,0 +1,50 @@
+"""Performance metrics tracking with OpenTelemetry support."""
+from typing import List, Dict, Any, Optional
+from opentelemetry import metrics
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import ConsoleMetricExporter, PeriodicExportingMetricReader
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+
+# Legacy lists for CLI compatibility
+TOKEN_LOG: List[Dict[str, Any]] = []
+LATENCY_LOG: List[Dict[str, Any]] = []
+
+_meter = metrics.get_meter("nl2sql.core")
+node_duration_histogram = _meter.create_histogram(
+    name="nl2sql.node.duration",
+    description="Duration of node execution in seconds",
+    unit="s",
+)
+token_usage_counter = _meter.create_counter(
+    name="nl2sql.token.usage",
+    description="Number of tokens used by LLM interactions",
+    unit="1",
+)
+
+
+def configure_metrics(exporter_type: str = "none", otlp_endpoint: Optional[str] = None):
+    """Configures the OpenTelemetry Metric Provider.
+    
+    Args:
+        exporter_type: 'none', 'console', or 'otlp'
+        otlp_endpoint: Optional endpoint for OTLP exporter
+    """
+    if exporter_type == "none":
+        return
+
+    reader = None
+    if exporter_type == "console":
+        reader = PeriodicExportingMetricReader(ConsoleMetricExporter())
+    elif exporter_type == "otlp":
+        endpoint = otlp_endpoint or "http://localhost:4317"
+        reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint))
+    
+    if reader:
+        provider = MeterProvider(metric_readers=[reader])
+        metrics.set_meter_provider(provider)
+
+
+def reset_usage():
+    """Resets the token and latency logs."""
+    TOKEN_LOG.clear()
+    LATENCY_LOG.clear()

nl2sql/common/resilience.py

@@ -0,0 +1,59 @@
+"""
+Resilience Module: Circuit Breakers and Fault Tolerance.
+
+This module centralizes the configuration of Circuit Breakers using `pybreaker`.
+It allows the system to Fail Fast when the vector store is unavailable.
+
+Features:
+- Global Breaker Instance (VECTOR)
+- Observability via CircuitBreakerListener
+- Safe Exclusion of "Soft Failures" (e.g., Rate Limits)
+"""
+import pybreaker
+from typing import Any, Optional, List, Type
+from nl2sql.common.logger import get_logger
+
+logger = get_logger("resilience")
+
+class ObservabilityListener(pybreaker.CircuitBreakerListener):
+    """Listener to export circuit breaker state changes and failures to logs/metrics."""
+    
+    def state_change(self, cb, old_state, new_state):
+        logger.warning(
+            f"Circuit Breaker '{cb.name}' changed state: {old_state.name} -> {new_state.name}"
+        )
+        # TODO: Emit metric: breaker_state_change{name=cb.name, state=new_state.name}
+
+    def failure(self, cb, exc):
+        logger.error(
+            f"Circuit Breaker '{cb.name}' recorded failure: {type(exc).__name__}: {exc}"
+        )
+        # TODO: Emit metric: breaker_failure{name=cb.name, error=type(exc).__name__}
+
+    def success(self, cb):
+        pass
+        # TODO: Emit metric: breaker_success{name=cb.name}
+
+
+def create_breaker(
+    name: str, 
+    fail_max: int = 5, 
+    reset_timeout: int = 60,
+    exclude: Optional[List[Type[Exception]]] = None
+) -> pybreaker.CircuitBreaker:
+    """Factory to create a configured Circuit Breaker."""
+    return pybreaker.CircuitBreaker(
+        fail_max=fail_max,
+        reset_timeout=reset_timeout,
+        name=name,
+        listeners=[ObservabilityListener()],
+        exclude=exclude or []
+    )
+
+
+# Vector Breaker: Retrieval Layer
+VECTOR_BREAKER = create_breaker(
+    name="VECTOR_BREAKER",
+    fail_max=5,
+    reset_timeout=30  # Faster recovery for infra blips
+)

nl2sql/common/settings.py

@@ -0,0 +1,195 @@
+from typing import Optional
+import os
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from nl2sql.common.logger import get_logger
+
+logger = get_logger(__name__)
+
+class Settings(BaseSettings):
+    """Application configuration settings backed by environment variables."""
+    
+    openai_api_key: Optional[str] = Field(default=None, validation_alias="OPENAI_API_KEY")
+    vector_store_path: Optional[str] = Field(
+        default="./chroma_db",
+        validation_alias="VECTOR_STORE",
+        description="Persist directory for the vector store."
+    )
+    vector_store_collection_name: str = Field(
+        default="nl2sql_store",
+        validation_alias="VECTOR_STORE_COLLECTION",
+        description="Chroma collection name for schema embeddings."
+    )
+    llm_config_path: str = Field(default="configs/llm.yaml", validation_alias="LLM_CONFIG")
+    datasource_config_path: str = Field(default="configs/datasources.yaml", validation_alias="DATASOURCE_CONFIG")
+    secrets_config_path: str = Field(default="configs/secrets.yaml", validation_alias="SECRETS_CONFIG")
+    embedding_model: str = Field(default="text-embedding-3-small", validation_alias="EMBEDDING_MODEL")
+    embedding_provider: str = Field(
+        default="openai",
+        validation_alias="EMBEDDING_PROVIDER",
+        description="Embedding backend: 'openai' (API key required) or 'local' (key-free ONNX model)."
+    )
+    tenant_id: str = Field(default="default_tenant", validation_alias="TENANT_ID")
+    sample_questions_path: str = Field(
+        default="configs/sample_questions.yaml", 
+        validation_alias="SAMPLE_QUESTIONS",
+        description="Path to the YAML file containing sample questions for routing."
+    )
+    policies_config_path: str = Field(
+        default="configs/policies.json",
+        validation_alias="POLICIES_CONFIG",
+        description="Path to the JSON file containing RBAC policies and permissions."
+    )
+    
+    global_timeout_sec: int = Field(
+        default=60,
+        validation_alias="GLOBAL_TIMEOUT_SEC",
+        description="Global timeout in seconds for pipeline execution."
+    )
+
+    sandbox_exec_workers: int = Field(
+        default=4,
+        validation_alias="SANDBOX_EXEC_WORKERS",
+        description="Max workers for latency-sensitive execution pool."
+    )
+
+    result_artifact_backend: str = Field(
+        default="local",
+        validation_alias="RESULT_ARTIFACT_BACKEND",
+        description="Artifact backend to store executor results: local, s3, adls."
+    )
+    result_artifact_base_uri: str = Field(
+        default="./artifacts",
+        validation_alias="RESULT_ARTIFACT_BASE_URI",
+        description="Base URI or path for artifact storage."
+    )
+    result_artifact_path_template: str = Field(
+        default="<tenant_id>/<request_id>.parquet",
+        validation_alias="RESULT_ARTIFACT_PATH_TEMPLATE",
+        description=(
+            "Template for artifact paths, relative to the backend root. "
+            "Placeholders are <key> names resolved from executor metadata; "
+            "the executor supplies tenant_id, request_id and schema_version."
+        )
+    )
+    result_artifact_s3_bucket: Optional[str] = Field(
+        default=None,
+        validation_alias="RESULT_ARTIFACT_S3_BUCKET",
+        description="S3 bucket for artifact storage."
+    )
+    result_artifact_s3_prefix: Optional[str] = Field(
+        default=None,
+        validation_alias="RESULT_ARTIFACT_S3_PREFIX",
+        description="S3 prefix for artifact storage."
+    )
+    result_artifact_adls_account: Optional[str] = Field(
+        default=None,
+        validation_alias="RESULT_ARTIFACT_ADLS_ACCOUNT",
+        description="ADLS storage account name."
+    )
+    result_artifact_adls_container: Optional[str] = Field(
+        default=None,
+        validation_alias="RESULT_ARTIFACT_ADLS_CONTAINER",
+        description="ADLS container name."
+    )
+    result_artifact_adls_connection_string: Optional[str] = Field(
+        default=None,
+        validation_alias="RESULT_ARTIFACT_ADLS_CONNECTION_STRING",
+        description="ADLS connection string, if using key-based auth."
+    )
+
+    schema_store_backend: str = Field(
+        default="sqlite",
+        validation_alias="SCHEMA_STORE_BACKEND",
+        description="Schema store backend identifier (e.g., 'sqlite', 'memory')."
+    )
+    schema_store_path: str = Field(
+        default="data/schema_store.db",
+        validation_alias="SCHEMA_STORE_PATH",
+        description="SQLite database path for schema store persistence."
+    )
+    schema_store_max_versions: int = Field(
+        default=3,
+        validation_alias="SCHEMA_STORE_MAX_VERSIONS",
+        description="Max versions to retain per datasource in schema store."
+    )
+    schema_version_mismatch_policy: str = Field(
+        default="warn",
+        validation_alias="SCHEMA_VERSION_MISMATCH_POLICY",
+        description="Action when chunk schema_version differs from SchemaStore: warn, fail, or ignore."
+    )
+
+    logical_validator_strict_columns: bool = Field(
+        default=False,
+        validation_alias="LOGICAL_VALIDATOR_STRICT_COLUMNS",
+        description="Treat missing columns as errors in logical validation."
+    )
+
+    sql_agent_max_retries: int = Field(
+        default=3,
+        validation_alias="SQL_AGENT_MAX_RETRIES",
+        description="Max retry attempts for SQL agent refinement loop."
+    )
+    sql_agent_retry_base_delay_sec: float = Field(
+        default=1.0,
+        validation_alias="SQL_AGENT_RETRY_BASE_DELAY_SEC",
+        description="Base delay for SQL agent retries (seconds)."
+    )
+    sql_agent_retry_max_delay_sec: float = Field(
+        default=10.0,
+        validation_alias="SQL_AGENT_RETRY_MAX_DELAY_SEC",
+        description="Max delay for SQL agent retries (seconds)."
+    )
+    sql_agent_retry_jitter_sec: float = Field(
+        default=0.5,
+        validation_alias="SQL_AGENT_RETRY_JITTER_SEC",
+        description="Max jitter added to SQL agent retry delays (seconds)."
+    )
+
+    observability_exporter: str = Field(
+        default="none",
+        validation_alias="OBSERVABILITY_EXPORTER",
+        description="Exporter for metrics/traces: 'none', 'console', 'otlp'."
+    )
+
+    otlp_endpoint: Optional[str] = Field(
+        default=None,
+        validation_alias="OTEL_EXPORTER_OTLP_ENDPOINT",
+        description="Endpoint for OTLP exporter (e.g. http://localhost:4317)."
+    )
+
+    model_config = SettingsConfigDict(
+        env_file=".env", 
+        env_file_encoding="utf-8", 
+        extra="ignore"
+    )
+
+
+def load_settings() -> Settings:
+    """Load settings using ENV_FILE_PATH or ENV/APP_ENV when provided."""
+    env_file_path = os.getenv("ENV_FILE_PATH")
+    env_name = os.getenv("ENV") or os.getenv("APP_ENV")
+
+    if env_file_path:
+        logger.info(f"Loading settings from ENV_FILE_PATH={env_file_path}")
+        return Settings(_env_file=env_file_path)
+    if env_name:
+        env_file = f".env.{env_name}"
+        logger.info(f"Loading settings from {env_file}")
+        return Settings(_env_file=env_file)
+
+    logger.info("Loading settings from default .env")
+    return Settings()
+
+
+settings = load_settings()
+
+def reload_settings() -> Settings:
+    """Re-read settings from the environment and refresh the module singleton.
+
+    The ``settings`` object is created at import time, so switching environments
+    at runtime requires updating it in place; existing
+    ``from nl2sql.common.settings import settings`` references stay valid.
+    """
+    settings.__dict__.update(load_settings().__dict__)
+    return settings

nl2sql/configs/__init__.py

@@ -0,0 +1,6 @@
+
+from .datasources import DatasourceConfig, ConnectionConfig, DatasourceFileConfig
+from .llm import LLMFileConfig, AgentConfig
+from .policies import PolicyFileConfig, RolePolicy
+from .secrets import SecretProviderConfig
+from .manager import ConfigManager