d365fo-client 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl

d365fo_client/auth.py

@@ -5,6 +5,7 @@ from typing import Optional
 
 from azure.identity import ClientSecretCredential, DefaultAzureCredential
 
+from .credential_sources import CredentialManager, CredentialSource
 from .models import FOClientConfig
 
 
@@ -20,25 +21,46 @@ class AuthenticationManager:
         self.config = config
         self._token = None
         self._token_expires = None
-        self.credential = self._setup_credentials()
-
-    def _setup_credentials(self):
-        """Setup authentication credentials"""
-        if self.config.use_default_credentials:
-            return DefaultAzureCredential()
-        elif (
+        self._credential_manager = CredentialManager()
+        self.credential = None  # Will be set by _setup_credentials
+
+    async def _setup_credentials(self):
+        """Setup authentication credentials with support for credential sources"""
+        
+        # Check if credential source is specified in config
+        credential_source = getattr(self.config, 'credential_source', None)
+        
+        if credential_source is not None:
+            # Use credential source to get credentials
+            try:
+                client_id, client_secret, tenant_id = await self._credential_manager.get_credentials(credential_source)
+                self.credential = ClientSecretCredential(
+                    tenant_id=tenant_id,
+                    client_id=client_id,
+                    client_secret=client_secret,
+                )
+                return
+            except Exception as e:
+                raise ValueError(f"Failed to setup credentials from source: {e}")
+        
+        
+        # Fallback to existing logic for backward compatibility
+  
+        if (
             self.config.client_id
             and self.config.client_secret
             and self.config.tenant_id
         ):
-            return ClientSecretCredential(
+            self.credential = ClientSecretCredential(
                 tenant_id=self.config.tenant_id,
                 client_id=self.config.client_id,
                 client_secret=self.config.client_secret,
             )
+        elif self.config.use_default_credentials:
+            self.credential = DefaultAzureCredential()
         else:
             raise ValueError(
-                "Must provide either use_default_credentials=True or client credentials"
+                "Must provide either use_default_credentials=True, client credentials, or credential_source"
             )
 
     async def get_token(self) -> str:
@@ -51,6 +73,10 @@ class AuthenticationManager:
         if self._is_localhost():
             return "mock-token-for-localhost"
 
+        # Initialize credentials if not already set
+        if self.credential is None:
+            await self._setup_credentials()
+
         if (
             self._token
             and self._token_expires
@@ -91,3 +117,16 @@ class AuthenticationManager:
         """Invalidate cached token to force refresh"""
         self._token = None
         self._token_expires = None
+
+    async def invalidate_credentials(self):
+        """Invalidate cached credentials and token to force full refresh"""
+        self.invalidate_token()
+        self.credential = None
+        if hasattr(self, '_credential_manager'):
+            self._credential_manager.clear_cache()
+
+    def get_credential_cache_stats(self) -> dict:
+        """Get credential cache statistics for debugging"""
+        if hasattr(self, '_credential_manager'):
+            return self._credential_manager.get_cache_stats()
+        return {"total_cached": 0, "expired": 0, "active": 0}

d365fo_client/client.py

@@ -11,6 +11,7 @@ from .exceptions import FOClientError
 from .labels import LabelOperations, resolve_labels_generic
 from .metadata_api import MetadataAPIOperations
 from .metadata_v2 import MetadataCacheV2, SmartSyncManagerV2
+from .metadata_v2.sync_session_manager import SyncSessionManager
 from .models import (
     ActionInfo,
     DataEntityInfo,
@@ -58,6 +59,7 @@ class FOClient:
         # Initialize new metadata cache and sync components
         self.metadata_cache = None
         self.sync_manager = None
+        self._sync_session_manager = None
         self._metadata_initialized = False
         self._background_sync_task = None
 
@@ -83,6 +85,10 @@ class FOClient:
 
         await self.session_manager.close()
 
+    
+    async def initialize_metadata(self):
+        await self._ensure_metadata_initialized()
+
     async def _ensure_metadata_initialized(self):
         """Ensure metadata cache and sync manager are initialized"""
         if not self._metadata_initialized and self.config.enable_metadata_cache:
@@ -104,6 +110,9 @@ class FOClient:
                     self.metadata_cache, self.metadata_api_ops
                 )
 
+                # Initialize sync message with session
+                self._sync_session_manager = SyncSessionManager(self.metadata_cache, self.metadata_api_ops)
+
                 self._metadata_initialized = True
                 self.logger.debug("Metadata cache v2 with label caching initialized")
 
@@ -144,18 +153,9 @@ class FOClient:
                 f"Starting background metadata sync for version {global_version_id}"
             )
 
-            # Use self as the fo_client for sync - SmartSyncManagerV2 expects a client with metadata API operations
-            result = await self.sync_manager.sync_metadata(global_version_id)
-
-            if result.success:
-                self.logger.info(
-                    f"Background sync completed: "
-                    f"{result.entity_count} entities, "
-                    f"{result.enumeration_count} enumerations, "
-                    f"{result.duration_ms:.2f}ms"
-                )
-            else:
-                self.logger.warning(f"Background sync failed: {result.error}")
+ 
+            self.sync_session_manager.start_sync_session(global_version_id=global_version_id,initiated_by="background_task")
+            
 
         except Exception as e:
             self.logger.error(f"Background sync error: {e}")
@@ -172,6 +172,27 @@ class FOClient:
             and not self._background_sync_task.done()
         )
 
+    @property
+    def sync_session_manager(self) -> SyncSessionManager:
+        """Get sync session manager (lazy initialization).
+        
+        Returns:
+            SyncSessionManager instance for enhanced sync progress tracking
+            
+        Raises:
+            RuntimeError: If metadata cache is not initialized
+        """
+        if self._sync_session_manager is None:
+            if not self.metadata_cache:
+                raise RuntimeError("Metadata cache must be initialized before accessing sync session manager")
+            
+            self._sync_session_manager = SyncSessionManager(
+                cache=self.metadata_cache,
+                metadata_api=self.metadata_api_ops
+            )
+        
+        return self._sync_session_manager
+
     async def _get_from_cache_first(
         self,
         cache_method,
@@ -687,17 +708,16 @@ class FOClient:
 
     async def get_data_entities(
         self, options: Optional[QueryOptions] = None
-    ) -> List[DataEntityInfo]:
-        """Get data entities - updated to return list for v2 sync compatibility
+    ) -> Dict[str, Any]:
+        """Get data entities from DataEntities metadata endpoint
 
         Args:
-            options: OData query options (ignored for now)
+            options: OData query options
 
         Returns:
-            List of DataEntityInfo objects
+            Response containing data entities
         """
-        # For sync manager compatibility, return list of DataEntityInfo objects
-        return await self.metadata_api_ops.search_data_entities("")  # Get all entities
+        return await self.metadata_api_ops.get_data_entities(options)
 
     async def get_data_entities_raw(
         self, options: Optional[QueryOptions] = None
@@ -934,7 +954,7 @@ class FOClient:
         Returns:
             Response containing public enumerations
         """
-        self._ensure_metadata_initialized()
+        await self._ensure_metadata_initialized()
 
         return await self.metadata_api_ops.get_public_enumerations(options)
 
@@ -1188,7 +1208,7 @@ class FOClient:
                     "advanced_cache_enabled": True,
                     "cache_v2_enabled": True,
                     "cache_initialized": self._metadata_initialized,
-                    "sync_manager_available": self.sync_manager is not None,
+                    "sync_manager_available": self.sync_manager is not None or self._sync_session_manager is not None,
                     "background_sync_running": self._is_background_sync_running(),
                     "statistics": stats,
                 }

d365fo_client/credential_sources.py

@@ -0,0 +1,431 @@
+"""Credential source management for D365 F&O client authentication."""
+
+import logging
+import os
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Any, Dict, Optional, Tuple, Union
+
+from azure.identity import ClientSecretCredential, DefaultAzureCredential
+from azure.keyvault.secrets import SecretClient
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class CredentialSource:
+    """Base credential source configuration."""
+    
+    source_type: str  # "environment", "keyvault", "file", etc.
+    
+    def to_dict(self) -> Dict[str, str]:
+        """Convert to dictionary for serialization."""
+        return {"source_type": self.source_type}
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "CredentialSource":
+        """Create CredentialSource from dictionary."""
+        source_type = data.get("source_type")
+        
+        if source_type == "environment":
+            return EnvironmentCredentialSource(
+                client_id_var=data.get("client_id_var", "D365FO_CLIENT_ID"),
+                client_secret_var=data.get("client_secret_var", "D365FO_CLIENT_SECRET"),
+                tenant_id_var=data.get("tenant_id_var", "D365FO_TENANT_ID")
+            )
+        elif source_type == "keyvault":
+            return KeyVaultCredentialSource(
+                vault_url=data.get("vault_url", ""),
+                client_id_secret_name=data.get("client_id_secret_name", "D365FO_CLIENT_ID"),
+                client_secret_secret_name=data.get("client_secret_secret_name", "D365FO_CLIENT_SECRET"),
+                tenant_id_secret_name=data.get("tenant_id_secret_name", "D365FO_TENANT_ID"),
+                keyvault_auth_mode=data.get("keyvault_auth_mode", "default"),
+                keyvault_client_id=data.get("keyvault_client_id"),
+                keyvault_client_secret=data.get("keyvault_client_secret"),
+                keyvault_tenant_id=data.get("keyvault_tenant_id")
+            )
+        else:
+            raise ValueError(f"Unknown credential source type: {source_type}")
+
+
+@dataclass 
+class EnvironmentCredentialSource(CredentialSource):
+    """Environment variable credential source."""
+    
+    client_id_var: str = "D365FO_CLIENT_ID"
+    client_secret_var: str = "D365FO_CLIENT_SECRET"
+    tenant_id_var: str = "D365FO_TENANT_ID"
+
+    def __init__(self, client_id_var: str = "D365FO_CLIENT_ID",
+                 client_secret_var: str = "D365FO_CLIENT_SECRET",
+                 tenant_id_var: str = "D365FO_TENANT_ID"):
+        """Initialize environment credential source."""
+        super().__init__(source_type="environment")
+        self.client_id_var = client_id_var
+        self.client_secret_var = client_secret_var
+        self.tenant_id_var = tenant_id_var
+    
+    def to_dict(self) -> Dict[str, str]:
+        """Convert to dictionary for serialization."""
+        return {
+            "source_type": self.source_type,
+            "client_id_var": self.client_id_var,
+            "client_secret_var": self.client_secret_var,
+            "tenant_id_var": self.tenant_id_var,
+        }
+
+
+@dataclass
+class KeyVaultCredentialSource(CredentialSource):
+    """Azure Key Vault credential source."""
+    
+    vault_url: str
+    client_id_secret_name: str
+    client_secret_secret_name: str
+    tenant_id_secret_name: str
+    # Authentication to Key Vault itself
+    keyvault_auth_mode: str = "default"  # "default" or "client_secret"
+    keyvault_client_id: Optional[str] = None
+    keyvault_client_secret: Optional[str] = None
+    keyvault_tenant_id: Optional[str] = None
+    
+    def __init__(self, vault_url: str, client_id_secret_name: str,
+                 client_secret_secret_name: str, tenant_id_secret_name: str,
+                 keyvault_auth_mode: str = "default",
+                 keyvault_client_id: Optional[str] = None,
+                 keyvault_client_secret: Optional[str] = None,
+                 keyvault_tenant_id: Optional[str] = None):
+        """Initialize Key Vault credential source."""
+        super().__init__(source_type="keyvault")
+        self.vault_url = vault_url
+        self.client_id_secret_name = client_id_secret_name
+        self.client_secret_secret_name = client_secret_secret_name
+        self.tenant_id_secret_name = tenant_id_secret_name
+        self.keyvault_auth_mode = keyvault_auth_mode
+        self.keyvault_client_id = keyvault_client_id
+        self.keyvault_client_secret = keyvault_client_secret
+        self.keyvault_tenant_id = keyvault_tenant_id
+    
+    def to_dict(self) -> Dict[str, str]:
+        """Convert to dictionary for serialization."""
+        result = {
+            "source_type": self.source_type,
+            "vault_url": self.vault_url,
+            "client_id_secret_name": self.client_id_secret_name,
+            "client_secret_secret_name": self.client_secret_secret_name,
+            "tenant_id_secret_name": self.tenant_id_secret_name,
+            "keyvault_auth_mode": self.keyvault_auth_mode,
+        }
+        
+        # Only include Key Vault auth credentials if using client_secret mode
+        if self.keyvault_auth_mode == "client_secret":
+            result.update({
+                "keyvault_client_id": self.keyvault_client_id or "",
+                "keyvault_client_secret": self.keyvault_client_secret or "",
+                "keyvault_tenant_id": self.keyvault_tenant_id or "",
+            })
+        
+        return result
+
+
+@dataclass
+class CachedCredentials:
+    """Cached credential information with expiry."""
+    
+    client_id: str
+    client_secret: str
+    tenant_id: str
+    expires_at: datetime
+    source_hash: str  # Hash of source configuration for invalidation
+    
+    def is_expired(self) -> bool:
+        """Check if credentials are expired."""
+        return datetime.now() >= self.expires_at
+    
+    def is_valid_for_source(self, source_hash: str) -> bool:
+        """Check if cached credentials are valid for the given source."""
+        return not self.is_expired() and self.source_hash == source_hash
+
+
+class CredentialProvider(ABC):
+    """Base class for credential providers."""
+    
+    @abstractmethod
+    async def get_credentials(self, source: CredentialSource) -> Tuple[str, str, str]:
+        """
+        Retrieve credentials from the source.
+        
+        Args:
+            source: Credential source configuration
+            
+        Returns:
+            Tuple of (client_id, client_secret, tenant_id)
+            
+        Raises:
+            ValueError: If credentials cannot be retrieved or are invalid
+            Exception: For other retrieval errors
+        """
+        pass
+    
+    def _get_source_hash(self, source: CredentialSource) -> str:
+        """Generate a hash for the credential source configuration."""
+        import hashlib
+        source_data = str(source.to_dict())
+        return hashlib.sha256(source_data.encode()).hexdigest()[:16]
+
+
+class EnvironmentCredentialProvider(CredentialProvider):
+    """Provides credentials from environment variables."""
+    
+    async def get_credentials(self, source: CredentialSource) -> Tuple[str, str, str]:
+        """
+        Retrieve credentials from environment variables.
+        
+        Args:
+            source: EnvironmentCredentialSource configuration
+            
+        Returns:
+            Tuple of (client_id, client_secret, tenant_id)
+            
+        Raises:
+            ValueError: If required environment variables are missing
+        """
+        if not isinstance(source, EnvironmentCredentialSource):
+            raise ValueError(f"Expected EnvironmentCredentialSource, got {type(source)}")
+        
+        client_id = os.getenv(source.client_id_var)
+        client_secret = os.getenv(source.client_secret_var)
+        tenant_id = os.getenv(source.tenant_id_var)
+        
+        missing_vars = []
+        if not client_id:
+            missing_vars.append(source.client_id_var)
+        if not client_secret:
+            missing_vars.append(source.client_secret_var)
+        if not tenant_id:
+            missing_vars.append(source.tenant_id_var)
+            
+        if missing_vars:
+            raise ValueError(f"Missing required environment variables: {', '.join(missing_vars)}")
+        
+        logger.debug(f"Retrieved credentials from environment variables: {source.client_id_var}, {source.client_secret_var}, {source.tenant_id_var}")
+        return client_id, client_secret, tenant_id
+
+
+class KeyVaultCredentialProvider(CredentialProvider):
+    """Provides credentials from Azure Key Vault."""
+    
+    def __init__(self):
+        """Initialize Key Vault credential provider."""
+        self._secret_clients: Dict[str, SecretClient] = {}
+    
+    async def get_credentials(self, source: CredentialSource) -> Tuple[str, str, str]:
+        """
+        Retrieve credentials from Azure Key Vault.
+        
+        Args:
+            source: KeyVaultCredentialSource configuration
+            
+        Returns:
+            Tuple of (client_id, client_secret, tenant_id)
+            
+        Raises:
+            ValueError: If Key Vault access fails or secrets are missing
+        """
+        if not isinstance(source, KeyVaultCredentialSource):
+            raise ValueError(f"Expected KeyVaultCredentialSource, got {type(source)}")
+        
+        try:
+            secret_client = self._get_secret_client(source)
+            
+            # Retrieve secrets from Key Vault
+            client_id_secret = secret_client.get_secret(source.client_id_secret_name)
+            client_secret_secret = secret_client.get_secret(source.client_secret_secret_name)
+            tenant_id_secret = secret_client.get_secret(source.tenant_id_secret_name)
+            
+            client_id = client_id_secret.value
+            client_secret = client_secret_secret.value
+            tenant_id = tenant_id_secret.value
+            
+            if not all([client_id, client_secret, tenant_id]):
+                raise ValueError("One or more secrets retrieved from Key Vault are empty")
+            
+            logger.debug(f"Retrieved credentials from Key Vault: {source.vault_url}")
+            return client_id, client_secret, tenant_id
+            
+        except Exception as e:
+            logger.error(f"Failed to retrieve credentials from Key Vault {source.vault_url}: {e}")
+            raise ValueError(f"Key Vault credential retrieval failed: {e}")
+    
+    def _get_secret_client(self, source: KeyVaultCredentialSource) -> SecretClient:
+        """
+        Get or create a SecretClient for the Key Vault.
+        
+        Args:
+            source: KeyVaultCredentialSource configuration
+            
+        Returns:
+            SecretClient instance
+        """
+        vault_url = source.vault_url
+        
+        # Return cached client if available
+        if vault_url in self._secret_clients:
+            return self._secret_clients[vault_url]
+        
+        # Create credential for Key Vault authentication
+        if source.keyvault_auth_mode == "default":
+            credential = DefaultAzureCredential()
+            logger.debug(f"Using default credentials for Key Vault authentication: {vault_url}")
+        elif source.keyvault_auth_mode == "client_secret":
+            if not all([source.keyvault_client_id, source.keyvault_client_secret, source.keyvault_tenant_id]):
+                raise ValueError("Key Vault client_secret authentication requires keyvault_client_id, keyvault_client_secret, and keyvault_tenant_id")
+            
+            credential = ClientSecretCredential(
+                tenant_id=source.keyvault_tenant_id,
+                client_id=source.keyvault_client_id,
+                client_secret=source.keyvault_client_secret,
+            )
+            logger.debug(f"Using client secret credentials for Key Vault authentication: {vault_url}")
+        else:
+            raise ValueError(f"Invalid keyvault_auth_mode: {source.keyvault_auth_mode}. Must be 'default' or 'client_secret'")
+        
+        # Create and cache the SecretClient
+        secret_client = SecretClient(vault_url=vault_url, credential=credential)
+        self._secret_clients[vault_url] = secret_client
+        
+        return secret_client
+
+
+class CredentialManager:
+    """
+    Manages credential retrieval from multiple sources with caching.
+    
+    This class provides a unified interface for retrieving credentials from
+    various sources (environment variables, Key Vault, etc.) with automatic
+    caching and validation.
+    """
+    
+    def __init__(self, cache_ttl_minutes: int = 30):
+        """
+        Initialize credential manager.
+        
+        Args:
+            cache_ttl_minutes: Time-to-live for cached credentials in minutes
+        """
+        self.cache_ttl_minutes = cache_ttl_minutes
+        self._credential_cache: Dict[str, CachedCredentials] = {}
+        self._providers: Dict[str, CredentialProvider] = {
+            "environment": EnvironmentCredentialProvider(),
+            "keyvault": KeyVaultCredentialProvider(),
+        }
+    
+    async def get_credentials(self, source: CredentialSource) -> Tuple[str, str, str]:
+        """
+        Retrieve credentials from the specified source with caching.
+        
+        Args:
+            source: Credential source configuration
+            
+        Returns:
+            Tuple of (client_id, client_secret, tenant_id)
+            
+        Raises:
+            ValueError: If source type is unsupported or credentials cannot be retrieved
+        """
+        # Check cache first
+        source_hash = self._get_source_hash(source)
+        cache_key = f"{source.source_type}:{source_hash}"
+        
+        if cache_key in self._credential_cache:
+            cached = self._credential_cache[cache_key]
+            if cached.is_valid_for_source(source_hash):
+                logger.debug(f"Using cached credentials for source type: {source.source_type}")
+                return cached.client_id, cached.client_secret, cached.tenant_id
+            else:
+                # Remove expired cache entry
+                del self._credential_cache[cache_key]
+                logger.debug(f"Expired cached credentials removed for source type: {source.source_type}")
+        
+        # Retrieve from provider
+        provider = self._providers.get(source.source_type)
+        if not provider:
+            raise ValueError(f"Unsupported credential source type: {source.source_type}")
+        
+        client_id, client_secret, tenant_id = await provider.get_credentials(source)
+        
+        # Cache the credentials
+        expires_at = datetime.now() + timedelta(minutes=self.cache_ttl_minutes)
+        cached_credentials = CachedCredentials(
+            client_id=client_id,
+            client_secret=client_secret,
+            tenant_id=tenant_id,
+            expires_at=expires_at,
+            source_hash=source_hash,
+        )
+        self._credential_cache[cache_key] = cached_credentials
+        
+        logger.debug(f"Retrieved and cached credentials from source type: {source.source_type}")
+        return client_id, client_secret, tenant_id
+    
+    def _get_source_hash(self, source: CredentialSource) -> str:
+        """Generate a hash for the credential source configuration."""
+        import hashlib
+        source_data = str(source.to_dict())
+        return hashlib.sha256(source_data.encode()).hexdigest()[:16]
+    
+    def clear_cache(self, source_type: Optional[str] = None) -> None:
+        """
+        Clear credential cache.
+        
+        Args:
+            source_type: If specified, only clear cache for this source type.
+                        If None, clear all cached credentials.
+        """
+        if source_type is None:
+            self._credential_cache.clear()
+            logger.debug("Cleared all cached credentials")
+        else:
+            keys_to_remove = [key for key in self._credential_cache.keys() if key.startswith(f"{source_type}:")]
+            for key in keys_to_remove:
+                del self._credential_cache[key]
+            logger.debug(f"Cleared cached credentials for source type: {source_type}")
+    
+    def get_cache_stats(self) -> Dict[str, int]:
+        """
+        Get cache statistics.
+        
+        Returns:
+            Dictionary with cache statistics
+        """
+        total_cached = len(self._credential_cache)
+        expired_count = sum(1 for cached in self._credential_cache.values() if cached.is_expired())
+        
+        return {
+            "total_cached": total_cached,
+            "expired": expired_count,
+            "active": total_cached - expired_count,
+        }
+
+
+def create_credential_source(source_type: str, **kwargs) -> CredentialSource:
+    """
+    Factory function to create credential source instances.
+    
+    Args:
+        source_type: Type of credential source ("environment", "keyvault")
+        **kwargs: Source-specific configuration parameters
+        
+    Returns:
+        CredentialSource instance
+        
+    Raises:
+        ValueError: If source_type is unsupported
+    """
+    if source_type == "environment":
+        return EnvironmentCredentialSource(**kwargs)
+    elif source_type == "keyvault":
+        return KeyVaultCredentialSource(**kwargs)
+    else:
+        raise ValueError(f"Unsupported credential source type: {source_type}")

d365fo_client/mcp/client_manager.py

@@ -201,6 +201,14 @@ class D365FOClientManager:
                 raise ValueError(
                     f"Profile '{env_profile.name}' has no base_url configured"
                 )
+            
+            # Check if legacy config should override certain settings
+            default_config = self.config.get("default_environment", {})
+            if default_config and profile == "default":
+                # Override use_default_credentials if specified in legacy config
+                if "use_default_credentials" in default_config:
+                    config.use_default_credentials = default_config["use_default_credentials"]
+            
             return config
 
         # Fallback to legacy config-based profiles