alita-sdk 0.3.497__py3-none-any.whl → 0.3.516__py3-none-any.whl

alita_sdk/cli/inventory.py

@@ -1048,209 +1048,26 @@ def _get_checkpoint_path(graph: str, source_name: str) -> str:
 
 
 def _load_toolkit_config(toolkit_path: str) -> Dict[str, Any]:
-    """
-    Load and parse a toolkit config JSON file.
-    
-    Supports environment variable substitution for values like ${GITHUB_PAT}.
-    """
-    with open(toolkit_path, 'r') as f:
-        config = json.load(f)
-    
-    # Recursively resolve environment variables
-    def resolve_env_vars(obj):
-        if isinstance(obj, str):
-            # Match ${VAR_NAME} pattern
-            pattern = r'\$\{([^}]+)\}'
-            matches = re.findall(pattern, obj)
-            for var_name in matches:
-                env_value = os.environ.get(var_name, '')
-                obj = obj.replace(f'${{{var_name}}}', env_value)
-            return obj
-        elif isinstance(obj, dict):
-            return {k: resolve_env_vars(v) for k, v in obj.items()}
-        elif isinstance(obj, list):
-            return [resolve_env_vars(item) for item in obj]
-        return obj
-    
-    return resolve_env_vars(config)
+    """Deprecated: Use alita_sdk.community.inventory.toolkit_utils.load_toolkit_config instead."""
+    from alita_sdk.community.inventory.toolkit_utils import load_toolkit_config
+    return load_toolkit_config(toolkit_path)
 
 
 def _get_llm(ctx, model: Optional[str] = None, temperature: float = 0.0):
-    """Get LLM instance from Alita client context."""
+    """Deprecated: Use alita_sdk.community.inventory.toolkit_utils.get_llm_for_config instead."""
     from .cli import get_client
+    from alita_sdk.community.inventory.toolkit_utils import get_llm_for_config
     
-    # Get Alita client - this will raise ClickException if not configured
     client = get_client(ctx)
-    
-    # Get model name from parameter or default
-    model_name = model or 'gpt-4o-mini'
-    
-    # Use client.get_llm() which is the actual method
-    return client.get_llm(
-        model_name=model_name,
-        model_config={
-            'temperature': temperature,
-            'max_tokens': 4096
-        }
-    )
+    return get_llm_for_config(client, model=model, temperature=temperature)
 
 
 def _get_source_toolkit(toolkit_config: Dict[str, Any]):
-    """
-    Get configured source toolkit instance from toolkit config.
-    
-    Uses the SDK's toolkit factory pattern - all toolkits extend BaseCodeToolApiWrapper
-    or BaseVectorStoreToolApiWrapper, which have loader() and chunker() methods.
-    
-    Also supports CLI-specific toolkits like 'filesystem' for local document loading.
+    """Deprecated: Use alita_sdk.community.inventory.toolkit_utils.get_source_toolkit instead."""
+    from alita_sdk.community.inventory.toolkit_utils import get_source_toolkit
     
-    Args:
-        toolkit_config: Toolkit configuration dict with 'type' and settings
-        
-    Returns:
-        API wrapper instance with loader() method
-    """
-    source = toolkit_config.get('type')
-    if not source:
-        raise click.ClickException("Toolkit config missing 'type' field")
-    
-    # Handle filesystem type (CLI-specific, not in AVAILABLE_TOOLS)
-    if source == 'filesystem':
-        from .tools.filesystem import FilesystemApiWrapper
-        
-        base_directory = (
-            toolkit_config.get('base_directory') or
-            toolkit_config.get('path') or
-            toolkit_config.get('git_root_dir')
-        )
-        
-        if not base_directory:
-            raise click.ClickException(
-                "Filesystem toolkit requires 'base_directory' or 'path' field"
-            )
-        
-        return FilesystemApiWrapper(
-            base_directory=base_directory,
-            recursive=toolkit_config.get('recursive', True),
-            follow_symlinks=toolkit_config.get('follow_symlinks', False),
-        )
-    
-    # Handle standard SDK toolkits via AVAILABLE_TOOLS registry
-    from alita_sdk.tools import AVAILABLE_TOOLS
-    
-    # Check if toolkit type is available
-    if source not in AVAILABLE_TOOLS:
-        raise click.ClickException(
-            f"Unknown toolkit type: {source}. "
-            f"Available: {', '.join(list(AVAILABLE_TOOLS.keys()) + ['filesystem'])}"
-        )
-    
-    toolkit_info = AVAILABLE_TOOLS[source]
-    
-    # Get the toolkit class
-    if 'toolkit_class' not in toolkit_info:
-        raise click.ClickException(
-            f"Toolkit '{source}' does not have a toolkit_class registered"
-        )
-    
-    toolkit_class = toolkit_info['toolkit_class']
-    
-    # Build kwargs from toolkit config - we need to map config to API wrapper params
-    kwargs = dict(toolkit_config)
-    
-    # Remove fields that aren't needed for the API wrapper
-    kwargs.pop('type', None)
-    kwargs.pop('toolkit_name', None)
-    kwargs.pop('selected_tools', None)
-    kwargs.pop('excluded_tools', None)
-    
-    # Handle common config patterns - flatten nested configurations
-    config_key = f"{source}_configuration"
-    if config_key in kwargs:
-        nested_config = kwargs.pop(config_key)
-        if isinstance(nested_config, dict):
-            kwargs.update(nested_config)
-    
-    # Handle ADO-specific config pattern
-    if 'ado_configuration' in kwargs:
-        ado_config = kwargs.pop('ado_configuration')
-        if isinstance(ado_config, dict):
-            kwargs.update(ado_config)
-    
-    # Expand environment variables in string values (e.g., ${GITHUB_PAT})
-    def expand_env_vars(value):
-        """Recursively expand environment variables in values."""
-        if isinstance(value, str):
-            import re
-            # Match ${VAR} or $VAR patterns
-            pattern = r'\$\{([^}]+)\}|\$([A-Za-z_][A-Za-z0-9_]*)'
-            def replace(match):
-                var_name = match.group(1) or match.group(2)
-                return os.environ.get(var_name, match.group(0))
-            return re.sub(pattern, replace, value)
-        elif isinstance(value, dict):
-            return {k: expand_env_vars(v) for k, v in value.items()}
-        elif isinstance(value, list):
-            return [expand_env_vars(v) for v in value]
-        return value
-    
-    kwargs = expand_env_vars(kwargs)
-    
-    # Map common field names to API wrapper expected names
-    # GitHub: personal_access_token -> github_access_token
-    if 'personal_access_token' in kwargs and source == 'github':
-        kwargs['github_access_token'] = kwargs.pop('personal_access_token')
-    
-    # GitHub: repository -> github_repository
-    if 'repository' in kwargs and source == 'github':
-        kwargs['github_repository'] = kwargs.pop('repository')
-    
-    # Ensure active_branch has a default
-    if 'active_branch' not in kwargs:
-        kwargs['active_branch'] = kwargs.get('base_branch', 'main')
-    
-    # Get the API wrapper class from the toolkit
-    # Introspect toolkit to find the API wrapper class it uses
     try:
-        # Try to get the API wrapper class from the toolkit's module
-        import importlib
-        module_path = f"alita_sdk.tools.{source}.api_wrapper"
-        try:
-            wrapper_module = importlib.import_module(module_path)
-        except ImportError:
-            # Try alternate path for nested modules
-            module_path = f"alita_sdk.tools.{source.replace('_', '.')}.api_wrapper"
-            wrapper_module = importlib.import_module(module_path)
-        
-        # Find the API wrapper class - look for class containing ApiWrapper/APIWrapper
-        api_wrapper_class = None
-        for name in dir(wrapper_module):
-            obj = getattr(wrapper_module, name)
-            if (isinstance(obj, type) and 
-                ('ApiWrapper' in name or 'APIWrapper' in name) and 
-                name not in ('BaseCodeToolApiWrapper', 'BaseVectorStoreToolApiWrapper', 'BaseToolApiWrapper')):
-                api_wrapper_class = obj
-                break
-        
-        if not api_wrapper_class:
-            raise click.ClickException(
-                f"Could not find API wrapper class in {module_path}"
-            )
-        
-        # Instantiate the API wrapper directly
-        api_wrapper = api_wrapper_class(**kwargs)
-        
-        # Verify it has loader method
-        if not hasattr(api_wrapper, 'loader'):
-            raise click.ClickException(
-                f"API wrapper '{api_wrapper_class.__name__}' has no loader() method"
-            )
-        
-        return api_wrapper
-        
-    except ImportError as e:
-        logger.exception(f"Failed to import API wrapper for {source}")
-        raise click.ClickException(f"Failed to import {source} API wrapper: {e}")
-    except Exception as e:
-        logger.exception(f"Failed to instantiate toolkit {source}")
-        raise click.ClickException(f"Failed to create {source} toolkit: {e}")
+        return get_source_toolkit(toolkit_config)
+    except ValueError as e:
+        # Convert ValueError to ClickException for CLI context
+        raise click.ClickException(str(e))

alita_sdk/community/inventory/__init__.py

@@ -85,6 +85,13 @@ from .ingestion import (
 # Retrieval Toolkit - for querying graphs
 from .retrieval import InventoryRetrievalApiWrapper
 
+# Toolkit utilities - for configuration and instantiation
+from .toolkit_utils import (
+    load_toolkit_config,
+    get_llm_for_config,
+    get_source_toolkit,
+)
+
 # Core graph types
 from .knowledge_graph import KnowledgeGraph, Citation
 
@@ -187,6 +194,11 @@ __all__ = [
     'InventoryRetrievalToolkit',
     'InventoryRetrievalApiWrapper',
     
+    # Toolkit utilities
+    'load_toolkit_config',
+    'get_llm_for_config',
+    'get_source_toolkit',
+    
     # Core types
     'KnowledgeGraph',
     'Citation',

alita_sdk/community/inventory/toolkit.py

@@ -15,7 +15,7 @@ from pydantic import BaseModel, Field, ConfigDict, create_model
 
 from .retrieval import InventoryRetrievalApiWrapper
 from ...tools.base.tool import BaseAction
-from ...tools.utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ...tools.utils import clean_string, get_max_toolkit_length
 
 
 class InventoryRetrievalToolkit(BaseToolkit):
@@ -144,17 +144,21 @@ class InventoryRetrievalToolkit(BaseToolkit):
         # Build tool mapping
         tool_map = {t['name']: t for t in available_tools}
         
-        # Create tools with toolkit prefix
-        prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
+        # Use clean toolkit name for context (max 1000 chars in description)
+        toolkit_context = f" [Toolkit: {clean_string(toolkit_name, 0)}]" if toolkit_name else ''
         
         tools = []
         for tool_name in selected_tools:
             if tool_name in tool_map:
                 tool_info = tool_map[tool_name]
+                # Add toolkit context to description with character limit
+                description = tool_info['description']
+                if toolkit_context and len(description + toolkit_context) <= 1000:
+                    description = description + toolkit_context
                 tools.append(BaseAction(
                     api_wrapper=api_wrapper,
-                    name=f"{prefix}{tool_name}",
-                    description=tool_info['description'],
+                    name=tool_name,
+                    description=description,
                     args_schema=tool_info['args_schema']
                 ))
         

alita_sdk/community/inventory/toolkit_utils.py

@@ -0,0 +1,176 @@
+"""
+Toolkit configuration and instantiation utilities for inventory ingestion.
+
+This module provides functions to load toolkit configurations, instantiate source
+toolkits from various sources (filesystem, GitHub, ADO), and get LLM instances
+for entity extraction.
+"""
+
+import json
+import os
+import re
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from alita_sdk.alita_client import AlitaClient
+
+
+def load_toolkit_config(toolkit_path: str) -> Dict[str, Any]:
+    """
+    Load and parse a toolkit config JSON file.
+    
+    Supports environment variable substitution for values like ${GITHUB_PAT}.
+    
+    Args:
+        toolkit_path: Path to the toolkit configuration JSON file
+        
+    Returns:
+        Dictionary containing the parsed and environment-resolved configuration
+        
+    Example:
+        >>> config = load_toolkit_config("configs/github_toolkit.json")
+        >>> config['type']
+        'github'
+    """
+    with open(toolkit_path, 'r') as f:
+        config = json.load(f)
+    
+    # Recursively resolve environment variables
+    def resolve_env_vars(obj):
+        if isinstance(obj, str):
+            # Match ${VAR_NAME} pattern
+            pattern = r'\$\{([^}]+)\}'
+            matches = re.findall(pattern, obj)
+            for var_name in matches:
+                env_value = os.environ.get(var_name, '')
+                obj = obj.replace(f'${{{var_name}}}', env_value)
+            return obj
+        elif isinstance(obj, dict):
+            return {k: resolve_env_vars(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [resolve_env_vars(item) for item in obj]
+        return obj
+    
+    return resolve_env_vars(config)
+
+
+def get_llm_for_config(
+    client: AlitaClient,
+    model: Optional[str] = None,
+    temperature: float = 0.0
+):
+    """
+    Get LLM instance from Alita client for entity extraction.
+    
+    Args:
+        client: AlitaClient instance
+        model: Model name (defaults to 'gpt-4o-mini' if not specified)
+        temperature: Temperature for the model (default 0.0 for deterministic output)
+        
+    Returns:
+        LLM instance configured with the specified model and parameters
+        
+    Example:
+        >>> client = AlitaClient(...)
+        >>> llm = get_llm_for_config(client, model='gpt-4o', temperature=0.0)
+    """
+    model_name = model or 'gpt-4o-mini'
+    
+    return client.get_llm(
+        model_name=model_name,
+        model_config={
+            'temperature': temperature,
+            'max_tokens': 4096
+        }
+    )
+
+
+def get_source_toolkit(toolkit_config: Dict[str, Any]):
+    """
+    Instantiate a source toolkit from configuration.
+    
+    Supports filesystem, GitHub, and Azure DevOps (ADO) toolkit types. For SDK-based
+    toolkits (GitHub, ADO), automatically handles configuration mapping and toolkit
+    instantiation from the registry.
+    
+    Args:
+        toolkit_config: Toolkit configuration dictionary with 'type' key
+                       and type-specific parameters
+                       
+    Returns:
+        Instantiated toolkit object ready for ingestion
+        
+    Raises:
+        ValueError: If toolkit type is unsupported or configuration is invalid
+        
+    Example:
+        >>> # Filesystem toolkit
+        >>> config = {'type': 'filesystem', 'base_path': '/path/to/code'}
+        >>> toolkit = get_source_toolkit(config)
+        
+        >>> # GitHub toolkit
+        >>> config = {
+        ...     'type': 'github',
+        ...     'github_token': 'ghp_...',
+        ...     'github_repository': 'owner/repo',
+        ...     'github_branch': 'main'
+        ... }
+        >>> toolkit = get_source_toolkit(config)
+    """
+    from alita_sdk.community.inventory.filesystem_toolkit import FilesystemToolkit
+    from alita_sdk.community.toolkits import AVAILABLE_TOOLS
+    
+    toolkit_type = toolkit_config.get('type')
+    
+    if toolkit_type == 'filesystem':
+        base_path = toolkit_config.get('base_path')
+        if not base_path:
+            raise ValueError("Filesystem toolkit requires 'base_path' configuration")
+        return FilesystemToolkit(base_path=Path(base_path))
+    
+    # Handle SDK toolkits (GitHub, ADO)
+    if toolkit_type not in AVAILABLE_TOOLS:
+        raise ValueError(
+            f"Unknown toolkit type: {toolkit_type}. "
+            f"Available types: filesystem, {', '.join(AVAILABLE_TOOLS.keys())}"
+        )
+    
+    toolkit_class = AVAILABLE_TOOLS[toolkit_type]
+    
+    # Flatten nested config if needed
+    config_for_init = {}
+    for key, value in toolkit_config.items():
+        if key == 'type':
+            continue
+        if isinstance(value, dict):
+            # Flatten nested dicts
+            config_for_init.update(value)
+        else:
+            config_for_init[key] = value
+    
+    # Map field names for specific toolkit types
+    if toolkit_type == 'github':
+        field_mapping = {
+            'github_token': 'token',
+            'github_repository': 'repository',
+            'github_branch': 'branch'
+        }
+        config_for_init = {
+            field_mapping.get(k, k): v 
+            for k, v in config_for_init.items()
+        }
+    elif toolkit_type == 'ado':
+        field_mapping = {
+            'ado_token': 'token',
+            'ado_organization': 'organization',
+            'ado_project': 'project',
+            'ado_repository': 'repository',
+            'ado_branch': 'branch'
+        }
+        config_for_init = {
+            field_mapping.get(k, k): v 
+            for k, v in config_for_init.items()
+        }
+    
+    # Instantiate toolkit
+    return toolkit_class(**config_for_init)

alita_sdk/configurations/ado.py

@@ -1,5 +1,8 @@
+import re
 from typing import Optional
+from urllib.parse import quote
 
+import requests
 from pydantic import BaseModel, ConfigDict, Field, SecretStr
 
 
@@ -19,6 +22,147 @@ class AdoConfiguration(BaseModel):
     project: str = Field(description="ADO project")
     token: Optional[SecretStr] = Field(description="ADO Token")
 
+    @staticmethod
+    def check_connection(settings: dict) -> str | None:
+        """
+        Test the connection to Azure DevOps API.
+
+        Args:
+            settings: Dictionary containing 'organization_url', 'project', and optionally 'token'
+
+        Returns:
+            None if connection is successful, error message string otherwise
+        """
+        organization_url = settings.get("organization_url")
+        if organization_url is None or organization_url == "":
+            if organization_url == "":
+                return "Organization URL cannot be empty"
+            return "Organization URL is required"
+
+        # Validate organization URL format
+        if not isinstance(organization_url, str):
+            return "Organization URL must be a string"
+
+        organization_url = organization_url.strip()
+        if not organization_url:
+            return "Organization URL cannot be empty"
+
+        if not organization_url.startswith(("http://", "https://")):
+            return "Organization URL must start with http:// or https://"
+
+        # Remove trailing slash for consistency
+        organization_url = organization_url.rstrip("/")
+
+        project = settings.get("project")
+        if project is None or project == "":
+            if project == "":
+                return "Project cannot be empty"
+            return "Project is required"
+
+        # Validate project format
+        if not isinstance(project, str):
+            return "Project must be a string"
+
+        project = project.strip()
+        if not project:
+            return "Project cannot be empty"
+
+        token = settings.get("token")
+
+        # Extract secret value if it's a SecretStr
+        if token is not None and hasattr(token, "get_secret_value"):
+            token = token.get_secret_value()
+
+        # Validate token if provided
+        if token is not None and (not token or not token.strip()):
+            return "Token cannot be empty if provided"
+
+        # NOTE on verification strategy:
+        # - Project endpoints can work anonymously for public projects.
+        #   That makes them a weak signal for detecting a bad/expired token.
+        # - If a token is provided, first validate it against a profile endpoint
+        #   that requires authentication, then check project access.
+
+        # Strictly require a canonical organization URL so we can build reliable API URLs.
+        # Supported formats:
+        # - https://dev.azure.com/<org>
+        # - https://<org>.visualstudio.com
+        org_name: str | None = None
+        org_url_kind: str | None = None  # 'dev.azure.com' | '*.visualstudio.com'
+        m = re.match(r"^https?://dev\.azure\.com/(?P<org>[^/]+)$", organization_url, flags=re.IGNORECASE)
+        if m:
+            org_name = m.group('org')
+            org_url_kind = 'dev.azure.com'
+        else:
+            m = re.match(r"^https?://(?P<org>[^/.]+)\.visualstudio\.com$", organization_url, flags=re.IGNORECASE)
+            if m:
+                org_name = m.group('org')
+                org_url_kind = '*.visualstudio.com'
+
+        if org_name is None:
+            return (
+                "Organization URL format is invalid. Use 'https://dev.azure.com/<org>' "
+                "(recommended) or 'https://<org>.visualstudio.com'."
+            )
+
+        project_encoded = quote(project, safe="")
+        project_url = f"{organization_url}/_apis/projects/{project_encoded}?api-version=7.0"
+        # Auth-required endpoint to validate PAT (works regardless of project visibility)
+        if org_url_kind == 'dev.azure.com':
+            profile_url = f"https://vssps.dev.azure.com/{org_name}/_apis/profile/profiles/me?api-version=7.1-preview.3"
+        else:
+            # For legacy org URLs, use the matching vssps host
+            profile_url = f"https://{org_name}.vssps.visualstudio.com/_apis/profile/profiles/me?api-version=7.1-preview.3"
+
+        try:
+            headers = {}
+            if token:
+                # Use Basic Auth with PAT token (username can be empty)
+                from requests.auth import HTTPBasicAuth
+                auth = HTTPBasicAuth("", token)
+
+                # 1) Validate token first (strong signal)
+                profile_resp = requests.get(profile_url, auth=auth, timeout=10)
+                if profile_resp.status_code == 200:
+                    pass
+                elif profile_resp.status_code == 401:
+                    return "Invalid or expired token (PAT). Please generate a new token and try again."
+                elif profile_resp.status_code == 403:
+                    return "Token is valid but lacks permission to access profile. Check PAT scopes/permissions."
+                elif profile_resp.status_code == 404:
+                    return "Organization not found. Verify the Organization URL."
+                else:
+                    return f"Token validation failed (HTTP {profile_resp.status_code})."
+
+                # 2) Validate project access
+                response = requests.get(project_url, auth=auth, timeout=10)
+            else:
+                # Try without authentication (works for public projects)
+                response = requests.get(project_url, headers=headers, timeout=10)
+
+            if response.status_code == 200:
+                return None  # Connection successful
+            elif response.status_code == 401:
+                if token:
+                    return "Not authorized. Token may be invalid for this organization or expired."
+                else:
+                    return "Authentication required - project may be private"
+            elif response.status_code == 403:
+                return "Access forbidden - token may lack required permissions for this project"
+            elif response.status_code == 404:
+                return f"Project '{project}' not found or not accessible. Check project name and organization URL."
+            else:
+                return f"Connection failed (HTTP {response.status_code})."
+
+        except requests.exceptions.Timeout:
+            return "Connection timeout - Azure DevOps did not respond within 10 seconds"
+        except requests.exceptions.ConnectionError:
+            return "Connection error - unable to reach Azure DevOps. Check the Organization URL and your network."
+        except requests.exceptions.RequestException as e:
+            return f"Request failed: {str(e)}"
+        except Exception:
+            return "Unexpected error during Azure DevOps connection check"
+
 
 class AdoReposConfiguration(BaseModel):
     model_config = ConfigDict(