signalwire-agents 0.1.18__py3-none-any.whl → 0.1.20__py3-none-any.whl

signalwire_agents/__init__.py

@@ -18,7 +18,7 @@ A package for building AI agents using SignalWire's AI and SWML capabilities.
 from .core.logging_config import configure_logging
 configure_logging()
 
-__version__ = "0.1.18"
+__version__ = "0.1.20"
 
 # Import core classes for easier access
 from .core.agent_base import AgentBase

signalwire_agents/cli/test_swaig.py

@@ -1079,183 +1079,326 @@ def simple_template_expand(template: str, data: Dict[str, Any]) -> str:
 def execute_datamap_function(datamap_config: Dict[str, Any], args: Dict[str, Any], 
                            verbose: bool = False) -> Dict[str, Any]:
     """
-    Execute a DataMap function by processing its configuration and making HTTP requests
+    Execute a DataMap function following the actual DataMap processing pipeline:
+    1. Expressions (pattern matching)
+    2. Webhooks (try each sequentially until one succeeds)
+    3. Foreach (within successful webhook)
+    4. Output (from successful webhook)
+    5. Fallback output (if all webhooks fail)
     
     Args:
-        datamap_config: The complete DataMap function configuration
+        datamap_config: DataMap configuration dictionary
         args: Function arguments
-        verbose: Whether to show verbose output
+        verbose: Enable verbose output
         
     Returns:
-        Final function result
+        Function result (should be string or dict with 'response' key)
     """
-    # Extract data_map configuration
-    data_map = datamap_config.get('data_map', {})
-    
-    # Check if this is a simple DataMap (webhooks directly) or complex (with expressions)
-    if 'webhooks' in data_map and 'expressions' not in data_map:
-        # Simple DataMap structure - webhooks directly under data_map
-        webhooks = data_map.get('webhooks', [])
-        fallback_output = data_map.get('output', 'Function completed')
+    if verbose:
+        print("=== DataMap Function Execution ===")
+        print(f"Config: {json.dumps(datamap_config, indent=2)}")
+        print(f"Args: {json.dumps(args, indent=2)}")
+    
+    # Extract the actual data_map configuration
+    # DataMap configs have the structure: {"function": "...", "data_map": {...}}
+    actual_datamap = datamap_config.get("data_map", datamap_config)
+    
+    if verbose:
+        print(f"Extracted data_map: {json.dumps(actual_datamap, indent=2)}")
+    
+    # Initialize context with function arguments
+    context = {"args": args}
+    context.update(args)  # Also make args available at top level for backward compatibility
+    
+    if verbose:
+        print(f"Initial context: {json.dumps(context, indent=2)}")
+    
+    # Step 1: Process expressions first (pattern matching)
+    if "expressions" in actual_datamap:
         if verbose:
-            print(f"Simple DataMap structure with {len(webhooks)} webhook(s)")
-    else:
-        # Complex DataMap structure - with expressions containing webhooks
-        expressions = data_map.get('expressions', [])
-        matched_expression = None
-        
+            print("\n--- Processing Expressions ---")
+        for expr in actual_datamap["expressions"]:
+            # Simple expression evaluation - in real implementation this would be more sophisticated
+            if "pattern" in expr and "output" in expr:
+                # For testing, we'll just match simple strings
+                pattern = expr["pattern"]
+                if pattern in str(args):
+                    if verbose:
+                        print(f"Expression matched: {pattern}")
+                    result = simple_template_expand(str(expr["output"]), context)
+                    if verbose:
+                        print(f"Expression result: {result}")
+                    return result
+    
+    # Step 2: Process webhooks sequentially
+    if "webhooks" in actual_datamap:
         if verbose:
-            print(f"Complex DataMap structure with {len(expressions)} expression(s)...")
+            print("\n--- Processing Webhooks ---")
         
-        for expr in expressions:
-            pattern = expr.get('pattern', '.*')  # Default to match everything
-            string_value = expr.get('string', '')
+        for i, webhook in enumerate(actual_datamap["webhooks"]):
+            if verbose:
+                print(f"\n=== Webhook {i+1}/{len(actual_datamap['webhooks'])} ===")
             
-            # Create a test string from the arguments
-            test_string = json.dumps(args)
+            url = webhook.get("url", "")
+            method = webhook.get("method", "POST").upper()
+            headers = webhook.get("headers", {})
+            
+            # Expand template variables in URL and headers
+            url = simple_template_expand(url, context)
+            expanded_headers = {}
+            for key, value in headers.items():
+                expanded_headers[key] = simple_template_expand(str(value), context)
             
             if verbose:
-                print(f"  Testing pattern '{pattern}' against '{test_string}'")
+                print(f"Making {method} request to: {url}")
+                print(f"Headers: {json.dumps(expanded_headers, indent=2)}")
+            
+            # Prepare request data
+            request_data = None
+            if method in ["POST", "PUT", "PATCH"]:
+                # Check for 'params' (SignalWire style) or 'data' (generic style) or 'body'
+                if "params" in webhook:
+                    # Expand template variables in params
+                    expanded_params = {}
+                    for key, value in webhook["params"].items():
+                        expanded_params[key] = simple_template_expand(str(value), context)
+                    request_data = json.dumps(expanded_params)
+                elif "body" in webhook:
+                    # Expand template variables in body
+                    if isinstance(webhook["body"], str):
+                        request_data = simple_template_expand(webhook["body"], context)
+                    else:
+                        expanded_body = {}
+                        for key, value in webhook["body"].items():
+                            expanded_body[key] = simple_template_expand(str(value), context)
+                        request_data = json.dumps(expanded_body)
+                elif "data" in webhook:
+                    # Expand template variables in data
+                    if isinstance(webhook["data"], str):
+                        request_data = simple_template_expand(webhook["data"], context)
+                    else:
+                        request_data = json.dumps(webhook["data"])
+                
+                if verbose and request_data:
+                    print(f"Request data: {request_data}")
+            
+            webhook_failed = False
+            response_data = None
             
-            # Use regex to match
             try:
-                if re.search(pattern, test_string):
-                    matched_expression = expr
-                    if verbose:
-                        print(f"  ✓ Pattern matched!")
-                    break
-                elif re.search(pattern, string_value):
-                    matched_expression = expr
+                # Make the HTTP request
+                if method == "GET":
+                    response = requests.get(url, headers=expanded_headers, timeout=30)
+                elif method == "POST":
+                    response = requests.post(url, data=request_data, headers=expanded_headers, timeout=30)
+                elif method == "PUT":
+                    response = requests.put(url, data=request_data, headers=expanded_headers, timeout=30)
+                elif method == "PATCH":
+                    response = requests.patch(url, data=request_data, headers=expanded_headers, timeout=30)
+                elif method == "DELETE":
+                    response = requests.delete(url, headers=expanded_headers, timeout=30)
+                else:
+                    raise ValueError(f"Unsupported HTTP method: {method}")
+                
+                if verbose:
+                    print(f"Response status: {response.status_code}")
+                    print(f"Response headers: {dict(response.headers)}")
+                
+                # Parse response
+                try:
+                    response_data = response.json()
+                except json.JSONDecodeError:
+                    response_data = {"text": response.text, "status_code": response.status_code}
+                    # Add parse_error like server does
+                    response_data["parse_error"] = True
+                    response_data["raw_response"] = response.text
+                
+                if verbose:
+                    print(f"Response data: {json.dumps(response_data, indent=2)}")
+                
+                # Check for webhook failure following server logic
+                
+                # 1. Check HTTP status code (fix the server bug - should be OR not AND)
+                if response.status_code < 200 or response.status_code > 299:
+                    webhook_failed = True
                     if verbose:
-                        print(f"  ✓ Pattern matched against string value!")
-                    break
-            except re.error as e:
+                        print(f"Webhook failed: HTTP status {response.status_code} outside 200-299 range")
+                
+                # 2. Check for explicit error keys (parse_error, protocol_error)
+                if not webhook_failed:
+                    explicit_error_keys = ["parse_error", "protocol_error"]
+                    for error_key in explicit_error_keys:
+                        if error_key in response_data and response_data[error_key]:
+                            webhook_failed = True
+                            if verbose:
+                                print(f"Webhook failed: Found explicit error key '{error_key}' = {response_data[error_key]}")
+                            break
+                
+                # 3. Check for custom error_keys from webhook config
+                if not webhook_failed and "error_keys" in webhook:
+                    error_keys = webhook["error_keys"]
+                    if isinstance(error_keys, str):
+                        error_keys = [error_keys]  # Convert single string to list
+                    elif not isinstance(error_keys, list):
+                        error_keys = []
+                    
+                    for error_key in error_keys:
+                        if error_key in response_data and response_data[error_key]:
+                            webhook_failed = True
+                            if verbose:
+                                print(f"Webhook failed: Found custom error key '{error_key}' = {response_data[error_key]}")
+                            break
+                
+            except Exception as e:
+                webhook_failed = True
                 if verbose:
-                    print(f"  ✗ Invalid regex pattern: {e}")
-        
-        if not matched_expression:
-            if verbose:
-                print("  No expressions matched, using first expression if available")
-            matched_expression = expressions[0] if expressions else {}
-        
-        # Get webhooks from matched expression
-        webhooks = matched_expression.get('webhooks', [])
-        fallback_output = matched_expression.get('output', 'Function completed')
-    webhook_result = None
-    
-    if verbose:
-        print(f"Processing {len(webhooks)} webhook(s)...")
-    
-    for i, webhook in enumerate(webhooks):
-        url = webhook.get('url', '')
-        method = webhook.get('method', 'POST').upper()
-        headers = webhook.get('headers', {})
-        
-        if verbose:
-            print(f"  Webhook {i+1}: {method} {url}")
-        
-        # Prepare request data
-        request_data = args.copy()
-        
-        # Expand URL template with arguments
-        template_context = {"args": args, "array": [], **args}
-        expanded_url = simple_template_expand(url, template_context)
-        
-        if verbose:
-            print(f"    Original URL: {url}")
-            print(f"    Template context: {template_context}")
-            print(f"    Expanded URL: {expanded_url}")
-        
-        try:
-            if method == 'GET':
-                response = requests.get(expanded_url, params=request_data, headers=headers, timeout=10)
-            else:
-                response = requests.post(expanded_url, json=request_data, headers=headers, timeout=10)
+                    print(f"Webhook failed: HTTP request exception: {e}")
+                # Create error response like server does
+                response_data = {
+                    "protocol_error": True,
+                    "error": str(e)
+                }
             
-            if response.status_code == 200:
-                try:
-                    webhook_data = response.json()
+            # If webhook succeeded, process its output
+            if not webhook_failed:
+                if verbose:
+                    print(f"Webhook {i+1} succeeded!")
+                
+                # Add response data to context
+                webhook_context = context.copy()
+                
+                # Handle different response types
+                if isinstance(response_data, list):
+                    # For array responses, use ${array[0].field} syntax
+                    webhook_context["array"] = response_data
+                    if verbose:
+                        print(f"Array response: {len(response_data)} items")
+                else:
+                    # For object responses, use ${response.field} syntax
+                    webhook_context["response"] = response_data
+                    if verbose:
+                        print("Object response")
+                
+                # Step 3: Process webhook-level foreach (if present)
+                if "foreach" in webhook:
+                    foreach_config = webhook["foreach"]
                     if verbose:
-                        print(f"  ✓ Webhook succeeded: {response.status_code}")
-                        print(f"    Response: {json.dumps(webhook_data, indent=4)}")
+                        print(f"\n--- Processing Webhook Foreach ---")
+                        print(f"Foreach config: {json.dumps(foreach_config, indent=2)}")
                     
-                    # Process output template if defined in this webhook
-                    webhook_output = webhook.get('output')
-                    if webhook_output:
-                        # Create context for template expansion
-                        template_context = {
-                            "args": args,
-                            "array": webhook_data if isinstance(webhook_data, list) else [webhook_data]
-                        }
+                    input_key = foreach_config.get("input_key", "data")
+                    output_key = foreach_config.get("output_key", "result")
+                    max_items = foreach_config.get("max", 100)
+                    append_template = foreach_config.get("append", "${this.value}")
+                    
+                    # Look for the input data in the response
+                    input_data = None
+                    if input_key in response_data and isinstance(response_data[input_key], list):
+                        input_data = response_data[input_key]
+                        if verbose:
+                            print(f"Found array data in response.{input_key}: {len(input_data)} items")
+                    
+                    if input_data:
+                        result_parts = []
+                        items_to_process = input_data[:max_items]
                         
-                        # Add webhook_data contents to context if it's a dict
-                        if isinstance(webhook_data, dict):
-                            template_context.update(webhook_data)
+                        for item in items_to_process:
+                            if isinstance(item, dict):
+                                # For objects, make properties available as ${this.property}
+                                item_context = {"this": item}
+                                expanded = simple_template_expand(append_template, item_context)
+                            else:
+                                # For non-dict items, make them available as ${this.value}
+                                item_context = {"this": {"value": item}}
+                                expanded = simple_template_expand(append_template, item_context)
+                            result_parts.append(expanded)
                         
-                        if isinstance(webhook_output, dict):
-                            # Process dict output template (e.g., {"response": "template", "action": [...]} )
-                            webhook_result = {}
-                            for key, template in webhook_output.items():
-                                if isinstance(template, str):
-                                    webhook_result[key] = simple_template_expand(template, template_context)
-                                else:
-                                    webhook_result[key] = template
-                        elif isinstance(webhook_output, str):
-                            # Simple string template
-                            webhook_result = {"response": simple_template_expand(webhook_output, template_context)}
-                        else:
-                            # Other types
-                            webhook_result = {"response": str(webhook_output)}
+                        # Store the concatenated result
+                        foreach_result = "".join(result_parts)
+                        webhook_context[output_key] = foreach_result
                         
                         if verbose:
-                            print(f"    Processed output: {webhook_result}")
+                            print(f"Processed {len(items_to_process)} items")
+                            print(f"Foreach result ({output_key}): {foreach_result[:200]}{'...' if len(foreach_result) > 200 else ''}")
                     else:
-                        webhook_result = webhook_data if isinstance(webhook_data, dict) else {"response": str(webhook_data)}
+                        if verbose:
+                            print(f"No array data found for foreach input_key: {input_key}")
+                
+                # Step 4: Process webhook-level output (this is the final result)
+                if "output" in webhook:
+                    webhook_output = webhook["output"]
+                    if verbose:
+                        print(f"\n--- Processing Webhook Output ---")
+                        print(f"Output template: {json.dumps(webhook_output, indent=2)}")
+                    
+                    if isinstance(webhook_output, dict):
+                        # Process each key-value pair in the output
+                        final_result = {}
+                        for key, template in webhook_output.items():
+                            expanded_value = simple_template_expand(str(template), webhook_context)
+                            final_result[key] = expanded_value
+                            if verbose:
+                                print(f"Set {key} = {expanded_value}")
+                    else:
+                        # Single output value (string template)
+                        final_result = simple_template_expand(str(webhook_output), webhook_context)
+                        if verbose:
+                            print(f"Final result = {final_result}")
                     
-                    break
-                except json.JSONDecodeError:
-                    webhook_result = {"response": response.text}
                     if verbose:
-                        print(f"  ✓ Webhook succeeded (text): {response.status_code}")
-                    break
+                        print(f"\n--- Webhook {i+1} Final Result ---")
+                        print(f"Result: {json.dumps(final_result, indent=2) if isinstance(final_result, dict) else final_result}")
+                    
+                    return final_result
+                
+                else:
+                    # No output template defined, return the response data
+                    if verbose:
+                        print("No output template defined, returning response data")
+                    return response_data
+            
             else:
+                # This webhook failed, try next webhook
                 if verbose:
-                    print(f"  ✗ Webhook failed: {response.status_code}")
-        except requests.RequestException as e:
-            if verbose:
-                print(f"  ✗ Webhook request failed: {e}")
+                    print(f"Webhook {i+1} failed, trying next webhook...")
+                continue
     
-    # If no webhook succeeded, use fallback
-    if webhook_result is None:
+    # Step 5: All webhooks failed, use fallback output if available
+    if "output" in actual_datamap:
         if verbose:
-            print("All webhooks failed, using fallback output...")
-        
-        # Use the fallback output determined earlier
-        output_template = fallback_output
-        
-        # Handle both string and dict output templates
-        if isinstance(output_template, dict):
-            # Process dict output template (e.g., {"response": "template"})
-            webhook_result = {}
-            for key, template in output_template.items():
-                if isinstance(template, str):
-                    webhook_result[key] = simple_template_expand(template, {"args": args})
-                else:
-                    webhook_result[key] = template
-        elif isinstance(output_template, str):
-            # Simple string template
-            webhook_result = {"response": simple_template_expand(output_template, {"args": args})}
+            print(f"\n--- Using DataMap Fallback Output ---")
+        datamap_output = actual_datamap["output"]
+        if verbose:
+            print(f"Fallback output template: {json.dumps(datamap_output, indent=2)}")
+        
+        if isinstance(datamap_output, dict):
+            # Process each key-value pair in the fallback output
+            final_result = {}
+            for key, template in datamap_output.items():
+                expanded_value = simple_template_expand(str(template), context)
+                final_result[key] = expanded_value
+                if verbose:
+                    print(f"Fallback: Set {key} = {expanded_value}")
+            result = final_result
         else:
-            # Other types (shouldn't happen but be safe)
-            webhook_result = {"response": str(output_template)}
+            # Single fallback output value
+            result = simple_template_expand(str(datamap_output), context)
+            if verbose:
+                print(f"Fallback result = {result}")
         
         if verbose:
-            print(f"Fallback result = {webhook_result}")
+            print(f"\n--- DataMap Fallback Final Result ---")
+            print(f"Result: {json.dumps(result, indent=2) if isinstance(result, dict) else result}")
+        
+        return result
     
-    # Process foreach (not implemented in this simple version)
-    # This would iterate over array results and apply templates
+    # No fallback defined, return generic error
+    error_result = {"error": "All webhooks failed and no fallback output defined", "status": "failed"}
+    if verbose:
+        print(f"\n--- DataMap Error Result ---")
+        print(f"Result: {json.dumps(error_result, indent=2)}")
     
-    return webhook_result
+    return error_result
 
 
 def execute_external_webhook_function(func: 'SWAIGFunction', function_name: str, function_args: Dict[str, Any], 

signalwire_agents/search/index_builder.py

@@ -367,16 +367,57 @@ class IndexBuilder:
                      global_tags: Optional[List[str]] = None) -> List[Dict[str, Any]]:
         """Process single file into chunks"""
         try:
-            # Try to read as text first
-            try:
-                content = file_path.read_text(encoding='utf-8')
-            except UnicodeDecodeError:
+            relative_path = str(file_path.relative_to(source_dir))
+            file_extension = file_path.suffix.lower()
+            
+            # Handle different file types appropriately
+            if file_extension == '.pdf':
+                # Use document processor for PDF extraction
+                content_result = self.doc_processor._extract_text_from_file(str(file_path))
+                if isinstance(content_result, str) and content_result.startswith('{"error"'):
+                    if self.verbose:
+                        print(f"Skipping PDF file (extraction failed): {file_path}")
+                    return []
+                content = content_result
+            elif file_extension in ['.docx', '.xlsx', '.pptx']:
+                # Use document processor for Office documents
+                content_result = self.doc_processor._extract_text_from_file(str(file_path))
+                if isinstance(content_result, str) and content_result.startswith('{"error"'):
+                    if self.verbose:
+                        print(f"Skipping office document (extraction failed): {file_path}")
+                    return []
+                content = content_result
+            elif file_extension == '.html':
+                # Use document processor for HTML
+                content_result = self.doc_processor._extract_text_from_file(str(file_path))
+                if isinstance(content_result, str) and content_result.startswith('{"error"'):
+                    if self.verbose:
+                        print(f"Skipping HTML file (extraction failed): {file_path}")
+                    return []
+                content = content_result
+            elif file_extension == '.rtf':
+                # Use document processor for RTF
+                content_result = self.doc_processor._extract_text_from_file(str(file_path))
+                if isinstance(content_result, str) and content_result.startswith('{"error"'):
+                    if self.verbose:
+                        print(f"Skipping RTF file (extraction failed): {file_path}")
+                    return []
+                content = content_result
+            else:
+                # Try to read as text file (markdown, txt, code, etc.)
+                try:
+                    content = file_path.read_text(encoding='utf-8')
+                except UnicodeDecodeError:
+                    if self.verbose:
+                        print(f"Skipping binary file: {file_path}")
+                    return []
+            
+            # Validate content
+            if not content or (isinstance(content, str) and len(content.strip()) == 0):
                 if self.verbose:
-                    print(f"Skipping binary file: {file_path}")
+                    print(f"Skipping empty file: {file_path}")
                 return []
             
-            relative_path = str(file_path.relative_to(source_dir))
-            
             # Create chunks using document processor - pass content directly, not file path
             chunks = self.doc_processor.create_chunks(
                 content=content,  # Pass the actual content, not the file path

signalwire_agents/search/query_processor.py

@@ -118,15 +118,28 @@ stopwords_language_map = {
 # Function to ensure NLTK resources are downloaded
 def ensure_nltk_resources():
     """Download required NLTK resources if not already present"""
-    resources = ['punkt', 'wordnet', 'averaged_perceptron_tagger', 'stopwords']
+    resources = ['punkt', 'punkt_tab', 'wordnet', 'averaged_perceptron_tagger', 'stopwords']
     for resource in resources:
         try:
-            nltk.data.find(f'corpora/{resource}')
+            # Try different paths for different resource types
+            if resource in ['punkt', 'punkt_tab']:
+                nltk.data.find(f'tokenizers/{resource}')
+            elif resource in ['wordnet']:
+                nltk.data.find(f'corpora/{resource}')
+            elif resource in ['averaged_perceptron_tagger']:
+                nltk.data.find(f'taggers/{resource}')
+            elif resource in ['stopwords']:
+                nltk.data.find(f'corpora/{resource}')
+            else:
+                nltk.data.find(f'corpora/{resource}')
         except LookupError:
             try:
+                logger.info(f"Downloading NLTK resource '{resource}'...")
                 nltk.download(resource, quiet=True)
+                logger.info(f"Successfully downloaded NLTK resource '{resource}'")
             except Exception as e:
                 logger.warning(f"Failed to download NLTK resource '{resource}': {e}")
+                # Continue without this resource - some functionality may be degraded
 
 # Initialize NLTK resources
 ensure_nltk_resources()
@@ -246,7 +259,20 @@ def preprocess_query(query: str, language: str = 'en', pos_to_expand: Optional[L
         query_nlp_backend = 'nltk'
     
     # Tokenization and stop word removal
-    tokens = nltk.word_tokenize(query)
+    try:
+        tokens = nltk.word_tokenize(query)
+    except LookupError as e:
+        # If tokenization fails, try to download punkt resources
+        logger.warning(f"NLTK tokenization failed: {e}")
+        try:
+            nltk.download('punkt', quiet=True)
+            nltk.download('punkt_tab', quiet=True)
+            tokens = nltk.word_tokenize(query)
+        except Exception as fallback_error:
+            # If all else fails, use simple split as fallback
+            logger.warning(f"NLTK tokenization fallback failed: {fallback_error}. Using simple word splitting.")
+            tokens = query.split()
+    
     nltk_language = stopwords_language_map.get(language, 'english')
     
     try:
@@ -279,14 +305,29 @@ def preprocess_query(query: str, language: str = 'en', pos_to_expand: Optional[L
             logger.info(f"POS Tagging Results (spaCy): {pos_tags}")
     else:
         # Use NLTK (default or fallback)
-        nltk_pos_tags = nltk.pos_tag(tokens)
-        for token, pos_tag in nltk_pos_tags:
-            lemma = lemmatizer.lemmatize(token, get_wordnet_pos(pos_tag)).lower()
-            stemmed = stemmer.stem(lemma)
-            lemmas.append((token.lower(), stemmed))
-            pos_tags[token.lower()] = pos_tag
-        if debug:
-            logger.info(f"POS Tagging Results (NLTK): {pos_tags}")
+        try:
+            nltk_pos_tags = nltk.pos_tag(tokens)
+            for token, pos_tag in nltk_pos_tags:
+                try:
+                    lemma = lemmatizer.lemmatize(token, get_wordnet_pos(pos_tag)).lower()
+                except Exception:
+                    # Fallback if lemmatization fails
+                    lemma = token.lower()
+                stemmed = stemmer.stem(lemma)
+                lemmas.append((token.lower(), stemmed))
+                pos_tags[token.lower()] = pos_tag
+            if debug:
+                logger.info(f"POS Tagging Results (NLTK): {pos_tags}")
+        except Exception as pos_error:
+            # Fallback if POS tagging fails completely
+            logger.warning(f"NLTK POS tagging failed: {pos_error}. Using basic token processing.")
+            for token in tokens:
+                lemma = token.lower()
+                stemmed = stemmer.stem(lemma)
+                lemmas.append((token.lower(), stemmed))
+                pos_tags[token.lower()] = 'NN'  # Default to noun
+            if debug:
+                logger.info(f"Using fallback token processing for: {tokens}")
 
     # Expanding query with synonyms
     expanded_query_set = set()

signalwire_agents/skills/datasphere_serverless/skill.py

@@ -83,7 +83,6 @@ class DataSphereServerlessSkill(SkillBase):
         webhook_params = {
             "document_id": self.document_id,
             "query_string": "${args.query}",  # Only this is dynamic from user input
-            "distance": self.distance,
             "count": self.count
         }
         

signalwire_agents/skills/native_vector_search/skill.py

@@ -41,18 +41,7 @@ class NativeVectorSearchSkill(SkillBase):
     def setup(self) -> bool:
         """Setup the native vector search skill"""
         
-        # Check if search functionality is available
-        try:
-            from signalwire_agents.search import IndexBuilder, SearchEngine
-            from signalwire_agents.search.query_processor import preprocess_query
-            self.search_available = True
-        except ImportError as e:
-            self.search_available = False
-            self.import_error = str(e)
-            self.logger.warning(f"Search dependencies not available: {e}")
-            # Don't fail setup - we'll provide helpful error messages at runtime
-        
-        # Get configuration
+        # Get configuration first
         self.tool_name = self.params.get('tool_name', 'search_knowledge')
         self.index_file = self.params.get('index_file')
         self.build_index = self.params.get('build_index', False)
@@ -74,7 +63,34 @@ class NativeVectorSearchSkill(SkillBase):
         # SWAIG fields for function fillers
         self.swaig_fields = self.params.get('swaig_fields', {})
         
-        # NLP backend configuration
+        # **EARLY REMOTE CHECK - Option 1**
+        # If remote URL is configured, skip all heavy local imports and just validate remote connectivity
+        if self.remote_url:
+            self.use_remote = True
+            self.search_engine = None  # No local search engine needed
+            self.logger.info(f"Using remote search server: {self.remote_url}")
+            
+            # Test remote connection (lightweight check)
+            try:
+                import requests
+                response = requests.get(f"{self.remote_url}/health", timeout=5)
+                if response.status_code == 200:
+                    self.logger.info("Remote search server is available")
+                    self.search_available = True
+                    return True  # Success - skip all local setup
+                else:
+                    self.logger.error(f"Remote search server returned status {response.status_code}")
+                    self.search_available = False
+                    return False
+            except Exception as e:
+                self.logger.error(f"Failed to connect to remote search server: {e}")
+                self.search_available = False
+                return False
+        
+        # **LOCAL MODE SETUP - Only when no remote URL**
+        self.use_remote = False
+        
+        # NLP backend configuration (only needed for local mode)
         self.nlp_backend = self.params.get('nlp_backend')  # Backward compatibility
         self.index_nlp_backend = self.params.get('index_nlp_backend', 'nltk')  # Default to fast NLTK for indexing
         self.query_nlp_backend = self.params.get('query_nlp_backend', 'nltk')  # Default to fast NLTK for search
@@ -95,6 +111,17 @@ class NativeVectorSearchSkill(SkillBase):
             self.logger.warning(f"Invalid query_nlp_backend '{self.query_nlp_backend}', using 'nltk'")
             self.query_nlp_backend = 'nltk'
         
+        # Check if local search functionality is available (heavy imports only for local mode)
+        try:
+            from signalwire_agents.search import IndexBuilder, SearchEngine
+            from signalwire_agents.search.query_processor import preprocess_query
+            self.search_available = True
+        except ImportError as e:
+            self.search_available = False
+            self.import_error = str(e)
+            self.logger.warning(f"Search dependencies not available: {e}")
+            # Don't fail setup - we'll provide helpful error messages at runtime
+        
         # Auto-build index if requested and search is available
         if self.build_index and self.source_dir and self.search_available:
             if not self.index_file:
@@ -124,7 +151,7 @@ class NativeVectorSearchSkill(SkillBase):
                     self.logger.error(f"Failed to build search index: {e}")
                     self.search_available = False
         
-        # Initialize search engine
+        # Initialize local search engine
         self.search_engine = None
         if self.search_available and self.index_file and os.path.exists(self.index_file):
             try:
@@ -134,24 +161,6 @@ class NativeVectorSearchSkill(SkillBase):
                 self.logger.error(f"Failed to load search index {self.index_file}: {e}")
                 self.search_available = False
         
-        # Check if we should use remote search mode
-        self.use_remote = bool(self.remote_url)
-        if self.use_remote:
-            self.logger.info(f"Using remote search server: {self.remote_url}")
-            # Test remote connection
-            try:
-                import requests
-                response = requests.get(f"{self.remote_url}/health", timeout=5)
-                if response.status_code == 200:
-                    self.logger.info("Remote search server is available")
-                    self.search_available = True
-                else:
-                    self.logger.error(f"Remote search server returned status {response.status_code}")
-                    self.search_available = False
-            except Exception as e:
-                self.logger.error(f"Failed to connect to remote search server: {e}")
-                self.search_available = False
-        
         return True
         
     def register_tools(self) -> None:
@@ -184,6 +193,11 @@ class NativeVectorSearchSkill(SkillBase):
     def _search_handler(self, args, raw_data):
         """Handle search requests"""
         
+        # Debug logging to see what arguments are being passed
+        self.logger.info(f"Search handler called with args: {args}")
+        self.logger.info(f"Args type: {type(args)}")
+        self.logger.info(f"Raw data: {raw_data}")
+        
         if not self.search_available:
             return SwaigFunctionResult(
                 f"Search functionality is not available. {getattr(self, 'import_error', '')}\n"
@@ -196,21 +210,27 @@ class NativeVectorSearchSkill(SkillBase):
                 f"{'Index file not found: ' + (self.index_file or 'not specified') if self.index_file else 'No index file configured'}"
             )
         
+        # Get arguments - the framework handles parsing correctly
         query = args.get('query', '').strip()
+        self.logger.error(f"DEBUG: Extracted query: '{query}' (length: {len(query)})")
+        self.logger.info(f"Query bool value: {bool(query)}")
+        
         if not query:
+            self.logger.error(f"Query validation failed - returning error message")
             return SwaigFunctionResult("Please provide a search query.")
         
+        self.logger.info(f"Query validation passed - proceeding with search")
         count = args.get('count', self.count)
         
         try:
-            # Preprocess the query
-            from signalwire_agents.search.query_processor import preprocess_query
-            enhanced = preprocess_query(query, language='en', vector=True, query_nlp_backend=self.query_nlp_backend)
-            
             # Perform search (local or remote)
             if self.use_remote:
-                results = self._search_remote(query, enhanced, count)
+                # For remote searches, let the server handle query preprocessing
+                results = self._search_remote(query, None, count)
             else:
+                # For local searches, preprocess the query locally
+                from signalwire_agents.search.query_processor import preprocess_query
+                enhanced = preprocess_query(query, language='en', vector=True, query_nlp_backend=self.query_nlp_backend)
                 results = self.search_engine.search(
                     query_vector=enhanced.get('vector', []),
                     enhanced_text=enhanced['enhanced_text'],
@@ -256,7 +276,24 @@ class NativeVectorSearchSkill(SkillBase):
             return SwaigFunctionResult("\n".join(response_parts))
             
         except Exception as e:
-            return SwaigFunctionResult(f"Search error: {str(e)}")
+            # Log the full error details for debugging
+            self.logger.error(f"Search error for query '{query}': {str(e)}", exc_info=True)
+            
+            # Return user-friendly error message
+            user_msg = "I'm sorry, I encountered an issue while searching. "
+            
+            # Check for specific error types and provide helpful guidance
+            error_str = str(e).lower()
+            if 'punkt' in error_str or 'nltk' in error_str:
+                user_msg += "It looks like some language processing resources are missing. Please try again in a moment."
+            elif 'vector' in error_str or 'embedding' in error_str:
+                user_msg += "There was an issue with the search indexing. Please try rephrasing your question."
+            elif 'timeout' in error_str or 'connection' in error_str:
+                user_msg += "The search service is temporarily unavailable. Please try again later."
+            else:
+                user_msg += "Please try rephrasing your question or contact support if the issue persists."
+                
+            return SwaigFunctionResult(user_msg)
     
     def _search_remote(self, query: str, enhanced: dict, count: int) -> list:
         """Perform search using remote search server"""

{signalwire_agents-0.1.18.dist-info → signalwire_agents-0.1.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: signalwire_agents
-Version: 0.1.18
+Version: 0.1.20
 Summary: SignalWire AI Agents SDK
 Author-email: SignalWire Team <info@signalwire.com>
 Project-URL: Homepage, https://github.com/signalwire/signalwire-ai-agents
@@ -26,6 +26,11 @@ Requires-Dist: structlog==25.3.0
 Requires-Dist: uvicorn==0.34.2
 Requires-Dist: beautifulsoup4==4.12.3
 Requires-Dist: pytz==2023.3
+Provides-Extra: search-queryonly
+Requires-Dist: numpy>=1.24.0; extra == "search-queryonly"
+Requires-Dist: scikit-learn>=1.3.0; extra == "search-queryonly"
+Requires-Dist: sentence-transformers>=2.2.0; extra == "search-queryonly"
+Requires-Dist: nltk>=3.8; extra == "search-queryonly"
 Provides-Extra: search
 Requires-Dist: sentence-transformers>=2.2.0; extra == "search"
 Requires-Dist: scikit-learn>=1.3.0; extra == "search"
@@ -488,7 +493,10 @@ The SDK includes optional local search capabilities that can be installed separa
 #### Search Installation Options
 
 ```bash
-# Basic search (vector search + keyword search)
+# Query existing .swsearch files only (smallest footprint)
+pip install signalwire-agents[search-queryonly]
+
+# Basic search (vector search + keyword search + building indexes)
 pip install signalwire-agents[search]
 
 # Full search with document processing (PDF, DOCX, etc.)
@@ -505,11 +513,18 @@ pip install signalwire-agents[search-all]
 
 | Option | Size | Features |
 |--------|------|----------|
+| `search-queryonly` | ~400MB | Query existing .swsearch files only (no building/processing) |
 | `search` | ~500MB | Vector embeddings, keyword search, basic text processing |
 | `search-full` | ~600MB | + PDF, DOCX, Excel, PowerPoint, HTML, Markdown processing |
 | `search-nlp` | ~600MB | + Advanced spaCy NLP features |
 | `search-all` | ~700MB | All search features combined |
 
+**When to use `search-queryonly`:**
+- Production containers with pre-built `.swsearch` files
+- Lambda/serverless deployments
+- Agents that only need to query knowledge bases (not build them)
+- Smaller deployment footprint requirements
+
 #### Search Features
 
 - **Local/Offline Search**: No external API dependencies

{signalwire_agents-0.1.18.dist-info → signalwire_agents-0.1.20.dist-info}/RECORD

@@ -1,9 +1,9 @@
-signalwire_agents/__init__.py,sha256=3YN2p2ynw-LH_LXSsuM52E0U5fCBzjAoxlOaQ7H1bvA,2707
+signalwire_agents/__init__.py,sha256=Y9kuYxQVXPYUbHXioNln_a1vqo6wUCdzdJWt3Hd_BPs,2707
 signalwire_agents/agent_server.py,sha256=3Or8rIMAqW750V-XitBUMgOpW9BAIXmKXoGq7LkejAA,24988
 signalwire_agents/schema.json,sha256=M8Mn6pQda2P9jhbmkALrLr1wt-fRuhYRqdmEi9Rbhqk,178075
 signalwire_agents/cli/__init__.py,sha256=Iy2BfWDWBEZoA1cyHTDsooBSVMx4vH5Ddhr3sEuFe8c,197
 signalwire_agents/cli/build_search.py,sha256=KLQJBqVSADFyGcKAi0KLwU_UoUd5rtRoKAdfcH91u70,28652
-signalwire_agents/cli/test_swaig.py,sha256=CqfdBWE8_fiFnFWHbv9ouU1eUA_rGUUHwyOf_iUaZFU,101851
+signalwire_agents/cli/test_swaig.py,sha256=IWjFULRM-KhYanwdVfVZalnLBkwxHTLcffggYNkfATI,109758
 signalwire_agents/core/__init__.py,sha256=mVDLbpq1pg_WwiqsQR28NNZwJ6-VUXFIfg-vN7pk0ew,806
 signalwire_agents/core/agent_base.py,sha256=O-sQ9k6L8_qE94Xzpds3bSS52onxqVs8rYU9_1B0LV8,159349
 signalwire_agents/core/contexts.py,sha256=RQGIZGR92AC90nv8_hGa8S4YGY-SmMa1giR4g7jLgQI,21283
@@ -31,8 +31,8 @@ signalwire_agents/prefabs/receptionist.py,sha256=8EyQ6M0Egs3g7KKWukHFiO9UPoVUxT4
 signalwire_agents/prefabs/survey.py,sha256=IIIQfgvMlfVNjEEEdWUn4lAJqVsCDlBsIAkOJ1ckyAE,14796
 signalwire_agents/search/__init__.py,sha256=x7saU_MDbhoOIzcvCT1-gnqyH2rrMpzB4ZUqk-av-lI,3958
 signalwire_agents/search/document_processor.py,sha256=J4OG640qbqGslbVevvD4J2cbTmFCZiGJ1bLX2yDayaE,43699
-signalwire_agents/search/index_builder.py,sha256=OLITsAidBJ_tkTtl7JAMVF-v4lUxaAmLC5pdn0SdpAE,22816
-signalwire_agents/search/query_processor.py,sha256=idQTGWO87I4IT5mrbGAuC3AvBiYNNj7Dx22KEv6YhV0,14668
+signalwire_agents/search/index_builder.py,sha256=kPnM62qBEV6gunBr50EFEBrTM8Oy0-L3hYAEScb63bM,25190
+signalwire_agents/search/query_processor.py,sha256=WMm_jjArQ6-Jpy0Cc0sUI4saidOtDRKx_XLv0qi3N3k,16739
 signalwire_agents/search/search_engine.py,sha256=KFF33aPsBdwb2N1aTJmAA0xo3KPANoXjwV53uSxih9o,14798
 signalwire_agents/search/search_service.py,sha256=Src_REgjkcddHBDCaLv2BMJnDSxcE4XV_1U8YStJOh8,8719
 signalwire_agents/skills/__init__.py,sha256=j0HambEKTkrFNP2vsaSmfrMJNDabq4EuvYJqRKZXfxU,342
@@ -40,7 +40,7 @@ signalwire_agents/skills/registry.py,sha256=lnr0XFOQ5YC_WgsAT6Id3RMAJ2-nf2ZCdqB7
 signalwire_agents/skills/datasphere/__init__.py,sha256=SJJlmeMSeezjINPgkuWN1XzDPN_Z3GzZ_StzO1BtxQs,257
 signalwire_agents/skills/datasphere/skill.py,sha256=L6GrGwej3sKPcHljKBNf4it5g4DaGzR18KlQx65_XKg,9598
 signalwire_agents/skills/datasphere_serverless/__init__.py,sha256=65hu8_0eqiczLSZ-aJgASpMQqTUjzTQUI1fC8GI7qTI,70
-signalwire_agents/skills/datasphere_serverless/skill.py,sha256=C3wynBZT2qCezPwb1_F8d37die98bMTNWX8mz5ugiE4,6826
+signalwire_agents/skills/datasphere_serverless/skill.py,sha256=zgEoTY8Jm6YKJBzM1kn3j7tf492K-NiKG7DbE3GReeE,6787
 signalwire_agents/skills/datetime/__init__.py,sha256=coPaY-k2EyZWuYckGunhSJ65Y1Jwz66h-iOo0QWb5WY,43
 signalwire_agents/skills/datetime/skill.py,sha256=aBBdcPV5xWX6uWi9W0VqZ0kKOqC3JHrAhG1bZn58ro8,3900
 signalwire_agents/skills/joke/__init__.py,sha256=R-iS9UMMvOdpkxL9aooVik16eCddJw14Rz4PmFqCdsM,53
@@ -48,7 +48,7 @@ signalwire_agents/skills/joke/skill.py,sha256=AFaf6fMy0sxUPJHvcnf3CWMuPqpJP4ODsc
 signalwire_agents/skills/math/__init__.py,sha256=lGAFWEmJH2fuwkuZUdDTY5dmucrIwtjfNT8bE2hOSP8,39
 signalwire_agents/skills/math/skill.py,sha256=5sErd5x1rFHJg2GlmdJB3LvrmvTNOrZsA2jRnG67Zw8,3342
 signalwire_agents/skills/native_vector_search/__init__.py,sha256=buvncVoH5u8MJA0SLlz1JQgIuyBTQW5aql-ydnc7Wh8,29
-signalwire_agents/skills/native_vector_search/skill.py,sha256=da-3eM1NT7Tcju1GiQipxOrYhP7OIry68l2dLKpDV4I,15859
+signalwire_agents/skills/native_vector_search/skill.py,sha256=6rIsyoXPAuzl_C550On2Hp0mQm5a5c8q7yiePjowefQ,18161
 signalwire_agents/skills/web_search/__init__.py,sha256=wJlptYDExYw-nxZJVzlTLOgkKkDOLUUt1ZdoLt44ixs,45
 signalwire_agents/skills/web_search/skill.py,sha256=6EwoNABxEH5UkEdXsPT72PQzoVlFUbWsFJR6NuyhglI,10363
 signalwire_agents/skills/wikipedia_search/__init__.py,sha256=8Db_aE0ly7QoXg7n2RDvCqKupkyR-UYlK9uFUnGNCE8,184
@@ -58,10 +58,10 @@ signalwire_agents/utils/pom_utils.py,sha256=4Mr7baQ_xR_hfJ72YxQRAT_GFa663YjFX_Pu
 signalwire_agents/utils/schema_utils.py,sha256=i4okv_O9bUApwT_jJf4Yoij3bLCrGrW3DC-vzSy2RuY,16392
 signalwire_agents/utils/token_generators.py,sha256=4Mr7baQ_xR_hfJ72YxQRAT_GFa663YjFX_PumJ35Xds,191
 signalwire_agents/utils/validators.py,sha256=4Mr7baQ_xR_hfJ72YxQRAT_GFa663YjFX_PumJ35Xds,191
-signalwire_agents-0.1.18.data/data/schema.json,sha256=M8Mn6pQda2P9jhbmkALrLr1wt-fRuhYRqdmEi9Rbhqk,178075
-signalwire_agents-0.1.18.dist-info/licenses/LICENSE,sha256=NYvAsB-rTcSvG9cqHt9EUHAWLiA9YzM4Qfz-mPdvDR0,1067
-signalwire_agents-0.1.18.dist-info/METADATA,sha256=OdI3xK6tg17jbWrZkWhAN4hsK2G67oG4VYSQ5BVTIMA,35364
-signalwire_agents-0.1.18.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-signalwire_agents-0.1.18.dist-info/entry_points.txt,sha256=LRwltbVfaKUFMYmQoMxJGTT_-iQm0ftzXK0xPfD64Is,138
-signalwire_agents-0.1.18.dist-info/top_level.txt,sha256=kDGS6ZYv84K9P5Kyg9_S8P_pbUXoHkso0On_DB5bbWc,18
-signalwire_agents-0.1.18.dist-info/RECORD,,
+signalwire_agents-0.1.20.data/data/schema.json,sha256=M8Mn6pQda2P9jhbmkALrLr1wt-fRuhYRqdmEi9Rbhqk,178075
+signalwire_agents-0.1.20.dist-info/licenses/LICENSE,sha256=NYvAsB-rTcSvG9cqHt9EUHAWLiA9YzM4Qfz-mPdvDR0,1067
+signalwire_agents-0.1.20.dist-info/METADATA,sha256=7EJOLAnXfsbnlBjo_2iP3Hzt_7Lg0RyRU4_8QawL2yM,36104
+signalwire_agents-0.1.20.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+signalwire_agents-0.1.20.dist-info/entry_points.txt,sha256=7RBxC9wwFdsqP7g1F4JzsJ3AIHsjGtb3h0mxEGmANlI,151
+signalwire_agents-0.1.20.dist-info/top_level.txt,sha256=kDGS6ZYv84K9P5Kyg9_S8P_pbUXoHkso0On_DB5bbWc,18
+signalwire_agents-0.1.20.dist-info/RECORD,,

{signalwire_agents-0.1.18.dist-info → signalwire_agents-0.1.20.dist-info}/entry_points.txt

@@ -1,3 +1,3 @@
 [console_scripts]
-sw-search = signalwire_agents.cli.sw_search_fast:main
+sw-search = signalwire_agents.cli.build_search:console_entry_point
 swaig-test = signalwire_agents.cli.test_swaig:console_entry_point