cicada-mcp 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

cicada/indexer.py

@@ -11,7 +11,14 @@ import sys
 from datetime import datetime
 from pathlib import Path
 
+from cicada.dependency_analyzer import (
+    calculate_function_end_line,
+    extract_function_dependencies,
+    extract_module_dependencies,
+)
+from cicada.git_helper import GitHelper
 from cicada.parser import ElixirParser
+from cicada.tier import read_keyword_extraction_config
 from cicada.utils import (
     load_index,
     merge_indexes_incremental,
@@ -24,42 +31,7 @@ from cicada.utils.hash_utils import (
     load_file_hashes,
     save_file_hashes,
 )
-from cicada.utils.storage import get_config_path
-
-
-def read_keyword_extraction_config(repo_path: Path) -> tuple[str, str]:
-    """
-    Read keyword extraction configuration from config.yaml.
-
-    Args:
-        repo_path: Path to the repository
-
-    Returns:
-        tuple[str, str]: (method, tier) where method is 'lemminflect' or 'bert',
-                        and tier is 'fast', 'regular', or 'max'.
-                        Returns ('lemminflect', 'regular') as default if config not found.
-    """
-    try:
-        import yaml
-
-        config_path = get_config_path(repo_path)
-        if not config_path.exists():
-            # Default to lemminflect if config doesn't exist
-            return ("lemminflect", "regular")
-
-        with open(config_path) as f:
-            config = yaml.safe_load(f)
-
-        if config and "keyword_extraction" in config:
-            method = config["keyword_extraction"].get("method", "lemminflect")
-            tier = config["keyword_extraction"].get("tier", "regular")
-            return (method, tier)
-
-        # Default to lemminflect if keyword_extraction section not found
-        return ("lemminflect", "regular")
-    except Exception:
-        # If anything goes wrong, default to lemminflect
-        return ("lemminflect", "regular")
+from cicada.version_check import get_version_string, version_mismatch
 
 
 class ElixirIndexer:
@@ -68,6 +40,10 @@ class ElixirIndexer:
     # Progress reporting interval - report every N files processed
     PROGRESS_REPORT_INTERVAL = 10
 
+    # Keyword expansion parameters
+    DEFAULT_EXPANSION_TOP_N = 3
+    DEFAULT_EXPANSION_THRESHOLD = 0.2
+
     def __init__(self, verbose: bool = False):
         """Initialize the indexer with a parser."""
         self.parser = ElixirParser()
@@ -82,6 +58,33 @@ class ElixirIndexer:
         }
         self._interrupted = False
 
+    def _extract_dependencies(self, module_data: dict, functions: list) -> tuple[dict, list]:
+        """
+        Extract module and function level dependencies.
+
+        Args:
+            module_data: Parsed module data containing calls, aliases, etc.
+            functions: List of function data dictionaries
+
+        Returns:
+            Tuple of (module_dependencies, modified_functions_list)
+        """
+        # Extract module-level dependencies
+        module_dependencies = extract_module_dependencies(module_data)
+
+        # Extract function-level dependencies
+        all_calls = module_data.get("calls", [])
+        for i, func in enumerate(functions):
+            # Calculate function end line
+            next_func_line = functions[i + 1]["line"] if i + 1 < len(functions) else None
+            func_end_line = calculate_function_end_line(func, next_func_line)
+
+            # Extract dependencies for this function
+            func_deps = extract_function_dependencies(module_data, func, all_calls, func_end_line)
+            func["dependencies"] = func_deps
+
+        return module_dependencies, functions
+
     def _handle_interrupt(self, _signum, _frame):
         """Handle interrupt signals (Ctrl-C, SIGTERM) gracefully."""
         print("\n\n⚠️  Interrupt received. Finishing current file and saving progress...")
@@ -113,6 +116,7 @@ class ElixirIndexer:
         repo_path: str,
         output_path: str,
         extract_keywords: bool = False,
+        compute_timestamps: bool = False,
     ):
         """
         Index an Elixir repository.
@@ -121,6 +125,7 @@ class ElixirIndexer:
             repo_path: Path to the Elixir repository root
             output_path: Path where the index JSON file will be saved
             extract_keywords: If True, extract keywords from documentation using NLP
+            compute_timestamps: If True, compute git history timestamps for functions
 
         Returns:
             Dictionary containing the index data
@@ -134,39 +139,61 @@ class ElixirIndexer:
             print(f"Indexing repository: {repo_path_obj}")
             if extract_keywords:
                 # Read and display keyword extraction config
-                method, tier = read_keyword_extraction_config(repo_path_obj)
-                print(f"Keyword extraction: {method.upper()} ({tier})")
+                extraction_method, expansion_method = read_keyword_extraction_config(repo_path_obj)
+                print(
+                    f"Keyword extraction: {extraction_method.upper()} + {expansion_method.upper()}"
+                )
 
         # Set up signal handlers for graceful interruption
         signal.signal(signal.SIGINT, self._handle_interrupt)
         signal.signal(signal.SIGTERM, self._handle_interrupt)
         self._interrupted = False
 
-        # Initialize keyword extractor if requested
+        # Initialize keyword extractor and expander if requested
         keyword_extractor = None
+        keyword_expander = None
         if extract_keywords:
             try:
                 # Read keyword extraction config from config.yaml
-                method, tier = read_keyword_extraction_config(repo_path_obj)
+                extraction_method, expansion_method = read_keyword_extraction_config(repo_path_obj)
 
-                if method == "bert":
-                    # Initialize KeyBERT extractor
-                    from cicada.keybert_extractor import KeyBERTExtractor
+                # Initialize extraction method
+                if extraction_method == "bert":
+                    from cicada.extractors.keybert import KeyBERTExtractor
 
-                    keyword_extractor = KeyBERTExtractor(model_tier=tier, verbose=self.verbose)
+                    keyword_extractor = KeyBERTExtractor(verbose=self.verbose)
                 else:
-                    # Initialize lemminflect extractor (default)
-                    from cicada.lightweight_keyword_extractor import (
-                        LightweightKeywordExtractor,
-                    )
+                    # Use regular (TF-based) extractor as default
+                    from cicada.extractors.keyword import RegularKeywordExtractor
+
+                    keyword_extractor = RegularKeywordExtractor(verbose=self.verbose)
+
+                # Initialize expansion method
+                from cicada.keyword_expander import KeywordExpander
+
+                keyword_expander = KeywordExpander(
+                    expansion_type=expansion_method, verbose=self.verbose
+                )
 
-                    keyword_extractor = LightweightKeywordExtractor(verbose=self.verbose)
             except Exception as e:
                 if self.verbose:
-                    print(f"Warning: Could not initialize keyword extractor: {e}")
+                    print(f"Warning: Could not initialize keyword extractor/expander: {e}")
                     print("Continuing without keyword extraction...")
                 extract_keywords = False
 
+        # Initialize git helper if timestamps are requested
+        git_helper = None
+        if compute_timestamps:
+            try:
+                git_helper = GitHelper(str(repo_path_obj))
+                if self.verbose:
+                    print("Git history tracking enabled - computing function timestamps")
+            except Exception as e:
+                if self.verbose:
+                    print(f"Warning: Could not initialize git helper: {e}")
+                    print("Continuing without timestamp computation...")
+                compute_timestamps = False
+
         # Find all Elixir files
         elixir_files = self._find_elixir_files(repo_path_obj)
         total_files = len(elixir_files)
@@ -193,13 +220,48 @@ class ElixirIndexer:
                         public_count = sum(1 for f in functions if f["type"] == "def")
                         private_count = sum(1 for f in functions if f["type"] == "defp")
 
-                        # Extract keywords if enabled
+                        # Extract and expand keywords if enabled
                         module_keywords = None
                         if keyword_extractor and module_data.get("moduledoc"):
                             try:
-                                module_keywords = keyword_extractor.extract_keywords_simple(
+                                # Step 1: Extract keywords with scores
+                                extraction_result = keyword_extractor.extract_keywords(
                                     module_data["moduledoc"], top_n=10
                                 )
+                                extracted_keywords = [
+                                    kw for kw, _ in extraction_result["top_keywords"]
+                                ]
+                                keyword_scores = {
+                                    kw.lower(): score
+                                    for kw, score in extraction_result["top_keywords"]
+                                }
+
+                                # Step 2: Expand keywords with scores
+                                if keyword_expander and extracted_keywords:
+                                    expansion_result = keyword_expander.expand_keywords(
+                                        extracted_keywords,
+                                        top_n=self.DEFAULT_EXPANSION_TOP_N,
+                                        threshold=self.DEFAULT_EXPANSION_THRESHOLD,
+                                        return_scores=True,
+                                        keyword_scores=keyword_scores,
+                                    )
+                                    # Convert to dict: word -> max_score
+                                    module_keywords = {}
+                                    # When return_scores=True, expansion_result is a dict
+                                    if not isinstance(expansion_result, dict):
+                                        raise TypeError(
+                                            "Expected dict from expand_keywords with return_scores=True"
+                                        )
+                                    for item in expansion_result["words"]:
+                                        word = item["word"]
+                                        score = item["score"]
+                                        if (
+                                            word not in module_keywords
+                                            or score > module_keywords[word]
+                                        ):
+                                            module_keywords[word] = score
+                                else:
+                                    module_keywords = keyword_scores
                             except Exception as e:
                                 keyword_extraction_failures += 1
                                 if self.verbose:
@@ -208,27 +270,97 @@ class ElixirIndexer:
                                         file=sys.stderr,
                                     )
 
-                        # Extract keywords from function docs
-                        if keyword_extractor:
-                            for func in functions:
-                                if func.get("doc"):
-                                    func_name = func.get("name", "")
-                                    try:
-                                        # Include function name in text for keyword extraction
-                                        # This ensures the function name identifier gets 10x weight
-                                        text_for_keywords = f"{func_name} {func['doc']}"
-                                        func_keywords = keyword_extractor.extract_keywords_simple(
-                                            text_for_keywords, top_n=10
+                        # Enrich function metadata (keywords and timestamps)
+                        for func in functions:
+                            func_name = func.get("name", "")
+
+                            # Extract and expand keywords from function docs
+                            if keyword_extractor and func.get("doc"):
+                                try:
+                                    # Include function name in text for keyword extraction
+                                    # This ensures the function name identifier gets 10x weight
+                                    text_for_keywords = f"{func_name} {func['doc']}"
+                                    # Step 1: Extract keywords with scores
+                                    extraction_result = keyword_extractor.extract_keywords(
+                                        text_for_keywords, top_n=10
+                                    )
+                                    extracted_keywords = [
+                                        kw for kw, _ in extraction_result["top_keywords"]
+                                    ]
+                                    keyword_scores = {
+                                        kw.lower(): score
+                                        for kw, score in extraction_result["top_keywords"]
+                                    }
+
+                                    # Step 2: Expand keywords with scores
+                                    if keyword_expander and extracted_keywords:
+                                        expansion_result = keyword_expander.expand_keywords(
+                                            extracted_keywords,
+                                            top_n=self.DEFAULT_EXPANSION_TOP_N,
+                                            threshold=self.DEFAULT_EXPANSION_THRESHOLD,
+                                            return_scores=True,
+                                            keyword_scores=keyword_scores,
                                         )
-                                        if func_keywords:
-                                            func["keywords"] = func_keywords
-                                    except Exception as e:
-                                        keyword_extraction_failures += 1
-                                        if self.verbose:
-                                            print(
-                                                f"Warning: Keyword extraction failed for {module_name}.{func_name}: {e}",
-                                                file=sys.stderr,
+                                        # Convert to dict: word -> max_score
+                                        func_keywords = {}
+                                        # When return_scores=True, expansion_result is a dict
+                                        if not isinstance(expansion_result, dict):
+                                            raise TypeError(
+                                                "Expected dict from expand_keywords with return_scores=True"
                                             )
+                                        for item in expansion_result["words"]:
+                                            word = item["word"]
+                                            score = item["score"]
+                                            if (
+                                                word not in func_keywords
+                                                or score > func_keywords[word]
+                                            ):
+                                                func_keywords[word] = score
+                                    else:
+                                        func_keywords = keyword_scores
+
+                                    if func_keywords:
+                                        func["keywords"] = func_keywords
+                                except Exception as e:
+                                    keyword_extraction_failures += 1
+                                    if self.verbose:
+                                        print(
+                                            f"Warning: Keyword extraction failed for {module_name}.{func_name}: {e}",
+                                            file=sys.stderr,
+                                        )
+
+                            # Compute git history timestamps if enabled
+                            if git_helper and func_name:
+                                try:
+                                    # Get function evolution metadata
+                                    evolution = git_helper.get_function_evolution(
+                                        file_path=str(file_path.relative_to(repo_path_obj)),
+                                        function_name=func_name,
+                                    )
+
+                                    if evolution:
+                                        # Add timestamp fields to function
+                                        func["created_at"] = evolution["created_at"]["date"]
+                                        func["last_modified_at"] = evolution["last_modified"][
+                                            "date"
+                                        ]
+                                        func["last_modified_sha"] = evolution["last_modified"][
+                                            "sha"
+                                        ]
+                                        func["modification_count"] = evolution[
+                                            "total_modifications"
+                                        ]
+                                except Exception as e:
+                                    if self.verbose:
+                                        print(
+                                            f"Warning: Could not compute timestamps for {module_name}.{func_name}: {e}",
+                                            file=sys.stderr,
+                                        )
+
+                        # Extract dependencies
+                        module_dependencies, functions = self._extract_dependencies(
+                            module_data, functions
+                        )
 
                         # Store module info
                         module_info = {
@@ -246,6 +378,7 @@ class ElixirIndexer:
                             "behaviours": module_data.get("behaviours", []),
                             "value_mentions": module_data.get("value_mentions", []),
                             "calls": module_data.get("calls", []),
+                            "dependencies": module_dependencies,
                         }
 
                         # Add module keywords if extracted
@@ -282,6 +415,7 @@ class ElixirIndexer:
                 "total_modules": len(all_modules),
                 "total_functions": total_functions,
                 "repo_path": str(repo_path_obj),
+                "cicada_version": get_version_string(),
             },
         }
 
@@ -390,6 +524,20 @@ class ElixirIndexer:
                     )
                 existing_index = None
 
+        # Check for version mismatch - if cicada version differs, force full reindex
+        if existing_index:
+            stored_version = existing_index.get("metadata", {}).get("cicada_version")
+            current_version = get_version_string()
+            if version_mismatch(stored_version, current_version):
+                if self.verbose:
+                    print(
+                        f"Warning: Cicada version mismatch. "
+                        f"Index was built with {stored_version}, current version is {current_version}. "
+                        f"Performing full reindex..."
+                    )
+                existing_index = None
+                existing_hashes = {}
+
         # If no existing data, do full index
         if not existing_index or not existing_hashes:
             if self.verbose:
@@ -398,9 +546,9 @@ class ElixirIndexer:
 
         if self.verbose:
             # Read and display keyword extraction config
-            method, tier = read_keyword_extraction_config(repo_path_obj)
+            extraction_method, expansion_method = read_keyword_extraction_config(repo_path_obj)
             print(f"Performing incremental index of: {repo_path_obj}")
-            print(f"Keyword extraction: {method.upper()} ({tier})")
+            print(f"Keyword extraction: {extraction_method.upper()} + {expansion_method.upper()}")
 
         # Set up signal handlers for graceful interruption
         signal.signal(signal.SIGINT, self._handle_interrupt)
@@ -436,27 +584,34 @@ class ElixirIndexer:
         if files_to_process:
             print(f"\nProcessing {len(files_to_process)} changed file(s)...")
 
-        # Initialize keyword extractor if requested
+        # Initialize keyword extractor and expander if requested
         keyword_extractor = None
+        keyword_expander = None
         if extract_keywords:
             try:
                 # Read keyword extraction config from config.yaml
-                method, tier = read_keyword_extraction_config(repo_path_obj)
+                extraction_method, expansion_method = read_keyword_extraction_config(repo_path_obj)
 
-                if method == "bert":
-                    # Initialize KeyBERT extractor
-                    from cicada.keybert_extractor import KeyBERTExtractor
+                # Initialize extraction method
+                if extraction_method == "bert":
+                    from cicada.extractors.keybert import KeyBERTExtractor
 
-                    keyword_extractor = KeyBERTExtractor(model_tier=tier, verbose=self.verbose)
+                    keyword_extractor = KeyBERTExtractor(verbose=self.verbose)
                 else:
-                    # Initialize lemminflect extractor (default)
-                    from cicada.lightweight_keyword_extractor import (
-                        LightweightKeywordExtractor,
-                    )
+                    # Use regular (TF-based) extractor as default
+                    from cicada.extractors.keyword import RegularKeywordExtractor
+
+                    keyword_extractor = RegularKeywordExtractor(verbose=self.verbose)
+
+                # Initialize expansion method
+                from cicada.keyword_expander import KeywordExpander
+
+                keyword_expander = KeywordExpander(
+                    expansion_type=expansion_method, verbose=self.verbose
+                )
 
-                    keyword_extractor = LightweightKeywordExtractor(verbose=self.verbose)
             except Exception as e:
-                print(f"Warning: Could not initialize keyword extractor: {e}")
+                print(f"Warning: Could not initialize keyword extractor/expander: {e}")
                 print("Continuing without keyword extraction...")
                 extract_keywords = False
 
@@ -480,31 +635,107 @@ class ElixirIndexer:
                         public_count = sum(1 for f in functions if f["type"] == "def")
                         private_count = sum(1 for f in functions if f["type"] == "defp")
 
-                        # Extract keywords if enabled
+                        # Extract and expand keywords if enabled
                         module_keywords = None
                         if keyword_extractor and module_data.get("moduledoc"):
                             try:
-                                module_keywords = keyword_extractor.extract_keywords_simple(
+                                # Step 1: Extract keywords with scores
+                                extraction_result = keyword_extractor.extract_keywords(
                                     module_data["moduledoc"], top_n=10
                                 )
+                                extracted_keywords = [
+                                    kw for kw, _ in extraction_result["top_keywords"]
+                                ]
+                                keyword_scores = {
+                                    kw.lower(): score
+                                    for kw, score in extraction_result["top_keywords"]
+                                }
+
+                                # Step 2: Expand keywords with scores
+                                if keyword_expander and extracted_keywords:
+                                    expansion_result = keyword_expander.expand_keywords(
+                                        extracted_keywords,
+                                        top_n=self.DEFAULT_EXPANSION_TOP_N,
+                                        threshold=self.DEFAULT_EXPANSION_THRESHOLD,
+                                        return_scores=True,
+                                        keyword_scores=keyword_scores,
+                                    )
+                                    # Convert to dict: word -> max_score
+                                    module_keywords = {}
+                                    # When return_scores=True, expansion_result is a dict
+                                    if not isinstance(expansion_result, dict):
+                                        raise TypeError(
+                                            "Expected dict from expand_keywords with return_scores=True"
+                                        )
+                                    for item in expansion_result["words"]:
+                                        word = item["word"]
+                                        score = item["score"]
+                                        if (
+                                            word not in module_keywords
+                                            or score > module_keywords[word]
+                                        ):
+                                            module_keywords[word] = score
+                                else:
+                                    module_keywords = keyword_scores
                             except Exception:
                                 keyword_extraction_failures += 1
 
-                        # Extract keywords from function docs
+                        # Extract and expand keywords from function docs
                         if keyword_extractor:
                             for func in functions:
                                 if func.get("doc"):
                                     try:
                                         func_name = func.get("name", "")
                                         text_for_keywords = f"{func_name} {func['doc']}"
-                                        func_keywords = keyword_extractor.extract_keywords_simple(
+                                        # Step 1: Extract keywords with scores
+                                        extraction_result = keyword_extractor.extract_keywords(
                                             text_for_keywords, top_n=10
                                         )
+                                        extracted_keywords = [
+                                            kw for kw, _ in extraction_result["top_keywords"]
+                                        ]
+                                        keyword_scores = {
+                                            kw.lower(): score
+                                            for kw, score in extraction_result["top_keywords"]
+                                        }
+
+                                        # Step 2: Expand keywords with scores
+                                        if keyword_expander and extracted_keywords:
+                                            expansion_result = keyword_expander.expand_keywords(
+                                                extracted_keywords,
+                                                top_n=self.DEFAULT_EXPANSION_TOP_N,
+                                                threshold=self.DEFAULT_EXPANSION_THRESHOLD,
+                                                return_scores=True,
+                                                keyword_scores=keyword_scores,
+                                            )
+                                            # Convert to dict: word -> max_score
+                                            func_keywords = {}
+                                            # When return_scores=True, expansion_result is a dict
+                                            if not isinstance(expansion_result, dict):
+                                                raise TypeError(
+                                                    "Expected dict from expand_keywords with return_scores=True"
+                                                )
+                                            for item in expansion_result["words"]:
+                                                word = item["word"]
+                                                score = item["score"]
+                                                if (
+                                                    word not in func_keywords
+                                                    or score > func_keywords[word]
+                                                ):
+                                                    func_keywords[word] = score
+                                        else:
+                                            func_keywords = keyword_scores
+
                                         if func_keywords:
                                             func["keywords"] = func_keywords
                                     except Exception:
                                         keyword_extraction_failures += 1
 
+                        # Extract dependencies
+                        module_dependencies, functions = self._extract_dependencies(
+                            module_data, functions
+                        )
+
                         # Store module info
                         module_info = {
                             "file": relative_file,
@@ -521,6 +752,7 @@ class ElixirIndexer:
                             "behaviours": module_data.get("behaviours", []),
                             "value_mentions": module_data.get("value_mentions", []),
                             "calls": module_data.get("calls", []),
+                            "dependencies": module_dependencies,
                         }
 
                         # Add module keywords if extracted
@@ -549,6 +781,7 @@ class ElixirIndexer:
             "metadata": {
                 "indexed_at": datetime.now().isoformat(),
                 "repo_path": str(repo_path_obj),
+                "cicada_version": get_version_string(),
             },
         }