mcp-code-indexer 1.1.5__py3-none-any.whl → 1.2.1__py3-none-any.whl

mcp_code_indexer/data/stop_words_english.txt

@@ -0,0 +1,851 @@
+able
+about
+above
+abroad
+according
+accordingly
+across
+actually
+adj
+after
+afterwards
+again
+against
+ago
+ahead
+ain't
+all
+allow
+allows
+almost
+alone
+along
+alongside
+already
+also
+although
+always
+am
+amid
+amidst
+among
+amongst
+an
+and
+another
+any
+anybody
+anyhow
+anyone
+anything
+anyway
+anyways
+anywhere
+apart
+appear
+appreciate
+appropriate
+are
+aren't
+around
+as
+a's
+aside
+ask
+asking
+associated
+at
+available
+away
+awfully
+back
+backward
+backwards
+be
+became
+because
+become
+becomes
+becoming
+been
+before
+beforehand
+begin
+behind
+being
+believe
+below
+beside
+besides
+best
+better
+between
+beyond
+both
+brief
+but
+by
+came
+can
+cannot
+cant
+can't
+caption
+cause
+causes
+certain
+certainly
+changes
+clearly
+c'mon
+co
+co.
+com
+come
+comes
+concerning
+consequently
+consider
+considering
+contain
+containing
+contains
+corresponding
+could
+couldn't
+course
+c's
+currently
+dare
+daren't
+definitely
+described
+despite
+did
+didn't
+different
+directly
+do
+does
+doesn't
+doing
+done
+don't
+down
+downwards
+during
+each
+edu
+eg
+eight
+eighty
+either
+else
+elsewhere
+end
+ending
+enough
+entirely
+especially
+et
+etc
+even
+ever
+evermore
+every
+everybody
+everyone
+everything
+everywhere
+ex
+exactly
+example
+except
+fairly
+far
+farther
+few
+fewer
+fifth
+first
+five
+followed
+following
+follows
+for
+forever
+former
+formerly
+forth
+forward
+found
+four
+from
+further
+furthermore
+get
+gets
+getting
+given
+gives
+go
+goes
+going
+gone
+got
+gotten
+greetings
+had
+hadn't
+half
+happens
+hardly
+has
+hasn't
+have
+haven't
+having
+he
+he'd
+he'll
+hello
+help
+hence
+her
+here
+hereafter
+hereby
+herein
+here's
+hereupon
+hers
+herself
+he's
+hi
+him
+himself
+his
+hither
+hopefully
+how
+howbeit
+however
+hundred
+i'd
+ie
+if
+ignored
+i'll
+i'm
+immediate
+in
+inasmuch
+inc
+inc.
+indeed
+indicate
+indicated
+indicates
+inner
+inside
+insofar
+instead
+into
+inward
+is
+isn't
+it
+it'd
+it'll
+its
+it's
+itself
+i've
+just
+k
+keep
+keeps
+kept
+know
+known
+knows
+last
+lately
+later
+latter
+latterly
+least
+less
+lest
+let
+let's
+like
+liked
+likely
+likewise
+little
+look
+looking
+looks
+low
+lower
+ltd
+made
+mainly
+make
+makes
+many
+may
+maybe
+mayn't
+me
+mean
+meantime
+meanwhile
+merely
+might
+mightn't
+mine
+minus
+miss
+more
+moreover
+most
+mostly
+mr
+mrs
+much
+must
+mustn't
+my
+myself
+name
+namely
+nd
+near
+nearly
+necessary
+need
+needn't
+needs
+neither
+never
+neverf
+neverless
+nevertheless
+new
+next
+nine
+ninety
+no
+nobody
+non
+none
+nonetheless
+noone
+no-one
+nor
+normally
+not
+nothing
+notwithstanding
+novel
+now
+nowhere
+obviously
+of
+off
+often
+oh
+ok
+okay
+old
+on
+once
+one
+ones
+one's
+only
+onto
+opposite
+or
+other
+others
+otherwise
+ought
+oughtn't
+our
+ours
+ourselves
+out
+outside
+over
+overall
+own
+particular
+particularly
+past
+per
+perhaps
+placed
+please
+plus
+possible
+presumably
+probably
+provided
+provides
+que
+quite
+qv
+rather
+rd
+re
+really
+reasonably
+recent
+recently
+regarding
+regardless
+regards
+relatively
+respectively
+right
+round
+said
+same
+saw
+say
+saying
+says
+second
+secondly
+see
+seeing
+seem
+seemed
+seeming
+seems
+seen
+self
+selves
+sensible
+sent
+serious
+seriously
+seven
+several
+shall
+shan't
+she
+she'd
+she'll
+she's
+should
+shouldn't
+since
+six
+so
+some
+somebody
+someday
+somehow
+someone
+something
+sometime
+sometimes
+somewhat
+somewhere
+soon
+sorry
+specified
+specify
+specifying
+still
+sub
+such
+sup
+sure
+take
+taken
+taking
+tell
+tends
+th
+than
+thank
+thanks
+thanx
+that
+that'll
+thats
+that's
+that've
+the
+their
+theirs
+them
+themselves
+then
+thence
+there
+thereafter
+thereby
+there'd
+therefore
+therein
+there'll
+there're
+theres
+there's
+thereupon
+there've
+these
+they
+they'd
+they'll
+they're
+they've
+thing
+things
+think
+third
+thirty
+this
+thorough
+thoroughly
+those
+though
+three
+through
+throughout
+thru
+thus
+till
+to
+together
+too
+took
+toward
+towards
+tried
+tries
+truly
+try
+trying
+t's
+twice
+two
+un
+under
+underneath
+undoing
+unfortunately
+unless
+unlike
+unlikely
+until
+unto
+up
+upon
+upwards
+us
+use
+used
+useful
+uses
+using
+usually
+v
+value
+various
+versus
+very
+via
+viz
+vs
+want
+wants
+was
+wasn't
+way
+we
+we'd
+welcome
+well
+we'll
+went
+were
+we're
+weren't
+we've
+what
+whatever
+what'll
+what's
+what've
+when
+whence
+whenever
+where
+whereafter
+whereas
+whereby
+wherein
+where's
+whereupon
+wherever
+whether
+which
+whichever
+while
+whilst
+whither
+who
+who'd
+whoever
+whole
+who'll
+whom
+whomever
+who's
+whose
+why
+will
+willing
+wish
+with
+within
+without
+wonder
+won't
+would
+wouldn't
+yes
+yet
+you
+you'd
+you'll
+your
+you're
+yours
+yourself
+yourselves
+you've
+zero
+a
+how's
+i
+when's
+why's
+b
+c
+d
+e
+f
+g
+h
+j
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+uucp
+w
+x
+y
+z
+I
+www
+amount
+bill
+bottom
+call
+computer
+con
+couldnt
+cry
+de
+describe
+detail
+due
+eleven
+empty
+fifteen
+fifty
+fill
+find
+fire
+forty
+front
+full
+give
+hasnt
+herse
+himse
+interest
+itse”
+mill
+move
+myse”
+part
+put
+show
+side
+sincere
+sixty
+system
+ten
+thick
+thin
+top
+twelve
+twenty
+abst
+accordance
+act
+added
+adopted
+affected
+affecting
+affects
+ah
+announce
+anymore
+apparently
+approximately
+aren
+arent
+arise
+auth
+beginning
+beginnings
+begins
+biol
+briefly
+ca
+date
+ed
+effect
+et-al
+ff
+fix
+gave
+giving
+heres
+hes
+hid
+home
+id
+im
+immediately
+importance
+important
+index
+information
+invention
+itd
+keys
+kg
+km
+largely
+lets
+line
+'ll
+means
+mg
+million
+ml
+mug
+na
+nay
+necessarily
+nos
+noted
+obtain
+obtained
+omitted
+ord
+owing
+page
+pages
+poorly
+possibly
+potentially
+pp
+predominantly
+present
+previously
+primarily
+promptly
+proud
+quickly
+ran
+readily
+ref
+refs
+related
+research
+resulted
+resulting
+results
+run
+sec
+section
+shed
+shes
+showed
+shown
+showns
+shows
+significant
+significantly
+similar
+similarly
+slightly
+somethan
+specifically
+state
+states
+stop
+strongly
+substantially
+successfully
+sufficiently
+suggest
+thered
+thereof
+therere
+thereto
+theyd
+theyre
+thou
+thoughh
+thousand
+throug
+til
+tip
+ts
+ups
+usefully
+usefulness
+'ve
+vol
+vols
+wed
+whats
+wheres
+whim
+whod
+whos
+widely
+words
+world
+youd
+youre

mcp_code_indexer/database/database.py

@@ -17,7 +17,7 @@ import aiosqlite
 
 from mcp_code_indexer.database.models import (
     Project, FileDescription, MergeConflict, SearchResult,
-    CodebaseSizeInfo
+    CodebaseSizeInfo, ProjectOverview, WordFrequencyResult, WordFrequencyTerm
 )
 
 logger = logging.getLogger(__name__)
@@ -538,3 +538,157 @@ class DatabaseManager:
         # Check if project has any descriptions
         file_count = await self.get_file_count(project.id, "main")
         return file_count == 0
+    
+    # Project Overview operations
+    
+    async def create_project_overview(self, overview: ProjectOverview) -> None:
+        """Create or update a project overview."""
+        async with self.get_connection() as db:
+            await db.execute(
+                """
+                INSERT OR REPLACE INTO project_overviews 
+                (project_id, branch, overview, last_modified, total_files, total_tokens)
+                VALUES (?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    overview.project_id,
+                    overview.branch,
+                    overview.overview,
+                    overview.last_modified,
+                    overview.total_files,
+                    overview.total_tokens
+                )
+            )
+            await db.commit()
+            logger.debug(f"Created/updated overview for project {overview.project_id}, branch {overview.branch}")
+    
+    async def get_project_overview(self, project_id: str, branch: str) -> Optional[ProjectOverview]:
+        """Get project overview by ID and branch."""
+        async with self.get_connection() as db:
+            cursor = await db.execute(
+                "SELECT * FROM project_overviews WHERE project_id = ? AND branch = ?",
+                (project_id, branch)
+            )
+            row = await cursor.fetchone()
+            
+            if row:
+                return ProjectOverview(
+                    project_id=row['project_id'],
+                    branch=row['branch'],
+                    overview=row['overview'],
+                    last_modified=datetime.fromisoformat(row['last_modified']),
+                    total_files=row['total_files'],
+                    total_tokens=row['total_tokens']
+                )
+            return None
+    
+    async def cleanup_missing_files(self, project_id: str, branch: str, project_root: Path) -> List[str]:
+        """
+        Remove descriptions for files that no longer exist on disk.
+        
+        Args:
+            project_id: Project identifier
+            branch: Branch name
+            project_root: Path to project root directory
+            
+        Returns:
+            List of file paths that were cleaned up
+        """
+        removed_files = []
+        
+        async with self.get_connection() as db:
+            # Get all file descriptions for this project/branch
+            cursor = await db.execute(
+                "SELECT file_path FROM file_descriptions WHERE project_id = ? AND branch = ?",
+                (project_id, branch)
+            )
+            
+            rows = await cursor.fetchall()
+            
+            # Check which files no longer exist
+            to_remove = []
+            for row in rows:
+                file_path = row['file_path']
+                full_path = project_root / file_path
+                
+                if not full_path.exists():
+                    to_remove.append(file_path)
+                    removed_files.append(file_path)
+            
+            # Remove descriptions for missing files
+            if to_remove:
+                await db.executemany(
+                    "DELETE FROM file_descriptions WHERE project_id = ? AND branch = ? AND file_path = ?",
+                    [(project_id, branch, path) for path in to_remove]
+                )
+                await db.commit()
+                logger.info(f"Cleaned up {len(to_remove)} missing files from {project_id}/{branch}")
+        
+        return removed_files
+    
+    async def analyze_word_frequency(self, project_id: str, branch: str, limit: int = 200) -> WordFrequencyResult:
+        """
+        Analyze word frequency across all file descriptions for a project/branch.
+        
+        Args:
+            project_id: Project identifier
+            branch: Branch name
+            limit: Maximum number of top terms to return
+            
+        Returns:
+            WordFrequencyResult with top terms and statistics
+        """
+        from collections import Counter
+        import re
+        
+        # Load stop words from bundled file
+        stop_words_path = Path(__file__).parent.parent / "data" / "stop_words_english.txt"
+        stop_words = set()
+        
+        if stop_words_path.exists():
+            with open(stop_words_path, 'r', encoding='utf-8') as f:
+                for line in f:
+                    # Parse lines like "1: word" and extract just the word
+                    parts = line.strip().split(': ', 1)
+                    if len(parts) == 2:
+                        stop_words.add(parts[1].lower())
+        
+        # Add common programming keywords to stop words
+        programming_keywords = {
+            'if', 'else', 'for', 'while', 'do', 'break', 'continue', 'return',
+            'function', 'class', 'def', 'var', 'let', 'const', 'public', 'private',
+            'static', 'async', 'await', 'import', 'export', 'from', 'true', 'false',
+            'null', 'undefined', 'this', 'that', 'self', 'super', 'new', 'delete'
+        }
+        stop_words.update(programming_keywords)
+        
+        async with self.get_connection() as db:
+            # Get all descriptions for this project/branch
+            cursor = await db.execute(
+                "SELECT description FROM file_descriptions WHERE project_id = ? AND branch = ?",
+                (project_id, branch)
+            )
+            
+            rows = await cursor.fetchall()
+            
+            # Combine all descriptions
+            all_text = " ".join(row['description'] for row in rows)
+            
+            # Tokenize and filter
+            words = re.findall(r'\b[a-zA-Z]{2,}\b', all_text.lower())
+            filtered_words = [word for word in words if word not in stop_words]
+            
+            # Count frequencies
+            word_counts = Counter(filtered_words)
+            
+            # Create result
+            top_terms = [
+                WordFrequencyTerm(term=term, frequency=count)
+                for term, count in word_counts.most_common(limit)
+            ]
+            
+            return WordFrequencyResult(
+                top_terms=top_terms,
+                total_terms_analyzed=len(filtered_words),
+                total_unique_terms=len(word_counts)
+            )

mcp_code_indexer/database/models.py

@@ -62,6 +62,22 @@ class MergeConflict(BaseModel):
     created: datetime = Field(default_factory=datetime.utcnow, description="Creation timestamp")
 
 
+class ProjectOverview(BaseModel):
+    """
+    Represents a condensed, interpretive overview of an entire codebase.
+    
+    Stores a comprehensive narrative that captures architecture, components,
+    relationships, and design patterns in a single document rather than
+    individual file descriptions.
+    """
+    project_id: str = Field(..., description="Reference to project")
+    branch: str = Field(..., description="Git branch name")
+    overview: str = Field(..., description="Comprehensive codebase narrative")
+    last_modified: datetime = Field(default_factory=datetime.utcnow, description="Last update timestamp")
+    total_files: int = Field(..., description="Number of files in codebase")
+    total_tokens: int = Field(..., description="Total tokens in individual descriptions")
+
+
 class CodebaseOverview(BaseModel):
     """
     Represents a complete codebase structure with file descriptions.
@@ -116,6 +132,25 @@ class CodebaseSizeInfo(BaseModel):
     is_large: bool = Field(..., description="Whether codebase exceeds token limit")
     recommendation: str = Field(..., description="Recommended approach (use_search or use_overview)")
     token_limit: int = Field(..., description="Configured token limit")
+    cleaned_up_files: List[str] = Field(default_factory=list, description="Files removed during cleanup")
+    cleaned_up_count: int = Field(default=0, description="Number of files cleaned up")
+
+
+class WordFrequencyTerm(BaseModel):
+    """
+    Represents a term and its frequency from word analysis.
+    """
+    term: str = Field(..., description="The word/term")
+    frequency: int = Field(..., description="Number of occurrences")
+
+
+class WordFrequencyResult(BaseModel):
+    """
+    Results from word frequency analysis of file descriptions.
+    """
+    top_terms: List[WordFrequencyTerm] = Field(..., description="Top frequent terms")
+    total_terms_analyzed: int = Field(..., description="Total terms processed")
+    total_unique_terms: int = Field(..., description="Number of unique terms found")
 
 
 # Enable forward references for recursive models

mcp_code_indexer/server/mcp_server.py

@@ -26,7 +26,8 @@ from mcp_code_indexer.file_scanner import FileScanner
 from mcp_code_indexer.token_counter import TokenCounter
 from mcp_code_indexer.database.models import (
     Project, FileDescription, CodebaseOverview, SearchResult,
-    CodebaseSizeInfo, FolderNode, FileNode
+    CodebaseSizeInfo, FolderNode, FileNode, ProjectOverview,
+    WordFrequencyResult
 )
 from mcp_code_indexer.error_handler import setup_error_handling, ErrorHandler
 from mcp_code_indexer.middleware.error_middleware import create_tool_middleware, AsyncTaskManager
@@ -297,8 +298,8 @@ class MCPCodeIndexServer:
                     }
                 ),
                 types.Tool(
-                    name="get_codebase_overview",
-                    description="Returns the complete file and folder structure of a codebase with all descriptions. For large codebases, this will recommend using search_descriptions instead.",
+                    name="get_all_descriptions",
+                    description="Returns the complete file-by-file structure of a codebase with individual descriptions for each file. For large codebases, consider using get_codebase_overview for a condensed summary instead.",
                     inputSchema={
                         "type": "object",
                         "properties": {
@@ -338,6 +339,87 @@ class MCPCodeIndexServer:
                         },
                         "required": ["projectName", "folderPath", "sourceBranch", "targetBranch"]
                     }
+                ),
+                types.Tool(
+                    name="get_codebase_overview",
+                    description="Returns a condensed, interpretive overview of the entire codebase. This is a single comprehensive narrative that captures the architecture, key components, relationships, and design patterns. Unlike get_all_descriptions which lists every file, this provides a holistic view suitable for understanding the codebase's structure and purpose. If no overview exists, returns empty string.",
+                    inputSchema={
+                        "type": "object",
+                        "properties": {
+                            "projectName": {"type": "string", "description": "The name of the project"},
+                            "folderPath": {"type": "string", "description": "Absolute path to the project folder on disk"},
+                            "branch": {"type": "string", "description": "Git branch name"},
+                            "remoteOrigin": {"type": "string", "description": "Git remote origin URL if available"},
+                            "upstreamOrigin": {"type": "string", "description": "Upstream repository URL if this is a fork"}
+                        },
+                        "required": ["projectName", "folderPath", "branch"]
+                    }
+                ),
+                types.Tool(
+                    name="update_codebase_overview",
+                    description="""Updates the condensed codebase overview. Create a comprehensive narrative that would help a new developer understand this codebase. Include: (1) A visual directory tree showing the main folders and their purposes, (2) Overall architecture - how components fit together, (3) Core business logic and main workflows, (4) Key technical patterns and conventions used, (5) Important dependencies and integrations, (6) Database schema overview if applicable, (7) API structure if applicable, (8) Testing approach, (9) Build and deployment notes. Write in a clear, structured format with headers and sections. Be thorough but organized - imagine writing a technical onboarding document. The overview should be substantial (think 10-20 pages of text) but well-structured so specific sections can be found easily.
+
+Example Structure:
+
+````
+## Directory Structure
+```
+src/
+├── api/          # REST API endpoints and middleware
+├── models/       # Database models and business logic  
+├── services/     # External service integrations
+├── utils/        # Shared utilities and helpers
+└── tests/        # Test suites
+```
+
+## Architecture Overview
+[Describe how components interact, data flow, key design decisions]
+
+## Core Components
+### API Layer
+[Details about API structure, authentication, routing]
+
+### Data Model
+[Key entities, relationships, database design]
+
+## Key Workflows
+1. User Authentication Flow
+   [Step-by-step description]
+2. Data Processing Pipeline
+   [How data moves through the system]
+
+[Continue with other sections...]"
+````                    
+
+                    """,
+                    inputSchema={
+                        "type": "object",
+                        "properties": {
+                            "projectName": {"type": "string", "description": "The name of the project"},
+                            "folderPath": {"type": "string", "description": "Absolute path to the project folder on disk"},
+                            "branch": {"type": "string", "description": "Git branch name"},
+                            "remoteOrigin": {"type": "string", "description": "Git remote origin URL if available"},
+                            "upstreamOrigin": {"type": "string", "description": "Upstream repository URL if this is a fork"},
+                            "overview": {"type": "string", "description": "Comprehensive narrative overview of the codebase (10-30k tokens recommended)"}
+                        },
+                        "required": ["projectName", "folderPath", "branch", "overview"]
+                    }
+                ),
+                types.Tool(
+                    name="get_word_frequency",
+                    description="Analyzes all file descriptions to find the most frequently used technical terms. Filters out common English stop words and symbols, returning the top 200 meaningful terms. Useful for understanding the codebase's domain vocabulary and finding all functions/files related to specific concepts.",
+                    inputSchema={
+                        "type": "object",
+                        "properties": {
+                            "projectName": {"type": "string", "description": "The name of the project"},
+                            "folderPath": {"type": "string", "description": "Absolute path to the project folder on disk"},
+                            "branch": {"type": "string", "description": "Git branch name"},
+                            "remoteOrigin": {"type": "string", "description": "Git remote origin URL if available"},
+                            "upstreamOrigin": {"type": "string", "description": "Upstream repository URL if this is a fork"},
+                            "limit": {"type": "integer", "default": 200, "description": "Number of top terms to return"}
+                        },
+                        "required": ["projectName", "folderPath", "branch"]
+                    }
                 )
             ]
         
@@ -351,7 +433,10 @@ class MCPCodeIndexServer:
                 "check_codebase_size": self._handle_check_codebase_size,
                 "find_missing_descriptions": self._handle_find_missing_descriptions,
                 "search_descriptions": self._handle_search_descriptions,
-                "get_codebase_overview": self._handle_get_codebase_overview,
+                "get_all_descriptions": self._handle_get_codebase_overview,
+                "get_codebase_overview": self._handle_get_condensed_overview,
+                "update_codebase_overview": self._handle_update_codebase_overview,
+                "get_word_frequency": self._handle_get_word_frequency,
                 "merge_branch_descriptions": self._handle_merge_branch_descriptions,
             }
             
@@ -679,8 +764,16 @@ class MCPCodeIndexServer:
         """Handle check_codebase_size tool calls."""
         project_id = await self._get_or_create_project_id(arguments)
         resolved_branch = await self._resolve_branch(project_id, arguments["branch"])
+        folder_path = Path(arguments["folderPath"])
         
-        # Get file descriptions for this project/branch
+        # Clean up descriptions for files that no longer exist
+        cleaned_up_files = await self.db_manager.cleanup_missing_files(
+            project_id=project_id,
+            branch=resolved_branch,
+            project_root=folder_path
+        )
+        
+        # Get file descriptions for this project/branch (after cleanup)
         file_descriptions = await self.db_manager.get_all_file_descriptions(
             project_id=project_id,
             branch=resolved_branch
@@ -699,7 +792,9 @@ class MCPCodeIndexServer:
             "isLarge": is_large,
             "recommendation": recommendation,
             "tokenLimit": token_limit,
-            "totalFiles": len(file_descriptions)
+            "totalFiles": len(file_descriptions),
+            "cleanedUpFiles": cleaned_up_files,
+            "cleanedUpCount": len(cleaned_up_files)
         }
     
     async def _handle_find_missing_descriptions(self, arguments: Dict[str, Any]) -> Dict[str, Any]:
@@ -787,18 +882,7 @@ class MCPCodeIndexServer:
         total_tokens = self.token_counter.calculate_codebase_tokens(file_descriptions)
         is_large = self.token_counter.is_large_codebase(total_tokens)
         
-        # If large, recommend search instead
-        if is_large:
-            return {
-                "isLarge": True,
-                "totalTokens": total_tokens,
-                "tokenLimit": self.token_counter.token_limit,
-                "totalFiles": len(file_descriptions),
-                "recommendation": "use_search",
-                "message": f"Codebase has {total_tokens} tokens (limit: {self.token_counter.token_limit}). Use search_descriptions instead for better performance."
-            }
-        
-        # Build folder structure
+        # Always build and return the folder structure - if the AI called this tool, it wants the overview
         structure = self._build_folder_structure(file_descriptions)
         
         return {
@@ -915,6 +999,83 @@ class MCPCodeIndexServer:
                 **result
             }
     
+    async def _handle_get_condensed_overview(self, arguments: Dict[str, Any]) -> Dict[str, Any]:
+        """Handle get_codebase_overview tool calls for condensed overviews."""
+        project_id = await self._get_or_create_project_id(arguments)
+        resolved_branch = await self._resolve_branch(project_id, arguments["branch"])
+        
+        # Try to get existing overview
+        overview = await self.db_manager.get_project_overview(project_id, resolved_branch)
+        
+        if overview:
+            return {
+                "overview": overview.overview,
+                "lastModified": overview.last_modified.isoformat(),
+                "totalFiles": overview.total_files,
+                "totalTokensInFullDescriptions": overview.total_tokens
+            }
+        else:
+            return {
+                "overview": "",
+                "lastModified": "",
+                "totalFiles": 0,
+                "totalTokensInFullDescriptions": 0
+            }
+    
+    async def _handle_update_codebase_overview(self, arguments: Dict[str, Any]) -> Dict[str, Any]:
+        """Handle update_codebase_overview tool calls."""
+        project_id = await self._get_or_create_project_id(arguments)
+        resolved_branch = await self._resolve_branch(project_id, arguments["branch"])
+        folder_path = Path(arguments["folderPath"])
+        
+        # Get current file count and total tokens for context
+        file_descriptions = await self.db_manager.get_all_file_descriptions(
+            project_id=project_id,
+            branch=resolved_branch
+        )
+        
+        total_files = len(file_descriptions)
+        total_tokens = self.token_counter.calculate_codebase_tokens(file_descriptions)
+        
+        # Create overview record
+        overview = ProjectOverview(
+            project_id=project_id,
+            branch=resolved_branch,
+            overview=arguments["overview"],
+            last_modified=datetime.utcnow(),
+            total_files=total_files,
+            total_tokens=total_tokens
+        )
+        
+        await self.db_manager.create_project_overview(overview)
+        
+        return {
+            "success": True,
+            "message": f"Overview updated for {total_files} files",
+            "totalFiles": total_files,
+            "totalTokens": total_tokens,
+            "overviewLength": len(arguments["overview"])
+        }
+    
+    async def _handle_get_word_frequency(self, arguments: Dict[str, Any]) -> Dict[str, Any]:
+        """Handle get_word_frequency tool calls."""
+        project_id = await self._get_or_create_project_id(arguments)
+        resolved_branch = await self._resolve_branch(project_id, arguments["branch"])
+        limit = arguments.get("limit", 200)
+        
+        # Analyze word frequency
+        result = await self.db_manager.analyze_word_frequency(
+            project_id=project_id,
+            branch=resolved_branch,
+            limit=limit
+        )
+        
+        return {
+            "topTerms": [{"term": term.term, "frequency": term.frequency} for term in result.top_terms],
+            "totalTermsAnalyzed": result.total_terms_analyzed,
+            "totalUniqueTerms": result.total_unique_terms
+        }
+    
     async def _run_session_with_retry(self, read_stream, write_stream, initialization_options) -> None:
         """Run a single MCP session with error handling and retry logic."""
         max_retries = 3

{mcp_code_indexer-1.1.5.dist-info → mcp_code_indexer-1.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mcp-code-indexer
-Version: 1.1.5
+Version: 1.2.1
 Summary: MCP server that tracks file descriptions across codebases, enabling AI agents to efficiently navigate and understand code through searchable summaries and token-aware overviews.
 Author: MCP Code Indexer Contributors
 Maintainer: MCP Code Indexer Contributors
@@ -158,12 +158,12 @@ mypy src/
 
 ## 🛠️ MCP Tools Available
 
-The server provides **8 powerful MCP tools** for intelligent codebase management:
+The server provides **11 powerful MCP tools** for intelligent codebase management:
 
 ### Core Operations
 - **`get_file_description`** - Retrieve stored file descriptions instantly
 - **`update_file_description`** - Store detailed file summaries and metadata
-- **`check_codebase_size`** - Get token count and size-based recommendations
+- **`check_codebase_size`** - Get token count and size-based recommendations with automatic file cleanup
 
 ### Batch Operations
 - **`find_missing_descriptions`** - Scan projects for files without descriptions
@@ -171,10 +171,13 @@ The server provides **8 powerful MCP tools** for intelligent codebase management
 
 ### Search & Discovery
 - **`search_descriptions`** - Fast full-text search across all descriptions
-- **`get_codebase_overview`** - Complete hierarchical project structure
+- **`get_all_descriptions`** - Complete hierarchical project structure
+- **`get_codebase_overview`** - Condensed narrative overview of entire codebase
+- **`get_word_frequency`** - Technical vocabulary analysis with stop-word filtering
 
 ### Advanced Features
 - **`merge_branch_descriptions`** - Two-phase merge with conflict resolution
+- **`update_codebase_overview`** - Create comprehensive codebase documentation
 
 ## 🏗️ Architecture Highlights
 

{mcp_code_indexer-1.1.5.dist-info → mcp_code_indexer-1.2.1.dist-info}/RECORD

@@ -6,18 +6,19 @@ mcp_code_indexer/logging_config.py,sha256=5L1cYIG8IAX91yCjc5pzkbO_KPt0bvm_ABHB53
 mcp_code_indexer/main.py,sha256=eRc0Vl3DVDGS5XtuPCDBArgmqcBIi92O97LbE8HYGGA,13601
 mcp_code_indexer/merge_handler.py,sha256=lJR8eVq2qSrF6MW9mR3Fy8UzrNAaQ7RsI2FMNXne3vQ,14692
 mcp_code_indexer/token_counter.py,sha256=WrifOkbF99nWWHlRlhCHAB2KN7qr83GOHl7apE-hJcE,8460
+mcp_code_indexer/data/stop_words_english.txt,sha256=7Zdd9ameVgA6tN_zuXROvHXD4hkWeELVywPhb7FJEkw,6343
 mcp_code_indexer/database/__init__.py,sha256=aPq_aaRp0aSwOBIq9GkuMNjmLxA411zg2vhdrAuHm-w,38
-mcp_code_indexer/database/database.py,sha256=eG2xY5cd-oxRZ6mgGkqqBiJJfGCPqJgzoFq6kR99WfA,20300
-mcp_code_indexer/database/models.py,sha256=3wOxHKb6j3zKPWFSwB5g1TLpI507vLNZcqsxZR4VuRs,5528
+mcp_code_indexer/database/database.py,sha256=ziePr0QHkPwv-plLRdySB8ei8fcXc3SOIgC0uRi47KI,26600
+mcp_code_indexer/database/models.py,sha256=_vCmJnPXZSiInRzyvs4c7QUWuNNW8qsOoDlGX8J-Gnk,7124
 mcp_code_indexer/middleware/__init__.py,sha256=p-mP0pMsfiU2yajCPvokCUxUEkh_lu4XJP1LyyMW2ug,220
 mcp_code_indexer/middleware/error_middleware.py,sha256=v6jaHmPxf3qerYdb85X1tHIXLxgcbybpitKVakFLQTA,10109
 mcp_code_indexer/server/__init__.py,sha256=16xMcuriUOBlawRqWNBk6niwrvtv_JD5xvI36X1Vsmk,41
-mcp_code_indexer/server/mcp_server.py,sha256=DtfoeoOF72kdWO7fCYU9qvhg3LUyJU12KpjCCqw_Uxw,50589
+mcp_code_indexer/server/mcp_server.py,sha256=X6_TV9ac5fZX-5bKblhx-gKHP1Cba_KDatYYLqBDvkc,59474
 mcp_code_indexer/tiktoken_cache/9b5ad71b2ce5302211f9c61530b329a4922fc6a4,sha256=Ijkht27pm96ZW3_3OFE-7xAPtR0YyTWXoRO8_-hlsqc,1681126
 mcp_code_indexer/tools/__init__.py,sha256=m01mxML2UdD7y5rih_XNhNSCMzQTz7WQ_T1TeOcYlnE,49
-mcp_code_indexer-1.1.5.dist-info/licenses/LICENSE,sha256=JN9dyPPgYwH9C-UjYM7FLNZjQ6BF7kAzpF3_4PwY4rY,1086
-mcp_code_indexer-1.1.5.dist-info/METADATA,sha256=anW6T4WSZBQjIgvFaViNibdh3mgIeLW35vxiaj3wrjM,11930
-mcp_code_indexer-1.1.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mcp_code_indexer-1.1.5.dist-info/entry_points.txt,sha256=8HqWOw1Is7jOP1bvIgaSwouvT9z_Boe-9hd4NzyJOhY,68
-mcp_code_indexer-1.1.5.dist-info/top_level.txt,sha256=yKYCM-gMGt-cnupGfAhnZaoEsROLB6DQ1KFUuyKx4rw,17
-mcp_code_indexer-1.1.5.dist-info/RECORD,,
+mcp_code_indexer-1.2.1.dist-info/licenses/LICENSE,sha256=JN9dyPPgYwH9C-UjYM7FLNZjQ6BF7kAzpF3_4PwY4rY,1086
+mcp_code_indexer-1.2.1.dist-info/METADATA,sha256=NcWRwgD-W2qG8g6aDsT7uKxA5Ya4hFFKnGlMPmwEef0,12201
+mcp_code_indexer-1.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mcp_code_indexer-1.2.1.dist-info/entry_points.txt,sha256=8HqWOw1Is7jOP1bvIgaSwouvT9z_Boe-9hd4NzyJOhY,68
+mcp_code_indexer-1.2.1.dist-info/top_level.txt,sha256=yKYCM-gMGt-cnupGfAhnZaoEsROLB6DQ1KFUuyKx4rw,17
+mcp_code_indexer-1.2.1.dist-info/RECORD,,