schema-search 0.1.2__tar.gz

schema_search-0.1.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Adib Hasan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

schema_search-0.1.2/MANIFEST.in

@@ -0,0 +1,4 @@
+include README.md
+include LICENSE
+include config.yml
+

schema_search-0.1.2/PKG-INFO

@@ -0,0 +1,275 @@
+Metadata-Version: 2.4
+Name: schema-search
+Version: 0.1.2
+Summary: Natural language search for database schemas with graph-aware semantic retrieval
+Home-page: https://github.com/neehan/schema-search
+Author: 
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: sqlalchemy>=1.4.0
+Requires-Dist: sentence-transformers>=2.2.0
+Requires-Dist: networkx>=2.8.0
+Requires-Dist: bm25s>=0.2.0
+Requires-Dist: numpy>=1.21.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: tqdm>=4.65.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: rapidfuzz>=3.0.0
+Provides-Extra: mcp
+Requires-Dist: fastmcp>=2.0.0; extra == "mcp"
+Provides-Extra: test
+Requires-Dist: pytest>=7.0.0; extra == "test"
+Requires-Dist: python-dotenv>=1.0.0; extra == "test"
+Requires-Dist: psutil>=5.9.0; extra == "test"
+Requires-Dist: datasets>=2.0.0; extra == "test"
+Provides-Extra: postgres
+Requires-Dist: psycopg2-binary>=2.9.0; extra == "postgres"
+Provides-Extra: mysql
+Requires-Dist: pymysql>=1.0.0; extra == "mysql"
+Provides-Extra: snowflake
+Requires-Dist: snowflake-sqlalchemy>=1.4.0; extra == "snowflake"
+Requires-Dist: snowflake-connector-python>=3.0.0; extra == "snowflake"
+Provides-Extra: bigquery
+Requires-Dist: sqlalchemy-bigquery>=1.6.0; extra == "bigquery"
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Schema Search
+
+An MCP Server for Natural Language Search over RDBMS Schemas. Find exact tables you need, with all their relationships mapped out, in milliseconds. No vector database setup is required.
+
+## Why
+
+You have 200 tables in your database. Someone asks "where are user refunds stored?"
+
+You could:
+- Grep through SQL files for 20 minutes
+- Pass the full schema to an LLM and watch it struggle with 200 tables
+
+Or **build schematic embeddings of your tables, store in-memory, and query in natural language in an MCP server**.
+
+### Benefits
+- No vector database setup is required
+- Small memory footprint -- easily scales up to 1000 tables and 10,000+ columns.
+- Millisecond query latency
+
+## Install
+
+```bash
+# With uv - PostgreSQL (recommended)
+uv pip install "schema-search[postgres,mcp]"
+
+# With pip - PostgreSQL
+pip install "schema-search[postgres,mcp]"
+
+# Other databases
+uv pip install "schema-search[mysql,mcp]"      # MySQL
+uv pip install "schema-search[snowflake,mcp]"  # Snowflake
+uv pip install "schema-search[bigquery,mcp]"   # BigQuery
+```
+
+## MCP Server
+
+Integrate with Claude Desktop or any MCP client.
+
+### Setup
+
+Add to your MCP config (e.g., `~/.cursor/mcp.json` or Claude Desktop config):
+
+**Using uv (Recommended):**
+```json
+{
+  "mcpServers": {
+    "schema-search": {
+      "command": "uvx",
+      "args": ["schema-search[postgres,mcp]", "postgresql://user:pass@localhost/db", "optional config.yml path", "optional llm_api_key", "optional llm_base_url"]
+    }
+  }
+}
+```
+
+**Using pip:**
+```json
+{
+  "mcpServers": {
+    "schema-search": {
+      "command": "path/to/schema-search-mcp", // conda: /Users/<username>/opt/miniconda3/envs/<your env>/bin/schema-search-mcp",
+      "args": ["postgresql://user:pass@localhost/db", "optional config.yml path", "optional llm_api_key", "optional llm_base_url"]
+    }
+  }
+}
+```
+
+
+The LLM API key and base url are only required if you use LLM-generated schema summaries (`config.chunking.strategy = 'llm'`).
+
+### CLI Usage
+
+```bash
+schema-search-mcp "postgresql://user:pass@localhost/db"
+```
+
+Optional args: `[config_path] [llm_api_key] [llm_base_url]`
+
+The server exposes `schema_search(query, hops, limit)` for natural language schema queries.
+
+## Python Use
+
+```python
+from sqlalchemy import create_engine
+from schema_search import SchemaSearch
+
+engine = create_engine("postgresql://user:pass@localhost/db")
+search = SchemaSearch(engine)
+
+search.index(force=False) # default is False
+results = search.search("where are user refunds stored?")
+
+for result in results['results']:
+    print(result['table'])           # "refund_transactions"
+    print(result['schema'])           # Full column info, types, constraints
+    print(result['related_tables'])   # ["users", "payments", "transactions"]
+
+# Override hops, limit, search strategy
+results = search.search("user_table", hops=0, limit=5, search_type="semantic")
+
+```
+
+`SchemaSearch.index()` automatically detects schema changes and refreshes cached metadata, so you rarely need to force a reindex manually.
+
+## Configuration
+
+Edit `[config.yml](config.yml)`:
+
+```yaml
+logging:
+  level: "WARNING"
+
+embedding:
+  location: "memory" # Options: "memory", "vectordb" (coming soon)
+  model: "multi-qa-MiniLM-L6-cos-v1"
+  metric: "cosine" # Options: "cosine", "euclidean", "manhattan", "dot"
+  batch_size: 32
+  show_progress: false
+  cache_dir: "/tmp/.schema_search_cache"
+
+chunking:
+  strategy: "raw" # Options: "raw", "llm"
+  max_tokens: 256
+  overlap_tokens: 50
+  model: "gpt-4o-mini"
+
+search:
+  # Search strategy: "semantic" (embeddings), "bm25" (BM25 lexical), "fuzzy" (fuzzy string matching), "hybrid" (semantic + bm25)
+  strategy: "hybrid"
+  initial_top_k: 20
+  rerank_top_k: 5
+  semantic_weight: 0.67 # For hybrid search (bm25_weight = 1 - semantic_weight)
+  hops: 1 # Number of foreign key hops for graph expansion (0-2 recommended)
+
+reranker:
+  # CrossEncoder model for reranking. Set to null to disable reranking
+  model: null # "Alibaba-NLP/gte-reranker-modernbert-base"
+
+schema:
+  include_columns: true
+  include_indices: true
+  include_foreign_keys: true
+  include_constraints: true
+```
+
+## Search Strategies
+
+Schema Search supports four search strategies:
+
+- **semantic**: Embedding-based similarity search using sentence transformers
+- **bm25**: Lexical search using BM25 ranking algorithm
+- **fuzzy**: String matching on table/column names using fuzzy matching
+- **hybrid**: Combines semantic and bm25 scores (default: 67% semantic, 33% fuzzy)
+
+Each strategy performs its own initial ranking, then optionally applies CrossEncoder reranking if `reranker.model` is configured. Set `reranker.model` to `null` to disable reranking.
+
+## Performance Comparison
+We [benchmarked](/tests/test_spider_eval.py) on the Spider dataset (1,234 train queries across 18 databases) using the default `config.yml`.  
+
+**Memory:** The embedding model requires ~90 MB and the optional reranker adds ~155 MB. Actual process memory depends on your Python runtime.
+
+### Without Reranker (`reranker.model: null`)
+![Without Reranker](img/spider_benchmark_without_reranker.png)
+- **Indexing:** 0.22s ± 0.08s per database (18 total).
+- **Accuracy:** Hybrid leads with Recall@1 62% / MRR 0.93; Semantic follows at Recall@1 58% / MRR 0.89.
+- **Latency:** BM25 and Fuzzy return in ~5ms; Semantic spends ~15ms; Hybrid (semantic + fuzzy) averages 52ms.
+- **Fuzzy baseline:** Recall@1 22%, highlighting the need for semantic signals on natural-language queries.
+
+### With Reranker (`Alibaba-NLP/gte-reranker-modernbert-base`)
+![With Reranker](img/spider_benchmark_with_reranker.png)
+- **Indexing:** 0.25s ± 0.05s per database (same 18 DBs).
+- **Accuracy:** All strategies converge around Recall@1 62% and MRR ≈ 0.92; Fuzzy jumps from 51% → 92% MRR.
+- **Latency trade-off:** Extra CrossEncoder pass lifts per-query latency to ~0.18–0.29s depending on strategy.
+- **Recommendation:** Enable the reranker when accuracy matters most; disable it for ultra-low-latency lookups.
+
+
+You can override the search strategy, hops, and limit at query time:
+
+```python
+# Use fuzzy search instead of default
+results = search.search("user_table", search_type="fuzzy")
+
+# Use BM25 for keyword-based search
+results = search.search("transactions payments", search_type="bm25")
+
+# Use hybrid for best of both worlds
+results = search.search("where are user refunds?", search_type="hybrid")
+
+# Override hops and limit
+results = search.search("user refunds", hops=2, limit=10)  # Expand 2 hops, return 10 tables
+
+# Disable graph expansion
+results = search.search("user_table", hops=0)  # Only direct matches, no foreign key traversal
+```
+
+### LLM Chunking
+
+Use LLM to generate semantic summaries instead of raw schema text:
+
+1. Set `strategy: "llm"` in `config.yml`
+2. Pass API credentials:
+
+```python
+search = SchemaSearch(
+    engine,
+    llm_api_key="sk-...",
+    llm_base_url="https://api.openai.com/v1/"  # optional
+)
+```
+
+## How It Works
+
+1. **Extract schemas** from database using SQLAlchemy inspector
+2. **Chunk schemas** into digestible pieces (markdown or LLM-generated summaries)
+3. **Initial search** using selected strategy (semantic/BM25/fuzzy)
+4. **Expand via foreign keys** to find related tables (configurable hops)
+5. **Optional reranking** with CrossEncoder to refine results
+6. Return top tables with full schema and relationships
+
+Cache stored in `.schema_search_cache/` (configurable in `config.yml`)
+
+## License
+
+MIT

schema_search-0.1.2/README.md

@@ -0,0 +1,223 @@
+# Schema Search
+
+An MCP Server for Natural Language Search over RDBMS Schemas. Find exact tables you need, with all their relationships mapped out, in milliseconds. No vector database setup is required.
+
+## Why
+
+You have 200 tables in your database. Someone asks "where are user refunds stored?"
+
+You could:
+- Grep through SQL files for 20 minutes
+- Pass the full schema to an LLM and watch it struggle with 200 tables
+
+Or **build schematic embeddings of your tables, store in-memory, and query in natural language in an MCP server**.
+
+### Benefits
+- No vector database setup is required
+- Small memory footprint -- easily scales up to 1000 tables and 10,000+ columns.
+- Millisecond query latency
+
+## Install
+
+```bash
+# With uv - PostgreSQL (recommended)
+uv pip install "schema-search[postgres,mcp]"
+
+# With pip - PostgreSQL
+pip install "schema-search[postgres,mcp]"
+
+# Other databases
+uv pip install "schema-search[mysql,mcp]"      # MySQL
+uv pip install "schema-search[snowflake,mcp]"  # Snowflake
+uv pip install "schema-search[bigquery,mcp]"   # BigQuery
+```
+
+## MCP Server
+
+Integrate with Claude Desktop or any MCP client.
+
+### Setup
+
+Add to your MCP config (e.g., `~/.cursor/mcp.json` or Claude Desktop config):
+
+**Using uv (Recommended):**
+```json
+{
+  "mcpServers": {
+    "schema-search": {
+      "command": "uvx",
+      "args": ["schema-search[postgres,mcp]", "postgresql://user:pass@localhost/db", "optional config.yml path", "optional llm_api_key", "optional llm_base_url"]
+    }
+  }
+}
+```
+
+**Using pip:**
+```json
+{
+  "mcpServers": {
+    "schema-search": {
+      "command": "path/to/schema-search-mcp", // conda: /Users/<username>/opt/miniconda3/envs/<your env>/bin/schema-search-mcp",
+      "args": ["postgresql://user:pass@localhost/db", "optional config.yml path", "optional llm_api_key", "optional llm_base_url"]
+    }
+  }
+}
+```
+
+
+The LLM API key and base url are only required if you use LLM-generated schema summaries (`config.chunking.strategy = 'llm'`).
+
+### CLI Usage
+
+```bash
+schema-search-mcp "postgresql://user:pass@localhost/db"
+```
+
+Optional args: `[config_path] [llm_api_key] [llm_base_url]`
+
+The server exposes `schema_search(query, hops, limit)` for natural language schema queries.
+
+## Python Use
+
+```python
+from sqlalchemy import create_engine
+from schema_search import SchemaSearch
+
+engine = create_engine("postgresql://user:pass@localhost/db")
+search = SchemaSearch(engine)
+
+search.index(force=False) # default is False
+results = search.search("where are user refunds stored?")
+
+for result in results['results']:
+    print(result['table'])           # "refund_transactions"
+    print(result['schema'])           # Full column info, types, constraints
+    print(result['related_tables'])   # ["users", "payments", "transactions"]
+
+# Override hops, limit, search strategy
+results = search.search("user_table", hops=0, limit=5, search_type="semantic")
+
+```
+
+`SchemaSearch.index()` automatically detects schema changes and refreshes cached metadata, so you rarely need to force a reindex manually.
+
+## Configuration
+
+Edit `[config.yml](config.yml)`:
+
+```yaml
+logging:
+  level: "WARNING"
+
+embedding:
+  location: "memory" # Options: "memory", "vectordb" (coming soon)
+  model: "multi-qa-MiniLM-L6-cos-v1"
+  metric: "cosine" # Options: "cosine", "euclidean", "manhattan", "dot"
+  batch_size: 32
+  show_progress: false
+  cache_dir: "/tmp/.schema_search_cache"
+
+chunking:
+  strategy: "raw" # Options: "raw", "llm"
+  max_tokens: 256
+  overlap_tokens: 50
+  model: "gpt-4o-mini"
+
+search:
+  # Search strategy: "semantic" (embeddings), "bm25" (BM25 lexical), "fuzzy" (fuzzy string matching), "hybrid" (semantic + bm25)
+  strategy: "hybrid"
+  initial_top_k: 20
+  rerank_top_k: 5
+  semantic_weight: 0.67 # For hybrid search (bm25_weight = 1 - semantic_weight)
+  hops: 1 # Number of foreign key hops for graph expansion (0-2 recommended)
+
+reranker:
+  # CrossEncoder model for reranking. Set to null to disable reranking
+  model: null # "Alibaba-NLP/gte-reranker-modernbert-base"
+
+schema:
+  include_columns: true
+  include_indices: true
+  include_foreign_keys: true
+  include_constraints: true
+```
+
+## Search Strategies
+
+Schema Search supports four search strategies:
+
+- **semantic**: Embedding-based similarity search using sentence transformers
+- **bm25**: Lexical search using BM25 ranking algorithm
+- **fuzzy**: String matching on table/column names using fuzzy matching
+- **hybrid**: Combines semantic and bm25 scores (default: 67% semantic, 33% fuzzy)
+
+Each strategy performs its own initial ranking, then optionally applies CrossEncoder reranking if `reranker.model` is configured. Set `reranker.model` to `null` to disable reranking.
+
+## Performance Comparison
+We [benchmarked](/tests/test_spider_eval.py) on the Spider dataset (1,234 train queries across 18 databases) using the default `config.yml`.  
+
+**Memory:** The embedding model requires ~90 MB and the optional reranker adds ~155 MB. Actual process memory depends on your Python runtime.
+
+### Without Reranker (`reranker.model: null`)
+![Without Reranker](img/spider_benchmark_without_reranker.png)
+- **Indexing:** 0.22s ± 0.08s per database (18 total).
+- **Accuracy:** Hybrid leads with Recall@1 62% / MRR 0.93; Semantic follows at Recall@1 58% / MRR 0.89.
+- **Latency:** BM25 and Fuzzy return in ~5ms; Semantic spends ~15ms; Hybrid (semantic + fuzzy) averages 52ms.
+- **Fuzzy baseline:** Recall@1 22%, highlighting the need for semantic signals on natural-language queries.
+
+### With Reranker (`Alibaba-NLP/gte-reranker-modernbert-base`)
+![With Reranker](img/spider_benchmark_with_reranker.png)
+- **Indexing:** 0.25s ± 0.05s per database (same 18 DBs).
+- **Accuracy:** All strategies converge around Recall@1 62% and MRR ≈ 0.92; Fuzzy jumps from 51% → 92% MRR.
+- **Latency trade-off:** Extra CrossEncoder pass lifts per-query latency to ~0.18–0.29s depending on strategy.
+- **Recommendation:** Enable the reranker when accuracy matters most; disable it for ultra-low-latency lookups.
+
+
+You can override the search strategy, hops, and limit at query time:
+
+```python
+# Use fuzzy search instead of default
+results = search.search("user_table", search_type="fuzzy")
+
+# Use BM25 for keyword-based search
+results = search.search("transactions payments", search_type="bm25")
+
+# Use hybrid for best of both worlds
+results = search.search("where are user refunds?", search_type="hybrid")
+
+# Override hops and limit
+results = search.search("user refunds", hops=2, limit=10)  # Expand 2 hops, return 10 tables
+
+# Disable graph expansion
+results = search.search("user_table", hops=0)  # Only direct matches, no foreign key traversal
+```
+
+### LLM Chunking
+
+Use LLM to generate semantic summaries instead of raw schema text:
+
+1. Set `strategy: "llm"` in `config.yml`
+2. Pass API credentials:
+
+```python
+search = SchemaSearch(
+    engine,
+    llm_api_key="sk-...",
+    llm_base_url="https://api.openai.com/v1/"  # optional
+)
+```
+
+## How It Works
+
+1. **Extract schemas** from database using SQLAlchemy inspector
+2. **Chunk schemas** into digestible pieces (markdown or LLM-generated summaries)
+3. **Initial search** using selected strategy (semantic/BM25/fuzzy)
+4. **Expand via foreign keys** to find related tables (configurable hops)
+5. **Optional reranking** with CrossEncoder to refine results
+6. Return top tables with full schema and relationships
+
+Cache stored in `.schema_search_cache/` (configurable in `config.yml`)
+
+## License
+
+MIT

schema_search-0.1.2/config.yml

@@ -0,0 +1,34 @@
+logging:
+  level: "WARNING"
+
+embedding:
+  location: "memory" # Options: "memory", "vectordb" (coming soon)
+  model: "multi-qa-MiniLM-L6-cos-v1"
+  metric: "cosine" # Options: "cosine", "euclidean", "manhattan", "dot"
+  batch_size: 32
+  show_progress: false
+  cache_dir: "/tmp/.schema_search_cache"
+
+chunking:
+  strategy: "raw" # Options: "raw", "llm"
+  max_tokens: 256
+  overlap_tokens: 50
+  model: "gpt-4o-mini"
+
+search:
+  # Search strategy: "semantic" (embeddings), "bm25" (BM25 lexical), "fuzzy" (fuzzy string matching), "hybrid" (semantic + bm25)
+  strategy: "bm25"
+  initial_top_k: 20
+  rerank_top_k: 5
+  semantic_weight: 0.67 # For hybrid search (bm25_weight = 1 - semantic_weight)
+  hops: 1 # Number of foreign key hops for graph expansion (0-2 recommended)
+
+reranker:
+  # CrossEncoder model for reranking. Set to null to disable reranking
+  model: # "Alibaba-NLP/gte-reranker-modernbert-base"
+
+schema:
+  include_columns: true
+  include_indices: true
+  include_foreign_keys: true
+  include_constraints: true

schema_search-0.1.2/schema_search/__init__.py

@@ -0,0 +1,26 @@
+from schema_search.schema_search import SchemaSearch
+from schema_search.types import (
+    IndexResult,
+    SearchResult,
+    SearchResultItem,
+    SearchType,
+    TableSchema,
+    ColumnInfo,
+    ForeignKeyInfo,
+    IndexInfo,
+    ConstraintInfo,
+)
+
+__version__ = "0.1.0"
+__all__ = [
+    "SchemaSearch",
+    "IndexResult",
+    "SearchResult",
+    "SearchResultItem",
+    "SearchType",
+    "TableSchema",
+    "ColumnInfo",
+    "ForeignKeyInfo",
+    "IndexInfo",
+    "ConstraintInfo",
+]

schema_search-0.1.2/schema_search/chunkers/__init__.py

@@ -0,0 +1,6 @@
+from schema_search.chunkers.base import Chunk, BaseChunker
+from schema_search.chunkers.markdown import MarkdownChunker
+from schema_search.chunkers.llm import LLMChunker
+from schema_search.chunkers.factory import create_chunker
+
+__all__ = ["Chunk", "BaseChunker", "MarkdownChunker", "LLMChunker", "create_chunker"]

schema_search-0.1.2/schema_search/chunkers/base.py

@@ -0,0 +1,95 @@
+from typing import Dict, List
+from dataclasses import dataclass
+from abc import ABC, abstractmethod
+
+from tqdm import tqdm
+
+from schema_search.types import TableSchema
+
+
+@dataclass
+class Chunk:
+    table_name: str
+    content: str
+    chunk_id: int
+    token_count: int
+
+
+class BaseChunker(ABC):
+    def __init__(self, max_tokens: int, overlap_tokens: int, show_progress: bool = False):
+        self.max_tokens = max_tokens
+        self.overlap_tokens = overlap_tokens
+        self.show_progress = show_progress
+
+    def chunk_schemas(self, schemas: Dict[str, TableSchema]) -> List[Chunk]:
+        chunks: List[Chunk] = []
+        chunk_id = 0
+
+        iterator = schemas.items()
+        if self.show_progress:
+            iterator = tqdm(iterator, desc="Chunking tables", unit="table")
+
+        for table_name, schema in iterator:
+            table_chunks = self._chunk_table(table_name, schema, chunk_id)
+            chunks.extend(table_chunks)
+            chunk_id += len(table_chunks)
+
+        return chunks
+
+    @abstractmethod
+    def _generate_content(self, table_name: str, schema: TableSchema) -> str:
+        pass
+
+    def _chunk_table(
+        self, table_name: str, schema: TableSchema, start_id: int
+    ) -> List[Chunk]:
+        content = self._generate_content(table_name, schema)
+        lines = content.split("\n")
+
+        header = f"Table: {table_name}"
+        header_tokens = self._estimate_tokens(header)
+
+        chunks: List[Chunk] = []
+        current_chunk_lines = [header]
+        current_tokens = header_tokens
+        chunk_id = start_id
+
+        for line in lines[1:]:
+            line_tokens = self._estimate_tokens(line)
+
+            if (
+                current_tokens + line_tokens > self.max_tokens
+                and len(current_chunk_lines) > 1
+            ):
+                chunk_content = "\n".join(current_chunk_lines)
+                chunks.append(
+                    Chunk(
+                        table_name=table_name,
+                        content=chunk_content,
+                        chunk_id=chunk_id,
+                        token_count=current_tokens,
+                    )
+                )
+                chunk_id += 1
+
+                current_chunk_lines = [header]
+                current_tokens = header_tokens
+
+            current_chunk_lines.append(line)
+            current_tokens += line_tokens
+
+        if len(current_chunk_lines) > 1:
+            chunk_content = "\n".join(current_chunk_lines)
+            chunks.append(
+                Chunk(
+                    table_name=table_name,
+                    content=chunk_content,
+                    chunk_id=chunk_id,
+                    token_count=current_tokens,
+                )
+            )
+
+        return chunks
+
+    def _estimate_tokens(self, text: str) -> int:
+        return len(text.split()) + len(text) // 4