schema-search 0.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of schema-search might be problematic. Click here for more details.

Files changed (38) hide show
  1. schema_search/__init__.py +26 -0
  2. schema_search/chunkers/__init__.py +6 -0
  3. schema_search/chunkers/base.py +95 -0
  4. schema_search/chunkers/factory.py +31 -0
  5. schema_search/chunkers/llm.py +51 -0
  6. schema_search/chunkers/markdown.py +25 -0
  7. schema_search/embedding_cache/__init__.py +5 -0
  8. schema_search/embedding_cache/base.py +40 -0
  9. schema_search/embedding_cache/bm25.py +63 -0
  10. schema_search/embedding_cache/factory.py +20 -0
  11. schema_search/embedding_cache/inmemory.py +112 -0
  12. schema_search/graph_builder.py +69 -0
  13. schema_search/mcp_server.py +82 -0
  14. schema_search/metrics.py +33 -0
  15. schema_search/rankers/__init__.py +5 -0
  16. schema_search/rankers/base.py +45 -0
  17. schema_search/rankers/cross_encoder.py +34 -0
  18. schema_search/rankers/factory.py +11 -0
  19. schema_search/schema_extractor.py +135 -0
  20. schema_search/schema_search.py +263 -0
  21. schema_search/search/__init__.py +15 -0
  22. schema_search/search/base.py +85 -0
  23. schema_search/search/bm25.py +48 -0
  24. schema_search/search/factory.py +61 -0
  25. schema_search/search/fuzzy.py +56 -0
  26. schema_search/search/hybrid.py +82 -0
  27. schema_search/search/semantic.py +49 -0
  28. schema_search/types.py +57 -0
  29. schema_search-0.1.2.dist-info/METADATA +275 -0
  30. schema_search-0.1.2.dist-info/RECORD +38 -0
  31. schema_search-0.1.2.dist-info/WHEEL +5 -0
  32. schema_search-0.1.2.dist-info/entry_points.txt +2 -0
  33. schema_search-0.1.2.dist-info/licenses/LICENSE +21 -0
  34. schema_search-0.1.2.dist-info/top_level.txt +2 -0
  35. tests/__init__.py +0 -0
  36. tests/test_integration.py +352 -0
  37. tests/test_llm_sql_generation.py +320 -0
  38. tests/test_spider_eval.py +484 -0
@@ -0,0 +1,61 @@
1
+ from typing import Callable, Dict, Optional
2
+
3
+ from schema_search.search.semantic import SemanticSearchStrategy
4
+ from schema_search.search.fuzzy import FuzzySearchStrategy
5
+ from schema_search.search.bm25 import BM25SearchStrategy
6
+ from schema_search.search.hybrid import HybridSearchStrategy
7
+ from schema_search.search.base import BaseSearchStrategy
8
+ from schema_search.embedding_cache import BaseEmbeddingCache
9
+ from schema_search.rankers.base import BaseRanker
10
+
11
+
12
+ def create_search_strategy(
13
+ config: Dict,
14
+ get_embedding_cache: Callable[[], BaseEmbeddingCache],
15
+ get_bm25_cache: Callable,
16
+ get_reranker: Callable[[], Optional[BaseRanker]],
17
+ strategy_type: Optional[str],
18
+ ) -> BaseSearchStrategy:
19
+ search_config = config["search"]
20
+ strategy_type = strategy_type or search_config["strategy"]
21
+
22
+ initial_top_k = search_config["initial_top_k"]
23
+ rerank_top_k = search_config["rerank_top_k"]
24
+
25
+ reranker = get_reranker()
26
+
27
+ if strategy_type == "semantic":
28
+ return SemanticSearchStrategy(
29
+ embedding_cache=get_embedding_cache(),
30
+ initial_top_k=initial_top_k,
31
+ rerank_top_k=rerank_top_k,
32
+ reranker=reranker,
33
+ )
34
+
35
+ if strategy_type == "bm25":
36
+ return BM25SearchStrategy(
37
+ bm25_cache=get_bm25_cache(),
38
+ initial_top_k=initial_top_k,
39
+ rerank_top_k=rerank_top_k,
40
+ reranker=reranker,
41
+ )
42
+
43
+ if strategy_type == "fuzzy":
44
+ return FuzzySearchStrategy(
45
+ initial_top_k=initial_top_k,
46
+ rerank_top_k=rerank_top_k,
47
+ reranker=reranker,
48
+ )
49
+
50
+ if strategy_type == "hybrid":
51
+ semantic_weight = search_config["semantic_weight"]
52
+ return HybridSearchStrategy(
53
+ embedding_cache=get_embedding_cache(),
54
+ bm25_cache=get_bm25_cache(),
55
+ initial_top_k=initial_top_k,
56
+ rerank_top_k=rerank_top_k,
57
+ reranker=reranker,
58
+ semantic_weight=semantic_weight,
59
+ )
60
+
61
+ raise ValueError(f"Unknown search strategy: {strategy_type}")
@@ -0,0 +1,56 @@
1
+ from typing import Dict, List, Optional
2
+
3
+ from rapidfuzz import fuzz
4
+
5
+ from schema_search.search.base import BaseSearchStrategy
6
+ from schema_search.types import TableSchema, SearchResultItem
7
+ from schema_search.chunkers import Chunk
8
+ from schema_search.graph_builder import GraphBuilder
9
+ from schema_search.rankers.base import BaseRanker
10
+
11
+
12
+ class FuzzySearchStrategy(BaseSearchStrategy):
13
+ def __init__(
14
+ self, initial_top_k: int, rerank_top_k: int, reranker: Optional[BaseRanker]
15
+ ):
16
+ super().__init__(reranker, initial_top_k, rerank_top_k)
17
+
18
+ def _initial_ranking(
19
+ self,
20
+ query: str,
21
+ schemas: Dict[str, TableSchema],
22
+ chunks: List[Chunk],
23
+ graph_builder: GraphBuilder,
24
+ hops: int,
25
+ ) -> List[SearchResultItem]:
26
+ scored_tables: List[tuple[str, float]] = []
27
+
28
+ for table_name, schema in schemas.items():
29
+ searchable_text = self._build_searchable_text(table_name, schema)
30
+ score = fuzz.ratio(query, searchable_text, score_cutoff=0) / 100.0
31
+ scored_tables.append((table_name, score))
32
+
33
+ scored_tables.sort(key=lambda x: x[1], reverse=True)
34
+
35
+ results: List[SearchResultItem] = []
36
+ for table_name, score in scored_tables[: self.initial_top_k]:
37
+ result = self._build_result_item(
38
+ table_name=table_name,
39
+ score=score,
40
+ schema=schemas[table_name],
41
+ matched_chunks=[],
42
+ graph_builder=graph_builder,
43
+ hops=hops,
44
+ )
45
+ results.append(result)
46
+
47
+ return results
48
+
49
+ def _build_searchable_text(self, table_name: str, schema: TableSchema) -> str:
50
+ parts = [table_name]
51
+
52
+ if schema["indices"]:
53
+ for idx in schema["indices"]:
54
+ parts.append(idx["name"])
55
+
56
+ return " ".join(parts)
@@ -0,0 +1,82 @@
1
+ from typing import Dict, List, Optional, TYPE_CHECKING
2
+
3
+ import numpy as np
4
+
5
+ from schema_search.search.base import BaseSearchStrategy
6
+ from schema_search.types import TableSchema, SearchResultItem
7
+ from schema_search.chunkers import Chunk
8
+ from schema_search.graph_builder import GraphBuilder
9
+ from schema_search.embedding_cache import BaseEmbeddingCache
10
+ from schema_search.rankers.base import BaseRanker
11
+
12
+ if TYPE_CHECKING:
13
+ from schema_search.embedding_cache.bm25 import BM25Cache
14
+
15
+
16
+ class HybridSearchStrategy(BaseSearchStrategy):
17
+ def __init__(
18
+ self,
19
+ embedding_cache: BaseEmbeddingCache,
20
+ bm25_cache: "BM25Cache",
21
+ initial_top_k: int,
22
+ rerank_top_k: int,
23
+ reranker: Optional[BaseRanker],
24
+ semantic_weight: float,
25
+ ):
26
+ super().__init__(reranker, initial_top_k, rerank_top_k)
27
+ assert 0 <= semantic_weight <= 1, "semantic_weight must be between 0 and 1"
28
+ self.embedding_cache = embedding_cache
29
+ self.bm25_cache = bm25_cache
30
+ self.semantic_weight = semantic_weight
31
+ self.bm25_weight = 1 - semantic_weight
32
+
33
+ def _initial_ranking(
34
+ self,
35
+ query: str,
36
+ schemas: Dict[str, TableSchema],
37
+ chunks: List[Chunk],
38
+ graph_builder: GraphBuilder,
39
+ hops: int,
40
+ ) -> List[SearchResultItem]:
41
+ query_embedding = self.embedding_cache.encode_query(query)
42
+ semantic_scores = self.embedding_cache.compute_similarities(query_embedding)
43
+
44
+ bm25_scores = self.bm25_cache.get_scores(query)
45
+
46
+ semantic_min = semantic_scores.min()
47
+ semantic_max = semantic_scores.max()
48
+ semantic_range = semantic_max - semantic_min
49
+ if semantic_range > 0:
50
+ semantic_scores_norm = (semantic_scores - semantic_min) / semantic_range
51
+ else:
52
+ semantic_scores_norm = np.zeros_like(semantic_scores)
53
+
54
+ bm25_min = bm25_scores.min()
55
+ bm25_max = bm25_scores.max()
56
+ bm25_range = bm25_max - bm25_min
57
+ if bm25_range > 0:
58
+ bm25_scores_norm = (bm25_scores - bm25_min) / bm25_range
59
+ else:
60
+ bm25_scores_norm = np.zeros_like(bm25_scores)
61
+
62
+ hybrid_scores = (
63
+ self.semantic_weight * semantic_scores_norm
64
+ + self.bm25_weight * bm25_scores_norm
65
+ )
66
+
67
+ top_indices = hybrid_scores.argsort()[::-1][: self.initial_top_k]
68
+
69
+ results: List[SearchResultItem] = []
70
+ for idx in top_indices:
71
+ chunk = chunks[idx]
72
+ result = self._build_result_item(
73
+ table_name=chunk.table_name,
74
+ score=float(hybrid_scores[idx]),
75
+ schema=schemas[chunk.table_name],
76
+ matched_chunks=[chunk.content],
77
+ graph_builder=graph_builder,
78
+ hops=hops,
79
+ )
80
+ results.append(result)
81
+
82
+ return results
@@ -0,0 +1,49 @@
1
+ from typing import Dict, List, Optional
2
+
3
+ import numpy as np
4
+
5
+ from schema_search.search.base import BaseSearchStrategy
6
+ from schema_search.types import TableSchema, SearchResultItem
7
+ from schema_search.chunkers import Chunk
8
+ from schema_search.graph_builder import GraphBuilder
9
+ from schema_search.embedding_cache import BaseEmbeddingCache
10
+ from schema_search.rankers.base import BaseRanker
11
+
12
+
13
+ class SemanticSearchStrategy(BaseSearchStrategy):
14
+ def __init__(
15
+ self,
16
+ embedding_cache: BaseEmbeddingCache,
17
+ initial_top_k: int,
18
+ rerank_top_k: int,
19
+ reranker: Optional[BaseRanker],
20
+ ):
21
+ super().__init__(reranker, initial_top_k, rerank_top_k)
22
+ self.embedding_cache = embedding_cache
23
+
24
+ def _initial_ranking(
25
+ self,
26
+ query: str,
27
+ schemas: Dict[str, TableSchema],
28
+ chunks: List[Chunk],
29
+ graph_builder: GraphBuilder,
30
+ hops: int,
31
+ ) -> List[SearchResultItem]:
32
+ query_embedding = self.embedding_cache.encode_query(query)
33
+ embedding_scores = self.embedding_cache.compute_similarities(query_embedding)
34
+ top_indices = embedding_scores.argsort()[::-1][: self.initial_top_k]
35
+
36
+ results: List[SearchResultItem] = []
37
+ for idx in top_indices:
38
+ chunk = chunks[idx]
39
+ result = self._build_result_item(
40
+ table_name=chunk.table_name,
41
+ score=float(embedding_scores[idx]),
42
+ schema=schemas[chunk.table_name],
43
+ matched_chunks=[chunk.content],
44
+ graph_builder=graph_builder,
45
+ hops=hops,
46
+ )
47
+ results.append(result)
48
+
49
+ return results
schema_search/types.py ADDED
@@ -0,0 +1,57 @@
1
+ from typing import TypedDict, List, Literal, Optional
2
+
3
+
4
+ SearchType = Literal["semantic", "fuzzy", "bm25", "hybrid"]
5
+
6
+
7
+ class ColumnInfo(TypedDict):
8
+ name: str
9
+ type: str
10
+ nullable: bool
11
+ default: Optional[str]
12
+
13
+
14
+ class ForeignKeyInfo(TypedDict):
15
+ constrained_columns: List[str]
16
+ referred_table: str
17
+ referred_columns: List[str]
18
+
19
+
20
+ class IndexInfo(TypedDict):
21
+ name: str
22
+ columns: List[str]
23
+ unique: bool
24
+
25
+
26
+ class ConstraintInfo(TypedDict):
27
+ name: Optional[str]
28
+ columns: List[str]
29
+
30
+
31
+ class TableSchema(TypedDict):
32
+ name: str
33
+ primary_keys: List[str]
34
+ columns: Optional[List[ColumnInfo]]
35
+ foreign_keys: Optional[List[ForeignKeyInfo]]
36
+ indices: Optional[List[IndexInfo]]
37
+ unique_constraints: Optional[List[ConstraintInfo]]
38
+ check_constraints: Optional[List[ConstraintInfo]]
39
+
40
+
41
+ class IndexResult(TypedDict):
42
+ tables: int
43
+ chunks: int
44
+ latency_sec: float
45
+
46
+
47
+ class SearchResultItem(TypedDict):
48
+ table: str
49
+ score: float
50
+ schema: TableSchema
51
+ matched_chunks: List[str]
52
+ related_tables: List[str]
53
+
54
+
55
+ class SearchResult(TypedDict):
56
+ results: List[SearchResultItem]
57
+ latency_sec: float
@@ -0,0 +1,275 @@
1
+ Metadata-Version: 2.4
2
+ Name: schema-search
3
+ Version: 0.1.2
4
+ Summary: Natural language search for database schemas with graph-aware semantic retrieval
5
+ Home-page: https://github.com/neehan/schema-search
6
+ Author:
7
+ Classifier: Development Status :: 3 - Alpha
8
+ Classifier: Intended Audience :: Developers
9
+ Classifier: License :: OSI Approved :: MIT License
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: 3.8
12
+ Classifier: Programming Language :: Python :: 3.9
13
+ Classifier: Programming Language :: Python :: 3.10
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Requires-Python: >=3.8
16
+ Description-Content-Type: text/markdown
17
+ License-File: LICENSE
18
+ Requires-Dist: sqlalchemy>=1.4.0
19
+ Requires-Dist: sentence-transformers>=2.2.0
20
+ Requires-Dist: networkx>=2.8.0
21
+ Requires-Dist: bm25s>=0.2.0
22
+ Requires-Dist: numpy>=1.21.0
23
+ Requires-Dist: pyyaml>=6.0
24
+ Requires-Dist: tqdm>=4.65.0
25
+ Requires-Dist: openai>=1.0.0
26
+ Requires-Dist: rapidfuzz>=3.0.0
27
+ Provides-Extra: mcp
28
+ Requires-Dist: fastmcp>=2.0.0; extra == "mcp"
29
+ Provides-Extra: test
30
+ Requires-Dist: pytest>=7.0.0; extra == "test"
31
+ Requires-Dist: python-dotenv>=1.0.0; extra == "test"
32
+ Requires-Dist: psutil>=5.9.0; extra == "test"
33
+ Requires-Dist: datasets>=2.0.0; extra == "test"
34
+ Provides-Extra: postgres
35
+ Requires-Dist: psycopg2-binary>=2.9.0; extra == "postgres"
36
+ Provides-Extra: mysql
37
+ Requires-Dist: pymysql>=1.0.0; extra == "mysql"
38
+ Provides-Extra: snowflake
39
+ Requires-Dist: snowflake-sqlalchemy>=1.4.0; extra == "snowflake"
40
+ Requires-Dist: snowflake-connector-python>=3.0.0; extra == "snowflake"
41
+ Provides-Extra: bigquery
42
+ Requires-Dist: sqlalchemy-bigquery>=1.6.0; extra == "bigquery"
43
+ Dynamic: classifier
44
+ Dynamic: description
45
+ Dynamic: description-content-type
46
+ Dynamic: home-page
47
+ Dynamic: license-file
48
+ Dynamic: provides-extra
49
+ Dynamic: requires-dist
50
+ Dynamic: requires-python
51
+ Dynamic: summary
52
+
53
+ # Schema Search
54
+
55
+ An MCP Server for Natural Language Search over RDBMS Schemas. Find exact tables you need, with all their relationships mapped out, in milliseconds. No vector database setup is required.
56
+
57
+ ## Why
58
+
59
+ You have 200 tables in your database. Someone asks "where are user refunds stored?"
60
+
61
+ You could:
62
+ - Grep through SQL files for 20 minutes
63
+ - Pass the full schema to an LLM and watch it struggle with 200 tables
64
+
65
+ Or **build schematic embeddings of your tables, store in-memory, and query in natural language in an MCP server**.
66
+
67
+ ### Benefits
68
+ - No vector database setup is required
69
+ - Small memory footprint -- easily scales up to 1000 tables and 10,000+ columns.
70
+ - Millisecond query latency
71
+
72
+ ## Install
73
+
74
+ ```bash
75
+ # With uv - PostgreSQL (recommended)
76
+ uv pip install "schema-search[postgres,mcp]"
77
+
78
+ # With pip - PostgreSQL
79
+ pip install "schema-search[postgres,mcp]"
80
+
81
+ # Other databases
82
+ uv pip install "schema-search[mysql,mcp]" # MySQL
83
+ uv pip install "schema-search[snowflake,mcp]" # Snowflake
84
+ uv pip install "schema-search[bigquery,mcp]" # BigQuery
85
+ ```
86
+
87
+ ## MCP Server
88
+
89
+ Integrate with Claude Desktop or any MCP client.
90
+
91
+ ### Setup
92
+
93
+ Add to your MCP config (e.g., `~/.cursor/mcp.json` or Claude Desktop config):
94
+
95
+ **Using uv (Recommended):**
96
+ ```json
97
+ {
98
+ "mcpServers": {
99
+ "schema-search": {
100
+ "command": "uvx",
101
+ "args": ["schema-search[postgres,mcp]", "postgresql://user:pass@localhost/db", "optional config.yml path", "optional llm_api_key", "optional llm_base_url"]
102
+ }
103
+ }
104
+ }
105
+ ```
106
+
107
+ **Using pip:**
108
+ ```json
109
+ {
110
+ "mcpServers": {
111
+ "schema-search": {
112
+ "command": "path/to/schema-search-mcp", // conda: /Users/<username>/opt/miniconda3/envs/<your env>/bin/schema-search-mcp",
113
+ "args": ["postgresql://user:pass@localhost/db", "optional config.yml path", "optional llm_api_key", "optional llm_base_url"]
114
+ }
115
+ }
116
+ }
117
+ ```
118
+
119
+
120
+ The LLM API key and base url are only required if you use LLM-generated schema summaries (`config.chunking.strategy = 'llm'`).
121
+
122
+ ### CLI Usage
123
+
124
+ ```bash
125
+ schema-search-mcp "postgresql://user:pass@localhost/db"
126
+ ```
127
+
128
+ Optional args: `[config_path] [llm_api_key] [llm_base_url]`
129
+
130
+ The server exposes `schema_search(query, hops, limit)` for natural language schema queries.
131
+
132
+ ## Python Use
133
+
134
+ ```python
135
+ from sqlalchemy import create_engine
136
+ from schema_search import SchemaSearch
137
+
138
+ engine = create_engine("postgresql://user:pass@localhost/db")
139
+ search = SchemaSearch(engine)
140
+
141
+ search.index(force=False) # default is False
142
+ results = search.search("where are user refunds stored?")
143
+
144
+ for result in results['results']:
145
+ print(result['table']) # "refund_transactions"
146
+ print(result['schema']) # Full column info, types, constraints
147
+ print(result['related_tables']) # ["users", "payments", "transactions"]
148
+
149
+ # Override hops, limit, search strategy
150
+ results = search.search("user_table", hops=0, limit=5, search_type="semantic")
151
+
152
+ ```
153
+
154
+ `SchemaSearch.index()` automatically detects schema changes and refreshes cached metadata, so you rarely need to force a reindex manually.
155
+
156
+ ## Configuration
157
+
158
+ Edit `[config.yml](config.yml)`:
159
+
160
+ ```yaml
161
+ logging:
162
+ level: "WARNING"
163
+
164
+ embedding:
165
+ location: "memory" # Options: "memory", "vectordb" (coming soon)
166
+ model: "multi-qa-MiniLM-L6-cos-v1"
167
+ metric: "cosine" # Options: "cosine", "euclidean", "manhattan", "dot"
168
+ batch_size: 32
169
+ show_progress: false
170
+ cache_dir: "/tmp/.schema_search_cache"
171
+
172
+ chunking:
173
+ strategy: "raw" # Options: "raw", "llm"
174
+ max_tokens: 256
175
+ overlap_tokens: 50
176
+ model: "gpt-4o-mini"
177
+
178
+ search:
179
+ # Search strategy: "semantic" (embeddings), "bm25" (BM25 lexical), "fuzzy" (fuzzy string matching), "hybrid" (semantic + bm25)
180
+ strategy: "hybrid"
181
+ initial_top_k: 20
182
+ rerank_top_k: 5
183
+ semantic_weight: 0.67 # For hybrid search (bm25_weight = 1 - semantic_weight)
184
+ hops: 1 # Number of foreign key hops for graph expansion (0-2 recommended)
185
+
186
+ reranker:
187
+ # CrossEncoder model for reranking. Set to null to disable reranking
188
+ model: null # "Alibaba-NLP/gte-reranker-modernbert-base"
189
+
190
+ schema:
191
+ include_columns: true
192
+ include_indices: true
193
+ include_foreign_keys: true
194
+ include_constraints: true
195
+ ```
196
+
197
+ ## Search Strategies
198
+
199
+ Schema Search supports four search strategies:
200
+
201
+ - **semantic**: Embedding-based similarity search using sentence transformers
202
+ - **bm25**: Lexical search using BM25 ranking algorithm
203
+ - **fuzzy**: String matching on table/column names using fuzzy matching
204
+ - **hybrid**: Combines semantic and bm25 scores (default: 67% semantic, 33% fuzzy)
205
+
206
+ Each strategy performs its own initial ranking, then optionally applies CrossEncoder reranking if `reranker.model` is configured. Set `reranker.model` to `null` to disable reranking.
207
+
208
+ ## Performance Comparison
209
+ We [benchmarked](/tests/test_spider_eval.py) on the Spider dataset (1,234 train queries across 18 databases) using the default `config.yml`.
210
+
211
+ **Memory:** The embedding model requires ~90 MB and the optional reranker adds ~155 MB. Actual process memory depends on your Python runtime.
212
+
213
+ ### Without Reranker (`reranker.model: null`)
214
+ ![Without Reranker](img/spider_benchmark_without_reranker.png)
215
+ - **Indexing:** 0.22s ± 0.08s per database (18 total).
216
+ - **Accuracy:** Hybrid leads with Recall@1 62% / MRR 0.93; Semantic follows at Recall@1 58% / MRR 0.89.
217
+ - **Latency:** BM25 and Fuzzy return in ~5ms; Semantic spends ~15ms; Hybrid (semantic + fuzzy) averages 52ms.
218
+ - **Fuzzy baseline:** Recall@1 22%, highlighting the need for semantic signals on natural-language queries.
219
+
220
+ ### With Reranker (`Alibaba-NLP/gte-reranker-modernbert-base`)
221
+ ![With Reranker](img/spider_benchmark_with_reranker.png)
222
+ - **Indexing:** 0.25s ± 0.05s per database (same 18 DBs).
223
+ - **Accuracy:** All strategies converge around Recall@1 62% and MRR ≈ 0.92; Fuzzy jumps from 51% → 92% MRR.
224
+ - **Latency trade-off:** Extra CrossEncoder pass lifts per-query latency to ~0.18–0.29s depending on strategy.
225
+ - **Recommendation:** Enable the reranker when accuracy matters most; disable it for ultra-low-latency lookups.
226
+
227
+
228
+ You can override the search strategy, hops, and limit at query time:
229
+
230
+ ```python
231
+ # Use fuzzy search instead of default
232
+ results = search.search("user_table", search_type="fuzzy")
233
+
234
+ # Use BM25 for keyword-based search
235
+ results = search.search("transactions payments", search_type="bm25")
236
+
237
+ # Use hybrid for best of both worlds
238
+ results = search.search("where are user refunds?", search_type="hybrid")
239
+
240
+ # Override hops and limit
241
+ results = search.search("user refunds", hops=2, limit=10) # Expand 2 hops, return 10 tables
242
+
243
+ # Disable graph expansion
244
+ results = search.search("user_table", hops=0) # Only direct matches, no foreign key traversal
245
+ ```
246
+
247
+ ### LLM Chunking
248
+
249
+ Use LLM to generate semantic summaries instead of raw schema text:
250
+
251
+ 1. Set `strategy: "llm"` in `config.yml`
252
+ 2. Pass API credentials:
253
+
254
+ ```python
255
+ search = SchemaSearch(
256
+ engine,
257
+ llm_api_key="sk-...",
258
+ llm_base_url="https://api.openai.com/v1/" # optional
259
+ )
260
+ ```
261
+
262
+ ## How It Works
263
+
264
+ 1. **Extract schemas** from database using SQLAlchemy inspector
265
+ 2. **Chunk schemas** into digestible pieces (markdown or LLM-generated summaries)
266
+ 3. **Initial search** using selected strategy (semantic/BM25/fuzzy)
267
+ 4. **Expand via foreign keys** to find related tables (configurable hops)
268
+ 5. **Optional reranking** with CrossEncoder to refine results
269
+ 6. Return top tables with full schema and relationships
270
+
271
+ Cache stored in `.schema_search_cache/` (configurable in `config.yml`)
272
+
273
+ ## License
274
+
275
+ MIT
@@ -0,0 +1,38 @@
1
+ schema_search/__init__.py,sha256=06680k1q7pUf1m-1MNhKJGgHyT2NYiyJTLUIOP74dJY,486
2
+ schema_search/graph_builder.py,sha256=oKiVdVI_EB_ZmnxNiIV7Dt-jyKjV8B1RlbiSWpOSe30,2140
3
+ schema_search/mcp_server.py,sha256=LRAr0DjyjjTucKuwfH8m2wMY3qPVX-0uEJg886js_N8,2479
4
+ schema_search/metrics.py,sha256=veyPo23aysiU_1MCwTVbBcVNreZFr_RGJwMCKBq1RAs,913
5
+ schema_search/schema_extractor.py,sha256=tpFF5FNPT694qZNoPZoRBjMSZySDt0CxUU0Ljtno6Z8,4280
6
+ schema_search/schema_search.py,sha256=60Xk8R3K--Sjsg4TUOiciEKqVe-lNns7K7O1iFsoxq0,8845
7
+ schema_search/types.py,sha256=0CbG57j6orJawBaKjAMG28sfFARh3jSusoQ6gsA4PRc,1156
8
+ schema_search/chunkers/__init__.py,sha256=nBZZCZHIvqpmWBR5Noef7j5yyTEcFZz_ZNZuDCYuQt0,314
9
+ schema_search/chunkers/base.py,sha256=J7K-EO5SCZ9x0m7mnpjCjud9z5mLyq5zB-wt1ziHheY,2841
10
+ schema_search/chunkers/factory.py,sha256=ue2M9MUIEtLZN_3sHXumHmUKXDrYD4Hu7L6EIYgBk1s,1108
11
+ schema_search/chunkers/llm.py,sha256=oSkAKZA6naH14gftViKUnqwKV_cPtBSXGYkE5l3xyeg,1618
12
+ schema_search/chunkers/markdown.py,sha256=LBFr9E3LVQV0UPW6X-qyNlfTYSfJGE8oxCl_FzbkAEw,940
13
+ schema_search/embedding_cache/__init__.py,sha256=cOOPGCcQqw_ywnMxcZbOJXfHHoqTRVS1Mj638kHjJoU,299
14
+ schema_search/embedding_cache/base.py,sha256=D9Izn3h319_HQIz9I_SrjWrdemnYg4aV8dQ0rppa3Y4,978
15
+ schema_search/embedding_cache/bm25.py,sha256=D_wiXWnigJhnOU_wD3p4xbjtUB70sL_7yYrV6eAuTy0,1940
16
+ schema_search/embedding_cache/factory.py,sha256=XJ6_PJV0Vsk-XNwW0xFJcuMO12vNExOhDcrgrZEH_u8,739
17
+ schema_search/embedding_cache/inmemory.py,sha256=7n2zJQ2OeaqPPs37mRAdpw1R2s3-gPzVCejLKJNEJ7Y,3571
18
+ schema_search/rankers/__init__.py,sha256=0pNYKAvWSuspeksB9uaCLYuXD82rT7jqT_jEBXzJ-rw,238
19
+ schema_search/rankers/base.py,sha256=HpYM_ljsRseMXJPgjKcDse58VdVRhg0aAYgmLJm-rZU,1467
20
+ schema_search/rankers/cross_encoder.py,sha256=pD-Wzqi67XQc3rwx-oOjvZzi-XMd_rRkIsNYtEjfJDI,1210
21
+ schema_search/rankers/factory.py,sha256=EVwd_kaHyg4TlVji1gt6Qb9BQS7D8kP7DsSq8DoNt4M,368
22
+ schema_search/search/__init__.py,sha256=pWMX755FaxAr0hbaZp4Qk2V8KH5W4CDjSbxFRavMmgw,545
23
+ schema_search/search/base.py,sha256=XmG8UewFfr0f4bEw0aMZPVoX2HxInGHZ-5ebB1x9ZEY,2573
24
+ schema_search/search/bm25.py,sha256=jQHRFTuKGhIcSt3UdM7JVd5MIbpsuIi9uAevRn6ranE,1539
25
+ schema_search/search/factory.py,sha256=wgcx-xnZ8c7uSvu6oP3Fpoabd2Gl8FyJxn7zu3zZYMs,2062
26
+ schema_search/search/fuzzy.py,sha256=Urn2GtJ5h6j0R3HsRkrMfQCLSTU8jtGaHdfYXL_Nb3A,1865
27
+ schema_search/search/hybrid.py,sha256=T1O46SLCPgpCOnTw2bznnCWmqP9EUkUBLqu5AeQu7oQ,2864
28
+ schema_search/search/semantic.py,sha256=brw7x2hZMCep6QK7WWMT451RnpVcSMuNIZtp51kC6Bo,1673
29
+ schema_search-0.1.2.dist-info/licenses/LICENSE,sha256=jOHFAJEjJCD7iBjS2dBe73X5IGDJdAWGosGOUxfCHTM,1067
30
+ tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
31
+ tests/test_integration.py,sha256=8Iiq9NAwAxMoZcnfR19oOcBEGTyIOmt6nSafG6LWpj0,11959
32
+ tests/test_llm_sql_generation.py,sha256=bj6iwTqXfNEvlrSXnbPxbrgEM2nscbrmYHbT-rNBJZ4,11834
33
+ tests/test_spider_eval.py,sha256=xQwrNXpipaDxk-vIKqSy0nOIl-3Nadtof58nZpsAsZA,15333
34
+ schema_search-0.1.2.dist-info/METADATA,sha256=f7dxa2ZNVeHaY2gUpTbBofEFv5bMu6lBomgAPhCQgOw,9268
35
+ schema_search-0.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
36
+ schema_search-0.1.2.dist-info/entry_points.txt,sha256=I4R8SZvsYXlrDbnGydQrWSw-s-QnZ2VmQl5s_q3qUHI,68
37
+ schema_search-0.1.2.dist-info/top_level.txt,sha256=NZTdQFHoJMezNIhtZICGPOuXlCXQkQduQV925Oqf4sk,20
38
+ schema_search-0.1.2.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (80.9.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ schema-search-mcp = schema_search.mcp_server:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Adib Hasan
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,2 @@
1
+ schema_search
2
+ tests
tests/__init__.py ADDED
File without changes