schema-search 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of schema-search might be problematic. Click here for more details.
- schema_search/__init__.py +26 -0
- schema_search/chunkers/__init__.py +6 -0
- schema_search/chunkers/base.py +95 -0
- schema_search/chunkers/factory.py +31 -0
- schema_search/chunkers/llm.py +51 -0
- schema_search/chunkers/markdown.py +25 -0
- schema_search/embedding_cache/__init__.py +5 -0
- schema_search/embedding_cache/base.py +40 -0
- schema_search/embedding_cache/bm25.py +63 -0
- schema_search/embedding_cache/factory.py +20 -0
- schema_search/embedding_cache/inmemory.py +112 -0
- schema_search/graph_builder.py +69 -0
- schema_search/mcp_server.py +82 -0
- schema_search/metrics.py +33 -0
- schema_search/rankers/__init__.py +5 -0
- schema_search/rankers/base.py +45 -0
- schema_search/rankers/cross_encoder.py +34 -0
- schema_search/rankers/factory.py +11 -0
- schema_search/schema_extractor.py +135 -0
- schema_search/schema_search.py +263 -0
- schema_search/search/__init__.py +15 -0
- schema_search/search/base.py +85 -0
- schema_search/search/bm25.py +48 -0
- schema_search/search/factory.py +61 -0
- schema_search/search/fuzzy.py +56 -0
- schema_search/search/hybrid.py +82 -0
- schema_search/search/semantic.py +49 -0
- schema_search/types.py +57 -0
- schema_search-0.1.2.dist-info/METADATA +275 -0
- schema_search-0.1.2.dist-info/RECORD +38 -0
- schema_search-0.1.2.dist-info/WHEEL +5 -0
- schema_search-0.1.2.dist-info/entry_points.txt +2 -0
- schema_search-0.1.2.dist-info/licenses/LICENSE +21 -0
- schema_search-0.1.2.dist-info/top_level.txt +2 -0
- tests/__init__.py +0 -0
- tests/test_integration.py +352 -0
- tests/test_llm_sql_generation.py +320 -0
- tests/test_spider_eval.py +484 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
from typing import Callable, Dict, Optional
|
|
2
|
+
|
|
3
|
+
from schema_search.search.semantic import SemanticSearchStrategy
|
|
4
|
+
from schema_search.search.fuzzy import FuzzySearchStrategy
|
|
5
|
+
from schema_search.search.bm25 import BM25SearchStrategy
|
|
6
|
+
from schema_search.search.hybrid import HybridSearchStrategy
|
|
7
|
+
from schema_search.search.base import BaseSearchStrategy
|
|
8
|
+
from schema_search.embedding_cache import BaseEmbeddingCache
|
|
9
|
+
from schema_search.rankers.base import BaseRanker
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def create_search_strategy(
|
|
13
|
+
config: Dict,
|
|
14
|
+
get_embedding_cache: Callable[[], BaseEmbeddingCache],
|
|
15
|
+
get_bm25_cache: Callable,
|
|
16
|
+
get_reranker: Callable[[], Optional[BaseRanker]],
|
|
17
|
+
strategy_type: Optional[str],
|
|
18
|
+
) -> BaseSearchStrategy:
|
|
19
|
+
search_config = config["search"]
|
|
20
|
+
strategy_type = strategy_type or search_config["strategy"]
|
|
21
|
+
|
|
22
|
+
initial_top_k = search_config["initial_top_k"]
|
|
23
|
+
rerank_top_k = search_config["rerank_top_k"]
|
|
24
|
+
|
|
25
|
+
reranker = get_reranker()
|
|
26
|
+
|
|
27
|
+
if strategy_type == "semantic":
|
|
28
|
+
return SemanticSearchStrategy(
|
|
29
|
+
embedding_cache=get_embedding_cache(),
|
|
30
|
+
initial_top_k=initial_top_k,
|
|
31
|
+
rerank_top_k=rerank_top_k,
|
|
32
|
+
reranker=reranker,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
if strategy_type == "bm25":
|
|
36
|
+
return BM25SearchStrategy(
|
|
37
|
+
bm25_cache=get_bm25_cache(),
|
|
38
|
+
initial_top_k=initial_top_k,
|
|
39
|
+
rerank_top_k=rerank_top_k,
|
|
40
|
+
reranker=reranker,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
if strategy_type == "fuzzy":
|
|
44
|
+
return FuzzySearchStrategy(
|
|
45
|
+
initial_top_k=initial_top_k,
|
|
46
|
+
rerank_top_k=rerank_top_k,
|
|
47
|
+
reranker=reranker,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
if strategy_type == "hybrid":
|
|
51
|
+
semantic_weight = search_config["semantic_weight"]
|
|
52
|
+
return HybridSearchStrategy(
|
|
53
|
+
embedding_cache=get_embedding_cache(),
|
|
54
|
+
bm25_cache=get_bm25_cache(),
|
|
55
|
+
initial_top_k=initial_top_k,
|
|
56
|
+
rerank_top_k=rerank_top_k,
|
|
57
|
+
reranker=reranker,
|
|
58
|
+
semantic_weight=semantic_weight,
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
raise ValueError(f"Unknown search strategy: {strategy_type}")
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
from typing import Dict, List, Optional
|
|
2
|
+
|
|
3
|
+
from rapidfuzz import fuzz
|
|
4
|
+
|
|
5
|
+
from schema_search.search.base import BaseSearchStrategy
|
|
6
|
+
from schema_search.types import TableSchema, SearchResultItem
|
|
7
|
+
from schema_search.chunkers import Chunk
|
|
8
|
+
from schema_search.graph_builder import GraphBuilder
|
|
9
|
+
from schema_search.rankers.base import BaseRanker
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class FuzzySearchStrategy(BaseSearchStrategy):
|
|
13
|
+
def __init__(
|
|
14
|
+
self, initial_top_k: int, rerank_top_k: int, reranker: Optional[BaseRanker]
|
|
15
|
+
):
|
|
16
|
+
super().__init__(reranker, initial_top_k, rerank_top_k)
|
|
17
|
+
|
|
18
|
+
def _initial_ranking(
|
|
19
|
+
self,
|
|
20
|
+
query: str,
|
|
21
|
+
schemas: Dict[str, TableSchema],
|
|
22
|
+
chunks: List[Chunk],
|
|
23
|
+
graph_builder: GraphBuilder,
|
|
24
|
+
hops: int,
|
|
25
|
+
) -> List[SearchResultItem]:
|
|
26
|
+
scored_tables: List[tuple[str, float]] = []
|
|
27
|
+
|
|
28
|
+
for table_name, schema in schemas.items():
|
|
29
|
+
searchable_text = self._build_searchable_text(table_name, schema)
|
|
30
|
+
score = fuzz.ratio(query, searchable_text, score_cutoff=0) / 100.0
|
|
31
|
+
scored_tables.append((table_name, score))
|
|
32
|
+
|
|
33
|
+
scored_tables.sort(key=lambda x: x[1], reverse=True)
|
|
34
|
+
|
|
35
|
+
results: List[SearchResultItem] = []
|
|
36
|
+
for table_name, score in scored_tables[: self.initial_top_k]:
|
|
37
|
+
result = self._build_result_item(
|
|
38
|
+
table_name=table_name,
|
|
39
|
+
score=score,
|
|
40
|
+
schema=schemas[table_name],
|
|
41
|
+
matched_chunks=[],
|
|
42
|
+
graph_builder=graph_builder,
|
|
43
|
+
hops=hops,
|
|
44
|
+
)
|
|
45
|
+
results.append(result)
|
|
46
|
+
|
|
47
|
+
return results
|
|
48
|
+
|
|
49
|
+
def _build_searchable_text(self, table_name: str, schema: TableSchema) -> str:
|
|
50
|
+
parts = [table_name]
|
|
51
|
+
|
|
52
|
+
if schema["indices"]:
|
|
53
|
+
for idx in schema["indices"]:
|
|
54
|
+
parts.append(idx["name"])
|
|
55
|
+
|
|
56
|
+
return " ".join(parts)
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
from typing import Dict, List, Optional, TYPE_CHECKING
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
from schema_search.search.base import BaseSearchStrategy
|
|
6
|
+
from schema_search.types import TableSchema, SearchResultItem
|
|
7
|
+
from schema_search.chunkers import Chunk
|
|
8
|
+
from schema_search.graph_builder import GraphBuilder
|
|
9
|
+
from schema_search.embedding_cache import BaseEmbeddingCache
|
|
10
|
+
from schema_search.rankers.base import BaseRanker
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from schema_search.embedding_cache.bm25 import BM25Cache
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class HybridSearchStrategy(BaseSearchStrategy):
|
|
17
|
+
def __init__(
|
|
18
|
+
self,
|
|
19
|
+
embedding_cache: BaseEmbeddingCache,
|
|
20
|
+
bm25_cache: "BM25Cache",
|
|
21
|
+
initial_top_k: int,
|
|
22
|
+
rerank_top_k: int,
|
|
23
|
+
reranker: Optional[BaseRanker],
|
|
24
|
+
semantic_weight: float,
|
|
25
|
+
):
|
|
26
|
+
super().__init__(reranker, initial_top_k, rerank_top_k)
|
|
27
|
+
assert 0 <= semantic_weight <= 1, "semantic_weight must be between 0 and 1"
|
|
28
|
+
self.embedding_cache = embedding_cache
|
|
29
|
+
self.bm25_cache = bm25_cache
|
|
30
|
+
self.semantic_weight = semantic_weight
|
|
31
|
+
self.bm25_weight = 1 - semantic_weight
|
|
32
|
+
|
|
33
|
+
def _initial_ranking(
|
|
34
|
+
self,
|
|
35
|
+
query: str,
|
|
36
|
+
schemas: Dict[str, TableSchema],
|
|
37
|
+
chunks: List[Chunk],
|
|
38
|
+
graph_builder: GraphBuilder,
|
|
39
|
+
hops: int,
|
|
40
|
+
) -> List[SearchResultItem]:
|
|
41
|
+
query_embedding = self.embedding_cache.encode_query(query)
|
|
42
|
+
semantic_scores = self.embedding_cache.compute_similarities(query_embedding)
|
|
43
|
+
|
|
44
|
+
bm25_scores = self.bm25_cache.get_scores(query)
|
|
45
|
+
|
|
46
|
+
semantic_min = semantic_scores.min()
|
|
47
|
+
semantic_max = semantic_scores.max()
|
|
48
|
+
semantic_range = semantic_max - semantic_min
|
|
49
|
+
if semantic_range > 0:
|
|
50
|
+
semantic_scores_norm = (semantic_scores - semantic_min) / semantic_range
|
|
51
|
+
else:
|
|
52
|
+
semantic_scores_norm = np.zeros_like(semantic_scores)
|
|
53
|
+
|
|
54
|
+
bm25_min = bm25_scores.min()
|
|
55
|
+
bm25_max = bm25_scores.max()
|
|
56
|
+
bm25_range = bm25_max - bm25_min
|
|
57
|
+
if bm25_range > 0:
|
|
58
|
+
bm25_scores_norm = (bm25_scores - bm25_min) / bm25_range
|
|
59
|
+
else:
|
|
60
|
+
bm25_scores_norm = np.zeros_like(bm25_scores)
|
|
61
|
+
|
|
62
|
+
hybrid_scores = (
|
|
63
|
+
self.semantic_weight * semantic_scores_norm
|
|
64
|
+
+ self.bm25_weight * bm25_scores_norm
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
top_indices = hybrid_scores.argsort()[::-1][: self.initial_top_k]
|
|
68
|
+
|
|
69
|
+
results: List[SearchResultItem] = []
|
|
70
|
+
for idx in top_indices:
|
|
71
|
+
chunk = chunks[idx]
|
|
72
|
+
result = self._build_result_item(
|
|
73
|
+
table_name=chunk.table_name,
|
|
74
|
+
score=float(hybrid_scores[idx]),
|
|
75
|
+
schema=schemas[chunk.table_name],
|
|
76
|
+
matched_chunks=[chunk.content],
|
|
77
|
+
graph_builder=graph_builder,
|
|
78
|
+
hops=hops,
|
|
79
|
+
)
|
|
80
|
+
results.append(result)
|
|
81
|
+
|
|
82
|
+
return results
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from typing import Dict, List, Optional
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
from schema_search.search.base import BaseSearchStrategy
|
|
6
|
+
from schema_search.types import TableSchema, SearchResultItem
|
|
7
|
+
from schema_search.chunkers import Chunk
|
|
8
|
+
from schema_search.graph_builder import GraphBuilder
|
|
9
|
+
from schema_search.embedding_cache import BaseEmbeddingCache
|
|
10
|
+
from schema_search.rankers.base import BaseRanker
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class SemanticSearchStrategy(BaseSearchStrategy):
|
|
14
|
+
def __init__(
|
|
15
|
+
self,
|
|
16
|
+
embedding_cache: BaseEmbeddingCache,
|
|
17
|
+
initial_top_k: int,
|
|
18
|
+
rerank_top_k: int,
|
|
19
|
+
reranker: Optional[BaseRanker],
|
|
20
|
+
):
|
|
21
|
+
super().__init__(reranker, initial_top_k, rerank_top_k)
|
|
22
|
+
self.embedding_cache = embedding_cache
|
|
23
|
+
|
|
24
|
+
def _initial_ranking(
|
|
25
|
+
self,
|
|
26
|
+
query: str,
|
|
27
|
+
schemas: Dict[str, TableSchema],
|
|
28
|
+
chunks: List[Chunk],
|
|
29
|
+
graph_builder: GraphBuilder,
|
|
30
|
+
hops: int,
|
|
31
|
+
) -> List[SearchResultItem]:
|
|
32
|
+
query_embedding = self.embedding_cache.encode_query(query)
|
|
33
|
+
embedding_scores = self.embedding_cache.compute_similarities(query_embedding)
|
|
34
|
+
top_indices = embedding_scores.argsort()[::-1][: self.initial_top_k]
|
|
35
|
+
|
|
36
|
+
results: List[SearchResultItem] = []
|
|
37
|
+
for idx in top_indices:
|
|
38
|
+
chunk = chunks[idx]
|
|
39
|
+
result = self._build_result_item(
|
|
40
|
+
table_name=chunk.table_name,
|
|
41
|
+
score=float(embedding_scores[idx]),
|
|
42
|
+
schema=schemas[chunk.table_name],
|
|
43
|
+
matched_chunks=[chunk.content],
|
|
44
|
+
graph_builder=graph_builder,
|
|
45
|
+
hops=hops,
|
|
46
|
+
)
|
|
47
|
+
results.append(result)
|
|
48
|
+
|
|
49
|
+
return results
|
schema_search/types.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
from typing import TypedDict, List, Literal, Optional
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
SearchType = Literal["semantic", "fuzzy", "bm25", "hybrid"]
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ColumnInfo(TypedDict):
|
|
8
|
+
name: str
|
|
9
|
+
type: str
|
|
10
|
+
nullable: bool
|
|
11
|
+
default: Optional[str]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ForeignKeyInfo(TypedDict):
|
|
15
|
+
constrained_columns: List[str]
|
|
16
|
+
referred_table: str
|
|
17
|
+
referred_columns: List[str]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class IndexInfo(TypedDict):
|
|
21
|
+
name: str
|
|
22
|
+
columns: List[str]
|
|
23
|
+
unique: bool
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ConstraintInfo(TypedDict):
|
|
27
|
+
name: Optional[str]
|
|
28
|
+
columns: List[str]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class TableSchema(TypedDict):
|
|
32
|
+
name: str
|
|
33
|
+
primary_keys: List[str]
|
|
34
|
+
columns: Optional[List[ColumnInfo]]
|
|
35
|
+
foreign_keys: Optional[List[ForeignKeyInfo]]
|
|
36
|
+
indices: Optional[List[IndexInfo]]
|
|
37
|
+
unique_constraints: Optional[List[ConstraintInfo]]
|
|
38
|
+
check_constraints: Optional[List[ConstraintInfo]]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class IndexResult(TypedDict):
|
|
42
|
+
tables: int
|
|
43
|
+
chunks: int
|
|
44
|
+
latency_sec: float
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class SearchResultItem(TypedDict):
|
|
48
|
+
table: str
|
|
49
|
+
score: float
|
|
50
|
+
schema: TableSchema
|
|
51
|
+
matched_chunks: List[str]
|
|
52
|
+
related_tables: List[str]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class SearchResult(TypedDict):
|
|
56
|
+
results: List[SearchResultItem]
|
|
57
|
+
latency_sec: float
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: schema-search
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: Natural language search for database schemas with graph-aware semantic retrieval
|
|
5
|
+
Home-page: https://github.com/neehan/schema-search
|
|
6
|
+
Author:
|
|
7
|
+
Classifier: Development Status :: 3 - Alpha
|
|
8
|
+
Classifier: Intended Audience :: Developers
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Requires-Python: >=3.8
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: sqlalchemy>=1.4.0
|
|
19
|
+
Requires-Dist: sentence-transformers>=2.2.0
|
|
20
|
+
Requires-Dist: networkx>=2.8.0
|
|
21
|
+
Requires-Dist: bm25s>=0.2.0
|
|
22
|
+
Requires-Dist: numpy>=1.21.0
|
|
23
|
+
Requires-Dist: pyyaml>=6.0
|
|
24
|
+
Requires-Dist: tqdm>=4.65.0
|
|
25
|
+
Requires-Dist: openai>=1.0.0
|
|
26
|
+
Requires-Dist: rapidfuzz>=3.0.0
|
|
27
|
+
Provides-Extra: mcp
|
|
28
|
+
Requires-Dist: fastmcp>=2.0.0; extra == "mcp"
|
|
29
|
+
Provides-Extra: test
|
|
30
|
+
Requires-Dist: pytest>=7.0.0; extra == "test"
|
|
31
|
+
Requires-Dist: python-dotenv>=1.0.0; extra == "test"
|
|
32
|
+
Requires-Dist: psutil>=5.9.0; extra == "test"
|
|
33
|
+
Requires-Dist: datasets>=2.0.0; extra == "test"
|
|
34
|
+
Provides-Extra: postgres
|
|
35
|
+
Requires-Dist: psycopg2-binary>=2.9.0; extra == "postgres"
|
|
36
|
+
Provides-Extra: mysql
|
|
37
|
+
Requires-Dist: pymysql>=1.0.0; extra == "mysql"
|
|
38
|
+
Provides-Extra: snowflake
|
|
39
|
+
Requires-Dist: snowflake-sqlalchemy>=1.4.0; extra == "snowflake"
|
|
40
|
+
Requires-Dist: snowflake-connector-python>=3.0.0; extra == "snowflake"
|
|
41
|
+
Provides-Extra: bigquery
|
|
42
|
+
Requires-Dist: sqlalchemy-bigquery>=1.6.0; extra == "bigquery"
|
|
43
|
+
Dynamic: classifier
|
|
44
|
+
Dynamic: description
|
|
45
|
+
Dynamic: description-content-type
|
|
46
|
+
Dynamic: home-page
|
|
47
|
+
Dynamic: license-file
|
|
48
|
+
Dynamic: provides-extra
|
|
49
|
+
Dynamic: requires-dist
|
|
50
|
+
Dynamic: requires-python
|
|
51
|
+
Dynamic: summary
|
|
52
|
+
|
|
53
|
+
# Schema Search
|
|
54
|
+
|
|
55
|
+
An MCP Server for Natural Language Search over RDBMS Schemas. Find exact tables you need, with all their relationships mapped out, in milliseconds. No vector database setup is required.
|
|
56
|
+
|
|
57
|
+
## Why
|
|
58
|
+
|
|
59
|
+
You have 200 tables in your database. Someone asks "where are user refunds stored?"
|
|
60
|
+
|
|
61
|
+
You could:
|
|
62
|
+
- Grep through SQL files for 20 minutes
|
|
63
|
+
- Pass the full schema to an LLM and watch it struggle with 200 tables
|
|
64
|
+
|
|
65
|
+
Or **build schematic embeddings of your tables, store in-memory, and query in natural language in an MCP server**.
|
|
66
|
+
|
|
67
|
+
### Benefits
|
|
68
|
+
- No vector database setup is required
|
|
69
|
+
- Small memory footprint -- easily scales up to 1000 tables and 10,000+ columns.
|
|
70
|
+
- Millisecond query latency
|
|
71
|
+
|
|
72
|
+
## Install
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
# With uv - PostgreSQL (recommended)
|
|
76
|
+
uv pip install "schema-search[postgres,mcp]"
|
|
77
|
+
|
|
78
|
+
# With pip - PostgreSQL
|
|
79
|
+
pip install "schema-search[postgres,mcp]"
|
|
80
|
+
|
|
81
|
+
# Other databases
|
|
82
|
+
uv pip install "schema-search[mysql,mcp]" # MySQL
|
|
83
|
+
uv pip install "schema-search[snowflake,mcp]" # Snowflake
|
|
84
|
+
uv pip install "schema-search[bigquery,mcp]" # BigQuery
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## MCP Server
|
|
88
|
+
|
|
89
|
+
Integrate with Claude Desktop or any MCP client.
|
|
90
|
+
|
|
91
|
+
### Setup
|
|
92
|
+
|
|
93
|
+
Add to your MCP config (e.g., `~/.cursor/mcp.json` or Claude Desktop config):
|
|
94
|
+
|
|
95
|
+
**Using uv (Recommended):**
|
|
96
|
+
```json
|
|
97
|
+
{
|
|
98
|
+
"mcpServers": {
|
|
99
|
+
"schema-search": {
|
|
100
|
+
"command": "uvx",
|
|
101
|
+
"args": ["schema-search[postgres,mcp]", "postgresql://user:pass@localhost/db", "optional config.yml path", "optional llm_api_key", "optional llm_base_url"]
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
**Using pip:**
|
|
108
|
+
```json
|
|
109
|
+
{
|
|
110
|
+
"mcpServers": {
|
|
111
|
+
"schema-search": {
|
|
112
|
+
"command": "path/to/schema-search-mcp", // conda: /Users/<username>/opt/miniconda3/envs/<your env>/bin/schema-search-mcp",
|
|
113
|
+
"args": ["postgresql://user:pass@localhost/db", "optional config.yml path", "optional llm_api_key", "optional llm_base_url"]
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
The LLM API key and base url are only required if you use LLM-generated schema summaries (`config.chunking.strategy = 'llm'`).
|
|
121
|
+
|
|
122
|
+
### CLI Usage
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
schema-search-mcp "postgresql://user:pass@localhost/db"
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Optional args: `[config_path] [llm_api_key] [llm_base_url]`
|
|
129
|
+
|
|
130
|
+
The server exposes `schema_search(query, hops, limit)` for natural language schema queries.
|
|
131
|
+
|
|
132
|
+
## Python Use
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
from sqlalchemy import create_engine
|
|
136
|
+
from schema_search import SchemaSearch
|
|
137
|
+
|
|
138
|
+
engine = create_engine("postgresql://user:pass@localhost/db")
|
|
139
|
+
search = SchemaSearch(engine)
|
|
140
|
+
|
|
141
|
+
search.index(force=False) # default is False
|
|
142
|
+
results = search.search("where are user refunds stored?")
|
|
143
|
+
|
|
144
|
+
for result in results['results']:
|
|
145
|
+
print(result['table']) # "refund_transactions"
|
|
146
|
+
print(result['schema']) # Full column info, types, constraints
|
|
147
|
+
print(result['related_tables']) # ["users", "payments", "transactions"]
|
|
148
|
+
|
|
149
|
+
# Override hops, limit, search strategy
|
|
150
|
+
results = search.search("user_table", hops=0, limit=5, search_type="semantic")
|
|
151
|
+
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
`SchemaSearch.index()` automatically detects schema changes and refreshes cached metadata, so you rarely need to force a reindex manually.
|
|
155
|
+
|
|
156
|
+
## Configuration
|
|
157
|
+
|
|
158
|
+
Edit `[config.yml](config.yml)`:
|
|
159
|
+
|
|
160
|
+
```yaml
|
|
161
|
+
logging:
|
|
162
|
+
level: "WARNING"
|
|
163
|
+
|
|
164
|
+
embedding:
|
|
165
|
+
location: "memory" # Options: "memory", "vectordb" (coming soon)
|
|
166
|
+
model: "multi-qa-MiniLM-L6-cos-v1"
|
|
167
|
+
metric: "cosine" # Options: "cosine", "euclidean", "manhattan", "dot"
|
|
168
|
+
batch_size: 32
|
|
169
|
+
show_progress: false
|
|
170
|
+
cache_dir: "/tmp/.schema_search_cache"
|
|
171
|
+
|
|
172
|
+
chunking:
|
|
173
|
+
strategy: "raw" # Options: "raw", "llm"
|
|
174
|
+
max_tokens: 256
|
|
175
|
+
overlap_tokens: 50
|
|
176
|
+
model: "gpt-4o-mini"
|
|
177
|
+
|
|
178
|
+
search:
|
|
179
|
+
# Search strategy: "semantic" (embeddings), "bm25" (BM25 lexical), "fuzzy" (fuzzy string matching), "hybrid" (semantic + bm25)
|
|
180
|
+
strategy: "hybrid"
|
|
181
|
+
initial_top_k: 20
|
|
182
|
+
rerank_top_k: 5
|
|
183
|
+
semantic_weight: 0.67 # For hybrid search (bm25_weight = 1 - semantic_weight)
|
|
184
|
+
hops: 1 # Number of foreign key hops for graph expansion (0-2 recommended)
|
|
185
|
+
|
|
186
|
+
reranker:
|
|
187
|
+
# CrossEncoder model for reranking. Set to null to disable reranking
|
|
188
|
+
model: null # "Alibaba-NLP/gte-reranker-modernbert-base"
|
|
189
|
+
|
|
190
|
+
schema:
|
|
191
|
+
include_columns: true
|
|
192
|
+
include_indices: true
|
|
193
|
+
include_foreign_keys: true
|
|
194
|
+
include_constraints: true
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
## Search Strategies
|
|
198
|
+
|
|
199
|
+
Schema Search supports four search strategies:
|
|
200
|
+
|
|
201
|
+
- **semantic**: Embedding-based similarity search using sentence transformers
|
|
202
|
+
- **bm25**: Lexical search using BM25 ranking algorithm
|
|
203
|
+
- **fuzzy**: String matching on table/column names using fuzzy matching
|
|
204
|
+
- **hybrid**: Combines semantic and bm25 scores (default: 67% semantic, 33% fuzzy)
|
|
205
|
+
|
|
206
|
+
Each strategy performs its own initial ranking, then optionally applies CrossEncoder reranking if `reranker.model` is configured. Set `reranker.model` to `null` to disable reranking.
|
|
207
|
+
|
|
208
|
+
## Performance Comparison
|
|
209
|
+
We [benchmarked](/tests/test_spider_eval.py) on the Spider dataset (1,234 train queries across 18 databases) using the default `config.yml`.
|
|
210
|
+
|
|
211
|
+
**Memory:** The embedding model requires ~90 MB and the optional reranker adds ~155 MB. Actual process memory depends on your Python runtime.
|
|
212
|
+
|
|
213
|
+
### Without Reranker (`reranker.model: null`)
|
|
214
|
+

|
|
215
|
+
- **Indexing:** 0.22s ± 0.08s per database (18 total).
|
|
216
|
+
- **Accuracy:** Hybrid leads with Recall@1 62% / MRR 0.93; Semantic follows at Recall@1 58% / MRR 0.89.
|
|
217
|
+
- **Latency:** BM25 and Fuzzy return in ~5ms; Semantic spends ~15ms; Hybrid (semantic + fuzzy) averages 52ms.
|
|
218
|
+
- **Fuzzy baseline:** Recall@1 22%, highlighting the need for semantic signals on natural-language queries.
|
|
219
|
+
|
|
220
|
+
### With Reranker (`Alibaba-NLP/gte-reranker-modernbert-base`)
|
|
221
|
+

|
|
222
|
+
- **Indexing:** 0.25s ± 0.05s per database (same 18 DBs).
|
|
223
|
+
- **Accuracy:** All strategies converge around Recall@1 62% and MRR ≈ 0.92; Fuzzy jumps from 51% → 92% MRR.
|
|
224
|
+
- **Latency trade-off:** Extra CrossEncoder pass lifts per-query latency to ~0.18–0.29s depending on strategy.
|
|
225
|
+
- **Recommendation:** Enable the reranker when accuracy matters most; disable it for ultra-low-latency lookups.
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
You can override the search strategy, hops, and limit at query time:
|
|
229
|
+
|
|
230
|
+
```python
|
|
231
|
+
# Use fuzzy search instead of default
|
|
232
|
+
results = search.search("user_table", search_type="fuzzy")
|
|
233
|
+
|
|
234
|
+
# Use BM25 for keyword-based search
|
|
235
|
+
results = search.search("transactions payments", search_type="bm25")
|
|
236
|
+
|
|
237
|
+
# Use hybrid for best of both worlds
|
|
238
|
+
results = search.search("where are user refunds?", search_type="hybrid")
|
|
239
|
+
|
|
240
|
+
# Override hops and limit
|
|
241
|
+
results = search.search("user refunds", hops=2, limit=10) # Expand 2 hops, return 10 tables
|
|
242
|
+
|
|
243
|
+
# Disable graph expansion
|
|
244
|
+
results = search.search("user_table", hops=0) # Only direct matches, no foreign key traversal
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
### LLM Chunking
|
|
248
|
+
|
|
249
|
+
Use LLM to generate semantic summaries instead of raw schema text:
|
|
250
|
+
|
|
251
|
+
1. Set `strategy: "llm"` in `config.yml`
|
|
252
|
+
2. Pass API credentials:
|
|
253
|
+
|
|
254
|
+
```python
|
|
255
|
+
search = SchemaSearch(
|
|
256
|
+
engine,
|
|
257
|
+
llm_api_key="sk-...",
|
|
258
|
+
llm_base_url="https://api.openai.com/v1/" # optional
|
|
259
|
+
)
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
## How It Works
|
|
263
|
+
|
|
264
|
+
1. **Extract schemas** from database using SQLAlchemy inspector
|
|
265
|
+
2. **Chunk schemas** into digestible pieces (markdown or LLM-generated summaries)
|
|
266
|
+
3. **Initial search** using selected strategy (semantic/BM25/fuzzy)
|
|
267
|
+
4. **Expand via foreign keys** to find related tables (configurable hops)
|
|
268
|
+
5. **Optional reranking** with CrossEncoder to refine results
|
|
269
|
+
6. Return top tables with full schema and relationships
|
|
270
|
+
|
|
271
|
+
Cache stored in `.schema_search_cache/` (configurable in `config.yml`)
|
|
272
|
+
|
|
273
|
+
## License
|
|
274
|
+
|
|
275
|
+
MIT
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
schema_search/__init__.py,sha256=06680k1q7pUf1m-1MNhKJGgHyT2NYiyJTLUIOP74dJY,486
|
|
2
|
+
schema_search/graph_builder.py,sha256=oKiVdVI_EB_ZmnxNiIV7Dt-jyKjV8B1RlbiSWpOSe30,2140
|
|
3
|
+
schema_search/mcp_server.py,sha256=LRAr0DjyjjTucKuwfH8m2wMY3qPVX-0uEJg886js_N8,2479
|
|
4
|
+
schema_search/metrics.py,sha256=veyPo23aysiU_1MCwTVbBcVNreZFr_RGJwMCKBq1RAs,913
|
|
5
|
+
schema_search/schema_extractor.py,sha256=tpFF5FNPT694qZNoPZoRBjMSZySDt0CxUU0Ljtno6Z8,4280
|
|
6
|
+
schema_search/schema_search.py,sha256=60Xk8R3K--Sjsg4TUOiciEKqVe-lNns7K7O1iFsoxq0,8845
|
|
7
|
+
schema_search/types.py,sha256=0CbG57j6orJawBaKjAMG28sfFARh3jSusoQ6gsA4PRc,1156
|
|
8
|
+
schema_search/chunkers/__init__.py,sha256=nBZZCZHIvqpmWBR5Noef7j5yyTEcFZz_ZNZuDCYuQt0,314
|
|
9
|
+
schema_search/chunkers/base.py,sha256=J7K-EO5SCZ9x0m7mnpjCjud9z5mLyq5zB-wt1ziHheY,2841
|
|
10
|
+
schema_search/chunkers/factory.py,sha256=ue2M9MUIEtLZN_3sHXumHmUKXDrYD4Hu7L6EIYgBk1s,1108
|
|
11
|
+
schema_search/chunkers/llm.py,sha256=oSkAKZA6naH14gftViKUnqwKV_cPtBSXGYkE5l3xyeg,1618
|
|
12
|
+
schema_search/chunkers/markdown.py,sha256=LBFr9E3LVQV0UPW6X-qyNlfTYSfJGE8oxCl_FzbkAEw,940
|
|
13
|
+
schema_search/embedding_cache/__init__.py,sha256=cOOPGCcQqw_ywnMxcZbOJXfHHoqTRVS1Mj638kHjJoU,299
|
|
14
|
+
schema_search/embedding_cache/base.py,sha256=D9Izn3h319_HQIz9I_SrjWrdemnYg4aV8dQ0rppa3Y4,978
|
|
15
|
+
schema_search/embedding_cache/bm25.py,sha256=D_wiXWnigJhnOU_wD3p4xbjtUB70sL_7yYrV6eAuTy0,1940
|
|
16
|
+
schema_search/embedding_cache/factory.py,sha256=XJ6_PJV0Vsk-XNwW0xFJcuMO12vNExOhDcrgrZEH_u8,739
|
|
17
|
+
schema_search/embedding_cache/inmemory.py,sha256=7n2zJQ2OeaqPPs37mRAdpw1R2s3-gPzVCejLKJNEJ7Y,3571
|
|
18
|
+
schema_search/rankers/__init__.py,sha256=0pNYKAvWSuspeksB9uaCLYuXD82rT7jqT_jEBXzJ-rw,238
|
|
19
|
+
schema_search/rankers/base.py,sha256=HpYM_ljsRseMXJPgjKcDse58VdVRhg0aAYgmLJm-rZU,1467
|
|
20
|
+
schema_search/rankers/cross_encoder.py,sha256=pD-Wzqi67XQc3rwx-oOjvZzi-XMd_rRkIsNYtEjfJDI,1210
|
|
21
|
+
schema_search/rankers/factory.py,sha256=EVwd_kaHyg4TlVji1gt6Qb9BQS7D8kP7DsSq8DoNt4M,368
|
|
22
|
+
schema_search/search/__init__.py,sha256=pWMX755FaxAr0hbaZp4Qk2V8KH5W4CDjSbxFRavMmgw,545
|
|
23
|
+
schema_search/search/base.py,sha256=XmG8UewFfr0f4bEw0aMZPVoX2HxInGHZ-5ebB1x9ZEY,2573
|
|
24
|
+
schema_search/search/bm25.py,sha256=jQHRFTuKGhIcSt3UdM7JVd5MIbpsuIi9uAevRn6ranE,1539
|
|
25
|
+
schema_search/search/factory.py,sha256=wgcx-xnZ8c7uSvu6oP3Fpoabd2Gl8FyJxn7zu3zZYMs,2062
|
|
26
|
+
schema_search/search/fuzzy.py,sha256=Urn2GtJ5h6j0R3HsRkrMfQCLSTU8jtGaHdfYXL_Nb3A,1865
|
|
27
|
+
schema_search/search/hybrid.py,sha256=T1O46SLCPgpCOnTw2bznnCWmqP9EUkUBLqu5AeQu7oQ,2864
|
|
28
|
+
schema_search/search/semantic.py,sha256=brw7x2hZMCep6QK7WWMT451RnpVcSMuNIZtp51kC6Bo,1673
|
|
29
|
+
schema_search-0.1.2.dist-info/licenses/LICENSE,sha256=jOHFAJEjJCD7iBjS2dBe73X5IGDJdAWGosGOUxfCHTM,1067
|
|
30
|
+
tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
31
|
+
tests/test_integration.py,sha256=8Iiq9NAwAxMoZcnfR19oOcBEGTyIOmt6nSafG6LWpj0,11959
|
|
32
|
+
tests/test_llm_sql_generation.py,sha256=bj6iwTqXfNEvlrSXnbPxbrgEM2nscbrmYHbT-rNBJZ4,11834
|
|
33
|
+
tests/test_spider_eval.py,sha256=xQwrNXpipaDxk-vIKqSy0nOIl-3Nadtof58nZpsAsZA,15333
|
|
34
|
+
schema_search-0.1.2.dist-info/METADATA,sha256=f7dxa2ZNVeHaY2gUpTbBofEFv5bMu6lBomgAPhCQgOw,9268
|
|
35
|
+
schema_search-0.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
36
|
+
schema_search-0.1.2.dist-info/entry_points.txt,sha256=I4R8SZvsYXlrDbnGydQrWSw-s-QnZ2VmQl5s_q3qUHI,68
|
|
37
|
+
schema_search-0.1.2.dist-info/top_level.txt,sha256=NZTdQFHoJMezNIhtZICGPOuXlCXQkQduQV925Oqf4sk,20
|
|
38
|
+
schema_search-0.1.2.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Adib Hasan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
tests/__init__.py
ADDED
|
File without changes
|