pghybrid 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pghybrid-0.1.0/.gitignore +33 -0
- pghybrid-0.1.0/PKG-INFO +134 -0
- pghybrid-0.1.0/README.md +93 -0
- pghybrid-0.1.0/pyproject.toml +105 -0
- pghybrid-0.1.0/src/pghybrid/__init__.py +100 -0
- pghybrid-0.1.0/src/pghybrid/adapters.py +187 -0
- pghybrid-0.1.0/src/pghybrid/cli.py +465 -0
- pghybrid-0.1.0/src/pghybrid/config.py +272 -0
- pghybrid-0.1.0/src/pghybrid/doctor.py +1795 -0
- pghybrid-0.1.0/src/pghybrid/explain.py +1503 -0
- pghybrid-0.1.0/src/pghybrid/py.typed +0 -0
- pghybrid-0.1.0/src/pghybrid/schema.py +1485 -0
- pghybrid-0.1.0/src/pghybrid/search.py +407 -0
- pghybrid-0.1.0/src/pghybrid/sql.py +582 -0
- pghybrid-0.1.0/src/pghybrid/textquery.py +81 -0
- pghybrid-0.1.0/tests/conftest.py +101 -0
- pghybrid-0.1.0/tests/golden/canonical_search.sql +60 -0
- pghybrid-0.1.0/tests/test_cli.py +306 -0
- pghybrid-0.1.0/tests/test_config.py +394 -0
- pghybrid-0.1.0/tests/test_integration.py +1584 -0
- pghybrid-0.1.0/tests/test_sql.py +1014 -0
- pghybrid-0.1.0/tests/test_textquery.py +386 -0
- pghybrid-0.1.0/uv.lock +3398 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
.venv/
|
|
5
|
+
venv/
|
|
6
|
+
dist/
|
|
7
|
+
build/
|
|
8
|
+
*.egg-info/
|
|
9
|
+
.pytest_cache/
|
|
10
|
+
.ruff_cache/
|
|
11
|
+
.mypy_cache/
|
|
12
|
+
.coverage
|
|
13
|
+
htmlcov/
|
|
14
|
+
|
|
15
|
+
# Node
|
|
16
|
+
node_modules/
|
|
17
|
+
js/dist/
|
|
18
|
+
*.tsbuildinfo
|
|
19
|
+
.npm/
|
|
20
|
+
|
|
21
|
+
# Editors / OS
|
|
22
|
+
.DS_Store
|
|
23
|
+
.idea/
|
|
24
|
+
.vscode/
|
|
25
|
+
*.swp
|
|
26
|
+
|
|
27
|
+
# Local
|
|
28
|
+
.env
|
|
29
|
+
.env.local
|
|
30
|
+
|
|
31
|
+
# Scratch files
|
|
32
|
+
js/dbg*.mjs
|
|
33
|
+
js/e2e-check.mjs
|
pghybrid-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pghybrid
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Hybrid search on the Postgres you already have. Vector + full-text + Reciprocal Rank Fusion on plain pgvector, with no extensions to install.
|
|
5
|
+
Project-URL: Homepage, https://github.com/pavangupta352/pghybrid
|
|
6
|
+
Project-URL: Repository, https://github.com/pavangupta352/pghybrid
|
|
7
|
+
Project-URL: Issues, https://github.com/pavangupta352/pghybrid/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/pavangupta352/pghybrid/blob/main/CHANGELOG.md
|
|
9
|
+
Author-email: Pavan Gupta <pavan.gupta.352@gmail.com>
|
|
10
|
+
License: MIT
|
|
11
|
+
Keywords: bm25,full-text-search,hybrid-search,pgvector,postgres,postgresql,rag,reciprocal-rank-fusion,retrieval,rrf,semantic-search,supabase,tsvector,vector-search
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Programming Language :: SQL
|
|
22
|
+
Classifier: Topic :: Database
|
|
23
|
+
Classifier: Topic :: Text Processing :: Indexing
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.9
|
|
26
|
+
Provides-Extra: cli
|
|
27
|
+
Requires-Dist: psycopg[binary]>=3.1; extra == 'cli'
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: asyncpg>=0.29; extra == 'dev'
|
|
30
|
+
Requires-Dist: django>=4.2; extra == 'dev'
|
|
31
|
+
Requires-Dist: langchain-core>=0.3; extra == 'dev'
|
|
32
|
+
Requires-Dist: mypy>=1.11; extra == 'dev'
|
|
33
|
+
Requires-Dist: psycopg-pool>=3.2; extra == 'dev'
|
|
34
|
+
Requires-Dist: psycopg[binary]>=3.1; extra == 'dev'
|
|
35
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
36
|
+
Requires-Dist: pytest-cov>=5.0; extra == 'dev'
|
|
37
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
38
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
39
|
+
Requires-Dist: sqlalchemy>=2.0; extra == 'dev'
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# pghybrid
|
|
43
|
+
|
|
44
|
+
**Hybrid search on the Postgres you already have.**
|
|
45
|
+
|
|
46
|
+
Vector similarity + full-text search, combined by Reciprocal Rank Fusion, on plain
|
|
47
|
+
`pgvector`. No `pg_search`, no VectorChord, no Elasticsearch, no vector database, no
|
|
48
|
+
extension you need superuser to install.
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
pip install pghybrid
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Why
|
|
55
|
+
|
|
56
|
+
Hybrid search on Postgres is a solved problem *if you can install a C extension*.
|
|
57
|
+
`pg_search`, VectorChord and `pg_textsearch` are all excellent, and all of them are
|
|
58
|
+
extensions — and on managed Postgres you usually cannot install extensions at all.
|
|
59
|
+
|
|
60
|
+
**`pgvector` is available almost everywhere. BM25 extensions are available almost
|
|
61
|
+
nowhere:** not on RDS, Aurora, Cloud SQL, Azure Database, Supabase or Heroku, and
|
|
62
|
+
[removed from Neon for new projects in March 2026](https://neon.com/docs/extensions/pg_search).
|
|
63
|
+
|
|
64
|
+
This is not a claim to search better than ParadeDB — `pg_search`'s BM25 genuinely beats
|
|
65
|
+
`ts_rank_cd`, and if you can install it you probably should. It is search you can
|
|
66
|
+
actually install.
|
|
67
|
+
|
|
68
|
+
## The case it exists for
|
|
69
|
+
|
|
70
|
+
A contract. Someone asks **"renewal notice period"**. The clause that answers it says
|
|
71
|
+
*"sixty days written notice prior to the anniversary date"* — it never uses the word
|
|
72
|
+
*renewal*. Vector search puts a plausible-but-wrong clause first. Keyword search puts the
|
|
73
|
+
clause that uses all three words and answers none of them first. The right answer is
|
|
74
|
+
second on both signals and first on neither, which is exactly what rank fusion is for.
|
|
75
|
+
|
|
76
|
+
## Use it
|
|
77
|
+
|
|
78
|
+
<!-- check:python -->
|
|
79
|
+
```python
|
|
80
|
+
from pghybrid import Config, HybridSearch
|
|
81
|
+
|
|
82
|
+
search = HybridSearch(
|
|
83
|
+
Config(table="chunks", text_column="content", vector_column="embedding",
|
|
84
|
+
tsvector_column="fts", extra_columns=["title"], paramstyle="pyformat"),
|
|
85
|
+
execute=lambda sql, params: conn.execute(sql, params).fetchall(),
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
for row in search.search("renewal notice period", embedding=query_vector, limit=10):
|
|
89
|
+
print(row.score, row.matched_by, row.get("title"))
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
You pass the embedding in. `pghybrid` never calls a model, so it works with OpenAI,
|
|
93
|
+
Cohere, Voyage, a local sentence-transformer or anything else, and needs no API key. It
|
|
94
|
+
never opens a connection either — it generates SQL and hands it to the driver you already
|
|
95
|
+
use. `for_psycopg`, `for_sqlalchemy`, `for_asyncpg` and `for_django` set the placeholder
|
|
96
|
+
style for you.
|
|
97
|
+
|
|
98
|
+
`row.matched_by` tells you which signal found each row — `both`, `vector` or `text` —
|
|
99
|
+
which is usually the first thing you want when a search returns the wrong thing.
|
|
100
|
+
|
|
101
|
+
## Read the statement instead of running it
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
sql, params = search.build_sql("renewal notice period", query_vector, limit=10)
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
It is one query with a candidate CTE per signal, fused by RRF. Nothing is hidden, and
|
|
108
|
+
`pghybrid sql` prints it without a database at all.
|
|
109
|
+
|
|
110
|
+
## Command line
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
pghybrid init --dsn "$DATABASE_URL" --table chunks # inspects, writes the migration
|
|
114
|
+
pghybrid doctor --dsn "$DATABASE_URL" --table chunks # measured recall, read-only
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
`doctor` measures recall@k against exact search, sweeps `ef_search`/`probes`, catches
|
|
118
|
+
queries that silently fall back to a sequential scan, and reports a `tsvector` that has
|
|
119
|
+
stopped matching its text — a failure that returns wrong answers without erroring.
|
|
120
|
+
|
|
121
|
+
`explain` decomposes a single result set: both ranks, both raw scores, each signal's
|
|
122
|
+
contribution, and the near-miss band just below your cut-off.
|
|
123
|
+
|
|
124
|
+
## Also in TypeScript
|
|
125
|
+
|
|
126
|
+
[`npm install pghybrid`](https://www.npmjs.com/package/pghybrid) generates byte-identical
|
|
127
|
+
SQL, checked on every commit rather than assumed.
|
|
128
|
+
|
|
129
|
+
---
|
|
130
|
+
|
|
131
|
+
**Full documentation, the reasoning, and copy-and-paste SQL:
|
|
132
|
+
[github.com/pavangupta352/pghybrid](https://github.com/pavangupta352/pghybrid#readme)**
|
|
133
|
+
|
|
134
|
+
MIT © Pavan Gupta
|
pghybrid-0.1.0/README.md
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# pghybrid
|
|
2
|
+
|
|
3
|
+
**Hybrid search on the Postgres you already have.**
|
|
4
|
+
|
|
5
|
+
Vector similarity + full-text search, combined by Reciprocal Rank Fusion, on plain
|
|
6
|
+
`pgvector`. No `pg_search`, no VectorChord, no Elasticsearch, no vector database, no
|
|
7
|
+
extension you need superuser to install.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
pip install pghybrid
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Why
|
|
14
|
+
|
|
15
|
+
Hybrid search on Postgres is a solved problem *if you can install a C extension*.
|
|
16
|
+
`pg_search`, VectorChord and `pg_textsearch` are all excellent, and all of them are
|
|
17
|
+
extensions — and on managed Postgres you usually cannot install extensions at all.
|
|
18
|
+
|
|
19
|
+
**`pgvector` is available almost everywhere. BM25 extensions are available almost
|
|
20
|
+
nowhere:** not on RDS, Aurora, Cloud SQL, Azure Database, Supabase or Heroku, and
|
|
21
|
+
[removed from Neon for new projects in March 2026](https://neon.com/docs/extensions/pg_search).
|
|
22
|
+
|
|
23
|
+
This is not a claim to search better than ParadeDB — `pg_search`'s BM25 genuinely beats
|
|
24
|
+
`ts_rank_cd`, and if you can install it you probably should. It is search you can
|
|
25
|
+
actually install.
|
|
26
|
+
|
|
27
|
+
## The case it exists for
|
|
28
|
+
|
|
29
|
+
A contract. Someone asks **"renewal notice period"**. The clause that answers it says
|
|
30
|
+
*"sixty days written notice prior to the anniversary date"* — it never uses the word
|
|
31
|
+
*renewal*. Vector search puts a plausible-but-wrong clause first. Keyword search puts the
|
|
32
|
+
clause that uses all three words and answers none of them first. The right answer is
|
|
33
|
+
second on both signals and first on neither, which is exactly what rank fusion is for.
|
|
34
|
+
|
|
35
|
+
## Use it
|
|
36
|
+
|
|
37
|
+
<!-- check:python -->
|
|
38
|
+
```python
|
|
39
|
+
from pghybrid import Config, HybridSearch
|
|
40
|
+
|
|
41
|
+
search = HybridSearch(
|
|
42
|
+
Config(table="chunks", text_column="content", vector_column="embedding",
|
|
43
|
+
tsvector_column="fts", extra_columns=["title"], paramstyle="pyformat"),
|
|
44
|
+
execute=lambda sql, params: conn.execute(sql, params).fetchall(),
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
for row in search.search("renewal notice period", embedding=query_vector, limit=10):
|
|
48
|
+
print(row.score, row.matched_by, row.get("title"))
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
You pass the embedding in. `pghybrid` never calls a model, so it works with OpenAI,
|
|
52
|
+
Cohere, Voyage, a local sentence-transformer or anything else, and needs no API key. It
|
|
53
|
+
never opens a connection either — it generates SQL and hands it to the driver you already
|
|
54
|
+
use. `for_psycopg`, `for_sqlalchemy`, `for_asyncpg` and `for_django` set the placeholder
|
|
55
|
+
style for you.
|
|
56
|
+
|
|
57
|
+
`row.matched_by` tells you which signal found each row — `both`, `vector` or `text` —
|
|
58
|
+
which is usually the first thing you want when a search returns the wrong thing.
|
|
59
|
+
|
|
60
|
+
## Read the statement instead of running it
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
sql, params = search.build_sql("renewal notice period", query_vector, limit=10)
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
It is one query with a candidate CTE per signal, fused by RRF. Nothing is hidden, and
|
|
67
|
+
`pghybrid sql` prints it without a database at all.
|
|
68
|
+
|
|
69
|
+
## Command line
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
pghybrid init --dsn "$DATABASE_URL" --table chunks # inspects, writes the migration
|
|
73
|
+
pghybrid doctor --dsn "$DATABASE_URL" --table chunks # measured recall, read-only
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
`doctor` measures recall@k against exact search, sweeps `ef_search`/`probes`, catches
|
|
77
|
+
queries that silently fall back to a sequential scan, and reports a `tsvector` that has
|
|
78
|
+
stopped matching its text — a failure that returns wrong answers without erroring.
|
|
79
|
+
|
|
80
|
+
`explain` decomposes a single result set: both ranks, both raw scores, each signal's
|
|
81
|
+
contribution, and the near-miss band just below your cut-off.
|
|
82
|
+
|
|
83
|
+
## Also in TypeScript
|
|
84
|
+
|
|
85
|
+
[`npm install pghybrid`](https://www.npmjs.com/package/pghybrid) generates byte-identical
|
|
86
|
+
SQL, checked on every commit rather than assumed.
|
|
87
|
+
|
|
88
|
+
---
|
|
89
|
+
|
|
90
|
+
**Full documentation, the reasoning, and copy-and-paste SQL:
|
|
91
|
+
[github.com/pavangupta352/pghybrid](https://github.com/pavangupta352/pghybrid#readme)**
|
|
92
|
+
|
|
93
|
+
MIT © Pavan Gupta
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "pghybrid"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Hybrid search on the Postgres you already have. Vector + full-text + Reciprocal Rank Fusion on plain pgvector, with no extensions to install."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
authors = [{ name = "Pavan Gupta", email = "pavan.gupta.352@gmail.com" }]
|
|
13
|
+
keywords = [
|
|
14
|
+
"postgres", "postgresql", "pgvector", "hybrid-search", "rrf",
|
|
15
|
+
"reciprocal-rank-fusion", "full-text-search", "vector-search",
|
|
16
|
+
"semantic-search", "rag", "retrieval", "tsvector", "bm25", "supabase",
|
|
17
|
+
]
|
|
18
|
+
classifiers = [
|
|
19
|
+
"Development Status :: 4 - Beta",
|
|
20
|
+
"Intended Audience :: Developers",
|
|
21
|
+
"License :: OSI Approved :: MIT License",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Programming Language :: Python :: 3.9",
|
|
24
|
+
"Programming Language :: Python :: 3.10",
|
|
25
|
+
"Programming Language :: Python :: 3.11",
|
|
26
|
+
"Programming Language :: Python :: 3.12",
|
|
27
|
+
"Programming Language :: Python :: 3.13",
|
|
28
|
+
"Programming Language :: SQL",
|
|
29
|
+
"Topic :: Database",
|
|
30
|
+
"Topic :: Text Processing :: Indexing",
|
|
31
|
+
"Typing :: Typed",
|
|
32
|
+
]
|
|
33
|
+
# No runtime dependencies. The package generates SQL and hands it to whatever
|
|
34
|
+
# driver the caller already uses.
|
|
35
|
+
dependencies = []
|
|
36
|
+
|
|
37
|
+
[project.optional-dependencies]
|
|
38
|
+
cli = ["psycopg[binary]>=3.1"]
|
|
39
|
+
dev = [
|
|
40
|
+
"psycopg[binary]>=3.1",
|
|
41
|
+
# The ORM adapters are only worth shipping if they are exercised against a real
|
|
42
|
+
# server, so the libraries they wrap are test dependencies.
|
|
43
|
+
"sqlalchemy>=2.0",
|
|
44
|
+
"asyncpg>=0.29",
|
|
45
|
+
# The concurrency test holds one HybridSearch across threads backed by a pool,
|
|
46
|
+
# which is how a web application uses it.
|
|
47
|
+
"psycopg-pool>=3.2",
|
|
48
|
+
# For examples/langchain_retriever.py. Not a dependency of the package: the glue
|
|
49
|
+
# lives in an example so this library never owns LangChain's version matrix. CI runs
|
|
50
|
+
# it so the example cannot rot unnoticed.
|
|
51
|
+
"langchain-core>=0.3",
|
|
52
|
+
"django>=4.2",
|
|
53
|
+
"pytest>=8.0",
|
|
54
|
+
"pytest-asyncio>=0.24",
|
|
55
|
+
"pytest-cov>=5.0",
|
|
56
|
+
"ruff>=0.6",
|
|
57
|
+
"mypy>=1.11",
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
[project.urls]
|
|
61
|
+
Homepage = "https://github.com/pavangupta352/pghybrid"
|
|
62
|
+
Repository = "https://github.com/pavangupta352/pghybrid"
|
|
63
|
+
Issues = "https://github.com/pavangupta352/pghybrid/issues"
|
|
64
|
+
Changelog = "https://github.com/pavangupta352/pghybrid/blob/main/CHANGELOG.md"
|
|
65
|
+
|
|
66
|
+
[project.scripts]
|
|
67
|
+
pghybrid = "pghybrid.cli:main"
|
|
68
|
+
|
|
69
|
+
[tool.hatch.build.targets.wheel]
|
|
70
|
+
packages = ["src/pghybrid"]
|
|
71
|
+
# PEP 561. Without this marker a consumer's type checker ignores the annotations
|
|
72
|
+
# entirely and reports every symbol as Any, however strictly the package checks itself.
|
|
73
|
+
artifacts = ["src/pghybrid/py.typed"]
|
|
74
|
+
|
|
75
|
+
[tool.pytest.ini_options]
|
|
76
|
+
testpaths = ["tests"]
|
|
77
|
+
markers = [
|
|
78
|
+
"integration: requires a live Postgres with pgvector",
|
|
79
|
+
"needs_database: the test talks to a server and needs the demo corpus seeded",
|
|
80
|
+
]
|
|
81
|
+
asyncio_mode = "auto"
|
|
82
|
+
|
|
83
|
+
[tool.ruff]
|
|
84
|
+
line-length = 100
|
|
85
|
+
target-version = "py39"
|
|
86
|
+
|
|
87
|
+
[tool.ruff.lint]
|
|
88
|
+
select = ["E", "F", "I", "UP", "B", "SIM"]
|
|
89
|
+
# UP045 rewrites Optional[X] to X | None. Annotations are strings here thanks to
|
|
90
|
+
# `from __future__ import annotations`, so both forms run on 3.9 — but only the
|
|
91
|
+
# Optional form survives a runtime typing.get_type_hints() call there, and the
|
|
92
|
+
# package supports 3.9. Not worth 65 rewrites to lose that.
|
|
93
|
+
ignore = ["UP045", "UP007"]
|
|
94
|
+
|
|
95
|
+
[tool.mypy]
|
|
96
|
+
# mypy no longer accepts 3.9 as a target. requires-python stays >=3.9;
|
|
97
|
+
# this only sets which syntax mypy assumes while checking.
|
|
98
|
+
python_version = "3.10"
|
|
99
|
+
strict = true
|
|
100
|
+
|
|
101
|
+
# Django ships no py.typed marker. The adapter touches exactly one symbol from it and
|
|
102
|
+
# is exercised against a real Django connection in the integration suite.
|
|
103
|
+
[[tool.mypy.overrides]]
|
|
104
|
+
module = ["django.*"]
|
|
105
|
+
ignore_missing_imports = true
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Hybrid search on the Postgres you already have.
|
|
2
|
+
|
|
3
|
+
Vector similarity and full-text search, combined by Reciprocal Rank Fusion, over plain
|
|
4
|
+
``pgvector``. No extension beyond ``pgvector`` is required, which is the entire point:
|
|
5
|
+
``pg_search`` and its peers are unavailable on most managed Postgres.
|
|
6
|
+
|
|
7
|
+
from pghybrid import Config, HybridSearch
|
|
8
|
+
|
|
9
|
+
search = HybridSearch(
|
|
10
|
+
Config(table="chunks", text_column="content", vector_column="embedding",
|
|
11
|
+
tsvector_column="fts", paramstyle="pyformat"),
|
|
12
|
+
execute=lambda sql, params: conn.execute(sql, params).fetchall(),
|
|
13
|
+
)
|
|
14
|
+
rows = search.search("renewal notice period", embedding=vector, limit=10)
|
|
15
|
+
|
|
16
|
+
``execute`` is whatever driver you already have. This package opens no connections and
|
|
17
|
+
calls no embedding model.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from .config import (
|
|
21
|
+
COSINE,
|
|
22
|
+
INNER_PRODUCT,
|
|
23
|
+
L1,
|
|
24
|
+
L2,
|
|
25
|
+
METRICS,
|
|
26
|
+
Config,
|
|
27
|
+
Metric,
|
|
28
|
+
Recency,
|
|
29
|
+
Weights,
|
|
30
|
+
)
|
|
31
|
+
from .explain import ExplainReport, ExplainRow, FindReport, explain, measure_weights
|
|
32
|
+
from .schema import (
|
|
33
|
+
Statement,
|
|
34
|
+
TableInfo,
|
|
35
|
+
TableNotFound,
|
|
36
|
+
build_migration,
|
|
37
|
+
dbapi_executor,
|
|
38
|
+
introspect,
|
|
39
|
+
suggest_config,
|
|
40
|
+
)
|
|
41
|
+
from .search import AsyncHybridSearch, HybridSearch, SearchResult
|
|
42
|
+
from .sql import IdentifierError, build_search_sql, quote_ident
|
|
43
|
+
from .textquery import ParsedQuery, parse_query
|
|
44
|
+
|
|
45
|
+
__version__ = "0.1.0"
|
|
46
|
+
|
|
47
|
+
__all__ = [
|
|
48
|
+
# Configuration
|
|
49
|
+
"Config",
|
|
50
|
+
"Metric",
|
|
51
|
+
"Recency",
|
|
52
|
+
"Weights",
|
|
53
|
+
"METRICS",
|
|
54
|
+
"COSINE",
|
|
55
|
+
"L2",
|
|
56
|
+
"INNER_PRODUCT",
|
|
57
|
+
"L1",
|
|
58
|
+
# Searching
|
|
59
|
+
"HybridSearch",
|
|
60
|
+
"AsyncHybridSearch",
|
|
61
|
+
"SearchResult",
|
|
62
|
+
# Diagnosing
|
|
63
|
+
"explain",
|
|
64
|
+
"measure_weights",
|
|
65
|
+
"ExplainReport",
|
|
66
|
+
"ExplainRow",
|
|
67
|
+
"FindReport",
|
|
68
|
+
# Schema
|
|
69
|
+
"introspect",
|
|
70
|
+
"suggest_config",
|
|
71
|
+
"build_migration",
|
|
72
|
+
"dbapi_executor",
|
|
73
|
+
"TableInfo",
|
|
74
|
+
"TableNotFound",
|
|
75
|
+
"Statement",
|
|
76
|
+
# The SQL itself, for people who want to read or steal it
|
|
77
|
+
"build_search_sql",
|
|
78
|
+
"quote_ident",
|
|
79
|
+
"IdentifierError",
|
|
80
|
+
"parse_query",
|
|
81
|
+
"ParsedQuery",
|
|
82
|
+
]
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def __getattr__(name: str) -> object:
|
|
86
|
+
"""Expose ``doctor`` lazily.
|
|
87
|
+
|
|
88
|
+
It is the heaviest module in the package and the only one most callers never touch,
|
|
89
|
+
so importing it eagerly would make ``import pghybrid`` pay for a feature reserved
|
|
90
|
+
for the moment something has already gone wrong.
|
|
91
|
+
"""
|
|
92
|
+
if name == "doctor":
|
|
93
|
+
from .doctor import doctor
|
|
94
|
+
|
|
95
|
+
return doctor
|
|
96
|
+
if name == "DoctorReport":
|
|
97
|
+
from .doctor import DoctorReport
|
|
98
|
+
|
|
99
|
+
return DoctorReport
|
|
100
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""One-line wiring for the drivers people actually have.
|
|
2
|
+
|
|
3
|
+
:class:`~pghybrid.search.HybridSearch` takes an ``execute`` callable, which keeps the
|
|
4
|
+
package free of driver dependencies but leaves every user writing the same lambda — and
|
|
5
|
+
choosing a placeholder style, which is the single thing most likely to be got wrong.
|
|
6
|
+
``$1`` and ``%s`` are not interchangeable, and picking the wrong one fails with a message
|
|
7
|
+
about parameter counts that says nothing about the cause.
|
|
8
|
+
|
|
9
|
+
So each helper here does both: it wires the executor for one driver *and* sets the
|
|
10
|
+
placeholder style that driver needs. Nothing is imported at module scope, so the package
|
|
11
|
+
keeps its promise of zero runtime dependencies whichever of these you never use.
|
|
12
|
+
|
|
13
|
+
from pghybrid.adapters import for_psycopg
|
|
14
|
+
|
|
15
|
+
search = for_psycopg(conn, table="chunks", text_column="content",
|
|
16
|
+
vector_column="embedding", tsvector_column="fts")
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from collections.abc import Mapping
|
|
22
|
+
from dataclasses import replace
|
|
23
|
+
from typing import Any, Optional
|
|
24
|
+
|
|
25
|
+
from .config import Config, ParamStyle
|
|
26
|
+
from .search import AsyncHybridSearch, HybridSearch
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"for_psycopg",
|
|
30
|
+
"for_asyncpg",
|
|
31
|
+
"for_sqlalchemy",
|
|
32
|
+
"for_django",
|
|
33
|
+
"psycopg_executor",
|
|
34
|
+
"asyncpg_executor",
|
|
35
|
+
"sqlalchemy_executor",
|
|
36
|
+
"django_executor",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _as_mappings(cursor: Any) -> list[dict[str, Any]]:
|
|
41
|
+
"""Read a DBAPI cursor as dicts, whatever row factory it was configured with.
|
|
42
|
+
|
|
43
|
+
A connection may already be returning mappings — ``psycopg.rows.dict_row`` is common
|
|
44
|
+
— and zipping column names against one of those pairs the names with the dict's keys
|
|
45
|
+
instead of its values, so every column comes back holding its own name.
|
|
46
|
+
"""
|
|
47
|
+
if cursor.description is None:
|
|
48
|
+
return []
|
|
49
|
+
columns = [column[0] for column in cursor.description]
|
|
50
|
+
rows = cursor.fetchall()
|
|
51
|
+
if rows and isinstance(rows[0], Mapping):
|
|
52
|
+
return [dict(row) for row in rows]
|
|
53
|
+
return [dict(zip(columns, row)) for row in rows]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _resolve(config: Optional[Config], paramstyle: ParamStyle, **overrides: Any) -> Config:
|
|
57
|
+
"""Build or adjust a Config, forcing the placeholder style the driver requires.
|
|
58
|
+
|
|
59
|
+
Passing ``paramstyle`` yourself is not an error worth respecting: a psycopg
|
|
60
|
+
connection cannot execute ``$1`` no matter how firmly it was requested.
|
|
61
|
+
"""
|
|
62
|
+
if config is None:
|
|
63
|
+
if not overrides:
|
|
64
|
+
raise TypeError(
|
|
65
|
+
"pass a Config, or the keyword arguments to build one "
|
|
66
|
+
"(table=..., text_column=..., vector_column=...)"
|
|
67
|
+
)
|
|
68
|
+
overrides.pop("paramstyle", None)
|
|
69
|
+
return Config(paramstyle=paramstyle, **overrides)
|
|
70
|
+
|
|
71
|
+
if overrides:
|
|
72
|
+
overrides.pop("paramstyle", None)
|
|
73
|
+
config = replace(config, **overrides)
|
|
74
|
+
return replace(config, paramstyle=paramstyle)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
# ------------------------------------------------------------------------- psycopg
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def psycopg_executor(connection: Any) -> Any:
|
|
81
|
+
"""An executor over a psycopg 2 or 3 connection, cursor factory notwithstanding.
|
|
82
|
+
|
|
83
|
+
Rows are returned as mappings whatever the connection was configured for, because
|
|
84
|
+
the caller asked for search results rather than for tuples they then have to zip
|
|
85
|
+
against a column list.
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
def execute(sql: str, params: list[Any]) -> list[dict[str, Any]]:
|
|
89
|
+
with connection.cursor() as cursor:
|
|
90
|
+
cursor.execute(sql, params)
|
|
91
|
+
return _as_mappings(cursor)
|
|
92
|
+
|
|
93
|
+
return execute
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def for_psycopg(connection: Any, config: Optional[Config] = None, **kwargs: Any) -> HybridSearch:
|
|
97
|
+
"""Wire a psycopg connection. psycopg speaks ``%s``, so the style is set for you."""
|
|
98
|
+
return HybridSearch(_resolve(config, "pyformat", **kwargs), psycopg_executor(connection))
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# ------------------------------------------------------------------------- asyncpg
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def asyncpg_executor(connection: Any) -> Any:
|
|
105
|
+
"""An executor over an asyncpg connection or pool.
|
|
106
|
+
|
|
107
|
+
Both expose ``fetch``, so one function covers them, and asyncpg's ``Record`` is
|
|
108
|
+
already a mapping.
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
async def execute(sql: str, params: list[Any]) -> list[Any]:
|
|
112
|
+
return list(await connection.fetch(sql, *params))
|
|
113
|
+
|
|
114
|
+
return execute
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def for_asyncpg(
|
|
118
|
+
connection: Any, config: Optional[Config] = None, **kwargs: Any
|
|
119
|
+
) -> AsyncHybridSearch:
|
|
120
|
+
"""Wire an asyncpg connection or pool. asyncpg speaks ``$1``."""
|
|
121
|
+
return AsyncHybridSearch(_resolve(config, "numeric", **kwargs), asyncpg_executor(connection))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
# ---------------------------------------------------------------------- SQLAlchemy
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def sqlalchemy_executor(bind: Any) -> Any:
|
|
128
|
+
"""An executor over a SQLAlchemy Session, Connection or Engine.
|
|
129
|
+
|
|
130
|
+
Uses ``exec_driver_sql``, which hands the statement to the DBAPI untouched, rather
|
|
131
|
+
than ``text()``. ``text()`` would have to bind ``:p0`` style parameters, and in
|
|
132
|
+
Postgres ``:`` is ambiguous with the cast operator: SQLAlchemy silently declines to
|
|
133
|
+
bind ``:p0`` in ``:p0::vector`` and the server then reports a syntax error at a colon
|
|
134
|
+
that looks nothing like the cause. The alternative — emitting ``CAST(x AS vector)``
|
|
135
|
+
throughout — would make the generated SQL less idiomatic for every user in order to
|
|
136
|
+
suit one ORM.
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
def execute(sql: str, params: list[Any]) -> list[dict[str, Any]]:
|
|
140
|
+
# A Session has .connection(); an Engine has .connect(); a Connection is already
|
|
141
|
+
# what we need. Engines are handled last because a Session also exposes .connect
|
|
142
|
+
# on some versions.
|
|
143
|
+
if hasattr(bind, "connection") and callable(bind.connection):
|
|
144
|
+
connection = bind.connection()
|
|
145
|
+
result = connection.exec_driver_sql(sql, tuple(params))
|
|
146
|
+
return [dict(row) for row in result.mappings()]
|
|
147
|
+
if hasattr(bind, "exec_driver_sql"):
|
|
148
|
+
result = bind.exec_driver_sql(sql, tuple(params))
|
|
149
|
+
return [dict(row) for row in result.mappings()]
|
|
150
|
+
with bind.connect() as connection:
|
|
151
|
+
result = connection.exec_driver_sql(sql, tuple(params))
|
|
152
|
+
return [dict(row) for row in result.mappings()]
|
|
153
|
+
|
|
154
|
+
return execute
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def for_sqlalchemy(bind: Any, config: Optional[Config] = None, **kwargs: Any) -> HybridSearch:
|
|
158
|
+
"""Wire a SQLAlchemy Session, Connection or Engine."""
|
|
159
|
+
return HybridSearch(_resolve(config, "pyformat", **kwargs), sqlalchemy_executor(bind))
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
# -------------------------------------------------------------------------- Django
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def django_executor(using: str = "default") -> Any:
|
|
166
|
+
"""An executor over a Django database connection.
|
|
167
|
+
|
|
168
|
+
Django's Postgres backend is psycopg, so the placeholder style is ``%s`` and the
|
|
169
|
+
connection is taken from ``django.db.connections`` at call time rather than at
|
|
170
|
+
import time — the alias may not be configured yet when the module is imported.
|
|
171
|
+
"""
|
|
172
|
+
|
|
173
|
+
def execute(sql: str, params: list[Any]) -> list[dict[str, Any]]:
|
|
174
|
+
from django.db import connections
|
|
175
|
+
|
|
176
|
+
with connections[using].cursor() as cursor:
|
|
177
|
+
cursor.execute(sql, params)
|
|
178
|
+
return _as_mappings(cursor)
|
|
179
|
+
|
|
180
|
+
return execute
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def for_django(
|
|
184
|
+
config: Optional[Config] = None, using: str = "default", **kwargs: Any
|
|
185
|
+
) -> HybridSearch:
|
|
186
|
+
"""Wire a Django database connection by alias."""
|
|
187
|
+
return HybridSearch(_resolve(config, "pyformat", **kwargs), django_executor(using))
|