surreal-orm-lite 0.36.5__tar.gz → 0.37.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/CHANGELOG.md +74 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/PKG-INFO +194 -11
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/README.md +193 -10
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/pyproject.toml +1 -1
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/__init__.py +1 -1
- surreal_orm_lite-0.37.0/src/surreal_orm_lite/hybrid.py +186 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/query_set.py +342 -15
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/search.py +16 -14
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/vector.py +40 -31
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/.gitignore +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/LICENSE +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/Makefile +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/_sdk.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/aggregations.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/auth.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/cache.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/changefeed.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/concurrency.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/connection_config.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/connection_manager.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/constants.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/decoding.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/define_parser.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/edge.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/embedded.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/encoding.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/enum.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/exceptions.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/functions.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/geo.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/introspection.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/live.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/model_auth.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/model_base.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/model_ddl.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/model_generator.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/py.typed +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/q.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/routing.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/schema.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/schema_state.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/signals.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/subquery.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/surql_literal.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/testing/__init__.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/testing/database.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/testing/fixtures.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/time_travel.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/transaction.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/types.py +0 -0
- {surreal_orm_lite-0.36.5 → surreal_orm_lite-0.37.0}/src/surreal_orm_lite/utils.py +0 -0
|
@@ -5,6 +5,80 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.37.0] - 2026-10-24
|
|
9
|
+
|
|
10
|
+
Hybrid search: a queryset fuses the records nearest to a query vector, as `similar_to()` finds
|
|
11
|
+
them, and the best matches of a text, as `search()` ranks them, into one ranking by Reciprocal
|
|
12
|
+
Rank Fusion, with the full SurrealDB-ORM's `hybrid_search()`. SurrealDB 3.x fuses the two lists
|
|
13
|
+
with `search::rrf()`; SurrealDB 2.x and the embedded engine have no such function, so the ORM
|
|
14
|
+
fuses them with the same formula: the same two lists give the same records, scores and order on
|
|
15
|
+
every line. The third and last minor of Tier 3's search phase.
|
|
16
|
+
|
|
17
|
+
### Added
|
|
18
|
+
|
|
19
|
+
- **`QuerySet.hybrid_search(*, vector_field, vector, vector_limit=20, text_field, text_query, text_limit=20, rrf_k=60, ef=None, dist=None)`**,
|
|
20
|
+
the full SurrealDB-ORM's method: each record gets 1 / (`rrf_k` + rank) from each list it is in,
|
|
21
|
+
its rank counted from 1, and the records come best fused score first. It returns a list of
|
|
22
|
+
dicts, as the full ORM does, each holding the record's fields keyed by their Python names (as
|
|
23
|
+
`exec()`'s dict rows are), `_rrf_score`, and `_knn_distance` and `_search_score` (`None` for a
|
|
24
|
+
record that only the other search found). The query vector and the text are bound as query
|
|
25
|
+
variables, never written into the statement. **`dist=`** is an addition: an exact scan that
|
|
26
|
+
needs no vector index, as in `similar_to()`. The queryset applies: its conditions select the
|
|
27
|
+
records of both lists, `select()` and `fetch()` apply to both (`id` always read),
|
|
28
|
+
`limit()`/`offset()` page through the fused records (at most `vector_limit + text_limit`), and
|
|
29
|
+
`cache()`, `objects(tx=)` and named connections apply as for `exec()`; the full ORM ignores them.
|
|
30
|
+
- README section 38, "Hybrid search (`hybrid_search()`, Reciprocal Rank Fusion)" (search, with the
|
|
31
|
+
queryset, what SurrealDB decides, what is not supported, a note for users of the full
|
|
32
|
+
SurrealDB-ORM), four rows of the "ORM behaviour: SurrealDB 2.x vs 3.x" table and one of the
|
|
33
|
+
embedded engine's, and the example notebook `examples/v0.37.0_hybrid_search.ipynb`.
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
results = await Passage.objects().filter(topic="tech").hybrid_search(
|
|
37
|
+
vector_field="embedding", vector=[0.0, 1.0, 0.0], vector_limit=4,
|
|
38
|
+
text_field="title", text_query="programming", text_limit=4,
|
|
39
|
+
)
|
|
40
|
+
for row in results: # best fused score first
|
|
41
|
+
print(row["id"], row["_rrf_score"], row["_knn_distance"], row["_search_score"])
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
### Notes
|
|
45
|
+
|
|
46
|
+
- **SurrealDB 3.x fuses with `search::rrf()`**, given both lists in one statement; **SurrealDB
|
|
47
|
+
2.x and the embedded engine get the same fusion from the ORM**: the same records, scores and
|
|
48
|
+
order for the same two lists. The line is read from `client.version()`, as for `search()`'s
|
|
49
|
+
index keyword. Other conditions apply to both lists; on SurrealDB 2.x and the embedded engine
|
|
50
|
+
the vector list selects the records matching them first, as `similar_to()` does.
|
|
51
|
+
- **Equal fused scores come in a fixed order**: the vector list's first, then the full-text list's
|
|
52
|
+
for the records it alone found, on every line (SurrealDB's `search::rrf()` alone returns them in
|
|
53
|
+
no fixed order), so the same two lists always give the same page. Each list is SurrealDB's own:
|
|
54
|
+
BM25 scores depend on the line (a term found in more than half the documents scores `0.0` on 3.x
|
|
55
|
+
and below zero on 2.x and the embedded engine), and on SurrealDB 2.x and the embedded engine an
|
|
56
|
+
exact scan (`dist=`) picks among records at an equal distance at the `vector_limit`-th place in
|
|
57
|
+
no fixed order, as `similar_to()`'s does since v0.36.0: the vector list, and a page of the fused
|
|
58
|
+
records, can then change between two identical calls. The HNSW index, and SurrealDB 3.x's exact
|
|
59
|
+
scan, return the same list on every call.
|
|
60
|
+
- **Checked before any request**: both fields, the vector, `vector_limit` and `text_limit` (1 to
|
|
61
|
+
10 000), `rrf_k` (0 to 10 000), `ef` and `dist`; the full ORM checks the field names only.
|
|
62
|
+
- **Refused before any request**: `order_by()`, `annotate()`, `values()`, `search()` and
|
|
63
|
+
`similar_to()` on the queryset (`ValueError`), and `as_of()` (`SurrealDbError`: the indexes hold
|
|
64
|
+
the records' current values). A model field stored under one of the seven names the method uses
|
|
65
|
+
for its own values (`rrf_score`, `_rrf_score`, `_knn_distance`, `_search_score`, `_hybrid_pos`,
|
|
66
|
+
`_hybrid_vec_pos`, `_hybrid_text_pos`) raises `ValueError`: store it under another name with
|
|
67
|
+
`Field(alias=...)`. On a SCHEMALESS table, a column with one of these names that the model does
|
|
68
|
+
not declare is not returned, even when `select()` names it.
|
|
69
|
+
- **Each search needs its index.** Without a full-text index on `text_field`, SurrealDB 3.3, 2.x
|
|
70
|
+
and the embedded engine refuse the search and `hybrid_search()` raises `SurrealDbError` naming
|
|
71
|
+
the field; SurrealDB 3.1 and 3.2 match nothing, so the vector list alone is fused. Without a
|
|
72
|
+
vector index on `vector_field`, the vector list is empty on SurrealDB 3.3, 2.x and the embedded
|
|
73
|
+
engine, so the full-text ranking alone is fused; SurrealDB 3.1 and 3.2 list the first
|
|
74
|
+
`vector_limit` records by id, with no distance. `dist=` needs no index.
|
|
75
|
+
- **Not yet**: `search::linear()` (weights and score normalisation), several full-text fields,
|
|
76
|
+
the full-text side's highlights, and search annotations next to `similar_to()` on SurrealDB 2.x
|
|
77
|
+
and the embedded engine, which the ORM still refuses there; `hybrid_search()` gives each record
|
|
78
|
+
found by both searches its BM25 score and its distance, on every line.
|
|
79
|
+
- Server pins unchanged: tested against SurrealDB 3.3.4 and 2.7.0 and on the embedded engine,
|
|
80
|
+
plus 3.1.5 for backward compatibility; the new behaviour rows were also measured on 3.2.6.
|
|
81
|
+
|
|
8
82
|
## [0.36.5] - 2026-10-24
|
|
9
83
|
|
|
10
84
|
Follow-up to v0.36.1 (#315): a queryset write that the ORM refuses, and an instance write
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: surreal-orm-lite
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.37.0
|
|
4
4
|
Summary: Lightweight Django-style ORM for SurrealDB using the official Python SDK. Async support with Pydantic validation.
|
|
5
5
|
Project-URL: Homepage, https://github.com/EulogySnowfall/SurrealDB-ORM-lite
|
|
6
6
|
Project-URL: Documentation, https://github.com/EulogySnowfall/SurrealDB-ORM-lite
|
|
@@ -271,6 +271,7 @@ embedded engine refuse one above `4294967295` (`2**32 - 1`), SurrealDB 3.x reads
|
|
|
271
271
|
| Model table DDL (`define_table()`, `schema_mode`, `permissions`) | ✅ |
|
|
272
272
|
| Full-text search (`search()`, BM25 score and highlights) | ✅ |
|
|
273
273
|
| Vector search (`similar_to()`, KNN, HNSW indexes) | ✅ |
|
|
274
|
+
| Hybrid search (`hybrid_search()`, Reciprocal Rank Fusion) | ✅ |
|
|
274
275
|
|
|
275
276
|
### Supported Filter Lookups
|
|
276
277
|
|
|
@@ -6109,10 +6110,25 @@ print(await NoteV2.objects().search(text="physics").count())
|
|
|
6109
6110
|
has already written records to the model's table with the ORM (`save()`, `objects(tx=…).bulk_create()`, …). The
|
|
6110
6111
|
refusal covers every kind of index, not only full-text ones: a search would miss those records, and a `UNIQUE` index
|
|
6111
6112
|
missing them would not enforce uniqueness for them. The refusal does not fail the block. `define_table(tx=…)`,
|
|
6112
|
-
`define_relation(tx=…)` and a write to another table do not count, nor does a write the ORM refused
|
|
6113
|
-
materialized view
|
|
6114
|
-
`
|
|
6115
|
-
operations
|
|
6113
|
+
`define_relation(tx=…)` and a write to another table do not count, nor does a write the ORM refused before sending or
|
|
6114
|
+
queuing it: one refused for a materialized view, a snapshot read with `as_of()`, an auth model with a
|
|
6115
|
+
`Field(exclude=True)` field SIGNUP writes, a `tx=` of another connection, or one of its arguments (a missing id,
|
|
6116
|
+
invalid patch operations, an invalid `server_values`/`extra_vars` pair or either one on an edge's `save()`, a computed
|
|
6117
|
+
field or an edge endpoint among the fields written, an atomic helper's field, amount or values, a subquery value in
|
|
6118
|
+
`bulk_update()`, a snapshot in `bulk_create()`'s list); a raw `tx.add()` is not tracked. Until
|
|
6119
|
+
[EulogySnowfall/SurrealDB-ORM-lite#325](https://github.com/EulogySnowfall/SurrealDB-ORM-lite/issues/325) is fixed,
|
|
6120
|
+
some refusals made while the write is built or its values are encoded still count, although nothing was sent or
|
|
6121
|
+
queued: `upsert()` without an id, `save(tx=…)` without an id in a buffered transaction, a field name or an
|
|
6122
|
+
`extra_vars` key refused while the statement of `save(server_values=…)` or `merge(server_values=…)` is compiled (an
|
|
6123
|
+
invalid name, a key that collides with an internal binding), the argument checks of `relate()`, `remove_relation()`
|
|
6124
|
+
and `remove_all_relations()` (an invalid edge name, or an empty list of edges, counts as a write to every table, so
|
|
6125
|
+
every later `define_indexes(tx=…)` in the block is refused), a field name `bulk_update()` refuses, a filter refused
|
|
6126
|
+
while the statement of `objects(tx=…).bulk_update()`, `bulk_delete()` or `patch()` is compiled, an edge id of another
|
|
6127
|
+
table in an edge's `save()` or `bulk_create()`, and a value the ORM cannot send, in any write (a field, a filter, a
|
|
6128
|
+
patch operation, an atomic helper's value or amount, a relation's data, `extra_vars`): a `field_serializer` that
|
|
6129
|
+
raises, or one of the [values refused before sending](#values-refused-before-sending), such as a negative duration,
|
|
6130
|
+
an `int` outside the signed 64-bit range or a `Decimal` out of range. A write refused first for one of the arguments
|
|
6131
|
+
listed above, such as an `update()` or a `merge()` without an id, does not count.
|
|
6116
6132
|
- Prefer `define_indexes()` outside the block: on SurrealDB 2.x, a write to a table that held records, after its index
|
|
6117
6133
|
in the same block, fails the commit with a retryable `SurrealDbConflictError` that a retry meets again; so does a
|
|
6118
6134
|
block that removes an index of a table that held records and defines it again, where the index stands as before.
|
|
@@ -6186,7 +6202,7 @@ print(await NoteV2.objects().search(text="physics").count())
|
|
|
6186
6202
|
indexes (section 37).
|
|
6187
6203
|
- **`filter(x__match=...)` stays a regular expression**: this library's `match` lookup, not SurrealDB's `@@`;
|
|
6188
6204
|
`search()` is the full-text path.
|
|
6189
|
-
- **`hybrid_search()`
|
|
6205
|
+
- **`hybrid_search()` fuses `search()`'s ranking with `similar_to()`'s** (section 38, v0.37.0).
|
|
6190
6206
|
|
|
6191
6207
|
### 37. Vector search (`similar_to()`, `VectorField`, `Index(hnsw=True)`)
|
|
6192
6208
|
|
|
@@ -6468,7 +6484,8 @@ except ValueError as error:
|
|
|
6468
6484
|
|
|
6469
6485
|
- Search annotations with `similar_to()` on SurrealDB 2.x and the embedded engine, for the reads that return records
|
|
6470
6486
|
(`exec()`, `first()`, `get()` without an id): `SurrealDbError` before the statement is sent. `search()` as a
|
|
6471
|
-
condition stays available there, and `count()`, `exists()` and the aggregations answer.
|
|
6487
|
+
condition stays available there, and `count()`, `exists()` and the aggregations answer. `hybrid_search()` gives each
|
|
6488
|
+
record found by both searches its BM25 score and its distance, on every line.
|
|
6472
6489
|
- `as_of()` ([section 27](#27-time-travel-as_of)): `similar_to()` with `as_of()` raises `SurrealDbError` before any
|
|
6473
6490
|
request, in either order and in a subquery under it. The index holds the records' current vectors, so a time-travel
|
|
6474
6491
|
read would pick its neighbours by their vectors now.
|
|
@@ -6505,12 +6522,172 @@ except ValueError as error:
|
|
|
6505
6522
|
- **No `MTREE`, `MINKOWSKI`, `JACCARD` or `PEARSON`, and no `concurrently` yet**; the full ORM writes `MINKOWSKI`
|
|
6506
6523
|
without its order, which SurrealDB refuses.
|
|
6507
6524
|
- **Vector functions in `annotate()`** (`vector::similarity::cosine()`…) are not available yet; `hybrid_search()`
|
|
6508
|
-
|
|
6525
|
+
(section 38) fuses this ranking with `search()`'s.
|
|
6509
6526
|
- **`get_vector_info()` and `is_vector_field()` are imported from `surreal_orm_lite.vector`**: the full ORM has them
|
|
6510
6527
|
in `surreal_orm.fields`, a module this library does not have.
|
|
6511
6528
|
- **`generate_models_from_db()` does not write `VectorField` yet**, while the full ORM's generator writes it for a
|
|
6512
6529
|
field with an HNSW index.
|
|
6513
6530
|
|
|
6531
|
+
### 38. Hybrid search (`hybrid_search()`, Reciprocal Rank Fusion)
|
|
6532
|
+
|
|
6533
|
+
Since v0.37.0 a queryset fuses two rankings of its records with the full SurrealDB-ORM's `hybrid_search()`: the
|
|
6534
|
+
records nearest to a query vector, as `similar_to()` finds them
|
|
6535
|
+
([section 37](#37-vector-search-similar_to-vectorfield-indexhnswtrue)), and the best matches of a text, as `search()`
|
|
6536
|
+
ranks them by BM25 score ([section 36](#36-full-text-search-search-searchscore-searchhighlight-analyzer-index)), are
|
|
6537
|
+
fused into one ranking by Reciprocal Rank Fusion. SurrealDB 3.x fuses the two lists with its `search::rrf()`;
|
|
6538
|
+
SurrealDB 2.x and the embedded engine have no such function, so the ORM fuses them with the same formula: the same
|
|
6539
|
+
two lists give the same records, scores and order on every line. Each record comes back as a dict, as in the full
|
|
6540
|
+
ORM. The examples assume a configured connection: they run as written, in order, on the embedded engine (`mem://`).
|
|
6541
|
+
|
|
6542
|
+
#### Search
|
|
6543
|
+
|
|
6544
|
+
```python
|
|
6545
|
+
from surreal_orm_lite import Analyzer, BaseSurrealModel, Index, SurrealConfigDict, SurrealDbError, VectorField
|
|
6546
|
+
|
|
6547
|
+
WORDS = Analyzer("words", ["blank"], ["lowercase"])
|
|
6548
|
+
|
|
6549
|
+
|
|
6550
|
+
class Passage(BaseSurrealModel):
|
|
6551
|
+
model_config = SurrealConfigDict(
|
|
6552
|
+
table_name="passage",
|
|
6553
|
+
indexes=[
|
|
6554
|
+
Index("passage_embedding", "embedding", hnsw=True, dimension=3, dist="cosine"),
|
|
6555
|
+
Index("passage_title", "title", search_analyzer=WORDS),
|
|
6556
|
+
],
|
|
6557
|
+
)
|
|
6558
|
+
|
|
6559
|
+
id: str | None = None
|
|
6560
|
+
title: str
|
|
6561
|
+
topic: str = ""
|
|
6562
|
+
embedding: VectorField[3]
|
|
6563
|
+
|
|
6564
|
+
|
|
6565
|
+
await Passage.define_table()
|
|
6566
|
+
await Passage.define_indexes()
|
|
6567
|
+
for key, title, topic, embedding in [
|
|
6568
|
+
("p1", "Cats and dogs", "pets", [1.0, 0.0, 0.0]),
|
|
6569
|
+
("p2", "Cats at home", "pets", [0.9, 0.1, 0.0]),
|
|
6570
|
+
("p3", "Rust programming", "tech", [0.0, 1.0, 0.0]),
|
|
6571
|
+
("p4", "Python programming", "tech", [0.1, 0.9, 0.0]),
|
|
6572
|
+
("p5", "Gardening", "home", [0.0, 0.0, 1.0]),
|
|
6573
|
+
("p6", "Cats and code", "news", [0.5, 0.5, 0.0]),
|
|
6574
|
+
("p7", "Rust and Python programming", "tech", [0.2, 0.8, 0.0]),
|
|
6575
|
+
("p8", "Cooking", "home", [0.0, 0.2, 0.8]),
|
|
6576
|
+
]:
|
|
6577
|
+
await Passage(id=key, title=title, topic=topic, embedding=embedding).save()
|
|
6578
|
+
|
|
6579
|
+
results = await Passage.objects().hybrid_search(
|
|
6580
|
+
vector_field="embedding", vector=[1.0, 0.0, 0.0], vector_limit=3,
|
|
6581
|
+
text_field="title", text_query="programming", text_limit=3,
|
|
6582
|
+
)
|
|
6583
|
+
for row in results:
|
|
6584
|
+
print(row["id"], f"{row['_rrf_score']:.4f}", row["_knn_distance"] is not None, row["_search_score"] is not None)
|
|
6585
|
+
# passage:p1 0.0164 True False
|
|
6586
|
+
# passage:p3 0.0164 False True
|
|
6587
|
+
# passage:p2 0.0161 True False
|
|
6588
|
+
# passage:p4 0.0161 False True
|
|
6589
|
+
# passage:p6 0.0159 True False
|
|
6590
|
+
# passage:p7 0.0159 False True
|
|
6591
|
+
```
|
|
6592
|
+
|
|
6593
|
+
Each record gets 1 / (`rrf_k` + rank) from each list it is in, its rank counted from 1 (`rrf_k` is 60 by default):
|
|
6594
|
+
`p1`, nearest, and `p3`, the best match, both score 1/61. Records with equal scores keep the vector list's order
|
|
6595
|
+
first, then the full-text list's for the records it alone found, on every line — SurrealDB's `search::rrf()` alone
|
|
6596
|
+
returns them in no fixed order. `_knn_distance` is `None` for a record only the full-text search found,
|
|
6597
|
+
`_search_score` for one only the vector search found.
|
|
6598
|
+
|
|
6599
|
+
#### With the queryset
|
|
6600
|
+
|
|
6601
|
+
```python
|
|
6602
|
+
tech = await Passage.objects().filter(topic="tech").limit(2).hybrid_search(
|
|
6603
|
+
vector_field="embedding", vector=[0.0, 1.0, 0.0], vector_limit=4,
|
|
6604
|
+
text_field="title", text_query="programming", text_limit=4,
|
|
6605
|
+
)
|
|
6606
|
+
for row in tech:
|
|
6607
|
+
print(row["id"], f"{row['_rrf_score']:.4f}")
|
|
6608
|
+
# passage:p3 0.0328
|
|
6609
|
+
# passage:p4 0.0323
|
|
6610
|
+
page = await Passage.objects().offset(1).limit(2).hybrid_search(
|
|
6611
|
+
vector_field="embedding", vector=[1.0, 0.0, 0.0], vector_limit=3,
|
|
6612
|
+
text_field="title", text_query="programming", text_limit=3,
|
|
6613
|
+
)
|
|
6614
|
+
print([str(row["id"]) for row in page])
|
|
6615
|
+
# ['passage:p3', 'passage:p2']
|
|
6616
|
+
```
|
|
6617
|
+
|
|
6618
|
+
The queryset's conditions select the records of both lists. `select()` and `fetch()` apply to both, `id` always read;
|
|
6619
|
+
`limit()` and `offset()` page through the fused records, of which there are at most `vector_limit + text_limit`.
|
|
6620
|
+
`cache()`, `objects(tx=)` and named connections apply as for `exec()`, and `dist=` scans exactly, as in section 37.
|
|
6621
|
+
|
|
6622
|
+
#### What SurrealDB decides
|
|
6623
|
+
|
|
6624
|
+
```python
|
|
6625
|
+
try:
|
|
6626
|
+
await Passage.objects().hybrid_search(
|
|
6627
|
+
vector_field="embedding", vector=[1.0, 0.0, 0.0], text_field="topic", text_query="tech",
|
|
6628
|
+
)
|
|
6629
|
+
except SurrealDbError as error:
|
|
6630
|
+
print(error)
|
|
6631
|
+
# hybrid_search(): no full-text index on field 'topic'. SurrealDB matches only a field with a FULLTEXT (3.x) or SEARCH (2.x) index: declare Index(..., search_analyzer=...) on the model and apply define_indexes().
|
|
6632
|
+
```
|
|
6633
|
+
|
|
6634
|
+
- Each list is SurrealDB's own: the vector list as `similar_to()` finds it — HNSW is approximate (section 37) — and
|
|
6635
|
+
the full-text list as `search()` ranks it, with section 36's BM25 scores. Those depend on the line: a term found in
|
|
6636
|
+
more than half the documents scores `0.0` on SurrealDB 3.x and below zero on 2.x and the embedded engine
|
|
6637
|
+
([2.x/3.x table](#orm-behaviour-surrealdb-2x-vs-3x)), so the full-text list, and the fused ranking with it, can
|
|
6638
|
+
differ between lines.
|
|
6639
|
+
- **An exact scan (`dist=`) on SurrealDB 2.x and the embedded engine** picks among records at an equal distance at the
|
|
6640
|
+
`vector_limit`-th place in no fixed order, as `similar_to()`'s does: the vector list, and the fused records with it,
|
|
6641
|
+
can change between two identical calls, and a page read with `offset()` and `limit()` can then skip or repeat a
|
|
6642
|
+
record. The HNSW index (no `dist=`), and SurrealDB 3.x's exact scan, return the same list on every call.
|
|
6643
|
+
- Each search needs its index. Without a full-text index on `text_field`, SurrealDB 3.3, 2.x and the embedded engine
|
|
6644
|
+
refuse the search, and `hybrid_search()` raises `SurrealDbError` naming the field, as above; SurrealDB 3.1 and 3.2
|
|
6645
|
+
match nothing, so the vector list alone is fused.
|
|
6646
|
+
- Without a vector index on `vector_field`, the vector search finds nothing on SurrealDB 3.3, 2.x and the embedded
|
|
6647
|
+
engine, so the full-text ranking alone is fused, every `_knn_distance` `None`; SurrealDB 3.1 and 3.2 list the first
|
|
6648
|
+
`vector_limit` records by id, with no distance. Pass `dist=` for an exact scan that needs no index.
|
|
6649
|
+
|
|
6650
|
+
#### Not supported
|
|
6651
|
+
|
|
6652
|
+
```python
|
|
6653
|
+
try:
|
|
6654
|
+
await Passage.objects().order_by("title").hybrid_search(
|
|
6655
|
+
vector_field="embedding", vector=[1.0, 0.0, 0.0], text_field="title", text_query="cats",
|
|
6656
|
+
)
|
|
6657
|
+
except ValueError as error:
|
|
6658
|
+
print(error)
|
|
6659
|
+
# hybrid_search() cannot be combined with order_by(): it runs its own vector and full-text searches and orders the records by their fused score. Remove them from the queryset.
|
|
6660
|
+
```
|
|
6661
|
+
|
|
6662
|
+
- `order_by()`, `annotate()`, `values()`, `search()` and `similar_to()` raise `ValueError` before any request, as
|
|
6663
|
+
above; `as_of()` raises `SurrealDbError`: the indexes hold the records' current values.
|
|
6664
|
+
- Several full-text fields, weights and score normalisation (`search::linear()`), and the full-text side's highlights
|
|
6665
|
+
are not available yet.
|
|
6666
|
+
- **A field stored under a name `hybrid_search()` uses for its own values** — `rrf_score`, `_rrf_score`,
|
|
6667
|
+
`_knn_distance`, `_search_score`, `_hybrid_pos`, `_hybrid_vec_pos` or `_hybrid_text_pos` — raises `ValueError`
|
|
6668
|
+
before any request: store it under another name (`Field(alias=...)`). On a SCHEMALESS table, a column with one of
|
|
6669
|
+
these names that the model does not declare is not returned, on every line, even when `select()` names it.
|
|
6670
|
+
|
|
6671
|
+
#### Coming from the full SurrealDB-ORM
|
|
6672
|
+
|
|
6673
|
+
- **`hybrid_search()` keeps the full ORM's signature**: keyword-only `vector_field`, `vector`, `vector_limit=20`,
|
|
6674
|
+
`text_field`, `text_query`, `text_limit=20`, `rrf_k=60`; `ef` is `None` for 100, so that `dist=` (an exact scan,
|
|
6675
|
+
section 37) can be given instead.
|
|
6676
|
+
- **It returns dicts, as the full ORM does**: each holds the record's fields, keyed by their Python names as
|
|
6677
|
+
`exec()`'s dict rows are, and `_rrf_score`, plus `_knn_distance` and `_search_score`, which the full ORM does not
|
|
6678
|
+
return.
|
|
6679
|
+
- **It applies the queryset**: its conditions, `select()`, `fetch()`, `limit()`/`offset()`, `cache()`, transactions
|
|
6680
|
+
and named connections; the full ORM ignores them.
|
|
6681
|
+
- **SurrealDB's `search::rrf()` fuses the lists on 3.x**, given both lists in one statement, and the ORM fuses them on
|
|
6682
|
+
2.x and the embedded engine; the full ORM calls `search::rrf()` with arguments SurrealDB 3.3 refuses, and runs on
|
|
6683
|
+
3.x only.
|
|
6684
|
+
- **Equal fused scores have a fixed order** (the vector list's first), so the same two lists always give the same
|
|
6685
|
+
page; on SurrealDB 2.x and the embedded engine, an exact scan (`dist=`) can change the vector list itself between
|
|
6686
|
+
two identical calls, as above.
|
|
6687
|
+
- **Checked before any request**: both fields, the vector, `vector_limit` and `text_limit` (1 to 10 000), `rrf_k`
|
|
6688
|
+
(0 to 10 000), `ef` and `dist`; the full ORM checks the field names only.
|
|
6689
|
+
- **Reciprocal Rank Fusion only**: `search::linear()` (weights and score normalisation) is not offered yet.
|
|
6690
|
+
|
|
6514
6691
|
---
|
|
6515
6692
|
|
|
6516
6693
|
## Configuration Options
|
|
@@ -6633,7 +6810,7 @@ features introduced in SurrealDB 3.x. On 2.x the ORM degrades gracefully. Capabi
|
|
|
6633
6810
|
listed behave the same on both lines. The 2.x column was measured on 2.6.5 and holds unchanged
|
|
6634
6811
|
on 2.7.0. The 3.x column was measured on 3.2.4 and holds on 3.3.0, except where a row says
|
|
6635
6812
|
otherwise. The full suite and every divergence below were re-verified on both new versions.
|
|
6636
|
-
Rows added in v0.20.0 were measured directly on 2.7.0 and 3.3.0; rows added in v0.21.0 on 2.7.0 and 3.3.2, re-verified on 3.3.3, with 3.1.5 where it differs; rows added in v0.22.0 on 2.7.0 and 3.3.3, with 3.1.5 where it differs; rows added in v0.23.0 on 2.7.0 and 3.3.4, with 3.1.5 where it differs; rows added in v0.24.0 on 2.7.0 and 3.3.4, identical on 3.1.5; rows added in v0.26.0 on 2.7.0 and 3.3.4, in memory and on versioned storage, identical on 3.1.5; rows added in v0.27.0 on 2.7.0 and 3.3.4, with 3.1.5 where it differs; rows added in v0.28.0 on 2.7.0 and 3.3.4, identical on 3.1.5; the rows added in v0.28.1 on 2.7.0 and 3.3.4, identical on 3.1.5; rows added in v0.29.0 on 2.7.0 and 3.3.4 in one process, identical on 3.1.5. The row added in v0.29.2 was measured on 2.7.0, 3.3.4 and the embedded engine, identical on 3.1.5. Rows added in v0.30.0 were measured on 2.7.0, 3.3.4 and the embedded engine, with 3.1.5 where it differs. The row added in v0.30.3 was measured on 2.7.0 and 3.3.4, over WebSocket and HTTP, and on the embedded engine, identical on 3.1.5. The row added in v0.30.7 was measured on 2.7.0, 3.3.4 and the embedded engine, identical on 3.1.5: its parser refuses a statement that nests six objects, and `update()` nests at most four. Rows added in v0.31.0 were measured on 2.7.0, 3.3.4 and the embedded engine, identical on 3.1.5. The row added in v0.31.1 was measured on 2.7.0, 3.3.4 and the embedded engine, identical on 3.1.5. Rows added in v0.32.0 were measured on 2.7.0, 3.3.4 and the embedded engine; 3.1.5 behaves as 3.3.4 but accepts raw writes to a view. Rows added in v0.33.0 were measured on 2.7.0, 3.3.4 and the embedded engine; 3.1.5 behaves as 3.3.4. Rows added in v0.34.0 were measured on 2.7.0, 3.3.4 and the embedded engine; 3.1.5 behaves as 3.3.4 except that it refuses a Python `set` written to a `set` field, and an object item of an optional untyped `list`. Rows added in v0.35.0 were measured on 2.7.0, 3.3.4 and the embedded engine, with 3.1.5 where it differs. The row added in v0.35.2 was measured on 2.7.0 and 3.3.4, identical on 3.1.5. The row added in v0.35.3 was measured on 2.7.0 and 3.3.4, over WebSocket and HTTP, and on the embedded engine, identical on 3.1.5. Rows added in v0.36.0 were measured on 2.7.0, 3.3.4 and the embedded engine, with 3.1.5 where it differs. The row added in v0.36.2 was measured on 2.7.0 and 3.3.4, over WebSocket and HTTP, and on the embedded engine, identical on 3.1.5.
|
|
6813
|
+
Rows added in v0.20.0 were measured directly on 2.7.0 and 3.3.0; rows added in v0.21.0 on 2.7.0 and 3.3.2, re-verified on 3.3.3, with 3.1.5 where it differs; rows added in v0.22.0 on 2.7.0 and 3.3.3, with 3.1.5 where it differs; rows added in v0.23.0 on 2.7.0 and 3.3.4, with 3.1.5 where it differs; rows added in v0.24.0 on 2.7.0 and 3.3.4, identical on 3.1.5; rows added in v0.26.0 on 2.7.0 and 3.3.4, in memory and on versioned storage, identical on 3.1.5; rows added in v0.27.0 on 2.7.0 and 3.3.4, with 3.1.5 where it differs; rows added in v0.28.0 on 2.7.0 and 3.3.4, identical on 3.1.5; the rows added in v0.28.1 on 2.7.0 and 3.3.4, identical on 3.1.5; rows added in v0.29.0 on 2.7.0 and 3.3.4 in one process, identical on 3.1.5. The row added in v0.29.2 was measured on 2.7.0, 3.3.4 and the embedded engine, identical on 3.1.5. Rows added in v0.30.0 were measured on 2.7.0, 3.3.4 and the embedded engine, with 3.1.5 where it differs. The row added in v0.30.3 was measured on 2.7.0 and 3.3.4, over WebSocket and HTTP, and on the embedded engine, identical on 3.1.5. The row added in v0.30.7 was measured on 2.7.0, 3.3.4 and the embedded engine, identical on 3.1.5: its parser refuses a statement that nests six objects, and `update()` nests at most four. Rows added in v0.31.0 were measured on 2.7.0, 3.3.4 and the embedded engine, identical on 3.1.5. The row added in v0.31.1 was measured on 2.7.0, 3.3.4 and the embedded engine, identical on 3.1.5. Rows added in v0.32.0 were measured on 2.7.0, 3.3.4 and the embedded engine; 3.1.5 behaves as 3.3.4 but accepts raw writes to a view. Rows added in v0.33.0 were measured on 2.7.0, 3.3.4 and the embedded engine; 3.1.5 behaves as 3.3.4. Rows added in v0.34.0 were measured on 2.7.0, 3.3.4 and the embedded engine; 3.1.5 behaves as 3.3.4 except that it refuses a Python `set` written to a `set` field, and an object item of an optional untyped `list`. Rows added in v0.35.0 were measured on 2.7.0, 3.3.4 and the embedded engine, with 3.1.5 where it differs. The row added in v0.35.2 was measured on 2.7.0 and 3.3.4, identical on 3.1.5. The row added in v0.35.3 was measured on 2.7.0 and 3.3.4, over WebSocket and HTTP, and on the embedded engine, identical on 3.1.5. Rows added in v0.36.0 were measured on 2.7.0, 3.3.4 and the embedded engine, with 3.1.5 where it differs. The row added in v0.36.2 was measured on 2.7.0 and 3.3.4, over WebSocket and HTTP, and on the embedded engine, identical on 3.1.5. Rows added in v0.37.0 were measured on 2.7.0, 3.3.4, 3.2.6 and the embedded engine, with 3.1.5 and 3.2.6 where they differ from 3.3.4.
|
|
6637
6814
|
The SDK's in-process engine has a table of its own: [ORM behaviour: the embedded engine](#orm-behaviour-the-embedded-engine).
|
|
6638
6815
|
|
|
6639
6816
|
| ORM capability | SurrealDB 2.6.x / 2.7.x | SurrealDB 3.2.x / 3.3.x | Since |
|
|
@@ -6794,6 +6971,10 @@ The SDK's in-process engine has a table of its own: [ORM behaviour: the embedded
|
|
|
6794
6971
|
| `similar_to(dist=…)` with a condition on a linked record's field | the ORM selects the matching records first (a `LET`): 2.x's exact scan would return every matching record, ignoring `limit` | the condition stays next to the KNN operator | v0.36.0 |
|
|
6795
6972
|
| `similar_to()` on a field without a vector index | no record | no record; 3.1 answers every record, with no distance | v0.36.0 |
|
|
6796
6973
|
| `remove_index(tx=)` then `define_indexes(tx=)` in one block, the index stored before the block | on a table holding records, the commit fails with a retryable `SurrealDbConflictError`, again on retry: nothing committed, the index kept; on an empty table, defined again | the index is defined again after its removal and committed, serving the records committed before the block | v0.36.2 |
|
|
6974
|
+
| `hybrid_search()`: the fusion | the ORM fuses the two lists with `search::rrf()`'s formula, which SurrealDB 2.x does not have: the same records, scores and order for the same two lists | SurrealDB's `search::rrf()` fuses them; the ORM orders equal scores, which SurrealDB returns in no fixed order | v0.37.0 |
|
|
6975
|
+
| `hybrid_search()` with other conditions | applied to both lists; the vector list selects the records first (a `LET`), as `similar_to()` does | applied to both lists, during each search | v0.37.0 |
|
|
6976
|
+
| `hybrid_search()` on a field without a vector or full-text index | no vector index: the vector list is empty, so the full-text ranking alone is fused; no full-text index: `SurrealDbError` naming the field | the same; 3.1 and 3.2 put the first `vector_limit` records by id in the vector list, with no distance, and match no record for the text | v0.37.0 |
|
|
6977
|
+
| An exact scan (`similar_to(dist=…)`, `hybrid_search(dist=…)`) with records at an equal distance at the last place | the scan picks among them in no fixed order: the records, and a page of them, can change between two identical calls; the HNSW index (no `dist=`) returns the same ones | the same records on every call, as with the HNSW index | pre-existing (documented in v0.37.0) |
|
|
6797
6978
|
| Full suite and every divergence in this table, re-run on SurrealDB 2.7.0 and 3.3.0 | 2.7.0 identical to 2.6.5 (same passes, same 3.x-only skips) | 3.3.0 identical to 3.2.4, except the system-user signin row | v0.19.1 |
|
|
6798
6979
|
|
|
6799
6980
|
> **Note on record IDs**: A record loaded from the database has its `id` field set to a native `surrealdb.RecordID` object, not a plain string. Use `model.get_raw_id()` to obtain the bare identifier string (e.g. `"alice"`), or compare directly with `model.id == RecordID("User", "alice")`. In-memory instances you construct yourself retain whatever value you assign.
|
|
@@ -6834,6 +7015,7 @@ Capabilities not listed behave as on a 2.x server.
|
|
|
6834
7015
|
| `define_indexes(tx=)` in a block that writes to the model's table | an ORM write before it: refused all the same, although this engine indexes its records; a write after it is committed and indexed | refused (left out of the index); on 2.x a write after it, to a table holding records, fails the commit | v0.35.3 |
|
|
6835
7016
|
| Vector search (v0.36.0) | as on a 2.x server: `MTREE` accepted, at most `ef` records returned (the ORM raises `ef` to `limit`), other conditions selected first in a `LET` (search annotations refused), writes made by the ids of the nearest, fewer records with a selective condition; an index without `TYPE` is `F64` (servers: `F32`), so the ORM always writes `TYPE` | 3.x refuses `MTREE`, applies the conditions during the index search and keeps search annotations | v0.36.0 |
|
|
6836
7017
|
| `remove_index(tx=)` then `define_indexes(tx=)` in one block | the index is defined again after its removal and committed, on a table holding records too | the same on 3.x; on 2.x, on a table holding records, the commit fails with a retryable conflict | v0.36.2 |
|
|
7018
|
+
| Hybrid search (v0.37.0) | as on a 2.x server: the ORM fuses the two lists with `search::rrf()`'s formula, which the engine does not have, and the vector list selects the other conditions first (a `LET`); an exact scan (`dist=`) picks among records at an equal distance at the `vector_limit`-th place in no fixed order, as `similar_to(dist=…)` does | 3.x fuses with `search::rrf()`, each list applying the conditions; its exact scan returns the same list | v0.37.0 |
|
|
6837
7019
|
| Everything else | CRUD, QuerySet, aggregations, relations, typed edges, rich and geo fields, upsert/patch, server values, computed fields, stored functions, signals: the same as on a 2.x server | unchanged | v0.25.0 |
|
|
6838
7020
|
|
|
6839
7021
|
---
|
|
@@ -6885,7 +7067,8 @@ Contributions are welcome! Please:
|
|
|
6885
7067
|
| v0.34.0 | Tier 3 — Model table DDL | ✅ Released |
|
|
6886
7068
|
| v0.35.0 | Tier 3 — Full-text search | ✅ Released |
|
|
6887
7069
|
| v0.36.0 | Tier 3 — Vector search | ✅ Released |
|
|
6888
|
-
| v0.37.0
|
|
7070
|
+
| v0.37.0 | Tier 3 — Hybrid search | ✅ Released |
|
|
7071
|
+
| v0.38.0 – v0.40.0 | Tier 3 — Advanced (migrations, CLI) | 📋 Planned |
|
|
6889
7072
|
| v0.41.0 | Beta Phase (API freeze, hardening) | 📋 Planned |
|
|
6890
7073
|
| v2.0.0 | Production / GA (aligned with SDK 2.0) | 📋 Planned |
|
|
6891
7074
|
|
|
@@ -6946,7 +7129,7 @@ SDK) and **server support**. Everything below is on the lite roadmap via the off
|
|
|
6946
7129
|
| Model table DDL | ✅ v0.34.0 (`define_table()`) | ✅ |
|
|
6947
7130
|
| Full-Text Search | ✅ v0.35.0 (`search()`) | ✅ |
|
|
6948
7131
|
| Vector Search (KNN/HNSW) | ✅ v0.36.0 (`similar_to()`) | ✅ |
|
|
6949
|
-
| Hybrid Search (RRF) | v0.37.0
|
|
7132
|
+
| Hybrid Search (RRF) | ✅ v0.37.0 (`hybrid_search()`) | ✅ |
|
|
6950
7133
|
| Migrations & CLI | v0.38 – v0.39 | ✅ |
|
|
6951
7134
|
| Test Fixtures & Factories | ✅ v0.25.0 SurrealFixture; factories v0.40.0 | ✅ |
|
|
6952
7135
|
| Retry, Logging, Metrics | v0.41.0 | ✅ |
|