pyobvector 0.2.17__tar.gz → 0.2.19__tar.gz

{pyobvector-0.2.17 → pyobvector-0.2.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyobvector
-Version: 0.2.17
+Version: 0.2.19
 Summary: A python SDK for OceanBase Vector Store, based on SQLAlchemy, compatible with Milvus API.
 License-File: LICENSE
 Author: shanhaikang.shk
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: aiomysql (>=0.3.2,<0.4.0)
-Requires-Dist: numpy (>=1.17.0,<2.0.0)
+Requires-Dist: numpy (>=1.17.0)
 Requires-Dist: pydantic (>=2.7.0,<3)
 Requires-Dist: pymysql (>=1.1.1,<2.0.0)
 Requires-Dist: sqlalchemy (>=1.4,<=3)
@@ -38,7 +38,7 @@ poetry install
 - install with pip:
 
 ```shell
-pip install pyobvector==0.2.17
+pip install pyobvector==0.2.19
 ```
 
 ## Build Doc
@@ -50,12 +50,17 @@ mkdir build
 make html
 ```
 
+## Release Notes
+
+For detailed release notes and changelog, see [RELEASE_NOTES.md](RELEASE_NOTES.md).
+
 ## Usage
 
-`pyobvector` supports two modes:
+`pyobvector` supports three modes:
 
 - `Milvus compatible mode`: You can use the `MilvusLikeClient` class to use vector storage in a way similar to the Milvus API
 - `SQLAlchemy hybrid mode`: You can use the vector storage function provided by the `ObVecClient` class and execute the relational database statement with the SQLAlchemy library. In this mode, you can regard `pyobvector` as an extension of SQLAlchemy.
+- `Hybrid Search mode`: You can use the `HybridSearch` class to perform hybrid search that combines full-text search and vector similarity search, with Elasticsearch-compatible query syntax.
 
 ### Milvus compatible mode
 
@@ -229,22 +234,21 @@ res = self.client.ann_search(
 The `ann_search` method supports flexible output column selection through the `output_columns` parameter:
 
 - **`output_columns`** (recommended): Accepts SQLAlchemy Column objects, expressions, or a mix of both
+
   - Column objects: `table.c.id`, `table.c.name`
   - Expressions: `(table.c.age + 10).label('age_plus_10')`
   - JSON queries: `text("JSON_EXTRACT(meta, '$.key') as extracted_key")`
   - String functions: `func.concat(table.c.name, ' (', table.c.age, ')').label('name_age')`
-
 - **`output_column_names`** (legacy): Accepts list of column name strings
-  - Example: `['id', 'name', 'meta']`
 
+  - Example: `['id', 'name', 'meta']`
 - **Parameter Priority**: `output_columns` takes precedence over `output_column_names` when both are provided
-
 - **`distance_threshold`** (optional): Filter results by distance threshold
+
   - Type: `Optional[float]`
   - Only returns results where `distance <= threshold`
   - Example: `distance_threshold=0.5` returns only results with distance <= 0.5
   - Use case: Quality control for similarity search, only return highly similar results
-
 - If you want to use pure `SQLAlchemy` API with `OceanBase` dialect, you can just get an `SQLAlchemy.engine` via `client.engine`. The engine can also be created as following:
 
 ```python
@@ -283,3 +287,170 @@ engine = create_async_engine(connection_str)
 
 - For further usage in pure `SQLAlchemy` mode, please refer to [SQLAlchemy](https://www.sqlalchemy.org/)
 
+### Hybrid Search Mode
+
+`pyobvector` supports hybrid search that combines full-text search and vector similarity search, with query syntax compatible with Elasticsearch. This allows you to perform semantic search with both keyword matching and vector similarity in a single query.
+
+- setup a client:
+
+```python
+from pyobvector import *
+from pyobvector.client.hybrid_search import HybridSearch
+from sqlalchemy import Column, Integer, VARCHAR
+
+client = HybridSearch(uri="127.0.0.1:2881", user="test@test")
+```
+
+**Note**: Hybrid search requires OceanBase version >= 4.4.1.0, or SeekDB.
+
+- create a table with both vector index and full-text index:
+
+```python
+test_table_name = "hybrid_search_test"
+
+# create table with vector and text columns
+client.create_table(
+    table_name=test_table_name,
+    columns=[
+        Column("id", Integer, primary_key=True, autoincrement=False),
+        Column("source_id", VARCHAR(32)),
+        Column("enabled", Integer),
+        Column("vector", VECTOR(3)),  # vector column
+        Column("title", VARCHAR(255)),  # text column for full-text search
+        Column("content", VARCHAR(255)),  # text column for full-text search
+    ],
+    indexes=[
+        VectorIndex("vec_idx", "vector", params="distance=l2, type=hnsw, lib=vsag"),
+    ],
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_unicode_ci',
+)
+
+# create full-text indexes for text columns
+from pyobvector import FtsIndexParam, FtsParser
+
+for col in ["title", "content"]:
+    client.create_fts_idx_with_fts_index_param(
+        table_name=test_table_name,
+        fts_idx_param=FtsIndexParam(
+            index_name=f"fts_idx_{col}",
+            field_names=[col],
+            parser_type=FtsParser.IK,  # or other parser types
+        ),
+    )
+```
+
+- insert data:
+
+```python
+client.insert(
+    table_name=test_table_name,
+    data=[
+        {
+            "id": 1,
+            "source_id": "3b767712b57211f09c170242ac130008",
+            "enabled": 1,
+            "vector": [1, 1, 1],
+            "title": "企业版和社区版的功能差异",
+            "content": "OceanBase 数据库提供企业版和社区版两种形态。",
+        },
+        {
+            "id": 2,
+            "vector": [1, 2, 3],
+            "enabled": 1,
+            "source_id": "3b791472b57211f09c170242ac130008",
+            "title": "快速体验 OceanBase 社区版",
+            "content": "本文根据使用场景详细介绍如何快速部署 OceanBase 数据库。",
+        },
+        # ... more data
+    ]
+)
+```
+
+- perform hybrid search with Elasticsearch-compatible query syntax:
+
+```python
+# build query body (compatible with Elasticsearch syntax)
+query = {
+    "bool": {
+        "must": [
+            {
+                "query_string": {
+                    "fields": ["title^10", "content"],  # field weights
+                    "type": "best_fields",
+                    "query": "oceanbase 数据 迁移",
+                    "minimum_should_match": "30%",
+                    "boost": 1
+                }
+            }
+        ],
+        "filter": [
+            {
+                "terms": {
+                    "source_id": [
+                        "3b791472b57211f09c170242ac130008",
+                        "3b7af31eb57211f09c170242ac130008"
+                    ]
+                }
+            },
+            {
+                "bool": {
+                    "must_not": [
+                        {
+                            "range": {
+                                "enabled": {"lt": 1}
+                            }
+                        }
+                    ]
+                }
+            }
+        ],
+        "boost": 0.7
+    }
+}
+
+body = {
+    "query": query,
+    "knn": {  # vector similarity search
+        "field": "vector",
+        "k": 1024,
+        "num_candidates": 1024,
+        "query_vector": [1, 2, 3],
+        "filter": query,  # optional: apply same filter to KNN
+        "similarity": 0.2  # similarity threshold
+    },
+    "from": 0,  # pagination offset
+    "size": 60  # pagination size
+}
+
+# execute hybrid search
+results = client.search(index=test_table_name, body=body)
+# results is a list of matching documents
+```
+
+#### Supported Query Types
+
+The hybrid search supports Elasticsearch-compatible query syntax:
+
+- **`bool` query**: Combine multiple queries with `must`, `must_not`, `should`, `filter`
+- **`query_string`**: Full-text search with field weights, boost, and matching options
+- **`terms`**: Exact match filtering for multiple values
+- **`range`**: Range queries (`lt`, `lte`, `gt`, `gte`)
+- **`knn`**: Vector similarity search (KNN) with:
+  - `field`: Vector field name
+  - `query_vector`: Query vector
+  - `k`: Number of results to return
+  - `num_candidates`: Number of candidates to consider
+  - `filter`: Optional filter to apply to KNN search
+  - `similarity`: Similarity threshold
+- **Pagination**: `from` and `size` parameters
+
+#### Get SQL Query
+
+You can also get the actual SQL that will be executed:
+
+```python
+sql = client.get_sql(index=test_table_name, body=body)
+print(sql)  # prints the SQL query
+```
+

{pyobvector-0.2.17 → pyobvector-0.2.19}/README.md

@@ -15,7 +15,7 @@ poetry install
 - install with pip:
 
 ```shell
-pip install pyobvector==0.2.17
+pip install pyobvector==0.2.19
 ```
 
 ## Build Doc
@@ -27,12 +27,17 @@ mkdir build
 make html
 ```
 
+## Release Notes
+
+For detailed release notes and changelog, see [RELEASE_NOTES.md](RELEASE_NOTES.md).
+
 ## Usage
 
-`pyobvector` supports two modes:
+`pyobvector` supports three modes:
 
 - `Milvus compatible mode`: You can use the `MilvusLikeClient` class to use vector storage in a way similar to the Milvus API
 - `SQLAlchemy hybrid mode`: You can use the vector storage function provided by the `ObVecClient` class and execute the relational database statement with the SQLAlchemy library. In this mode, you can regard `pyobvector` as an extension of SQLAlchemy.
+- `Hybrid Search mode`: You can use the `HybridSearch` class to perform hybrid search that combines full-text search and vector similarity search, with Elasticsearch-compatible query syntax.
 
 ### Milvus compatible mode
 
@@ -206,22 +211,21 @@ res = self.client.ann_search(
 The `ann_search` method supports flexible output column selection through the `output_columns` parameter:
 
 - **`output_columns`** (recommended): Accepts SQLAlchemy Column objects, expressions, or a mix of both
+
   - Column objects: `table.c.id`, `table.c.name`
   - Expressions: `(table.c.age + 10).label('age_plus_10')`
   - JSON queries: `text("JSON_EXTRACT(meta, '$.key') as extracted_key")`
   - String functions: `func.concat(table.c.name, ' (', table.c.age, ')').label('name_age')`
-
 - **`output_column_names`** (legacy): Accepts list of column name strings
-  - Example: `['id', 'name', 'meta']`
 
+  - Example: `['id', 'name', 'meta']`
 - **Parameter Priority**: `output_columns` takes precedence over `output_column_names` when both are provided
-
 - **`distance_threshold`** (optional): Filter results by distance threshold
+
   - Type: `Optional[float]`
   - Only returns results where `distance <= threshold`
   - Example: `distance_threshold=0.5` returns only results with distance <= 0.5
   - Use case: Quality control for similarity search, only return highly similar results
-
 - If you want to use pure `SQLAlchemy` API with `OceanBase` dialect, you can just get an `SQLAlchemy.engine` via `client.engine`. The engine can also be created as following:
 
 ```python
@@ -259,3 +263,170 @@ engine = create_async_engine(connection_str)
 ```
 
 - For further usage in pure `SQLAlchemy` mode, please refer to [SQLAlchemy](https://www.sqlalchemy.org/)
+
+### Hybrid Search Mode
+
+`pyobvector` supports hybrid search that combines full-text search and vector similarity search, with query syntax compatible with Elasticsearch. This allows you to perform semantic search with both keyword matching and vector similarity in a single query.
+
+- setup a client:
+
+```python
+from pyobvector import *
+from pyobvector.client.hybrid_search import HybridSearch
+from sqlalchemy import Column, Integer, VARCHAR
+
+client = HybridSearch(uri="127.0.0.1:2881", user="test@test")
+```
+
+**Note**: Hybrid search requires OceanBase version >= 4.4.1.0, or SeekDB.
+
+- create a table with both vector index and full-text index:
+
+```python
+test_table_name = "hybrid_search_test"
+
+# create table with vector and text columns
+client.create_table(
+    table_name=test_table_name,
+    columns=[
+        Column("id", Integer, primary_key=True, autoincrement=False),
+        Column("source_id", VARCHAR(32)),
+        Column("enabled", Integer),
+        Column("vector", VECTOR(3)),  # vector column
+        Column("title", VARCHAR(255)),  # text column for full-text search
+        Column("content", VARCHAR(255)),  # text column for full-text search
+    ],
+    indexes=[
+        VectorIndex("vec_idx", "vector", params="distance=l2, type=hnsw, lib=vsag"),
+    ],
+    mysql_charset='utf8mb4',
+    mysql_collate='utf8mb4_unicode_ci',
+)
+
+# create full-text indexes for text columns
+from pyobvector import FtsIndexParam, FtsParser
+
+for col in ["title", "content"]:
+    client.create_fts_idx_with_fts_index_param(
+        table_name=test_table_name,
+        fts_idx_param=FtsIndexParam(
+            index_name=f"fts_idx_{col}",
+            field_names=[col],
+            parser_type=FtsParser.IK,  # or other parser types
+        ),
+    )
+```
+
+- insert data:
+
+```python
+client.insert(
+    table_name=test_table_name,
+    data=[
+        {
+            "id": 1,
+            "source_id": "3b767712b57211f09c170242ac130008",
+            "enabled": 1,
+            "vector": [1, 1, 1],
+            "title": "企业版和社区版的功能差异",
+            "content": "OceanBase 数据库提供企业版和社区版两种形态。",
+        },
+        {
+            "id": 2,
+            "vector": [1, 2, 3],
+            "enabled": 1,
+            "source_id": "3b791472b57211f09c170242ac130008",
+            "title": "快速体验 OceanBase 社区版",
+            "content": "本文根据使用场景详细介绍如何快速部署 OceanBase 数据库。",
+        },
+        # ... more data
+    ]
+)
+```
+
+- perform hybrid search with Elasticsearch-compatible query syntax:
+
+```python
+# build query body (compatible with Elasticsearch syntax)
+query = {
+    "bool": {
+        "must": [
+            {
+                "query_string": {
+                    "fields": ["title^10", "content"],  # field weights
+                    "type": "best_fields",
+                    "query": "oceanbase 数据 迁移",
+                    "minimum_should_match": "30%",
+                    "boost": 1
+                }
+            }
+        ],
+        "filter": [
+            {
+                "terms": {
+                    "source_id": [
+                        "3b791472b57211f09c170242ac130008",
+                        "3b7af31eb57211f09c170242ac130008"
+                    ]
+                }
+            },
+            {
+                "bool": {
+                    "must_not": [
+                        {
+                            "range": {
+                                "enabled": {"lt": 1}
+                            }
+                        }
+                    ]
+                }
+            }
+        ],
+        "boost": 0.7
+    }
+}
+
+body = {
+    "query": query,
+    "knn": {  # vector similarity search
+        "field": "vector",
+        "k": 1024,
+        "num_candidates": 1024,
+        "query_vector": [1, 2, 3],
+        "filter": query,  # optional: apply same filter to KNN
+        "similarity": 0.2  # similarity threshold
+    },
+    "from": 0,  # pagination offset
+    "size": 60  # pagination size
+}
+
+# execute hybrid search
+results = client.search(index=test_table_name, body=body)
+# results is a list of matching documents
+```
+
+#### Supported Query Types
+
+The hybrid search supports Elasticsearch-compatible query syntax:
+
+- **`bool` query**: Combine multiple queries with `must`, `must_not`, `should`, `filter`
+- **`query_string`**: Full-text search with field weights, boost, and matching options
+- **`terms`**: Exact match filtering for multiple values
+- **`range`**: Range queries (`lt`, `lte`, `gt`, `gte`)
+- **`knn`**: Vector similarity search (KNN) with:
+  - `field`: Vector field name
+  - `query_vector`: Query vector
+  - `k`: Number of results to return
+  - `num_candidates`: Number of candidates to consider
+  - `filter`: Optional filter to apply to KNN search
+  - `similarity`: Similarity threshold
+- **Pagination**: `from` and `size` parameters
+
+#### Get SQL Query
+
+You can also get the actual SQL that will be executed:
+
+```python
+sql = client.get_sql(index=test_table_name, body=body)
+print(sql)  # prints the SQL query
+```

{pyobvector-0.2.17 → pyobvector-0.2.19}/pyobvector/client/exceptions.py

@@ -111,5 +111,5 @@ class ExceptionsMessage:
     MetricTypeValueInvalid = "MetricType should be 'l2'/'ip'/'neg_ip'/'cosine' in ann search."
     UsingInIDsWhenMultiPrimaryKey = "Using 'ids' when table has multi primary key."
     ClusterVersionIsLow = (
-        "OceanBase Vector Store is not supported because cluster version is below 4.3.3.0."
+        "OceanBase %s feature is not supported because cluster version is below %s."
     )

pyobvector-0.2.19/pyobvector/client/hybrid_search.py

@@ -0,0 +1,91 @@
+"""OceanBase Hybrid Search Client."""
+import json
+import logging
+from typing import Dict, Any
+
+from sqlalchemy import text
+
+from .exceptions import ClusterVersionException, ErrorCode, ExceptionsMessage
+from .ob_vec_client import ObVecClient as Client
+from ..util import ObVersion
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+
+class HybridSearch(Client):
+    """The OceanBase Hybrid Search Client"""
+
+    def __init__(
+        self,
+        uri: str = "127.0.0.1:2881",
+        user: str = "root@test",
+        password: str = "",
+        db_name: str = "test",
+        **kwargs,
+    ):
+        super().__init__(uri, user, password, db_name, **kwargs)
+
+        min_required_version = ObVersion.from_db_version_nums(4, 4, 1, 0)
+        
+        if self.ob_version < min_required_version:
+            # For versions < 4.4.1.0, check if it's SeekDB
+            with self.engine.connect() as conn:
+                with conn.begin():
+                    res = conn.execute(text("SELECT version()"))
+                    version_str = [r[0] for r in res][0]
+                    if "SeekDB" in version_str:
+                        logger.info(f"SeekDB detected in version string: {version_str}, allowing hybrid search")
+                        return
+            raise ClusterVersionException(
+                code=ErrorCode.NOT_SUPPORTED,
+                message=ExceptionsMessage.ClusterVersionIsLow % ("Hybrid Search", "4.4.1.0"),
+            )
+
+    def search(
+        self,
+        index: str,
+        body: Dict[str, Any],
+        **kwargs,
+    ):
+        """Execute hybrid search with parameter compatible with Elasticsearch.
+
+        Args:
+            index: The name of the table to search
+            body: The search query body
+            **kwargs: Additional search parameters
+
+        Returns:
+            Search results
+        """
+        body_str = json.dumps(body)
+
+        sql = text("SELECT DBMS_HYBRID_SEARCH.SEARCH(:index, :body_str)")
+
+        with self.engine.connect() as conn:
+            with conn.begin():
+                res = conn.execute(sql, {"index": index, "body_str": body_str}).fetchone()
+                return json.loads(res[0])
+
+    def get_sql(
+        self,
+        index: str,
+        body: Dict[str, Any],
+    ) -> str:
+        """Get the SQL actually to be executed in hybrid search.
+
+        Args:
+            index: The name of the table to search
+            body: The hybrid search query body
+
+        Returns:
+            The SQL actually to be executed
+        """
+        body_str = json.dumps(body)
+
+        sql = text("SELECT DBMS_HYBRID_SEARCH.GET_SQL(:index, :body_str)")
+
+        with self.engine.connect() as conn:
+            with conn.begin():
+                res = conn.execute(sql, {"index": index, "body_str": body_str}).fetchone()
+                return res[0]

{pyobvector-0.2.17 → pyobvector-0.2.19}/pyobvector/client/ob_vec_client.py

@@ -44,7 +44,7 @@ class ObVecClient(ObClient):
         if self.ob_version < ObVersion.from_db_version_nums(4, 3, 3, 0):
             raise ClusterVersionException(
                 code=ErrorCode.NOT_SUPPORTED,
-                message=ExceptionsMessage.ClusterVersionIsLow,
+                message=ExceptionsMessage.ClusterVersionIsLow % ("Vector Store", "4.3.3.0"),
             )
 
     def _get_sparse_vector_index_params(

{pyobvector-0.2.17 → pyobvector-0.2.19}/pyproject.toml

@@ -1,13 +1,13 @@
 [tool.poetry]
 name = "pyobvector"
-version = "0.2.17"
+version = "0.2.19"
 description = "A python SDK for OceanBase Vector Store, based on SQLAlchemy, compatible with Milvus API."
 authors = ["shanhaikang.shk <shanhaikang.shk@oceanbase.com>"]
 readme = "README.md"
 
 [tool.poetry.dependencies]
 python = ">=3.9,<4.0"
-numpy = ">=1.17.0,<2.0.0"
+numpy = ">=1.17.0"
 sqlalchemy = ">=1.4,<=3"
 pymysql = "^1.1.1"
 aiomysql = "^0.3.2"