pyobvector 0.2.12__tar.gz → 0.2.14__tar.gz

{pyobvector-0.2.12 → pyobvector-0.2.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pyobvector
-Version: 0.2.12
+Version: 0.2.14
 Summary: A python SDK for OceanBase Vector Store, based on SQLAlchemy, compatible with Milvus API.
 Author: shanhaikang.shk
 Author-email: shanhaikang.shk@oceanbase.com
@@ -36,7 +36,7 @@ poetry install
 - install with pip:
 
 ```shell
-pip install pyobvector==0.2.12
+pip install pyobvector==0.2.14
 ```
 
 ## Build Doc

{pyobvector-0.2.12 → pyobvector-0.2.14}/README.md

@@ -15,7 +15,7 @@ poetry install
 - install with pip:
 
 ```shell
-pip install pyobvector==0.2.12
+pip install pyobvector==0.2.14
 ```
 
 ## Build Doc

{pyobvector-0.2.12 → pyobvector-0.2.14}/pyobvector/client/ob_vec_client.py

@@ -89,6 +89,21 @@ class ObVecClient:
                         message=ExceptionsMessage.ClusterVersionIsLow,
                     )
 
+    def refresh_metadata(self, tables: Optional[list[str]] = None):
+        """Reload metadata from the database.
+        
+        Args:
+            tables (Optional[list[str]]): names of the tables to refresh. If None, refresh all tables.
+        """
+        if tables is not None:
+            for table_name in tables:
+                if table_name in self.metadata_obj.tables:
+                    self.metadata_obj.remove(Table(table_name, self.metadata_obj))
+            self.metadata_obj.reflect(bind=self.engine, only=tables, extend_existing=True)
+        else:
+            self.metadata_obj.clear()
+            self.metadata_obj.reflect(bind=self.engine, extend_existing=True)
+
     def _insert_partition_hint_for_query_sql(self, sql: str, partition_hint: str):
         from_index = sql.find("FROM")
         assert from_index != -1
@@ -801,3 +816,47 @@ class ObVecClient:
         with self.engine.connect() as conn:
             with conn.begin():
                 return conn.execute(text(text_sql))
+
+    def add_columns(
+        self,
+        table_name: str,
+        columns: list[Column],
+    ):
+        """Add multiple columns to an existing table.
+        
+        Args:
+            table_name (string): table name
+            columns (list[Column]): list of SQLAlchemy Column objects representing the new columns
+        """
+        compiler = self.engine.dialect.ddl_compiler(self.engine.dialect, None)
+        column_specs = [compiler.get_column_specification(column) for column in columns]
+        columns_ddl = ", ".join(f"ADD COLUMN {spec}" for spec in column_specs)
+        
+        with self.engine.connect() as conn:
+            with conn.begin():
+                conn.execute(
+                    text(f"ALTER TABLE `{table_name}` {columns_ddl}")
+                )
+                
+        self.refresh_metadata([table_name])
+
+    def drop_columns(
+        self,
+        table_name: str,
+        column_names: list[str],
+    ):
+        """Drop multiple columns from an existing table.
+        
+        Args:
+            table_name (string): table name
+            column_names (list[str]): names of the columns to drop
+        """
+        columns_ddl = ", ".join(f"DROP COLUMN `{name}`" for name in column_names)
+        
+        with self.engine.connect() as conn:
+            with conn.begin():
+                conn.execute(
+                    text(f"ALTER TABLE `{table_name}` {columns_ddl}")
+                )
+                
+        self.refresh_metadata([table_name])

{pyobvector-0.2.12 → pyobvector-0.2.14}/pyobvector/client/ob_vec_json_table_client.py

@@ -236,6 +236,7 @@ class ObVecJsonTableClient(ObVecClient):
             raise ValueError("Table name duplicated")
         
         session = self.session()
+        session.execute(text("SET @@session.autocommit=0"))
         new_meta_cache_items = []
         col_id = 16
         for col_def in ast.find_all(exp.ColumnDef):
@@ -607,6 +608,7 @@ class ObVecJsonTableClient(ObVecClient):
             raise ValueError(f"Table {jtable_name} does not exists")
         
         session = self.session()
+        session.execute(text("SET @@session.autocommit=0"))
         for action in ast.actions:
             if isinstance(action, ChangeColumn):
                 self._handle_alter_jtable_change_column(
@@ -681,6 +683,7 @@ class ObVecJsonTableClient(ObVecClient):
             raise ValueError(f"Invalid ast type {ast.this}")
 
         session = self.session()
+        session.execute(text("SET @@session.autocommit=0"))
         n_new_records = 0
         for tuple in ast.expression.expressions:
             expr_list = tuple.expressions

{pyobvector-0.2.12 → pyobvector-0.2.14}/pyobvector/schema/array.py

@@ -1,6 +1,6 @@
 """ARRAY: An extended data type for SQLAlchemy"""
 import json
-from typing import Any, List, Optional, Sequence, Union, Type
+from typing import Any, List, Optional, Sequence, Union
 
 from sqlalchemy.sql.type_api import TypeEngine
 from sqlalchemy.types import UserDefinedType, String
@@ -10,7 +10,6 @@ class ARRAY(UserDefinedType):
     """ARRAY data type definition with support for up to 6 levels of nesting."""
     cache_ok = True
     _string = String()
-    _max_nesting_level = 6
 
     def __init__(self, item_type: Union[TypeEngine, type]):
         """Construct an ARRAY.
@@ -18,25 +17,17 @@ class ARRAY(UserDefinedType):
         Args:
             item_type: The data type of items in this array. For nested arrays,
                       pass another ARRAY type.
-
-        Raises:
-            ValueError: If nesting level exceeds the maximum allowed level (6).
         """
         super(UserDefinedType, self).__init__()
         if isinstance(item_type, type):
             item_type = item_type()
         self.item_type = item_type
-        self._validate_nesting_level()
-
-    def _validate_nesting_level(self):
-        """Validate that the nesting level does not exceed the maximum allowed level."""
-        level = 1
-        current_type = self.item_type
-        while isinstance(current_type, ARRAY):
-            level += 1
-            if level > self._max_nesting_level:
-                raise ValueError(f"Maximum nesting level of {self._max_nesting_level} exceeded")
-            current_type = current_type.item_type
+        if isinstance(item_type, ARRAY):
+            self.dim = item_type.dim + 1
+        else:
+            self.dim = 1
+        if self.dim > 6:
+            raise ValueError("Maximum nesting level of 6 exceeded")
 
     def get_col_spec(self, **kw):  # pylint: disable=unused-argument
         """Parse to array data type definition in text SQL."""
@@ -46,12 +37,33 @@ class ARRAY(UserDefinedType):
             base_type = str(self.item_type)
         return f"ARRAY({base_type})"
 
+    def _get_list_depth(self, value: Any) -> int:
+        if not isinstance(value, list):
+            return 0
+        max_depth = 0
+        for element in value:
+            current_depth = self._get_list_depth(element)
+            if current_depth > max_depth:
+                max_depth = current_depth
+        return 1 + max_depth
+
+    def _validate_dimension(self, value: list[Any]):
+        arr_depth = self._get_list_depth(value)
+        assert arr_depth == self.dim, "Array dimension mismatch, expected {}, got {}".format(self.dim, arr_depth)
+
     def bind_processor(self, dialect):
-        item_proc = self.item_type.dialect_impl(dialect).bind_processor(dialect)
+        item_type = self.item_type
+        while isinstance(item_type, ARRAY):
+            item_type = item_type.item_type
 
-        def process(value: Optional[Sequence[Any]]) -> Optional[str]:
+        item_proc = item_type.dialect_impl(dialect).bind_processor(dialect)
+
+        def process(value: Optional[Sequence[Any] | str]) -> Optional[str]:
             if value is None:
                 return None
+            if isinstance(value, str):
+                self._validate_dimension(json.loads(value))
+                return value
 
             def convert(val):
                 if isinstance(val, (list, tuple)):
@@ -61,12 +73,17 @@ class ARRAY(UserDefinedType):
                 return val
 
             processed = convert(value)
+            self._validate_dimension(processed)
             return json.dumps(processed)
 
         return process
 
     def result_processor(self, dialect, coltype):
-        item_proc = self.item_type.dialect_impl(dialect).result_processor(dialect, coltype)
+        item_type = self.item_type
+        while isinstance(item_type, ARRAY):
+            item_type = item_type.item_type
+
+        item_proc = item_type.dialect_impl(dialect).result_processor(dialect, coltype)
 
         def process(value: Optional[str]) -> Optional[List[Any]]:
             if value is None:
@@ -85,7 +102,11 @@ class ARRAY(UserDefinedType):
         return process
 
     def literal_processor(self, dialect):
-        item_proc = self.item_type.dialect_impl(dialect).literal_processor(dialect)
+        item_type = self.item_type
+        while isinstance(item_type, ARRAY):
+            item_type = item_type.item_type
+
+        item_proc = item_type.dialect_impl(dialect).literal_processor(dialect)
 
         def process(value: Sequence[Any]) -> str:
             def convert(val):
@@ -99,44 +120,3 @@ class ARRAY(UserDefinedType):
             return json.dumps(processed)
 
         return process
-
-    def __repr__(self):
-        """Return a string representation of the array type."""
-        current_type = self.item_type
-        nesting_level = 1
-        base_type = current_type
-
-        # Find the innermost type and count nesting level
-        while isinstance(current_type, ARRAY):
-            nesting_level += 1
-            current_type = current_type.item_type
-            if not isinstance(current_type, ARRAY):
-                base_type = current_type
-
-        return f"{nesting_level}D_Array({base_type})"
-
-
-def nested_array(dim: int) -> Type[ARRAY]:
-    """Create a nested array type class with specified dimensions.
-    
-    Args:
-        dim: The number of dimensions for the array type (1-6)
-        
-    Returns:
-        A class type that can be instantiated with an item_type to create a nested array
-        
-    Raises:
-        ValueError: If dim is not between 1 and 6
-    """
-    if not 1 <= dim <= 6:
-        raise ValueError("Dimension must be between 1 and 6")
-
-    class ArrayType(ARRAY):
-        def __init__(self, item_type: Union[TypeEngine, type]):
-            nested_type = item_type
-            for _ in range(dim - 1):
-                nested_type = ARRAY(nested_type)
-            super().__init__(nested_type)
-
-    ArrayType.__name__ = f"{dim}D_Array"
-    return ArrayType

{pyobvector-0.2.12 → pyobvector-0.2.14}/pyobvector/schema/reflection.py

@@ -3,7 +3,7 @@ import re
 import logging
 from sqlalchemy.dialects.mysql.reflection import MySQLTableDefinitionParser, _re_compile, cleanup_text
 
-from pyobvector.schema.array import nested_array
+from pyobvector.schema.array import ARRAY
 
 logger = logging.getLogger(__name__)
 
@@ -100,7 +100,9 @@ class OceanBaseTableDefinitionParser(MySQLTableDefinitionParser):
                 item_type_args = [int(v) for v in self._re_csv_int.findall(item_type_arg)]
 
             nested_level = coltype_with_args.lower().count('array')
-            type_instance = nested_array(nested_level)(item_type(*item_type_args))
+            type_instance = item_type(*item_type_args)
+            for _ in range(nested_level):
+                type_instance = ARRAY(type_instance)
 
             col_kw = {}
 

{pyobvector-0.2.12 → pyobvector-0.2.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pyobvector"
-version = "0.2.12"
+version = "0.2.14"
 description = "A python SDK for OceanBase Vector Store, based on SQLAlchemy, compatible with Milvus API."
 authors = ["shanhaikang.shk <shanhaikang.shk@oceanbase.com>"]
 readme = "README.md"