matrixone-python-sdk 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- matrixone/async_client.py +94 -3
- matrixone/async_metadata_manager.py +3 -1
- matrixone/async_vector_index_manager.py +3 -1
- matrixone/client.py +116 -3
- matrixone/connection_hooks.py +1 -0
- matrixone/exceptions.py +0 -24
- matrixone/index_utils.py +121 -0
- matrixone/metadata.py +3 -2
- matrixone/moctl.py +0 -2
- matrixone/sqlalchemy_ext/__init__.py +1 -0
- matrixone/sqlalchemy_ext/vector_index.py +28 -0
- matrixone/version.py +0 -2
- {matrixone_python_sdk-0.1.1.dist-info → matrixone_python_sdk-0.1.3.dist-info}/METADATA +1 -1
- {matrixone_python_sdk-0.1.1.dist-info → matrixone_python_sdk-0.1.3.dist-info}/RECORD +20 -17
- tests/online/test_async_index_verification_online.py +259 -0
- tests/online/test_index_verification_online.py +194 -0
- {matrixone_python_sdk-0.1.1.dist-info → matrixone_python_sdk-0.1.3.dist-info}/WHEEL +0 -0
- {matrixone_python_sdk-0.1.1.dist-info → matrixone_python_sdk-0.1.3.dist-info}/entry_points.txt +0 -0
- {matrixone_python_sdk-0.1.1.dist-info → matrixone_python_sdk-0.1.3.dist-info}/licenses/LICENSE +0 -0
- {matrixone_python_sdk-0.1.1.dist-info → matrixone_python_sdk-0.1.3.dist-info}/top_level.txt +0 -0
matrixone/async_client.py
CHANGED
@@ -34,9 +34,10 @@ from datetime import datetime
|
|
34
34
|
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
35
35
|
|
36
36
|
from .account import Account, User
|
37
|
+
from .async_metadata_manager import AsyncMetadataManager
|
37
38
|
from .async_vector_index_manager import AsyncVectorManager
|
38
39
|
from .base_client import BaseMatrixOneClient, BaseMatrixOneExecutor
|
39
|
-
from .connection_hooks import
|
40
|
+
from .connection_hooks import ConnectionAction, ConnectionHook, create_connection_hook
|
40
41
|
from .exceptions import (
|
41
42
|
AccountError,
|
42
43
|
ConnectionError,
|
@@ -48,7 +49,6 @@ from .exceptions import (
|
|
48
49
|
SnapshotError,
|
49
50
|
)
|
50
51
|
from .logger import MatrixOneLogger, create_default_logger
|
51
|
-
from .async_metadata_manager import AsyncMetadataManager
|
52
52
|
from .pitr import Pitr
|
53
53
|
from .pubsub import Publication, Subscription
|
54
54
|
from .snapshot import Snapshot, SnapshotLevel
|
@@ -1837,7 +1837,6 @@ class AsyncClient(BaseMatrixOneClient):
|
|
1837
1837
|
"""Cleanup when object is garbage collected"""
|
1838
1838
|
# Don't try to cleanup in __del__ as it can cause issues with event loops
|
1839
1839
|
# The fixture should handle proper cleanup
|
1840
|
-
pass
|
1841
1840
|
|
1842
1841
|
def get_sqlalchemy_engine(self) -> AsyncEngine:
|
1843
1842
|
"""
|
@@ -1920,6 +1919,7 @@ class AsyncClient(BaseMatrixOneClient):
|
|
1920
1919
|
and other SDK components. External users should use execute() instead.
|
1921
1920
|
"""
|
1922
1921
|
import time
|
1922
|
+
|
1923
1923
|
from sqlalchemy import text
|
1924
1924
|
|
1925
1925
|
start_time = time.time()
|
@@ -2554,6 +2554,97 @@ class AsyncClient(BaseMatrixOneClient):
|
|
2554
2554
|
except Exception as e:
|
2555
2555
|
raise QueryError(f"Failed to get git version: {e}")
|
2556
2556
|
|
2557
|
+
async def get_secondary_index_tables(self, table_name: str) -> List[str]:
|
2558
|
+
"""
|
2559
|
+
Get all secondary index table names for a given table (async version).
|
2560
|
+
|
2561
|
+
Args:
|
2562
|
+
table_name: Name of the table to get secondary indexes for
|
2563
|
+
|
2564
|
+
Returns:
|
2565
|
+
List of secondary index table names
|
2566
|
+
|
2567
|
+
Examples::
|
2568
|
+
|
2569
|
+
>>> async with AsyncClient() as client:
|
2570
|
+
... await client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2571
|
+
... index_tables = await client.get_secondary_index_tables('cms_all_content_chunk_info')
|
2572
|
+
... print(index_tables)
|
2573
|
+
"""
|
2574
|
+
from .index_utils import build_get_index_tables_sql
|
2575
|
+
|
2576
|
+
sql, params = build_get_index_tables_sql(table_name)
|
2577
|
+
result = await self.execute(sql, params)
|
2578
|
+
return [row[0] for row in result.fetchall()]
|
2579
|
+
|
2580
|
+
async def get_secondary_index_table_by_name(self, table_name: str, index_name: str) -> Optional[str]:
|
2581
|
+
"""
|
2582
|
+
Get the physical table name of a secondary index by its index name (async version).
|
2583
|
+
|
2584
|
+
Args:
|
2585
|
+
table_name: Name of the table
|
2586
|
+
index_name: Name of the secondary index
|
2587
|
+
|
2588
|
+
Returns:
|
2589
|
+
Physical table name of the secondary index, or None if not found
|
2590
|
+
|
2591
|
+
Examples::
|
2592
|
+
|
2593
|
+
>>> async with AsyncClient() as client:
|
2594
|
+
... await client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2595
|
+
... index_table = await client.get_secondary_index_table_by_name('cms_all_content_chunk_info', 'cms_id')
|
2596
|
+
... print(index_table)
|
2597
|
+
"""
|
2598
|
+
from .index_utils import build_get_index_table_by_name_sql
|
2599
|
+
|
2600
|
+
sql, params = build_get_index_table_by_name_sql(table_name, index_name)
|
2601
|
+
result = await self.execute(sql, params)
|
2602
|
+
row = result.fetchone()
|
2603
|
+
return row[0] if row else None
|
2604
|
+
|
2605
|
+
async def verify_table_index_counts(self, table_name: str) -> int:
|
2606
|
+
"""
|
2607
|
+
Verify that the main table and all its secondary index tables have the same row count (async version).
|
2608
|
+
|
2609
|
+
This method compares the COUNT(*) of the main table with all its secondary index tables
|
2610
|
+
in a single SQL query for consistency. If counts don't match, raises an exception.
|
2611
|
+
|
2612
|
+
Args:
|
2613
|
+
table_name: Name of the table to verify
|
2614
|
+
|
2615
|
+
Returns:
|
2616
|
+
Row count (int) if verification succeeds
|
2617
|
+
|
2618
|
+
Raises:
|
2619
|
+
ValueError: If any secondary index table has a different count than the main table,
|
2620
|
+
with details about all counts in the error message
|
2621
|
+
|
2622
|
+
Examples::
|
2623
|
+
|
2624
|
+
>>> async with AsyncClient() as client:
|
2625
|
+
... await client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2626
|
+
... count = await client.verify_table_index_counts('cms_all_content_chunk_info')
|
2627
|
+
... print(f"✓ Verification passed, row count: {count}")
|
2628
|
+
|
2629
|
+
>>> # If verification fails:
|
2630
|
+
>>> try:
|
2631
|
+
... count = await client.verify_table_index_counts('some_table')
|
2632
|
+
... except ValueError as e:
|
2633
|
+
... print(f"Verification failed: {e}")
|
2634
|
+
"""
|
2635
|
+
from .index_utils import build_verify_counts_sql, process_verify_result
|
2636
|
+
|
2637
|
+
# Get all secondary index tables
|
2638
|
+
index_tables = await self.get_secondary_index_tables(table_name)
|
2639
|
+
|
2640
|
+
# Build and execute verification SQL
|
2641
|
+
sql = build_verify_counts_sql(table_name, index_tables)
|
2642
|
+
result = await self.execute(sql)
|
2643
|
+
row = result.fetchone()
|
2644
|
+
|
2645
|
+
# Process result and raise exception if verification fails
|
2646
|
+
return process_verify_result(table_name, index_tables, row)
|
2647
|
+
|
2557
2648
|
async def __aenter__(self):
|
2558
2649
|
return self
|
2559
2650
|
|
@@ -19,8 +19,10 @@ This module provides async metadata scanning capabilities for MatrixOne tables,
|
|
19
19
|
allowing users to analyze table statistics, column information, and data distribution.
|
20
20
|
"""
|
21
21
|
|
22
|
-
from typing import
|
22
|
+
from typing import Any, Dict, List, Optional, Union
|
23
|
+
|
23
24
|
from sqlalchemy.engine import Result
|
25
|
+
|
24
26
|
from .metadata import BaseMetadataManager, MetadataColumn, MetadataRow
|
25
27
|
|
26
28
|
|
@@ -395,7 +395,9 @@ class AsyncVectorManager:
|
|
395
395
|
|
396
396
|
# Convert distance type to enum
|
397
397
|
if distance_type == "l2":
|
398
|
-
distance_func = DistanceFunction.
|
398
|
+
distance_func = DistanceFunction.L2 # Use L2 for consistency with ORM l2_distance()
|
399
|
+
elif distance_type == "l2_sq":
|
400
|
+
distance_func = DistanceFunction.L2_SQ # Explicitly use squared distance if needed
|
399
401
|
elif distance_type == "cosine":
|
400
402
|
distance_func = DistanceFunction.COSINE
|
401
403
|
elif distance_type == "inner_product":
|
matrixone/client.py
CHANGED
@@ -30,7 +30,7 @@ from sqlalchemy.engine import Engine
|
|
30
30
|
|
31
31
|
from .account import AccountManager, TransactionAccountManager
|
32
32
|
from .base_client import BaseMatrixOneClient, BaseMatrixOneExecutor
|
33
|
-
from .connection_hooks import
|
33
|
+
from .connection_hooks import ConnectionAction, ConnectionHook, create_connection_hook
|
34
34
|
from .exceptions import ConnectionError, QueryError
|
35
35
|
from .logger import MatrixOneLogger, create_default_logger
|
36
36
|
from .metadata import MetadataManager, TransactionMetadataManager
|
@@ -1698,8 +1698,9 @@ class Client(BaseMatrixOneClient):
|
|
1698
1698
|
table_name = model_class.__tablename__
|
1699
1699
|
table = model_class.__table__
|
1700
1700
|
|
1701
|
+
from sqlalchemy.schema import CreateIndex, CreateTable
|
1702
|
+
|
1701
1703
|
from .sqlalchemy_ext import FulltextIndex, VectorIndex
|
1702
|
-
from sqlalchemy.schema import CreateTable, CreateIndex
|
1703
1704
|
|
1704
1705
|
try:
|
1705
1706
|
engine_context = self.get_sqlalchemy_engine().begin()
|
@@ -2612,6 +2613,99 @@ class Client(BaseMatrixOneClient):
|
|
2612
2613
|
|
2613
2614
|
return self
|
2614
2615
|
|
2616
|
+
def get_secondary_index_tables(self, table_name: str) -> List[str]:
|
2617
|
+
"""
|
2618
|
+
Get all secondary index table names for a given table.
|
2619
|
+
|
2620
|
+
Args:
|
2621
|
+
table_name: Name of the table to get secondary indexes for
|
2622
|
+
|
2623
|
+
Returns:
|
2624
|
+
List of secondary index table names
|
2625
|
+
|
2626
|
+
Examples::
|
2627
|
+
|
2628
|
+
>>> client = Client()
|
2629
|
+
>>> client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2630
|
+
>>> index_tables = client.get_secondary_index_tables('cms_all_content_chunk_info')
|
2631
|
+
>>> print(index_tables)
|
2632
|
+
['__mo_index_secondary_..._cms_id', '__mo_index_secondary_..._idx_all_content_length']
|
2633
|
+
"""
|
2634
|
+
from .index_utils import build_get_index_tables_sql
|
2635
|
+
|
2636
|
+
sql, params = build_get_index_tables_sql(table_name)
|
2637
|
+
result = self.execute(sql, params)
|
2638
|
+
return [row[0] for row in result.fetchall()]
|
2639
|
+
|
2640
|
+
def get_secondary_index_table_by_name(self, table_name: str, index_name: str) -> Optional[str]:
|
2641
|
+
"""
|
2642
|
+
Get the physical table name of a secondary index by its index name.
|
2643
|
+
|
2644
|
+
Args:
|
2645
|
+
table_name: Name of the table
|
2646
|
+
index_name: Name of the secondary index
|
2647
|
+
|
2648
|
+
Returns:
|
2649
|
+
Physical table name of the secondary index, or None if not found
|
2650
|
+
|
2651
|
+
Examples::
|
2652
|
+
|
2653
|
+
>>> client = Client()
|
2654
|
+
>>> client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2655
|
+
>>> index_table = client.get_secondary_index_table_by_name('cms_all_content_chunk_info', 'cms_id')
|
2656
|
+
>>> print(index_table)
|
2657
|
+
'__mo_index_secondary_018cfbda-bde1-7c3e-805c-3f8e71769f75_cms_id'
|
2658
|
+
"""
|
2659
|
+
from .index_utils import build_get_index_table_by_name_sql
|
2660
|
+
|
2661
|
+
sql, params = build_get_index_table_by_name_sql(table_name, index_name)
|
2662
|
+
result = self.execute(sql, params)
|
2663
|
+
row = result.fetchone()
|
2664
|
+
return row[0] if row else None
|
2665
|
+
|
2666
|
+
def verify_table_index_counts(self, table_name: str) -> int:
|
2667
|
+
"""
|
2668
|
+
Verify that the main table and all its secondary index tables have the same row count.
|
2669
|
+
|
2670
|
+
This method compares the COUNT(*) of the main table with all its secondary index tables
|
2671
|
+
in a single SQL query for consistency. If counts don't match, raises an exception.
|
2672
|
+
|
2673
|
+
Args:
|
2674
|
+
table_name: Name of the table to verify
|
2675
|
+
|
2676
|
+
Returns:
|
2677
|
+
Row count (int) if verification succeeds
|
2678
|
+
|
2679
|
+
Raises:
|
2680
|
+
ValueError: If any secondary index table has a different count than the main table,
|
2681
|
+
with details about all counts in the error message
|
2682
|
+
|
2683
|
+
Examples::
|
2684
|
+
|
2685
|
+
>>> client = Client()
|
2686
|
+
>>> client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2687
|
+
>>> count = client.verify_table_index_counts('cms_all_content_chunk_info')
|
2688
|
+
>>> print(f"✓ Verification passed, row count: {count}")
|
2689
|
+
|
2690
|
+
>>> # If verification fails:
|
2691
|
+
>>> try:
|
2692
|
+
... count = client.verify_table_index_counts('some_table')
|
2693
|
+
... except ValueError as e:
|
2694
|
+
... print(f"Verification failed: {e}")
|
2695
|
+
"""
|
2696
|
+
from .index_utils import build_verify_counts_sql, process_verify_result
|
2697
|
+
|
2698
|
+
# Get all secondary index tables
|
2699
|
+
index_tables = self.get_secondary_index_tables(table_name)
|
2700
|
+
|
2701
|
+
# Build and execute verification SQL
|
2702
|
+
sql = build_verify_counts_sql(table_name, index_tables)
|
2703
|
+
result = self.execute(sql)
|
2704
|
+
row = result.fetchone()
|
2705
|
+
|
2706
|
+
# Process result and raise exception if verification fails
|
2707
|
+
return process_verify_result(table_name, index_tables, row)
|
2708
|
+
|
2615
2709
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
2616
2710
|
self.disconnect()
|
2617
2711
|
|
@@ -3632,6 +3726,12 @@ class VectorManager:
|
|
3632
3726
|
if not success:
|
3633
3727
|
raise Exception(f"Failed to create IVFFLAT vector index {name} on table {table_name}")
|
3634
3728
|
|
3729
|
+
# Add summary log
|
3730
|
+
op_type_str = op_type.value if hasattr(op_type, 'value') else op_type
|
3731
|
+
self.client.logger.info(
|
3732
|
+
f"✓ Created IVF index '{name}' on {table_name}.{column} | " f"lists={lists} | op_type={op_type_str}"
|
3733
|
+
)
|
3734
|
+
|
3635
3735
|
return self
|
3636
3736
|
|
3637
3737
|
def create_hnsw(
|
@@ -3685,6 +3785,7 @@ class VectorManager:
|
|
3685
3785
|
if op_type is None:
|
3686
3786
|
op_type = VectorOpType.VECTOR_L2_OPS
|
3687
3787
|
|
3788
|
+
# Build index creation SQL
|
3688
3789
|
success = HnswVectorIndex.create_index(
|
3689
3790
|
engine=self.client.get_sqlalchemy_engine(),
|
3690
3791
|
table_name=table_name,
|
@@ -3699,6 +3800,13 @@ class VectorManager:
|
|
3699
3800
|
if not success:
|
3700
3801
|
raise Exception(f"Failed to create HNSW vector index {name} on table {table_name}")
|
3701
3802
|
|
3803
|
+
# Add summary log
|
3804
|
+
op_type_str = op_type.value if hasattr(op_type, 'value') else op_type
|
3805
|
+
self.client.logger.info(
|
3806
|
+
f"✓ Created HNSW index '{name}' on {table_name}.{column} | "
|
3807
|
+
f"m={m} | ef_construction={ef_construction} | ef_search={ef_search} | op_type={op_type_str}"
|
3808
|
+
)
|
3809
|
+
|
3702
3810
|
return self
|
3703
3811
|
|
3704
3812
|
def create_ivf_in_transaction(
|
@@ -3838,6 +3946,9 @@ class VectorManager:
|
|
3838
3946
|
if not success:
|
3839
3947
|
raise Exception(f"Failed to drop vector index {name} from table {table_name}")
|
3840
3948
|
|
3949
|
+
# Add summary log
|
3950
|
+
self.client.logger.info(f"✓ Dropped vector index '{name}' from {table_name}")
|
3951
|
+
|
3841
3952
|
return self
|
3842
3953
|
|
3843
3954
|
def enable_ivf(self, probe_limit: int = 1) -> "VectorManager":
|
@@ -4051,7 +4162,9 @@ class VectorManager:
|
|
4051
4162
|
|
4052
4163
|
# Convert distance type to enum
|
4053
4164
|
if distance_type == "l2":
|
4054
|
-
distance_func = DistanceFunction.
|
4165
|
+
distance_func = DistanceFunction.L2 # Use L2 for consistency with ORM l2_distance()
|
4166
|
+
elif distance_type == "l2_sq":
|
4167
|
+
distance_func = DistanceFunction.L2_SQ # Explicitly use squared distance if needed
|
4055
4168
|
elif distance_type == "cosine":
|
4056
4169
|
distance_func = DistanceFunction.COSINE
|
4057
4170
|
elif distance_type == "inner_product":
|
matrixone/connection_hooks.py
CHANGED
matrixone/exceptions.py
CHANGED
@@ -20,70 +20,46 @@ MatrixOne SDK Exceptions
|
|
20
20
|
class MatrixOneError(Exception):
|
21
21
|
"""Base exception for all MatrixOne SDK errors"""
|
22
22
|
|
23
|
-
pass
|
24
|
-
|
25
23
|
|
26
24
|
class ConnectionError(MatrixOneError):
|
27
25
|
"""Raised when connection to MatrixOne fails"""
|
28
26
|
|
29
|
-
pass
|
30
|
-
|
31
27
|
|
32
28
|
class QueryError(MatrixOneError):
|
33
29
|
"""Raised when SQL query execution fails"""
|
34
30
|
|
35
|
-
pass
|
36
|
-
|
37
31
|
|
38
32
|
class ConfigurationError(MatrixOneError):
|
39
33
|
"""Raised when configuration is invalid"""
|
40
34
|
|
41
|
-
pass
|
42
|
-
|
43
35
|
|
44
36
|
class SnapshotError(MatrixOneError):
|
45
37
|
"""Raised when snapshot operations fail"""
|
46
38
|
|
47
|
-
pass
|
48
|
-
|
49
39
|
|
50
40
|
class CloneError(MatrixOneError):
|
51
41
|
"""Raised when clone operations fail"""
|
52
42
|
|
53
|
-
pass
|
54
|
-
|
55
43
|
|
56
44
|
class MoCtlError(MatrixOneError):
|
57
45
|
"""Raised when mo_ctl operations fail"""
|
58
46
|
|
59
|
-
pass
|
60
|
-
|
61
47
|
|
62
48
|
class RestoreError(MatrixOneError):
|
63
49
|
"""Raised when restore operations fail"""
|
64
50
|
|
65
|
-
pass
|
66
|
-
|
67
51
|
|
68
52
|
class PitrError(MatrixOneError):
|
69
53
|
"""Raised when PITR operations fail"""
|
70
54
|
|
71
|
-
pass
|
72
|
-
|
73
55
|
|
74
56
|
class PubSubError(MatrixOneError):
|
75
57
|
"""Raised when publish-subscribe operations fail"""
|
76
58
|
|
77
|
-
pass
|
78
|
-
|
79
59
|
|
80
60
|
class AccountError(MatrixOneError):
|
81
61
|
"""Raised when account management operations fail"""
|
82
62
|
|
83
|
-
pass
|
84
|
-
|
85
63
|
|
86
64
|
class VersionError(MatrixOneError):
|
87
65
|
"""Raised when version compatibility check fails"""
|
88
|
-
|
89
|
-
pass
|
matrixone/index_utils.py
ADDED
@@ -0,0 +1,121 @@
|
|
1
|
+
# Copyright 2021 - 2022 Matrix Origin
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
"""
|
16
|
+
Index utilities - Shared logic for secondary index operations
|
17
|
+
"""
|
18
|
+
|
19
|
+
from typing import List, Tuple
|
20
|
+
|
21
|
+
|
22
|
+
def build_get_index_tables_sql(table_name: str) -> Tuple[str, Tuple]:
|
23
|
+
"""
|
24
|
+
Build SQL to get all secondary index table names for a given table.
|
25
|
+
|
26
|
+
Args:
|
27
|
+
table_name: Name of the table
|
28
|
+
|
29
|
+
Returns:
|
30
|
+
Tuple of (sql, params)
|
31
|
+
"""
|
32
|
+
sql = """
|
33
|
+
SELECT DISTINCT index_table_name
|
34
|
+
FROM mo_catalog.mo_indexes
|
35
|
+
JOIN mo_catalog.mo_tables ON mo_indexes.table_id = mo_tables.rel_id
|
36
|
+
WHERE relname = ? AND type = 'MULTIPLE'
|
37
|
+
"""
|
38
|
+
return sql, (table_name,)
|
39
|
+
|
40
|
+
|
41
|
+
def build_get_index_table_by_name_sql(table_name: str, index_name: str) -> Tuple[str, Tuple]:
|
42
|
+
"""
|
43
|
+
Build SQL to get the physical table name of a secondary index by its index name.
|
44
|
+
|
45
|
+
Args:
|
46
|
+
table_name: Name of the table
|
47
|
+
index_name: Name of the secondary index
|
48
|
+
|
49
|
+
Returns:
|
50
|
+
Tuple of (sql, params)
|
51
|
+
"""
|
52
|
+
sql = """
|
53
|
+
SELECT DISTINCT index_table_name
|
54
|
+
FROM mo_catalog.mo_indexes
|
55
|
+
JOIN mo_catalog.mo_tables ON mo_indexes.table_id = mo_tables.rel_id
|
56
|
+
WHERE relname = ? AND name = ?
|
57
|
+
"""
|
58
|
+
return sql, (table_name, index_name)
|
59
|
+
|
60
|
+
|
61
|
+
def build_verify_counts_sql(table_name: str, index_tables: List[str]) -> str:
|
62
|
+
"""
|
63
|
+
Build SQL to verify counts of main table and all index tables in a single query.
|
64
|
+
|
65
|
+
Args:
|
66
|
+
table_name: Name of the main table
|
67
|
+
index_tables: List of index table names
|
68
|
+
|
69
|
+
Returns:
|
70
|
+
SQL string
|
71
|
+
"""
|
72
|
+
if not index_tables:
|
73
|
+
return f"SELECT COUNT(*) FROM `{table_name}`"
|
74
|
+
|
75
|
+
select_parts = [f"(SELECT COUNT(*) FROM `{table_name}`) as main_count"]
|
76
|
+
for idx, index_table in enumerate(index_tables):
|
77
|
+
select_parts.append(f"(SELECT COUNT(*) FROM `{index_table}`) as idx{idx}_count")
|
78
|
+
|
79
|
+
return "SELECT " + ", ".join(select_parts)
|
80
|
+
|
81
|
+
|
82
|
+
def process_verify_result(table_name: str, index_tables: List[str], row: Tuple) -> int:
|
83
|
+
"""
|
84
|
+
Process the verification result and raise exception if counts don't match.
|
85
|
+
|
86
|
+
Args:
|
87
|
+
table_name: Name of the main table
|
88
|
+
index_tables: List of index table names
|
89
|
+
row: Result row from the verification SQL
|
90
|
+
|
91
|
+
Returns:
|
92
|
+
Row count if verification succeeds
|
93
|
+
|
94
|
+
Raises:
|
95
|
+
ValueError: If any index table has a different count
|
96
|
+
"""
|
97
|
+
main_count = row[0]
|
98
|
+
|
99
|
+
if not index_tables:
|
100
|
+
return main_count
|
101
|
+
|
102
|
+
index_counts = {}
|
103
|
+
mismatch = []
|
104
|
+
|
105
|
+
for idx, index_table in enumerate(index_tables):
|
106
|
+
index_count = row[idx + 1]
|
107
|
+
index_counts[index_table] = index_count
|
108
|
+
if index_count != main_count:
|
109
|
+
mismatch.append(index_table)
|
110
|
+
|
111
|
+
# If there's a mismatch, raise an exception with details
|
112
|
+
if mismatch:
|
113
|
+
error_details = [f"Main table '{table_name}': {main_count} rows"]
|
114
|
+
for index_table, count in index_counts.items():
|
115
|
+
status = "✗ MISMATCH" if index_table in mismatch else "✓"
|
116
|
+
error_details.append(f"{status} Index '{index_table}': {count} rows")
|
117
|
+
|
118
|
+
error_msg = "Index count verification failed!\n" + "\n".join(error_details)
|
119
|
+
raise ValueError(error_msg)
|
120
|
+
|
121
|
+
return main_count
|
matrixone/metadata.py
CHANGED
@@ -19,10 +19,11 @@ This module provides metadata scanning capabilities for MatrixOne tables,
|
|
19
19
|
allowing users to analyze table statistics, column information, and data distribution.
|
20
20
|
"""
|
21
21
|
|
22
|
-
from typing import Optional, List, Dict, Any, Union
|
23
|
-
from sqlalchemy.engine import Result
|
24
22
|
from dataclasses import dataclass
|
25
23
|
from enum import Enum
|
24
|
+
from typing import Any, Dict, List, Optional, Union
|
25
|
+
|
26
|
+
from sqlalchemy.engine import Result
|
26
27
|
|
27
28
|
|
28
29
|
class MetadataColumn(Enum):
|
matrixone/moctl.py
CHANGED
@@ -247,7 +247,21 @@ class IVFVectorIndex(Index):
|
|
247
247
|
# Enable IVF indexing
|
248
248
|
_exec_sql_safe(conn, "SET experimental_ivf_index = 1")
|
249
249
|
_exec_sql_safe(conn, "SET probe_limit = 1")
|
250
|
+
|
251
|
+
# Execute CREATE INDEX
|
250
252
|
_exec_sql_safe(conn, sql)
|
253
|
+
|
254
|
+
# Try to log using the connection's info attribute if available
|
255
|
+
try:
|
256
|
+
from ..client import logger
|
257
|
+
|
258
|
+
logger.info(
|
259
|
+
f"✓ | CREATE INDEX {name} USING ivfflat ON {table_name}({column}) "
|
260
|
+
f"LISTS {lists} op_type '{op_type}'"
|
261
|
+
)
|
262
|
+
except Exception:
|
263
|
+
pass # Silently skip logging if not available (for tests)
|
264
|
+
|
251
265
|
return True
|
252
266
|
except Exception as e:
|
253
267
|
print(f"Failed to create IVFFLAT vector index: {e}")
|
@@ -648,7 +662,21 @@ class HnswVectorIndex(Index):
|
|
648
662
|
with engine.begin() as conn:
|
649
663
|
# Enable HNSW indexing
|
650
664
|
_exec_sql_safe(conn, "SET experimental_hnsw_index = 1")
|
665
|
+
|
666
|
+
# Execute CREATE INDEX
|
651
667
|
_exec_sql_safe(conn, sql)
|
668
|
+
|
669
|
+
# Try to log using the connection's info attribute if available
|
670
|
+
try:
|
671
|
+
from ..client import logger
|
672
|
+
|
673
|
+
logger.info(
|
674
|
+
f"✓ | CREATE INDEX {name} USING hnsw ON {table_name}({column}) "
|
675
|
+
f"M {m} EF_CONSTRUCTION {ef_construction} EF_SEARCH {ef_search} op_type '{op_type}'"
|
676
|
+
)
|
677
|
+
except Exception:
|
678
|
+
pass # Silently skip logging if not available (for tests)
|
679
|
+
|
652
680
|
return True
|
653
681
|
except Exception as e:
|
654
682
|
print(f"Failed to create HNSW vector index: {e}")
|
matrixone/version.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: matrixone-python-sdk
|
3
|
-
Version: 0.1.
|
3
|
+
Version: 0.1.3
|
4
4
|
Summary: A comprehensive Python SDK for MatrixOne database operations with vector search, fulltext search, and advanced features
|
5
5
|
Home-page: https://github.com/matrixorigin/matrixone
|
6
6
|
Author: MatrixOne Team
|
@@ -1,17 +1,18 @@
|
|
1
1
|
matrixone/__init__.py,sha256=7bePzwzerZTiZIlbUT9zI_u4fp49DvQNcIqyiIvEVVs,3966
|
2
2
|
matrixone/account.py,sha256=0r9xLNTiUfXa3xWZQUhEJ_uUcNp_gja-YulOR5iYDU4,24712
|
3
|
-
matrixone/async_client.py,sha256=
|
4
|
-
matrixone/async_metadata_manager.py,sha256=
|
3
|
+
matrixone/async_client.py,sha256=KKvWIzcV2PtwaFT31jgawRLt1CJhI_rePQdAs8xLzEc,155275
|
4
|
+
matrixone/async_metadata_manager.py,sha256=W3TJBCy56diDPBzimpBImFrmf9DBc7iiS5CdBHLAh_Q,10232
|
5
5
|
matrixone/async_orm.py,sha256=O4Rf85MnZg0_fuLvbHGfOPrHI2OLznYwLk9eam5J5ik,4864
|
6
|
-
matrixone/async_vector_index_manager.py,sha256=
|
6
|
+
matrixone/async_vector_index_manager.py,sha256=fVnS00zgbUXw4YmZR-wRCzm541n30TLlmXXmXV34ELU,23134
|
7
7
|
matrixone/base_client.py,sha256=PUhq0c7Si9OIk83B4ZZ73ozwaHDRZ1zi0Q94YixseqY,6952
|
8
|
-
matrixone/client.py,sha256=
|
8
|
+
matrixone/client.py,sha256=bxdKUVTzo18yMDFmgcUdSIfoKozC-pfUOlKuWCwwGwI,187341
|
9
9
|
matrixone/config.py,sha256=jrm1rLBhLWrX3dVdPx67sFaZ_9M8Vj4hImVrAmFaYbw,13917
|
10
|
-
matrixone/connection_hooks.py,sha256=
|
11
|
-
matrixone/exceptions.py,sha256=
|
10
|
+
matrixone/connection_hooks.py,sha256=FrCGWipg5KUrdBt-cJMORaOUfoIqgSZNZCtDnn9la50,13531
|
11
|
+
matrixone/exceptions.py,sha256=VsGgESLl4vm5-9QbigyE5RTwFujyoYXhQuGhGKoujFw,1667
|
12
|
+
matrixone/index_utils.py,sha256=ND_Md-mj1Yh8Qq4RGW5tYxwqkgwItkWzWDLL_SQ9AMI,3759
|
12
13
|
matrixone/logger.py,sha256=aZmetje_AqtyxFxsqZvp-3R68n8I_ldplRGfV2MHg48,26105
|
13
|
-
matrixone/metadata.py,sha256=
|
14
|
-
matrixone/moctl.py,sha256=
|
14
|
+
matrixone/metadata.py,sha256=Yf0eJywe3BUqg6aEywSWh9neRcVsrgf6g-uaZFvqLOA,31707
|
15
|
+
matrixone/moctl.py,sha256=b_qQPSK52JiyyUH4PYzAuhJPk8dAD6LRfT_sFGsf23o,6704
|
15
16
|
matrixone/orm.py,sha256=QWNdfJ067KcgWU-eB__vXwJuANBaDUeVVG7thFH5kiU,90473
|
16
17
|
matrixone/pitr.py,sha256=NPsexPsJXIAiJTH3oMyFOwiXWjvfohPzWQyFHhQKh5k,21632
|
17
18
|
matrixone/pubsub.py,sha256=ckaG6Z3IuEtcdwp11qvB88PrCDHYymocSOJ1v4TftzY,26893
|
@@ -19,8 +20,8 @@ matrixone/restore.py,sha256=xAbsRUL-2hCk6YpuouVLbiwc3GwUNzCoIVEFtcmghkg,15791
|
|
19
20
|
matrixone/search_vector_index.py,sha256=NjQpluaPcja66herm-OYq3nNeNAqY_-FGGL4SOBOWNA,46075
|
20
21
|
matrixone/snapshot.py,sha256=cOatk26Z6Jt6aAdasOMejxxiv73S7Z2o2if_va20Ihg,18103
|
21
22
|
matrixone/sql_builder.py,sha256=ZR9TpRkV4cN_RiDJQXMLwSzodpESyxAqV-TTIr--ZbY,29009
|
22
|
-
matrixone/version.py,sha256=
|
23
|
-
matrixone/sqlalchemy_ext/__init__.py,sha256=
|
23
|
+
matrixone/version.py,sha256=7wpMFVqDjGE16bFtnvFwbcnAI3mzz2zMZPpc9v7G9-c,19913
|
24
|
+
matrixone/sqlalchemy_ext/__init__.py,sha256=YrcXthWkpN-mkRq6293t6Su5D39MdsEKYu_g1lz4qgI,4029
|
24
25
|
matrixone/sqlalchemy_ext/adapters.py,sha256=-PoQtxN8uveto4Me2BTyTq1FoeSj8qo3TkUKAhoUd4w,5286
|
25
26
|
matrixone/sqlalchemy_ext/dialect.py,sha256=wrAQDjDDpCz6HT2Atrz7QXAzbyxaNTbRXcauh2REa1A,21100
|
26
27
|
matrixone/sqlalchemy_ext/fulltext_index.py,sha256=OceS34yg1kG-1uopG1lDf7GbVyRiUnxzKHDUHYLNLuA,27614
|
@@ -28,9 +29,9 @@ matrixone/sqlalchemy_ext/fulltext_search.py,sha256=1pCMxw9GyTjRhjsJ9KbKq_YsMEPuC
|
|
28
29
|
matrixone/sqlalchemy_ext/hnsw_config.py,sha256=5NYZXK1jsIKlSTrtqpv8n2V_yhbMsmOhItFrbmS5Vs4,5498
|
29
30
|
matrixone/sqlalchemy_ext/ivf_config.py,sha256=-esAijWMfyF1GUJivpZh-F4lvyKRmY8LYIzVgmMVtVo,6367
|
30
31
|
matrixone/sqlalchemy_ext/table_builder.py,sha256=JVritBPnCXZt0eJUivDrmGdlpaYO-uL7i2n24HiarEE,12870
|
31
|
-
matrixone/sqlalchemy_ext/vector_index.py,sha256=
|
32
|
+
matrixone/sqlalchemy_ext/vector_index.py,sha256=dasW3kT4f69io9y6DLQMarvf1FTbzh0UN-HALs0kBVs,55907
|
32
33
|
matrixone/sqlalchemy_ext/vector_type.py,sha256=HaOJ9dRdW_yrecD9qGUucW9bMfM3zCxbDC-0Ca32Kmk,30669
|
33
|
-
matrixone_python_sdk-0.1.
|
34
|
+
matrixone_python_sdk-0.1.3.dist-info/licenses/LICENSE,sha256=-PpUMwDyMyFlH9H7cnzkTh0Uo42tRvz43k7hnxe7G_I,11252
|
34
35
|
tests/__init__.py,sha256=odB22tIaJIHSwRhumhDlQYD6Fug_C0opWa07dSKkeQs,694
|
35
36
|
tests/offline/__init__.py,sha256=M13mz7gtVDS0_dJUW1EFyyiAGhEj282k3ia7eWA3dPs,703
|
36
37
|
tests/offline/conftest.py,sha256=Mz_NT6GBOxqSZsSCR2SXe1pkSpLGKT2-ssWNHhh9xOg,2494
|
@@ -77,6 +78,7 @@ tests/online/test_account_management.py,sha256=mWIF3kPaegcb-JEhHBM9LrSae2xZ3ppjk
|
|
77
78
|
tests/online/test_advanced_features.py,sha256=f0oOe1bB6dT4bXZqx4LzrWd4ljwwcr9iIZLDHnTdPhU,13641
|
78
79
|
tests/online/test_async_client_interfaces.py,sha256=ZT2kWm-ZT_O-_ZkGE7mVLE5c52_eRsbCC_wEksrUis0,12295
|
79
80
|
tests/online/test_async_client_online.py,sha256=LZp0QXk4l4FfuQOHNNg__degRTwh9K61StFtEmeUmZ8,10906
|
81
|
+
tests/online/test_async_index_verification_online.py,sha256=zevl8BKAIGRyJ0ndnUmW_oQpC_rc2_W0ATPb152OMag,10162
|
80
82
|
tests/online/test_async_model_insert_online.py,sha256=YPlcNjuaeimvG355JjjgyZUPfa9J0HtC5FVJr9uIp_k,10857
|
81
83
|
tests/online/test_async_orm_online.py,sha256=8wjQb9oXMyNh4HQ1YkwiFYfHo73_YVPicOR_aiXrbEg,11039
|
82
84
|
tests/online/test_async_simple_query_online.py,sha256=nBMsCv-GaEiSRxh1juLNCiXNNsfUwWRApQvStzLeGr4,31021
|
@@ -91,6 +93,7 @@ tests/online/test_filter_logical_in_online.py,sha256=-OrmHSMyk6nYap5OCICaImt9AQ5
|
|
91
93
|
tests/online/test_fulltext_comprehensive.py,sha256=lJ_XMkZnHe1HPsbXS9O5F9tBBsPKTBNUnx78KgKTv3w,74063
|
92
94
|
tests/online/test_fulltext_label_online.py,sha256=1ExVrGIOA7dfG6UiB7Ul6xeLrLY9u6oAjjT4hYNtr8E,16534
|
93
95
|
tests/online/test_fulltext_search_online.py,sha256=711WdigiDR71nwtqLlVl39_lNocbWlNnJR5W6hBhl50,33660
|
96
|
+
tests/online/test_index_verification_online.py,sha256=Uz8Gjvjm2SdNhpXeun_Y8KOQ1VOs601WLMmxWTMzNa0,7510
|
94
97
|
tests/online/test_ivf_stats_online.py,sha256=Fdi_-bleJVXtWzHNfeA6eCgVdlvSvpEfop7GGUZTQ9E,19845
|
95
98
|
tests/online/test_logger_integration.py,sha256=iv0TrKZGRg-oxjbp55fYixbfpdqLFWQDR8mXeaSKArg,11824
|
96
99
|
tests/online/test_matrixone_query_orm.py,sha256=WYH6HxcfnNqtQ05kKHRcmwmWHXtDz8Ts0pUwuSdgo_U,20130
|
@@ -115,8 +118,8 @@ tests/online/test_transaction_query_methods.py,sha256=VaPvHOYskwJVyM7lvVvBJ6Hqhd
|
|
115
118
|
tests/online/test_unified_filter_online.py,sha256=86JwEKUaBLNuLGJlR4E0zjzdWL5xVFyYLvyxbzkdEVI,21214
|
116
119
|
tests/online/test_vector_comprehensive.py,sha256=vmnwwVXLA7lUI_zSK3fJcf1HKx2AvrVDcUT0_d-gQwg,26980
|
117
120
|
tests/online/test_version_management.py,sha256=BU8Vc1fDKNCwhRliZi6XmEnd0HYdHuki9Xxi09vnriA,11416
|
118
|
-
matrixone_python_sdk-0.1.
|
119
|
-
matrixone_python_sdk-0.1.
|
120
|
-
matrixone_python_sdk-0.1.
|
121
|
-
matrixone_python_sdk-0.1.
|
122
|
-
matrixone_python_sdk-0.1.
|
121
|
+
matrixone_python_sdk-0.1.3.dist-info/METADATA,sha256=lNJxaTy1NjfSwoOdr8YOVZ-E9ISSjL2U-t9ZDtZsMnI,20815
|
122
|
+
matrixone_python_sdk-0.1.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
123
|
+
matrixone_python_sdk-0.1.3.dist-info/entry_points.txt,sha256=4wUGPC_7_f5ZDR33JRo1LZmmTuyfkYAv41_5H5Qy-Ik,138
|
124
|
+
matrixone_python_sdk-0.1.3.dist-info/top_level.txt,sha256=LQZabpBx_dtQk8JbKeH3MbjmC8HYDLE8UQeEf6NfQJA,16
|
125
|
+
matrixone_python_sdk-0.1.3.dist-info/RECORD,,
|
@@ -0,0 +1,259 @@
|
|
1
|
+
# Copyright 2021 - 2022 Matrix Origin
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
"""
|
16
|
+
Online tests for async secondary index verification functionality
|
17
|
+
"""
|
18
|
+
|
19
|
+
import pytest
|
20
|
+
import pytest_asyncio
|
21
|
+
import os
|
22
|
+
import sys
|
23
|
+
|
24
|
+
# Add the matrixone package to the path
|
25
|
+
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
|
26
|
+
|
27
|
+
from matrixone import AsyncClient
|
28
|
+
from matrixone.orm import declarative_base
|
29
|
+
from sqlalchemy import Column, String, Integer, Index
|
30
|
+
from .test_config import online_config
|
31
|
+
|
32
|
+
|
33
|
+
class TestAsyncIndexVerificationOnline:
|
34
|
+
"""Online tests for async secondary index verification"""
|
35
|
+
|
36
|
+
@pytest_asyncio.fixture(scope="function")
|
37
|
+
async def async_client_with_test_table(self):
|
38
|
+
"""Create AsyncClient with test table"""
|
39
|
+
host, port, user, password, database = online_config.get_connection_params()
|
40
|
+
client = AsyncClient()
|
41
|
+
await client.connect(host=host, port=port, user=user, password=password, database=database)
|
42
|
+
|
43
|
+
test_table = "test_async_index_table"
|
44
|
+
|
45
|
+
try:
|
46
|
+
# Define ORM model with secondary indexes
|
47
|
+
Base = declarative_base()
|
48
|
+
|
49
|
+
class TestTable(Base):
|
50
|
+
__tablename__ = test_table
|
51
|
+
|
52
|
+
id = Column(Integer, primary_key=True)
|
53
|
+
name = Column(String(100))
|
54
|
+
category = Column(String(50))
|
55
|
+
value = Column(Integer)
|
56
|
+
|
57
|
+
# Define secondary indexes
|
58
|
+
__table_args__ = (
|
59
|
+
Index('idx_async_name', 'name'),
|
60
|
+
Index('idx_async_category', 'category'),
|
61
|
+
Index('idx_async_value', 'value'),
|
62
|
+
)
|
63
|
+
|
64
|
+
# Drop table if exists and create new one using SDK API
|
65
|
+
try:
|
66
|
+
await client.drop_table(TestTable)
|
67
|
+
except:
|
68
|
+
pass
|
69
|
+
|
70
|
+
# Create table with indexes using SDK API
|
71
|
+
await client.create_table(TestTable)
|
72
|
+
|
73
|
+
# Insert test data using SDK API
|
74
|
+
test_data = [
|
75
|
+
{'id': i, 'name': f'async_name_{i}', 'category': f'cat_{i % 5}', 'value': i * 10} for i in range(1, 101)
|
76
|
+
]
|
77
|
+
await client.batch_insert(TestTable, test_data)
|
78
|
+
|
79
|
+
yield client, test_table, TestTable
|
80
|
+
|
81
|
+
finally:
|
82
|
+
# Clean up using SDK API
|
83
|
+
try:
|
84
|
+
await client.drop_table(TestTable)
|
85
|
+
await client.disconnect()
|
86
|
+
except Exception as e:
|
87
|
+
print(f"Cleanup failed: {e}")
|
88
|
+
|
89
|
+
@pytest.mark.asyncio
|
90
|
+
async def test_async_get_secondary_index_tables(self, async_client_with_test_table):
|
91
|
+
"""Test async getting all secondary index table names"""
|
92
|
+
client, test_table, TestTable = async_client_with_test_table
|
93
|
+
|
94
|
+
index_tables = await client.get_secondary_index_tables(test_table)
|
95
|
+
|
96
|
+
# Should have 3 secondary indexes
|
97
|
+
assert len(index_tables) == 3, f"Expected 3 indexes, got {len(index_tables)}"
|
98
|
+
|
99
|
+
# All index tables should start with __mo_index_secondary_
|
100
|
+
for index_table in index_tables:
|
101
|
+
assert index_table.startswith(
|
102
|
+
'__mo_index_secondary_'
|
103
|
+
), f"Index table name should start with '__mo_index_secondary_', got {index_table}"
|
104
|
+
|
105
|
+
|
106
|
+
class TestAsyncIndexVerificationOnline:
|
107
|
+
"""Online tests for async secondary index verification"""
|
108
|
+
|
109
|
+
@pytest_asyncio.fixture(scope="function")
|
110
|
+
async def async_client_with_test_table(self):
|
111
|
+
"""Create AsyncClient with test table"""
|
112
|
+
host, port, user, password, database = online_config.get_connection_params()
|
113
|
+
client = AsyncClient()
|
114
|
+
await client.connect(host=host, port=port, user=user, password=password, database=database)
|
115
|
+
|
116
|
+
test_table = "test_async_index_table"
|
117
|
+
|
118
|
+
try:
|
119
|
+
# Define ORM model with secondary indexes
|
120
|
+
Base = declarative_base()
|
121
|
+
|
122
|
+
class TestTable(Base):
|
123
|
+
__tablename__ = test_table
|
124
|
+
|
125
|
+
id = Column(Integer, primary_key=True)
|
126
|
+
name = Column(String(100))
|
127
|
+
category = Column(String(50))
|
128
|
+
value = Column(Integer)
|
129
|
+
|
130
|
+
# Define secondary indexes
|
131
|
+
__table_args__ = (
|
132
|
+
Index('idx_async_name', 'name'),
|
133
|
+
Index('idx_async_category', 'category'),
|
134
|
+
Index('idx_async_value', 'value'),
|
135
|
+
)
|
136
|
+
|
137
|
+
# Drop table if exists and create new one using SDK API
|
138
|
+
try:
|
139
|
+
await client.drop_table(TestTable)
|
140
|
+
except:
|
141
|
+
pass
|
142
|
+
|
143
|
+
# Create table with indexes using SDK API
|
144
|
+
await client.create_table(TestTable)
|
145
|
+
|
146
|
+
# Insert test data using SDK API
|
147
|
+
test_data = [
|
148
|
+
{'id': i, 'name': f'async_name_{i}', 'category': f'cat_{i % 5}', 'value': i * 10} for i in range(1, 101)
|
149
|
+
]
|
150
|
+
await client.batch_insert(TestTable, test_data)
|
151
|
+
|
152
|
+
yield client, test_table, TestTable
|
153
|
+
|
154
|
+
finally:
|
155
|
+
# Clean up using SDK API
|
156
|
+
try:
|
157
|
+
await client.drop_table(TestTable)
|
158
|
+
await client.disconnect()
|
159
|
+
except Exception as e:
|
160
|
+
print(f"Cleanup failed: {e}")
|
161
|
+
|
162
|
+
@pytest.mark.asyncio
|
163
|
+
async def test_async_get_secondary_index_table_by_name(self, async_client_with_test_table):
|
164
|
+
"""Test async getting physical table name of a secondary index by its name"""
|
165
|
+
client, test_table, TestTable = async_client_with_test_table
|
166
|
+
|
167
|
+
# Test getting idx_async_name
|
168
|
+
idx_name_table = await client.get_secondary_index_table_by_name(test_table, 'idx_async_name')
|
169
|
+
assert idx_name_table is not None, "idx_async_name should exist"
|
170
|
+
assert idx_name_table.startswith('__mo_index_secondary_')
|
171
|
+
|
172
|
+
# Test getting idx_async_category
|
173
|
+
idx_category_table = await client.get_secondary_index_table_by_name(test_table, 'idx_async_category')
|
174
|
+
assert idx_category_table is not None, "idx_async_category should exist"
|
175
|
+
assert idx_category_table.startswith('__mo_index_secondary_')
|
176
|
+
|
177
|
+
# Test getting idx_async_value
|
178
|
+
idx_value_table = await client.get_secondary_index_table_by_name(test_table, 'idx_async_value')
|
179
|
+
assert idx_value_table is not None, "idx_async_value should exist"
|
180
|
+
assert idx_value_table.startswith('__mo_index_secondary_')
|
181
|
+
|
182
|
+
# Test non-existent index
|
183
|
+
non_existent = await client.get_secondary_index_table_by_name(test_table, 'idx_non_existent')
|
184
|
+
assert non_existent is None, "Non-existent index should return None"
|
185
|
+
|
186
|
+
@pytest.mark.asyncio
|
187
|
+
async def test_async_verify_table_index_counts(self, async_client_with_test_table):
|
188
|
+
"""Test async verify_table_index_counts when counts match"""
|
189
|
+
client, test_table, TestTable = async_client_with_test_table
|
190
|
+
|
191
|
+
# All indexes should have the same count as the main table
|
192
|
+
count = await client.verify_table_index_counts(test_table)
|
193
|
+
|
194
|
+
# Should return 100 (number of inserted rows)
|
195
|
+
assert count == 100, f"Expected count 100, got {count}"
|
196
|
+
|
197
|
+
# Verify count matches query result using SDK API
|
198
|
+
actual_count = await client.query(TestTable).count()
|
199
|
+
assert count == actual_count, "Verify result should match actual count"
|
200
|
+
|
201
|
+
@pytest.mark.asyncio
|
202
|
+
async def test_async_verify_table_without_indexes(self, async_client_with_test_table):
|
203
|
+
"""Test async verify_table_index_counts on table without secondary indexes"""
|
204
|
+
client, test_table, TestTable = async_client_with_test_table
|
205
|
+
|
206
|
+
# Define a simple model without secondary indexes
|
207
|
+
Base = declarative_base()
|
208
|
+
|
209
|
+
class SimpleTable(Base):
|
210
|
+
__tablename__ = "test_async_simple_table"
|
211
|
+
id = Column(Integer, primary_key=True)
|
212
|
+
value = Column(Integer)
|
213
|
+
|
214
|
+
# Create table using SDK API
|
215
|
+
try:
|
216
|
+
await client.drop_table(SimpleTable)
|
217
|
+
except:
|
218
|
+
pass
|
219
|
+
|
220
|
+
await client.create_table(SimpleTable)
|
221
|
+
|
222
|
+
# Insert data using SDK API
|
223
|
+
test_data = [{'id': 1, 'value': 100}, {'id': 2, 'value': 200}]
|
224
|
+
await client.batch_insert(SimpleTable, test_data)
|
225
|
+
|
226
|
+
# Verification should succeed and return count
|
227
|
+
count = await client.verify_table_index_counts("test_async_simple_table")
|
228
|
+
assert count == 2, f"Expected count 2, got {count}"
|
229
|
+
|
230
|
+
# Cleanup using SDK API
|
231
|
+
await client.drop_table(SimpleTable)
|
232
|
+
|
233
|
+
@pytest.mark.asyncio
|
234
|
+
async def test_async_index_table_mapping(self, async_client_with_test_table):
|
235
|
+
"""Test that all indexes can be retrieved both ways (async)"""
|
236
|
+
client, test_table, TestTable = async_client_with_test_table
|
237
|
+
|
238
|
+
# Get all index tables
|
239
|
+
all_index_tables = await client.get_secondary_index_tables(test_table)
|
240
|
+
|
241
|
+
# Get each index by name
|
242
|
+
index_names = ['idx_async_name', 'idx_async_category', 'idx_async_value']
|
243
|
+
retrieved_tables = []
|
244
|
+
|
245
|
+
for index_name in index_names:
|
246
|
+
table = await client.get_secondary_index_table_by_name(test_table, index_name)
|
247
|
+
assert table is not None, f"Index {index_name} should exist"
|
248
|
+
retrieved_tables.append(table)
|
249
|
+
|
250
|
+
# All retrieved tables should be in the all_index_tables list
|
251
|
+
for table in retrieved_tables:
|
252
|
+
assert table in all_index_tables, f"Retrieved table {table} should be in all index tables"
|
253
|
+
|
254
|
+
# Both lists should have the same length
|
255
|
+
assert len(retrieved_tables) == len(all_index_tables), "Number of retrieved tables should match total index tables"
|
256
|
+
|
257
|
+
|
258
|
+
if __name__ == '__main__':
|
259
|
+
pytest.main([__file__, '-v'])
|
@@ -0,0 +1,194 @@
|
|
1
|
+
# Copyright 2021 - 2022 Matrix Origin
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
"""
|
16
|
+
Online tests for secondary index verification functionality
|
17
|
+
"""
|
18
|
+
|
19
|
+
import unittest
|
20
|
+
import os
|
21
|
+
import sys
|
22
|
+
|
23
|
+
# Add the matrixone package to the path
|
24
|
+
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
|
25
|
+
|
26
|
+
from matrixone import Client
|
27
|
+
from matrixone.orm import declarative_base
|
28
|
+
from sqlalchemy import Column, String, Integer, Index
|
29
|
+
|
30
|
+
|
31
|
+
class TestIndexVerificationOnline(unittest.TestCase):
|
32
|
+
"""Online tests for secondary index verification"""
|
33
|
+
|
34
|
+
@classmethod
|
35
|
+
def setUpClass(cls):
|
36
|
+
"""Set up test database connection"""
|
37
|
+
cls.client = Client(
|
38
|
+
host=os.getenv('MATRIXONE_HOST', '127.0.0.1'),
|
39
|
+
port=int(os.getenv('MATRIXONE_PORT', '6001')),
|
40
|
+
user=os.getenv('MATRIXONE_USER', 'root'),
|
41
|
+
password=os.getenv('MATRIXONE_PASSWORD', '111'),
|
42
|
+
database=os.getenv('MATRIXONE_DATABASE', 'test'),
|
43
|
+
)
|
44
|
+
|
45
|
+
cls.test_table = "test_index_table"
|
46
|
+
|
47
|
+
try:
|
48
|
+
# Define ORM model with secondary indexes
|
49
|
+
Base = declarative_base()
|
50
|
+
|
51
|
+
class TestTable(Base):
|
52
|
+
__tablename__ = cls.test_table
|
53
|
+
|
54
|
+
id = Column(Integer, primary_key=True)
|
55
|
+
name = Column(String(100))
|
56
|
+
category = Column(String(50))
|
57
|
+
value = Column(Integer)
|
58
|
+
|
59
|
+
# Define secondary indexes
|
60
|
+
__table_args__ = (
|
61
|
+
Index('idx_name', 'name'),
|
62
|
+
Index('idx_category', 'category'),
|
63
|
+
Index('idx_value', 'value'),
|
64
|
+
)
|
65
|
+
|
66
|
+
cls.TestTable = TestTable
|
67
|
+
|
68
|
+
# Drop table if exists and create new one
|
69
|
+
try:
|
70
|
+
cls.client.drop_table(TestTable)
|
71
|
+
except:
|
72
|
+
pass
|
73
|
+
|
74
|
+
# Create table with indexes using SDK API
|
75
|
+
cls.client.create_table(TestTable)
|
76
|
+
|
77
|
+
# Insert test data using SDK API
|
78
|
+
test_data = [{'id': i, 'name': f'name_{i}', 'category': f'cat_{i % 5}', 'value': i * 10} for i in range(1, 101)]
|
79
|
+
cls.client.batch_insert(TestTable, test_data)
|
80
|
+
|
81
|
+
except Exception as e:
|
82
|
+
print(f"Setup failed: {e}")
|
83
|
+
raise
|
84
|
+
|
85
|
+
@classmethod
|
86
|
+
def tearDownClass(cls):
|
87
|
+
"""Clean up test database"""
|
88
|
+
try:
|
89
|
+
cls.client.drop_table(cls.TestTable)
|
90
|
+
except Exception as e:
|
91
|
+
print(f"Cleanup failed: {e}")
|
92
|
+
|
93
|
+
def test_get_secondary_index_tables(self):
|
94
|
+
"""Test getting all secondary index table names"""
|
95
|
+
index_tables = self.client.get_secondary_index_tables(self.test_table)
|
96
|
+
|
97
|
+
# Should have 3 secondary indexes
|
98
|
+
self.assertEqual(len(index_tables), 3, f"Expected 3 indexes, got {len(index_tables)}")
|
99
|
+
|
100
|
+
# All index tables should start with __mo_index_secondary_
|
101
|
+
for index_table in index_tables:
|
102
|
+
self.assertTrue(
|
103
|
+
index_table.startswith('__mo_index_secondary_'),
|
104
|
+
f"Index table name should start with '__mo_index_secondary_', got {index_table}",
|
105
|
+
)
|
106
|
+
|
107
|
+
def test_get_secondary_index_table_by_name(self):
|
108
|
+
"""Test getting physical table name of a secondary index by its name"""
|
109
|
+
# Test getting idx_name
|
110
|
+
idx_name_table = self.client.get_secondary_index_table_by_name(self.test_table, 'idx_name')
|
111
|
+
self.assertIsNotNone(idx_name_table, "idx_name should exist")
|
112
|
+
self.assertTrue(idx_name_table.startswith('__mo_index_secondary_'))
|
113
|
+
|
114
|
+
# Test getting idx_category
|
115
|
+
idx_category_table = self.client.get_secondary_index_table_by_name(self.test_table, 'idx_category')
|
116
|
+
self.assertIsNotNone(idx_category_table, "idx_category should exist")
|
117
|
+
self.assertTrue(idx_category_table.startswith('__mo_index_secondary_'))
|
118
|
+
|
119
|
+
# Test getting idx_value
|
120
|
+
idx_value_table = self.client.get_secondary_index_table_by_name(self.test_table, 'idx_value')
|
121
|
+
self.assertIsNotNone(idx_value_table, "idx_value should exist")
|
122
|
+
self.assertTrue(idx_value_table.startswith('__mo_index_secondary_'))
|
123
|
+
|
124
|
+
# Test non-existent index
|
125
|
+
non_existent = self.client.get_secondary_index_table_by_name(self.test_table, 'idx_non_existent')
|
126
|
+
self.assertIsNone(non_existent, "Non-existent index should return None")
|
127
|
+
|
128
|
+
def test_verify_table_index_counts(self):
|
129
|
+
"""Test verify_table_index_counts when counts match"""
|
130
|
+
# All indexes should have the same count as the main table
|
131
|
+
count = self.client.verify_table_index_counts(self.test_table)
|
132
|
+
|
133
|
+
# Should return 100 (number of inserted rows)
|
134
|
+
self.assertEqual(count, 100, f"Expected count 100, got {count}")
|
135
|
+
|
136
|
+
# Verify count matches query result using SDK API
|
137
|
+
actual_count = self.client.query(self.TestTable).count()
|
138
|
+
self.assertEqual(count, actual_count, "Verify result should match actual count")
|
139
|
+
|
140
|
+
def test_verify_table_without_indexes(self):
|
141
|
+
"""Test verify_table_index_counts on table without secondary indexes"""
|
142
|
+
# Define a simple model without secondary indexes
|
143
|
+
Base = declarative_base()
|
144
|
+
|
145
|
+
class SimpleTable(Base):
|
146
|
+
__tablename__ = "test_simple_table"
|
147
|
+
id = Column(Integer, primary_key=True)
|
148
|
+
value = Column(Integer)
|
149
|
+
|
150
|
+
# Create table using SDK API
|
151
|
+
try:
|
152
|
+
self.client.drop_table(SimpleTable)
|
153
|
+
except:
|
154
|
+
pass
|
155
|
+
|
156
|
+
self.client.create_table(SimpleTable)
|
157
|
+
|
158
|
+
# Insert data using SDK API
|
159
|
+
test_data = [{'id': 1, 'value': 100}, {'id': 2, 'value': 200}]
|
160
|
+
self.client.batch_insert(SimpleTable, test_data)
|
161
|
+
|
162
|
+
# Verification should succeed and return count
|
163
|
+
count = self.client.verify_table_index_counts("test_simple_table")
|
164
|
+
self.assertEqual(count, 2, f"Expected count 2, got {count}")
|
165
|
+
|
166
|
+
# Cleanup using SDK API
|
167
|
+
self.client.drop_table(SimpleTable)
|
168
|
+
|
169
|
+
def test_index_table_mapping(self):
|
170
|
+
"""Test that all indexes can be retrieved both ways"""
|
171
|
+
# Get all index tables
|
172
|
+
all_index_tables = self.client.get_secondary_index_tables(self.test_table)
|
173
|
+
|
174
|
+
# Get each index by name
|
175
|
+
index_names = ['idx_name', 'idx_category', 'idx_value']
|
176
|
+
retrieved_tables = []
|
177
|
+
|
178
|
+
for index_name in index_names:
|
179
|
+
table = self.client.get_secondary_index_table_by_name(self.test_table, index_name)
|
180
|
+
self.assertIsNotNone(table, f"Index {index_name} should exist")
|
181
|
+
retrieved_tables.append(table)
|
182
|
+
|
183
|
+
# All retrieved tables should be in the all_index_tables list
|
184
|
+
for table in retrieved_tables:
|
185
|
+
self.assertIn(table, all_index_tables, f"Retrieved table {table} should be in all index tables")
|
186
|
+
|
187
|
+
# Both lists should have the same length
|
188
|
+
self.assertEqual(
|
189
|
+
len(retrieved_tables), len(all_index_tables), "Number of retrieved tables should match total index tables"
|
190
|
+
)
|
191
|
+
|
192
|
+
|
193
|
+
if __name__ == '__main__':
|
194
|
+
unittest.main()
|
File without changes
|
{matrixone_python_sdk-0.1.1.dist-info → matrixone_python_sdk-0.1.3.dist-info}/entry_points.txt
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1.dist-info → matrixone_python_sdk-0.1.3.dist-info}/licenses/LICENSE
RENAMED
File without changes
|
File without changes
|