matrixone-python-sdk 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- matrixone/async_client.py +100 -3
- matrixone/async_metadata_manager.py +3 -1
- matrixone/client.py +102 -2
- matrixone/connection_hooks.py +8 -7
- matrixone/exceptions.py +0 -24
- matrixone/index_utils.py +143 -0
- matrixone/metadata.py +3 -2
- matrixone/moctl.py +0 -2
- matrixone/sqlalchemy_ext/__init__.py +1 -0
- matrixone/version.py +0 -2
- {matrixone_python_sdk-0.1.2.dist-info → matrixone_python_sdk-0.1.4.dist-info}/METADATA +1 -1
- {matrixone_python_sdk-0.1.2.dist-info → matrixone_python_sdk-0.1.4.dist-info}/RECORD +19 -16
- tests/offline/test_connection_hooks_offline.py +8 -8
- tests/online/test_async_index_verification_online.py +259 -0
- tests/online/test_index_verification_online.py +194 -0
- {matrixone_python_sdk-0.1.2.dist-info → matrixone_python_sdk-0.1.4.dist-info}/WHEEL +0 -0
- {matrixone_python_sdk-0.1.2.dist-info → matrixone_python_sdk-0.1.4.dist-info}/entry_points.txt +0 -0
- {matrixone_python_sdk-0.1.2.dist-info → matrixone_python_sdk-0.1.4.dist-info}/licenses/LICENSE +0 -0
- {matrixone_python_sdk-0.1.2.dist-info → matrixone_python_sdk-0.1.4.dist-info}/top_level.txt +0 -0
matrixone/async_client.py
CHANGED
@@ -34,9 +34,10 @@ from datetime import datetime
|
|
34
34
|
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
35
35
|
|
36
36
|
from .account import Account, User
|
37
|
+
from .async_metadata_manager import AsyncMetadataManager
|
37
38
|
from .async_vector_index_manager import AsyncVectorManager
|
38
39
|
from .base_client import BaseMatrixOneClient, BaseMatrixOneExecutor
|
39
|
-
from .connection_hooks import
|
40
|
+
from .connection_hooks import ConnectionAction, ConnectionHook, create_connection_hook
|
40
41
|
from .exceptions import (
|
41
42
|
AccountError,
|
42
43
|
ConnectionError,
|
@@ -48,7 +49,6 @@ from .exceptions import (
|
|
48
49
|
SnapshotError,
|
49
50
|
)
|
50
51
|
from .logger import MatrixOneLogger, create_default_logger
|
51
|
-
from .async_metadata_manager import AsyncMetadataManager
|
52
52
|
from .pitr import Pitr
|
53
53
|
from .pubsub import Publication, Subscription
|
54
54
|
from .snapshot import Snapshot, SnapshotLevel
|
@@ -1837,7 +1837,6 @@ class AsyncClient(BaseMatrixOneClient):
|
|
1837
1837
|
"""Cleanup when object is garbage collected"""
|
1838
1838
|
# Don't try to cleanup in __del__ as it can cause issues with event loops
|
1839
1839
|
# The fixture should handle proper cleanup
|
1840
|
-
pass
|
1841
1840
|
|
1842
1841
|
def get_sqlalchemy_engine(self) -> AsyncEngine:
|
1843
1842
|
"""
|
@@ -1920,6 +1919,7 @@ class AsyncClient(BaseMatrixOneClient):
|
|
1920
1919
|
and other SDK components. External users should use execute() instead.
|
1921
1920
|
"""
|
1922
1921
|
import time
|
1922
|
+
|
1923
1923
|
from sqlalchemy import text
|
1924
1924
|
|
1925
1925
|
start_time = time.time()
|
@@ -2554,6 +2554,103 @@ class AsyncClient(BaseMatrixOneClient):
|
|
2554
2554
|
except Exception as e:
|
2555
2555
|
raise QueryError(f"Failed to get git version: {e}")
|
2556
2556
|
|
2557
|
+
async def get_secondary_index_tables(self, table_name: str) -> List[str]:
|
2558
|
+
"""
|
2559
|
+
Get all secondary index table names for a given table in the current database (async version).
|
2560
|
+
|
2561
|
+
Args:
|
2562
|
+
table_name: Name of the table to get secondary indexes for
|
2563
|
+
|
2564
|
+
Returns:
|
2565
|
+
List of secondary index table names
|
2566
|
+
|
2567
|
+
Examples::
|
2568
|
+
|
2569
|
+
>>> async with AsyncClient() as client:
|
2570
|
+
... await client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2571
|
+
... index_tables = await client.get_secondary_index_tables('cms_all_content_chunk_info')
|
2572
|
+
... print(index_tables)
|
2573
|
+
"""
|
2574
|
+
from .index_utils import build_get_index_tables_sql
|
2575
|
+
|
2576
|
+
# Get current database from connection params
|
2577
|
+
database = self._connection_params.get('database') if hasattr(self, '_connection_params') else None
|
2578
|
+
|
2579
|
+
sql, params = build_get_index_tables_sql(table_name, database)
|
2580
|
+
result = await self.execute(sql, params)
|
2581
|
+
return [row[0] for row in result.fetchall()]
|
2582
|
+
|
2583
|
+
async def get_secondary_index_table_by_name(self, table_name: str, index_name: str) -> Optional[str]:
|
2584
|
+
"""
|
2585
|
+
Get the physical table name of a secondary index by its index name in the current database (async version).
|
2586
|
+
|
2587
|
+
Args:
|
2588
|
+
table_name: Name of the table
|
2589
|
+
index_name: Name of the secondary index
|
2590
|
+
|
2591
|
+
Returns:
|
2592
|
+
Physical table name of the secondary index, or None if not found
|
2593
|
+
|
2594
|
+
Examples::
|
2595
|
+
|
2596
|
+
>>> async with AsyncClient() as client:
|
2597
|
+
... await client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2598
|
+
... index_table = await client.get_secondary_index_table_by_name('cms_all_content_chunk_info', 'cms_id')
|
2599
|
+
... print(index_table)
|
2600
|
+
"""
|
2601
|
+
from .index_utils import build_get_index_table_by_name_sql
|
2602
|
+
|
2603
|
+
# Get current database from connection params
|
2604
|
+
database = self._connection_params.get('database') if hasattr(self, '_connection_params') else None
|
2605
|
+
|
2606
|
+
sql, params = build_get_index_table_by_name_sql(table_name, index_name, database)
|
2607
|
+
result = await self.execute(sql, params)
|
2608
|
+
row = result.fetchone()
|
2609
|
+
return row[0] if row else None
|
2610
|
+
|
2611
|
+
async def verify_table_index_counts(self, table_name: str) -> int:
|
2612
|
+
"""
|
2613
|
+
Verify that the main table and all its secondary index tables have the same row count (async version).
|
2614
|
+
|
2615
|
+
This method compares the COUNT(*) of the main table with all its secondary index tables
|
2616
|
+
in a single SQL query for consistency. If counts don't match, raises an exception.
|
2617
|
+
|
2618
|
+
Args:
|
2619
|
+
table_name: Name of the table to verify
|
2620
|
+
|
2621
|
+
Returns:
|
2622
|
+
Row count (int) if verification succeeds
|
2623
|
+
|
2624
|
+
Raises:
|
2625
|
+
ValueError: If any secondary index table has a different count than the main table,
|
2626
|
+
with details about all counts in the error message
|
2627
|
+
|
2628
|
+
Examples::
|
2629
|
+
|
2630
|
+
>>> async with AsyncClient() as client:
|
2631
|
+
... await client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2632
|
+
... count = await client.verify_table_index_counts('cms_all_content_chunk_info')
|
2633
|
+
... print(f"✓ Verification passed, row count: {count}")
|
2634
|
+
|
2635
|
+
>>> # If verification fails:
|
2636
|
+
>>> try:
|
2637
|
+
... count = await client.verify_table_index_counts('some_table')
|
2638
|
+
... except ValueError as e:
|
2639
|
+
... print(f"Verification failed: {e}")
|
2640
|
+
"""
|
2641
|
+
from .index_utils import build_verify_counts_sql, process_verify_result
|
2642
|
+
|
2643
|
+
# Get all secondary index tables
|
2644
|
+
index_tables = await self.get_secondary_index_tables(table_name)
|
2645
|
+
|
2646
|
+
# Build and execute verification SQL
|
2647
|
+
sql = build_verify_counts_sql(table_name, index_tables)
|
2648
|
+
result = await self.execute(sql)
|
2649
|
+
row = result.fetchone()
|
2650
|
+
|
2651
|
+
# Process result and raise exception if verification fails
|
2652
|
+
return process_verify_result(table_name, index_tables, row)
|
2653
|
+
|
2557
2654
|
async def __aenter__(self):
|
2558
2655
|
return self
|
2559
2656
|
|
@@ -19,8 +19,10 @@ This module provides async metadata scanning capabilities for MatrixOne tables,
|
|
19
19
|
allowing users to analyze table statistics, column information, and data distribution.
|
20
20
|
"""
|
21
21
|
|
22
|
-
from typing import
|
22
|
+
from typing import Any, Dict, List, Optional, Union
|
23
|
+
|
23
24
|
from sqlalchemy.engine import Result
|
25
|
+
|
24
26
|
from .metadata import BaseMetadataManager, MetadataColumn, MetadataRow
|
25
27
|
|
26
28
|
|
matrixone/client.py
CHANGED
@@ -30,7 +30,7 @@ from sqlalchemy.engine import Engine
|
|
30
30
|
|
31
31
|
from .account import AccountManager, TransactionAccountManager
|
32
32
|
from .base_client import BaseMatrixOneClient, BaseMatrixOneExecutor
|
33
|
-
from .connection_hooks import
|
33
|
+
from .connection_hooks import ConnectionAction, ConnectionHook, create_connection_hook
|
34
34
|
from .exceptions import ConnectionError, QueryError
|
35
35
|
from .logger import MatrixOneLogger, create_default_logger
|
36
36
|
from .metadata import MetadataManager, TransactionMetadataManager
|
@@ -1698,8 +1698,9 @@ class Client(BaseMatrixOneClient):
|
|
1698
1698
|
table_name = model_class.__tablename__
|
1699
1699
|
table = model_class.__table__
|
1700
1700
|
|
1701
|
+
from sqlalchemy.schema import CreateIndex, CreateTable
|
1702
|
+
|
1701
1703
|
from .sqlalchemy_ext import FulltextIndex, VectorIndex
|
1702
|
-
from sqlalchemy.schema import CreateTable, CreateIndex
|
1703
1704
|
|
1704
1705
|
try:
|
1705
1706
|
engine_context = self.get_sqlalchemy_engine().begin()
|
@@ -2612,6 +2613,105 @@ class Client(BaseMatrixOneClient):
|
|
2612
2613
|
|
2613
2614
|
return self
|
2614
2615
|
|
2616
|
+
def get_secondary_index_tables(self, table_name: str) -> List[str]:
|
2617
|
+
"""
|
2618
|
+
Get all secondary index table names for a given table in the current database.
|
2619
|
+
|
2620
|
+
Args:
|
2621
|
+
table_name: Name of the table to get secondary indexes for
|
2622
|
+
|
2623
|
+
Returns:
|
2624
|
+
List of secondary index table names
|
2625
|
+
|
2626
|
+
Examples::
|
2627
|
+
|
2628
|
+
>>> client = Client()
|
2629
|
+
>>> client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2630
|
+
>>> index_tables = client.get_secondary_index_tables('cms_all_content_chunk_info')
|
2631
|
+
>>> print(index_tables)
|
2632
|
+
['__mo_index_secondary_..._cms_id', '__mo_index_secondary_..._idx_all_content_length']
|
2633
|
+
"""
|
2634
|
+
from .index_utils import build_get_index_tables_sql
|
2635
|
+
|
2636
|
+
# Get current database from connection params
|
2637
|
+
database = self._connection_params.get('database') if hasattr(self, '_connection_params') else None
|
2638
|
+
|
2639
|
+
sql, params = build_get_index_tables_sql(table_name, database)
|
2640
|
+
result = self.execute(sql, params)
|
2641
|
+
return [row[0] for row in result.fetchall()]
|
2642
|
+
|
2643
|
+
def get_secondary_index_table_by_name(self, table_name: str, index_name: str) -> Optional[str]:
|
2644
|
+
"""
|
2645
|
+
Get the physical table name of a secondary index by its index name in the current database.
|
2646
|
+
|
2647
|
+
Args:
|
2648
|
+
table_name: Name of the table
|
2649
|
+
index_name: Name of the secondary index
|
2650
|
+
|
2651
|
+
Returns:
|
2652
|
+
Physical table name of the secondary index, or None if not found
|
2653
|
+
|
2654
|
+
Examples::
|
2655
|
+
|
2656
|
+
>>> client = Client()
|
2657
|
+
>>> client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2658
|
+
>>> index_table = client.get_secondary_index_table_by_name('cms_all_content_chunk_info', 'cms_id')
|
2659
|
+
>>> print(index_table)
|
2660
|
+
'__mo_index_secondary_018cfbda-bde1-7c3e-805c-3f8e71769f75_cms_id'
|
2661
|
+
"""
|
2662
|
+
from .index_utils import build_get_index_table_by_name_sql
|
2663
|
+
|
2664
|
+
# Get current database from connection params
|
2665
|
+
database = self._connection_params.get('database') if hasattr(self, '_connection_params') else None
|
2666
|
+
|
2667
|
+
sql, params = build_get_index_table_by_name_sql(table_name, index_name, database)
|
2668
|
+
result = self.execute(sql, params)
|
2669
|
+
row = result.fetchone()
|
2670
|
+
return row[0] if row else None
|
2671
|
+
|
2672
|
+
def verify_table_index_counts(self, table_name: str) -> int:
|
2673
|
+
"""
|
2674
|
+
Verify that the main table and all its secondary index tables have the same row count.
|
2675
|
+
|
2676
|
+
This method compares the COUNT(*) of the main table with all its secondary index tables
|
2677
|
+
in a single SQL query for consistency. If counts don't match, raises an exception.
|
2678
|
+
|
2679
|
+
Args:
|
2680
|
+
table_name: Name of the table to verify
|
2681
|
+
|
2682
|
+
Returns:
|
2683
|
+
Row count (int) if verification succeeds
|
2684
|
+
|
2685
|
+
Raises:
|
2686
|
+
ValueError: If any secondary index table has a different count than the main table,
|
2687
|
+
with details about all counts in the error message
|
2688
|
+
|
2689
|
+
Examples::
|
2690
|
+
|
2691
|
+
>>> client = Client()
|
2692
|
+
>>> client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2693
|
+
>>> count = client.verify_table_index_counts('cms_all_content_chunk_info')
|
2694
|
+
>>> print(f"✓ Verification passed, row count: {count}")
|
2695
|
+
|
2696
|
+
>>> # If verification fails:
|
2697
|
+
>>> try:
|
2698
|
+
... count = client.verify_table_index_counts('some_table')
|
2699
|
+
... except ValueError as e:
|
2700
|
+
... print(f"Verification failed: {e}")
|
2701
|
+
"""
|
2702
|
+
from .index_utils import build_verify_counts_sql, process_verify_result
|
2703
|
+
|
2704
|
+
# Get all secondary index tables
|
2705
|
+
index_tables = self.get_secondary_index_tables(table_name)
|
2706
|
+
|
2707
|
+
# Build and execute verification SQL
|
2708
|
+
sql = build_verify_counts_sql(table_name, index_tables)
|
2709
|
+
result = self.execute(sql)
|
2710
|
+
row = result.fetchone()
|
2711
|
+
|
2712
|
+
# Process result and raise exception if verification fails
|
2713
|
+
return process_verify_result(table_name, index_tables, row)
|
2714
|
+
|
2615
2715
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
2616
2716
|
self.disconnect()
|
2617
2717
|
|
matrixone/connection_hooks.py
CHANGED
@@ -18,6 +18,7 @@ Connection hooks for MatrixOne clients
|
|
18
18
|
|
19
19
|
from enum import Enum
|
20
20
|
from typing import Callable, List, Optional, Union
|
21
|
+
|
21
22
|
from sqlalchemy import event
|
22
23
|
from sqlalchemy.engine import Engine
|
23
24
|
from sqlalchemy.ext.asyncio import AsyncEngine
|
@@ -68,13 +69,13 @@ class ConnectionHook:
|
|
68
69
|
event.listen(engine.sync_engine, "connect", self._on_connect_sync)
|
69
70
|
event.listen(engine.sync_engine, "before_cursor_execute", self._on_before_cursor_execute)
|
70
71
|
if hasattr(self._client_ref, 'logger'):
|
71
|
-
self._client_ref.logger.
|
72
|
+
self._client_ref.logger.debug("Attached connection hook to async engine")
|
72
73
|
else:
|
73
74
|
# For sync engines, listen to both connect and before_cursor_execute events
|
74
75
|
event.listen(engine, "connect", self._on_connect_sync)
|
75
76
|
event.listen(engine, "before_cursor_execute", self._on_before_cursor_execute)
|
76
77
|
if hasattr(self._client_ref, 'logger'):
|
77
|
-
self._client_ref.logger.
|
78
|
+
self._client_ref.logger.debug("Attached connection hook to sync engine")
|
78
79
|
|
79
80
|
def _on_connect_sync(self, dbapi_connection, connection_record):
|
80
81
|
"""SQLAlchemy event handler for new connections (sync)"""
|
@@ -85,7 +86,7 @@ class ConnectionHook:
|
|
85
86
|
try:
|
86
87
|
# Log that the hook is being executed
|
87
88
|
if hasattr(self._client_ref, 'logger'):
|
88
|
-
self._client_ref.logger.
|
89
|
+
self._client_ref.logger.debug(f"Executing connection hook on new connection {conn_id}")
|
89
90
|
# Pass the connection to avoid creating new connections
|
90
91
|
self.execute_sync_with_connection(self._client_ref, dbapi_connection)
|
91
92
|
self._executed_connections.add(conn_id)
|
@@ -103,7 +104,7 @@ class ConnectionHook:
|
|
103
104
|
try:
|
104
105
|
# Log that the hook is being executed
|
105
106
|
if hasattr(self._client_ref, 'logger'):
|
106
|
-
self._client_ref.logger.
|
107
|
+
self._client_ref.logger.debug(f"Executing connection hook on connection {conn_id}")
|
107
108
|
# Use the connection to avoid creating new connections
|
108
109
|
self.execute_sync_with_connection(self._client_ref, conn.connection)
|
109
110
|
self._executed_connections.add(conn_id)
|
@@ -211,7 +212,7 @@ class ConnectionHook:
|
|
211
212
|
cursor = dbapi_connection.cursor()
|
212
213
|
cursor.execute("SET experimental_ivf_index = 1")
|
213
214
|
cursor.close()
|
214
|
-
client.logger.
|
215
|
+
client.logger.debug("✓ Enabled IVF vector operations")
|
215
216
|
except Exception as e:
|
216
217
|
client.logger.warning(f"Failed to enable IVF: {e}")
|
217
218
|
|
@@ -222,7 +223,7 @@ class ConnectionHook:
|
|
222
223
|
cursor = dbapi_connection.cursor()
|
223
224
|
cursor.execute("SET experimental_hnsw_index = 1")
|
224
225
|
cursor.close()
|
225
|
-
client.logger.
|
226
|
+
client.logger.debug("✓ Enabled HNSW vector operations")
|
226
227
|
except Exception as e:
|
227
228
|
client.logger.warning(f"Failed to enable HNSW: {e}")
|
228
229
|
|
@@ -233,7 +234,7 @@ class ConnectionHook:
|
|
233
234
|
cursor = dbapi_connection.cursor()
|
234
235
|
cursor.execute("SET experimental_fulltext_index = 1")
|
235
236
|
cursor.close()
|
236
|
-
client.logger.
|
237
|
+
client.logger.debug("✓ Enabled fulltext search operations")
|
237
238
|
except Exception as e:
|
238
239
|
client.logger.warning(f"Failed to enable fulltext: {e}")
|
239
240
|
|
matrixone/exceptions.py
CHANGED
@@ -20,70 +20,46 @@ MatrixOne SDK Exceptions
|
|
20
20
|
class MatrixOneError(Exception):
|
21
21
|
"""Base exception for all MatrixOne SDK errors"""
|
22
22
|
|
23
|
-
pass
|
24
|
-
|
25
23
|
|
26
24
|
class ConnectionError(MatrixOneError):
|
27
25
|
"""Raised when connection to MatrixOne fails"""
|
28
26
|
|
29
|
-
pass
|
30
|
-
|
31
27
|
|
32
28
|
class QueryError(MatrixOneError):
|
33
29
|
"""Raised when SQL query execution fails"""
|
34
30
|
|
35
|
-
pass
|
36
|
-
|
37
31
|
|
38
32
|
class ConfigurationError(MatrixOneError):
|
39
33
|
"""Raised when configuration is invalid"""
|
40
34
|
|
41
|
-
pass
|
42
|
-
|
43
35
|
|
44
36
|
class SnapshotError(MatrixOneError):
|
45
37
|
"""Raised when snapshot operations fail"""
|
46
38
|
|
47
|
-
pass
|
48
|
-
|
49
39
|
|
50
40
|
class CloneError(MatrixOneError):
|
51
41
|
"""Raised when clone operations fail"""
|
52
42
|
|
53
|
-
pass
|
54
|
-
|
55
43
|
|
56
44
|
class MoCtlError(MatrixOneError):
|
57
45
|
"""Raised when mo_ctl operations fail"""
|
58
46
|
|
59
|
-
pass
|
60
|
-
|
61
47
|
|
62
48
|
class RestoreError(MatrixOneError):
|
63
49
|
"""Raised when restore operations fail"""
|
64
50
|
|
65
|
-
pass
|
66
|
-
|
67
51
|
|
68
52
|
class PitrError(MatrixOneError):
|
69
53
|
"""Raised when PITR operations fail"""
|
70
54
|
|
71
|
-
pass
|
72
|
-
|
73
55
|
|
74
56
|
class PubSubError(MatrixOneError):
|
75
57
|
"""Raised when publish-subscribe operations fail"""
|
76
58
|
|
77
|
-
pass
|
78
|
-
|
79
59
|
|
80
60
|
class AccountError(MatrixOneError):
|
81
61
|
"""Raised when account management operations fail"""
|
82
62
|
|
83
|
-
pass
|
84
|
-
|
85
63
|
|
86
64
|
class VersionError(MatrixOneError):
|
87
65
|
"""Raised when version compatibility check fails"""
|
88
|
-
|
89
|
-
pass
|
matrixone/index_utils.py
ADDED
@@ -0,0 +1,143 @@
|
|
1
|
+
# Copyright 2021 - 2022 Matrix Origin
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
"""
|
16
|
+
Index utilities - Shared logic for secondary index operations
|
17
|
+
"""
|
18
|
+
|
19
|
+
from typing import List, Tuple
|
20
|
+
|
21
|
+
|
22
|
+
def build_get_index_tables_sql(table_name: str, database: str = None) -> Tuple[str, Tuple]:
|
23
|
+
"""
|
24
|
+
Build SQL to get all secondary index table names for a given table.
|
25
|
+
|
26
|
+
Args:
|
27
|
+
table_name: Name of the table
|
28
|
+
database: Name of the database (optional, but recommended to avoid cross-database conflicts)
|
29
|
+
|
30
|
+
Returns:
|
31
|
+
Tuple of (sql, params)
|
32
|
+
"""
|
33
|
+
if database:
|
34
|
+
sql = """
|
35
|
+
SELECT DISTINCT index_table_name
|
36
|
+
FROM mo_catalog.mo_indexes
|
37
|
+
JOIN mo_catalog.mo_tables ON mo_indexes.table_id = mo_tables.rel_id
|
38
|
+
WHERE relname = ? AND reldatabase = ? AND type = 'MULTIPLE'
|
39
|
+
"""
|
40
|
+
return sql, (table_name, database)
|
41
|
+
else:
|
42
|
+
# Fallback to old behavior if database is not provided
|
43
|
+
sql = """
|
44
|
+
SELECT DISTINCT index_table_name
|
45
|
+
FROM mo_catalog.mo_indexes
|
46
|
+
JOIN mo_catalog.mo_tables ON mo_indexes.table_id = mo_tables.rel_id
|
47
|
+
WHERE relname = ? AND type = 'MULTIPLE'
|
48
|
+
"""
|
49
|
+
return sql, (table_name,)
|
50
|
+
|
51
|
+
|
52
|
+
def build_get_index_table_by_name_sql(table_name: str, index_name: str, database: str = None) -> Tuple[str, Tuple]:
|
53
|
+
"""
|
54
|
+
Build SQL to get the physical table name of a secondary index by its index name.
|
55
|
+
|
56
|
+
Args:
|
57
|
+
table_name: Name of the table
|
58
|
+
index_name: Name of the secondary index
|
59
|
+
database: Name of the database (optional, but recommended to avoid cross-database conflicts)
|
60
|
+
|
61
|
+
Returns:
|
62
|
+
Tuple of (sql, params)
|
63
|
+
"""
|
64
|
+
if database:
|
65
|
+
sql = """
|
66
|
+
SELECT DISTINCT index_table_name
|
67
|
+
FROM mo_catalog.mo_indexes
|
68
|
+
JOIN mo_catalog.mo_tables ON mo_indexes.table_id = mo_tables.rel_id
|
69
|
+
WHERE relname = ? AND name = ? AND reldatabase = ?
|
70
|
+
"""
|
71
|
+
return sql, (table_name, index_name, database)
|
72
|
+
else:
|
73
|
+
# Fallback to old behavior if database is not provided
|
74
|
+
sql = """
|
75
|
+
SELECT DISTINCT index_table_name
|
76
|
+
FROM mo_catalog.mo_indexes
|
77
|
+
JOIN mo_catalog.mo_tables ON mo_indexes.table_id = mo_tables.rel_id
|
78
|
+
WHERE relname = ? AND name = ?
|
79
|
+
"""
|
80
|
+
return sql, (table_name, index_name)
|
81
|
+
|
82
|
+
|
83
|
+
def build_verify_counts_sql(table_name: str, index_tables: List[str]) -> str:
|
84
|
+
"""
|
85
|
+
Build SQL to verify counts of main table and all index tables in a single query.
|
86
|
+
|
87
|
+
Args:
|
88
|
+
table_name: Name of the main table
|
89
|
+
index_tables: List of index table names
|
90
|
+
|
91
|
+
Returns:
|
92
|
+
SQL string
|
93
|
+
"""
|
94
|
+
if not index_tables:
|
95
|
+
return f"SELECT COUNT(*) FROM `{table_name}`"
|
96
|
+
|
97
|
+
select_parts = [f"(SELECT COUNT(*) FROM `{table_name}`) as main_count"]
|
98
|
+
for idx, index_table in enumerate(index_tables):
|
99
|
+
select_parts.append(f"(SELECT COUNT(*) FROM `{index_table}`) as idx{idx}_count")
|
100
|
+
|
101
|
+
return "SELECT " + ", ".join(select_parts)
|
102
|
+
|
103
|
+
|
104
|
+
def process_verify_result(table_name: str, index_tables: List[str], row: Tuple) -> int:
|
105
|
+
"""
|
106
|
+
Process the verification result and raise exception if counts don't match.
|
107
|
+
|
108
|
+
Args:
|
109
|
+
table_name: Name of the main table
|
110
|
+
index_tables: List of index table names
|
111
|
+
row: Result row from the verification SQL
|
112
|
+
|
113
|
+
Returns:
|
114
|
+
Row count if verification succeeds
|
115
|
+
|
116
|
+
Raises:
|
117
|
+
ValueError: If any index table has a different count
|
118
|
+
"""
|
119
|
+
main_count = row[0]
|
120
|
+
|
121
|
+
if not index_tables:
|
122
|
+
return main_count
|
123
|
+
|
124
|
+
index_counts = {}
|
125
|
+
mismatch = []
|
126
|
+
|
127
|
+
for idx, index_table in enumerate(index_tables):
|
128
|
+
index_count = row[idx + 1]
|
129
|
+
index_counts[index_table] = index_count
|
130
|
+
if index_count != main_count:
|
131
|
+
mismatch.append(index_table)
|
132
|
+
|
133
|
+
# If there's a mismatch, raise an exception with details
|
134
|
+
if mismatch:
|
135
|
+
error_details = [f"Main table '{table_name}': {main_count} rows"]
|
136
|
+
for index_table, count in index_counts.items():
|
137
|
+
status = "✗ MISMATCH" if index_table in mismatch else "✓"
|
138
|
+
error_details.append(f"{status} Index '{index_table}': {count} rows")
|
139
|
+
|
140
|
+
error_msg = "Index count verification failed!\n" + "\n".join(error_details)
|
141
|
+
raise ValueError(error_msg)
|
142
|
+
|
143
|
+
return main_count
|
matrixone/metadata.py
CHANGED
@@ -19,10 +19,11 @@ This module provides metadata scanning capabilities for MatrixOne tables,
|
|
19
19
|
allowing users to analyze table statistics, column information, and data distribution.
|
20
20
|
"""
|
21
21
|
|
22
|
-
from typing import Optional, List, Dict, Any, Union
|
23
|
-
from sqlalchemy.engine import Result
|
24
22
|
from dataclasses import dataclass
|
25
23
|
from enum import Enum
|
24
|
+
from typing import Any, Dict, List, Optional, Union
|
25
|
+
|
26
|
+
from sqlalchemy.engine import Result
|
26
27
|
|
27
28
|
|
28
29
|
class MetadataColumn(Enum):
|
matrixone/moctl.py
CHANGED
matrixone/version.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: matrixone-python-sdk
|
3
|
-
Version: 0.1.
|
3
|
+
Version: 0.1.4
|
4
4
|
Summary: A comprehensive Python SDK for MatrixOne database operations with vector search, fulltext search, and advanced features
|
5
5
|
Home-page: https://github.com/matrixorigin/matrixone
|
6
6
|
Author: MatrixOne Team
|
@@ -1,17 +1,18 @@
|
|
1
1
|
matrixone/__init__.py,sha256=7bePzwzerZTiZIlbUT9zI_u4fp49DvQNcIqyiIvEVVs,3966
|
2
2
|
matrixone/account.py,sha256=0r9xLNTiUfXa3xWZQUhEJ_uUcNp_gja-YulOR5iYDU4,24712
|
3
|
-
matrixone/async_client.py,sha256=
|
4
|
-
matrixone/async_metadata_manager.py,sha256=
|
3
|
+
matrixone/async_client.py,sha256=yluma410CLUy3FhxMxeOm-5uhRwhD-ONzMIJd-dEuvg,155669
|
4
|
+
matrixone/async_metadata_manager.py,sha256=W3TJBCy56diDPBzimpBImFrmf9DBc7iiS5CdBHLAh_Q,10232
|
5
5
|
matrixone/async_orm.py,sha256=O4Rf85MnZg0_fuLvbHGfOPrHI2OLznYwLk9eam5J5ik,4864
|
6
6
|
matrixone/async_vector_index_manager.py,sha256=fVnS00zgbUXw4YmZR-wRCzm541n30TLlmXXmXV34ELU,23134
|
7
7
|
matrixone/base_client.py,sha256=PUhq0c7Si9OIk83B4ZZ73ozwaHDRZ1zi0Q94YixseqY,6952
|
8
|
-
matrixone/client.py,sha256=
|
8
|
+
matrixone/client.py,sha256=NONid82vkq2GYWVfJqjHgFjeTBkXauEEvcoYtSoeuYs,187735
|
9
9
|
matrixone/config.py,sha256=jrm1rLBhLWrX3dVdPx67sFaZ_9M8Vj4hImVrAmFaYbw,13917
|
10
|
-
matrixone/connection_hooks.py,sha256=
|
11
|
-
matrixone/exceptions.py,sha256=
|
10
|
+
matrixone/connection_hooks.py,sha256=5o7heheRQ8pLguoUQxEoo023519rT2-OhloPW-vORpE,13538
|
11
|
+
matrixone/exceptions.py,sha256=VsGgESLl4vm5-9QbigyE5RTwFujyoYXhQuGhGKoujFw,1667
|
12
|
+
matrixone/index_utils.py,sha256=ofEm0PHKE4VIGuq4ymeJPtA5NvV1afwZfHPLN2bd6Dw,4862
|
12
13
|
matrixone/logger.py,sha256=aZmetje_AqtyxFxsqZvp-3R68n8I_ldplRGfV2MHg48,26105
|
13
|
-
matrixone/metadata.py,sha256=
|
14
|
-
matrixone/moctl.py,sha256=
|
14
|
+
matrixone/metadata.py,sha256=Yf0eJywe3BUqg6aEywSWh9neRcVsrgf6g-uaZFvqLOA,31707
|
15
|
+
matrixone/moctl.py,sha256=b_qQPSK52JiyyUH4PYzAuhJPk8dAD6LRfT_sFGsf23o,6704
|
15
16
|
matrixone/orm.py,sha256=QWNdfJ067KcgWU-eB__vXwJuANBaDUeVVG7thFH5kiU,90473
|
16
17
|
matrixone/pitr.py,sha256=NPsexPsJXIAiJTH3oMyFOwiXWjvfohPzWQyFHhQKh5k,21632
|
17
18
|
matrixone/pubsub.py,sha256=ckaG6Z3IuEtcdwp11qvB88PrCDHYymocSOJ1v4TftzY,26893
|
@@ -19,8 +20,8 @@ matrixone/restore.py,sha256=xAbsRUL-2hCk6YpuouVLbiwc3GwUNzCoIVEFtcmghkg,15791
|
|
19
20
|
matrixone/search_vector_index.py,sha256=NjQpluaPcja66herm-OYq3nNeNAqY_-FGGL4SOBOWNA,46075
|
20
21
|
matrixone/snapshot.py,sha256=cOatk26Z6Jt6aAdasOMejxxiv73S7Z2o2if_va20Ihg,18103
|
21
22
|
matrixone/sql_builder.py,sha256=ZR9TpRkV4cN_RiDJQXMLwSzodpESyxAqV-TTIr--ZbY,29009
|
22
|
-
matrixone/version.py,sha256=
|
23
|
-
matrixone/sqlalchemy_ext/__init__.py,sha256=
|
23
|
+
matrixone/version.py,sha256=7wpMFVqDjGE16bFtnvFwbcnAI3mzz2zMZPpc9v7G9-c,19913
|
24
|
+
matrixone/sqlalchemy_ext/__init__.py,sha256=YrcXthWkpN-mkRq6293t6Su5D39MdsEKYu_g1lz4qgI,4029
|
24
25
|
matrixone/sqlalchemy_ext/adapters.py,sha256=-PoQtxN8uveto4Me2BTyTq1FoeSj8qo3TkUKAhoUd4w,5286
|
25
26
|
matrixone/sqlalchemy_ext/dialect.py,sha256=wrAQDjDDpCz6HT2Atrz7QXAzbyxaNTbRXcauh2REa1A,21100
|
26
27
|
matrixone/sqlalchemy_ext/fulltext_index.py,sha256=OceS34yg1kG-1uopG1lDf7GbVyRiUnxzKHDUHYLNLuA,27614
|
@@ -30,7 +31,7 @@ matrixone/sqlalchemy_ext/ivf_config.py,sha256=-esAijWMfyF1GUJivpZh-F4lvyKRmY8LYI
|
|
30
31
|
matrixone/sqlalchemy_ext/table_builder.py,sha256=JVritBPnCXZt0eJUivDrmGdlpaYO-uL7i2n24HiarEE,12870
|
31
32
|
matrixone/sqlalchemy_ext/vector_index.py,sha256=dasW3kT4f69io9y6DLQMarvf1FTbzh0UN-HALs0kBVs,55907
|
32
33
|
matrixone/sqlalchemy_ext/vector_type.py,sha256=HaOJ9dRdW_yrecD9qGUucW9bMfM3zCxbDC-0Ca32Kmk,30669
|
33
|
-
matrixone_python_sdk-0.1.
|
34
|
+
matrixone_python_sdk-0.1.4.dist-info/licenses/LICENSE,sha256=-PpUMwDyMyFlH9H7cnzkTh0Uo42tRvz43k7hnxe7G_I,11252
|
34
35
|
tests/__init__.py,sha256=odB22tIaJIHSwRhumhDlQYD6Fug_C0opWa07dSKkeQs,694
|
35
36
|
tests/offline/__init__.py,sha256=M13mz7gtVDS0_dJUW1EFyyiAGhEj282k3ia7eWA3dPs,703
|
36
37
|
tests/offline/conftest.py,sha256=Mz_NT6GBOxqSZsSCR2SXe1pkSpLGKT2-ssWNHhh9xOg,2494
|
@@ -38,7 +39,7 @@ tests/offline/test_account.py,sha256=uCHKPOBGiJrpEwXdyF2CoNoS5yqxuyKJgLTPMvKZQbk
|
|
38
39
|
tests/offline/test_async_client_query_comprehensive.py,sha256=HmLg8iYlD6l5K4lJN5VSXhJDtKGg17vym-9KaX6WDzY,45416
|
39
40
|
tests/offline/test_basic.py,sha256=pVyPnidD5jIoC8_40U1iXND29399G-0O5e4itm-2uhc,1710
|
40
41
|
tests/offline/test_case_sensitivity.py,sha256=aYlhA5F8RSlSy6wl3HWIrPylE3Z3eAUf4RWQohwD0eA,8715
|
41
|
-
tests/offline/test_connection_hooks_offline.py,sha256=
|
42
|
+
tests/offline/test_connection_hooks_offline.py,sha256=mqnummeGGEvqeevG2R7-0VexnBquo4CJyTtmLhMa7Ps,11686
|
42
43
|
tests/offline/test_dialect_schema_handling.py,sha256=WK22Cu3cI1uZ2RtmXmkwnWrYXrwMdcXkhqHTwp0Bqqo,24380
|
43
44
|
tests/offline/test_explain_methods.py,sha256=QGSwqGGulZ-8e0pnshQ9A0JwC-CQZ6GcE7mkp_LIYIw,12683
|
44
45
|
tests/offline/test_filter_logical_in.py,sha256=Qb-bJiy5lhQaVlTeVNVZ75v6Xyx4Qd2IS_faaxC4VUg,8967
|
@@ -77,6 +78,7 @@ tests/online/test_account_management.py,sha256=mWIF3kPaegcb-JEhHBM9LrSae2xZ3ppjk
|
|
77
78
|
tests/online/test_advanced_features.py,sha256=f0oOe1bB6dT4bXZqx4LzrWd4ljwwcr9iIZLDHnTdPhU,13641
|
78
79
|
tests/online/test_async_client_interfaces.py,sha256=ZT2kWm-ZT_O-_ZkGE7mVLE5c52_eRsbCC_wEksrUis0,12295
|
79
80
|
tests/online/test_async_client_online.py,sha256=LZp0QXk4l4FfuQOHNNg__degRTwh9K61StFtEmeUmZ8,10906
|
81
|
+
tests/online/test_async_index_verification_online.py,sha256=zevl8BKAIGRyJ0ndnUmW_oQpC_rc2_W0ATPb152OMag,10162
|
80
82
|
tests/online/test_async_model_insert_online.py,sha256=YPlcNjuaeimvG355JjjgyZUPfa9J0HtC5FVJr9uIp_k,10857
|
81
83
|
tests/online/test_async_orm_online.py,sha256=8wjQb9oXMyNh4HQ1YkwiFYfHo73_YVPicOR_aiXrbEg,11039
|
82
84
|
tests/online/test_async_simple_query_online.py,sha256=nBMsCv-GaEiSRxh1juLNCiXNNsfUwWRApQvStzLeGr4,31021
|
@@ -91,6 +93,7 @@ tests/online/test_filter_logical_in_online.py,sha256=-OrmHSMyk6nYap5OCICaImt9AQ5
|
|
91
93
|
tests/online/test_fulltext_comprehensive.py,sha256=lJ_XMkZnHe1HPsbXS9O5F9tBBsPKTBNUnx78KgKTv3w,74063
|
92
94
|
tests/online/test_fulltext_label_online.py,sha256=1ExVrGIOA7dfG6UiB7Ul6xeLrLY9u6oAjjT4hYNtr8E,16534
|
93
95
|
tests/online/test_fulltext_search_online.py,sha256=711WdigiDR71nwtqLlVl39_lNocbWlNnJR5W6hBhl50,33660
|
96
|
+
tests/online/test_index_verification_online.py,sha256=Uz8Gjvjm2SdNhpXeun_Y8KOQ1VOs601WLMmxWTMzNa0,7510
|
94
97
|
tests/online/test_ivf_stats_online.py,sha256=Fdi_-bleJVXtWzHNfeA6eCgVdlvSvpEfop7GGUZTQ9E,19845
|
95
98
|
tests/online/test_logger_integration.py,sha256=iv0TrKZGRg-oxjbp55fYixbfpdqLFWQDR8mXeaSKArg,11824
|
96
99
|
tests/online/test_matrixone_query_orm.py,sha256=WYH6HxcfnNqtQ05kKHRcmwmWHXtDz8Ts0pUwuSdgo_U,20130
|
@@ -115,8 +118,8 @@ tests/online/test_transaction_query_methods.py,sha256=VaPvHOYskwJVyM7lvVvBJ6Hqhd
|
|
115
118
|
tests/online/test_unified_filter_online.py,sha256=86JwEKUaBLNuLGJlR4E0zjzdWL5xVFyYLvyxbzkdEVI,21214
|
116
119
|
tests/online/test_vector_comprehensive.py,sha256=vmnwwVXLA7lUI_zSK3fJcf1HKx2AvrVDcUT0_d-gQwg,26980
|
117
120
|
tests/online/test_version_management.py,sha256=BU8Vc1fDKNCwhRliZi6XmEnd0HYdHuki9Xxi09vnriA,11416
|
118
|
-
matrixone_python_sdk-0.1.
|
119
|
-
matrixone_python_sdk-0.1.
|
120
|
-
matrixone_python_sdk-0.1.
|
121
|
-
matrixone_python_sdk-0.1.
|
122
|
-
matrixone_python_sdk-0.1.
|
121
|
+
matrixone_python_sdk-0.1.4.dist-info/METADATA,sha256=EOKIuyVQVsFc316C84HjVLJYxD-7fMtq7a307ypoJ54,20815
|
122
|
+
matrixone_python_sdk-0.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
123
|
+
matrixone_python_sdk-0.1.4.dist-info/entry_points.txt,sha256=4wUGPC_7_f5ZDR33JRo1LZmmTuyfkYAv41_5H5Qy-Ik,138
|
124
|
+
matrixone_python_sdk-0.1.4.dist-info/top_level.txt,sha256=LQZabpBx_dtQk8JbKeH3MbjmC8HYDLE8UQeEf6NfQJA,16
|
125
|
+
matrixone_python_sdk-0.1.4.dist-info/RECORD,,
|
@@ -106,7 +106,7 @@ class TestConnectionHook:
|
|
106
106
|
# Verify that cursor.execute was called with the correct SQL
|
107
107
|
self.mock_cursor.execute.assert_called_with("SET experimental_ivf_index = 1")
|
108
108
|
self.mock_cursor.close.assert_called_once()
|
109
|
-
self.mock_client.logger.
|
109
|
+
self.mock_client.logger.debug.assert_called_with("✓ Enabled IVF vector operations")
|
110
110
|
|
111
111
|
def test_execute_sync_with_hnsw_action(self):
|
112
112
|
"""Test synchronous execution with HNSW action"""
|
@@ -117,7 +117,7 @@ class TestConnectionHook:
|
|
117
117
|
# Verify that cursor.execute was called with the correct SQL
|
118
118
|
self.mock_cursor.execute.assert_called_with("SET experimental_hnsw_index = 1")
|
119
119
|
self.mock_cursor.close.assert_called_once()
|
120
|
-
self.mock_client.logger.
|
120
|
+
self.mock_client.logger.debug.assert_called_with("✓ Enabled HNSW vector operations")
|
121
121
|
|
122
122
|
def test_execute_sync_with_fulltext_action(self):
|
123
123
|
"""Test synchronous execution with fulltext action"""
|
@@ -128,7 +128,7 @@ class TestConnectionHook:
|
|
128
128
|
# Verify that cursor.execute was called with the correct SQL
|
129
129
|
self.mock_cursor.execute.assert_called_with("SET experimental_fulltext_index = 1")
|
130
130
|
self.mock_cursor.close.assert_called_once()
|
131
|
-
self.mock_client.logger.
|
131
|
+
self.mock_client.logger.debug.assert_called_with("✓ Enabled fulltext search operations")
|
132
132
|
|
133
133
|
def test_execute_sync_with_vector_action(self):
|
134
134
|
"""Test synchronous execution with vector action (enables both IVF and HNSW)"""
|
@@ -140,8 +140,8 @@ class TestConnectionHook:
|
|
140
140
|
expected_calls = [call("SET experimental_ivf_index = 1"), call("SET experimental_hnsw_index = 1")]
|
141
141
|
self.mock_cursor.execute.assert_has_calls(expected_calls, any_order=True)
|
142
142
|
self.mock_cursor.close.assert_called()
|
143
|
-
self.mock_client.logger.
|
144
|
-
self.mock_client.logger.
|
143
|
+
self.mock_client.logger.debug.assert_any_call("✓ Enabled IVF vector operations")
|
144
|
+
self.mock_client.logger.debug.assert_any_call("✓ Enabled HNSW vector operations")
|
145
145
|
|
146
146
|
def test_execute_sync_with_all_action(self):
|
147
147
|
"""Test synchronous execution with all action"""
|
@@ -157,9 +157,9 @@ class TestConnectionHook:
|
|
157
157
|
]
|
158
158
|
self.mock_cursor.execute.assert_has_calls(expected_calls, any_order=True)
|
159
159
|
self.mock_cursor.close.assert_called()
|
160
|
-
self.mock_client.logger.
|
161
|
-
self.mock_client.logger.
|
162
|
-
self.mock_client.logger.
|
160
|
+
self.mock_client.logger.debug.assert_any_call("✓ Enabled IVF vector operations")
|
161
|
+
self.mock_client.logger.debug.assert_any_call("✓ Enabled HNSW vector operations")
|
162
|
+
self.mock_client.logger.debug.assert_any_call("✓ Enabled fulltext search operations")
|
163
163
|
|
164
164
|
def test_execute_sync_with_custom_hook(self):
|
165
165
|
"""Test synchronous execution with custom hook"""
|
@@ -0,0 +1,259 @@
|
|
1
|
+
# Copyright 2021 - 2022 Matrix Origin
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
"""
|
16
|
+
Online tests for async secondary index verification functionality
|
17
|
+
"""
|
18
|
+
|
19
|
+
import pytest
|
20
|
+
import pytest_asyncio
|
21
|
+
import os
|
22
|
+
import sys
|
23
|
+
|
24
|
+
# Add the matrixone package to the path
|
25
|
+
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
|
26
|
+
|
27
|
+
from matrixone import AsyncClient
|
28
|
+
from matrixone.orm import declarative_base
|
29
|
+
from sqlalchemy import Column, String, Integer, Index
|
30
|
+
from .test_config import online_config
|
31
|
+
|
32
|
+
|
33
|
+
class TestAsyncIndexVerificationOnline:
|
34
|
+
"""Online tests for async secondary index verification"""
|
35
|
+
|
36
|
+
@pytest_asyncio.fixture(scope="function")
|
37
|
+
async def async_client_with_test_table(self):
|
38
|
+
"""Create AsyncClient with test table"""
|
39
|
+
host, port, user, password, database = online_config.get_connection_params()
|
40
|
+
client = AsyncClient()
|
41
|
+
await client.connect(host=host, port=port, user=user, password=password, database=database)
|
42
|
+
|
43
|
+
test_table = "test_async_index_table"
|
44
|
+
|
45
|
+
try:
|
46
|
+
# Define ORM model with secondary indexes
|
47
|
+
Base = declarative_base()
|
48
|
+
|
49
|
+
class TestTable(Base):
|
50
|
+
__tablename__ = test_table
|
51
|
+
|
52
|
+
id = Column(Integer, primary_key=True)
|
53
|
+
name = Column(String(100))
|
54
|
+
category = Column(String(50))
|
55
|
+
value = Column(Integer)
|
56
|
+
|
57
|
+
# Define secondary indexes
|
58
|
+
__table_args__ = (
|
59
|
+
Index('idx_async_name', 'name'),
|
60
|
+
Index('idx_async_category', 'category'),
|
61
|
+
Index('idx_async_value', 'value'),
|
62
|
+
)
|
63
|
+
|
64
|
+
# Drop table if exists and create new one using SDK API
|
65
|
+
try:
|
66
|
+
await client.drop_table(TestTable)
|
67
|
+
except:
|
68
|
+
pass
|
69
|
+
|
70
|
+
# Create table with indexes using SDK API
|
71
|
+
await client.create_table(TestTable)
|
72
|
+
|
73
|
+
# Insert test data using SDK API
|
74
|
+
test_data = [
|
75
|
+
{'id': i, 'name': f'async_name_{i}', 'category': f'cat_{i % 5}', 'value': i * 10} for i in range(1, 101)
|
76
|
+
]
|
77
|
+
await client.batch_insert(TestTable, test_data)
|
78
|
+
|
79
|
+
yield client, test_table, TestTable
|
80
|
+
|
81
|
+
finally:
|
82
|
+
# Clean up using SDK API
|
83
|
+
try:
|
84
|
+
await client.drop_table(TestTable)
|
85
|
+
await client.disconnect()
|
86
|
+
except Exception as e:
|
87
|
+
print(f"Cleanup failed: {e}")
|
88
|
+
|
89
|
+
@pytest.mark.asyncio
|
90
|
+
async def test_async_get_secondary_index_tables(self, async_client_with_test_table):
|
91
|
+
"""Test async getting all secondary index table names"""
|
92
|
+
client, test_table, TestTable = async_client_with_test_table
|
93
|
+
|
94
|
+
index_tables = await client.get_secondary_index_tables(test_table)
|
95
|
+
|
96
|
+
# Should have 3 secondary indexes
|
97
|
+
assert len(index_tables) == 3, f"Expected 3 indexes, got {len(index_tables)}"
|
98
|
+
|
99
|
+
# All index tables should start with __mo_index_secondary_
|
100
|
+
for index_table in index_tables:
|
101
|
+
assert index_table.startswith(
|
102
|
+
'__mo_index_secondary_'
|
103
|
+
), f"Index table name should start with '__mo_index_secondary_', got {index_table}"
|
104
|
+
|
105
|
+
|
106
|
+
class TestAsyncIndexVerificationOnline:
|
107
|
+
"""Online tests for async secondary index verification"""
|
108
|
+
|
109
|
+
@pytest_asyncio.fixture(scope="function")
|
110
|
+
async def async_client_with_test_table(self):
|
111
|
+
"""Create AsyncClient with test table"""
|
112
|
+
host, port, user, password, database = online_config.get_connection_params()
|
113
|
+
client = AsyncClient()
|
114
|
+
await client.connect(host=host, port=port, user=user, password=password, database=database)
|
115
|
+
|
116
|
+
test_table = "test_async_index_table"
|
117
|
+
|
118
|
+
try:
|
119
|
+
# Define ORM model with secondary indexes
|
120
|
+
Base = declarative_base()
|
121
|
+
|
122
|
+
class TestTable(Base):
|
123
|
+
__tablename__ = test_table
|
124
|
+
|
125
|
+
id = Column(Integer, primary_key=True)
|
126
|
+
name = Column(String(100))
|
127
|
+
category = Column(String(50))
|
128
|
+
value = Column(Integer)
|
129
|
+
|
130
|
+
# Define secondary indexes
|
131
|
+
__table_args__ = (
|
132
|
+
Index('idx_async_name', 'name'),
|
133
|
+
Index('idx_async_category', 'category'),
|
134
|
+
Index('idx_async_value', 'value'),
|
135
|
+
)
|
136
|
+
|
137
|
+
# Drop table if exists and create new one using SDK API
|
138
|
+
try:
|
139
|
+
await client.drop_table(TestTable)
|
140
|
+
except:
|
141
|
+
pass
|
142
|
+
|
143
|
+
# Create table with indexes using SDK API
|
144
|
+
await client.create_table(TestTable)
|
145
|
+
|
146
|
+
# Insert test data using SDK API
|
147
|
+
test_data = [
|
148
|
+
{'id': i, 'name': f'async_name_{i}', 'category': f'cat_{i % 5}', 'value': i * 10} for i in range(1, 101)
|
149
|
+
]
|
150
|
+
await client.batch_insert(TestTable, test_data)
|
151
|
+
|
152
|
+
yield client, test_table, TestTable
|
153
|
+
|
154
|
+
finally:
|
155
|
+
# Clean up using SDK API
|
156
|
+
try:
|
157
|
+
await client.drop_table(TestTable)
|
158
|
+
await client.disconnect()
|
159
|
+
except Exception as e:
|
160
|
+
print(f"Cleanup failed: {e}")
|
161
|
+
|
162
|
+
@pytest.mark.asyncio
|
163
|
+
async def test_async_get_secondary_index_table_by_name(self, async_client_with_test_table):
|
164
|
+
"""Test async getting physical table name of a secondary index by its name"""
|
165
|
+
client, test_table, TestTable = async_client_with_test_table
|
166
|
+
|
167
|
+
# Test getting idx_async_name
|
168
|
+
idx_name_table = await client.get_secondary_index_table_by_name(test_table, 'idx_async_name')
|
169
|
+
assert idx_name_table is not None, "idx_async_name should exist"
|
170
|
+
assert idx_name_table.startswith('__mo_index_secondary_')
|
171
|
+
|
172
|
+
# Test getting idx_async_category
|
173
|
+
idx_category_table = await client.get_secondary_index_table_by_name(test_table, 'idx_async_category')
|
174
|
+
assert idx_category_table is not None, "idx_async_category should exist"
|
175
|
+
assert idx_category_table.startswith('__mo_index_secondary_')
|
176
|
+
|
177
|
+
# Test getting idx_async_value
|
178
|
+
idx_value_table = await client.get_secondary_index_table_by_name(test_table, 'idx_async_value')
|
179
|
+
assert idx_value_table is not None, "idx_async_value should exist"
|
180
|
+
assert idx_value_table.startswith('__mo_index_secondary_')
|
181
|
+
|
182
|
+
# Test non-existent index
|
183
|
+
non_existent = await client.get_secondary_index_table_by_name(test_table, 'idx_non_existent')
|
184
|
+
assert non_existent is None, "Non-existent index should return None"
|
185
|
+
|
186
|
+
@pytest.mark.asyncio
|
187
|
+
async def test_async_verify_table_index_counts(self, async_client_with_test_table):
|
188
|
+
"""Test async verify_table_index_counts when counts match"""
|
189
|
+
client, test_table, TestTable = async_client_with_test_table
|
190
|
+
|
191
|
+
# All indexes should have the same count as the main table
|
192
|
+
count = await client.verify_table_index_counts(test_table)
|
193
|
+
|
194
|
+
# Should return 100 (number of inserted rows)
|
195
|
+
assert count == 100, f"Expected count 100, got {count}"
|
196
|
+
|
197
|
+
# Verify count matches query result using SDK API
|
198
|
+
actual_count = await client.query(TestTable).count()
|
199
|
+
assert count == actual_count, "Verify result should match actual count"
|
200
|
+
|
201
|
+
@pytest.mark.asyncio
|
202
|
+
async def test_async_verify_table_without_indexes(self, async_client_with_test_table):
|
203
|
+
"""Test async verify_table_index_counts on table without secondary indexes"""
|
204
|
+
client, test_table, TestTable = async_client_with_test_table
|
205
|
+
|
206
|
+
# Define a simple model without secondary indexes
|
207
|
+
Base = declarative_base()
|
208
|
+
|
209
|
+
class SimpleTable(Base):
|
210
|
+
__tablename__ = "test_async_simple_table"
|
211
|
+
id = Column(Integer, primary_key=True)
|
212
|
+
value = Column(Integer)
|
213
|
+
|
214
|
+
# Create table using SDK API
|
215
|
+
try:
|
216
|
+
await client.drop_table(SimpleTable)
|
217
|
+
except:
|
218
|
+
pass
|
219
|
+
|
220
|
+
await client.create_table(SimpleTable)
|
221
|
+
|
222
|
+
# Insert data using SDK API
|
223
|
+
test_data = [{'id': 1, 'value': 100}, {'id': 2, 'value': 200}]
|
224
|
+
await client.batch_insert(SimpleTable, test_data)
|
225
|
+
|
226
|
+
# Verification should succeed and return count
|
227
|
+
count = await client.verify_table_index_counts("test_async_simple_table")
|
228
|
+
assert count == 2, f"Expected count 2, got {count}"
|
229
|
+
|
230
|
+
# Cleanup using SDK API
|
231
|
+
await client.drop_table(SimpleTable)
|
232
|
+
|
233
|
+
@pytest.mark.asyncio
|
234
|
+
async def test_async_index_table_mapping(self, async_client_with_test_table):
|
235
|
+
"""Test that all indexes can be retrieved both ways (async)"""
|
236
|
+
client, test_table, TestTable = async_client_with_test_table
|
237
|
+
|
238
|
+
# Get all index tables
|
239
|
+
all_index_tables = await client.get_secondary_index_tables(test_table)
|
240
|
+
|
241
|
+
# Get each index by name
|
242
|
+
index_names = ['idx_async_name', 'idx_async_category', 'idx_async_value']
|
243
|
+
retrieved_tables = []
|
244
|
+
|
245
|
+
for index_name in index_names:
|
246
|
+
table = await client.get_secondary_index_table_by_name(test_table, index_name)
|
247
|
+
assert table is not None, f"Index {index_name} should exist"
|
248
|
+
retrieved_tables.append(table)
|
249
|
+
|
250
|
+
# All retrieved tables should be in the all_index_tables list
|
251
|
+
for table in retrieved_tables:
|
252
|
+
assert table in all_index_tables, f"Retrieved table {table} should be in all index tables"
|
253
|
+
|
254
|
+
# Both lists should have the same length
|
255
|
+
assert len(retrieved_tables) == len(all_index_tables), "Number of retrieved tables should match total index tables"
|
256
|
+
|
257
|
+
|
258
|
+
if __name__ == '__main__':
|
259
|
+
pytest.main([__file__, '-v'])
|
@@ -0,0 +1,194 @@
|
|
1
|
+
# Copyright 2021 - 2022 Matrix Origin
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
"""
|
16
|
+
Online tests for secondary index verification functionality
|
17
|
+
"""
|
18
|
+
|
19
|
+
import unittest
|
20
|
+
import os
|
21
|
+
import sys
|
22
|
+
|
23
|
+
# Add the matrixone package to the path
|
24
|
+
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
|
25
|
+
|
26
|
+
from matrixone import Client
|
27
|
+
from matrixone.orm import declarative_base
|
28
|
+
from sqlalchemy import Column, String, Integer, Index
|
29
|
+
|
30
|
+
|
31
|
+
class TestIndexVerificationOnline(unittest.TestCase):
|
32
|
+
"""Online tests for secondary index verification"""
|
33
|
+
|
34
|
+
@classmethod
|
35
|
+
def setUpClass(cls):
|
36
|
+
"""Set up test database connection"""
|
37
|
+
cls.client = Client(
|
38
|
+
host=os.getenv('MATRIXONE_HOST', '127.0.0.1'),
|
39
|
+
port=int(os.getenv('MATRIXONE_PORT', '6001')),
|
40
|
+
user=os.getenv('MATRIXONE_USER', 'root'),
|
41
|
+
password=os.getenv('MATRIXONE_PASSWORD', '111'),
|
42
|
+
database=os.getenv('MATRIXONE_DATABASE', 'test'),
|
43
|
+
)
|
44
|
+
|
45
|
+
cls.test_table = "test_index_table"
|
46
|
+
|
47
|
+
try:
|
48
|
+
# Define ORM model with secondary indexes
|
49
|
+
Base = declarative_base()
|
50
|
+
|
51
|
+
class TestTable(Base):
|
52
|
+
__tablename__ = cls.test_table
|
53
|
+
|
54
|
+
id = Column(Integer, primary_key=True)
|
55
|
+
name = Column(String(100))
|
56
|
+
category = Column(String(50))
|
57
|
+
value = Column(Integer)
|
58
|
+
|
59
|
+
# Define secondary indexes
|
60
|
+
__table_args__ = (
|
61
|
+
Index('idx_name', 'name'),
|
62
|
+
Index('idx_category', 'category'),
|
63
|
+
Index('idx_value', 'value'),
|
64
|
+
)
|
65
|
+
|
66
|
+
cls.TestTable = TestTable
|
67
|
+
|
68
|
+
# Drop table if exists and create new one
|
69
|
+
try:
|
70
|
+
cls.client.drop_table(TestTable)
|
71
|
+
except:
|
72
|
+
pass
|
73
|
+
|
74
|
+
# Create table with indexes using SDK API
|
75
|
+
cls.client.create_table(TestTable)
|
76
|
+
|
77
|
+
# Insert test data using SDK API
|
78
|
+
test_data = [{'id': i, 'name': f'name_{i}', 'category': f'cat_{i % 5}', 'value': i * 10} for i in range(1, 101)]
|
79
|
+
cls.client.batch_insert(TestTable, test_data)
|
80
|
+
|
81
|
+
except Exception as e:
|
82
|
+
print(f"Setup failed: {e}")
|
83
|
+
raise
|
84
|
+
|
85
|
+
@classmethod
|
86
|
+
def tearDownClass(cls):
|
87
|
+
"""Clean up test database"""
|
88
|
+
try:
|
89
|
+
cls.client.drop_table(cls.TestTable)
|
90
|
+
except Exception as e:
|
91
|
+
print(f"Cleanup failed: {e}")
|
92
|
+
|
93
|
+
def test_get_secondary_index_tables(self):
|
94
|
+
"""Test getting all secondary index table names"""
|
95
|
+
index_tables = self.client.get_secondary_index_tables(self.test_table)
|
96
|
+
|
97
|
+
# Should have 3 secondary indexes
|
98
|
+
self.assertEqual(len(index_tables), 3, f"Expected 3 indexes, got {len(index_tables)}")
|
99
|
+
|
100
|
+
# All index tables should start with __mo_index_secondary_
|
101
|
+
for index_table in index_tables:
|
102
|
+
self.assertTrue(
|
103
|
+
index_table.startswith('__mo_index_secondary_'),
|
104
|
+
f"Index table name should start with '__mo_index_secondary_', got {index_table}",
|
105
|
+
)
|
106
|
+
|
107
|
+
def test_get_secondary_index_table_by_name(self):
|
108
|
+
"""Test getting physical table name of a secondary index by its name"""
|
109
|
+
# Test getting idx_name
|
110
|
+
idx_name_table = self.client.get_secondary_index_table_by_name(self.test_table, 'idx_name')
|
111
|
+
self.assertIsNotNone(idx_name_table, "idx_name should exist")
|
112
|
+
self.assertTrue(idx_name_table.startswith('__mo_index_secondary_'))
|
113
|
+
|
114
|
+
# Test getting idx_category
|
115
|
+
idx_category_table = self.client.get_secondary_index_table_by_name(self.test_table, 'idx_category')
|
116
|
+
self.assertIsNotNone(idx_category_table, "idx_category should exist")
|
117
|
+
self.assertTrue(idx_category_table.startswith('__mo_index_secondary_'))
|
118
|
+
|
119
|
+
# Test getting idx_value
|
120
|
+
idx_value_table = self.client.get_secondary_index_table_by_name(self.test_table, 'idx_value')
|
121
|
+
self.assertIsNotNone(idx_value_table, "idx_value should exist")
|
122
|
+
self.assertTrue(idx_value_table.startswith('__mo_index_secondary_'))
|
123
|
+
|
124
|
+
# Test non-existent index
|
125
|
+
non_existent = self.client.get_secondary_index_table_by_name(self.test_table, 'idx_non_existent')
|
126
|
+
self.assertIsNone(non_existent, "Non-existent index should return None")
|
127
|
+
|
128
|
+
def test_verify_table_index_counts(self):
|
129
|
+
"""Test verify_table_index_counts when counts match"""
|
130
|
+
# All indexes should have the same count as the main table
|
131
|
+
count = self.client.verify_table_index_counts(self.test_table)
|
132
|
+
|
133
|
+
# Should return 100 (number of inserted rows)
|
134
|
+
self.assertEqual(count, 100, f"Expected count 100, got {count}")
|
135
|
+
|
136
|
+
# Verify count matches query result using SDK API
|
137
|
+
actual_count = self.client.query(self.TestTable).count()
|
138
|
+
self.assertEqual(count, actual_count, "Verify result should match actual count")
|
139
|
+
|
140
|
+
def test_verify_table_without_indexes(self):
|
141
|
+
"""Test verify_table_index_counts on table without secondary indexes"""
|
142
|
+
# Define a simple model without secondary indexes
|
143
|
+
Base = declarative_base()
|
144
|
+
|
145
|
+
class SimpleTable(Base):
|
146
|
+
__tablename__ = "test_simple_table"
|
147
|
+
id = Column(Integer, primary_key=True)
|
148
|
+
value = Column(Integer)
|
149
|
+
|
150
|
+
# Create table using SDK API
|
151
|
+
try:
|
152
|
+
self.client.drop_table(SimpleTable)
|
153
|
+
except:
|
154
|
+
pass
|
155
|
+
|
156
|
+
self.client.create_table(SimpleTable)
|
157
|
+
|
158
|
+
# Insert data using SDK API
|
159
|
+
test_data = [{'id': 1, 'value': 100}, {'id': 2, 'value': 200}]
|
160
|
+
self.client.batch_insert(SimpleTable, test_data)
|
161
|
+
|
162
|
+
# Verification should succeed and return count
|
163
|
+
count = self.client.verify_table_index_counts("test_simple_table")
|
164
|
+
self.assertEqual(count, 2, f"Expected count 2, got {count}")
|
165
|
+
|
166
|
+
# Cleanup using SDK API
|
167
|
+
self.client.drop_table(SimpleTable)
|
168
|
+
|
169
|
+
def test_index_table_mapping(self):
|
170
|
+
"""Test that all indexes can be retrieved both ways"""
|
171
|
+
# Get all index tables
|
172
|
+
all_index_tables = self.client.get_secondary_index_tables(self.test_table)
|
173
|
+
|
174
|
+
# Get each index by name
|
175
|
+
index_names = ['idx_name', 'idx_category', 'idx_value']
|
176
|
+
retrieved_tables = []
|
177
|
+
|
178
|
+
for index_name in index_names:
|
179
|
+
table = self.client.get_secondary_index_table_by_name(self.test_table, index_name)
|
180
|
+
self.assertIsNotNone(table, f"Index {index_name} should exist")
|
181
|
+
retrieved_tables.append(table)
|
182
|
+
|
183
|
+
# All retrieved tables should be in the all_index_tables list
|
184
|
+
for table in retrieved_tables:
|
185
|
+
self.assertIn(table, all_index_tables, f"Retrieved table {table} should be in all index tables")
|
186
|
+
|
187
|
+
# Both lists should have the same length
|
188
|
+
self.assertEqual(
|
189
|
+
len(retrieved_tables), len(all_index_tables), "Number of retrieved tables should match total index tables"
|
190
|
+
)
|
191
|
+
|
192
|
+
|
193
|
+
if __name__ == '__main__':
|
194
|
+
unittest.main()
|
File without changes
|
{matrixone_python_sdk-0.1.2.dist-info → matrixone_python_sdk-0.1.4.dist-info}/entry_points.txt
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.2.dist-info → matrixone_python_sdk-0.1.4.dist-info}/licenses/LICENSE
RENAMED
File without changes
|
File without changes
|