matrixone-python-sdk 0.1.1__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {matrixone_python_sdk-0.1.1/matrixone_python_sdk.egg-info → matrixone_python_sdk-0.1.3}/PKG-INFO +1 -1
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/README.md +25 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/async_client.py +94 -3
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/async_metadata_manager.py +3 -1
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/async_vector_index_manager.py +3 -1
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/client.py +116 -3
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/connection_hooks.py +1 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/exceptions.py +0 -24
- matrixone_python_sdk-0.1.3/matrixone/index_utils.py +121 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/metadata.py +3 -2
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/moctl.py +0 -2
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/__init__.py +1 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/vector_index.py +28 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/version.py +0 -2
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3/matrixone_python_sdk.egg-info}/PKG-INFO +1 -1
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone_python_sdk.egg-info/SOURCES.txt +1 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/pyproject.toml +1 -1
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/LICENSE +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/MANIFEST.in +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/README_USER.md +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_01_basic_connection.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_02_account_management.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_03_async_operations.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_04_transaction_management.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_05_snapshot_restore.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_06_sqlalchemy_integration.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_07_advanced_features.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_08_pubsub_operations.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_09_logger_integration.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_10_version_management.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_11_matrixone_version_demo.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_12_vector_basics.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_13_vector_indexes.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_14_vector_search.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_15_vector_advanced.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_18_snapshot_orm.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_19_sqlalchemy_style_orm.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_20_sqlalchemy_engine_integration.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_21_advanced_orm_features.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_22_unified_sql_builder.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_24_query_update.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_25_metadata_operations.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_connection_hooks.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_dynamic_logging.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_ivf_stats_complete.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/__init__.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/account.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/async_orm.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/base_client.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/config.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/logger.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/orm.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/pitr.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/pubsub.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/restore.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/search_vector_index.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/snapshot.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sql_builder.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/adapters.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/dialect.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/fulltext_index.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/fulltext_search.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/hnsw_config.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/ivf_config.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/table_builder.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/vector_type.py +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone_python_sdk.egg-info/dependency_links.txt +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone_python_sdk.egg-info/entry_points.txt +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone_python_sdk.egg-info/not-zip-safe +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone_python_sdk.egg-info/requires.txt +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone_python_sdk.egg-info/top_level.txt +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/requirements.txt +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/setup.cfg +0 -0
- {matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/setup.py +0 -0
{matrixone_python_sdk-0.1.1/matrixone_python_sdk.egg-info → matrixone_python_sdk-0.1.3}/PKG-INFO
RENAMED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: matrixone-python-sdk
|
3
|
-
Version: 0.1.
|
3
|
+
Version: 0.1.3
|
4
4
|
Summary: A comprehensive Python SDK for MatrixOne database operations with vector search, fulltext search, and advanced features
|
5
5
|
Home-page: https://github.com/matrixorigin/matrixone
|
6
6
|
Author: MatrixOne Team
|
@@ -31,6 +31,10 @@ A comprehensive Python SDK for MatrixOne that provides SQLAlchemy-like interface
|
|
31
31
|
- Table size and row count statistics
|
32
32
|
- Column data distribution analysis
|
33
33
|
- Index usage metrics
|
34
|
+
- 🔍 **Secondary Index Verification**: Verify consistency of secondary indexes with main table
|
35
|
+
- Get all secondary index table names
|
36
|
+
- Get specific index table by index name
|
37
|
+
- Verify row counts across main table and all indexes in single query
|
34
38
|
- 📸 **Snapshot Management**: Create and manage database snapshots at multiple levels
|
35
39
|
- ⏰ **Point-in-Time Recovery**: PITR functionality for precise data recovery
|
36
40
|
- 🔄 **Table Cloning**: Clone databases and tables efficiently with data replication
|
@@ -323,6 +327,27 @@ print(f"Original size: {table_stats['original_size']} bytes")
|
|
323
327
|
print(f"Compressed size: {table_stats['compress_size']} bytes")
|
324
328
|
```
|
325
329
|
|
330
|
+
### Secondary Index Verification
|
331
|
+
|
332
|
+
```python
|
333
|
+
# Get all secondary index tables
|
334
|
+
index_tables = client.get_secondary_index_tables('my_table')
|
335
|
+
print(f"Found {len(index_tables)} secondary indexes")
|
336
|
+
|
337
|
+
# Get specific index by name
|
338
|
+
physical_table = client.get_secondary_index_table_by_name('my_table', 'idx_name')
|
339
|
+
print(f"Index 'idx_name' -> {physical_table}")
|
340
|
+
|
341
|
+
# Verify all indexes have same count as main table
|
342
|
+
try:
|
343
|
+
count = client.verify_table_index_counts('my_table')
|
344
|
+
print(f"✓ Verification passed! Row count: {count}")
|
345
|
+
except ValueError as e:
|
346
|
+
print(f"✗ Index mismatch detected:")
|
347
|
+
print(e)
|
348
|
+
# Error includes all count details for debugging
|
349
|
+
```
|
350
|
+
|
326
351
|
### Version Management
|
327
352
|
|
328
353
|
```python
|
@@ -34,9 +34,10 @@ from datetime import datetime
|
|
34
34
|
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
35
35
|
|
36
36
|
from .account import Account, User
|
37
|
+
from .async_metadata_manager import AsyncMetadataManager
|
37
38
|
from .async_vector_index_manager import AsyncVectorManager
|
38
39
|
from .base_client import BaseMatrixOneClient, BaseMatrixOneExecutor
|
39
|
-
from .connection_hooks import
|
40
|
+
from .connection_hooks import ConnectionAction, ConnectionHook, create_connection_hook
|
40
41
|
from .exceptions import (
|
41
42
|
AccountError,
|
42
43
|
ConnectionError,
|
@@ -48,7 +49,6 @@ from .exceptions import (
|
|
48
49
|
SnapshotError,
|
49
50
|
)
|
50
51
|
from .logger import MatrixOneLogger, create_default_logger
|
51
|
-
from .async_metadata_manager import AsyncMetadataManager
|
52
52
|
from .pitr import Pitr
|
53
53
|
from .pubsub import Publication, Subscription
|
54
54
|
from .snapshot import Snapshot, SnapshotLevel
|
@@ -1837,7 +1837,6 @@ class AsyncClient(BaseMatrixOneClient):
|
|
1837
1837
|
"""Cleanup when object is garbage collected"""
|
1838
1838
|
# Don't try to cleanup in __del__ as it can cause issues with event loops
|
1839
1839
|
# The fixture should handle proper cleanup
|
1840
|
-
pass
|
1841
1840
|
|
1842
1841
|
def get_sqlalchemy_engine(self) -> AsyncEngine:
|
1843
1842
|
"""
|
@@ -1920,6 +1919,7 @@ class AsyncClient(BaseMatrixOneClient):
|
|
1920
1919
|
and other SDK components. External users should use execute() instead.
|
1921
1920
|
"""
|
1922
1921
|
import time
|
1922
|
+
|
1923
1923
|
from sqlalchemy import text
|
1924
1924
|
|
1925
1925
|
start_time = time.time()
|
@@ -2554,6 +2554,97 @@ class AsyncClient(BaseMatrixOneClient):
|
|
2554
2554
|
except Exception as e:
|
2555
2555
|
raise QueryError(f"Failed to get git version: {e}")
|
2556
2556
|
|
2557
|
+
async def get_secondary_index_tables(self, table_name: str) -> List[str]:
|
2558
|
+
"""
|
2559
|
+
Get all secondary index table names for a given table (async version).
|
2560
|
+
|
2561
|
+
Args:
|
2562
|
+
table_name: Name of the table to get secondary indexes for
|
2563
|
+
|
2564
|
+
Returns:
|
2565
|
+
List of secondary index table names
|
2566
|
+
|
2567
|
+
Examples::
|
2568
|
+
|
2569
|
+
>>> async with AsyncClient() as client:
|
2570
|
+
... await client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2571
|
+
... index_tables = await client.get_secondary_index_tables('cms_all_content_chunk_info')
|
2572
|
+
... print(index_tables)
|
2573
|
+
"""
|
2574
|
+
from .index_utils import build_get_index_tables_sql
|
2575
|
+
|
2576
|
+
sql, params = build_get_index_tables_sql(table_name)
|
2577
|
+
result = await self.execute(sql, params)
|
2578
|
+
return [row[0] for row in result.fetchall()]
|
2579
|
+
|
2580
|
+
async def get_secondary_index_table_by_name(self, table_name: str, index_name: str) -> Optional[str]:
|
2581
|
+
"""
|
2582
|
+
Get the physical table name of a secondary index by its index name (async version).
|
2583
|
+
|
2584
|
+
Args:
|
2585
|
+
table_name: Name of the table
|
2586
|
+
index_name: Name of the secondary index
|
2587
|
+
|
2588
|
+
Returns:
|
2589
|
+
Physical table name of the secondary index, or None if not found
|
2590
|
+
|
2591
|
+
Examples::
|
2592
|
+
|
2593
|
+
>>> async with AsyncClient() as client:
|
2594
|
+
... await client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2595
|
+
... index_table = await client.get_secondary_index_table_by_name('cms_all_content_chunk_info', 'cms_id')
|
2596
|
+
... print(index_table)
|
2597
|
+
"""
|
2598
|
+
from .index_utils import build_get_index_table_by_name_sql
|
2599
|
+
|
2600
|
+
sql, params = build_get_index_table_by_name_sql(table_name, index_name)
|
2601
|
+
result = await self.execute(sql, params)
|
2602
|
+
row = result.fetchone()
|
2603
|
+
return row[0] if row else None
|
2604
|
+
|
2605
|
+
async def verify_table_index_counts(self, table_name: str) -> int:
|
2606
|
+
"""
|
2607
|
+
Verify that the main table and all its secondary index tables have the same row count (async version).
|
2608
|
+
|
2609
|
+
This method compares the COUNT(*) of the main table with all its secondary index tables
|
2610
|
+
in a single SQL query for consistency. If counts don't match, raises an exception.
|
2611
|
+
|
2612
|
+
Args:
|
2613
|
+
table_name: Name of the table to verify
|
2614
|
+
|
2615
|
+
Returns:
|
2616
|
+
Row count (int) if verification succeeds
|
2617
|
+
|
2618
|
+
Raises:
|
2619
|
+
ValueError: If any secondary index table has a different count than the main table,
|
2620
|
+
with details about all counts in the error message
|
2621
|
+
|
2622
|
+
Examples::
|
2623
|
+
|
2624
|
+
>>> async with AsyncClient() as client:
|
2625
|
+
... await client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2626
|
+
... count = await client.verify_table_index_counts('cms_all_content_chunk_info')
|
2627
|
+
... print(f"✓ Verification passed, row count: {count}")
|
2628
|
+
|
2629
|
+
>>> # If verification fails:
|
2630
|
+
>>> try:
|
2631
|
+
... count = await client.verify_table_index_counts('some_table')
|
2632
|
+
... except ValueError as e:
|
2633
|
+
... print(f"Verification failed: {e}")
|
2634
|
+
"""
|
2635
|
+
from .index_utils import build_verify_counts_sql, process_verify_result
|
2636
|
+
|
2637
|
+
# Get all secondary index tables
|
2638
|
+
index_tables = await self.get_secondary_index_tables(table_name)
|
2639
|
+
|
2640
|
+
# Build and execute verification SQL
|
2641
|
+
sql = build_verify_counts_sql(table_name, index_tables)
|
2642
|
+
result = await self.execute(sql)
|
2643
|
+
row = result.fetchone()
|
2644
|
+
|
2645
|
+
# Process result and raise exception if verification fails
|
2646
|
+
return process_verify_result(table_name, index_tables, row)
|
2647
|
+
|
2557
2648
|
async def __aenter__(self):
|
2558
2649
|
return self
|
2559
2650
|
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/async_metadata_manager.py
RENAMED
@@ -19,8 +19,10 @@ This module provides async metadata scanning capabilities for MatrixOne tables,
|
|
19
19
|
allowing users to analyze table statistics, column information, and data distribution.
|
20
20
|
"""
|
21
21
|
|
22
|
-
from typing import
|
22
|
+
from typing import Any, Dict, List, Optional, Union
|
23
|
+
|
23
24
|
from sqlalchemy.engine import Result
|
25
|
+
|
24
26
|
from .metadata import BaseMetadataManager, MetadataColumn, MetadataRow
|
25
27
|
|
26
28
|
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/async_vector_index_manager.py
RENAMED
@@ -395,7 +395,9 @@ class AsyncVectorManager:
|
|
395
395
|
|
396
396
|
# Convert distance type to enum
|
397
397
|
if distance_type == "l2":
|
398
|
-
distance_func = DistanceFunction.
|
398
|
+
distance_func = DistanceFunction.L2 # Use L2 for consistency with ORM l2_distance()
|
399
|
+
elif distance_type == "l2_sq":
|
400
|
+
distance_func = DistanceFunction.L2_SQ # Explicitly use squared distance if needed
|
399
401
|
elif distance_type == "cosine":
|
400
402
|
distance_func = DistanceFunction.COSINE
|
401
403
|
elif distance_type == "inner_product":
|
@@ -30,7 +30,7 @@ from sqlalchemy.engine import Engine
|
|
30
30
|
|
31
31
|
from .account import AccountManager, TransactionAccountManager
|
32
32
|
from .base_client import BaseMatrixOneClient, BaseMatrixOneExecutor
|
33
|
-
from .connection_hooks import
|
33
|
+
from .connection_hooks import ConnectionAction, ConnectionHook, create_connection_hook
|
34
34
|
from .exceptions import ConnectionError, QueryError
|
35
35
|
from .logger import MatrixOneLogger, create_default_logger
|
36
36
|
from .metadata import MetadataManager, TransactionMetadataManager
|
@@ -1698,8 +1698,9 @@ class Client(BaseMatrixOneClient):
|
|
1698
1698
|
table_name = model_class.__tablename__
|
1699
1699
|
table = model_class.__table__
|
1700
1700
|
|
1701
|
+
from sqlalchemy.schema import CreateIndex, CreateTable
|
1702
|
+
|
1701
1703
|
from .sqlalchemy_ext import FulltextIndex, VectorIndex
|
1702
|
-
from sqlalchemy.schema import CreateTable, CreateIndex
|
1703
1704
|
|
1704
1705
|
try:
|
1705
1706
|
engine_context = self.get_sqlalchemy_engine().begin()
|
@@ -2612,6 +2613,99 @@ class Client(BaseMatrixOneClient):
|
|
2612
2613
|
|
2613
2614
|
return self
|
2614
2615
|
|
2616
|
+
def get_secondary_index_tables(self, table_name: str) -> List[str]:
|
2617
|
+
"""
|
2618
|
+
Get all secondary index table names for a given table.
|
2619
|
+
|
2620
|
+
Args:
|
2621
|
+
table_name: Name of the table to get secondary indexes for
|
2622
|
+
|
2623
|
+
Returns:
|
2624
|
+
List of secondary index table names
|
2625
|
+
|
2626
|
+
Examples::
|
2627
|
+
|
2628
|
+
>>> client = Client()
|
2629
|
+
>>> client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2630
|
+
>>> index_tables = client.get_secondary_index_tables('cms_all_content_chunk_info')
|
2631
|
+
>>> print(index_tables)
|
2632
|
+
['__mo_index_secondary_..._cms_id', '__mo_index_secondary_..._idx_all_content_length']
|
2633
|
+
"""
|
2634
|
+
from .index_utils import build_get_index_tables_sql
|
2635
|
+
|
2636
|
+
sql, params = build_get_index_tables_sql(table_name)
|
2637
|
+
result = self.execute(sql, params)
|
2638
|
+
return [row[0] for row in result.fetchall()]
|
2639
|
+
|
2640
|
+
def get_secondary_index_table_by_name(self, table_name: str, index_name: str) -> Optional[str]:
|
2641
|
+
"""
|
2642
|
+
Get the physical table name of a secondary index by its index name.
|
2643
|
+
|
2644
|
+
Args:
|
2645
|
+
table_name: Name of the table
|
2646
|
+
index_name: Name of the secondary index
|
2647
|
+
|
2648
|
+
Returns:
|
2649
|
+
Physical table name of the secondary index, or None if not found
|
2650
|
+
|
2651
|
+
Examples::
|
2652
|
+
|
2653
|
+
>>> client = Client()
|
2654
|
+
>>> client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2655
|
+
>>> index_table = client.get_secondary_index_table_by_name('cms_all_content_chunk_info', 'cms_id')
|
2656
|
+
>>> print(index_table)
|
2657
|
+
'__mo_index_secondary_018cfbda-bde1-7c3e-805c-3f8e71769f75_cms_id'
|
2658
|
+
"""
|
2659
|
+
from .index_utils import build_get_index_table_by_name_sql
|
2660
|
+
|
2661
|
+
sql, params = build_get_index_table_by_name_sql(table_name, index_name)
|
2662
|
+
result = self.execute(sql, params)
|
2663
|
+
row = result.fetchone()
|
2664
|
+
return row[0] if row else None
|
2665
|
+
|
2666
|
+
def verify_table_index_counts(self, table_name: str) -> int:
|
2667
|
+
"""
|
2668
|
+
Verify that the main table and all its secondary index tables have the same row count.
|
2669
|
+
|
2670
|
+
This method compares the COUNT(*) of the main table with all its secondary index tables
|
2671
|
+
in a single SQL query for consistency. If counts don't match, raises an exception.
|
2672
|
+
|
2673
|
+
Args:
|
2674
|
+
table_name: Name of the table to verify
|
2675
|
+
|
2676
|
+
Returns:
|
2677
|
+
Row count (int) if verification succeeds
|
2678
|
+
|
2679
|
+
Raises:
|
2680
|
+
ValueError: If any secondary index table has a different count than the main table,
|
2681
|
+
with details about all counts in the error message
|
2682
|
+
|
2683
|
+
Examples::
|
2684
|
+
|
2685
|
+
>>> client = Client()
|
2686
|
+
>>> client.connect(host='localhost', port=6001, user='root', password='111', database='test')
|
2687
|
+
>>> count = client.verify_table_index_counts('cms_all_content_chunk_info')
|
2688
|
+
>>> print(f"✓ Verification passed, row count: {count}")
|
2689
|
+
|
2690
|
+
>>> # If verification fails:
|
2691
|
+
>>> try:
|
2692
|
+
... count = client.verify_table_index_counts('some_table')
|
2693
|
+
... except ValueError as e:
|
2694
|
+
... print(f"Verification failed: {e}")
|
2695
|
+
"""
|
2696
|
+
from .index_utils import build_verify_counts_sql, process_verify_result
|
2697
|
+
|
2698
|
+
# Get all secondary index tables
|
2699
|
+
index_tables = self.get_secondary_index_tables(table_name)
|
2700
|
+
|
2701
|
+
# Build and execute verification SQL
|
2702
|
+
sql = build_verify_counts_sql(table_name, index_tables)
|
2703
|
+
result = self.execute(sql)
|
2704
|
+
row = result.fetchone()
|
2705
|
+
|
2706
|
+
# Process result and raise exception if verification fails
|
2707
|
+
return process_verify_result(table_name, index_tables, row)
|
2708
|
+
|
2615
2709
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
2616
2710
|
self.disconnect()
|
2617
2711
|
|
@@ -3632,6 +3726,12 @@ class VectorManager:
|
|
3632
3726
|
if not success:
|
3633
3727
|
raise Exception(f"Failed to create IVFFLAT vector index {name} on table {table_name}")
|
3634
3728
|
|
3729
|
+
# Add summary log
|
3730
|
+
op_type_str = op_type.value if hasattr(op_type, 'value') else op_type
|
3731
|
+
self.client.logger.info(
|
3732
|
+
f"✓ Created IVF index '{name}' on {table_name}.{column} | " f"lists={lists} | op_type={op_type_str}"
|
3733
|
+
)
|
3734
|
+
|
3635
3735
|
return self
|
3636
3736
|
|
3637
3737
|
def create_hnsw(
|
@@ -3685,6 +3785,7 @@ class VectorManager:
|
|
3685
3785
|
if op_type is None:
|
3686
3786
|
op_type = VectorOpType.VECTOR_L2_OPS
|
3687
3787
|
|
3788
|
+
# Build index creation SQL
|
3688
3789
|
success = HnswVectorIndex.create_index(
|
3689
3790
|
engine=self.client.get_sqlalchemy_engine(),
|
3690
3791
|
table_name=table_name,
|
@@ -3699,6 +3800,13 @@ class VectorManager:
|
|
3699
3800
|
if not success:
|
3700
3801
|
raise Exception(f"Failed to create HNSW vector index {name} on table {table_name}")
|
3701
3802
|
|
3803
|
+
# Add summary log
|
3804
|
+
op_type_str = op_type.value if hasattr(op_type, 'value') else op_type
|
3805
|
+
self.client.logger.info(
|
3806
|
+
f"✓ Created HNSW index '{name}' on {table_name}.{column} | "
|
3807
|
+
f"m={m} | ef_construction={ef_construction} | ef_search={ef_search} | op_type={op_type_str}"
|
3808
|
+
)
|
3809
|
+
|
3702
3810
|
return self
|
3703
3811
|
|
3704
3812
|
def create_ivf_in_transaction(
|
@@ -3838,6 +3946,9 @@ class VectorManager:
|
|
3838
3946
|
if not success:
|
3839
3947
|
raise Exception(f"Failed to drop vector index {name} from table {table_name}")
|
3840
3948
|
|
3949
|
+
# Add summary log
|
3950
|
+
self.client.logger.info(f"✓ Dropped vector index '{name}' from {table_name}")
|
3951
|
+
|
3841
3952
|
return self
|
3842
3953
|
|
3843
3954
|
def enable_ivf(self, probe_limit: int = 1) -> "VectorManager":
|
@@ -4051,7 +4162,9 @@ class VectorManager:
|
|
4051
4162
|
|
4052
4163
|
# Convert distance type to enum
|
4053
4164
|
if distance_type == "l2":
|
4054
|
-
distance_func = DistanceFunction.
|
4165
|
+
distance_func = DistanceFunction.L2 # Use L2 for consistency with ORM l2_distance()
|
4166
|
+
elif distance_type == "l2_sq":
|
4167
|
+
distance_func = DistanceFunction.L2_SQ # Explicitly use squared distance if needed
|
4055
4168
|
elif distance_type == "cosine":
|
4056
4169
|
distance_func = DistanceFunction.COSINE
|
4057
4170
|
elif distance_type == "inner_product":
|
@@ -20,70 +20,46 @@ MatrixOne SDK Exceptions
|
|
20
20
|
class MatrixOneError(Exception):
|
21
21
|
"""Base exception for all MatrixOne SDK errors"""
|
22
22
|
|
23
|
-
pass
|
24
|
-
|
25
23
|
|
26
24
|
class ConnectionError(MatrixOneError):
|
27
25
|
"""Raised when connection to MatrixOne fails"""
|
28
26
|
|
29
|
-
pass
|
30
|
-
|
31
27
|
|
32
28
|
class QueryError(MatrixOneError):
|
33
29
|
"""Raised when SQL query execution fails"""
|
34
30
|
|
35
|
-
pass
|
36
|
-
|
37
31
|
|
38
32
|
class ConfigurationError(MatrixOneError):
|
39
33
|
"""Raised when configuration is invalid"""
|
40
34
|
|
41
|
-
pass
|
42
|
-
|
43
35
|
|
44
36
|
class SnapshotError(MatrixOneError):
|
45
37
|
"""Raised when snapshot operations fail"""
|
46
38
|
|
47
|
-
pass
|
48
|
-
|
49
39
|
|
50
40
|
class CloneError(MatrixOneError):
|
51
41
|
"""Raised when clone operations fail"""
|
52
42
|
|
53
|
-
pass
|
54
|
-
|
55
43
|
|
56
44
|
class MoCtlError(MatrixOneError):
|
57
45
|
"""Raised when mo_ctl operations fail"""
|
58
46
|
|
59
|
-
pass
|
60
|
-
|
61
47
|
|
62
48
|
class RestoreError(MatrixOneError):
|
63
49
|
"""Raised when restore operations fail"""
|
64
50
|
|
65
|
-
pass
|
66
|
-
|
67
51
|
|
68
52
|
class PitrError(MatrixOneError):
|
69
53
|
"""Raised when PITR operations fail"""
|
70
54
|
|
71
|
-
pass
|
72
|
-
|
73
55
|
|
74
56
|
class PubSubError(MatrixOneError):
|
75
57
|
"""Raised when publish-subscribe operations fail"""
|
76
58
|
|
77
|
-
pass
|
78
|
-
|
79
59
|
|
80
60
|
class AccountError(MatrixOneError):
|
81
61
|
"""Raised when account management operations fail"""
|
82
62
|
|
83
|
-
pass
|
84
|
-
|
85
63
|
|
86
64
|
class VersionError(MatrixOneError):
|
87
65
|
"""Raised when version compatibility check fails"""
|
88
|
-
|
89
|
-
pass
|
@@ -0,0 +1,121 @@
|
|
1
|
+
# Copyright 2021 - 2022 Matrix Origin
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
"""
|
16
|
+
Index utilities - Shared logic for secondary index operations
|
17
|
+
"""
|
18
|
+
|
19
|
+
from typing import List, Tuple
|
20
|
+
|
21
|
+
|
22
|
+
def build_get_index_tables_sql(table_name: str) -> Tuple[str, Tuple]:
|
23
|
+
"""
|
24
|
+
Build SQL to get all secondary index table names for a given table.
|
25
|
+
|
26
|
+
Args:
|
27
|
+
table_name: Name of the table
|
28
|
+
|
29
|
+
Returns:
|
30
|
+
Tuple of (sql, params)
|
31
|
+
"""
|
32
|
+
sql = """
|
33
|
+
SELECT DISTINCT index_table_name
|
34
|
+
FROM mo_catalog.mo_indexes
|
35
|
+
JOIN mo_catalog.mo_tables ON mo_indexes.table_id = mo_tables.rel_id
|
36
|
+
WHERE relname = ? AND type = 'MULTIPLE'
|
37
|
+
"""
|
38
|
+
return sql, (table_name,)
|
39
|
+
|
40
|
+
|
41
|
+
def build_get_index_table_by_name_sql(table_name: str, index_name: str) -> Tuple[str, Tuple]:
|
42
|
+
"""
|
43
|
+
Build SQL to get the physical table name of a secondary index by its index name.
|
44
|
+
|
45
|
+
Args:
|
46
|
+
table_name: Name of the table
|
47
|
+
index_name: Name of the secondary index
|
48
|
+
|
49
|
+
Returns:
|
50
|
+
Tuple of (sql, params)
|
51
|
+
"""
|
52
|
+
sql = """
|
53
|
+
SELECT DISTINCT index_table_name
|
54
|
+
FROM mo_catalog.mo_indexes
|
55
|
+
JOIN mo_catalog.mo_tables ON mo_indexes.table_id = mo_tables.rel_id
|
56
|
+
WHERE relname = ? AND name = ?
|
57
|
+
"""
|
58
|
+
return sql, (table_name, index_name)
|
59
|
+
|
60
|
+
|
61
|
+
def build_verify_counts_sql(table_name: str, index_tables: List[str]) -> str:
|
62
|
+
"""
|
63
|
+
Build SQL to verify counts of main table and all index tables in a single query.
|
64
|
+
|
65
|
+
Args:
|
66
|
+
table_name: Name of the main table
|
67
|
+
index_tables: List of index table names
|
68
|
+
|
69
|
+
Returns:
|
70
|
+
SQL string
|
71
|
+
"""
|
72
|
+
if not index_tables:
|
73
|
+
return f"SELECT COUNT(*) FROM `{table_name}`"
|
74
|
+
|
75
|
+
select_parts = [f"(SELECT COUNT(*) FROM `{table_name}`) as main_count"]
|
76
|
+
for idx, index_table in enumerate(index_tables):
|
77
|
+
select_parts.append(f"(SELECT COUNT(*) FROM `{index_table}`) as idx{idx}_count")
|
78
|
+
|
79
|
+
return "SELECT " + ", ".join(select_parts)
|
80
|
+
|
81
|
+
|
82
|
+
def process_verify_result(table_name: str, index_tables: List[str], row: Tuple) -> int:
|
83
|
+
"""
|
84
|
+
Process the verification result and raise exception if counts don't match.
|
85
|
+
|
86
|
+
Args:
|
87
|
+
table_name: Name of the main table
|
88
|
+
index_tables: List of index table names
|
89
|
+
row: Result row from the verification SQL
|
90
|
+
|
91
|
+
Returns:
|
92
|
+
Row count if verification succeeds
|
93
|
+
|
94
|
+
Raises:
|
95
|
+
ValueError: If any index table has a different count
|
96
|
+
"""
|
97
|
+
main_count = row[0]
|
98
|
+
|
99
|
+
if not index_tables:
|
100
|
+
return main_count
|
101
|
+
|
102
|
+
index_counts = {}
|
103
|
+
mismatch = []
|
104
|
+
|
105
|
+
for idx, index_table in enumerate(index_tables):
|
106
|
+
index_count = row[idx + 1]
|
107
|
+
index_counts[index_table] = index_count
|
108
|
+
if index_count != main_count:
|
109
|
+
mismatch.append(index_table)
|
110
|
+
|
111
|
+
# If there's a mismatch, raise an exception with details
|
112
|
+
if mismatch:
|
113
|
+
error_details = [f"Main table '{table_name}': {main_count} rows"]
|
114
|
+
for index_table, count in index_counts.items():
|
115
|
+
status = "✗ MISMATCH" if index_table in mismatch else "✓"
|
116
|
+
error_details.append(f"{status} Index '{index_table}': {count} rows")
|
117
|
+
|
118
|
+
error_msg = "Index count verification failed!\n" + "\n".join(error_details)
|
119
|
+
raise ValueError(error_msg)
|
120
|
+
|
121
|
+
return main_count
|
@@ -19,10 +19,11 @@ This module provides metadata scanning capabilities for MatrixOne tables,
|
|
19
19
|
allowing users to analyze table statistics, column information, and data distribution.
|
20
20
|
"""
|
21
21
|
|
22
|
-
from typing import Optional, List, Dict, Any, Union
|
23
|
-
from sqlalchemy.engine import Result
|
24
22
|
from dataclasses import dataclass
|
25
23
|
from enum import Enum
|
24
|
+
from typing import Any, Dict, List, Optional, Union
|
25
|
+
|
26
|
+
from sqlalchemy.engine import Result
|
26
27
|
|
27
28
|
|
28
29
|
class MetadataColumn(Enum):
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/vector_index.py
RENAMED
@@ -247,7 +247,21 @@ class IVFVectorIndex(Index):
|
|
247
247
|
# Enable IVF indexing
|
248
248
|
_exec_sql_safe(conn, "SET experimental_ivf_index = 1")
|
249
249
|
_exec_sql_safe(conn, "SET probe_limit = 1")
|
250
|
+
|
251
|
+
# Execute CREATE INDEX
|
250
252
|
_exec_sql_safe(conn, sql)
|
253
|
+
|
254
|
+
# Try to log using the connection's info attribute if available
|
255
|
+
try:
|
256
|
+
from ..client import logger
|
257
|
+
|
258
|
+
logger.info(
|
259
|
+
f"✓ | CREATE INDEX {name} USING ivfflat ON {table_name}({column}) "
|
260
|
+
f"LISTS {lists} op_type '{op_type}'"
|
261
|
+
)
|
262
|
+
except Exception:
|
263
|
+
pass # Silently skip logging if not available (for tests)
|
264
|
+
|
251
265
|
return True
|
252
266
|
except Exception as e:
|
253
267
|
print(f"Failed to create IVFFLAT vector index: {e}")
|
@@ -648,7 +662,21 @@ class HnswVectorIndex(Index):
|
|
648
662
|
with engine.begin() as conn:
|
649
663
|
# Enable HNSW indexing
|
650
664
|
_exec_sql_safe(conn, "SET experimental_hnsw_index = 1")
|
665
|
+
|
666
|
+
# Execute CREATE INDEX
|
651
667
|
_exec_sql_safe(conn, sql)
|
668
|
+
|
669
|
+
# Try to log using the connection's info attribute if available
|
670
|
+
try:
|
671
|
+
from ..client import logger
|
672
|
+
|
673
|
+
logger.info(
|
674
|
+
f"✓ | CREATE INDEX {name} USING hnsw ON {table_name}({column}) "
|
675
|
+
f"M {m} EF_CONSTRUCTION {ef_construction} EF_SEARCH {ef_search} op_type '{op_type}'"
|
676
|
+
)
|
677
|
+
except Exception:
|
678
|
+
pass # Silently skip logging if not available (for tests)
|
679
|
+
|
652
680
|
return True
|
653
681
|
except Exception as e:
|
654
682
|
print(f"Failed to create HNSW vector index: {e}")
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3/matrixone_python_sdk.egg-info}/PKG-INFO
RENAMED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: matrixone-python-sdk
|
3
|
-
Version: 0.1.
|
3
|
+
Version: 0.1.3
|
4
4
|
Summary: A comprehensive Python SDK for MatrixOne database operations with vector search, fulltext search, and advanced features
|
5
5
|
Home-page: https://github.com/matrixorigin/matrixone
|
6
6
|
Author: MatrixOne Team
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
4
4
|
|
5
5
|
[project]
|
6
6
|
name = "matrixone-python-sdk"
|
7
|
-
version = "0.1.
|
7
|
+
version = "0.1.3"
|
8
8
|
description = "A comprehensive Python SDK for MatrixOne database operations with vector search, fulltext search, and advanced features"
|
9
9
|
readme = "README_USER.md"
|
10
10
|
license = {text = "Apache-2.0"}
|
File without changes
|
File without changes
|
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_01_basic_connection.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_02_account_management.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_03_async_operations.py
RENAMED
File without changes
|
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_05_snapshot_restore.py
RENAMED
File without changes
|
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_07_advanced_features.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_08_pubsub_operations.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_09_logger_integration.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_10_version_management.py
RENAMED
File without changes
|
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_12_vector_basics.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_13_vector_indexes.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_14_vector_search.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_15_vector_advanced.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_18_snapshot_orm.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_22_unified_sql_builder.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_24_query_update.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_25_metadata_operations.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_connection_hooks.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_dynamic_logging.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/examples/example_ivf_stats_complete.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/adapters.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/dialect.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/fulltext_index.py
RENAMED
File without changes
|
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/hnsw_config.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/ivf_config.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/table_builder.py
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/vector_type.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone_python_sdk.egg-info/not-zip-safe
RENAMED
File without changes
|
{matrixone_python_sdk-0.1.1 → matrixone_python_sdk-0.1.3}/matrixone_python_sdk.egg-info/requires.txt
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|