matrixone-python-sdk 0.1.2__tar.gz → 0.1.3__tar.gz

{matrixone_python_sdk-0.1.2/matrixone_python_sdk.egg-info → matrixone_python_sdk-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrixone-python-sdk
-Version: 0.1.2
+Version: 0.1.3
 Summary: A comprehensive Python SDK for MatrixOne database operations with vector search, fulltext search, and advanced features
 Home-page: https://github.com/matrixorigin/matrixone
 Author: MatrixOne Team

{matrixone_python_sdk-0.1.2 → matrixone_python_sdk-0.1.3}/README.md

@@ -31,6 +31,10 @@ A comprehensive Python SDK for MatrixOne that provides SQLAlchemy-like interface
   - Table size and row count statistics
   - Column data distribution analysis
   - Index usage metrics
+- 🔍 **Secondary Index Verification**: Verify consistency of secondary indexes with main table
+  - Get all secondary index table names
+  - Get specific index table by index name
+  - Verify row counts across main table and all indexes in single query
 - 📸 **Snapshot Management**: Create and manage database snapshots at multiple levels
 - ⏰ **Point-in-Time Recovery**: PITR functionality for precise data recovery
 - 🔄 **Table Cloning**: Clone databases and tables efficiently with data replication
@@ -323,6 +327,27 @@ print(f"Original size: {table_stats['original_size']} bytes")
 print(f"Compressed size: {table_stats['compress_size']} bytes")
 ```
 
+### Secondary Index Verification
+
+```python
+# Get all secondary index tables
+index_tables = client.get_secondary_index_tables('my_table')
+print(f"Found {len(index_tables)} secondary indexes")
+
+# Get specific index by name
+physical_table = client.get_secondary_index_table_by_name('my_table', 'idx_name')
+print(f"Index 'idx_name' -> {physical_table}")
+
+# Verify all indexes have same count as main table
+try:
+    count = client.verify_table_index_counts('my_table')
+    print(f"✓ Verification passed! Row count: {count}")
+except ValueError as e:
+    print(f"✗ Index mismatch detected:")
+    print(e)
+    # Error includes all count details for debugging
+```
+
 ### Version Management
 
 ```python

{matrixone_python_sdk-0.1.2 → matrixone_python_sdk-0.1.3}/matrixone/async_client.py

@@ -34,9 +34,10 @@ from datetime import datetime
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 from .account import Account, User
+from .async_metadata_manager import AsyncMetadataManager
 from .async_vector_index_manager import AsyncVectorManager
 from .base_client import BaseMatrixOneClient, BaseMatrixOneExecutor
-from .connection_hooks import ConnectionHook, ConnectionAction, create_connection_hook
+from .connection_hooks import ConnectionAction, ConnectionHook, create_connection_hook
 from .exceptions import (
     AccountError,
     ConnectionError,
@@ -48,7 +49,6 @@ from .exceptions import (
     SnapshotError,
 )
 from .logger import MatrixOneLogger, create_default_logger
-from .async_metadata_manager import AsyncMetadataManager
 from .pitr import Pitr
 from .pubsub import Publication, Subscription
 from .snapshot import Snapshot, SnapshotLevel
@@ -1837,7 +1837,6 @@ class AsyncClient(BaseMatrixOneClient):
         """Cleanup when object is garbage collected"""
         # Don't try to cleanup in __del__ as it can cause issues with event loops
         # The fixture should handle proper cleanup
-        pass
 
     def get_sqlalchemy_engine(self) -> AsyncEngine:
         """
@@ -1920,6 +1919,7 @@ class AsyncClient(BaseMatrixOneClient):
             and other SDK components. External users should use execute() instead.
         """
         import time
+
         from sqlalchemy import text
 
         start_time = time.time()
@@ -2554,6 +2554,97 @@ class AsyncClient(BaseMatrixOneClient):
         except Exception as e:
             raise QueryError(f"Failed to get git version: {e}")
 
+    async def get_secondary_index_tables(self, table_name: str) -> List[str]:
+        """
+        Get all secondary index table names for a given table (async version).
+
+        Args:
+            table_name: Name of the table to get secondary indexes for
+
+        Returns:
+            List of secondary index table names
+
+        Examples::
+
+            >>> async with AsyncClient() as client:
+            ...     await client.connect(host='localhost', port=6001, user='root', password='111', database='test')
+            ...     index_tables = await client.get_secondary_index_tables('cms_all_content_chunk_info')
+            ...     print(index_tables)
+        """
+        from .index_utils import build_get_index_tables_sql
+
+        sql, params = build_get_index_tables_sql(table_name)
+        result = await self.execute(sql, params)
+        return [row[0] for row in result.fetchall()]
+
+    async def get_secondary_index_table_by_name(self, table_name: str, index_name: str) -> Optional[str]:
+        """
+        Get the physical table name of a secondary index by its index name (async version).
+
+        Args:
+            table_name: Name of the table
+            index_name: Name of the secondary index
+
+        Returns:
+            Physical table name of the secondary index, or None if not found
+
+        Examples::
+
+            >>> async with AsyncClient() as client:
+            ...     await client.connect(host='localhost', port=6001, user='root', password='111', database='test')
+            ...     index_table = await client.get_secondary_index_table_by_name('cms_all_content_chunk_info', 'cms_id')
+            ...     print(index_table)
+        """
+        from .index_utils import build_get_index_table_by_name_sql
+
+        sql, params = build_get_index_table_by_name_sql(table_name, index_name)
+        result = await self.execute(sql, params)
+        row = result.fetchone()
+        return row[0] if row else None
+
+    async def verify_table_index_counts(self, table_name: str) -> int:
+        """
+        Verify that the main table and all its secondary index tables have the same row count (async version).
+
+        This method compares the COUNT(*) of the main table with all its secondary index tables
+        in a single SQL query for consistency. If counts don't match, raises an exception.
+
+        Args:
+            table_name: Name of the table to verify
+
+        Returns:
+            Row count (int) if verification succeeds
+
+        Raises:
+            ValueError: If any secondary index table has a different count than the main table,
+                       with details about all counts in the error message
+
+        Examples::
+
+            >>> async with AsyncClient() as client:
+            ...     await client.connect(host='localhost', port=6001, user='root', password='111', database='test')
+            ...     count = await client.verify_table_index_counts('cms_all_content_chunk_info')
+            ...     print(f"✓ Verification passed, row count: {count}")
+
+            >>> # If verification fails:
+            >>> try:
+            ...     count = await client.verify_table_index_counts('some_table')
+            ... except ValueError as e:
+            ...     print(f"Verification failed: {e}")
+        """
+        from .index_utils import build_verify_counts_sql, process_verify_result
+
+        # Get all secondary index tables
+        index_tables = await self.get_secondary_index_tables(table_name)
+
+        # Build and execute verification SQL
+        sql = build_verify_counts_sql(table_name, index_tables)
+        result = await self.execute(sql)
+        row = result.fetchone()
+
+        # Process result and raise exception if verification fails
+        return process_verify_result(table_name, index_tables, row)
+
     async def __aenter__(self):
         return self
 

{matrixone_python_sdk-0.1.2 → matrixone_python_sdk-0.1.3}/matrixone/async_metadata_manager.py

@@ -19,8 +19,10 @@ This module provides async metadata scanning capabilities for MatrixOne tables,
 allowing users to analyze table statistics, column information, and data distribution.
 """
 
-from typing import Optional, List, Dict, Any, Union
+from typing import Any, Dict, List, Optional, Union
+
 from sqlalchemy.engine import Result
+
 from .metadata import BaseMetadataManager, MetadataColumn, MetadataRow
 
 

{matrixone_python_sdk-0.1.2 → matrixone_python_sdk-0.1.3}/matrixone/client.py

@@ -30,7 +30,7 @@ from sqlalchemy.engine import Engine
 
 from .account import AccountManager, TransactionAccountManager
 from .base_client import BaseMatrixOneClient, BaseMatrixOneExecutor
-from .connection_hooks import ConnectionHook, ConnectionAction, create_connection_hook
+from .connection_hooks import ConnectionAction, ConnectionHook, create_connection_hook
 from .exceptions import ConnectionError, QueryError
 from .logger import MatrixOneLogger, create_default_logger
 from .metadata import MetadataManager, TransactionMetadataManager
@@ -1698,8 +1698,9 @@ class Client(BaseMatrixOneClient):
             table_name = model_class.__tablename__
             table = model_class.__table__
 
+            from sqlalchemy.schema import CreateIndex, CreateTable
+
             from .sqlalchemy_ext import FulltextIndex, VectorIndex
-            from sqlalchemy.schema import CreateTable, CreateIndex
 
             try:
                 engine_context = self.get_sqlalchemy_engine().begin()
@@ -2612,6 +2613,99 @@ class Client(BaseMatrixOneClient):
 
         return self
 
+    def get_secondary_index_tables(self, table_name: str) -> List[str]:
+        """
+        Get all secondary index table names for a given table.
+
+        Args:
+            table_name: Name of the table to get secondary indexes for
+
+        Returns:
+            List of secondary index table names
+
+        Examples::
+
+            >>> client = Client()
+            >>> client.connect(host='localhost', port=6001, user='root', password='111', database='test')
+            >>> index_tables = client.get_secondary_index_tables('cms_all_content_chunk_info')
+            >>> print(index_tables)
+            ['__mo_index_secondary_..._cms_id', '__mo_index_secondary_..._idx_all_content_length']
+        """
+        from .index_utils import build_get_index_tables_sql
+
+        sql, params = build_get_index_tables_sql(table_name)
+        result = self.execute(sql, params)
+        return [row[0] for row in result.fetchall()]
+
+    def get_secondary_index_table_by_name(self, table_name: str, index_name: str) -> Optional[str]:
+        """
+        Get the physical table name of a secondary index by its index name.
+
+        Args:
+            table_name: Name of the table
+            index_name: Name of the secondary index
+
+        Returns:
+            Physical table name of the secondary index, or None if not found
+
+        Examples::
+
+            >>> client = Client()
+            >>> client.connect(host='localhost', port=6001, user='root', password='111', database='test')
+            >>> index_table = client.get_secondary_index_table_by_name('cms_all_content_chunk_info', 'cms_id')
+            >>> print(index_table)
+            '__mo_index_secondary_018cfbda-bde1-7c3e-805c-3f8e71769f75_cms_id'
+        """
+        from .index_utils import build_get_index_table_by_name_sql
+
+        sql, params = build_get_index_table_by_name_sql(table_name, index_name)
+        result = self.execute(sql, params)
+        row = result.fetchone()
+        return row[0] if row else None
+
+    def verify_table_index_counts(self, table_name: str) -> int:
+        """
+        Verify that the main table and all its secondary index tables have the same row count.
+
+        This method compares the COUNT(*) of the main table with all its secondary index tables
+        in a single SQL query for consistency. If counts don't match, raises an exception.
+
+        Args:
+            table_name: Name of the table to verify
+
+        Returns:
+            Row count (int) if verification succeeds
+
+        Raises:
+            ValueError: If any secondary index table has a different count than the main table,
+                       with details about all counts in the error message
+
+        Examples::
+
+            >>> client = Client()
+            >>> client.connect(host='localhost', port=6001, user='root', password='111', database='test')
+            >>> count = client.verify_table_index_counts('cms_all_content_chunk_info')
+            >>> print(f"✓ Verification passed, row count: {count}")
+
+            >>> # If verification fails:
+            >>> try:
+            ...     count = client.verify_table_index_counts('some_table')
+            ... except ValueError as e:
+            ...     print(f"Verification failed: {e}")
+        """
+        from .index_utils import build_verify_counts_sql, process_verify_result
+
+        # Get all secondary index tables
+        index_tables = self.get_secondary_index_tables(table_name)
+
+        # Build and execute verification SQL
+        sql = build_verify_counts_sql(table_name, index_tables)
+        result = self.execute(sql)
+        row = result.fetchone()
+
+        # Process result and raise exception if verification fails
+        return process_verify_result(table_name, index_tables, row)
+
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.disconnect()
 

{matrixone_python_sdk-0.1.2 → matrixone_python_sdk-0.1.3}/matrixone/connection_hooks.py

@@ -18,6 +18,7 @@ Connection hooks for MatrixOne clients
 
 from enum import Enum
 from typing import Callable, List, Optional, Union
+
 from sqlalchemy import event
 from sqlalchemy.engine import Engine
 from sqlalchemy.ext.asyncio import AsyncEngine

{matrixone_python_sdk-0.1.2 → matrixone_python_sdk-0.1.3}/matrixone/exceptions.py

@@ -20,70 +20,46 @@ MatrixOne SDK Exceptions
 class MatrixOneError(Exception):
     """Base exception for all MatrixOne SDK errors"""
 
-    pass
-
 
 class ConnectionError(MatrixOneError):
     """Raised when connection to MatrixOne fails"""
 
-    pass
-
 
 class QueryError(MatrixOneError):
     """Raised when SQL query execution fails"""
 
-    pass
-
 
 class ConfigurationError(MatrixOneError):
     """Raised when configuration is invalid"""
 
-    pass
-
 
 class SnapshotError(MatrixOneError):
     """Raised when snapshot operations fail"""
 
-    pass
-
 
 class CloneError(MatrixOneError):
     """Raised when clone operations fail"""
 
-    pass
-
 
 class MoCtlError(MatrixOneError):
     """Raised when mo_ctl operations fail"""
 
-    pass
-
 
 class RestoreError(MatrixOneError):
     """Raised when restore operations fail"""
 
-    pass
-
 
 class PitrError(MatrixOneError):
     """Raised when PITR operations fail"""
 
-    pass
-
 
 class PubSubError(MatrixOneError):
     """Raised when publish-subscribe operations fail"""
 
-    pass
-
 
 class AccountError(MatrixOneError):
     """Raised when account management operations fail"""
 
-    pass
-
 
 class VersionError(MatrixOneError):
     """Raised when version compatibility check fails"""
-
-    pass

matrixone_python_sdk-0.1.3/matrixone/index_utils.py

@@ -0,0 +1,121 @@
+# Copyright 2021 - 2022 Matrix Origin
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Index utilities - Shared logic for secondary index operations
+"""
+
+from typing import List, Tuple
+
+
+def build_get_index_tables_sql(table_name: str) -> Tuple[str, Tuple]:
+    """
+    Build SQL to get all secondary index table names for a given table.
+
+    Args:
+        table_name: Name of the table
+
+    Returns:
+        Tuple of (sql, params)
+    """
+    sql = """
+        SELECT DISTINCT index_table_name
+        FROM mo_catalog.mo_indexes
+        JOIN mo_catalog.mo_tables ON mo_indexes.table_id = mo_tables.rel_id
+        WHERE relname = ? AND type = 'MULTIPLE'
+    """
+    return sql, (table_name,)
+
+
+def build_get_index_table_by_name_sql(table_name: str, index_name: str) -> Tuple[str, Tuple]:
+    """
+    Build SQL to get the physical table name of a secondary index by its index name.
+
+    Args:
+        table_name: Name of the table
+        index_name: Name of the secondary index
+
+    Returns:
+        Tuple of (sql, params)
+    """
+    sql = """
+        SELECT DISTINCT index_table_name
+        FROM mo_catalog.mo_indexes
+        JOIN mo_catalog.mo_tables ON mo_indexes.table_id = mo_tables.rel_id
+        WHERE relname = ? AND name = ?
+    """
+    return sql, (table_name, index_name)
+
+
+def build_verify_counts_sql(table_name: str, index_tables: List[str]) -> str:
+    """
+    Build SQL to verify counts of main table and all index tables in a single query.
+
+    Args:
+        table_name: Name of the main table
+        index_tables: List of index table names
+
+    Returns:
+        SQL string
+    """
+    if not index_tables:
+        return f"SELECT COUNT(*) FROM `{table_name}`"
+
+    select_parts = [f"(SELECT COUNT(*) FROM `{table_name}`) as main_count"]
+    for idx, index_table in enumerate(index_tables):
+        select_parts.append(f"(SELECT COUNT(*) FROM `{index_table}`) as idx{idx}_count")
+
+    return "SELECT " + ", ".join(select_parts)
+
+
+def process_verify_result(table_name: str, index_tables: List[str], row: Tuple) -> int:
+    """
+    Process the verification result and raise exception if counts don't match.
+
+    Args:
+        table_name: Name of the main table
+        index_tables: List of index table names
+        row: Result row from the verification SQL
+
+    Returns:
+        Row count if verification succeeds
+
+    Raises:
+        ValueError: If any index table has a different count
+    """
+    main_count = row[0]
+
+    if not index_tables:
+        return main_count
+
+    index_counts = {}
+    mismatch = []
+
+    for idx, index_table in enumerate(index_tables):
+        index_count = row[idx + 1]
+        index_counts[index_table] = index_count
+        if index_count != main_count:
+            mismatch.append(index_table)
+
+    # If there's a mismatch, raise an exception with details
+    if mismatch:
+        error_details = [f"Main table '{table_name}': {main_count} rows"]
+        for index_table, count in index_counts.items():
+            status = "✗ MISMATCH" if index_table in mismatch else "✓"
+            error_details.append(f"{status} Index '{index_table}': {count} rows")
+
+        error_msg = "Index count verification failed!\n" + "\n".join(error_details)
+        raise ValueError(error_msg)
+
+    return main_count

{matrixone_python_sdk-0.1.2 → matrixone_python_sdk-0.1.3}/matrixone/metadata.py

@@ -19,10 +19,11 @@ This module provides metadata scanning capabilities for MatrixOne tables,
 allowing users to analyze table statistics, column information, and data distribution.
 """
 
-from typing import Optional, List, Dict, Any, Union
-from sqlalchemy.engine import Result
 from dataclasses import dataclass
 from enum import Enum
+from typing import Any, Dict, List, Optional, Union
+
+from sqlalchemy.engine import Result
 
 
 class MetadataColumn(Enum):

{matrixone_python_sdk-0.1.2 → matrixone_python_sdk-0.1.3}/matrixone/moctl.py

@@ -40,8 +40,6 @@ from .exceptions import MatrixOneError
 class MoCtlError(MatrixOneError):
     """Raised when mo_ctl operations fail"""
 
-    pass
-
 
 class MoCtlManager:
     """

{matrixone_python_sdk-0.1.2 → matrixone_python_sdk-0.1.3}/matrixone/sqlalchemy_ext/__init__.py

@@ -147,6 +147,7 @@ __all__ = [
     "FulltextIndex",
     "FulltextAlgorithmType",
     "FulltextModeType",
+    "FulltextParserType",
     "FulltextSearchBuilder",
     "create_fulltext_index",
     "fulltext_search_builder",

{matrixone_python_sdk-0.1.2 → matrixone_python_sdk-0.1.3}/matrixone/version.py

@@ -519,8 +519,6 @@ def requires_version(
 class VersionError(MatrixOneError):
     """Raised when version compatibility check fails"""
 
-    pass
-
 
 # Initialize common feature requirements
 def _initialize_default_features():

{matrixone_python_sdk-0.1.2 → matrixone_python_sdk-0.1.3/matrixone_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrixone-python-sdk
-Version: 0.1.2
+Version: 0.1.3
 Summary: A comprehensive Python SDK for MatrixOne database operations with vector search, fulltext search, and advanced features
 Home-page: https://github.com/matrixorigin/matrixone
 Author: MatrixOne Team

{matrixone_python_sdk-0.1.2 → matrixone_python_sdk-0.1.3}/matrixone_python_sdk.egg-info/SOURCES.txt

@@ -41,6 +41,7 @@ matrixone/client.py
 matrixone/config.py
 matrixone/connection_hooks.py
 matrixone/exceptions.py
+matrixone/index_utils.py
 matrixone/logger.py
 matrixone/metadata.py
 matrixone/moctl.py

{matrixone_python_sdk-0.1.2 → matrixone_python_sdk-0.1.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "matrixone-python-sdk"
-version = "0.1.2"
+version = "0.1.3"
 description = "A comprehensive Python SDK for MatrixOne database operations with vector search, fulltext search, and advanced features"
 readme = "README_USER.md"
 license = {text = "Apache-2.0"}