scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

scitex/db/_inspect_optimized.py

@@ -0,0 +1,301 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-09-10 13:20:00 (ywatanabe)"
+# File: /data/gpfs/projects/punim2354/ywatanabe/SciTeX-Code/src/scitex/db/_inspect_optimized.py
+# ----------------------------------------
+from __future__ import annotations
+import os
+__FILE__ = __file__
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+import sqlite3
+from typing import Any, Dict, List, Optional, Tuple
+from contextlib import contextmanager
+
+
+class OptimizedInspector:
+    """Optimized database inspector with connection reuse and efficient queries."""
+    
+    def __init__(self, db_path: str):
+        if not os.path.exists(db_path):
+            raise FileNotFoundError(f"Database file not found: {db_path}")
+        self.db_path = db_path
+        self._conn = None
+        self._cursor = None
+    
+    @contextmanager
+    def connection(self):
+        """Context manager for database connection reuse."""
+        if self._conn is None:
+            self._conn = sqlite3.connect(self.db_path)
+            self._conn.row_factory = sqlite3.Row  # Enable column access by name
+            self._cursor = self._conn.cursor()
+        try:
+            yield self._cursor
+        except Exception:
+            self._conn.rollback()
+            raise
+    
+    def close(self):
+        """Close the database connection."""
+        if self._conn:
+            self._conn.close()
+            self._conn = None
+            self._cursor = None
+    
+    def __enter__(self):
+        return self
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+    
+    def get_table_names(self) -> List[str]:
+        """Retrieves all table names from the database."""
+        with self.connection() as cursor:
+            cursor.execute(
+                "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;"
+            )
+            return [row[0] for row in cursor.fetchall()]
+    
+    def get_table_info_batch(self, table_names: List[str]) -> Dict[str, List[Dict]]:
+        """Get table info for multiple tables in one go.
+        
+        Returns:
+            Dict mapping table_name to list of column info dictionaries
+        """
+        table_info = {}
+        
+        with self.connection() as cursor:
+            for table_name in table_names:
+                # Get column info
+                cursor.execute(f"PRAGMA table_info({table_name})")
+                columns = cursor.fetchall()
+                
+                # Get primary key info more efficiently
+                cursor.execute(f"""
+                    SELECT name FROM pragma_table_info('{table_name}') 
+                    WHERE pk > 0
+                """)
+                pk_columns = {row[0] for row in cursor.fetchall()}
+                
+                # Build column info
+                col_info = []
+                for col in columns:
+                    col_dict = {
+                        'cid': col[0],
+                        'name': col[1],
+                        'type': col[2],
+                        'notnull': col[3],
+                        'default': col[4],
+                        'pk': col[1] in pk_columns
+                    }
+                    col_info.append(col_dict)
+                
+                table_info[table_name] = col_info
+        
+        return table_info
+    
+    def get_table_stats_batch(self, table_names: List[str], 
+                             sample_size: int = 5,
+                             skip_count: bool = False) -> Dict[str, Dict]:
+        """Get statistics for multiple tables efficiently.
+        
+        Args:
+            table_names: List of table names to inspect
+            sample_size: Number of sample rows to retrieve
+            skip_count: If True, skip the COUNT(*) query for performance
+        
+        Returns:
+            Dict mapping table_name to statistics dictionary
+        """
+        stats = {}
+        
+        with self.connection() as cursor:
+            for table_name in table_names:
+                table_stats = {}
+                
+                # Get sample data
+                cursor.execute(f"SELECT * FROM {table_name} LIMIT {sample_size}")
+                table_stats['columns'] = [desc[0] for desc in cursor.description]
+                table_stats['sample_data'] = cursor.fetchall()
+                
+                # Get row count (can be slow for large tables)
+                if not skip_count:
+                    # Use approximate count if available
+                    cursor.execute(f"""
+                        SELECT COUNT(*) FROM (
+                            SELECT 1 FROM {table_name} LIMIT 100000
+                        )
+                    """)
+                    approx_count = cursor.fetchone()[0]
+                    
+                    if approx_count < 100000:
+                        # Small table, get exact count
+                        cursor.execute(f"SELECT COUNT(*) FROM {table_name}")
+                        table_stats['row_count'] = cursor.fetchone()[0]
+                        table_stats['is_approximate'] = False
+                    else:
+                        # Large table, use approximate count
+                        table_stats['row_count'] = f">{approx_count:,}"
+                        table_stats['is_approximate'] = True
+                else:
+                    table_stats['row_count'] = "Not counted"
+                    table_stats['is_approximate'] = None
+                
+                stats[table_name] = table_stats
+        
+        return stats
+    
+    def inspect_fast(
+        self,
+        table_names: Optional[List[str]] = None,
+        sample_size: int = 5,
+        skip_count: bool = False,
+        skip_blob_content: bool = True,
+        verbose: bool = True,
+    ) -> List[Dict[str, Any]]:
+        """Fast inspection of database tables.
+        
+        Args:
+            table_names: Tables to inspect (None for all)
+            sample_size: Number of sample rows
+            skip_count: Skip row counting for performance
+            skip_blob_content: Don't load BLOB content
+            verbose: Print results
+        
+        Returns:
+            List of inspection results
+        """
+        if table_names is None:
+            table_names = self.get_table_names()
+        
+        # Batch operations for efficiency
+        table_info = self.get_table_info_batch(table_names)
+        table_stats = self.get_table_stats_batch(
+            table_names, sample_size, skip_count
+        )
+        
+        results = []
+        for table_name in table_names:
+            result = {
+                'table_name': table_name,
+                'columns': table_info[table_name],
+                'row_count': table_stats[table_name]['row_count'],
+                'is_approximate': table_stats[table_name]['is_approximate'],
+                'sample_data': []
+            }
+            
+            # Format sample data
+            for row in table_stats[table_name]['sample_data']:
+                formatted_row = {}
+                for i, col_name in enumerate(table_stats[table_name]['columns']):
+                    value = row[i]
+                    if isinstance(value, bytes):
+                        if skip_blob_content:
+                            formatted_row[col_name] = f"<BLOB {len(value)} bytes>"
+                        else:
+                            formatted_row[col_name] = value
+                    else:
+                        formatted_row[col_name] = value
+                result['sample_data'].append(formatted_row)
+            
+            results.append(result)
+            
+            if verbose:
+                self._print_table_info(result)
+        
+        return results
+    
+    def _print_table_info(self, result: Dict):
+        """Pretty print table information."""
+        print(f"\n{'='*60}")
+        print(f"Table: {result['table_name']}")
+        print(f"Rows: {result['row_count']}" + 
+              (" (approximate)" if result['is_approximate'] else ""))
+        print(f"-"*60)
+        
+        # Print column info
+        print("Columns:")
+        for col in result['columns']:
+            pk_marker = " [PK]" if col['pk'] else ""
+            nn_marker = " NOT NULL" if col['notnull'] else ""
+            print(f"  {col['name']}: {col['type']}{pk_marker}{nn_marker}")
+        
+        # Print sample data
+        if result['sample_data']:
+            print(f"\nSample data ({len(result['sample_data'])} rows):")
+            # Get column names that aren't metadata columns
+            data_cols = [c for c in result['sample_data'][0].keys() 
+                        if not c.endswith(('_dtype', '_shape', '_compressed'))]
+            
+            # Print header
+            header = " | ".join(f"{col[:15]:<15}" for col in data_cols[:5])
+            print(f"  {header}")
+            print(f"  {'-'*len(header)}")
+            
+            # Print rows
+            for row in result['sample_data'][:3]:
+                values = []
+                for col in data_cols[:5]:
+                    val = str(row.get(col, ''))
+                    if len(val) > 15:
+                        val = val[:12] + "..."
+                    values.append(f"{val:<15}")
+                print(f"  {' | '.join(values)}")
+
+
+def inspect_optimized(
+    lpath_db: str,
+    table_names: Optional[List[str]] = None,
+    sample_size: int = 5,
+    skip_count: bool = False,
+    verbose: bool = True,
+) -> List[Dict[str, Any]]:
+    """
+    Optimized database inspection.
+    
+    Example:
+    >>> inspect_optimized('path/to/database.db')
+    >>> inspect_optimized('path/to/database.db', ['table1'], skip_count=True)
+    
+    Args:
+        lpath_db: Path to the SQLite database file
+        table_names: List of table names to inspect (None for all)
+        sample_size: Number of sample rows to retrieve
+        skip_count: Skip row counting for large tables (much faster)
+        verbose: Print inspection results
+    
+    Returns:
+        List of inspection results
+    """
+    with OptimizedInspector(lpath_db) as inspector:
+        return inspector.inspect_fast(
+            table_names=table_names,
+            sample_size=sample_size,
+            skip_count=skip_count,
+            verbose=verbose
+        )
+
+
+# Performance comparison example:
+if __name__ == "__main__":
+    import time
+    import sys
+    
+    if len(sys.argv) > 1:
+        db_path = sys.argv[1]
+        
+        print("Testing optimized version...")
+        start = time.time()
+        results = inspect_optimized(db_path, skip_count=True, verbose=False)
+        print(f"Optimized version took: {time.time() - start:.2f} seconds")
+        print(f"Inspected {len(results)} tables")
+        
+        # Show first table info
+        if results:
+            print(f"\nFirst table: {results[0]['table_name']}")
+            print(f"Columns: {len(results[0]['columns'])}")
+            print(f"Sample rows: {len(results[0]['sample_data'])}")
+
+# EOF

scitex/db/{_PostgreSQL.py → _postgresql/_PostgreSQL.py}

@@ -1,13 +1,13 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # Time-stamp: "2024-11-25 02:00:06 (ywatanabe)"
-# File: ./scitex_repo/src/scitex/db/_PostgreSQL.py
+# File: ./scitex_repo/src/scitex/db/_postgresql/_PostgreSQL.py
 
-THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_PostgreSQL.py"
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_postgresql/_PostgreSQL.py"
 
 from typing import List, Optional
 
-from ..str import printc as _printc
+from ...str import printc as _printc
 from typing import Optional
 import psycopg2
 from ._PostgreSQLMixins._BackupMixin import _BackupMixin

scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_BackupMixin.py

@@ -10,7 +10,7 @@ THIS_FILE = (
 import subprocess
 import os
 from typing import Optional, List
-from .._BaseMixins._BaseBackupMixin import _BaseBackupMixin
+from ..._BaseMixins._BaseBackupMixin import _BaseBackupMixin
 
 
 class _BackupMixin(_BaseBackupMixin):

scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_BatchMixin.py

@@ -9,7 +9,7 @@ THIS_FILE = (
 
 from typing import List, Any, Optional, Dict, Union
 import pandas as pd
-from .._BaseMixins._BaseBatchMixin import _BaseBatchMixin
+from ..._BaseMixins._BaseBatchMixin import _BaseBatchMixin
 
 
 class _BatchMixin(_BaseBatchMixin):

scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_BlobMixin.py

@@ -9,7 +9,7 @@ import psycopg2
 import numpy as np
 from typing import Any as _Any
 from typing import Dict, List, Optional, Tuple, Union
-from .._BaseMixins._BaseBlobMixin import _BaseBlobMixin
+from ..._BaseMixins._BaseBlobMixin import _BaseBlobMixin
 
 
 class _BlobMixin(_BaseBlobMixin):

scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_ConnectionMixin.py

@@ -10,7 +10,7 @@ THIS_FILE = (
 from typing import Any, Tuple
 import psycopg2
 
-from .._BaseMixins._BaseConnectionMixin import _BaseConnectionMixin
+from ..._BaseMixins._BaseConnectionMixin import _BaseConnectionMixin
 
 
 class _ConnectionMixin(_BaseConnectionMixin):

scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_MaintenanceMixin.py

@@ -13,7 +13,7 @@ from typing import ContextManager, Dict, List, Optional
 
 import psycopg2
 
-from .._BaseMixins._BaseMaintenanceMixin import _BaseMaintenanceMixin
+from ..._BaseMixins._BaseMaintenanceMixin import _BaseMaintenanceMixin
 
 
 class _MaintenanceMixin(_BaseMaintenanceMixin):

scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_QueryMixin.py

@@ -8,7 +8,7 @@ THIS_FILE = (
 )
 
 from typing import List, Dict, Any, Optional, Union, Tuple
-from .._BaseMixins._BaseQueryMixin import _BaseQueryMixin
+from ..._BaseMixins._BaseQueryMixin import _BaseQueryMixin
 
 
 class _QueryMixin(_BaseQueryMixin):

scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_SchemaMixin.py

@@ -8,7 +8,7 @@ THIS_FILE = (
 )
 
 from typing import List, Dict, Any, Optional
-from .._BaseMixins._BaseSchemaMixin import _BaseSchemaMixin
+from ..._BaseMixins._BaseSchemaMixin import _BaseSchemaMixin
 
 
 class _SchemaMixin(_BaseSchemaMixin):

scitex/db/{_PostgreSQLMixins → _postgresql/_PostgreSQLMixins}/_TransactionMixin.py

@@ -8,7 +8,7 @@ THIS_FILE = (
 )
 
 import psycopg2
-from .._BaseMixins._BaseTransactionMixin import _BaseTransactionMixin
+from ..._BaseMixins._BaseTransactionMixin import _BaseTransactionMixin
 
 
 class _TransactionMixin(_BaseTransactionMixin):

scitex/db/_postgresql/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+"""PostgreSQL database module."""
+
+from ._PostgreSQL import PostgreSQL
+
+__all__ = ["PostgreSQL"]

scitex/db/_sqlite3/_SQLite3.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-09-11 07:57:49 (ywatanabe)"
+# File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/db/_sqlite3/_SQLite3.py
+# ----------------------------------------
+from __future__ import annotations
+import os
+__FILE__ = __file__
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+import warnings
+from typing import List, Optional
+
+from ...str import printc as _printc
+from ._SQLite3Mixins._ArrayMixin import _ArrayMixin
+from ._SQLite3Mixins._BatchMixin import _BatchMixin
+from ._SQLite3Mixins._BlobMixin import _BlobMixin
+from ._SQLite3Mixins._ColumnMixin import _ColumnMixin
+from ._SQLite3Mixins._ConnectionMixin import _ConnectionMixin
+from ._SQLite3Mixins._GitMixin import _GitMixin
+from ._SQLite3Mixins._ImportExportMixin import _ImportExportMixin
+from ._SQLite3Mixins._IndexMixin import _IndexMixin
+from ._SQLite3Mixins._MaintenanceMixin import _MaintenanceMixin
+from ._SQLite3Mixins._QueryMixin import _QueryMixin
+from ._SQLite3Mixins._RowMixin import _RowMixin
+from ._SQLite3Mixins._TableMixin import _TableMixin
+from ._SQLite3Mixins._TransactionMixin import _TransactionMixin
+
+
+class SQLite3(
+    _ArrayMixin,
+    _ConnectionMixin,
+    _QueryMixin,
+    _TransactionMixin,
+    _ColumnMixin,
+    _TableMixin,
+    _IndexMixin,
+    _RowMixin,
+    _BatchMixin,
+    _BlobMixin,
+    _ImportExportMixin,
+    _MaintenanceMixin,
+    _GitMixin,
+):
+    """SQLite database manager with automatic metadata handling, numpy array storage, and compression.
+
+    This class provides a comprehensive interface for SQLite database operations with
+    automatic compression, thread-safe operations, and specialized numpy array handling.
+
+    Features:
+        - Automatic compression for BLOB data (70-90% reduction)
+        - Thread-safe operations with proper connection management
+        - Metadata handling for BLOB columns
+        - Batch processing support
+        - Context manager support for proper resource cleanup
+
+    Examples:
+        Basic usage with context manager (recommended):
+
+        >>> with SQLite3("data.db", compress_by_default=True) as db:
+        ...     db.create_table("experiments", {"id": "INTEGER PRIMARY KEY", "data": "BLOB"})
+        ...     data = np.random.random((1000, 100))
+        ...     db.save_array("experiments", data, column="data", additional_columns={"id": 1})
+
+        Array storage and retrieval:
+
+        >>> with SQLite3("data.db") as db:
+        ...     # Save numpy array
+        ...     db.save_array(
+        ...         table_name="measurements",
+        ...         data=np.random.random((1000, 100)),
+        ...         column="data",
+        ...         additional_columns={"name": "experiment_1", "timestamp": 1234567890}
+        ...     )
+        ...     # Load array
+        ...     loaded = db.load_array("measurements", "data", where="name = 'experiment_1'")
+
+        Generic object storage:
+
+        >>> with SQLite3("data.db") as db:
+        ...     db.save_blob(
+        ...         table_name="objects",
+        ...         data={"weights": array, "params": {"lr": 0.001}},
+        ...         key="model_v1"
+        ...     )
+        ...     loaded_obj = db.load_blob("objects", key="model_v1")
+
+    Notes:
+        - Always use context manager (with statement) for proper resource cleanup
+        - BLOB columns automatically get metadata columns: {column}_dtype, {column}_shape, {column}_compressed
+        - Compression is enabled by default for arrays > 1KB
+        - Thread-safe operations are supported
+    """
+
+    def __init__(
+        self,
+        db_path: str,
+        use_temp: bool = False,
+        compress_by_default: bool = False,
+        autocommit: bool = False,
+    ):
+        """Initialize SQLite database manager.
+
+        Parameters
+        ----------
+        db_path : str
+            Path to the SQLite database file
+        use_temp : bool, optional
+            Whether to use a temporary copy of the database, by default False
+        compress_by_default : bool, optional
+            Whether to compress BLOB data by default when not explicitly specified, by default False
+        autocommit : bool, optional
+            Whether to automatically commit transactions, by default False
+
+        Warnings
+        --------
+        UserWarning
+            If not used with context manager, warns about potential resource leaks
+        """
+
+        if not os.path.exists(db_path):
+            os.makedirs(os.path.dirname(db_path), exist_ok=True)
+
+        _ConnectionMixin.__init__(self, db_path, use_temp)
+        self.compress_by_default = compress_by_default
+        self.autocommit = autocommit
+        self._context_manager_used = False
+
+    def __enter__(self):
+        """Enter context manager."""
+        self._context_manager_used = True
+        return self
+
+    def _check_context_manager(self):
+        if not self._context_manager_used:
+            raise RuntimeError(
+                "SQLite3 must be used with context manager: 'with SQLite3(...) as db:'"
+            )
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Exit context manager and ensure proper cleanup."""
+        self.close()
+
+    def __del__(self):
+        """Destructor with context manager usage warning."""
+        if (
+            hasattr(self, "_context_manager_used")
+            and not self._context_manager_used
+        ):
+            warnings.warn(
+                "SQLite3 instance was not used with context manager. "
+                "Use 'with SQLite3(...) as db:' to ensure proper resource cleanup.",
+                UserWarning,
+                stacklevel=2,
+            )
+        if hasattr(self, "close"):
+            self.close()
+
+    def __call__(
+        self,
+        return_summary=False,
+        print_summary=True,
+        table_names: Optional[List[str]] = None,
+        verbose: bool = True,
+        limit: int = 5,
+    ):
+        """Display database summary information.
+
+        Parameters
+        ----------
+        return_summary : bool, optional
+            Whether to return summary dict, by default False
+        print_summary : bool, optional
+            Whether to print summary to console, by default True
+        table_names : Optional[List[str]], optional
+            Specific table names to summarize, by default None (all tables)
+        verbose : bool, optional
+            Whether to show detailed information, by default True
+        limit : int, optional
+            Maximum number of rows to display per table, by default 5
+
+        Returns
+        -------
+        dict or None
+            Summary dictionary if return_summary=True, else None
+        """
+
+        summary = self.get_summaries(
+            table_names=table_names,
+            verbose=verbose,
+            limit=limit,
+        )
+
+        if print_summary:
+            for k, v in summary.items():
+                _printc(f"{k}\n{v}")
+
+        if return_summary:
+            return summary
+
+    @property
+    def summary(self):
+        """Quick access to database summary."""
+        self()
+
+
+BaseSQLiteDB = SQLite3
+
+# EOF