scitex 2.14.0__py3-none-any.whl

scitex/db/_BaseMixins/_BaseBlobMixin.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-25 01:45:48 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_BaseMixins/_BaseBlobMixin.py
+
+THIS_FILE = (
+    "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_BaseMixins/_BaseBlobMixin.py"
+)
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Tuple, Union
+import numpy as np
+import pandas as pd
+
+
+class _BaseBlobMixin(ABC):
+    """Base class for BLOB data handling functionality"""
+
+    @abstractmethod
+    def save_array(
+        self,
+        table_name: str,
+        data: np.ndarray,
+        column: str = "data",
+        ids: Optional[Union[int, List[int]]] = None,
+        where: str = None,
+        additional_columns: Dict[str, Any] = None,
+        batch_size: int = 1000,
+    ) -> None:
+        """Save numpy array(s) to database"""
+        pass
+
+    @abstractmethod
+    def load_array(
+        self,
+        table_name: str,
+        column: str,
+        ids: Union[int, List[int], str] = "all",
+        where: str = None,
+        order_by: str = None,
+        batch_size: int = 128,
+        dtype: np.dtype = None,
+        shape: Optional[Tuple] = None,
+    ) -> Optional[np.ndarray]:
+        """Load numpy array(s) from database"""
+        pass
+
+    @abstractmethod
+    def binary_to_array(
+        self,
+        binary_data,
+        dtype_str=None,
+        shape_str=None,
+        dtype=None,
+        shape=None,
+    ) -> Optional[np.ndarray]:
+        """Convert binary data to numpy array"""
+        pass
+
+    @abstractmethod
+    def get_array_dict(
+        self,
+        df: pd.DataFrame,
+        columns: Optional[List[str]] = None,
+        dtype: Optional[np.dtype] = None,
+        shape: Optional[Tuple] = None,
+    ) -> Dict[str, np.ndarray]:
+        """Convert DataFrame columns to dictionary of arrays"""
+        pass
+
+    @abstractmethod
+    def decode_array_columns(
+        self,
+        df: pd.DataFrame,
+        columns: Optional[List[str]] = None,
+        dtype: Optional[np.dtype] = None,
+        shape: Optional[Tuple] = None,
+    ) -> pd.DataFrame:
+        """Decode binary columns in DataFrame to numpy arrays"""
+        pass
+
+
+# EOF

scitex/db/_BaseMixins/_BaseConnectionMixin.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-25 06:02:43 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_BaseMixins/_BaseConnectionMixin.py
+
+THIS_FILE = (
+    "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_BaseMixins/_BaseConnectionMixin.py"
+)
+
+import threading
+from typing import Optional
+
+
+class _BaseConnectionMixin:
+    def __init__(self):
+        self.lock = threading.Lock()
+        self._maintenance_lock = threading.Lock()
+        self.conn = None
+        self.cursor = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    def connect(self):
+        raise NotImplementedError
+
+    def close(self):
+        raise NotImplementedError
+
+    def reconnect(self):
+        raise NotImplementedError
+
+    def execute(self, query: str, parameters=()) -> None:
+        raise NotImplementedError
+
+    def executemany(self, query: str, parameters) -> None:
+        raise NotImplementedError
+
+
+# EOF

scitex/db/_BaseMixins/_BaseImportExportMixin.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-24 22:20:15 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_Basemodules/_BaseImportExportMixin.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_Basemodules/_BaseImportExportMixin.py"
+
+
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from typing import List
+
+
+class _BaseImportExportMixin:
+    def load_from_csv(
+        self,
+        table_name: str,
+        csv_path: str,
+        if_exists: str = "append",
+        batch_size: int = 10_000,
+        chunk_size: int = 100_000,
+    ) -> None:
+        raise NotImplementedError
+
+    def save_to_csv(
+        self,
+        table_name: str,
+        output_path: str,
+        columns: List[str] = ["*"],
+        where: str = None,
+        batch_size: int = 10_000,
+    ) -> None:
+        raise NotImplementedError
+
+
+# EOF

scitex/db/_BaseMixins/_BaseIndexMixin.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-24 22:20:26 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_Basemodules/_BaseIndexMixin.py
+
+THIS_FILE = (
+    "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_Basemodules/_BaseIndexMixin.py"
+)
+
+
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from typing import List
+
+
+class _BaseIndexMixin:
+    def create_index(
+        self,
+        table_name: str,
+        column_names: List[str],
+        index_name: str = None,
+        unique: bool = False,
+    ) -> None:
+        raise NotImplementedError
+
+    def drop_index(self, index_name: str) -> None:
+        raise NotImplementedError
+
+
+# EOF

scitex/db/_BaseMixins/_BaseMaintenanceMixin.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-24 22:12:07 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_Basemodules/_BaseMaintenanceMixin.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_Basemodules/_BaseMaintenanceMixin.py"
+
+from typing import Optional, List, Dict
+
+
+class _BaseMaintenanceMixin:
+    def vacuum(self, table: Optional[str] = None):
+        raise NotImplementedError
+
+    def analyze(self, table: Optional[str] = None):
+        raise NotImplementedError
+
+    def reindex(self, table: Optional[str] = None):
+        raise NotImplementedError
+
+    def get_table_size(self, table: str):
+        raise NotImplementedError
+
+    def get_database_size(self):
+        raise NotImplementedError
+
+    def get_table_info(self) -> List[Dict]:
+        raise NotImplementedError
+
+
+# EOF

scitex/db/_BaseMixins/_BaseQueryMixin.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-24 22:17:03 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_Basemodules/_BaseQueryMixin.py
+
+THIS_FILE = (
+    "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_Basemodules/_BaseQueryMixin.py"
+)
+
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from typing import List, Dict, Any, Optional, Union, Tuple
+
+
+class _BaseQueryMixin:
+    def select(
+        self,
+        table: str,
+        columns: Optional[List[str]] = None,
+        where: Optional[str] = None,
+        params: Optional[tuple] = None,
+        order_by: Optional[str] = None,
+        limit: Optional[int] = None,
+    ) -> List[Dict[str, Any]]:
+        raise NotImplementedError
+
+    def insert(self, table: str, data: Dict[str, Any]) -> None:
+        raise NotImplementedError
+
+    def update(
+        self,
+        table: str,
+        data: Dict[str, Any],
+        where: str,
+        params: Optional[tuple] = None,
+    ) -> int:
+        raise NotImplementedError
+
+    def delete(self, table: str, where: str, params: Optional[tuple] = None) -> int:
+        raise NotImplementedError
+
+    def execute_query(
+        self, query: str, params: Optional[tuple] = None
+    ) -> List[Dict[str, Any]]:
+        raise NotImplementedError
+
+    def count(
+        self, table: str, where: Optional[str] = None, params: Optional[tuple] = None
+    ) -> int:
+        raise NotImplementedError
+
+
+# EOF

scitex/db/_BaseMixins/_BaseRowMixin.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-24 22:21:03 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_Basemodules/_BaseRowMixin.py
+
+THIS_FILE = (
+    "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_Basemodules/_BaseRowMixin.py"
+)
+
+
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from typing import List, Optional
+
+
+class _BaseRowMixin:
+    def get_rows(
+        self,
+        table_name: str,
+        columns: List[str] = None,
+        where: str = None,
+        order_by: str = None,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        return_as: str = "dataframe",
+    ):
+        raise NotImplementedError
+
+    def get_row_count(self, table_name: str = None, where: str = None) -> int:
+        raise NotImplementedError
+
+
+# EOF

scitex/db/_BaseMixins/_BaseSchemaMixin.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-24 22:14:24 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_Basemodules/_BaseSchemaMixin.py
+
+THIS_FILE = (
+    "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_Basemodules/_BaseSchemaMixin.py"
+)
+
+from typing import List, Dict, Any, Optional
+
+
+class _BaseSchemaMixin:
+    def get_tables(self) -> List[str]:
+        raise NotImplementedError
+
+    def get_columns(self, table: str) -> List[Dict[str, Any]]:
+        raise NotImplementedError
+
+    def get_primary_keys(self, table: str) -> List[str]:
+        raise NotImplementedError
+
+    def get_foreign_keys(self, table: str) -> List[Dict[str, Any]]:
+        raise NotImplementedError
+
+    def get_indexes(self, table: str) -> List[Dict[str, Any]]:
+        raise NotImplementedError
+
+    def table_exists(self, table: str) -> bool:
+        raise NotImplementedError
+
+    def column_exists(self, table: str, column: str) -> bool:
+        raise NotImplementedError
+
+    def create_index(
+        self, table: str, columns: List[str], index_name: Optional[str] = None
+    ) -> None:
+        raise NotImplementedError
+
+    def drop_index(self, index_name: str) -> None:
+        raise NotImplementedError
+
+
+# EOF

scitex/db/_BaseMixins/_BaseTableMixin.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-24 22:21:17 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_Basemodules/_BaseTableMixin.py
+
+THIS_FILE = (
+    "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_Basemodules/_BaseTableMixin.py"
+)
+
+
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from typing import Any, Dict, List, Union
+
+
+class _BaseTableMixin:
+    def create_table(
+        self,
+        table_name: str,
+        columns: Dict[str, str],
+        foreign_keys: List[Dict[str, str]] = None,
+        if_not_exists: bool = True,
+    ) -> None:
+        raise NotImplementedError
+
+    def drop_table(self, table_name: str, if_exists: bool = True) -> None:
+        raise NotImplementedError
+
+    def rename_table(self, old_name: str, new_name: str) -> None:
+        raise NotImplementedError
+
+    def add_columns(
+        self,
+        table_name: str,
+        columns: Dict[str, str],
+        default_values: Dict[str, Any] = None,
+    ) -> None:
+        raise NotImplementedError
+
+    def add_column(
+        self,
+        table_name: str,
+        column_name: str,
+        column_type: str,
+        default_value: Any = None,
+    ) -> None:
+        raise NotImplementedError
+
+    def drop_columns(
+        self, table_name: str, columns: Union[str, List[str]], if_exists: bool = True
+    ) -> None:
+        raise NotImplementedError
+
+    def get_table_names(self) -> List[str]:
+        raise NotImplementedError
+
+    def get_table_schema(self, table_name: str):
+        raise NotImplementedError
+
+    def get_primary_key(self, table_name: str) -> str:
+        raise NotImplementedError
+
+    def get_table_stats(self, table_name: str) -> Dict[str, int]:
+        raise NotImplementedError
+
+
+# EOF

scitex/db/_BaseMixins/_BaseTransactionMixin.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-24 22:08:33 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_Basemodules/_BaseTransactionMixin.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_Basemodules/_BaseTransactionMixin.py"
+
+
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import contextlib
+
+
+class _BaseTransactionMixin:
+    @contextlib.contextmanager
+    def transaction(self):
+        try:
+            self.begin()
+            yield
+            self.commit()
+        except Exception as e:
+            self.rollback()
+            raise e
+
+    def begin(self):
+        raise NotImplementedError
+
+    def commit(self):
+        raise NotImplementedError
+
+    def rollback(self):
+        raise NotImplementedError
+
+    def enable_foreign_keys(self):
+        raise NotImplementedError
+
+    def disable_foreign_keys(self):
+        raise NotImplementedError
+
+    @property
+    def writable(self):
+        raise NotImplementedError
+
+    @writable.setter
+    def writable(self, state: bool):
+        raise NotImplementedError
+
+
+# EOF

scitex/db/_BaseMixins/__init__.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+"""Base mixins for database functionality."""
+
+from ._BaseBackupMixin import _BaseBackupMixin
+from ._BaseBatchMixin import _BaseBatchMixin
+from ._BaseBlobMixin import _BaseBlobMixin
+from ._BaseConnectionMixin import _BaseConnectionMixin
+from ._BaseImportExportMixin import _BaseImportExportMixin
+from ._BaseIndexMixin import _BaseIndexMixin
+from ._BaseMaintenanceMixin import _BaseMaintenanceMixin
+from ._BaseQueryMixin import _BaseQueryMixin
+from ._BaseRowMixin import _BaseRowMixin
+from ._BaseSchemaMixin import _BaseSchemaMixin
+from ._BaseTableMixin import _BaseTableMixin
+from ._BaseTransactionMixin import _BaseTransactionMixin
+
+__all__ = [
+    "_BaseBackupMixin",
+    "_BaseBatchMixin",
+    "_BaseBlobMixin",
+    "_BaseConnectionMixin",
+    "_BaseImportExportMixin",
+    "_BaseIndexMixin",
+    "_BaseMaintenanceMixin",
+    "_BaseQueryMixin",
+    "_BaseRowMixin",
+    "_BaseSchemaMixin",
+    "_BaseTableMixin",
+    "_BaseTransactionMixin",
+]

scitex/db/__init__.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-07-16 12:49:57 (ywatanabe)"
+# File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/db/__init__.py
+# ----------------------------------------
+import os
+
+__FILE__ = __file__
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+"""Database operations module for scitex."""
+
+from ._postgresql._PostgreSQL import PostgreSQL
+from ._sqlite3._SQLite3 import SQLite3
+from ._sqlite3._delete_duplicates import delete_sqlite3_duplicates
+from ._delete_duplicates import delete_duplicates
+from ._inspect import inspect
+from ._check_health import check_health, batch_health_check
+
+__all__ = [
+    "PostgreSQL",
+    "SQLite3",
+    "delete_duplicates",
+    "delete_sqlite3_duplicates",
+    "inspect",
+]
+
+
+# Clean up namespace - remove private modules from public access
+def _cleanup_namespace():
+    import sys
+
+    current_module = sys.modules[__name__]
+    names_to_remove = []
+    for name in list(vars(current_module).keys()):
+        if (
+            name.startswith("_")
+            and not name.startswith("__")
+            and name not in ["_cleanup_namespace"]
+        ):
+            names_to_remove.append(name)
+    for name in names_to_remove:
+        delattr(current_module, name)
+
+
+_cleanup_namespace()
+del _cleanup_namespace
+
+# EOF

scitex/db/__main__.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-09-10 07:44:49 (ywatanabe)"
+# File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/db/__main__.py
+# ----------------------------------------
+from __future__ import annotations
+import os
+
+__FILE__ = __file__
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+import argparse
+import sys
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Database utilities")
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # Inspect command
+    inspect_parser = subparsers.add_parser("inspect", help="Inspect database structure")
+    inspect_parser.add_argument("db_path", help="Database file path")
+    inspect_parser.add_argument(
+        "--tables", nargs="*", help="Specific tables to inspect"
+    )
+    inspect_parser.add_argument("--quiet", action="store_true", help="Minimal output")
+
+    # Health check command
+    health_parser = subparsers.add_parser("health", help="Check database health")
+    health_parser.add_argument("db_paths", nargs="+", help="Database file paths")
+    health_parser.add_argument(
+        "--fix", action="store_true", help="Attempt to fix issues"
+    )
+    health_parser.add_argument("--quiet", action="store_true", help="Minimal output")
+
+    args = parser.parse_args()
+
+    if args.command == "inspect":
+        from ._inspect import inspect
+
+        inspect(args.db_path, table_names=args.tables, verbose=not args.quiet)
+
+    elif args.command == "health":
+        from ._check_health import batch_health_check, check_health
+
+        if len(args.db_paths) == 1:
+            check_health(args.db_paths[0], verbose=not args.quiet, fix_issues=args.fix)
+        else:
+            batch_health_check(
+                args.db_paths, verbose=not args.quiet, fix_issues=args.fix
+            )
+
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()
+
+# EOF