scitex 2.0.0__py2.py3-none-any.whl

scitex/db/_SQLite3Mixins/_BatchMixin.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-29 04:36:14 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_SQLite3Mixins/_BatchMixin.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_SQLite3Mixins/_BatchMixin.py"
+
+"""
+Functionality:
+    * Provides batch database operations for SQLite3
+    * Handles batch inserts, updates, replacements with transaction safety
+    * Supports foreign key inheritance and where clause filtering
+
+Input:
+    * Table name, row data as dictionaries, and operation parameters
+    * Support for batch size control and conditional operations
+
+Output:
+    * None, but executes database operations with transaction safety
+
+Prerequisites:
+    * SQLite3 database connection
+    * Table schema must exist
+    * Foreign key constraints must be enabled if using inherit_foreign
+"""
+
+from typing import Any as _Any
+from typing import Dict, List, Optional
+from .._BaseMixins._BaseBatchMixin import _BaseBatchMixin
+import sqlite3
+
+
+class _BatchMixin:
+    """Batch operations functionality"""
+
+    def _run_many(
+        self,
+        sql_command,
+        table_name: str,
+        rows: List[Dict[str, _Any]],
+        batch_size: int = 1000,
+        inherit_foreign: bool = True,
+        where: Optional[str] = None,
+        columns: Optional[List[str]] = None,
+    ) -> None:
+        try:
+            if batch_size <= 0:
+                raise ValueError("Batch size must be positive")
+
+            table_name = f'"{table_name.replace("`", "``")}"'
+
+            # Validate table exists and get schema
+            schema = self.get_table_schema(table_name)
+            table_columns = set(schema["name"])
+
+            # Replace the problematic code block with:
+            if columns:
+                valid_columns = [
+                    col
+                    for col in columns
+                    if col in table_columns and col.isidentifier()
+                ]
+            else:
+                valid_columns = [
+                    col
+                    for col in rows[0].keys()
+                    if col in table_columns and col.isidentifier()
+                ]
+
+            if not valid_columns:
+                raise ValueError("No valid columns found")
+
+            if not table_name or not isinstance(table_name, str):
+                raise ValueError("Invalid table name")
+            if not isinstance(rows, list):
+                raise ValueError("Rows must be a list of dictionaries")
+            if rows and not all(isinstance(row, dict) for row in rows):
+                raise ValueError("All rows must be dictionaries")
+
+            # Validate table exists
+            self.execute(f"SELECT 1 FROM {table_name} LIMIT 1")
+
+            assert sql_command.upper() in [
+                "INSERT",
+                "REPLACE",
+                "INSERT OR REPLACE",
+                "UPDATE",
+            ]
+
+            if not rows:
+                return
+
+            if sql_command.upper() == "UPDATE":
+                valid_columns = columns if columns else [col for col in rows[0].keys()]
+                set_clause = ",".join([f"{col}=?" for col in valid_columns])
+                where_clause = where if where else "1=1"
+
+                # Modified query construction
+                query = f"UPDATE {table_name} SET {set_clause} WHERE {where_clause}"
+                params = []
+
+                for row in rows:
+                    row_params = [row[col] for col in valid_columns]
+                    if where:
+                        # Add any parameters needed for WHERE clause
+                        row_params.extend(
+                            [row[col] for col in where.split() if col in row]
+                        )
+                    params.append(tuple(row_params))
+
+                self.executemany(query, params)
+                return
+
+            if where:
+                filtered_rows = []
+                for row in rows:
+                    try:
+                        test_query = f"SELECT 1 FROM (SELECT {','.join('? as ' + k for k in row.keys())}) WHERE {where}"
+                        values = tuple(row.values())
+                        result = self.execute(test_query, values).fetchone()
+                        if result:
+                            filtered_rows.append(row)
+                    except Exception as e:
+                        print(f"Warning: Where clause evaluation failed for row: {e}")
+                rows = filtered_rows
+            schema = self.get_table_schema(table_name)
+            table_columns = set(schema["name"])
+            valid_columns = [col for col in rows[0].keys()]
+
+            if inherit_foreign:
+                fk_query = f"PRAGMA foreign_key_list({table_name})"
+                foreign_keys = self.execute(fk_query).fetchall()
+
+                for row in rows:
+                    for fk in foreign_keys:
+                        ref_table, from_col, to_col = fk[2], fk[3], fk[4]
+                        if from_col not in row or row[from_col] is None:
+                            if to_col in row:
+                                query = f"SELECT {from_col} FROM {ref_table} WHERE {to_col} = ?"
+                                result = self.execute(query, (row[to_col],)).fetchone()
+                                if result:
+                                    row[from_col] = result[0]
+
+            columns = valid_columns
+            placeholders = ",".join(["?" for _ in columns])
+            query = f"{sql_command} INTO {table_name} ({','.join(columns)}) VALUES ({placeholders})"
+
+            for idx in range(0, len(rows), batch_size):
+                batch = rows[idx : idx + batch_size]
+                values = [[row.get(col) for col in valid_columns] for row in batch]
+                self.executemany(query, values)
+
+        except sqlite3.Error as e:
+            self.rollback()
+            raise ValueError(f"Batch operation failed: {e}")
+
+    def update_many(
+        self,
+        table_name: str,
+        rows: List[Dict[str, _Any]],
+        batch_size: int = 1000,
+        where: Optional[str] = None,
+        columns: Optional[List[str]] = None,
+    ) -> None:
+        with self.transaction():
+            self._run_many(
+                sql_command="UPDATE",
+                table_name=table_name,
+                rows=rows,
+                batch_size=batch_size,
+                inherit_foreign=False,
+                where=where,
+                columns=columns,
+            )
+
+    def insert_many(
+        self,
+        table_name: str,
+        rows: List[Dict[str, _Any]],
+        batch_size: int = 1000,
+        inherit_foreign: bool = True,
+        where: Optional[str] = None,
+    ) -> None:
+        with self.transaction():
+            self._run_many(
+                sql_command="INSERT",
+                table_name=table_name,
+                rows=rows,
+                batch_size=batch_size,
+                inherit_foreign=inherit_foreign,
+                where=where,
+            )
+
+    def replace_many(
+        self,
+        table_name: str,
+        rows: List[Dict[str, _Any]],
+        batch_size: int = 1000,
+        inherit_foreign: bool = True,
+        where: Optional[str] = None,
+    ) -> None:
+        with self.transaction():
+            self._run_many(
+                sql_command="REPLACE",
+                table_name=table_name,
+                rows=rows,
+                batch_size=batch_size,
+                inherit_foreign=inherit_foreign,
+                where=where,
+            )
+
+    def delete_where(
+        self, table_name: str, where: str, limit: Optional[int] = None
+    ) -> None:
+        with self.transaction():
+            if not where or not isinstance(where, str):
+                raise ValueError("Invalid where clause")
+            params = []
+            query = f"DELETE FROM {table_name} WHERE {where}"
+            if limit is not None:
+                if not isinstance(limit, int) or limit <= 0:
+                    raise ValueError("Limit must be a positive integer")
+                query += f" LIMIT {limit}"
+            self.execute(query, params)
+
+    def update_where(
+        self,
+        table_name: str,
+        updates: Dict[str, _Any],
+        where: str,
+        limit: Optional[int] = None,
+    ) -> None:
+        with self.transaction():
+            if not updates:
+                raise ValueError("Updates dictionary cannot be empty")
+            set_clause = ", ".join([f"{col} = ?" for col in updates.keys()])
+            query = f"UPDATE {table_name} SET {set_clause} WHERE {where}"
+            if limit is not None:
+                query += f" LIMIT {limit}"
+            self.execute(query, tuple(updates.values()))
+
+
+# EOF

scitex/db/_SQLite3Mixins/_BlobMixin.py

@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-12-01 05:13:49 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_SQLite3Mixins/_BlobMixin.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_SQLite3Mixins/_BlobMixin.py"
+
+import sqlite3
+from typing import Any as _Any
+from typing import Dict, List, Optional, Tuple, Union
+
+import numpy as np
+from .._BaseMixins._BaseBlobMixin import _BaseBlobMixin
+
+
+class _BlobMixin:
+    """BLOB data handling functionality"""
+
+    def save_array(
+        self,
+        table_name: str,
+        data: np.ndarray,
+        column: str = "data",
+        ids: Optional[Union[int, List[int]]] = None,
+        where: str = None,
+        additional_columns: Dict[str, _Any] = None,
+        batch_size: int = 1000,
+    ) -> None:
+        with self.lock:
+            if not isinstance(data, (np.ndarray, list)):
+                raise ValueError("Input must be a NumPy array or list of arrays")
+
+            try:
+                if ids is not None:
+                    if isinstance(ids, int):
+                        ids = [ids]
+                        data = [data]
+                    if len(ids) != len(data):
+                        raise ValueError("Length of ids must match number of arrays")
+
+                    for id_, arr in zip(ids, data):
+                        if not isinstance(arr, np.ndarray):
+                            raise ValueError(
+                                f"Element for id {id_} must be a NumPy array"
+                            )
+
+                        binary = arr.tobytes()
+                        columns = [
+                            column,
+                            f"{column}_dtype",
+                            f"{column}_shape",
+                        ]
+                        values = [binary, str(arr.dtype), str(arr.shape)]
+
+                        if additional_columns:
+                            columns = list(additional_columns.keys()) + columns
+                            values = list(additional_columns.values()) + values
+
+                        update_cols = [f"{col}=?" for col in columns]
+                        query = f"UPDATE {table_name} SET {','.join(update_cols)} WHERE id=?"
+                        values.append(id_)
+                        self.execute(query, tuple(values))
+
+                else:
+                    if not isinstance(data, np.ndarray):
+                        raise ValueError("Single input must be a NumPy array")
+
+                    binary = data.tobytes()
+                    columns = [column, f"{column}_dtype", f"{column}_shape"]
+                    values = [binary, str(data.dtype), str(data.shape)]
+
+                    if additional_columns:
+                        columns = list(additional_columns.keys()) + columns
+                        values = list(additional_columns.values()) + values
+
+                    if where is not None:
+                        update_cols = [f"{col}=?" for col in columns]
+                        query = f"UPDATE {table_name} SET {','.join(update_cols)} WHERE {where}"
+                        self.execute(query, tuple(values))
+                    else:
+                        placeholders = ",".join(["?" for _ in columns])
+                        columns_str = ",".join(columns)
+                        query = f"INSERT INTO {table_name} ({columns_str}) VALUES ({placeholders})"
+                        self.execute(query, tuple(values))
+
+            except Exception as err:
+                raise ValueError(f"Failed to save array: {err}")
+
+    def load_array(
+        self,
+        table_name: str,
+        column: str,
+        ids: Union[int, List[int], str] = "all",
+        where: str = None,
+        order_by: str = None,
+        batch_size: int = 128,
+        dtype: np.dtype = None,
+        shape: Optional[Tuple] = None,
+    ) -> Optional[np.ndarray]:
+        try:
+            if ids == "all":
+                query = f"SELECT id FROM {table_name}"
+                if where:
+                    query += f" WHERE {where}"
+                self.cursor.execute(query)
+                ids = [row[0] for row in self.cursor.fetchall()]
+            elif isinstance(ids, int):
+                ids = [ids]
+
+            id_to_data = {}
+            unique_ids = list(set(ids))
+
+            for idx in range(0, len(unique_ids), batch_size):
+                batch_ids = unique_ids[idx : idx + batch_size]
+                placeholders = ",".join("?" for _ in batch_ids)
+
+                try:
+                    query = f"""
+                        SELECT id, {column},
+                               {column}_dtype,
+                               {column}_shape
+                        FROM {table_name}
+                        WHERE id IN ({placeholders})
+                    """
+                    self.cursor.execute(query, tuple(batch_ids))
+                    has_metadata = True
+                except sqlite3.OperationalError:
+                    query = f"SELECT id, {column} FROM {table_name} WHERE id IN ({placeholders})"
+                    self.cursor.execute(query, tuple(batch_ids))
+                    has_metadata = False
+
+                if where:
+                    query += f" AND {where}"
+                if order_by:
+                    query += f" ORDER BY {order_by}"
+
+                results = self.cursor.fetchall()
+                if results:
+                    for result in results:
+                        if has_metadata:
+                            id_val, blob, dtype_str, shape_str = result
+                            data = np.frombuffer(
+                                blob, dtype=np.dtype(dtype_str)
+                            ).reshape(eval(shape_str))
+                        else:
+                            id_val, blob = result
+                            data = (
+                                np.frombuffer(blob, dtype=dtype)
+                                if dtype
+                                else np.frombuffer(blob)
+                            )
+                            if shape:
+                                data = data.reshape(shape)
+                        id_to_data[id_val] = data
+
+            all_data = [id_to_data[id_val] for id_val in ids if id_val in id_to_data]
+            return np.stack(all_data, axis=0) if all_data else None
+
+        except Exception as err:
+            raise ValueError(f"Failed to load array: {err}")
+
+    def binary_to_array(
+        self,
+        binary_data,
+        dtype_str=None,
+        shape_str=None,
+        dtype=None,
+        shape=None,
+    ):
+        if binary_data is None:
+            return None
+
+        if dtype_str and shape_str:
+            return np.frombuffer(binary_data, dtype=np.dtype(dtype_str)).reshape(
+                eval(shape_str)
+            )
+        elif dtype and shape:
+            return np.frombuffer(binary_data, dtype=dtype).reshape(shape)
+        return binary_data
+
+    def get_array_dict(self, df, columns=None, dtype=None, shape=None):
+        result = {}
+        if columns is None:
+            columns = [
+                col
+                for col in df.columns
+                if not (col.endswith("_dtype") or col.endswith("_shape"))
+            ]
+
+        for col in columns:
+            if f"{col}_dtype" in df.columns and f"{col}_shape" in df.columns:
+                arrays = [
+                    self.binary_to_array(
+                        row[col], row[f"{col}_dtype"], row[f"{col}_shape"]
+                    )
+                    for _, row in df.iterrows()
+                ]
+            elif dtype and shape:
+                arrays = [
+                    self.binary_to_array(x, dtype=dtype, shape=shape) for x in df[col]
+                ]
+            result[col] = np.stack(arrays)
+
+        return result
+
+    def decode_array_columns(self, df, columns=None, dtype=None, shape=None):
+        if columns is None:
+            columns = [
+                col
+                for col in df.columns
+                if not (col.endswith("_dtype") or col.endswith("_shape"))
+            ]
+
+        for col in columns:
+            if f"{col}_dtype" in df.columns and f"{col}_shape" in df.columns:
+                df[col] = df.apply(
+                    lambda row: self.binary_to_array(
+                        row[col], row[f"{col}_dtype"], row[f"{col}_shape"]
+                    ),
+                    axis=1,
+                )
+            elif dtype and shape:
+                df[col] = df[col].apply(
+                    lambda x: self.binary_to_array(x, dtype=dtype, shape=shape)
+                )
+        return df
+
+
+# EOF

scitex/db/_SQLite3Mixins/_ConnectionMixin.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-29 04:33:58 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_SQLite3Mixins/_ConnectionMixin.py
+
+THIS_FILE = (
+    "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_SQLite3Mixins/_ConnectionMixin.py"
+)
+
+"""
+1. Functionality:
+   - Manages SQLite database connections with thread-safe operations
+   - Handles database journal files and transaction states
+2. Input:
+   - Database file path
+3. Output:
+   - Managed SQLite connection and cursor objects
+4. Prerequisites:
+   - sqlite3
+   - threading
+"""
+
+import sqlite3
+import threading
+from typing import Optional
+import os
+import shutil
+import tempfile
+from .._BaseMixins._BaseConnectionMixin import _BaseConnectionMixin
+import contextlib
+
+
+class _ConnectionMixin:
+    """Connection management functionality"""
+
+    def __init__(self, db_path: str, use_temp_db: bool = False):
+        self.lock = threading.Lock()
+        self._maintenance_lock = threading.Lock()
+        self.db_path = db_path
+        self.conn = None
+        self.cursor = None
+        self.temp_path = None  # Initialize temp_path attribute
+        if db_path:
+            self.connect(db_path, use_temp_db)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    def _create_temp_copy(self, db_path: str) -> str:
+        """Creates temporary copy of database."""
+        temp_dir = tempfile.gettempdir()
+        self.temp_path = os.path.join(temp_dir, f"temp_{os.path.basename(db_path)}")
+        shutil.copy2(db_path, self.temp_path)
+        return self.temp_path
+
+    def connect(self, db_path: str, use_temp_db: bool = False) -> None:
+        if self.conn:
+            self.close()
+
+        path_to_connect = self._create_temp_copy(db_path) if use_temp_db else db_path
+
+        self.conn = sqlite3.connect(path_to_connect, timeout=60.0)
+        self.cursor = self.conn.cursor()
+
+        with self.lock:
+            # WAL mode settings
+            self.cursor.execute("PRAGMA journal_mode = WAL")
+            self.cursor.execute("PRAGMA synchronous = NORMAL")
+            self.cursor.execute("PRAGMA busy_timeout = 60000")
+            self.cursor.execute("PRAGMA mmap_size = 30000000000")
+            self.cursor.execute("PRAGMA temp_store = MEMORY")
+            self.cursor.execute("PRAGMA cache_size = -2000")
+            self.conn.commit()
+
+    def close(self) -> None:
+        if self.cursor:
+            self.cursor.close()
+        if self.conn:
+            try:
+                self.conn.rollback()
+                self.conn.close()
+            except sqlite3.Error:
+                pass
+        self.cursor = None
+        self.conn = None
+
+        if (
+            hasattr(self, "temp_path")
+            and self.temp_path
+            and os.path.exists(self.temp_path)
+        ):
+            try:
+                os.remove(self.temp_path)
+                self.temp_path = None
+            except OSError:
+                pass
+
+    def reconnect(self, use_temp_db: bool = False) -> None:
+        if self.db_path:
+            self.connect(self.db_path, use_temp_db)
+        else:
+            raise ValueError("No database path specified for reconnection")
+
+
+# EOF

scitex/db/_SQLite3Mixins/_ImportExportMixin.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-25 01:36:18 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_SQLite3Mixins/_ImportExportMixin.py
+
+THIS_FILE = (
+    "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_SQLite3Mixins/_ImportExportMixin.py"
+)
+
+from typing import List
+
+import pandas as pd
+
+from .._BaseMixins._BaseImportExportMixin import _BaseImportExportMixin
+
+
+class _ImportExportMixin:
+    """Import/Export functionality"""
+
+    def load_from_csv(
+        self,
+        table_name: str,
+        csv_path: str,
+        if_exists: str = "append",
+        batch_size: int = 10_000,
+        chunk_size: int = 100_000,
+    ) -> None:
+        with self.transaction():
+            try:
+                for chunk in pd.read_csv(csv_path, chunksize=chunk_size):
+                    chunk.to_sql(
+                        table_name,
+                        self.conn,
+                        if_exists=if_exists,
+                        index=False,
+                        chunksize=batch_size,
+                    )
+                    if_exists = "append"
+            except FileNotFoundError:
+                raise FileNotFoundError(f"CSV file not found: {csv_path}")
+            except Exception as err:
+                raise ValueError(f"Failed to import from CSV: {err}")
+
+    def save_to_csv(
+        self,
+        table_name: str,
+        output_path: str,
+        columns: List[str] = ["*"],
+        where: str = None,
+        batch_size: int = 10_000,
+    ) -> None:
+        try:
+            df = self.get_rows(
+                columns=columns,
+                table_name=table_name,
+                where=where,
+                limit=batch_size,
+                offset=0,
+            )
+            df.to_csv(output_path, index=False, mode="w")
+
+            offset = batch_size
+            while len(df) == batch_size:
+                df = self.get_rows(
+                    columns=columns,
+                    table_name=table_name,
+                    where=where,
+                    limit=batch_size,
+                    offset=offset,
+                )
+                if len(df) > 0:
+                    df.to_csv(output_path, index=False, mode="a", header=False)
+                offset += batch_size
+        except PermissionError:
+            raise PermissionError(f"Cannot write to: {output_path}")
+        except Exception as err:
+            raise ValueError(f"Failed to export to CSV: {err}")
+
+
+# EOF

scitex/db/_SQLite3Mixins/_IndexMixin.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-25 01:36:45 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_SQLite3Mixins/_IndexMixin.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_SQLite3Mixins/_IndexMixin.py"
+
+from typing import List
+from .._BaseMixins._BaseIndexMixin import _BaseIndexMixin
+
+
+class _IndexMixin:
+    """Index management functionality"""
+
+    def create_index(
+        self,
+        table_name: str,
+        column_names: List[str],
+        index_name: str = None,
+        unique: bool = False,
+    ) -> None:
+        if index_name is None:
+            index_name = f"idx_{table_name}_{'_'.join(column_names)}"
+        unique_clause = "UNIQUE" if unique else ""
+        query = f"CREATE {unique_clause} INDEX IF NOT EXISTS {index_name} ON {table_name} ({','.join(column_names)})"
+        self.execute(query)
+
+    def drop_index(self, index_name: str) -> None:
+        self.execute(f"DROP INDEX IF EXISTS {index_name}")
+
+
+# EOF