scitex 2.0.0__py2.py3-none-any.whl

scitex/db/_SQLite3Mixins/_MaintenanceMixin.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-25 01:37:20 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_SQLite3Mixins/_MaintenanceMixin.py
+
+THIS_FILE = (
+    "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_SQLite3Mixins/_MaintenanceMixin.py"
+)
+
+import contextlib
+import sqlite3
+from typing import Callable
+from typing import ContextManager, Dict, List, Optional
+import pandas as pd
+
+from .._BaseMixins._BaseMaintenanceMixin import _BaseMaintenanceMixin
+
+
+class _MaintenanceMixin:
+    """Database maintenance functionality"""
+
+    @contextlib.contextmanager
+    def maintenance_lock(self) -> ContextManager[None]:
+        if not self._maintenance_lock.acquire(timeout=300):
+            raise TimeoutError("Could not acquire maintenance lock")
+        try:
+            yield
+        finally:
+            self._maintenance_lock.release()
+
+    def backup(
+        self,
+        backup_path: str,
+        pages: int = -1,
+        progress: Optional[Callable[[sqlite3.Connection, int, int], None]] = None,
+    ) -> None:
+        with self.maintenance_lock():
+            try:
+
+                def _progress(
+                    status: sqlite3.Connection, remaining: int, total: int
+                ) -> None:
+                    if progress:
+                        progress(total - remaining, total)
+
+                backup_conn = sqlite3.connect(backup_path)
+                with backup_conn:
+                    self.conn.backup(backup_conn, pages=pages, progress=_progress)
+                backup_conn.close()
+            except (sqlite3.Error, Exception) as err:
+                raise ValueError(f"Failed to create backup: {err}")
+
+    def vacuum(self, into: Optional[str] = None) -> None:
+        with self.maintenance_lock():
+            try:
+                if into:
+                    self.execute(f"VACUUM INTO '{into}'")
+                else:
+                    self.execute("VACUUM")
+            except sqlite3.Error as err:
+                raise ValueError(f"Vacuum operation failed: {err}")
+
+    def optimize(self, analyze: bool = True) -> None:
+        with self.maintenance_lock():
+            try:
+                self.execute("PRAGMA optimize")
+                self.vacuum()
+                if analyze:
+                    self.execute("ANALYZE")
+            except sqlite3.Error as err:
+                raise ValueError(f"Failed to optimize database: {err}")
+
+    def get_summaries(
+        self,
+        table_names: Optional[List[str]] = None,
+        verbose: bool = True,
+        limit: int = 5,
+    ) -> Dict[str, pd.DataFrame]:
+        if table_names is None:
+            table_names = self.get_table_names()
+        if isinstance(table_names, str):
+            table_names = [table_names]
+
+        sample_tables = {}
+        for table_name in table_names:
+            columns = self.get_table_schema(table_name)
+            table_sample = self.get_rows(table_name=table_name, limit=limit)
+
+            for column in table_sample.columns:
+                if table_sample[column].dtype == object:
+                    try:
+                        pd.to_datetime(table_sample[column])
+                        continue
+                    except:
+                        pass
+
+                    if table_sample[column].apply(lambda x: isinstance(x, str)).all():
+                        continue
+
+            sample_tables[table_name] = table_sample
+
+        return sample_tables
+
+    def fix_corruption(self) -> bool:
+        """Attempts to fix database corruption"""
+        with self.maintenance_lock():
+            try:
+                # Integrity check
+                integrity_check = self.execute("PRAGMA integrity_check").fetchall()
+                if integrity_check[0][0] == "ok":
+                    return True
+
+                # Backup good data
+                temp_db = f"{self.db_path}_temp"
+                self.execute("PRAGMA writable_schema = ON")
+                self.execute(".dump", output_file=temp_db)
+
+                # Recreate database
+                self.close()
+                os.remove(self.db_path)
+                self.connect(self.db_path)
+
+                # Restore from dump
+                self.execute(f".read {temp_db}")
+                os.remove(temp_db)
+                return True
+
+            except sqlite3.Error as error:
+                raise ValueError(f"Failed to fix corruption: {error}")
+
+    def fix_journal(self) -> bool:
+        """Removes stale journal files and resets journal mode"""
+        with self.maintenance_lock():
+            try:
+                self.close()
+
+                # Remove journal files
+                journal_file = f"{self.db_path}-journal"
+                wal_file = f"{self.db_path}-wal"
+                shm_file = f"{self.db_path}-shm"
+
+                for file in [journal_file, wal_file, shm_file]:
+                    if os.path.exists(file):
+                        os.remove(file)
+
+                # Reconnect and reset journal mode
+                self.connect(self.db_path)
+                self.execute("PRAGMA journal_mode = DELETE")
+                self.execute("PRAGMA synchronous = NORMAL")
+                return True
+
+            except (sqlite3.Error, OSError) as error:
+                raise ValueError(f"Failed to fix journal: {error}")
+
+    def fix_indexes(self) -> bool:
+        """Rebuilds all indexes"""
+        with self.maintenance_lock():
+            try:
+                # Get all indexes
+                indexes = self.execute(
+                    "SELECT name, tbl_name, sql FROM sqlite_master WHERE type='index'"
+                ).fetchall()
+
+                # Drop and recreate each index
+                for name, table, sql in indexes:
+                    if sql:  # Skip internal indexes
+                        self.execute(f"DROP INDEX IF EXISTS {name}")
+                        self.execute(sql)
+
+                return True
+
+            except sqlite3.Error as error:
+                raise ValueError(f"Failed to fix indexes: {error}")
+
+
+# EOF

scitex/db/_SQLite3Mixins/_QueryMixin.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-29 04:31:43 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_SQLite3Mixins/_QueryMixin.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_SQLite3Mixins/_QueryMixin.py"
+
+import sqlite3
+from typing import List, Tuple
+
+import pandas as pd
+from .._BaseMixins._BaseQueryMixin import _BaseQueryMixin
+
+
+class _QueryMixin:
+    """Query execution functionality"""
+
+    def _sanitize_parameters(self, parameters):
+        """Convert pandas Timestamp objects to strings"""
+        if isinstance(parameters, (list, tuple)):
+            return [str(p) if isinstance(p, pd.Timestamp) else p for p in parameters]
+        return parameters
+
+    def execute(self, query: str, parameters: Tuple = ()) -> None:
+        if not self.cursor:
+            raise ConnectionError("Database not connected")
+
+        if any(
+            keyword in query.upper()
+            for keyword in ["INSERT", "UPDATE", "DELETE", "DROP", "CREATE", "ALTER"]
+        ):
+            self._check_writable()
+
+        try:
+            parameters = self._sanitize_parameters(parameters)
+            self.cursor.execute(query, parameters)
+            self.conn.commit()
+            return self.cursor
+        except sqlite3.Error as err:
+            raise sqlite3.Error(f"Query execution failed: {err}")
+
+    def executemany(self, query: str, parameters: List[Tuple]) -> None:
+        if not self.cursor:
+            raise ConnectionError("Database not connected")
+
+        if any(
+            keyword in query.upper()
+            for keyword in ["INSERT", "UPDATE", "DELETE", "DROP", "CREATE", "ALTER"]
+        ):
+            self._check_writable()
+
+        try:
+            parameters = [self._sanitize_parameters(p) for p in parameters]
+            self.cursor.executemany(query, parameters)
+            self.conn.commit()
+        except sqlite3.Error as err:
+            raise sqlite3.Error(f"Batch query execution failed: {err}")
+
+    def executescript(self, script: str) -> None:
+        if not self.cursor:
+            raise ConnectionError("Database not connected")
+
+        if any(
+            keyword in script.upper()
+            for keyword in [
+                "INSERT",
+                "UPDATE",
+                "DELETE",
+                "DROP",
+                "CREATE",
+                "ALTER",
+            ]
+        ):
+            self._check_writable()
+
+        try:
+            self.cursor.executescript(script)
+            self.conn.commit()
+        except sqlite3.Error as err:
+            raise sqlite3.Error(f"Script execution failed: {err}")
+
+
+# EOF

scitex/db/_SQLite3Mixins/_RowMixin.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-25 01:38:17 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_SQLite3Mixins/_RowMixin.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_SQLite3Mixins/_RowMixin.py"
+
+import sqlite3
+from typing import List
+from typing import Optional
+import pandas as pd
+from .._BaseMixins._BaseRowMixin import _BaseRowMixin
+
+
+class _RowMixin:
+    """Row operations functionality"""
+
+    def get_rows(
+        self,
+        table_name: str,
+        columns: List[str] = None,
+        where: str = None,
+        order_by: str = None,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        return_as: str = "dataframe",
+    ):
+        if columns is None:
+            columns_str = "*"
+        elif isinstance(columns, str):
+            columns_str = f'"{columns}"'
+        else:
+            columns_str = ", ".join(f'"{col}"' for col in columns)
+
+        try:
+            query_parts = [f"SELECT {columns_str} FROM {table_name}"]
+
+            if where:
+                query_parts.append(f"WHERE {where}")
+            if order_by:
+                query_parts.append(f"ORDER BY {order_by}")
+            if limit is not None:
+                query_parts.append(f"LIMIT {limit}")
+            if offset is not None:
+                query_parts.append(f"OFFSET {offset}")
+
+            query = " ".join(query_parts)
+            self.cursor.execute(query)
+
+            column_names = [description[0] for description in self.cursor.description]
+            data = self.cursor.fetchall()
+
+            if return_as == "list":
+                return data
+            elif return_as == "dict":
+                return [dict(zip(column_names, row)) for row in data]
+            else:
+                return pd.DataFrame(data, columns=column_names)
+
+        except sqlite3.Error as error:
+            raise sqlite3.Error(f"Query execution failed: {str(error)}")
+
+    def get_row_count(self, table_name: str = None, where: str = None) -> int:
+        if table_name is None:
+            raise ValueError("Table name must be specified")
+
+        query = f"SELECT COUNT(*) FROM {table_name}"
+        if where:
+            query += f" WHERE {where}"
+
+        self.cursor.execute(query)
+        return self.cursor.fetchone()[0]
+
+
+# EOF

scitex/db/_SQLite3Mixins/_TableMixin.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-25 01:38:47 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_SQLite3Mixins/_TableMixin.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_SQLite3Mixins/_TableMixin.py"
+
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-11 19:13:19 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_BaseSQLiteDB_modules/_TableMixin.py
+
+import sqlite3
+from typing import Any, Dict, List, Union
+import pandas as pd
+from .._BaseMixins._BaseTableMixin import _BaseTableMixin
+
+
+class _TableMixin:
+    """Table management functionality"""
+
+    def create_table(
+        self,
+        table_name: str,
+        columns: Dict[str, str],
+        foreign_keys: List[Dict[str, str]] = None,
+        if_not_exists: bool = True,
+    ) -> None:
+        with self.transaction():
+            try:
+                exists_clause = "IF NOT EXISTS " if if_not_exists else ""
+                column_defs = []
+
+                for col_name, col_type in columns.items():
+                    column_defs.append(f"{col_name} {col_type}")
+                    if "BLOB" in col_type.upper():
+                        column_defs.extend(
+                            [
+                                f"{col_name}_dtype TEXT DEFAULT 'unknown'",
+                                f"{col_name}_shape TEXT DEFAULT 'unknown'",
+                            ]
+                        )
+
+                if foreign_keys:
+                    for fk in foreign_keys:
+                        column_defs.append(
+                            f"FOREIGN KEY ({fk['tgt_column']}) REFERENCES {fk['src_table']}({fk['src_column']})"
+                        )
+
+                query = f"CREATE TABLE {exists_clause}{table_name} ({', '.join(column_defs)})"
+                self.execute(query)
+
+            except sqlite3.Error as err:
+                raise ValueError(f"Failed to create table {table_name}: {err}")
+
+    def drop_table(self, table_name: str, if_exists: bool = True) -> None:
+        with self.transaction():
+            try:
+                exists_clause = "IF EXISTS " if if_exists else ""
+                query = f"DROP TABLE {exists_clause}{table_name}"
+                self.execute(query)
+            except sqlite3.Error as err:
+                raise ValueError(f"Failed to drop table: {err}")
+
+    def rename_table(self, old_name: str, new_name: str) -> None:
+        with self.transaction():
+            try:
+                query = f"ALTER TABLE {old_name} RENAME TO {new_name}"
+                self.execute(query)
+            except sqlite3.Error as err:
+                raise ValueError(f"Failed to rename table: {err}")
+
+    def add_columns(
+        self,
+        table_name: str,
+        columns: Dict[str, str],
+        default_values: Dict[str, Any] = None,
+    ) -> None:
+        with self.transaction():
+            if default_values is None:
+                default_values = {}
+
+            for column_name, column_type in columns.items():
+                self.add_column(
+                    table_name,
+                    column_name,
+                    column_type,
+                    default_values.get(column_name),
+                )
+
+    def add_column(
+        self,
+        table_name: str,
+        column_name: str,
+        column_type: str,
+        default_value: Any = None,
+    ) -> None:
+        with self.transaction():
+            schema = self.get_table_schema(table_name)
+            if column_name in schema["name"].values:
+                return
+
+            try:
+                query = (
+                    f"ALTER TABLE {table_name} ADD COLUMN {column_name} {column_type}"
+                )
+                if default_value is not None:
+                    query += f" DEFAULT {default_value}"
+                self.execute(query)
+
+                if "BLOB" in column_type.upper():
+                    self.add_column(
+                        table_name,
+                        f"{column_name}_dtype",
+                        "TEXT",
+                        default_value="'unknown'",
+                    )
+                    self.add_column(
+                        table_name,
+                        f"{column_name}_shape",
+                        "TEXT",
+                        default_value="'unknown'",
+                    )
+
+            except sqlite3.OperationalError as err:
+                raise ValueError(f"Failed to add column: {err}")
+
+    def drop_columns(
+        self,
+        table_name: str,
+        columns: Union[str, List[str]],
+        if_exists: bool = True,
+    ) -> None:
+        with self.transaction():
+            if isinstance(columns, str):
+                columns = [columns]
+            schema = self.get_table_schema(table_name)
+            existing_columns = schema["name"].values
+            columns_to_drop = (
+                [col for col in columns if col in existing_columns]
+                if if_exists
+                else columns
+            )
+
+            if not columns_to_drop:
+                return
+
+            # Drop multiple columns in a single ALTER TABLE statement
+            drop_clause = ", ".join(f"DROP COLUMN {col}" for col in columns_to_drop)
+            self.execute(f"ALTER TABLE {table_name} {drop_clause}")
+
+    def get_table_names(self) -> List[str]:
+        query = "SELECT name FROM sqlite_master WHERE type='table'"
+        self.cursor.execute(query)
+        return [table[0] for table in self.cursor.fetchall()]
+
+    def get_table_schema(self, table_name: str) -> pd.DataFrame:
+        query = f"PRAGMA table_info({table_name})"
+        self.cursor.execute(query)
+        columns = ["cid", "name", "type", "notnull", "dflt_value", "pk"]
+        return pd.DataFrame(self.cursor.fetchall(), columns=columns)
+
+    def get_primary_key(self, table_name: str) -> str:
+        schema = self.get_table_schema(table_name)
+        pk_col = schema[schema["pk"] == 1]["name"].values
+        return pk_col[0] if len(pk_col) > 0 else None
+
+    def get_table_stats(self, table_name: str) -> Dict[str, int]:
+        try:
+            pages = self.cursor.execute(f"PRAGMA page_count").fetchone()[0]
+            page_size = self.cursor.execute(f"PRAGMA page_size").fetchone()[0]
+            row_count = self.get_row_count(table_name)
+            return {
+                "pages": pages,
+                "page_size": page_size,
+                "total_size": pages * page_size,
+                "row_count": row_count,
+            }
+        except sqlite3.Error as err:
+            raise ValueError(f"Failed to get table size: {err}")
+
+
+# EOF

scitex/db/_SQLite3Mixins/_TransactionMixin.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-29 04:32:42 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_SQLite3Mixins/_TransactionMixin.py
+
+THIS_FILE = (
+    "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_SQLite3Mixins/_TransactionMixin.py"
+)
+
+import sqlite3
+import contextlib
+from .._BaseMixins._BaseTransactionMixin import _BaseTransactionMixin
+
+
+class _TransactionMixin:
+    """Transaction management functionality"""
+
+    @contextlib.contextmanager
+    def transaction(self):
+        with self.lock:
+            try:
+                self.begin()
+                yield
+                self.commit()
+            except Exception as e:
+                self.rollback()
+                raise e
+
+    def begin(self) -> None:
+        self.execute("BEGIN TRANSACTION")
+
+    def commit(self) -> None:
+        self.conn.commit()
+
+    def rollback(self) -> None:
+        self.conn.rollback()
+
+    def enable_foreign_keys(self) -> None:
+        self.execute("PRAGMA foreign_keys = ON")
+
+    def disable_foreign_keys(self) -> None:
+        self.execute("PRAGMA foreign_keys = OFF")
+
+    @property
+    def writable(self) -> bool:
+        try:
+            self.cursor.execute("SELECT value FROM _db_state WHERE key = 'writable'")
+            result = self.cursor.fetchone()
+            return result[0].lower() == "true" if result else True
+        except sqlite3.Error:
+            return True
+
+    @writable.setter
+    def writable(self, state: bool) -> None:
+        try:
+            self.execute("UPDATE _db_state SET protected = 0 WHERE key = 'writable'")
+            self.execute(
+                "UPDATE _db_state SET value = ? WHERE key = 'writable'",
+                (str(state).lower(),),
+            )
+            self.execute("UPDATE _db_state SET protected = 1 WHERE key = 'writable'")
+            self.execute("PRAGMA query_only = ?", (not state,))
+        except sqlite3.Error as err:
+            raise ValueError(f"Failed to set writable state: {err}")
+
+    def _check_writable(self) -> None:
+        if not self.writable:
+            raise ValueError("Database is in read-only mode")
+
+
+# EOF

scitex/db/_SQLite3Mixins/__init__.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-12 09:29:50 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_SQLite3Mixins/__init__.py
+
+from ._BatchMixin import _BatchMixin
+from ._BlobMixin import _BlobMixin
+from ._ConnectionMixin import _ConnectionMixin
+from ._ImportExportMixin import _ImportExportMixin
+from ._IndexMixin import _IndexMixin
+from ._MaintenanceMixin import _MaintenanceMixin
+from ._QueryMixin import _QueryMixin
+from ._RowMixin import _RowMixin
+from ._TableMixin import _TableMixin
+from ._TransactionMixin import _TransactionMixin
+
+__all__ = [
+    "_BatchMixin",
+    "_BlobMixin",
+    "_ConnectionMixin",
+    "_ImportExportMixin",
+    "_IndexMixin",
+    "_MaintenanceMixin",
+    "_QueryMixin",
+    "_RowMixin",
+    "_TableMixin",
+    "_TransactionMixin",
+]
+
+# EOF

scitex/db/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""Database operations module for scitex."""
+
+from ._PostgreSQL import PostgreSQL
+from ._SQLite3 import SQLite3
+from ._delete_duplicates import delete_duplicates
+from ._inspect import inspect
+
+__all__ = [
+    "PostgreSQL",
+    "SQLite3", 
+    "delete_duplicates",
+    "inspect",
+]