scitex 2.0.0__py2.py3-none-any.whl

scitex/db/_delete_duplicates.py

@@ -0,0 +1,397 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-11 14:16:58 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_delete_duplicates.py
+
+import sqlite3
+from typing import List, Optional, Tuple, Union
+import pandas as pd
+
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-10-20 02:17:10 (ywatanabe)"
+# /data/gpfs/projects/punim2354/ywatanabe/scitex_repo/src/scitex/db/_delete_duplicates_clean.py
+
+
+"""
+Functionality:
+    - Deletes duplicate entries from an SQLite database table
+Input:
+    - SQLite database file path, table name, columns to consider for duplicates
+Output:
+    - Updated SQLite database with duplicates removed
+Prerequisites:
+    - sqlite3, pandas, tqdm, scitex
+"""
+
+
+def _sort_db(cursor: sqlite3.Cursor, table_name: str, columns: List[str]) -> None:
+    """
+    Sorts the database table based on the specified columns.
+
+    Parameters
+    ----------
+    cursor : sqlite3.Cursor
+        The cursor object for executing SQL commands.
+    table_name : str
+        The name of the table to be sorted.
+    columns : List[str]
+        The list of column names to sort by, in order of priority.
+
+    Example
+    -------
+    >>> conn = sqlite3.connect('example.db')
+    >>> cursor = conn.cursor()
+    >>> _sort_db(cursor, 'my_table', ['column1', 'column2'])
+    >>> conn.commit()
+    >>> conn.close()
+    """
+    columns_str = ", ".join(columns)
+    temp_table = f"{table_name}_temp"
+
+    cursor.execute(
+        f"CREATE TABLE {temp_table} AS SELECT * FROM {table_name} ORDER BY {columns_str}"
+    )
+    cursor.execute(f"DROP TABLE {table_name}")
+    cursor.execute(f"ALTER TABLE {temp_table} RENAME TO {table_name}")
+
+
+def _determine_columns(
+    cursor: sqlite3.Cursor,
+    table_name: str,
+    columns: Union[str, List[str]],
+    include_blob: bool,
+) -> List[str]:
+    cursor.execute(f"PRAGMA table_info({table_name})")
+    table_info = cursor.fetchall()
+    all_columns = [col[1] for col in table_info]
+    column_types = {col[1]: col[2] for col in table_info}
+
+    if columns == "all":
+        columns = (
+            all_columns
+            if include_blob
+            else [col for col in all_columns if column_types[col].lower() != "blob"]
+        )
+    elif isinstance(columns, str):
+        columns = [columns]
+
+    columns_str = ", ".join(columns)
+    print(f"Columns considered for duplicates: {columns_str}")
+
+    return columns
+
+
+def _fetch_as_df(
+    cursor: sqlite3.Cursor, columns: List[str], table_name: str
+) -> pd.DataFrame:
+    print("\nFetching all database entries...")
+    columns_str = ", ".join(columns)
+    query = f"SELECT {columns_str} FROM {table_name}"
+    cursor.execute(query)
+    df_entries = cursor.fetchall()
+    return pd.DataFrame(df_entries, columns=columns)
+
+
+def _find_duplicated(df: pd.DataFrame) -> pd.DataFrame:
+    df_duplicated = df[df.duplicated(keep="first")].copy()
+    duplication_rate = len(df_duplicated) / (len(df) - len(df_duplicated))
+    print(f"\n{100*duplication_rate:.2f}% of data was duplicated. Cleaning up...")
+    print(f"\nOriginal entries:\n{df.head()}")
+    print(f"\nDuplicated entries:\n{df_duplicated.head()}")
+    return df_duplicated
+
+
+def verify_duplicated_index(
+    cursor: sqlite3.Cursor, duplicated_row: pd.Series, table_name: str, dry_run: bool
+) -> Tuple[str, bool]:
+    """Check if entry to delete is the one intended"""
+    columns = list(duplicated_row.index)
+    columns_str = ", ".join(columns)
+
+    where_conditions = " AND ".join([f"{col} = ?" for col in columns])
+    select_query = f"""
+        SELECT {columns_str}
+        FROM {table_name}
+        WHERE {where_conditions}
+    """
+    cursor.execute(select_query, tuple(duplicated_row))
+    entries = cursor.fetchall()
+
+    is_verified = len(entries) >= 1
+
+    if dry_run:
+        print(f"Expected duplicate entry: {tuple(duplicated_row)}")
+        print(f"Found entries: {entries}")
+        print(f"Verification {'succeeded' if is_verified else 'failed'}")
+
+    return select_query, is_verified
+
+
+def _delete_entry(
+    cursor: sqlite3.Cursor,
+    duplicated_row: pd.Series,
+    table_name: str,
+    dry_run: bool = True,
+) -> None:
+    select_query, is_verified = verify_duplicated_index(
+        cursor, duplicated_row, table_name, dry_run
+    )
+    if is_verified:
+        delete_query = select_query.replace("SELECT", "DELETE")
+        if dry_run:
+            print(f"[DRY RUN] Would delete entry:\n{duplicated_row}")
+        else:
+            cursor.execute(delete_query, tuple(duplicated_row))
+            print(f"Deleted entry:\n{duplicated_row}")
+    else:
+        print(f"Skipping entry (not found or already deleted):\n{duplicated_row}")
+
+
+# def delete_duplicates(
+#     lpath_db: str,
+#     table_name: str,
+#     columns: Union[str, List[str]] = "all",
+#     include_blob: bool = False,
+#     batch_size: int = 1000,
+#     reindex: bool = False,
+#     sort: bool = False,
+#     dry_run: bool = True,
+# ) -> Tuple[Optional[pd.DataFrame], Optional[pd.DataFrame]]:
+#     """
+#     Delete duplicate entries from an SQLite database table.
+
+#     Parameters
+#     ----------
+#     lpath_db : str
+#         Path to the SQLite database file.
+#     table_name : str
+#         Name of the table to remove duplicates from.
+#     columns : Union[str, List[str]], optional
+#         Columns to consider when identifying duplicates. Default is "all".
+#     include_blob : bool, optional
+#         Whether to include BLOB columns when considering duplicates. Default is False.
+#     batch_size : int, optional
+#         Number of rows to process in each batch. Default is 1000.
+#     reindex : bool, optional
+#         Whether to reindex the table after deletion. Default is False.
+#     dry_run : bool, optional
+#         If True, simulates the deletion without actually modifying the database. Default is True.
+
+#     Returns
+#     -------
+#     Tuple[Optional[pd.DataFrame], Optional[pd.DataFrame]]
+#         A tuple containing:
+#         - DataFrame of all entries after deletion process
+#         - DataFrame of remaining duplicates if any, None otherwise
+#     """
+#     try:
+#         conn = sqlite3.connect(lpath_db)
+#         cursor = conn.cursor()
+
+#         columns = _determine_columns(cursor, table_name, columns, include_blob)
+
+#         if sort:
+#             _sort_db(cursor, table_name, columns)
+
+#         df_orig = _fetch_as_df(cursor, columns, table_name)
+#         duplicates = _find_duplicated(df_orig)
+
+#         if duplicates.empty:
+#             print("Congratulations. Database is clean.")
+#             return df_orig, None
+
+#         columns_str = ", ".join(columns)
+#         where_conditions = " AND ".join([f"{col} = ?" for col in columns])
+#         delete_query = f"""
+#             DELETE FROM {table_name}
+#             WHERE {where_conditions}
+#         """
+
+#         for start in tqdm(range(0, len(duplicates), batch_size)):
+#             batch = duplicates.iloc[start:start+batch_size]
+#             batch_values = batch.values.tolist()
+
+#             if dry_run:
+#                 print(f"[DRY RUN] Would delete {len(batch)} entries")
+#             else:
+#                 cursor.executemany(delete_query, batch_values)
+#                 conn.commit()
+
+#         if not dry_run:
+#             conn.commit()
+
+#             if reindex:
+#                 print("Reindexing the table...")
+#                 cursor.execute(f"REINDEX {table_name}")
+#                 conn.commit()
+
+#         df_after = _fetch_as_df(cursor, columns, table_name)
+#         remaining_duplicates = _find_duplicated(df_after)
+
+#         if remaining_duplicates.empty:
+#             print("All duplicates successfully removed.")
+#             return df_after, None
+#         else:
+#             print(f"Warning: {len(remaining_duplicates)} duplicates still remain.\n{remaining_duplicates}")
+#             return df_after, remaining_duplicates
+
+#     except Exception as error:
+#         print(f"An error occurred: {error}")
+#         return None, None
+
+#     finally:
+#         conn.close()
+
+# def delete_duplicates(
+#     lpath_db: str,
+#     table_name: str,
+#     columns: Union[str, List[str]] = "all",
+#     include_blob: bool = False,
+#     batch_size: int = 1000,
+#     chunk_size: int = 100_000,
+#     reindex: bool = False,
+#     sort: bool = False,
+#     dry_run: bool = True,
+# ) -> Tuple[Optional[int], Optional[int]]:
+#     try:
+#         conn = sqlite3.connect(lpath_db)
+#         cursor = conn.cursor()
+
+#         columns = _determine_columns(cursor, table_name, columns, include_blob)
+
+#         if sort:
+#             _sort_db(cursor, table_name, columns)
+
+#         columns_str = ", ".join(columns)
+#         where_conditions = " AND ".join([f"{col} = ?" for col in columns])
+#         delete_query = f"""
+#             DELETE FROM {table_name}
+#             WHERE {where_conditions}
+#         """
+
+#         total_rows = cursor.execute(f"SELECT COUNT(*) FROM {table_name}").fetchone()[0]
+#         total_deleted = 0
+#         total_duplicates = 0
+
+#         for offset in tqdm(range(0, total_rows, chunk_size)):
+#             chunk_query = f"""
+#                 SELECT {columns_str}
+#                 FROM {table_name}
+#                 LIMIT {chunk_size} OFFSET {offset}
+#             """
+#             df_chunk = pd.read_sql_query(chunk_query, conn)
+#             duplicates = _find_duplicated(df_chunk)
+#             total_duplicates += len(duplicates)
+
+#             if duplicates.empty:
+#                 continue
+
+#             for start in range(0, len(duplicates), batch_size):
+#                 batch = duplicates.iloc[start:start+batch_size]
+#                 batch_values = batch.values.tolist()
+
+#                 if dry_run:
+#                     print(f"[DRY RUN] Would delete {len(batch)} entries")
+#                 else:
+#                     cursor.executemany(delete_query, batch_values)
+#                     conn.commit()
+#                     total_deleted += len(batch)
+
+#         if not dry_run:
+#             if reindex:
+#                 print("Reindexing the table...")
+#                 cursor.execute(f"REINDEX {table_name}")
+#                 conn.commit()
+
+#         print(f"Total duplicates found: {total_duplicates}")
+#         print(f"Total entries deleted: {total_deleted}")
+
+#         return total_duplicates, total_deleted
+
+#     except Exception as error:
+#         print(f"An error occurred: {error}")
+#         return None, None
+
+#     finally:
+#         conn.close()
+
+
+def delete_duplicates(
+    lpath_db: str,
+    table_name: str,
+    columns: Union[str, List[str]] = "all",
+    include_blob: bool = False,
+    chunk_size: int = 10_000,
+    dry_run: bool = True,
+) -> Tuple[Optional[int], Optional[int]]:
+    try:
+        conn = sqlite3.connect(lpath_db)
+        cursor = conn.cursor()
+
+        # Vacuum the database to free up space
+        if not dry_run:
+            cursor.execute("VACUUM")
+            conn.commit()
+
+        columns = _determine_columns(cursor, table_name, columns, include_blob)
+        columns_str = ", ".join(columns)
+
+        # Create a temporary table to store unique rows
+        temp_table = f"{table_name}_temp"
+        cursor.execute(
+            f"CREATE TABLE {temp_table} AS SELECT DISTINCT {columns_str} FROM {table_name} LIMIT 0"
+        )
+
+        # Process in small chunks
+        offset = 0
+        total_processed = 0
+        total_unique = 0
+
+        while True:
+            chunk_query = f"""
+                INSERT OR IGNORE INTO {temp_table}
+                SELECT DISTINCT {columns_str}
+                FROM {table_name}
+                LIMIT {chunk_size} OFFSET {offset}
+            """
+
+            if dry_run:
+                print(f"[DRY RUN] Would execute: {chunk_query}")
+            else:
+                cursor.execute(chunk_query)
+                conn.commit()
+
+            rows_affected = cursor.rowcount
+            if rows_affected == 0:
+                break
+
+            total_processed += chunk_size
+            total_unique += rows_affected
+            offset += chunk_size
+
+            print(f"Processed {total_processed} rows, {total_unique} unique")
+
+        total_duplicates = total_processed - total_unique
+
+        if not dry_run:
+            # Replace original table with the deduplicated one
+            cursor.execute(f"DROP TABLE {table_name}")
+            cursor.execute(f"ALTER TABLE {temp_table} RENAME TO {table_name}")
+            conn.commit()
+
+        print(f"Total rows processed: {total_processed}")
+        print(f"Total unique rows: {total_unique}")
+        print(f"Total duplicates removed: {total_duplicates}")
+
+        return total_processed, total_duplicates
+
+    except Exception as error:
+        print(f"An error occurred: {error}")
+        return None, None
+
+    finally:
+        conn.close()
+
+
+# EOF

scitex/db/_inspect.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-11 14:17:00 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_inspect.py
+
+import os
+import sqlite3
+from typing import Any, Dict, List, Optional, Tuple
+
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-10-24 13:13:33 (ywatanabe)"
+# /mnt/ssd/scitex_repo/src/scitex/db/_inspect.py
+
+
+class Inspector:
+    def __init__(self, db_path: str):
+        if not os.path.exists(db_path):
+            raise FileNotFoundError(f"Database file not found: {db_path}")
+        self.db_path = db_path
+
+    def get_table_names(self) -> List[str]:
+        """Retrieves all table names from the database.
+
+        Returns:
+            List[str]: List of table names
+        """
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+            return [table[0] for table in cursor.fetchall()]
+
+    def get_table_info(
+        self, table_name: str
+    ) -> List[Tuple[int, str, str, int, Any, int, str]]:
+        """Retrieves table structure information.
+
+        Args:
+            table_name (str): Name of the table
+
+        Returns:
+            List[Tuple[int, str, str, int, Any, int, str]]: List of column information tuples
+        """
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute(f"PRAGMA table_info({table_name})")
+            columns = cursor.fetchall()
+
+            cursor.execute(f"PRAGMA index_list({table_name})")
+            indexes = cursor.fetchall()
+            pk_columns = []
+            for idx in indexes:
+                if idx[2] == 1:  # Is primary key
+                    cursor.execute(f"PRAGMA index_info({idx[1]})")
+                    pk_columns.extend([info[2] for info in cursor.fetchall()])
+
+            enhanced_columns = []
+            for col in columns:
+                constraints = []
+                if col[1] in pk_columns:
+                    constraints.append("PRIMARY KEY")
+                if col[3] == 1:
+                    constraints.append("NOT NULL")
+                enhanced_columns.append(col + (" ".join(constraints),))
+
+            return enhanced_columns
+
+    def get_sample_data(
+        self, table_name: str, limit: int = 5
+    ) -> Tuple[List[str], List[Tuple], int]:
+        """Retrieves sample data from the specified table.
+
+        Args:
+            table_name (str): Name of the table
+            limit (int, optional): Number of rows to retrieve. Defaults to 5.
+
+        Returns:
+            Tuple[List[str], List[Tuple], int]: Column names, sample data rows, and total row count
+        """
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute(f"SELECT * FROM {table_name} LIMIT {limit}")
+            columns = [description[0] for description in cursor.description]
+            sample_data = cursor.fetchall()
+
+            cursor.execute(f"SELECT COUNT(*) FROM {table_name}")
+            total_rows = cursor.fetchone()[0]
+
+            return columns, sample_data, total_rows
+
+    def inspect(
+        self,
+        table_names: Optional[List[str]] = None,
+        verbose=True,
+    ) -> List[Dict[str, Any]]:
+        import pandas as pd
+
+        if table_names is None:
+            table_names = self.get_table_names()
+
+        data_tables = []
+        for table_name in table_names:
+            columns = self.get_table_info(table_name)
+            column_names, rows, total_rows = self.get_sample_data(table_name)
+
+            meta = {}
+            meta["table_name"] = table_name
+            meta["n_total_rows"] = total_rows
+
+            sample_data = pd.DataFrame(
+                [
+                    {
+                        col: (str(value) if not isinstance(value, bytes) else "<BLOB>")
+                        for col, value in zip(column_names, row)
+                    }
+                    for row in rows
+                ]
+            )
+
+            for k, v in meta.items():
+                sample_data[k] = v
+
+            sample_data = sample_data.set_index(["table_name", "n_total_rows"])
+
+            data_tables.append(sample_data)
+
+        # if len(data_tables) == 1:
+        #     return data_tables[0]
+        # else:
+        #     return tuple(data_tables)
+        return data_tables
+
+
+def inspect(
+    lpath_db: str, table_names: Optional[List[str]] = None, verbose: bool = True
+) -> None:
+    """
+    Inspects the specified SQLite database.
+
+    Example:
+    >>> inspect('path/to/database.db')
+    >>> inspect('path/to/database.db', ['table1', 'table2'])
+
+    Args:
+        lpath_db (str): Path to the SQLite database file
+        table_names (Optional[List[str]], optional): List of table names to inspect.
+            If None, inspects all tables. Defaults to None.
+    """
+    inspector = Inspector(lpath_db)
+    overviews_tables = inspector.inspect(table_names, verbose=verbose)
+    if verbose:
+        for dd in overviews_tables:
+            print(f"\n{dd}\n")
+    return overviews_tables
+
+
+# python -c "import scitex; scitex.db.inspect(\"./data/db_all/Patient_23_005.db\")"
+# python -c "import scitex; scitex.db.inspect(\"./data/db_all/Patient_23_005.db\", table_names=[\"eeg_data_reindexed\"])"
+# python -c "import scitex; scitex.db.inspect(\"./data/db_all/Patient_23_005.db\", table_names=[\"eeg_data\"])"
+# python -c "import scitex; scitex.db.inspect(\"./data/db_all/Patient_23_005.db\", table_names=[\"sqlite_sequence\"])"
+
+
+# EOF

scitex/decorators/__init__.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+"""Scitex decorators module."""
+
+from ._auto_order import AutoOrderDecorator, batch_fn, disable_auto_order, enable_auto_order, numpy_fn, pandas_fn, torch_fn
+from ._batch_fn import batch_fn
+from ._cache_disk import cache_disk
+from ._cache_mem import cache_mem
+from ._combined import batch_numpy_fn, batch_pandas_fn, batch_torch_fn, numpy_batch_fn, pandas_batch_fn, torch_batch_fn
+from ._converters import ConversionWarning, is_cuda, is_nested_decorator, is_torch, to_numpy, to_torch
+from ._deprecated import deprecated
+from ._not_implemented import not_implemented
+from ._numpy_fn import numpy_fn
+from ._pandas_fn import pandas_fn
+from ._preserve_doc import preserve_doc
+from ._signal_fn import signal_fn
+from ._timeout import timeout
+from ._torch_fn import torch_fn
+from ._wrap import wrap
+from ._xarray_fn import xarray_fn
+
+__all__ = [
+    "AutoOrderDecorator",
+    "ConversionWarning",
+    "batch_fn",
+    "batch_fn",
+    "batch_numpy_fn",
+    "batch_pandas_fn",
+    "batch_torch_fn",
+    "cache_disk",
+    "cache_mem",
+    "deprecated",
+    "disable_auto_order",
+    "enable_auto_order",
+    "is_cuda",
+    "is_nested_decorator",
+    "is_torch",
+    "not_implemented",
+    "numpy_batch_fn",
+    "numpy_fn",
+    "numpy_fn",
+    "pandas_batch_fn",
+    "pandas_fn",
+    "pandas_fn",
+    "preserve_doc",
+    "signal_fn",
+    "timeout",
+    "to_numpy",
+    "to_torch",
+    "torch_batch_fn",
+    "torch_fn",
+    "torch_fn",
+    "wrap",
+    "xarray_fn",
+]