junshan-kit 2.2.3__py2.py3-none-any.whl → 2.5.1__py2.py3-none-any.whl

junshan_kit/DataHub.py

@@ -0,0 +1,114 @@
+import torchvision, torch
+import torchvision.transforms as transforms
+import pandas as pd
+
+from junshan_kit import DataSets, DataProcessor
+
+
+
+
+
+
+
+
+
+
+
+def Adult_Income_Prediction(Paras):
+
+    df = DataSets.adult_income_prediction() 
+    transform = {
+        "train_size": 0.7,
+        "normalization": True
+    }
+    label_col='income'
+
+    train_dataset, test_dataset, transform = DataProcessor.Pandas_TO_Torch(df, label_col).to_torch(transform, Paras)
+
+    return train_dataset, test_dataset, transform
+
+
+def Credit_Card_Fraud_Detection(Paras):
+    df = DataSets.credit_card_fraud_detection()
+    transform = {
+        "train_size": 0.7,
+        "normalization": True
+    }
+    label_col='Class'
+
+    train_dataset, test_dataset, transform = DataProcessor.Pandas_TO_Torch(df, label_col).to_torch(transform, Paras)
+
+    return train_dataset, test_dataset, transform
+
+
+
+def MNIST(Paras, model_name):
+    """
+    Load the MNIST dataset and return both the training and test sets,
+    along with the transformation applied (ToTensor).
+    """
+    transform = torchvision.transforms.ToTensor()
+
+    train_dataset = torchvision.datasets.MNIST(
+        root='./exp_data/MNIST',
+        train=True,
+        download=True,
+        transform=transform
+    )
+
+    test_dataset = torchvision.datasets.MNIST(
+        root='./exp_data/MNIST',
+        train=False,
+        download=True,
+        transform=transform
+    )
+
+    if Paras["model_type"][model_name] == "binary":
+
+        train_mask = (train_dataset.targets == 0) | (train_dataset.targets == 1)
+        test_mask = (test_dataset.targets == 0) | (test_dataset.targets == 1)
+
+        train_indices = torch.nonzero(train_mask, as_tuple=True)[0]
+        test_indices = torch.nonzero(test_mask, as_tuple=True)[0]
+
+        train_dataset = torch.utils.data.Subset(train_dataset, train_indices.tolist())
+        test_dataset = torch.utils.data.Subset(test_dataset, test_indices.tolist())
+
+    return train_dataset, test_dataset, transform
+
+
+def CIFAR100(Paras, model_name):
+    """
+    Load the CIFAR-100 dataset with standard normalization and return both
+    the training and test sets, along with the transformation applied.
+    """
+    transform = transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize(mean=[0.5071, 0.4867, 0.4408],   
+                            std=[0.2675, 0.2565, 0.2761])     
+    ])
+
+    train_dataset = torchvision.datasets.CIFAR100(
+        root='./exp_data/CIFAR100',
+        train=True,
+        download=True,
+        transform=transform
+    )
+
+    test_dataset = torchvision.datasets.CIFAR100(
+        root='./exp_data/CIFAR100',
+        train=False,
+        download=True,
+        transform=transform
+    )
+    if Paras["model_type"][model_name] == "binary":
+        train_mask = (torch.tensor(train_dataset.targets) == 0) | (torch.tensor(train_dataset.targets) == 1)
+        test_mask = (torch.tensor(test_dataset.targets) == 0) | (torch.tensor(test_dataset.targets) == 1)
+
+        train_indices = torch.nonzero(train_mask, as_tuple=True)[0]
+        test_indices = torch.nonzero(test_mask, as_tuple=True)[0]
+
+        train_dataset = torch.utils.data.Subset(train_dataset, train_indices.tolist())
+        test_dataset = torch.utils.data.Subset(test_dataset, test_indices.tolist())
+
+    return train_dataset, test_dataset, transform

junshan_kit/DataProcessor.py

@@ -1,148 +1,164 @@
 """
 ----------------------------------------------------------------------
->>> Author       : Junshan Yin  
+>>> Author       : Junshan Yin
 >>> Last Updated : 2025-10-12
 ----------------------------------------------------------------------
 """
 
 import pandas as pd
-import os
+import torch
 from sklearn.preprocessing import StandardScaler
-import junshan_kit.datahub
-import zipfile
 
-class CSVToPandasMeta:
+
+class CSV_TO_Pandas:
     def __init__(self):
-        self.data_downloader = junshan_kit.datahub.kaggle_data()
-        
+        pass
+    
+    def _trans_time_fea(self, df, time_info: dict):
+        """
+        Transform and extract time-based features from a specified datetime column.
+
+        This function converts a given column to pandas datetime format and
+        extracts different time-related features based on the specified mode.
+        It supports two extraction modes:
+        - type = 0: Extracts basic components (year, month, day, hour)
+        - type = 1: Extracts hour, day of week, and weekend indicator
+
+        Parameters
+        ----------
+        df : pandas.DataFrame
+            Input DataFrame containing the datetime column.
+        time_info: 
+            - time_col_name : str
+                Name of the column containing time or datetime values.
+            - trans_type : int, optional, default=1
+                - 0 : Extract ['year', 'month', 'day', 'hour']
+                - 1 : Extract ['hour', 'dayofweek', 'is_weekend']
+
+        Returns
+        -------
+        pandas.DataFrame
+            The DataFrame with newly added time-based feature columns.
+
+        Notes
+        -----
+        - Rows that cannot be parsed as valid datetime will be dropped automatically.
+        - 'dayofweek' ranges from 0 (Monday) to 6 (Sunday).
+        - 'is_weekend' equals 1 if the day is Saturday or Sunday, otherwise 0.
+
+        Examples
+        --------
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({
+        ...     'timestamp': ['2023-08-01 12:30:00', '2023-08-05 08:15:00', 'invalid_time']
+        ... })
+        >>> df = handler._trans_time_fea(data, {"time_col_name": "timestamp", "trans_type": 1})
+        >>> print(df)
+                    timestamp  hour  dayofweek  is_weekend
+        0 2023-08-01 12:30:00    12          1           0
+        1 2023-08-05 08:15:00     8          5           1
+        """
+    
+        time_col_name, trans_type = time_info['time_col_name'], time_info['trans_type']
 
-    def read_csv(self, data_name):
-        self.csv_path = f'exp_data/{data_name}/{data_name}.csv'
-        if not os.path.exists(self.csv_path):
-            self.data_downloader.download_data(f'{data_name}', f'exp_data/{data_name}')  
+        df[time_col_name] = pd.to_datetime(df[time_col_name], errors="coerce")
 
-    # ----------------- ccfd_kaggle ----------------------------------
-    def ccfd_kaggle(self, data_name = 'ccfd-kaggle', show_info = True):
-        # download data if not exist
-        self.read_csv(data_name)
-        
-        df = pd.read_csv(self.csv_path)
-        m_before, n_before = df.shape
-        df = df.dropna(axis=0, how='any')
-        m_after, n_after = df.shape
-        df['Class'] = df['Class'].replace(0, -1)
-
-        if show_info:
-            pos_count = (df['Class'] == 1).sum()
-            neg_count = (df['Class'] == -1).sum()
-            
-            print('\n' + '='*60)
-            print(f"{'CCFD-Kaggle Dataset Info':^60}")
-            print('='*60)
-            print(f"{'Original size:':<25} {m_before} rows x {n_before} cols")
-            print(f"{'Size after dropping NaNs:':<25} {m_after} rows x {n_after} cols")
-            print(f"{'Positive samples (+1):':<25} {pos_count}")
-            print(f"{'Negative samples (-1):':<25} {neg_count}")
-            print(f"{'Export size:':<25} {m_after} rows x {n_after} cols")
-            print('-'*60)
-            print(f"More details: https://www.jianguoyun.com/p/Dd1clVgQ4ZThCxiwzZQGIAA")
-            print('='*60 + '\n')
+        # Drop rows where the datetime conversion failed, and make an explicit copy
+        df = df.dropna(subset=[time_col_name]).copy()
 
-        return df
-    
-    # ------------------------ 
-    def ghpdd_kaggle(self, data_name='ghpdd-kaggle', show_info=True):
-        # download data if not exist
-        self.read_csv(data_name)
-
-        # read csv
-        df = pd.read_csv(self.csv_path)
-        m_before, n_before = df.shape
-
-        # drop NaNs
-        df = df.dropna(axis=0, how='any')
-        m_after, n_after = df.shape
-
-        # drop unique identifier
-        if 'property_id' in df.columns:
-            df.drop(columns=['property_id'], inplace=True)
-
-        # Replace label 0 with -1
-        df['decision'] = df['decision'].replace(0, -1)
-
-        # Identify categorical and numerical columns
-        cat_cols = ['country', 'city', 'property_type', 'furnishing_status']
-        num_cols = [col for col in df.columns if col not in cat_cols + ['decision']]
-
-        # One-Hot encode categorical columns
-        df = pd.get_dummies(df, columns=cat_cols)
-
-        # Convert boolean columns to int
-        bool_cols = df.select_dtypes(include='bool').columns
-        for col in bool_cols:
-            df[col] = df[col].astype(int)
-
-        # Standardize numerical columns
-        scaler = StandardScaler()
-        df[num_cols] = scaler.fit_transform(df[num_cols])
-
-        # The size after export
-        m_export, n_export = df.shape
-
-        if show_info:
-            pos_count = (df['decision'] == 1).sum()
-            neg_count = (df['decision'] == -1).sum()
-            
-            print('\n' + '='*70)
-            print(f"{'GHPDD-Kaggle Dataset Info':^70}")
-            print('='*70)
-            print(f"{'Original size:':<35} {m_before} rows x {n_before} cols")
-            print(f"{'Size after dropping NaNs:':<35} {m_after} rows x {n_after} cols")
-            print(f"{'Positive samples (+1):':<35} {pos_count}")
-            print(f"{'Negative samples (-1):':<35} {neg_count}")
-            print(f"{'Export size (after encoding & scaling):':<35} {m_export} rows x {n_export} cols")
-            print('-'*70)
-            print(f"{'Note: categorical columns one-hot encoded, numerical standardized.'}")
-            print(f"More details: https://www.jianguoyun.com/p/DU6Lr9oQqdHDDRj5sI0GIAA")
-            print('='*70 + '\n')
+        if trans_type == 0:
+            df.loc[:, "year"] = df[time_col_name].dt.year
+            df.loc[:, "month"] = df[time_col_name].dt.month
+            df.loc[:, "day"] = df[time_col_name].dt.day
+            df.loc[:, "hour"] = df[time_col_name].dt.hour
 
-        return df
+            user_text_fea = ['year','month','day', 'hour']
+            df = pd.get_dummies(df, columns=user_text_fea, dtype=int)
 
+        elif trans_type == 1:
+            df.loc[:, "hour"] = df[time_col_name].dt.hour
+            df.loc[:, "dayofweek"] = df[time_col_name].dt.dayofweek
+            df.loc[:, "is_weekend"] = df["dayofweek"].isin([5, 6]).astype(int)
 
+            user_text_fea = ['hour','dayofweek','is_weekend']
+            df = pd.get_dummies(df, columns=user_text_fea, dtype=int)
 
-class CSV_TO_Pandas:
-    def __init__(self):
-        pass
+        elif trans_type == 2:
+            df.loc[:, "year"] = df[time_col_name].dt.year
+            df.loc[:, "month"] = df[time_col_name].dt.month
+            df.loc[:, "day"] = df[time_col_name].dt.day
 
-    def unzip_file(self, zip_path: str, unzip_folder: str):
-        """
-        Args:
-            zip_path (str): Path to the ZIP file to extract.
-            dest_folder (str, optional): Folder to extract files into. 
-                If None, the function will create a folder with the same 
-                name as the ZIP file (without extension).
-
-        Examples:
-            >>> zip_path = "./downloads/data.zip"
-            >>> unzip_folder = "./exp_data/data"
-            >>> unzip_file(zip_path, unzip_folder)
-        """
 
-        if unzip_folder is None:
-            unzip_folder = os.path.splitext(os.path.basename(zip_path))[0]
+            user_text_fea = ['year','month','day']
+            df = pd.get_dummies(df, columns=user_text_fea, dtype=int)
+        else:
+            print("error!")
 
-        os.makedirs(unzip_folder, exist_ok=True)
+        df = df.drop(columns=[time_col_name])
 
-        with zipfile.ZipFile(zip_path, 'r') as zip_ref:
-            zip_ref.extractall(unzip_folder)
+        return df
 
-        print(f"✅ Extracted '{zip_path}' to '{os.path.abspath(unzip_folder)}'")
-    
-    # -----------------------------------------------------
+    def preprocess_dataset(
+        self,
+        df,
+        drop_cols: list,
+        label_col: str,
+        label_map: dict,
+        title_name: str,
+        user_one_hot_cols=[],
+        print_info=False,
+        time_info: dict | None = None,
+        missing_strategy = 'drop',  # [drop, mode]
+    ):
+        """
+        Preprocess a CSV dataset by performing data cleaning, label mapping, and feature encoding.
+
+        This function loads a dataset from a CSV file, removes specified non-feature columns,
+        drops rows with missing values, maps the target label to numerical values, and
+        one-hot encodes categorical features. Optionally, it can print dataset statistics
+        before and after preprocessing.
 
-    def clean_data(self, csv_path, drop_cols: list, label_col: str, label_map: dict,  print_info = False):
+        Args:
+            csv_path (str):
+                Path to the input CSV dataset.
+            drop_cols (list):
+                List of column names to drop from the dataset.
+            label_col (str):
+                Name of the target label column.
+            label_map (dict):
+                Mapping dictionary for label conversion (e.g., {"yes": 1, "no": -1}).
+            print_info (bool, optional):
+                Whether to print preprocessing information and dataset statistics.
+                Defaults to False.
+            title_name (str):
+                Title used for the summary table or report that documents
+                the preprocessing steps and dataset statistics.
+
+        Returns:
+            pandas.DataFrame:
+                The cleaned and preprocessed dataset ready for model input.
+
+        Steps:
+            1. Load the dataset from CSV.
+            2. Drop non-informative or irrelevant columns.
+            3. Remove rows with missing values.
+            4. Map label column to numerical values according to `label_map`.
+            5. One-hot encode categorical (non-label) text features.
+            6. Optionally print dataset information and summary statistics.
+
+        Example:
+            >>> label_map = {"positive": 1, "negative": -1}
+            >>> df = data_handler.preprocess_dataset(
+            ...     csv_path="data/raw.csv",
+            ...     drop_cols=["id", "timestamp"],
+            ...     label_col="sentiment",
+            ...     label_map=label_map,
+            ...     print_info=True
+            ... )
+        """
         # Step 0: Load the dataset
-        df = pd.read_csv(csv_path)
+        # df = pd.read_csv(csv_path)
+        columns = df.columns
 
         # Save original size
         m_original, n_original = df.shape
@@ -151,17 +167,34 @@ class CSV_TO_Pandas:
         df = df.drop(columns=drop_cols)
 
         # Step 2: Remove rows with missing values
-        df = df.dropna(axis=0, how='any')
+        if missing_strategy == 'drop':
+            df = df.dropna(axis=0, how="any")
+
+        elif missing_strategy == 'mode':
+            for col in df.columns:
+                if df[col].notna().any():  
+                    mode_val = df[col].mode()[0]
+                    df[col] = df[col].fillna(mode_val)
+
         m_encoded, n_encoded = df.shape
-        
-        # Step 3: Map target label to -1 and +1
+
+        if time_info is not None:
+            df = self._trans_time_fea(df, time_info)
+
+        # Step 3: Map target label (to -1 and +1)
         df[label_col] = df[label_col].map(label_map)
 
         # Step 4: Encode categorical features (exclude label column)
-        text_feature_cols = df.select_dtypes(include=['object', 'string', 'category']).columns
-        text_feature_cols = [col for col in text_feature_cols if col != label_col]  # ✅ exclude label
-
-        df = pd.get_dummies(df, columns=text_feature_cols, dtype=int)
+        text_feature_cols = df.select_dtypes(
+            include=["object", "string", "category"]
+        ).columns
+        text_feature_cols = [
+            col for col in text_feature_cols if col != label_col
+        ]  # ✅ exclude label
+
+        df = pd.get_dummies(
+            df, columns=text_feature_cols + user_one_hot_cols, dtype=int
+        )
         m_cleaned, n_cleaned = df.shape
 
         # print info
@@ -170,23 +203,132 @@ class CSV_TO_Pandas:
             neg_count = (df[label_col] == -1).sum()
 
             # Step 6: Print dataset information
-            print('\n' + '='*80)
-            print(f"{'Dataset Info':^70}")
-            print('='*80)
+            print("\n" + "=" * 80)
+            print(f"{f'{title_name} - Summary':^70}")
+            print("=" * 80)
             print(f"{'Original size:':<40} {m_original} rows x {n_original} cols")
-            print(f"{'Size after dropping NaN & non-feature cols:':<40} {m_cleaned} rows x {n_cleaned} cols")
+            print(
+                f"{'Dropped non-feature columns:':<40} {', '.join(drop_cols) if drop_cols else 'None'}"
+            )
+            print(f"{'missing_strategy:':<40} {missing_strategy}")
+            print(
+                f"{'Dropping NaN & non-feature cols:':<40} {m_encoded} rows x {n_encoded} cols"
+            )
             print(f"{'Positive samples (+1):':<40} {pos_count}")
             print(f"{'Negative samples (-1):':<40} {neg_count}")
-            print(f"{'Size after one-hot encoding:':<40} {m_encoded} rows x {n_encoded} cols")
-            print('-'*80)
-            print(f"Note:")
+            print(
+                f"{'Size after one-hot encoding:':<40} {m_cleaned} rows x {n_cleaned} cols"
+            )
+            print("-" * 80)
+            print(f"{'More details about preprocessing':^70}")
+            print("-" * 80)
             print(f"{'Label column:':<40} {label_col}")
-            print(f"{'Dropped non-feature columns:':<40} {', '.join(drop_cols) if drop_cols else 'None'}")
-            print(f"{'text fetaure columns:':<40} {', '.join(list(text_feature_cols)) if list(text_feature_cols) else 'None'}")
-            print('='*80 + '\n')
+            print(f"{'label_map:':<40} {label_map}")
+            print(f"{'time column:':<40} {time_info}")
+            if time_info is not None:
+                if time_info["trans_type"] == 0:
+                    print("- 0 : Extract ['year', 'month', 'day', 'hour']")
+                elif time_info["trans_type"] == 1:
+                    print("- 1 : Extract ['hour', 'dayofweek', 'is_weekend']")
+                elif time_info["trans_type"] == 2:
+                    print("- 2 : Extract ['year', 'month', 'day']")
+                else:
+                    assert False
+            print(
+                f"{'text fetaure columns:':<40} {', '.join(list(text_feature_cols)) if list(text_feature_cols) else 'None'}"
+            )
+            # print("-" * 80)
+            # print("all columns:")
+            # print(list(columns))
+            # print("=" * 80 + "\n")
 
         return df
+    
+
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler
+from torch.utils.data import Dataset
+
+class Pandas_TO_Torch(Dataset):
+
+    def __init__(self, df: pd.DataFrame, 
+                 label_col: str, 
+                ):
+        self.df = df
+        self.label_col = label_col
+
+        # Identify feature columns automatically (all except the label)
+        self.label_col = label_col
+        self.feature_cols = [col for col in self.df.columns if col != label_col]
 
+        # Extract features and labels
+        self.features = self.df[self.feature_cols].values.astype("float32")
+        self.labels = self.df[self.label_col].values.astype("int64")
+
+
+    def __len__(self):
+        """Return the total number of samples."""
+        return len(self.features)
+
+    def __getitem__(self, idx):
+        x = torch.tensor(self.features[idx], dtype=torch.float32)
+        y = torch.tensor(self.labels[idx], dtype=torch.long)
         
+        return x, y
+
+    def __repr__(self):
+        info = (
+            f"Dataset CustomNumericDataset\n"
+            f"    Number of datapoints: {len(self)}\n"
+            f"    Features: {self.features.shape[1]}\n"
+        )
+        return info
+    
+    def to_torch(self, transform, Paras):
+        fea_cols = [col for col in self.df.columns if col != self.label_col]
 
+        if transform["normalization"]:
+            scaler = StandardScaler()
+            self.df[fea_cols] = scaler.fit_transform(self.df[fea_cols])
         
+        # Train/test split
+        
+        train_df, test_df = train_test_split(self.df, train_size=transform["train_size"], random_state=Paras["seed"], stratify=self.df[self.label_col])
+        
+        # Create datasets
+        train_dataset = Pandas_TO_Torch(train_df, self.label_col)
+        test_dataset  = Pandas_TO_Torch(test_df, self.label_col)
+
+        return train_dataset, test_dataset, transform
+
+
+class TXT_TO_Numpy:
+    def __init__(self):
+        pass
+
+
+class bz2_To_Numpy:
+    def __init__(self):
+        pass
+
+
+
+
+
+
+    
+
+class StepByStep:
+    def __init__(self):
+        pass
+
+    def print_text_fea(self, df, text_feature_cols):
+        for col in text_feature_cols:
+            print(f"\n{'-'*80}")
+            print(f'Feature: "{col}"')
+            print(f"{'-'*80}")
+            print(
+                f"Unique values ({len(df[col].unique())}): {df[col].unique().tolist()}"
+            )
+
+