junshan-kit 2.2.8__py2.py3-none-any.whl → 2.7.3__py2.py3-none-any.whl

junshan_kit/BenchmarkFunctions.py

@@ -0,0 +1,7 @@
+
+
+def rosenbrock(x, a=1.0, b=100.0):
+    # Optimal value: (a, a^2)
+    return (a - x[0])**2 + b * (x[1] - x[0]**2)**2
+
+

junshan_kit/Check_Info.py

@@ -0,0 +1,44 @@
+"""
+----------------------------------------------------------------------
+>>> Author       : Junshan Yin  
+>>> Last Updated : 2025-11-22
+----------------------------------------------------------------------
+"""
+
+from junshan_kit import ModelsHub
+
+def check_args(self, args, parser, allowed_models, allowed_optimizers, allowed_datasets):
+    # Parse and validate each train_group
+    for cfg in args.train:
+        try:
+            model, dataset, optimizer = cfg.split("-")
+
+            if model not in allowed_models:
+                parser.error(f"Invalid model '{model}'. Choose from {allowed_models}")
+            if optimizer not in allowed_optimizers:
+                parser.error(f"Invalid optimizer '{optimizer}'. Choose from {allowed_optimizers}")
+            if dataset not in allowed_datasets:
+                parser.error(f"Invalid dataset '{dataset}'. Choose from {allowed_datasets}")
+
+        except ValueError:
+            parser.error(f"Invalid format '{cfg}'. Use model-dataset-optimizer")
+
+    for cfg in args.train:
+        model_name, dataset_name, optimizer_name = cfg.split("-")
+        try:
+            f = getattr(ModelsHub, f"Build_{args.model_name_mapping[model_name]}_{args.data_name_mapping[dataset_name]}")
+
+        except:
+            print(getattr(ModelsHub, f"Build_{args.model_name_mapping[model_name]}_{args.data_name_mapping[dataset_name]}"))
+            assert False
+
+def check_subset_info(self, args, parser):
+        total = sum(args.subset)
+        if args.subset[0]>1:
+            # CHECK
+            for i in args.subset:
+                if i < 1:
+                    parser.error(f"Invalid --subset {args.subset}: The number of subdata must > 1")    
+        else:
+            if abs(total - 1.0) != 0.0:  
+                parser.error(f"Invalid --subset {args.subset}: the values must sum to 1.0 (current sum = {total:.6f}))")

junshan_kit/DataHub.py

@@ -0,0 +1,214 @@
+"""
+----------------------------------------------------------------------
+>>> Author       : Junshan Yin  
+>>> Last Updated : 2025-10-28
+----------------------------------------------------------------------
+"""
+
+import torchvision, torch
+import torchvision.transforms as transforms
+import pandas as pd
+from torch.utils.data import random_split, Subset
+
+from junshan_kit import DataSets, DataProcessor, ParametersHub
+
+def Adult_Income_Prediction(Paras):
+
+    df = DataSets.adult_income_prediction() 
+    transform = {
+        "train_size": 0.7,
+        "normalization": True
+    }
+    label_col='income'
+
+    train_dataset, test_dataset, transform = DataProcessor.Pandas_TO_Torch(df, label_col).to_torch(transform, Paras)
+
+    return train_dataset, test_dataset, transform
+
+
+def Credit_Card_Fraud_Detection(Paras):
+    df = DataSets.credit_card_fraud_detection()
+    transform = {
+        "train_size": 0.7,
+        "normalization": True
+    }
+    label_col='Class'
+
+    train_dataset, test_dataset, transform = DataProcessor.Pandas_TO_Torch(df, label_col).to_torch(transform, Paras)
+
+    return train_dataset, test_dataset, transform
+
+def Diabetes_Health_Indicators(Paras):
+    df = DataSets.diabetes_health_indicators()
+    transform = {
+        "train_size": 0.7,
+        "normalization": True
+    }
+    label_col='diagnosed_diabetes'
+
+    train_dataset, test_dataset, transform = DataProcessor.Pandas_TO_Torch(df, label_col).to_torch(transform, Paras)
+
+    return train_dataset, test_dataset, transform
+
+def Electric_Vehicle_Population(Paras): 
+    df = DataSets.electric_vehicle_population()
+    transform = {
+        "train_size": 0.7,
+        "normalization": True
+    }
+    label_col='Electric Vehicle Type'
+    train_dataset, test_dataset, transform = DataProcessor.Pandas_TO_Torch(df, label_col).to_torch(transform, Paras)
+
+    return train_dataset, test_dataset, transform
+
+def Global_House_Purchase(Paras): 
+    df = DataSets.global_house_purchase()
+    transform = {
+        "train_size": 0.7,
+        "normalization": True
+    }
+    label_col='decision'
+    train_dataset, test_dataset, transform = DataProcessor.Pandas_TO_Torch(df, label_col).to_torch(transform, Paras)
+
+    return train_dataset, test_dataset, transform
+
+def Health_Lifestyle(Paras): 
+    df = DataSets.health_lifestyle()
+    transform = {
+        "train_size": 0.7,
+        "normalization": True
+    }
+    label_col='disease_risk'
+    train_dataset, test_dataset, transform = DataProcessor.Pandas_TO_Torch(df, label_col).to_torch(transform, Paras)
+
+    return train_dataset, test_dataset, transform
+
+def Homesite_Quote_Conversion(Paras): 
+    df = DataSets.Homesite_Quote_Conversion()
+    transform = {
+        "train_size": 0.7,
+        "normalization": True
+    }
+    label_col='QuoteConversion_Flag'
+    train_dataset, test_dataset, transform = DataProcessor.Pandas_TO_Torch(df, label_col).to_torch(transform, Paras)
+
+    return train_dataset, test_dataset, transform
+
+def TN_Weather_2020_2025(Paras): 
+    df = DataSets.TamilNadu_weather_2020_2025()
+    transform = {
+        "train_size": 0.7,
+        "normalization": True
+    }
+    label_col='rain_tomorrow'
+    train_dataset, test_dataset, transform = DataProcessor.Pandas_TO_Torch(df, label_col).to_torch(transform, Paras)
+
+    return train_dataset, test_dataset, transform
+
+
+
+def MNIST(Paras, model_name):
+    """
+    Load the MNIST dataset and return both the training and test sets,
+    along with the transformation applied (ToTensor).
+    """
+    transform = torchvision.transforms.ToTensor()
+
+    train_dataset = torchvision.datasets.MNIST(
+        root='./exp_data/MNIST',
+        train=True,
+        download=True,
+        transform=transform
+    )
+
+    test_dataset = torchvision.datasets.MNIST(
+        root='./exp_data/MNIST',
+        train=False,
+        download=True,
+        transform=transform
+    )
+
+    if Paras["model_type"][model_name] == "binary":
+
+        train_mask = (train_dataset.targets == 0) | (train_dataset.targets == 1)
+        test_mask = (test_dataset.targets == 0) | (test_dataset.targets == 1)
+
+        train_indices = torch.nonzero(train_mask, as_tuple=True)[0]
+        test_indices = torch.nonzero(test_mask, as_tuple=True)[0]
+
+        train_dataset = torch.utils.data.Subset(train_dataset, train_indices.tolist())
+        test_dataset = torch.utils.data.Subset(test_dataset, test_indices.tolist())
+
+    return train_dataset, test_dataset, transform
+
+
+def CIFAR100(Paras, model_name):
+    """
+    Load the CIFAR-100 dataset with standard normalization and return both
+    the training and test sets, along with the transformation applied.
+    """
+    transform = transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize(mean=[0.5071, 0.4867, 0.4408],   
+                            std=[0.2675, 0.2565, 0.2761])     
+    ])
+
+    train_dataset = torchvision.datasets.CIFAR100(
+        root='./exp_data/CIFAR100',
+        train=True,
+        download=True,
+        transform=transform
+    )
+
+    test_dataset = torchvision.datasets.CIFAR100(
+        root='./exp_data/CIFAR100',
+        train=False,
+        download=True,
+        transform=transform
+    )
+
+    if Paras["model_type"][model_name] == "binary":
+        train_mask = (torch.tensor(train_dataset.targets) == 0) | (torch.tensor(train_dataset.targets) == 1)
+        test_mask = (torch.tensor(test_dataset.targets) == 0) | (torch.tensor(test_dataset.targets) == 1)
+
+        train_indices = torch.nonzero(train_mask, as_tuple=True)[0]
+        test_indices = torch.nonzero(test_mask, as_tuple=True)[0]
+
+        train_dataset = torch.utils.data.Subset(train_dataset, train_indices.tolist())
+        test_dataset = torch.utils.data.Subset(test_dataset, test_indices.tolist())
+
+    return train_dataset, test_dataset, transform
+
+
+def Caltech101_Resize_32(Paras, train_ratio=0.7, split=True):
+
+    transform = transforms.Compose([
+        # transforms.Lambda(convert_to_rgb),  
+        transforms.Grayscale(num_output_channels=3),
+        transforms.Resize((32, 32)),
+        transforms.ToTensor(),
+        transforms.Normalize(mean=[0.485, 0.456, 0.406],  
+                            std=[0.229, 0.224, 0.225])
+    ])
+    
+    full_dataset = torchvision.datasets.Caltech101(
+        root='./exp_data/Caltech101',
+        download=True,
+        transform=transform
+    )
+
+    if split:
+        total_size = len(full_dataset)
+        train_size = int(train_ratio * total_size)
+        test_size = total_size - train_size
+
+        train_dataset, test_dataset = random_split(full_dataset, [train_size, test_size])
+
+    else:
+        train_dataset = full_dataset
+        # Empty test dataset, keep the structure consistent
+        test_dataset = Subset(full_dataset, [])  
+
+    return train_dataset, test_dataset, transform
+
+# <caltech101_Resize_32>

junshan_kit/DataProcessor.py

@@ -6,19 +6,112 @@
 """
 
 import pandas as pd
+import numpy as np
+import torch, bz2
+from torch.utils.data import random_split, Subset
+from sklearn.datasets import load_svmlight_file
+from sklearn.preprocessing import StandardScaler
 
 
 class CSV_TO_Pandas:
     def __init__(self):
         pass
+    
+    def _trans_time_fea(self, df, time_info: dict):
+        """
+        Transform and extract time-based features from a specified datetime column.
+
+        This function converts a given column to pandas datetime format and
+        extracts different time-related features based on the specified mode.
+        It supports two extraction modes:
+        - type = 0: Extracts basic components (year, month, day, hour)
+        - type = 1: Extracts hour, day of week, and weekend indicator
+
+        Parameters
+        ----------
+        df : pandas.DataFrame
+            Input DataFrame containing the datetime column.
+        time_info: 
+            - time_col_name : str
+                Name of the column containing time or datetime values.
+            - trans_type : int, optional, default=1
+                - 0 : Extract ['year', 'month', 'day', 'hour']
+                - 1 : Extract ['hour', 'dayofweek', 'is_weekend']
+
+        Returns
+        -------
+        pandas.DataFrame
+            The DataFrame with newly added time-based feature columns.
+
+        Notes
+        -----
+        - Rows that cannot be parsed as valid datetime will be dropped automatically.
+        - 'dayofweek' ranges from 0 (Monday) to 6 (Sunday).
+        - 'is_weekend' equals 1 if the day is Saturday or Sunday, otherwise 0.
+
+        Examples
+        --------
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({
+        ...     'timestamp': ['2023-08-01 12:30:00', '2023-08-05 08:15:00', 'invalid_time']
+        ... })
+        >>> df = handler._trans_time_fea(data, {"time_col_name": "timestamp", "trans_type": 1})
+        >>> print(df)
+                    timestamp  hour  dayofweek  is_weekend
+        0 2023-08-01 12:30:00    12          1           0
+        1 2023-08-05 08:15:00     8          5           1
+        """
+    
+        time_col_name, trans_type = time_info['time_col_name'], time_info['trans_type']
+
+        df[time_col_name] = pd.to_datetime(df[time_col_name], errors="coerce")
+
+        # Drop rows where the datetime conversion failed, and make an explicit copy
+        df = df.dropna(subset=[time_col_name]).copy()
+
+        if trans_type == 0:
+            df.loc[:, "year"] = df[time_col_name].dt.year
+            df.loc[:, "month"] = df[time_col_name].dt.month
+            df.loc[:, "day"] = df[time_col_name].dt.day
+            df.loc[:, "hour"] = df[time_col_name].dt.hour
+
+            user_text_fea = ['year','month','day', 'hour']
+            df = pd.get_dummies(df, columns=user_text_fea, dtype=int)
+
+        elif trans_type == 1:
+            df.loc[:, "hour"] = df[time_col_name].dt.hour
+            df.loc[:, "dayofweek"] = df[time_col_name].dt.dayofweek
+            df.loc[:, "is_weekend"] = df["dayofweek"].isin([5, 6]).astype(int)
+
+            user_text_fea = ['hour','dayofweek','is_weekend']
+            df = pd.get_dummies(df, columns=user_text_fea, dtype=int)
+
+        elif trans_type == 2:
+            df.loc[:, "year"] = df[time_col_name].dt.year
+            df.loc[:, "month"] = df[time_col_name].dt.month
+            df.loc[:, "day"] = df[time_col_name].dt.day
+
+
+            user_text_fea = ['year','month','day']
+            df = pd.get_dummies(df, columns=user_text_fea, dtype=int)
+        else:
+            print("error!")
+
+        df = df.drop(columns=[time_col_name])
+
+        return df
 
     def preprocess_dataset(
         self,
-        csv_path,
+        df,
         drop_cols: list,
         label_col: str,
         label_map: dict,
+        title_name: str,
+        user_one_hot_cols=[],
         print_info=False,
+        time_info: dict | None = None,
+        missing_strategy = 'drop',  # [drop, mode]
     ):
         """
         Preprocess a CSV dataset by performing data cleaning, label mapping, and feature encoding.
@@ -40,6 +133,9 @@ class CSV_TO_Pandas:
             print_info (bool, optional):
                 Whether to print preprocessing information and dataset statistics.
                 Defaults to False.
+            title_name (str):
+                Title used for the summary table or report that documents
+                the preprocessing steps and dataset statistics.
 
         Returns:
             pandas.DataFrame:
@@ -64,7 +160,8 @@ class CSV_TO_Pandas:
             ... )
         """
         # Step 0: Load the dataset
-        df = pd.read_csv(csv_path)
+        # df = pd.read_csv(csv_path)
+        columns = df.columns
 
         # Save original size
         m_original, n_original = df.shape
@@ -73,10 +170,21 @@ class CSV_TO_Pandas:
         df = df.drop(columns=drop_cols)
 
         # Step 2: Remove rows with missing values
-        df = df.dropna(axis=0, how="any")
+        if missing_strategy == 'drop':
+            df = df.dropna(axis=0, how="any")
+
+        elif missing_strategy == 'mode':
+            for col in df.columns:
+                if df[col].notna().any():  
+                    mode_val = df[col].mode()[0]
+                    df[col] = df[col].fillna(mode_val)
+
         m_encoded, n_encoded = df.shape
 
-        # Step 3: Map target label (to -1 and +1)
+        if time_info is not None:
+            df = self._trans_time_fea(df, time_info)
+
+        # Step 3: Map target label (to 0 and +1)
         df[label_col] = df[label_col].map(label_map)
 
         # Step 4: Encode categorical features (exclude label column)
@@ -87,36 +195,218 @@ class CSV_TO_Pandas:
             col for col in text_feature_cols if col != label_col
         ]  # ✅ exclude label
 
-        df = pd.get_dummies(df, columns=text_feature_cols, dtype=int)
+        df = pd.get_dummies(
+            df, columns=text_feature_cols + user_one_hot_cols, dtype=int
+        )
         m_cleaned, n_cleaned = df.shape
 
         # print info
         if print_info:
             pos_count = (df[label_col] == 1).sum()
-            neg_count = (df[label_col] == -1).sum()
+            neg_count = (df[label_col] == 0).sum()
 
             # Step 6: Print dataset information
             print("\n" + "=" * 80)
-            print(f"{'Dataset Info':^70}")
+            print(f"{f'{title_name} - Summary':^70}")
             print("=" * 80)
             print(f"{'Original size:':<40} {m_original} rows x {n_original} cols")
             print(
-                f"{'Size after dropping NaN & non-feature cols:':<40} {m_cleaned} rows x {n_cleaned} cols"
+                f"{'Dropped non-feature columns:':<40} {', '.join(drop_cols) if drop_cols else 'None'}"
             )
-            print(f"{'Positive samples (+1):':<40} {pos_count}")
-            print(f"{'Negative samples (-1):':<40} {neg_count}")
+            print(f"{'missing_strategy:':<40} {missing_strategy}")
             print(
-                f"{'Size after one-hot encoding:':<40} {m_encoded} rows x {n_encoded} cols"
+                f"{'Dropping NaN & non-feature cols:':<40} {m_encoded} rows x {n_encoded} cols"
             )
-            print("-" * 80)
-            print(f"Note:")
-            print(f"{'Label column:':<40} {label_col}")
+            print(f"{'Positive samples (1):':<40} {pos_count}")
+            print(f"{'Negative samples (0):':<40} {neg_count}")
             print(
-                f"{'Dropped non-feature columns:':<40} {', '.join(drop_cols) if drop_cols else 'None'}"
+                f"{'Size after one-hot encoding:':<40} {m_cleaned} rows x {n_cleaned} cols"
             )
+            print("-" * 80)
+            print(f"{'More details about preprocessing':^70}")
+            print("-" * 80)
+            print(f"{'Label column:':<40} {label_col}")
+            print(f"{'label_map:':<40} {label_map}")
+            print(f"{'time column:':<40} {time_info}")
+            if time_info is not None:
+                if time_info["trans_type"] == 0:
+                    print("- 0 : Extract ['year', 'month', 'day', 'hour']")
+                elif time_info["trans_type"] == 1:
+                    print("- 1 : Extract ['hour', 'dayofweek', 'is_weekend']")
+                elif time_info["trans_type"] == 2:
+                    print("- 2 : Extract ['year', 'month', 'day']")
+                else:
+                    assert False
             print(
                 f"{'text fetaure columns:':<40} {', '.join(list(text_feature_cols)) if list(text_feature_cols) else 'None'}"
             )
-            print("=" * 80 + "\n")
+            # print("-" * 80)
+            # print("all columns:")
+            # print(list(columns))
+            # print("=" * 80 + "\n")
 
         return df
+    
+
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler
+from torch.utils.data import Dataset
+
+class Pandas_TO_Torch(Dataset):
+
+    def __init__(self, df: pd.DataFrame, 
+                label_col: str, 
+                ):
+        self.df = df
+        self.label_col = label_col
+
+        # Identify feature columns automatically (all except the label)
+        self.label_col = label_col
+        self.feature_cols = [col for col in self.df.columns if col != label_col]
+
+        # Extract features and labels
+        self.features = self.df[self.feature_cols].values.astype("float32")
+        self.labels = self.df[self.label_col].values.astype("int64")
+
+
+    def __len__(self):
+        """Return the total number of samples."""
+        return len(self.features)
+
+    def __getitem__(self, idx):
+        x = torch.tensor(self.features[idx], dtype=torch.float32)
+        y = torch.tensor(self.labels[idx], dtype=torch.long)
+        
+        return x, y
+
+    def __repr__(self):
+        info = (
+            f"Dataset CustomNumericDataset\n"
+            f"    Number of datapoints: {len(self)}\n"
+            f"    Features: {self.features.shape[1]}\n"
+        )
+        return info
+    
+    def to_torch(self, transform, Paras):
+        fea_cols = [col for col in self.df.columns if col != self.label_col]
+
+        if transform["normalization"]:
+            scaler = StandardScaler()
+            self.df[fea_cols] = scaler.fit_transform(self.df[fea_cols])
+        
+        # Train/test split
+        
+        train_df, test_df = train_test_split(self.df, train_size=transform["train_size"], random_state=Paras["seed"], stratify=self.df[self.label_col])
+        
+        # Create datasets
+        train_dataset = Pandas_TO_Torch(train_df, self.label_col)
+        test_dataset  = Pandas_TO_Torch(test_df, self.label_col)
+
+        return train_dataset, test_dataset, transform
+
+
+class TXT_TO_Numpy:
+    def __init__(self):
+        pass
+
+
+class bz2_To_Numpy:
+    def __init__(self):
+        pass
+
+
+
+
+
+
+class StepByStep:
+    def __init__(self):
+        pass
+
+    def print_text_fea(self, df, text_feature_cols):
+        for col in text_feature_cols:
+            print(f"\n{'-'*80}")
+            print(f'Feature: "{col}"')
+            print(f"{'-'*80}")
+            print(
+                f"Unique values ({len(df[col].unique())}): {df[col].unique().tolist()}"
+            )
+
+
+class LibSVMDataset_bz2(Dataset):
+    def __init__(self, path, data_name = None, Paras = None):
+        with bz2.open(path, 'rb') as f:
+            X, y = load_svmlight_file(f) # type: ignore
+
+        self.X, self.path = X, path
+        
+        y = np.asanyarray(y)
+
+        if data_name is not None:
+            data_name = data_name.lower()
+
+            # Binary classification, with the label -1/1
+            if data_name in ["rcv1"]:
+                y = (y > 0).astype(int)  # Convert to 0/1
+            
+            # Multi-category, labels usually start with 1
+            elif data_name in [""]:  
+                y = y - 1  # Start with 0
+
+        else:
+            # Default policy: Try to avoid CrossEntropyLoss errors
+            if np.min(y) < 0:  # e.g. [-1, 1]
+                y = (y > 0).astype(int)
+            elif np.min(y) >= 1:
+                y = y - 1
+
+        self.y = y
+
+    def __len__(self):
+        return self.X.shape[0]
+
+    def __getitem__(self, idx):
+        xi = torch.tensor(self.X.getrow(idx).toarray(), dtype=torch.float32).squeeze(0)
+        yi = torch.tensor(self.y[idx], dtype=torch.float32)
+        return xi, yi
+    
+    def __repr__(self):
+        num_samples = len(self.y)
+        num_features = self.X.shape[1]
+        num_classes = len(np.unique(self.y))
+        return (f"LibSVMDataset_bz2(\n"
+                f"  num_samples = {num_samples},\n"
+                f"  num_features = {num_features},\n"
+                f"  num_classes = {num_classes}\n"
+                f"  path = {self.path}\n"
+                f")")
+    
+def get_libsvm_bz2_data(train_path, test_path, data_name, Paras, split = True):
+    
+    transform = "-1 → 0 for binary, y-1 for multi-class"
+    train_data = LibSVMDataset_bz2(train_path)
+
+    if data_name in ["Duke", "Ijcnn"]:
+        test_data = LibSVMDataset_bz2(test_path)
+        split = False
+    else:
+        test_data = Subset(train_data, [])
+        
+    
+    if split:
+        total_size = len(train_data)
+        train_size = int(Paras["train_ratio"] * total_size)
+        test_size = total_size - train_size
+
+        train_dataset, test_dataset = random_split(train_data, [train_size, test_size])
+
+    else:
+        train_dataset = train_data
+        # # Empty test dataset, keep the structure consistent
+        # test_dataset = Subset(train_data, []) 
+        test_dataset = test_data
+
+    # print(test_dataset) 
+    # assert False
+
+    return train_dataset, test_dataset, transform