junshan-kit 2.3.9__py2.py3-none-any.whl → 2.4.1__py2.py3-none-any.whl

junshan_kit/ExperimentHub.py

@@ -0,0 +1,328 @@
+import sys, os, torch, random
+import numpy as np
+import torch.nn as nn
+import torch.utils.data as Data
+from torch.utils.data import Subset, random_split
+from junshan_kit import ComOptimizers, datahub, Models, TrainingParas, SPBM
+
+# -------------------------------------
+def set_seed(seed=42):
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    np.random.seed(seed)
+    random.seed(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
+def device(Paras):
+    device = torch.device(f"{Paras['cuda']}" if torch.cuda.is_available() else "cpu")
+    Paras["device"] = device
+    use_color = sys.stdout.isatty()
+    Paras["use_color"] = use_color
+
+    return Paras
+
+# -------------------------------------
+class Train_Steps:
+    def __init__(self, args) -> None:
+        self.args = args
+
+    def _model_map(self, model_name):
+        model_mapping = self.args.model_mapping
+
+        return model_mapping[model_name]
+    
+    def get_train_group(self):
+        training_group = []
+        for cfg in self.args.train_group:
+            model, dataset, optimizer = cfg.split("-")
+            training_group.append((self._model_map(model), dataset, optimizer))
+
+        return training_group
+    
+    def set_paras(self, results_folder_name, py_name, time_str, OtherParas):
+        Paras = {
+        # Name of the folder where results will be saved.
+        "results_folder_name": results_folder_name,
+        # Whether to draw loss/accuracy figures.
+        "DrawFigs": "ON",
+        # Whether to use log scale when drawing plots.
+        "use_log_scale": "ON",
+        # Print loss every N epochs.
+        "epoch_log_interval": 1,
+        # Timestamp string for result saving.
+        "time_str": time_str,
+        # Random seed
+        "seed": OtherParas['seed'],
+        # Device used for training.
+        "cuda": f"cuda:{self.args.cuda}",
+
+        # batch-size 
+        "batch_size": self.args.bs,
+
+        # epochs
+        "epochs": self.args.e,
+
+        # split_train_data
+        "split_train_data": self.args.s,
+
+        # select_subset
+        "select_subset": self.args.subset,
+
+        # subset_number_dict
+        "subset_number_dict": TrainingParas.subset_number_dict(OtherParas),
+
+        # validation
+        "validation": TrainingParas.validation(),
+
+        # validation_rate
+        "validation_rate": TrainingParas.validation_rate(),
+
+        # model list
+        "model_list" : TrainingParas.model_list(),
+
+        # model_type
+        "model_type": TrainingParas.model_type(),
+
+        # data_list
+        "data_list": TrainingParas.data_list(),
+
+        # optimizer_dict
+        "optimizer_dict": TrainingParas.optimizer_dict(OtherParas)
+        }
+        Paras["py_name"] = py_name
+        
+        return Paras
+    
+    # <Step_3> : Chosen_loss
+    def chosen_loss(self, model_name, Paras):
+        # ---------------------------------------------------
+        # There have an addition parameter
+        if model_name == "LogRegressionBinaryL2":
+            Paras["lambda"] = 1e-3
+        # ---------------------------------------------------
+
+        if model_name in ["LeastSquares"]:
+            loss_fn = nn.MSELoss()
+
+        else:
+            if Paras["model_type"][model_name] == "binary":
+                loss_fn = nn.BCEWithLogitsLoss()
+
+            elif Paras["model_type"][model_name] == "multi":
+                loss_fn = nn.CrossEntropyLoss()
+
+            else:
+                loss_fn = nn.MSELoss()
+                print("\033[91m The loss function is error!\033[0m")
+                assert False
+        Paras["loss_fn"] = loss_fn
+
+        return loss_fn, Paras
+    
+    # <Step_4> : import data --> step.py
+    def load_data(self, model_name, data_name, Paras):
+        # load data
+        train_path = f"./exp_data/{data_name}/training_data"
+        test_path = f"./exp_data/{data_name}/test_data"
+        # Paras["train_ratio"] = 1.0
+        # Paras["select_subset"].setdefault(data_name, False)
+        # Paras["validation"].setdefault(data_name, False)
+
+        if data_name == "MNIST":
+            train_dataset, test_dataset, transform = datahub.MNIST(Paras, model_name)
+
+        elif data_name == "CIFAR100":
+            train_dataset, test_dataset, transform = datahub.CIFAR100(Paras, model_name)
+
+        elif data_name == "CALTECH101_Resize_32":
+            Paras["train_ratio"] = 0.7
+            train_dataset, test_dataset, transform = datahub.caltech101_Resize_32(
+                Paras["seed"], Paras["train_ratio"], split=True
+            )
+
+        elif data_name in ["Vowel", "Letter", "Shuttle", "w8a"]:
+            Paras["train_ratio"] = Paras["split_train_data"][data_name]
+            train_dataset, test_dataset, transform = datahub.get_libsvm_data(
+                train_path + ".txt", test_path + ".txt", data_name
+            )
+
+        elif data_name in ["RCV1", "Duke", "Ijcnn"]:
+            Paras["train_ratio"] = Paras["split_train_data"][data_name]
+            train_dataset, test_dataset, transform = datahub.get_libsvm_bz2_data(
+                train_path + ".bz2", test_path + ".bz2", data_name, Paras
+            )
+
+        else:
+            transform = None
+            print(f"The data_name is error!")
+            assert False
+
+        return train_dataset, test_dataset, transform
+    # <Step_4>
+
+    # <subset> : Step 5.1 -->step.py
+    def set_subset(self, data_name, Paras, train_dataset, test_dataset):
+        if self.args.subset[0]>1:
+            train_num = self.args.subset[0]
+            test_num = self.args.subset[1]
+            train_subset_num = min(train_num, len(train_dataset))
+            test_subset_num = min(test_num, len(test_dataset))
+
+            train_subset_indices = list(range(int(train_subset_num)))
+            train_dataset = Subset(train_dataset, train_subset_indices)
+
+            test_subset_indices = list(range(int(test_subset_num)))
+            test_dataset = Subset(test_dataset, test_subset_indices)
+            
+        else:
+            train_ratios= self.args.subset[0]
+            test_ratios= self.args.subset[1]
+
+            train_subset_indices = list(range(int(train_ratios * len(train_dataset))))
+            train_dataset = Subset(train_dataset, train_subset_indices)
+
+            test_subset_indices = list(range(int(test_ratios * len(test_dataset))))
+            test_dataset = Subset(test_dataset, test_subset_indices)
+
+        return train_dataset, test_dataset
+
+    # <validation> : Step 5.2   --> step.py
+    def set_val_set(self, data_name, train_dataset, Paras):
+        if Paras["validation"][data_name]:
+            size_ = len(train_dataset)
+            val_size = int(size_ * Paras["validation_rate"][data_name])
+            train_size = size_ - val_size
+
+            train_dataset, val_dataset = random_split(
+                train_dataset,
+                [train_size, val_size],
+                generator=torch.Generator().manual_seed(Paras["seed"]),
+            )
+
+        else:
+            val_dataset = Subset(train_dataset, [])
+
+        return train_dataset, val_dataset
+    # <validation>
+
+
+    # <get_dataloader> Step 5.3 -->step.py
+    def get_dataloader(self, data_name, train_dataset, test_dataset, Paras):
+        set_seed(Paras["seed"])
+        g = torch.Generator()
+        g.manual_seed(Paras["seed"])
+
+        # Create training DataLoader
+        
+        train_loader = Data.DataLoader(
+            dataset=train_dataset,
+            shuffle=True,
+            batch_size=self.args.bs,
+            generator=g,
+            num_workers=4,
+        )
+
+        # test loader
+        test_loader = Data.DataLoader(
+            dataset=test_dataset,
+            shuffle=False,
+            batch_size=self.args.bs,
+            generator=g,
+            num_workers=4,
+        )
+    
+        return train_loader, test_loader
+    # <get_dataloader>
+
+    def hyperparas_and_path(
+        self,
+        model_name,
+        data_name,
+        optimizer_name,
+        Paras,
+    ):
+        params_gird = Paras["optimizer_dict"][optimizer_name]["params"]
+        keys, values = list(params_gird.keys()), list(params_gird.values())
+
+        # Set the path for saving results
+        folder_path = f'./{Paras["results_folder_name"]}/seed_{Paras["seed"]}/{model_name}/{data_name}/{optimizer_name}/train_{Paras["train_data_num"]}_test_{Paras["test_data_num"]}/Batch_size_{self.args.bs}/epoch_{self.args.e}/{Paras["time_str"]}'
+        os.makedirs(folder_path, exist_ok=True)
+
+        return keys, values, folder_path
+
+    
+    # <Reloading> Step 7.3 --> step.py
+    def reloading_model_dataloader(self,
+        base_model_fn,
+        initial_state_dict,
+        data_name,
+        train_dataset,
+        test_dataset,
+        Paras,
+    ):
+        set_seed(Paras["seed"])
+        model = base_model_fn()
+        model.load_state_dict(initial_state_dict)
+        model.to(Paras["device"])
+        train_loader, test_loader = self.get_dataloader(
+            data_name, train_dataset, test_dataset, Paras
+        )
+
+        return model, train_loader, test_loader
+    # <Reloading>
+
+    def chosen_optimizer(self, optimizer_name, model, hyperparams, Paras):
+        if optimizer_name == "SGD":
+            optimizer = torch.optim.SGD(model.parameters(), lr=hyperparams["alpha"])
+
+        elif optimizer_name == "ADAM":
+            optimizer = torch.optim.Adam(
+                model.parameters(),
+                lr=hyperparams["alpha"],
+                betas=(hyperparams["beta1"], hyperparams["beta2"]),
+                eps=hyperparams["epsilon"],
+            )
+
+        elif optimizer_name in ["SPBM-TR"]:
+            optimizer = SPBM.TR(model.parameters(), model, hyperparams, Paras)
+
+        elif optimizer_name in ["SPBM-TR-NoneLower"]:
+            optimizer = SPBM.TR_NoneLower(model.parameters(), model, hyperparams, Paras)
+        
+        elif optimizer_name in ["SPBM-TR-NoneSpecial"]:
+            optimizer = SPBM.TR_NoneSpecial(model.parameters(), model, hyperparams, Paras)
+        
+        elif optimizer_name in ["SPBM-TR-NoneCut"]:
+            optimizer = SPBM.TR_NoneCut(model.parameters(), model, hyperparams, Paras)
+
+        elif optimizer_name in ["SPBM-PF-NoneLower"]:
+            optimizer = SPBM.PF_NoneLower(model.parameters(), model, hyperparams, Paras)
+
+        elif optimizer_name in ["SPBM-PF"]:
+            optimizer = SPBM.PF(model.parameters(), model, hyperparams, Paras)
+        
+        elif optimizer_name in ["SPBM-PF-NoneCut"]:
+            optimizer = SPBM.PF_NoneCut(model.parameters(), model, hyperparams, Paras)
+
+        elif optimizer_name in ["SPSmax"]:
+            optimizer = ComOptimizers.SPSmax(
+                model.parameters(), model, hyperparams, Paras
+            )
+
+        elif optimizer_name in ["ALR-SMAG"]:
+            optimizer = ComOptimizers.ALR_SMAG(
+                model.parameters(), model, hyperparams, Paras
+            )
+
+        elif optimizer_name in ["Bundle"]:
+            optimizer = ComOptimizers.Bundle(
+                model.parameters(), model, hyperparams, Paras
+            )
+
+        else:
+            raise NotImplementedError(f"{optimizer_name} is not supported.")
+
+        return optimizer
+
+    

junshan_kit/Models.py

@@ -0,0 +1,207 @@
+import torchvision,torch, random
+import numpy as np
+from torchvision.models import resnet18,resnet34, ResNet18_Weights, ResNet34_Weights
+import torch.nn as nn
+
+
+# ---------------- Build ResNet18 - Caltech101 -----------------------
+def Build_ResNet18_CALTECH101_Resize_32():
+    
+    """
+    1. Modify the first convolutional layer for smaller input (e.g., 32x32 instead of 224x224)
+    Original: kernel_size=7, stride=2, padding=3 → changed to 3x3 kernel, stride=1, padding=1
+
+    2. Adjust the final fully connected layer to match the number of Caltech101 classes (101)
+    """
+    model = resnet18(weights=None)
+    model.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) # 1
+    model.fc = nn.Linear(model.fc.in_features, 101) # 2
+
+    return model
+
+
+# ---------------- Build ResNet18 - CIFAR100 -----------------------
+def Build_ResNet18_CIFAR100():
+    """
+    1. Modify the first convolutional layer for smaller input (e.g., 32x32 instead of 224x224)  
+    Original: kernel_size=7, stride=2, padding=3 → changed to 3x3 kernel, stride=1, padding=1
+
+    2. Adjust the final fully connected layer to match the number of CIFAR-100 classes (100)
+    """
+    
+    model = resnet18(weights=None)
+    # model = resnet18(weights=ResNet18_Weights.DEFAULT)
+    model.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)  # 1
+    model.fc = nn.Linear(model.fc.in_features, 100)  # 2
+
+    return model
+
+
+# ---------------- Build ResNet18 - MNIST ----------------------------
+def Build_ResNet18_MNIST():
+    """
+    1. Modify the first convolutional layer to accept grayscale input (1 channel instead of 3)  
+    Original: in_channels=3 → changed to in_channels=1
+
+    2. Adjust the final fully connected layer to match the number of MNIST classes (10)
+    """
+    
+    model = resnet18(weights=None)
+    model.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)  # 1
+    model.fc = nn.Linear(model.fc.in_features, 10)  # 2
+
+    return model
+
+
+# ---------------- Build ResNet34 - CIFAR100 -----------------------
+def Build_ResNet34_CIFAR100():
+    
+    model = resnet34(weights=None)
+    model.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) 
+    model.fc = nn.Linear(model.fc.in_features, 100)  
+    return model
+
+# ---------------- Build ResNet18 - MNIST ----------------------------
+def Build_ResNet34_MNIST():
+    # Do not load the pre-trained weights
+    model = resnet34(weights=None)  
+
+    model.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)  
+    model.fc = nn.Linear(model.fc.in_features, 10)  
+
+    return model
+
+# ---------------- Build ResNet34 - Caltech101 -----------------------
+def Build_ResNet34_CALTECH101_Resize_32():
+    
+    model = resnet34(weights=None)
+    model.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)
+    model.fc = nn.Linear(model.fc.in_features, 101)
+    return model
+
+
+#**************************************************************
+# ---------------------- LeastSquares -------------------------
+#**************************************************************
+# ---------------- LeastSquares - MNIST -----------------------
+def Build_LeastSquares_MNIST():
+    """
+    1. flatten MNIST images (1x28x28 → 784)
+    2. Use a linear layer for multi-classification
+    """
+    return nn.Sequential(
+        nn.Flatten(),                      
+        nn.Linear(28 * 28, 10))
+
+# ---------------- LeastSquares - CIFAR100 --------------------
+def Build_LeastSquares_CIFAR100():
+    """
+    1. flatten MNIST images (3 * 32 * 32 → 784)
+    2. Use a linear layer for multi-classification
+    """
+    return nn.Sequential(
+        nn.Flatten(),                      
+        nn.Linear(3 * 32 * 32, 100))
+
+# ---------------- LeastSquares - Caltech101 ------------------
+def Build_LeastSquares_CALTECH101_Resize_32():
+    return nn.Sequential(
+        nn.Flatten(),
+        nn.Linear(3*32*32, 101)
+    )
+
+
+#*************************************************************
+# --------------- LogRegressionBinary ------------------------
+#*************************************************************
+# -------------- LogRegressionBinary - MNIST ------------------
+def Build_LogRegressionBinary_MNIST():
+    """
+    1. flatten MNIST images (1x28x28 → 784)
+    2. Use a linear layer for binary classification
+    """
+    return nn.Sequential(
+        nn.Flatten(),                      
+        nn.Linear(28 * 28, 1))
+
+
+# --------------- LogRegressionBinary - CIFAR100 --------------
+def Build_LogRegressionBinary_CIFAR100():
+    """
+    1. flatten CIFAR100 images 
+    2. Use a linear layer for binary classification
+    """
+    return nn.Sequential(
+        nn.Flatten(),                      
+        nn.Linear(3* 32 * 32, 1))
+
+# -------------- LogRegressionBinary - RCV1 ------------------
+def Build_LogRegressionBinary_RCV1():
+    """
+    1. Use a linear layer for binary classification
+    """
+    return nn.Sequential(                    
+        nn.Linear(47236, 1))
+
+# <LogRegressionBinaryL2>
+#**************************************************************
+# ------------- LogRegressionBinaryL2 -------------------------
+#**************************************************************
+def Build_LogRegressionBinaryL2_RCV1():
+    """
+    1. Use a linear layer for binary classification
+    """
+    return nn.Sequential(                    
+        nn.Linear(47236, 1))
+# <LogRegressionBinaryL2>
+
+# ---------------------------------------------------------
+def Build_LogRegressionBinaryL2_MNIST():
+    """
+    1. flatten MNIST images (1x28x28 -> 784)
+    2. Use a linear layer for binary classification
+    """
+    return nn.Sequential(
+        nn.Flatten(),                      
+        nn.Linear(28 * 28, 1))
+
+# ---------------------------------------------------------
+def Build_LogRegressionBinaryL2_CIFAR100():
+    """
+    1. flatten CIFAR100 images 
+    2. Use a linear layer for binary classification
+    """
+    return nn.Sequential(
+        nn.Flatten(),                      
+        nn.Linear(3* 32 * 32, 1))
+
+# ---------------------------------------------------------
+def Build_LogRegressionBinaryL2_Duke():
+    """
+    Use a linear layer for binary classification
+    """
+    return nn.Sequential(
+        nn.Flatten(),                      
+        nn.Linear(7129, 1))
+
+# ---------------------------------------------------------
+def Build_LogRegressionBinaryL2_Ijcnn():
+    """
+    Use a linear layer for binary classification
+    """
+    return nn.Sequential(
+        nn.Flatten(),                      
+        nn.Linear(22, 1))
+
+# ---------------------------------------------------------
+def Build_LogRegressionBinaryL2_w8a():
+    """
+    Use a linear layer for binary classification
+    """
+    return nn.Sequential(
+        nn.Flatten(),                      
+        nn.Linear(300, 1))
+
+
+
+

junshan_kit/Print_Info.py

@@ -0,0 +1,59 @@
+
+
+
+# -------------------------------------------------------------
+def training_group(training_group):
+    print(f"--------------------- training_group ------------------")
+    for g in training_group:
+        print(g)
+    print(f"-------------------------------------------------------")
+
+
+def training_info(args, use_color, data_name, optimizer_name, folder_path, hyperparams, Paras, model_name):
+    if use_color:
+        print("\033[90m" + "-" * 115 + "\033[0m")
+        print(
+            f"\033[32m✅ \033[34mDataset:\033[32m {data_name}, \t\033[34mBatch-size:\033[32m {args.bs}, \t\033[34m(training, test) = \033[32m ({Paras['train_data_num']}, {Paras['test_data_num']}), \t\033[34m device:\033[32m {Paras['device']}"
+        )
+        print(
+            f"\033[32m✅ \033[34mOptimizer:\033[32m {optimizer_name}, \t\033[34mParams:\033[32m {hyperparams}"
+        )
+        print(
+            f'\033[32m✅ \033[34mmodel:\033[32m {model_name}, \t\033[34mmodel type:\033[32m {Paras["model_type"][model_name]},\t\033[34m loss_fn:\033[32m {Paras["loss_fn"]}'
+        )
+        print(f"\033[32m✅ \033[34mfolder_path:\033[32m {folder_path}")
+        print("\033[90m" + "-" * 115 + "\033[0m")
+
+    else:
+        print("-" * 115)
+        print(
+            f"✅ Dataset: {data_name}, \tBatch-size: {Paras['batch_size'][data_name]}, \t(training, val, test) = ({Paras['training_samples']}, {Paras['val_samples']}, {Paras['test_samples']}), \tdevice: {Paras['device']}"
+        )
+        print(f"✅ Optimizer: {optimizer_name}, \tParams: {hyperparams}")
+        print(
+            f'✅ model: {model_name}, \t model type: {Paras["model_type"][model_name]}, loss_fn: {Paras["loss_fn"]}'
+        )
+        print(f"✅ folder_path: {folder_path}")
+        print("-" * 115)
+        
+# <Step_7_2>
+
+def print_per_epoch_info(use_color, epoch, Paras, epoch_loss, training_loss, training_acc, test_loss, test_acc, run_time):
+    epochs = Paras["epochs"][Paras["data_name"]]
+    # result = [(k, f"{v:.4f}") for k, v in run_time.items()]
+    if use_color:
+        print(
+            f'\033[34m epoch = \033[32m{epoch+1}/{epochs}\033[0m,\t\b'
+            f'\033[34m epoch_loss = \033[32m{epoch_loss[epoch+1]:.4e}\033[0m,\t\b'
+            f'\033[34m train_loss = \033[32m{training_loss[epoch+1]:.4e}\033[0m,\t\b'
+            f'\033[34m train_acc = \033[32m{100 * training_acc[epoch+1]:.2f}%\033[0m,\t\b'
+            f'\033[34m test_acc = \033[32m{100 * test_acc[epoch+1]:.2f}%\033[0m,\t\b'
+            f'\033[34m time (ep, tr, te) = \033[32m({run_time["epoch"]:.2f}, {run_time["train"]:.2f}, {run_time["test"]:.2f})\033[0m')
+    else:
+        print(
+        f'epoch = {epoch+1}/{epochs},\t'
+        f'epoch_loss = {epoch_loss[epoch+1]:.4e},\t'
+        f'train_loss = {training_loss[epoch+1]:.4e},\t'
+        f'train_acc = {100 * training_acc[epoch+1]:.2f}%,\t'
+        f'test_acc = {100 * test_acc[epoch+1]:.2f}%,\t'
+        f'time (ep, tr, te) = ({run_time["epoch"]:.2f}, {run_time["train"]:.2f}, {run_time["test"]:.2f})')