junshan-kit 2.2.8__py2.py3-none-any.whl → 2.5.1__py2.py3-none-any.whl

junshan_kit/ParametersHub.py

@@ -0,0 +1,419 @@
+import numpy as np
+import sys, os, torch, random
+import argparse
+import junshan_kit.ModelsHub as ModelsHub
+
+
+class check_args:
+    def __init__(self):
+        pass
+
+    def get_args(self):
+        parser = argparse.ArgumentParser(description="Combined config argument example")
+
+        allowed_models = ["LS", "LRL2","ResNet18"]
+        allowed_optimizers = ["ADAM", "SGD", "Bundle"]
+
+        allowed_datasets = ["MNIST", 
+                            "CIFAR100",
+                            "AIP", 
+                            "CCFD",
+                            ]
+        
+        optimizers_mapping = {
+            "ADAM": "ADAM",
+            "SGD": "SGD",
+            "Bundle": "Bundle"
+        }
+
+        model_mapping = {
+            "LS": "LeastSquares",
+            "LRL2": "LogRegressionBinaryL2",
+            "ResNet18": "ResNet18"
+        }
+        
+        data_name_mapping = {
+            "MNIST": "MNIST",
+            "CIFAR100": "CIFAR100",
+            "AIP": "Adult_Income_Prediction",
+            "CCFD": "Credit_Card_Fraud_Detection"
+        }
+
+        
+
+        # Single combined argument that can appear multiple times
+        parser.add_argument(
+            "--train",
+            type=str,
+            nargs="+",                   # Allow multiple configs
+            required=True,
+            help = f"Format: model-dataset-optimizer (e.g., ResNet18-CIFAR10-Adam). model: {model_mapping}, \n datasets: {allowed_datasets}, optimizers: {allowed_optimizers},"
+        )
+
+        parser.add_argument(
+        "--e",
+        type=int,
+        required=True,
+        help="Number of training epochs. Example: --e 50"
+        )
+
+        parser.add_argument(
+            "--seed",
+            type=int,
+            default=42,
+            help="Random seed for experiment reproducibility. Default: 42"
+        )
+
+        parser.add_argument(
+            "--bs",
+            type=int,
+            required=True,
+            help="Batch size for training. Example: --bs 128"
+        )
+
+        parser.add_argument(
+            "--cuda",
+            type=int,
+            default=0,
+            required=True,
+            help="The number of cuda. Example: --cuda 1 (default=0) "
+        )
+
+        parser.add_argument(
+            "--s",
+            type=float, 
+            default=1.0, 
+            # required=True,
+            help="Proportion of dataset to use for training split. Example: --s 0.8 (default=1.0)"
+        )
+
+        parser.add_argument(
+        "--subset",
+        type=float,
+        nargs=2,
+        # required=True,
+        help = "Two subset ratios (train, test), e.g., --subset 0.7 0.3 or --subset 500 500"
+        )
+
+        args = parser.parse_args()
+        args.model_name_mapping = model_mapping
+        args.data_name_mapping = data_name_mapping
+        args.optimizers_name_mapping = optimizers_mapping
+
+
+        if args.subset is not None:
+            self.check_subset_info(args, parser)
+
+
+        self.check_args(args, parser, allowed_models, allowed_optimizers, allowed_datasets)
+
+        return args
+    
+    def check_subset_info(self, args, parser):
+        total = sum(args.subset)
+        if args.subset[0]>1:
+            # CHECK
+            for i in args.subset:
+                if i < 1:
+                    parser.error(f"Invalid --subset {args.subset}: The number of subdata must > 1")    
+        else:
+            if abs(total - 1.0) != 0.0:  
+                parser.error(f"Invalid --subset {args.subset}: the values must sum to 1.0 (current sum = {total:.6f}))")
+
+
+    def check_args(self, args, parser, allowed_models, allowed_optimizers, allowed_datasets):
+        # Parse and validate each train_group
+        for cfg in args.train:
+            try:
+                model, dataset, optimizer = cfg.split("-")
+            
+                if model not in allowed_models:
+                    parser.error(f"Invalid model '{model}'. Choose from {allowed_models}")
+                if optimizer not in allowed_optimizers:
+                    parser.error(f"Invalid optimizer '{optimizer}'. Choose from {allowed_optimizers}")
+                if dataset not in allowed_datasets:
+                    parser.error(f"Invalid dataset '{dataset}'. Choose from {allowed_datasets}")
+
+            except ValueError:
+                parser.error(f"Invalid format '{cfg}'. Use model-dataset-optimizer")
+
+        for cfg in args.train:
+            model_name, dataset_name, optimizer_name = cfg.split("-")
+            try:
+                f = getattr(ModelsHub, f"Build_{args.model_name_mapping[model_name]}_{args.data_name_mapping[dataset_name]}")
+
+            except:
+                print(getattr(ModelsHub, f"Build_{args.model_name_mapping[model_name]}_{args.data_name_mapping[dataset_name]}"))
+                assert False
+
+def get_train_group(args):
+    training_group = []
+    for cfg in args.train:
+        model, dataset, optimizer = cfg.split("-")
+        training_group.append((args.model_name_mapping[model], args.data_name_mapping[dataset], args.optimizers_name_mapping[optimizer]))
+
+    return training_group
+
+
+def set_paras(args, OtherParas):
+    Paras = {
+        # Name of the folder where results will be saved.
+        "results_folder_name": OtherParas["results_folder_name"],
+
+        # Print loss every N epochs.
+        "epoch_log_interval": 1,
+
+        # Timestamp string for result saving.
+        "time_str": ["time_str"],
+
+        # Random seed
+        "seed": args.seed,
+
+        # Device used for training.
+        "cuda": f"cuda:{args.cuda}",
+
+        # batch-size 
+        "batch_size": args.bs,
+
+        # epochs
+        "epochs": args.e,
+
+        # split_train_data
+        "split_train_data": args.s,
+
+        # select_subset
+        "select_subset": args.subset
+    }
+
+    Paras = model_list(Paras)
+    Paras = model_type(Paras)
+    Paras = data_list(Paras)
+    Paras = optimizer_dict(Paras, OtherParas)
+    Paras = device(Paras)
+        
+    return Paras
+
+
+def set_seed(seed=42):
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    np.random.seed(seed)
+    random.seed(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
+def device(Paras) -> dict:
+    device = torch.device(f"{Paras['cuda']}" if torch.cuda.is_available() else "cpu")
+    Paras["device"] = device
+    use_color = sys.stdout.isatty()
+    Paras["use_color"] = use_color
+
+    return Paras
+
+def model_list(Paras) -> dict:
+    model_list = [
+        "ResNet18",
+        "ResNet34",
+        "LeastSquares",
+        "LogRegressionBinary",
+        "LogRegressionBinaryL2",
+    ]
+    Paras["model_list"] = model_list
+    return Paras
+
+def model_type(Paras) -> dict:
+    model_type = {
+        "ResNet18": "multi",
+        "ResNet34": "multi",
+        "LeastSquares": "multi",
+        "LogRegressionBinary": "binary",
+        "LogRegressionBinaryL2": "binary",
+    }
+
+    Paras["model_type"] = model_type
+    return Paras
+
+def data_list(Paras) -> dict:
+    """
+    Attach a predefined list of dataset names to the parameter dictionary.
+
+    The predefined datasets include:
+    - Duke:
+        - classes: 2
+        - data: 42 (38 + 4)
+        - features: 7,129
+    - Ijcnn:
+        - classes: 2
+        - data: (35,000 + 91,701)
+        - features: 22
+    - w8a:
+        - classes: 2
+        - data: (49,749 + 14,951)
+        - features: 300
+    - RCV1
+    - Shuttle
+    - Letter
+    - Vowel
+    - MNIST
+    - CIFAR100
+    - CALTECH101_Resize_32
+    - Adult Income Prediction
+        - 
+    - Credit_Card_Fraud_Detection
+    """
+
+    data_list = [
+        "Duke",
+        "Ijcnn",
+        "w8a",
+        "RCV1",
+        "Shuttle",
+        "Letter",
+        "Vowel",
+        "MNIST",
+        "CIFAR100",
+        "CALTECH101_Resize_32",
+        "Adult_Income_Prediction",
+        "Credit_Card_Fraud_Detection"
+    ]
+    Paras["data_list"] = data_list
+    return Paras
+
+
+def optimizer_dict(Paras, OtherParas)->dict:
+    optimizer_dict = {
+    # ----------------- ADAM --------------------
+    "ADAM": {
+        "params": {
+            # "alpha": [2 * 1e-3],
+            "alpha": (
+                [0.5 * 1e-3, 1e-3, 2 * 1e-3]
+                if OtherParas["SeleParasOn"]
+                else [0.0005]
+            ),
+            "epsilon": [1e-8],
+            "beta1": [0.9],
+            "beta2": [0.999],
+        },
+    },
+    # ------------- ALR-SMAG --------------------
+    "ALR-SMAG": {
+        "params": {
+            "c": ([0.1, 0.5, 1, 5, 10] if OtherParas["SeleParasOn"] else [0.1]),
+            "eta_max": (
+                [2**i for i in range(-8, 9)]
+                if OtherParas["SeleParasOn"]
+                else [0.125]
+            ),
+            "beta": [0.9],
+        },
+    },
+    # ------------ Bundle -----------------------
+    "Bundle": {
+        "params": {
+            "delta": (
+                [2**i for i in range(-8, 9)]
+                if OtherParas["SeleParasOn"]
+                else [0.25]
+            ),
+            "cutting_number": [10],
+        },
+    },
+    # ------------------- SGD -------------------
+    "SGD": {
+        "params": {
+            "alpha": (
+                [2**i for i in range(-8, 9)] if OtherParas["SeleParasOn"] else [0.5]
+            )
+        }
+    },
+    # ------------------- SPSmax ----------------
+    "SPSmax": {
+        "params": {
+            "c": ([0.1, 0.5, 1, 5, 10] if OtherParas["SeleParasOn"] else [0.1]),
+            "gamma": (
+                [2**i for i in range(-8, 9)]
+                if OtherParas["SeleParasOn"]
+                else [0.125]),
+        },
+    },
+    # -------------- SPBM-PF --------------------
+    "SPBM-PF": {
+        "params": {
+            "M": [1e-5],
+            "delta": (
+                [2**i for i in range(9, 20)]
+                if OtherParas["SeleParasOn"]
+                else [1]
+            ),
+            "cutting_number": [10],
+        },
+    },
+    # -------------- SPBM-TR --------------------
+    "SPBM-TR": {
+        "params": {
+            "M": [1e-5],
+            "delta": (
+                [2**i for i in range(9, 20)]
+                if OtherParas["SeleParasOn"]
+                else [256]
+            ),
+            "cutting_number": [10],
+        },
+    },
+    
+    # ----------- SPBM-TR-NoneLower -------------
+    "SPBM-TR-NoneLower": {
+        "params": {
+            "M": [1e-5],
+            "delta": (
+                [2**i for i in range(0, 9)]
+                if OtherParas["SeleParasOn"]
+                else [256]
+            ),
+            "cutting_number": [10],
+        },
+    },
+    # ----------- SPBM-TR-NoneSpecial -----------
+    "SPBM-TR-NoneSpecial": {
+        "params": {
+            "M": [1e-5],
+            "delta": (
+                [2**i for i in range(-8, 9)]
+                if OtherParas["SeleParasOn"]
+                else [1]
+            ),
+            "cutting_number": [10],
+        },
+    },
+    # ------------- SPBM-PF-NoneLower -----------
+    "SPBM-PF-NoneLower": {
+        "params": {
+            "M": [1e-5],
+            "delta": (
+                [2**i for i in range(0, 9)]
+                if OtherParas["SeleParasOn"]
+                else [0]
+            ),
+            "cutting_number": [10],
+        },
+    },
+    }
+
+    Paras["optimizer_dict"] = optimizer_dict
+    return Paras
+
+
+
+def metrics()->dict:
+    metrics = {
+        "epoch_loss": [],
+        "training_loss": [],
+        "test_loss": [],
+        "iter_loss": [],
+        "training_acc": [],
+        "test_acc": [],
+        "grad_norm": [],
+        "per_epoch_loss": []
+    }
+    return metrics

junshan_kit/Print_Info.py

@@ -0,0 +1,63 @@
+from junshan_kit import ParametersHub
+
+
+# -------------------------------------------------------------
+def training_group(training_group):
+    print(f"--------------------- training_group ------------------")
+    for g in training_group:
+        print(g)
+    print(f"-------------------------------------------------------")
+
+
+def training_info(args, use_color, data_name, optimizer_name, folder_path, hyperparams, Paras, model_name):
+    if use_color:
+        print("\033[90m" + "-" * 115 + "\033[0m")
+        print(
+            f"\033[32m✅ \033[34mDataset:\033[32m {data_name}, \t\033[34mBatch-size:\033[32m {args.bs}, \t\033[34m(training, test) = \033[32m ({Paras['train_data_num']}, {Paras['test_data_num']}), \t\033[34m device:\033[32m {Paras['device']}"
+        )
+        print(
+            f"\033[32m✅ \033[34mOptimizer:\033[32m {optimizer_name}, \t\033[34mParams:\033[32m {hyperparams}"
+        )
+        print(
+            f'\033[32m✅ \033[34mmodel:\033[32m {model_name}, \t\033[34mmodel type:\033[32m {Paras["model_type"][model_name]},\t\033[34m loss_fn:\033[32m {Paras["loss_fn"]}'
+        )
+        print(f"\033[32m✅ \033[34mfolder_path:\033[32m {folder_path}")
+        print("\033[90m" + "-" * 115 + "\033[0m")
+
+    else:
+        print("-" * 115)
+        print(
+            f"✅ Dataset: {data_name}, \tBatch-size: {Paras['batch_size'][data_name]}, \t(training, val, test) = ({Paras['training_samples']}, {Paras['val_samples']}, {Paras['test_samples']}), \tdevice: {Paras['device']}"
+        )
+        print(f"✅ Optimizer: {optimizer_name}, \tParams: {hyperparams}")
+        print(
+            f'✅ model: {model_name}, \t model type: {Paras["model_type"][model_name]}, loss_fn: {Paras["loss_fn"]}'
+        )
+        print(f"✅ folder_path: {folder_path}")
+        print("-" * 115)
+        
+# <Step_7_2>
+
+def print_per_epoch_info(use_color, epoch, Paras, epoch_loss, training_loss, training_acc, test_loss, test_acc, run_time):
+    epochs = Paras["epochs"][Paras["data_name"]]
+    # result = [(k, f"{v:.4f}") for k, v in run_time.items()]
+    if use_color:
+        print(
+            f'\033[34m epoch = \033[32m{epoch+1}/{epochs}\033[0m,\t\b'
+            f'\033[34m epoch_loss = \033[32m{epoch_loss[epoch+1]:.4e}\033[0m,\t\b'
+            f'\033[34m train_loss = \033[32m{training_loss[epoch+1]:.4e}\033[0m,\t\b'
+            f'\033[34m train_acc = \033[32m{100 * training_acc[epoch+1]:.2f}%\033[0m,\t\b'
+            f'\033[34m test_acc = \033[32m{100 * test_acc[epoch+1]:.2f}%\033[0m,\t\b'
+            f'\033[34m time (ep, tr, te) = \033[32m({run_time["epoch"]:.2f}, {run_time["train"]:.2f}, {run_time["test"]:.2f})\033[0m')
+    else:
+        print(
+        f'epoch = {epoch+1}/{epochs},\t'
+        f'epoch_loss = {epoch_loss[epoch+1]:.4e},\t'
+        f'train_loss = {training_loss[epoch+1]:.4e},\t'
+        f'train_acc = {100 * training_acc[epoch+1]:.2f}%,\t'
+        f'test_acc = {100 * test_acc[epoch+1]:.2f}%,\t'
+        f'time (ep, tr, te) = ({run_time["epoch"]:.2f}, {run_time["train"]:.2f}, {run_time["test"]:.2f})')
+
+
+def data_info():
+    print(ParametersHub.data_list.__doc__)

junshan_kit/TrainingHub.py

@@ -0,0 +1,174 @@
+import torch, time
+import torch.nn as nn
+import numpy as np
+import torch.utils.data as Data
+from torch.nn.utils import parameters_to_vector
+from junshan_kit import DataHub, ParametersHub, TrainingHub, Evaluate_Metrics
+
+def chosen_loss_fn(model_name, Paras):
+    # ---------------------------------------
+    # There have an addition parameter
+    if model_name == "LogRegressionBinaryL2":
+        Paras["lambda"] = 1e-3
+    # ---------------------------------------
+
+    if model_name in ["LeastSquares"]:
+        loss_fn = nn.MSELoss()
+
+    else:
+        if Paras["model_type"][model_name] == "binary":
+            loss_fn = nn.BCEWithLogitsLoss()
+
+        elif Paras["model_type"][model_name] == "multi":
+            loss_fn = nn.CrossEntropyLoss()
+
+        else:
+            loss_fn = nn.MSELoss()
+            print("\033[91m The loss function is error!\033[0m")
+            assert False
+            
+    Paras["loss_fn"] = loss_fn
+
+    return loss_fn, Paras
+
+
+def load_data(model_name, data_name, Paras):
+    # load data
+    train_path = f"./exp_data/{data_name}/training_data"
+    test_path = f"./exp_data/{data_name}/test_data"
+
+    if data_name == "MNIST":
+        train_dataset, test_dataset, transform = DataHub.MNIST(Paras, model_name)
+
+    elif data_name == "CIFAR100":
+        train_dataset, test_dataset, transform = DataHub.CIFAR100(Paras, model_name)
+
+    elif data_name == "Adult_Income_Prediction":
+        train_dataset, test_dataset, transform = DataHub.Adult_Income_Prediction(Paras)
+
+    elif data_name == "Credit_Card_Fraud_Detection":
+        train_dataset, test_dataset, transform = DataHub.Credit_Card_Fraud_Detection(Paras)
+    
+
+    # elif data_name == "CALTECH101_Resize_32":
+    #     Paras["train_ratio"] = 0.7
+    #     train_dataset, test_dataset, transform = datahub.caltech101_Resize_32(
+    #         Paras["seed"], Paras["train_ratio"], split=True
+    #     )
+
+    # elif data_name in ["Vowel", "Letter", "Shuttle", "w8a"]:
+    #     Paras["train_ratio"] = Paras["split_train_data"][data_name]
+    #     train_dataset, test_dataset, transform = datahub.get_libsvm_data(
+    #         train_path + ".txt", test_path + ".txt", data_name
+    #     )
+
+    # elif data_name in ["RCV1", "Duke", "Ijcnn"]:
+    #     Paras["train_ratio"] = Paras["split_train_data"][data_name]
+    #     train_dataset, test_dataset, transform = datahub.get_libsvm_bz2_data(
+    #         train_path + ".bz2", test_path + ".bz2", data_name, Paras
+    #     )
+
+    else:
+        transform = None
+        print(f"The data_name is error!")
+        assert False
+
+    return train_dataset, test_dataset, transform
+
+
+def get_dataloader(data_name, train_dataset, test_dataset, Paras):
+    ParametersHub.set_seed(Paras["seed"])
+    g = torch.Generator()
+    g.manual_seed(Paras["seed"])
+    
+    train_loader = Data.DataLoader(
+            dataset=train_dataset,
+            shuffle=True,
+            batch_size=Paras["batch_size"],
+            generator=g,
+            num_workers=4,
+        )
+    
+    test_loader = Data.DataLoader(
+            dataset=test_dataset,
+            shuffle=False,
+            batch_size=Paras["batch_size"],
+            generator=g,
+            num_workers=4,
+        )
+    
+    return train_loader, test_loader
+
+def chosen_optimizer(optimizer_name, model, hyperparams, Paras):
+    if optimizer_name == "SGD":
+        optimizer = torch.optim.SGD(model.parameters(), lr=hyperparams["alpha"])
+
+    elif optimizer_name == "ADAM":
+        optimizer = torch.optim.Adam(
+            model.parameters(),
+            lr=hyperparams["alpha"],
+            betas=(hyperparams["beta1"], hyperparams["beta2"]),
+            eps=hyperparams["epsilon"],
+        )
+
+    else:
+        raise NotImplementedError(f"{optimizer_name} is not supported.")
+
+    return optimizer
+
+def load_model_dataloader(base_model_fun, initial_state_dict, data_name, train_dataset, test_dataset, Paras):
+    ParametersHub.set_seed(Paras["seed"])
+    model = base_model_fun()
+    model.load_state_dict(initial_state_dict)
+    model.to(Paras["device"])
+    train_loader, test_loader = TrainingHub.get_dataloader(data_name, train_dataset, test_dataset, Paras)
+
+    return model, train_loader, test_loader
+
+def train(train_loader, optimizer_name, optimizer, model, loss_fn, Paras):
+    metrics = ParametersHub.metrics()
+    for epoch in range(Paras["epochs"]):
+        for index, (X, Y) in enumerate(train_loader):
+            X, Y = X.to(Paras["device"]), Y.to(Paras["device"])
+
+            if optimizer_name in ["SGD", "ADAM"]:
+                optimizer.zero_grad()
+                loss = Evaluate_Metrics.compute_epoch_loss(X, Y, model, loss_fn, Paras)
+                loss.backward()
+                optimizer.step()
+
+            elif optimizer_name in [
+                "Bundle",
+                "SBPM",
+            ]:
+                def closure():
+                    optimizer.zero_grad()
+                    loss = Evaluate_Metrics.compute_epoch_loss(X, Y, model, loss_fn, Paras)
+                    loss.backward()
+                    return loss
+
+                loss = optimizer.step(closure)
+
+            else:
+                loss = 0
+                raise NotImplementedError(f"{optimizer_name} is not supported.")
+            
+            if index == 0 and epoch == 0:
+                metrics["iter_loss"].append(loss.item())
+                metrics["epoch_loss"].append(loss.item())
+                with torch.no_grad():
+                    g_k = parameters_to_vector(
+                        [
+                            p.grad if p.grad is not None else torch.zeros_like(p)
+                            for p in model.parameters()
+                        ]
+                    )
+                    metrics["grad_norm"].append(torch.norm(g_k, p=2).detach().cpu().item())
+        
+            metrics["per_epoch_loss"].append(loss.item())
+        metrics["epoch_loss"].append(np.mean(metrics["per_epoch_loss"]).item())
+
+        
+        
+        
+