junshan-kit 2.3.0__py2.py3-none-any.whl → 2.3.2__py2.py3-none-any.whl

junshan_kit/DataProcessor.py

@@ -6,7 +6,7 @@
 """
 
 import pandas as pd
-
+from sklearn.preprocessing import StandardScaler
 
 class CSV_TO_Pandas:
     def __init__(self):
@@ -18,7 +18,10 @@ class CSV_TO_Pandas:
         drop_cols: list,
         label_col: str,
         label_map: dict,
+        data_name: str,
+        user_one_hot_cols = [],
         print_info=False,
+        Standard = False
     ):
         """
         Preprocess a CSV dataset by performing data cleaning, label mapping, and feature encoding.
@@ -87,9 +90,15 @@ class CSV_TO_Pandas:
             col for col in text_feature_cols if col != label_col
         ]  # ✅ exclude label
 
-        df = pd.get_dummies(df, columns=text_feature_cols, dtype=int)
+        df = pd.get_dummies(df, columns=text_feature_cols + user_one_hot_cols, dtype=int)
         m_cleaned, n_cleaned = df.shape
 
+        if Standard:
+        # Identify numerical columns Standardize numerical columns
+            num_cols = [col for col in df.columns if col not in list(text_feature_cols) + [label_col] + [user_one_hot_cols]]
+            scaler = StandardScaler()
+            df[num_cols] = scaler.fit_transform(df[num_cols])
+
         # print info
         if print_info:
             pos_count = (df[label_col] == 1).sum()
@@ -97,20 +106,21 @@ class CSV_TO_Pandas:
 
             # Step 6: Print dataset information
             print("\n" + "=" * 80)
-            print(f"{'Dataset Info':^70}")
+            print(f"{f'{data_name} - Info':^70}")
             print("=" * 80)
             print(f"{'Original size:':<40} {m_original} rows x {n_original} cols")
             print(
-                f"{'Size after dropping NaN & non-feature cols:':<40} {m_cleaned} rows x {n_cleaned} cols"
+                f"{'Size after dropping NaN & non-feature cols:':<40} {m_encoded} rows x {n_encoded} cols"
             )
             print(f"{'Positive samples (+1):':<40} {pos_count}")
             print(f"{'Negative samples (-1):':<40} {neg_count}")
             print(
-                f"{'Size after one-hot encoding:':<40} {m_encoded} rows x {n_encoded} cols"
+                f"{'Size after one-hot encoding:':<40} {m_cleaned} rows x {n_cleaned} cols"
             )
             print("-" * 80)
             print(f"Note:")
-            print(f"{'Label column:':<40} {label_col}")
+            print(f"{'Label column:':<40} {label_col}") 
+            print(f"{'label_map:':<40} {label_map}")
             print(
                 f"{'Dropped non-feature columns:':<40} {', '.join(drop_cols) if drop_cols else 'None'}"
             )

junshan_kit/DataSets.py

@@ -36,16 +36,26 @@ def _download_data(data_name):
         else:
             print("❌ Invalid choice. Please enter 1 or 2.\n")
 
-def _run(csv_path, data_name, drop_cols, label_col, label_map, print_info):
+def _export_csv(df, data_name):
+    path = f'./data_trans_fea/{data_name}/'
+    os.makedirs(path, exist_ok=True)
+    df.to_csv(path + f'{data_name}.csv')
+    print(path + f'{data_name}.csv')
+
+
+def _run(csv_path, data_name, drop_cols, label_col, label_map, print_info, user_one_hot_cols = [], export_csv = False, ):
     if not os.path.exists(csv_path):
         print('\n' + '*'*60)
         print(f"Please download the data.")
         print(csv_path)
         _download_data(data_name)
-        junshan_kit.kit.unzip_file(f'./exp_data/{data_name}/{data_name}.zip', f'./exp_data/{data_name}')
-        
+        junshan_kit.kit.unzip_file(f'./exp_data/{data_name}/{data_name}.zip', f'./exp_data/{data_name}')   
+    
     cleaner = junshan_kit.DataProcessor.CSV_TO_Pandas()
-    df = cleaner.preprocess_dataset(csv_path, drop_cols, label_col, label_map, print_info=print_info)
+    df = cleaner.preprocess_dataset(csv_path, drop_cols, label_col, label_map, data_name,user_one_hot_cols, print_info=print_info)
+
+    if export_csv:
+        _export_csv(df, data_name)
 
     return df
 
@@ -67,7 +77,7 @@ def credit_card_fraud_detection(data_name = "Credit Card Fraud Detection", print
     return df
 
 
-def diabetes_health_indicators_dataset(data_name = "Diabetes Health Indicators Dataset", print_info = False):
+def diabetes_health_indicators_dataset(data_name = "Diabetes Health Indicators", print_info = False):
     csv_path = f'./exp_data/{data_name}/diabetes_dataset.csv'
     drop_cols = []
     label_col = 'diagnosed_diabetes'
@@ -78,7 +88,7 @@ def diabetes_health_indicators_dataset(data_name = "Diabetes Health Indicators D
     return df
 
 
-def electric_vehicle_population_data(data_name = "Electric Vehicle Population Data", print_info = False):
+def electric_vehicle_population_data(data_name = "Electric Vehicle Population", print_info = False):
     csv_path = f'./exp_data/{data_name}/Electric_Vehicle_Population_Data.csv'
     drop_cols = ['VIN (1-10)', 'DOL Vehicle ID', 'Vehicle Location']
     label_col = 'Electric Vehicle Type'
@@ -91,10 +101,10 @@ def electric_vehicle_population_data(data_name = "Electric Vehicle Population Da
 
     return df
 
-def global_house_purchase_dataset(data_name = "Global House Purchase Dataset", print_info = False):
+def global_house_purchase_dataset(data_name = "Global House Purchase", print_info = False):
     csv_path = f'./exp_data/{data_name}/global_house_purchase_dataset.csv'
     drop_cols = ['property_id']
-    label_col = 'Electric Vehicle Type'
+    label_col = 'decision'
     label_map = {0: -1, 1: 1}
 
     df = _run(csv_path, data_name, drop_cols, label_col, label_map, print_info)
@@ -102,10 +112,10 @@ def global_house_purchase_dataset(data_name = "Global House Purchase Dataset", p
     return df
 
 
-def health_lifestyle_dataset(data_name = "Health_lifestyle_dataset", print_info = False):
+def health_lifestyle_dataset(data_name = "Health Lifestyle", print_info = False):
     csv_path = f'./exp_data/{data_name}/health_lifestyle_dataset.csv'
     drop_cols = ['id']
-    label_col = 'decision'
+    label_col = 'disease_risk'
     label_map = {0: -1, 1: 1}
 
     df = _run(csv_path, data_name, drop_cols, label_col, label_map, print_info)
@@ -114,8 +124,123 @@ def health_lifestyle_dataset(data_name = "Health_lifestyle_dataset", print_info
 
 
 
+def medical_insurance_cost_prediction(data_name = "Medical Insurance Cost Prediction", print_info = False):
+    """
+    1. The missing values in this dataset are handled by directly removing the corresponding column. Since the `alcohol_freq` column contains a large number of missing values, deleting the rows would result in significant data loss, so the entire column is dropped instead.
+
+    2. There are several columns that could serve as binary classification labels, such as `is_high_risk`, `cardiovascular_disease`, and `liver_disease`. In this case, `is_high_risk` is chosen as the label column.
+    """
+    csv_path = f'./exp_data/{data_name}/medical_insurance.csv'
+    drop_cols = ['alcohol_freq']
+    label_col = 'is_high_risk'
+    label_map = {0: -1, 1: 1}
+
+    df = _run(csv_path, data_name, drop_cols, label_col, label_map, print_info)
+
+    return df
+
 
+def particle_physics_event_classification(data_name = "Particle Physics Event Classification", print_info = False):
+    csv_path = f'./exp_data/{data_name}/Particle Physics Event Classification.csv'
+    drop_cols = []
+    label_col = 'Label'
+    label_map = {'s': -1, 'b': 1}
 
-def wine_and_food_pairing_dataset():
-    pass 
+    df = _run(csv_path, data_name, drop_cols, label_col, label_map, print_info)
 
+    return df
+
+
+
+def adult_income_prediction(data_name = "Adult Income Prediction", print_info = False):
+    csv_path = f'./exp_data/{data_name}/adult.csv'
+    drop_cols = []
+    label_col = 'income'
+    label_map = {'<=50K': -1, '>50K': 1}
+
+    df = _run(csv_path, data_name, drop_cols, label_col, label_map, print_info)
+
+    return df
+
+
+
+
+def TamilNadu_weather_2020_2025(data_name = "TN Weather 2020-2025", print_info = False, export_csv = False):
+    csv_path = f'./exp_data/{data_name}/TNweather_1.8M.csv'
+    label_col = 'rain_tomorrow'
+    label_map = {0: -1, 1: 1}
+    
+    # Step 0: Load the dataset
+    df = pd.read_csv(csv_path)
+
+    df['time'] = pd.to_datetime(df['time'])
+    df['year'] = df['time'].dt.year
+    df['month'] = df['time'].dt.month
+    df['day'] = df['time'].dt.day
+    df['hour'] = df['time'].dt.hour
+    
+    user_one_hot_cols = ['year','month','day', 'hour']
+    drop_cols = ['Unnamed: 0', 'time']
+
+    # Save original size
+    m_original, n_original = df.shape
+
+    # Step 1: Drop non-informative columns
+    df = df.drop(columns=drop_cols)
+
+    # Step 2: Remove rows with missing values
+    df = df.dropna(axis=0, how="any")
+    m_encoded, n_encoded = df.shape
+
+    # Step 3: Map target label (to -1 and +1)
+    df[label_col] = df[label_col].map(label_map)
+
+    # Step 4: Encode categorical features (exclude label column)
+    text_feature_cols = df.select_dtypes(
+        include=["object", "string", "category"]
+    ).columns
+    text_feature_cols = [
+        col for col in text_feature_cols if col != label_col
+    ]  # ✅ exclude label
+
+    df = pd.get_dummies(df, columns=text_feature_cols + user_one_hot_cols, dtype=int)
+    m_cleaned, n_cleaned = df.shape
+
+    num_cols = [col for col in df.columns if col not in list(text_feature_cols) + [label_col] + user_one_hot_cols]
+    scaler = StandardScaler()
+    df[num_cols] = scaler.fit_transform(df[num_cols])
+
+    if export_csv:
+        _export_csv(df, data_name)
+
+    # print info
+    if print_info:
+        pos_count = (df[label_col] == 1).sum()
+        neg_count = (df[label_col] == -1).sum()
+
+        # Step 6: Print dataset information
+        print("\n" + "=" * 80)
+        print(f"{f'{data_name} - Info':^70}")
+        print("=" * 80)
+        print(f"{'Original size:':<40} {m_original} rows x {n_original} cols")
+        print(
+            f"{'Size after dropping NaN & non-feature cols:':<40} {m_encoded} rows x {n_encoded} cols"
+        )
+        print(f"{'Positive samples (+1):':<40} {pos_count}")
+        print(f"{'Negative samples (-1):':<40} {neg_count}")
+        print(
+            f"{'Size after one-hot encoding:':<40} {m_cleaned} rows x {n_cleaned} cols"
+        )
+        print("-" * 80)
+        print(f"Note:")
+        print(f"{'Label column:':<40} {label_col}") 
+        print(f"{'label_map:':<40} {label_map}")
+        print(
+            f"{'Dropped non-feature columns:':<40} {', '.join(drop_cols) if drop_cols else 'None'}"
+        )
+        print(
+            f"{'text fetaure columns:':<40} {', '.join(list(text_feature_cols)) if list(text_feature_cols) else 'None'}"
+        )
+        print("=" * 80 + "\n")
+
+    return df

{junshan_kit-2.3.0.dist-info → junshan_kit-2.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: junshan_kit
-Version: 2.3.0
+Version: 2.3.2
 Summary: This is an optimization tool.
 Author-email: Junshan Yin <junshanyin@163.com>
 Requires-Dist: kaggle==1.7.4.5

junshan_kit-2.3.2.dist-info/RECORD

@@ -0,0 +1,7 @@
+junshan_kit/DataProcessor.py,sha256=YIZMy2gnqnT8n9MMT-q7WtRB1bbA4ITwPxNBEasAnLQ,4966
+junshan_kit/DataSets.py,sha256=qN4lTVaUsKlu4b8tkZ3aMgHg9lyZTQJlYsgc0uLwUys,8570
+junshan_kit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+junshan_kit/kit.py,sha256=tB1TpW9hW1EweK1RQwHOdUo7uG1QU4vSeyR0fdaSydo,9569
+junshan_kit-2.3.2.dist-info/METADATA,sha256=OcuB7peGzVOuC0KIcohSPiCBdAs9BZ1mpZjX4ySSKpk,329
+junshan_kit-2.3.2.dist-info/WHEEL,sha256=tkmg4JIqwd9H8mL30xA7crRmoStyCtGp0VWshokd1Jc,105
+junshan_kit-2.3.2.dist-info/RECORD,,

junshan_kit-2.3.0.dist-info/RECORD

@@ -1,7 +0,0 @@
-junshan_kit/DataProcessor.py,sha256=eryVmS5BFZj8wjDN2QWVHqkbFgFuWU0HXV9s6TGf9QM,4442
-junshan_kit/DataSets.py,sha256=rf5AVlA9DxP7wBpXjSO1_xznCMuxEoK50TqExafwHhc,3972
-junshan_kit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-junshan_kit/kit.py,sha256=tB1TpW9hW1EweK1RQwHOdUo7uG1QU4vSeyR0fdaSydo,9569
-junshan_kit-2.3.0.dist-info/METADATA,sha256=9NlU4YOD0zx5F5tLIMREKKxn-LwYD8-7IFVtp7DvMNM,329
-junshan_kit-2.3.0.dist-info/WHEEL,sha256=tkmg4JIqwd9H8mL30xA7crRmoStyCtGp0VWshokd1Jc,105
-junshan_kit-2.3.0.dist-info/RECORD,,