junshan-kit 2.1.6__py2.py3-none-any.whl → 2.1.7__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
junshan_kit/DataProcessor.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
|
|
2
2
|
import pandas as pd
|
|
3
3
|
import os
|
|
4
|
-
|
|
4
|
+
from sklearn.preprocessing import StandardScaler
|
|
5
5
|
import junshan_kit.datahub
|
|
6
6
|
|
|
7
7
|
class CSVToPandas:
|
|
@@ -16,6 +16,7 @@ class CSVToPandas:
|
|
|
16
16
|
|
|
17
17
|
# ----------------- ccfd_kaggle ----------------------------------
|
|
18
18
|
def ccfd_kaggle(self, data_name = 'ccfd-kaggle', show_info = True):
|
|
19
|
+
# download data if not exist
|
|
19
20
|
self.read_csv(data_name)
|
|
20
21
|
|
|
21
22
|
df = pd.read_csv(self.csv_path)
|
|
@@ -33,13 +34,72 @@ class CSVToPandas:
|
|
|
33
34
|
print('='*60)
|
|
34
35
|
print(f"{'Original size:':<25} {m_before} rows x {n_before} cols")
|
|
35
36
|
print(f"{'Size after dropping NaNs:':<25} {m_after} rows x {n_after} cols")
|
|
36
|
-
print(f"{'Export size:':<25} {m_after} rows x {n_after} cols")
|
|
37
37
|
print(f"{'Positive samples (+1):':<25} {pos_count}")
|
|
38
38
|
print(f"{'Negative samples (-1):':<25} {neg_count}")
|
|
39
|
+
print(f"{'Export size:':<25} {m_after} rows x {n_after} cols")
|
|
39
40
|
print('-'*60)
|
|
40
41
|
print(f"More details: https://www.jianguoyun.com/p/Dd1clVgQ4ZThCxiwzZQGIAA")
|
|
41
42
|
print('='*60 + '\n')
|
|
42
43
|
|
|
43
44
|
return df
|
|
45
|
+
|
|
46
|
+
# ------------------------
|
|
47
|
+
def ghpdd_kaggle(self, data_name='ghpdd-kaggle', show_info=True):
|
|
48
|
+
# download data if not exist
|
|
49
|
+
self.read_csv(data_name)
|
|
50
|
+
|
|
51
|
+
# read csv
|
|
52
|
+
df = pd.read_csv(self.csv_path)
|
|
53
|
+
m_before, n_before = df.shape
|
|
54
|
+
|
|
55
|
+
# drop NaNs
|
|
56
|
+
df = df.dropna(axis=0, how='any')
|
|
57
|
+
m_after, n_after = df.shape
|
|
58
|
+
|
|
59
|
+
# drop unique identifier
|
|
60
|
+
if 'property_id' in df.columns:
|
|
61
|
+
df.drop(columns=['property_id'], inplace=True)
|
|
62
|
+
|
|
63
|
+
# Replace label 0 with -1
|
|
64
|
+
df['decision'] = df['decision'].replace(0, -1)
|
|
65
|
+
|
|
66
|
+
# Identify categorical and numerical columns
|
|
67
|
+
cat_cols = ['country', 'city', 'property_type', 'furnishing_status']
|
|
68
|
+
num_cols = [col for col in df.columns if col not in cat_cols + ['decision']]
|
|
69
|
+
|
|
70
|
+
# One-Hot encode categorical columns
|
|
71
|
+
df = pd.get_dummies(df, columns=cat_cols)
|
|
72
|
+
|
|
73
|
+
# Convert boolean columns to int
|
|
74
|
+
bool_cols = df.select_dtypes(include='bool').columns
|
|
75
|
+
for col in bool_cols:
|
|
76
|
+
df[col] = df[col].astype(int)
|
|
77
|
+
|
|
78
|
+
# Standardize numerical columns
|
|
79
|
+
scaler = StandardScaler()
|
|
80
|
+
df[num_cols] = scaler.fit_transform(df[num_cols])
|
|
81
|
+
|
|
82
|
+
# 导出后的大小
|
|
83
|
+
m_export, n_export = df.shape
|
|
84
|
+
|
|
85
|
+
if show_info:
|
|
86
|
+
pos_count = (df['decision'] == 1).sum()
|
|
87
|
+
neg_count = (df['decision'] == -1).sum()
|
|
88
|
+
|
|
89
|
+
print('\n' + '='*70)
|
|
90
|
+
print(f"{'GHPDD-Kaggle Dataset Info':^70}")
|
|
91
|
+
print('='*70)
|
|
92
|
+
print(f"{'Original size:':<35} {m_before} rows x {n_before} cols")
|
|
93
|
+
print(f"{'Size after dropping NaNs:':<35} {m_after} rows x {n_after} cols")
|
|
94
|
+
print(f"{'Export size (after encoding & scaling):':<35} {m_export} rows x {n_export} cols")
|
|
95
|
+
print(f"{'Positive samples (+1):':<35} {pos_count}")
|
|
96
|
+
print(f"{'Negative samples (-1):':<35} {neg_count}")
|
|
97
|
+
print('-'*70)
|
|
98
|
+
print(f"{'Note: categorical columns one-hot encoded, numerical standardized.':^70}")
|
|
99
|
+
print(f"More details: https://www.jianguoyun.com/p/DU6Lr9oQqdHDDRj5sI0GIAA")
|
|
100
|
+
print('='*70 + '\n')
|
|
101
|
+
|
|
102
|
+
return df
|
|
103
|
+
|
|
44
104
|
|
|
45
105
|
|
junshan_kit/test.py
CHANGED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
junshan_kit/DataProcessor.py,sha256=rp_w325h8EvKcLMSa12w5B-UA8G75O1qP0ogE6GDSE0,3886
|
|
2
|
+
junshan_kit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
junshan_kit/datahub.py,sha256=BWcG_TPW1xf_y_GzxRXanuOAB01WugBiO5r53EDbr8s,1815
|
|
4
|
+
junshan_kit/test.py,sha256=jyZQPgX40HlLM23vGMbuZFwFBk7YiFqzzh9xuOTzbw8,91
|
|
5
|
+
junshan_kit-2.1.7.dist-info/METADATA,sha256=ePQG7bT7y7yVU7iSI3CxnfNacwJLyAlSB7nEmAG3_NM,599
|
|
6
|
+
junshan_kit-2.1.7.dist-info/WHEEL,sha256=tkmg4JIqwd9H8mL30xA7crRmoStyCtGp0VWshokd1Jc,105
|
|
7
|
+
junshan_kit-2.1.7.dist-info/RECORD,,
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
junshan_kit/DataProcessor.py,sha256=9mlLYxdDiMX7baZmfJk5QuxT4vx_V728XIFbkXmCP0s,1594
|
|
2
|
-
junshan_kit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
-
junshan_kit/datahub.py,sha256=BWcG_TPW1xf_y_GzxRXanuOAB01WugBiO5r53EDbr8s,1815
|
|
4
|
-
junshan_kit/test.py,sha256=aEaobINtr4Ri0jX6D8u49xgftyA6SE12wx0P6m5x-2w,90
|
|
5
|
-
junshan_kit-2.1.6.dist-info/METADATA,sha256=KZTS690qvlgOduiYwo6oqshsk4dqY8m9HcVtWu-aXTI,599
|
|
6
|
-
junshan_kit-2.1.6.dist-info/WHEEL,sha256=tkmg4JIqwd9H8mL30xA7crRmoStyCtGp0VWshokd1Jc,105
|
|
7
|
-
junshan_kit-2.1.6.dist-info/RECORD,,
|
|
File without changes
|