edahub 0.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edahub-0.0.0/PKG-INFO +18 -0
- edahub-0.0.0/README.md +1 -0
- edahub-0.0.0/edahub/__init__.py +1 -0
- edahub-0.0.0/edahub/logic/__init__.py +0 -0
- edahub-0.0.0/edahub/logic/project.py +79 -0
- edahub-0.0.0/edahub/logic/stats_calculator.py +95 -0
- edahub-0.0.0/edahub/logic/vis/__init__.py +0 -0
- edahub-0.0.0/edahub/logic/vis/binning.py +166 -0
- edahub-0.0.0/edahub/logic/vis/histogram.py +54 -0
- edahub-0.0.0/edahub/widgets/__init__.py +1 -0
- edahub-0.0.0/edahub/widgets/column_summary.py +71 -0
- edahub-0.0.0/edahub/widgets/main.py +58 -0
- edahub-0.0.0/edahub.egg-info/PKG-INFO +18 -0
- edahub-0.0.0/edahub.egg-info/SOURCES.txt +17 -0
- edahub-0.0.0/edahub.egg-info/dependency_links.txt +1 -0
- edahub-0.0.0/edahub.egg-info/requires.txt +6 -0
- edahub-0.0.0/edahub.egg-info/top_level.txt +1 -0
- edahub-0.0.0/setup.cfg +4 -0
- edahub-0.0.0/setup.py +28 -0
edahub-0.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: edahub
|
|
3
|
+
Version: 0.0.0
|
|
4
|
+
Summary: T.B.A.
|
|
5
|
+
Home-page: https://github.com/not-so-fat/edahub
|
|
6
|
+
Author: @not-so-fat
|
|
7
|
+
Author-email: conjurer.not.so.fat@gmail.com
|
|
8
|
+
License: UNKNOWN
|
|
9
|
+
Platform: UNKNOWN
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Requires-Python: >=3.9
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# EDAHA
|
|
17
|
+
|
|
18
|
+
|
edahub-0.0.0/README.md
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# EDAHA
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .logic.project import EDAHub
|
|
File without changes
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
import threading
|
|
3
|
+
|
|
4
|
+
import pandas
|
|
5
|
+
|
|
6
|
+
from ..widgets import EDAHubWidget
|
|
7
|
+
from . import stats_calculator
|
|
8
|
+
from .vis import histogram
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
STATUS_COLUMNS = ["name", "#rows", "#cols", "status", "created"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class EDAHub:
|
|
15
|
+
def __init__(self, name=None):
|
|
16
|
+
self.name = name or _get_name()
|
|
17
|
+
self.tables = {}
|
|
18
|
+
self.status_table = pandas.DataFrame(columns=STATUS_COLUMNS)
|
|
19
|
+
self.stats_tables = {}
|
|
20
|
+
self.histograms = {}
|
|
21
|
+
self.eda_thread = None
|
|
22
|
+
self.widget = EDAHubWidget(self)
|
|
23
|
+
|
|
24
|
+
def add_table(self, name, df, overwrite=False):
|
|
25
|
+
if not overwrite and name in self.tables:
|
|
26
|
+
raise Exception(f"name={name} has been already registered. Use a different name or the option `overwrite=True`")
|
|
27
|
+
now = datetime.now().replace(microsecond=0)
|
|
28
|
+
new_row = pandas.DataFrame(
|
|
29
|
+
{k: [v] for k, v in zip(
|
|
30
|
+
STATUS_COLUMNS, [name, len(df), len(df.columns), "computing stats", now]
|
|
31
|
+
)}
|
|
32
|
+
)
|
|
33
|
+
if len(self.status_table):
|
|
34
|
+
self.status_table = pandas.concat([self.status_table, new_row], ignore_index=True)
|
|
35
|
+
else:
|
|
36
|
+
self.status_table = new_row
|
|
37
|
+
self.tables[name] = df
|
|
38
|
+
self.stats_tables[name] = self._get_schema(df)
|
|
39
|
+
self.widget.update(None)
|
|
40
|
+
self._compute_stats(name, df)
|
|
41
|
+
self._add_histograms(name, df)
|
|
42
|
+
|
|
43
|
+
def get_table_names(self):
|
|
44
|
+
#TODO: sort table names based on created
|
|
45
|
+
return list(self.tables.keys())
|
|
46
|
+
|
|
47
|
+
def _get_schema(self, df):
|
|
48
|
+
return pandas.DataFrame({
|
|
49
|
+
"column_name": df.columns,
|
|
50
|
+
"dtype": df.dtypes
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
def _compute_stats(self, name, df):
|
|
54
|
+
def target():
|
|
55
|
+
self.stats_tables[name] = stats_calculator.calc_column_stat(df)
|
|
56
|
+
self._update_status(name, "generating histograms")
|
|
57
|
+
self.widget.update(None)
|
|
58
|
+
|
|
59
|
+
self.eda_thread = threading.Thread(target=target)
|
|
60
|
+
self.eda_thread.start()
|
|
61
|
+
|
|
62
|
+
def _add_histograms(self, name, df):
|
|
63
|
+
def target():
|
|
64
|
+
self.histograms[name] = {
|
|
65
|
+
c: histogram.plot_histogram(self.tables[name][c])
|
|
66
|
+
for c in self.tables[name].columns
|
|
67
|
+
}
|
|
68
|
+
self._update_status(name, "done")
|
|
69
|
+
self.widget.update(None)
|
|
70
|
+
|
|
71
|
+
self.eda_thread = threading.Thread(target=target)
|
|
72
|
+
self.eda_thread.start()
|
|
73
|
+
|
|
74
|
+
def _update_status(self, name, status):
|
|
75
|
+
self.status_table.loc[self.status_table["name"]==name, "status"] = status
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _get_name():
|
|
79
|
+
return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import math
|
|
2
|
+
import copy
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
import numpy
|
|
6
|
+
import pandas
|
|
7
|
+
from pandas.api import types
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def calc_column_stat(df):
|
|
14
|
+
df_size = len(df)
|
|
15
|
+
output_column_names = [
|
|
16
|
+
"column_name", "dtype", "min", "max", "mean", "std", "ratio_na",
|
|
17
|
+
"ratio_zero", "unique_count", "is_unique"
|
|
18
|
+
]
|
|
19
|
+
def _calc_stat(column, df_size):
|
|
20
|
+
unique_count = len(get_unique_values(df, column))
|
|
21
|
+
logger.info("...calculating {} (dtype: {})".format(column, df.dtypes[column]))
|
|
22
|
+
return pandas.DataFrame(
|
|
23
|
+
{
|
|
24
|
+
"column_name": [column],
|
|
25
|
+
"dtype": df.dtypes[column],
|
|
26
|
+
"min": [df[column].min()] if _orderable(df.dtypes[column]) else [pandas.NA],
|
|
27
|
+
"max": [df[column].max()] if _orderable(df.dtypes[column]) else [pandas.NA],
|
|
28
|
+
"mean": [df[column].mean()] if types.is_numeric_dtype(df.dtypes[column]) else [pandas.NA],
|
|
29
|
+
"std": [df[column].std()] if types.is_numeric_dtype(df.dtypes[column]) else [pandas.NA],
|
|
30
|
+
"ratio_na": [pandas.isnull(df[column]).sum() / float(df_size)],
|
|
31
|
+
"ratio_zero": [len(df[df[column]==0]) / float(df_size)]\
|
|
32
|
+
if types.is_numeric_dtype(df.dtypes[column]) else [pandas.NA],
|
|
33
|
+
"unique_count": [unique_count],
|
|
34
|
+
"is_unique": [unique_count == df_size]
|
|
35
|
+
},
|
|
36
|
+
columns=output_column_names
|
|
37
|
+
)
|
|
38
|
+
return pandas.concat([_calc_stat(c, df_size) for c in df.columns], axis=0, ignore_index=True)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def get_unique_values(df, columns):
|
|
42
|
+
df_tmp = df[columns].dropna()
|
|
43
|
+
if isinstance(df_tmp, pandas.Series):
|
|
44
|
+
return set([v for v in df_tmp.values])
|
|
45
|
+
else:
|
|
46
|
+
return set([tuple(v) for v in df_tmp.values])
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def calculate_percentiles_for_df(df, column_list, ratio_list):
|
|
50
|
+
value_dic = {
|
|
51
|
+
**{
|
|
52
|
+
"name": ["min"] +
|
|
53
|
+
["{0:.2%}-percentile".format(ratio) for ratio in ratio_list]
|
|
54
|
+
+ ["max"]
|
|
55
|
+
},
|
|
56
|
+
**{
|
|
57
|
+
c: calculate_percentiles(df[column].values, ratio_list)
|
|
58
|
+
for c in column_list
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return pandas.DataFrame(value_dic, columns=["name"] + column_list).set_index("name")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def calculate_percentiles(array, ratio_list):
|
|
65
|
+
n_record0 = array.shape[0]
|
|
66
|
+
valid_array = array[numpy.isfinite(array)]
|
|
67
|
+
n_record = valid_array.shape[0]
|
|
68
|
+
if n_record0 - n_record > 0:
|
|
69
|
+
logger.info("{0} records have missing values (out of {1} records)".format(n_record0 - n_record, n_record0))
|
|
70
|
+
sorted_array = numpy.sort(valid_array)
|
|
71
|
+
ind_list = [_get_ind(n_record, ratio) for ratio in ratio_list]
|
|
72
|
+
return numpy.array([sorted_array[ind] for ind in ind_list])
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def calculate_series_stats(column_name, column_dtype, grb_obj):
|
|
76
|
+
if types.is_numeric_dtype(column_dtype):
|
|
77
|
+
return [f"{agg}({column_name})" for agg in ["min", "max", "mean", "std"]], \
|
|
78
|
+
[grb_obj[column_name].min(), grb_obj[column_name].max(),
|
|
79
|
+
grb_obj[column_name].mean(), grb_obj[column_name].std()]
|
|
80
|
+
elif types.is_datetime64_any_dtype(column_dtype):
|
|
81
|
+
return [f"{agg}({column_name})" for agg in ["min", "max"]], \
|
|
82
|
+
[grb_obj[column_name].min(), grb_obj[column_name].max()]
|
|
83
|
+
else:
|
|
84
|
+
return [f"nunique({column_name})"], [grb_obj[column_name].nunique()]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _get_ind(n_record, ratio):
|
|
88
|
+
return 0 if ratio == 0 else int(math.ceil(n_record * ratio)) - 1
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _orderable(dtype):
|
|
92
|
+
if types.is_numeric_dtype(dtype) or types.is_datetime64_any_dtype(dtype):
|
|
93
|
+
return True
|
|
94
|
+
else:
|
|
95
|
+
return False
|
|
File without changes
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
import numpy
|
|
4
|
+
import pandas
|
|
5
|
+
from pandas.api import types
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
logger = logging.getLogger(__name__)
|
|
9
|
+
|
|
10
|
+
FREQUENCY_CNAME="# of records"
|
|
11
|
+
RATIO_CNAME="ratio of records"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def create_frequency_table(series, num_bins=50, minv=None, maxv=None):
|
|
15
|
+
if is_quantitative(series, num_bins):
|
|
16
|
+
return count_frequency_quantitative(series, num_bins, minv, maxv)
|
|
17
|
+
else:
|
|
18
|
+
return count_frequency_categorical(series, num_bins)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def is_quantitative(series, num_bins):
|
|
22
|
+
if types.is_integer_dtype(series.dtype):
|
|
23
|
+
return len(series.value_counts()) >= num_bins
|
|
24
|
+
else:
|
|
25
|
+
return types.is_numeric_dtype(series.dtype) or types.is_datetime64_any_dtype(series.dtype)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def count_frequency_quantitative(series, num_bins, minv, maxv):
|
|
29
|
+
name = series.name or "value"
|
|
30
|
+
bins = create_bins_quantitative(series, num_bins, minv, maxv)
|
|
31
|
+
count_array = get_continuous_value_counts(series.values, bins)
|
|
32
|
+
ratio_array = count_array / count_array.sum()
|
|
33
|
+
return pandas.DataFrame({
|
|
34
|
+
"{}_lb".format(name): [b.lb for b in bins],
|
|
35
|
+
"{}_ub".format(name): [b.ub for b in bins],
|
|
36
|
+
FREQUENCY_CNAME: count_array,
|
|
37
|
+
RATIO_CNAME: ratio_array
|
|
38
|
+
})
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def count_frequency_categorical(series, num_bins):
|
|
42
|
+
name = series.name or "value"
|
|
43
|
+
vcounts = series.value_counts()
|
|
44
|
+
if len(vcounts) > num_bins:
|
|
45
|
+
index = vcounts[:num_bins].index
|
|
46
|
+
vcounts = get_categorical_value_counts(series, index)
|
|
47
|
+
total = vcounts.values.sum()
|
|
48
|
+
return pandas.DataFrame({
|
|
49
|
+
name: vcounts.index,
|
|
50
|
+
FREQUENCY_CNAME: vcounts.values,
|
|
51
|
+
RATIO_CNAME: [v/total for v in vcounts.values]
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def create_bins_quantitative(series, num_bins, minv=None, maxv=None):
|
|
56
|
+
array = series.values
|
|
57
|
+
if numpy.array(array[~numpy.isnan(array)]).size == 0:
|
|
58
|
+
raise BinCreationError("all values are null")
|
|
59
|
+
minv = minv if minv is not None else series.min() # cannot use "or" for the case minv==0
|
|
60
|
+
maxv = maxv if maxv is not None else series.max() # cannot use "or" for the case maxv==0
|
|
61
|
+
if maxv == minv:
|
|
62
|
+
raise BinCreationError("min == max")
|
|
63
|
+
lb_list, ub_list, bin_size = get_bin_config(minv, maxv, num_bins)
|
|
64
|
+
return [QuantitativeBin(lb, ub, False) for lb, ub in zip(lb_list[:-1], ub_list[:-1])] + \
|
|
65
|
+
[QuantitativeBin(lb_list[-1], ub_list[-1], True)]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def create_bins_categorical(series, num_bins):
|
|
69
|
+
vcounts = series.value_counts()
|
|
70
|
+
bins = [CategoricalBin([v]) for v in vcounts[:num_bins].index]
|
|
71
|
+
if len(vcounts) > num_bins:
|
|
72
|
+
bins.append(CategoricalBin([v for v in vcounts[num_bins:].index]))
|
|
73
|
+
return bins
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def get_bin_config(minv, maxv, num_bins):
|
|
77
|
+
bin_size = (maxv - minv) / num_bins
|
|
78
|
+
lower_list = [minv + i * bin_size for i in range(num_bins)]
|
|
79
|
+
upper_list = [minv + (i + 1) * bin_size for i in range(num_bins-1)] + [maxv]
|
|
80
|
+
return lower_list, upper_list, bin_size
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def get_continuous_value_counts(series, bins):
|
|
84
|
+
return numpy.array([len(b.filter(series)) for b in bins])
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def get_categorical_value_counts(series, index=None):
|
|
88
|
+
vcounts = series.value_counts()
|
|
89
|
+
if index is not None:
|
|
90
|
+
other_counts = vcounts[~vcounts.index.isin(index)].values.sum()
|
|
91
|
+
vcounts = vcounts[index]
|
|
92
|
+
vcounts["OTHER"] = other_counts
|
|
93
|
+
return vcounts
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def create_frequency_table_2d(df, column_x, column_y, num_bins_x=50, num_bins_y=50,
|
|
97
|
+
xmin=None, xmax=None, ymin=None, ymax=None):
|
|
98
|
+
is_quantitative_x = is_quantitative(df[column_x], num_bins_x)
|
|
99
|
+
is_quantitative_y = is_quantitative(df[column_y], num_bins_y)
|
|
100
|
+
bins_x = create_bins(df[column_x], num_bins_x, xmin, xmax, is_quantitative_x)
|
|
101
|
+
bins_y = create_bins(df[column_y], num_bins_y, ymin, ymax, is_quantitative_y)
|
|
102
|
+
frequency_list = []
|
|
103
|
+
for bin_x in bins_x:
|
|
104
|
+
for bin_y in bins_y:
|
|
105
|
+
frequency_list.append(
|
|
106
|
+
_count_2d(df, column_x, column_y, bin_x, bin_y)
|
|
107
|
+
)
|
|
108
|
+
count_df = pandas.concat(frequency_list, ignore_index=True)
|
|
109
|
+
count_df[RATIO_CNAME] = count_df[FREQUENCY_CNAME] / count_df[FREQUENCY_CNAME].sum()
|
|
110
|
+
return count_df
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def create_bins(series, num_bins, minv, maxv, is_quantitative):
|
|
114
|
+
if is_quantitative:
|
|
115
|
+
return create_bins_quantitative(series, num_bins, minv, maxv)
|
|
116
|
+
else:
|
|
117
|
+
return create_bins_categorical(series, num_bins)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _count_2d(df, column_x, column_y, bin_x, bin_y):
|
|
121
|
+
return pandas.DataFrame({
|
|
122
|
+
**bin_x.bin_info(column_x),
|
|
123
|
+
**bin_y.bin_info(column_y),
|
|
124
|
+
**{
|
|
125
|
+
FREQUENCY_CNAME: [len(bin_y.filter(bin_x.filter(df, column_x), column_y))]
|
|
126
|
+
}
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class QuantitativeBin(object):
|
|
131
|
+
def __init__(self, lb, ub, is_last_bin):
|
|
132
|
+
self.lb = lb
|
|
133
|
+
self.ub = ub
|
|
134
|
+
self.is_last_bin = is_last_bin
|
|
135
|
+
|
|
136
|
+
def filter(self, df, column_name=""):
|
|
137
|
+
if column_name:
|
|
138
|
+
return df[(df[column_name]>=self.lb)&(df[column_name]<=self.ub)] if self.is_last_bin \
|
|
139
|
+
else df[(df[column_name]>=self.lb)&(df[column_name]<self.ub)]
|
|
140
|
+
else:
|
|
141
|
+
return df[(df>=self.lb)&(df<=self.ub)] if self.is_last_bin \
|
|
142
|
+
else df[(df>=self.lb)&(df<self.ub)]
|
|
143
|
+
|
|
144
|
+
def bin_info(self, column_name=""):
|
|
145
|
+
return {
|
|
146
|
+
"{}_lb".format(column_name): [self.lb],
|
|
147
|
+
"{}_ub".format(column_name): [self.ub]
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class CategoricalBin(object):
|
|
152
|
+
def __init__(self, values):
|
|
153
|
+
self.values = values
|
|
154
|
+
|
|
155
|
+
def filter(self, df, column_name=""):
|
|
156
|
+
return df[df[column_name].isin(self.values)] if column_name else df[df.isin(self.values)]
|
|
157
|
+
|
|
158
|
+
def bin_info(self, column_name=""):
|
|
159
|
+
return {
|
|
160
|
+
column_name: self.values,
|
|
161
|
+
} if len(self.values) == 1 else {column_name: ["OTHERS"]}
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
class BinCreationError(Exception):
|
|
165
|
+
def __init__(self, message):
|
|
166
|
+
self.message = message
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
import altair as alt
|
|
4
|
+
from pandas.api import types
|
|
5
|
+
|
|
6
|
+
from . import binning
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def plot_histogram(values, num_bins=50, normalize=False, minv=None, maxv=None):
|
|
13
|
+
try:
|
|
14
|
+
bin_df = binning.create_frequency_table(values, num_bins, minv, maxv)
|
|
15
|
+
except Exception as e:
|
|
16
|
+
raise e
|
|
17
|
+
return plot_frequency_numeric(bin_df, normalize) if len(bin_df.columns) > 3\
|
|
18
|
+
else plot_frequency_category(bin_df, normalize)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def plot_histogram_for_stats(df, stat_df, num_bins=50, normalize=False):
|
|
22
|
+
for ind, column in enumerate(df.columns):
|
|
23
|
+
minv, maxv = \
|
|
24
|
+
stat_df[stat_df["column_name"] == column][
|
|
25
|
+
["min", "max"]].values.tolist()[0]
|
|
26
|
+
try:
|
|
27
|
+
plot_histogram(df[column], num_bins, normalize, minv, maxv).display()
|
|
28
|
+
except binning.BinCreationError as e:
|
|
29
|
+
logger.info("Histogram for {} was skipped: {}".format(column, e.message))
|
|
30
|
+
pass
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def plot_frequency_numeric(df, normalize, xname=None):
|
|
34
|
+
column_lb = df.columns[0]
|
|
35
|
+
column_ub = df.columns[1]
|
|
36
|
+
xname = xname or column_lb.replace("_lb", "")
|
|
37
|
+
column_y = binning.RATIO_CNAME if normalize else binning.FREQUENCY_CNAME
|
|
38
|
+
y_args = dict(axis=alt.Axis(format="%")) if normalize else {}
|
|
39
|
+
return alt.Chart(df).mark_bar().encode(
|
|
40
|
+
alt.X(column_lb, bin="binned", axis=alt.Axis(title=xname)),
|
|
41
|
+
x2=column_ub,
|
|
42
|
+
y=alt.Y("{}:Q".format(column_y), **y_args),
|
|
43
|
+
tooltip=[column_lb, column_ub, column_y]
|
|
44
|
+
).properties(height=100, width=400, title="Histogram of {}".format(xname)).interactive()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def plot_frequency_category(df, normalize, xname=None):
|
|
48
|
+
x_args = {} if types.is_integer_dtype(df.dtypes.iloc[0]) else dict(sort="-y")
|
|
49
|
+
y_args = dict(axis=alt.Axis(format="%")) if normalize else {}
|
|
50
|
+
column_y = binning.RATIO_CNAME if normalize else binning.FREQUENCY_CNAME
|
|
51
|
+
return alt.Chart(df).mark_bar().encode(
|
|
52
|
+
x=alt.X("{}:N".format(df.columns[0]), **x_args),
|
|
53
|
+
y=alt.Y("{}:Q".format(column_y), **y_args)
|
|
54
|
+
).properties(height=100, width=400, title="Histogram of {}".format(df.columns[0])).interactive()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .main import EDAHubWidget
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import ipywidgets as widgets
|
|
2
|
+
from IPython.display import display, HTML
|
|
3
|
+
|
|
4
|
+
from ..logic.vis import histogram
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class EDAHubWidgetColumnSummary:
|
|
8
|
+
def __init__(self):
|
|
9
|
+
self.output = widgets.Accordion()
|
|
10
|
+
|
|
11
|
+
def update(self, edahub):
|
|
12
|
+
table_names = edahub.get_table_names()
|
|
13
|
+
contents = []
|
|
14
|
+
titles = []
|
|
15
|
+
for table_name in table_names:
|
|
16
|
+
stats = widgets.Output(layout=widgets.Layout(width='100%'))
|
|
17
|
+
histogram = widgets.Output(layout=widgets.Layout(width='100%'))
|
|
18
|
+
select_chart, filter_text = self._get_histogram_selection(table_name, edahub, histogram)
|
|
19
|
+
if select_chart.options:
|
|
20
|
+
select_chart.value = select_chart.options[0]
|
|
21
|
+
with stats:
|
|
22
|
+
display(HTML(self._get_table_html(table_name, edahub.stats_tables[table_name])))
|
|
23
|
+
contents.append(
|
|
24
|
+
widgets.VBox(
|
|
25
|
+
[
|
|
26
|
+
stats,
|
|
27
|
+
widgets.HBox([select_chart, filter_text]),
|
|
28
|
+
histogram
|
|
29
|
+
]
|
|
30
|
+
)
|
|
31
|
+
)
|
|
32
|
+
titles.append(table_name)
|
|
33
|
+
self.output.children = contents
|
|
34
|
+
for i, title in enumerate(titles):
|
|
35
|
+
self.output.set_title(i, title)
|
|
36
|
+
|
|
37
|
+
def _get_table_html(self, table_name, table_df):
|
|
38
|
+
return table_df.to_html()
|
|
39
|
+
|
|
40
|
+
def _get_histogram_selection(self, table_name, edahub, histogram):
|
|
41
|
+
available_columns = list(edahub.histograms[table_name].keys()) if table_name in edahub.histograms \
|
|
42
|
+
else []
|
|
43
|
+
select_chart = widgets.Select(
|
|
44
|
+
options=available_columns,
|
|
45
|
+
description='Select Chart:',
|
|
46
|
+
disabled=False
|
|
47
|
+
)
|
|
48
|
+
filter_text = widgets.Text(
|
|
49
|
+
placeholder='',
|
|
50
|
+
description='Type to filter:',
|
|
51
|
+
disabled=False
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
def update_chart(change):
|
|
55
|
+
column_name = change['new']
|
|
56
|
+
chart = edahub.histograms[table_name].get(column_name)
|
|
57
|
+
if chart:
|
|
58
|
+
with histogram:
|
|
59
|
+
histogram.clear_output()
|
|
60
|
+
display(chart)
|
|
61
|
+
|
|
62
|
+
def filter_options(text):
|
|
63
|
+
filtered_options = [option for option in available_columns if text.lower() in option.lower()]
|
|
64
|
+
select_chart.options = filtered_options
|
|
65
|
+
|
|
66
|
+
def handle_text_change(change):
|
|
67
|
+
filter_options(change['new'])
|
|
68
|
+
|
|
69
|
+
select_chart.observe(update_chart, names='value')
|
|
70
|
+
filter_text.observe(handle_text_change, names='value')
|
|
71
|
+
return select_chart, filter_text
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import pandas
|
|
2
|
+
from sidecar import Sidecar
|
|
3
|
+
import ipywidgets as widgets
|
|
4
|
+
import time
|
|
5
|
+
from IPython.display import display, HTML
|
|
6
|
+
|
|
7
|
+
from .column_summary import EDAHubWidgetColumnSummary
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class EDAHubWidget:
|
|
11
|
+
def __init__(self, edahub):
|
|
12
|
+
self.edahub = edahub
|
|
13
|
+
tab_contents = ["Summary", "Schema & Stats", "Warnings"]
|
|
14
|
+
self.sidecar = Sidecar(title=f'EDAHub({self.edahub.name})', anchor='right')
|
|
15
|
+
self.update_button = widgets.Button(description="Update")
|
|
16
|
+
self.update_button.on_click(self.update)
|
|
17
|
+
self.summary = EDAHubWidgetSummary()
|
|
18
|
+
self.column_summary = EDAHubWidgetColumnSummary()
|
|
19
|
+
self.warnings = EDAHubWidgetWarnings()
|
|
20
|
+
with self.sidecar:
|
|
21
|
+
tab = widgets.Tab()
|
|
22
|
+
tab.children=[
|
|
23
|
+
self.summary.output,
|
|
24
|
+
self.column_summary.output,
|
|
25
|
+
self.warnings.output
|
|
26
|
+
]
|
|
27
|
+
tab.titles = tab_contents
|
|
28
|
+
display(widgets.VBox([self.update_button, tab]))
|
|
29
|
+
|
|
30
|
+
def update(self, _):
|
|
31
|
+
self.summary.update(self.edahub)
|
|
32
|
+
self.column_summary.update(self.edahub)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class EDAHubWidgetSummary:
|
|
36
|
+
def __init__(self):
|
|
37
|
+
self.status_text = widgets.Output(layout=widgets.Layout(width='100%'))
|
|
38
|
+
self.status_table = widgets.Output(layout=widgets.Layout(width='100%'))
|
|
39
|
+
self.output = widgets.VBox(
|
|
40
|
+
[self.status_text, self.status_table]
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
def update(self, edahub):
|
|
44
|
+
with self.status_text:
|
|
45
|
+
text = f"{len(edahub.tables)} tables"
|
|
46
|
+
self.status_text.clear_output()
|
|
47
|
+
display(HTML(f"<p>{text}</p>"))
|
|
48
|
+
|
|
49
|
+
with self.status_table:
|
|
50
|
+
self.status_table.clear_output()
|
|
51
|
+
display(HTML(edahub.status_table.to_html()))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class EDAHubWidgetWarnings:
|
|
57
|
+
def __init__(self):
|
|
58
|
+
self.output = widgets.Accordion(children=[])
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: edahub
|
|
3
|
+
Version: 0.0.0
|
|
4
|
+
Summary: T.B.A.
|
|
5
|
+
Home-page: https://github.com/not-so-fat/edahub
|
|
6
|
+
Author: @not-so-fat
|
|
7
|
+
Author-email: conjurer.not.so.fat@gmail.com
|
|
8
|
+
License: UNKNOWN
|
|
9
|
+
Platform: UNKNOWN
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Requires-Python: >=3.9
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# EDAHA
|
|
17
|
+
|
|
18
|
+
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
setup.py
|
|
3
|
+
edahub/__init__.py
|
|
4
|
+
edahub.egg-info/PKG-INFO
|
|
5
|
+
edahub.egg-info/SOURCES.txt
|
|
6
|
+
edahub.egg-info/dependency_links.txt
|
|
7
|
+
edahub.egg-info/requires.txt
|
|
8
|
+
edahub.egg-info/top_level.txt
|
|
9
|
+
edahub/logic/__init__.py
|
|
10
|
+
edahub/logic/project.py
|
|
11
|
+
edahub/logic/stats_calculator.py
|
|
12
|
+
edahub/logic/vis/__init__.py
|
|
13
|
+
edahub/logic/vis/binning.py
|
|
14
|
+
edahub/logic/vis/histogram.py
|
|
15
|
+
edahub/widgets/__init__.py
|
|
16
|
+
edahub/widgets/column_summary.py
|
|
17
|
+
edahub/widgets/main.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
edahub
|
edahub-0.0.0/setup.cfg
ADDED
edahub-0.0.0/setup.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import setuptools
|
|
2
|
+
import pkg_resources
|
|
3
|
+
|
|
4
|
+
with open("requirements.txt", "r") as f:
|
|
5
|
+
requirements = [str(req) for req in pkg_resources.parse_requirements(f)]
|
|
6
|
+
|
|
7
|
+
with open("README.md", "r") as fh:
|
|
8
|
+
long_description = fh.read()
|
|
9
|
+
|
|
10
|
+
setuptools.setup(
|
|
11
|
+
name="edahub",
|
|
12
|
+
version="0.0.0",
|
|
13
|
+
author="@not-so-fat",
|
|
14
|
+
author_email="conjurer.not.so.fat@gmail.com",
|
|
15
|
+
description="T.B.A.",
|
|
16
|
+
long_description=long_description,
|
|
17
|
+
long_description_content_type="text/markdown",
|
|
18
|
+
url="https://github.com/not-so-fat/edahub",
|
|
19
|
+
install_requires=requirements,
|
|
20
|
+
packages=setuptools.find_packages(),
|
|
21
|
+
classifiers=[
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"License :: OSI Approved :: MIT License",
|
|
24
|
+
"Operating System :: OS Independent",
|
|
25
|
+
],
|
|
26
|
+
python_requires='>=3.9',
|
|
27
|
+
)
|
|
28
|
+
|