dukit 0.2.7__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dukit-0.2.7 → dukit-0.3.1}/PKG-INFO +3 -2
- dukit-0.3.1/pyproject.toml +48 -0
- dukit-0.2.7/pyproject.toml → dukit-0.3.1/pyproject.toml.orig +4 -4
- {dukit-0.2.7 → dukit-0.3.1}/src/dukit/_test_utils.py +8 -8
- {dukit-0.2.7 → dukit-0.3.1}/src/dukit/diffing.py +8 -8
- {dukit-0.2.7 → dukit-0.3.1}/src/dukit/os.py +1 -1
- {dukit-0.2.7 → dukit-0.3.1}/src/dukit/pandas.py +29 -31
- {dukit-0.2.7 → dukit-0.3.1}/src/dukit/qlang/symbols.py +2 -2
- {dukit-0.2.7 → dukit-0.3.1}/README.md +0 -0
- {dukit-0.2.7 → dukit-0.3.1}/src/dukit/__init__.py +0 -0
- {dukit-0.2.7 → dukit-0.3.1}/src/dukit/excel.py +0 -0
- {dukit-0.2.7 → dukit-0.3.1}/src/dukit/interfaces.py +0 -0
- {dukit-0.2.7 → dukit-0.3.1}/src/dukit/py.typed +0 -0
- {dukit-0.2.7 → dukit-0.3.1}/src/dukit/qlang/__init__.py +0 -0
- {dukit-0.2.7 → dukit-0.3.1}/src/dukit/qlang/engine.py +0 -0
- {dukit-0.2.7 → dukit-0.3.1}/src/dukit/typing.py +0 -0
- {dukit-0.2.7 → dukit-0.3.1}/src/dukit/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dukit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: dukit - data utilities kit for exploring, modifying and visualizing small but messy data using pandas
|
|
5
5
|
Keywords: pandas,query,utility,data,diff
|
|
6
6
|
Author: MartinVoelkl94
|
|
@@ -10,7 +10,8 @@ Classifier: Programming Language :: Python
|
|
|
10
10
|
Classifier: Programming Language :: Python :: 3
|
|
11
11
|
Requires-Dist: ipython>=9.15.0
|
|
12
12
|
Requires-Dist: jinja2>=3.1.6
|
|
13
|
-
Requires-Dist:
|
|
13
|
+
Requires-Dist: openpyxl>=3.1.5
|
|
14
|
+
Requires-Dist: pandas>=3.1.0
|
|
14
15
|
Requires-Python: >=3.14
|
|
15
16
|
Project-URL: Repository, https://github.com/MartinVoelkl94/dukit
|
|
16
17
|
Description-Content-Type: text/markdown
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "dukit"
|
|
3
|
+
version = "0.3.1"
|
|
4
|
+
description = "dukit - data utilities kit for exploring, modifying and visualizing small but messy data using pandas"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
requires-python = ">=3.14"
|
|
8
|
+
classifiers = [
|
|
9
|
+
"Programming Language :: Python",
|
|
10
|
+
"Programming Language :: Python :: 3",
|
|
11
|
+
]
|
|
12
|
+
keywords = [
|
|
13
|
+
"pandas",
|
|
14
|
+
"query",
|
|
15
|
+
"utility",
|
|
16
|
+
"data",
|
|
17
|
+
"diff",
|
|
18
|
+
]
|
|
19
|
+
dependencies = [
|
|
20
|
+
"ipython>=9.15.0",
|
|
21
|
+
"jinja2>=3.1.6",
|
|
22
|
+
"openpyxl>=3.1.5",
|
|
23
|
+
"pandas>=3.1.0",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[[project.authors]]
|
|
27
|
+
name = "MartinVoelkl94"
|
|
28
|
+
email = "martinv16248@gmail.com"
|
|
29
|
+
|
|
30
|
+
[project.urls]
|
|
31
|
+
Repository = "https://github.com/MartinVoelkl94/dukit"
|
|
32
|
+
|
|
33
|
+
[build-system]
|
|
34
|
+
requires = ["uv_build>=0.10.9,<0.11.0"]
|
|
35
|
+
build-backend = "uv_build"
|
|
36
|
+
|
|
37
|
+
[dependency-groups]
|
|
38
|
+
dev = [
|
|
39
|
+
"faker>=40.28.1",
|
|
40
|
+
"flake8>=7.3.0",
|
|
41
|
+
"ipykernel>=7.3.0",
|
|
42
|
+
"ipywidgets>=8.1.8",
|
|
43
|
+
"pip>=26.1.2",
|
|
44
|
+
"pylance>=8.0.0",
|
|
45
|
+
"pytest>=9.2.0",
|
|
46
|
+
"pytest-cov>=7.1.0",
|
|
47
|
+
"viztracer>=1.1.1",
|
|
48
|
+
]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dukit"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.1"
|
|
4
4
|
description = "dukit - data utilities kit for exploring, modifying and visualizing small but messy data using pandas"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [{ name = "MartinVoelkl94", email = "martinv16248@gmail.com" }]
|
|
@@ -20,7 +20,8 @@ keywords = [
|
|
|
20
20
|
dependencies = [
|
|
21
21
|
"ipython>=9.15.0",
|
|
22
22
|
"jinja2>=3.1.6",
|
|
23
|
-
"
|
|
23
|
+
"openpyxl>=3.1.5",
|
|
24
|
+
"pandas>=3.1.0",
|
|
24
25
|
]
|
|
25
26
|
|
|
26
27
|
|
|
@@ -39,10 +40,9 @@ dev = [
|
|
|
39
40
|
"flake8>=7.3.0",
|
|
40
41
|
"ipykernel>=7.3.0",
|
|
41
42
|
"ipywidgets>=8.1.8",
|
|
42
|
-
"openpyxl>=3.1.5",
|
|
43
43
|
"pip>=26.1.2",
|
|
44
44
|
"pylance>=8.0.0",
|
|
45
|
-
"pytest>=9.
|
|
45
|
+
"pytest>=9.2.0",
|
|
46
46
|
"pytest-cov>=7.1.0",
|
|
47
47
|
"viztracer>=1.1.1",
|
|
48
48
|
]
|
|
@@ -63,7 +63,7 @@ def _get_expected_new():
|
|
|
63
63
|
expected.loc['x2', 'a'] = 1
|
|
64
64
|
expected.loc['z', 'a'] = pd.NA
|
|
65
65
|
|
|
66
|
-
expected.reset_index(drop=True
|
|
66
|
+
expected = expected.reset_index(drop=True)
|
|
67
67
|
|
|
68
68
|
return expected.convert_dtypes()
|
|
69
69
|
|
|
@@ -107,7 +107,7 @@ def _get_expected_newplus():
|
|
|
107
107
|
expected.loc['z', 'a *old'] = 3
|
|
108
108
|
|
|
109
109
|
expected['b *old'] = expected['b *old'].astype('Int64')
|
|
110
|
-
expected.reset_index(drop=True
|
|
110
|
+
expected = expected.reset_index(drop=True)
|
|
111
111
|
|
|
112
112
|
return expected.convert_dtypes()
|
|
113
113
|
|
|
@@ -142,7 +142,7 @@ def _get_expected_old():
|
|
|
142
142
|
expected.loc['y', 'c'] = 2
|
|
143
143
|
expected.loc['z', 'c'] = 3
|
|
144
144
|
|
|
145
|
-
expected.reset_index(drop=True
|
|
145
|
+
expected = expected.reset_index(drop=True)
|
|
146
146
|
|
|
147
147
|
return expected.convert_dtypes()
|
|
148
148
|
|
|
@@ -187,7 +187,7 @@ def _get_expected_mix():
|
|
|
187
187
|
expected.loc['z', 'c'] = 3
|
|
188
188
|
expected.loc['x', 'c'] = 1
|
|
189
189
|
|
|
190
|
-
expected.reset_index(drop=True
|
|
190
|
+
expected = expected.reset_index(drop=True)
|
|
191
191
|
|
|
192
192
|
return expected.convert_dtypes()
|
|
193
193
|
|
|
@@ -218,7 +218,7 @@ def _get_expected_style_new():
|
|
|
218
218
|
expected.loc['x2', 'a'] = f'background-color: {GREEN};'
|
|
219
219
|
expected.loc['z', 'a'] = f'background-color: {RED_LIGHT};'
|
|
220
220
|
|
|
221
|
-
expected.reset_index(drop=True
|
|
221
|
+
expected = expected.reset_index(drop=True)
|
|
222
222
|
|
|
223
223
|
return expected.convert_dtypes()
|
|
224
224
|
|
|
@@ -256,7 +256,7 @@ def _get_expected_style_newplus():
|
|
|
256
256
|
expected.loc['x2', 'a *old'] = f'font-style: italic;background-color: {GREEN};'
|
|
257
257
|
expected.loc['z', 'a *old'] = 'font-style: italic;'
|
|
258
258
|
|
|
259
|
-
expected.reset_index(drop=True
|
|
259
|
+
expected = expected.reset_index(drop=True)
|
|
260
260
|
|
|
261
261
|
return expected.convert_dtypes()
|
|
262
262
|
|
|
@@ -286,7 +286,7 @@ def _get_expected_style_old():
|
|
|
286
286
|
expected.loc['y', 'c'] = f'background-color: {RED};'
|
|
287
287
|
expected.loc['z', 'c'] = f'background-color: {RED};'
|
|
288
288
|
|
|
289
|
-
expected.reset_index(drop=True
|
|
289
|
+
expected = expected.reset_index(drop=True)
|
|
290
290
|
|
|
291
291
|
return expected.convert_dtypes()
|
|
292
292
|
|
|
@@ -325,6 +325,6 @@ def _get_expected_style_mix():
|
|
|
325
325
|
expected.loc['z', 'c'] = f'background-color: {RED};'
|
|
326
326
|
|
|
327
327
|
|
|
328
|
-
expected.reset_index(drop=True
|
|
328
|
+
expected = expected.reset_index(drop=True)
|
|
329
329
|
|
|
330
330
|
return expected.convert_dtypes()
|
|
@@ -1058,15 +1058,15 @@ def _set_uid(d: Diff, uid: typing.Any) -> Diff:
|
|
|
1058
1058
|
)
|
|
1059
1059
|
d.old.index = d.old[uid]
|
|
1060
1060
|
d.new.index = d.new[uid]
|
|
1061
|
-
d.old.drop(columns=uid
|
|
1062
|
-
d.new.drop(columns=uid
|
|
1061
|
+
d.old = d.old.drop(columns=uid)
|
|
1062
|
+
d.new = d.new.drop(columns=uid)
|
|
1063
1063
|
d.uid = uid
|
|
1064
1064
|
|
|
1065
1065
|
elif uid in d.old.columns and uid in d.new.columns:
|
|
1066
1066
|
d.old.index = d.old[uid]
|
|
1067
1067
|
d.new.index = d.new[uid]
|
|
1068
|
-
d.old.drop(columns=uid
|
|
1069
|
-
d.new.drop(columns=uid
|
|
1068
|
+
d.old = d.old.drop(columns=uid)
|
|
1069
|
+
d.new = d.new.drop(columns=uid)
|
|
1070
1070
|
d.uid = uid
|
|
1071
1071
|
|
|
1072
1072
|
else:
|
|
@@ -1295,7 +1295,7 @@ def _create_meta_cols(d: Diff) -> Diff:
|
|
|
1295
1295
|
|
|
1296
1296
|
df_meta = values.loc[:, col_mapping.keys()].copy()
|
|
1297
1297
|
df_meta.loc[:, :] = pd.NA
|
|
1298
|
-
df_meta.rename(columns=col_mapping
|
|
1298
|
+
df_meta = df_meta.rename(columns=col_mapping)
|
|
1299
1299
|
|
|
1300
1300
|
data = {
|
|
1301
1301
|
colname: ['font-style: italic;'] * len(values.index)
|
|
@@ -1577,8 +1577,8 @@ def _reset_index(d: Diff) -> Diff:
|
|
|
1577
1577
|
|
|
1578
1578
|
index_diff = d._values.index.difference(d._style.index)
|
|
1579
1579
|
if index_diff.empty:
|
|
1580
|
-
d._values.reset_index(drop=True
|
|
1581
|
-
d._style.reset_index(drop=True
|
|
1580
|
+
d._values = d._values.reset_index(drop=True)
|
|
1581
|
+
d._style = d._style.reset_index(drop=True)
|
|
1582
1582
|
else:
|
|
1583
1583
|
msg = (
|
|
1584
1584
|
'ERROR: index of values and style dfs do not match.'
|
|
@@ -1602,7 +1602,7 @@ def _apply_style(d: Diff) -> Diff:
|
|
|
1602
1602
|
)
|
|
1603
1603
|
log(msg, context, d.verbosity)
|
|
1604
1604
|
|
|
1605
|
-
d._style.fillna(''
|
|
1605
|
+
d._style = d._style.fillna('')
|
|
1606
1606
|
d.result = (
|
|
1607
1607
|
d._values
|
|
1608
1608
|
.style
|
|
@@ -442,7 +442,7 @@ def compose(
|
|
|
442
442
|
|
|
443
443
|
col_new = df[cols].apply(lambda row: template.format(*row), axis='columns')
|
|
444
444
|
df.insert(pos, col, col_new)
|
|
445
|
-
df.drop(columns=cols
|
|
445
|
+
df = df.drop(columns=cols)
|
|
446
446
|
|
|
447
447
|
return df
|
|
448
448
|
|
|
@@ -550,7 +550,7 @@ def stack(
|
|
|
550
550
|
_stack_check_uid_unique1(df, uid, verbosity)
|
|
551
551
|
|
|
552
552
|
uid_old = ensure_unique_string(uid, df.columns)
|
|
553
|
-
df.set_index(uid
|
|
553
|
+
df = df.set_index(uid)
|
|
554
554
|
df.index.name = uid_old
|
|
555
555
|
cols_keep = list_(cols_keep)
|
|
556
556
|
cols_stack = [col for col in df.columns if col not in cols_keep]
|
|
@@ -577,32 +577,30 @@ def stack(
|
|
|
577
577
|
_stack_check_successfull_match(not_matched, verbosity)
|
|
578
578
|
|
|
579
579
|
for subid_val in subdfs.keys():
|
|
580
|
-
subdf = subdfs[subid_val]
|
|
581
580
|
if dropna:
|
|
582
|
-
|
|
583
|
-
inplace=True,
|
|
584
|
-
how='all',
|
|
585
|
-
)
|
|
581
|
+
subdfs[subid_val] = subdfs[subid_val].dropna(how='all')
|
|
586
582
|
uid_vals = [
|
|
587
583
|
uid_template.format(uid=uid_old, subid=subid_val)
|
|
588
|
-
for uid_old in
|
|
584
|
+
for uid_old in subdfs[subid_val].index
|
|
589
585
|
]
|
|
590
|
-
|
|
586
|
+
subdfs[subid_val].insert(0, uid, uid_vals)
|
|
591
587
|
|
|
588
|
+
df_keep = df[cols_keep].reset_index()
|
|
592
589
|
df_new = pd.concat(list(subdfs.values()), axis='index')
|
|
590
|
+
df_new = df_new.reset_index()
|
|
593
591
|
df_new = pd.merge(
|
|
594
|
-
|
|
595
|
-
df_new
|
|
592
|
+
df_keep,
|
|
593
|
+
df_new,
|
|
596
594
|
on=uid_old,
|
|
597
595
|
how='right',
|
|
598
596
|
)
|
|
599
|
-
df_new.drop(columns=[uid_old]
|
|
597
|
+
df_new = df_new.drop(columns=[uid_old])
|
|
600
598
|
|
|
601
599
|
_stack_check_missed_values(df_new, uid, cols_keep, non_na_vals, verbosity)
|
|
602
600
|
|
|
603
601
|
if sort:
|
|
604
|
-
df_new.sort_values(by=uid
|
|
605
|
-
df_new.reset_index(drop=True
|
|
602
|
+
df_new = df_new.sort_values(by=uid)
|
|
603
|
+
df_new = df_new.reset_index(drop=True)
|
|
606
604
|
|
|
607
605
|
_stack_check_uid_unique2(uid, verbosity, df_new)
|
|
608
606
|
|
|
@@ -678,9 +676,9 @@ def unstack(
|
|
|
678
676
|
_unstack_check_successfull_extraction(df, uid, ids, expected_cols, verbosity)
|
|
679
677
|
|
|
680
678
|
|
|
681
|
-
ids.set_index('uid'
|
|
679
|
+
ids = ids.set_index('uid')
|
|
682
680
|
df[uid] = ids.index
|
|
683
|
-
df.set_index(uid
|
|
681
|
+
df = df.set_index(uid)
|
|
684
682
|
subid_col = ids['subid']
|
|
685
683
|
subid_vals = subid_col.unique()
|
|
686
684
|
cols_keep = list_(cols_keep)
|
|
@@ -720,9 +718,9 @@ def unstack(
|
|
|
720
718
|
how='left'
|
|
721
719
|
)
|
|
722
720
|
|
|
723
|
-
_unstack_check_missed_values(uid, cols_keep,
|
|
721
|
+
_unstack_check_missed_values(df_new, uid, cols_keep, non_na_vals, verbosity)
|
|
724
722
|
|
|
725
|
-
df_new.reset_index(
|
|
723
|
+
df_new = df_new.reset_index()
|
|
726
724
|
|
|
727
725
|
return df_new
|
|
728
726
|
|
|
@@ -762,22 +760,22 @@ def _unstack_check_successfull_extraction(df, uid, ids, expected_cols, verbosity
|
|
|
762
760
|
log(msg, 'dk.pandas.unstack', verbosity)
|
|
763
761
|
|
|
764
762
|
|
|
765
|
-
def _unstack_check_missed_values(uid, cols_keep,
|
|
763
|
+
def _unstack_check_missed_values(df_new, uid, cols_keep, non_na_vals, verbosity):
|
|
766
764
|
cols_stacked = [
|
|
767
765
|
col for col
|
|
768
766
|
in df_new.columns
|
|
769
767
|
if col not in cols_keep + [uid]
|
|
770
768
|
]
|
|
771
|
-
|
|
769
|
+
non_na_vals_new = int(
|
|
772
770
|
df_new[cols_stacked]
|
|
773
771
|
.notna()
|
|
774
772
|
.sum()
|
|
775
773
|
.sum()
|
|
776
774
|
)
|
|
777
|
-
if
|
|
775
|
+
if non_na_vals != non_na_vals_new:
|
|
778
776
|
msg = (
|
|
779
777
|
'WARNING: Non-NA value mismatch: '
|
|
780
|
-
f'before={
|
|
778
|
+
f'before={non_na_vals}, after={non_na_vals_new}'
|
|
781
779
|
)
|
|
782
780
|
log(msg, 'dk.pandas.unstack', verbosity)
|
|
783
781
|
|
|
@@ -809,8 +807,8 @@ def transpose(
|
|
|
809
807
|
df = df.T
|
|
810
808
|
df.columns = cols_new
|
|
811
809
|
df.insert(0, header, cols_old)
|
|
812
|
-
df.drop(header, axis=0
|
|
813
|
-
df.reset_index(drop=True
|
|
810
|
+
df = df.drop(header, axis=0)
|
|
811
|
+
df = df.reset_index(drop=True)
|
|
814
812
|
|
|
815
813
|
return df
|
|
816
814
|
|
|
@@ -857,10 +855,10 @@ def date_delta(
|
|
|
857
855
|
|
|
858
856
|
for col in df_dates.columns:
|
|
859
857
|
if df_dates[col].isna().all():
|
|
860
|
-
df_dates.drop(columns=col
|
|
858
|
+
df_dates = df_dates.drop(columns=col)
|
|
861
859
|
else:
|
|
862
860
|
cols_dates.append(col)
|
|
863
|
-
df.drop(columns=col
|
|
861
|
+
df = df.drop(columns=col)
|
|
864
862
|
|
|
865
863
|
df = pd.concat([df, df_dates], axis=1)[cols_all]
|
|
866
864
|
|
|
@@ -942,7 +940,7 @@ def date_table(
|
|
|
942
940
|
msg = f'WARNING: no dates found in reference col "{reference_col}"'
|
|
943
941
|
log(msg, 'dk.date_delta', verbosity)
|
|
944
942
|
else:
|
|
945
|
-
df.drop(columns=col
|
|
943
|
+
df = df.drop(columns=col)
|
|
946
944
|
|
|
947
945
|
reference = df[reference_col].apply(date_).apply(_fix_nas)
|
|
948
946
|
|
|
@@ -1088,7 +1086,7 @@ def split_col(
|
|
|
1088
1086
|
pos = df.columns.get_loc(col)
|
|
1089
1087
|
splits = df[col].str.partition(sep)
|
|
1090
1088
|
|
|
1091
|
-
df.drop(columns=[col]
|
|
1089
|
+
df = df.drop(columns=[col])
|
|
1092
1090
|
if new1 is None:
|
|
1093
1091
|
new1 = col
|
|
1094
1092
|
elif new1 in df.columns:
|
|
@@ -1123,7 +1121,7 @@ def collapse(
|
|
|
1123
1121
|
df = df.map(lambda x: _to_lines(x, template_item))
|
|
1124
1122
|
df.columns = [template_col.format(colname=col) for col in df.columns]
|
|
1125
1123
|
df.insert(0, on, df.index)
|
|
1126
|
-
df.reset_index(drop=True
|
|
1124
|
+
df = df.reset_index(drop=True)
|
|
1127
1125
|
return df
|
|
1128
1126
|
|
|
1129
1127
|
|
|
@@ -1253,7 +1251,7 @@ def flatten(
|
|
|
1253
1251
|
|
|
1254
1252
|
df = df[cols_new]
|
|
1255
1253
|
df.insert(0, on, df.index)
|
|
1256
|
-
df.reset_index(drop=True
|
|
1254
|
+
df = df.reset_index(drop=True)
|
|
1257
1255
|
|
|
1258
1256
|
return df
|
|
1259
1257
|
|
|
@@ -1311,7 +1309,7 @@ def stagger(
|
|
|
1311
1309
|
col_new.name = template.format(counter=i + 1, colname=col)
|
|
1312
1310
|
df_new = pd.concat([df_new, col_new], axis=1)
|
|
1313
1311
|
|
|
1314
|
-
df_new.reset_index(drop=True
|
|
1312
|
+
df_new = df_new.reset_index(drop=True)
|
|
1315
1313
|
|
|
1316
1314
|
return df_new
|
|
1317
1315
|
|
|
@@ -2879,7 +2879,7 @@ class GetIsNA(Symbol):
|
|
|
2879
2879
|
if 'strict' in self.flags:
|
|
2880
2880
|
mask = series.isna()
|
|
2881
2881
|
else:
|
|
2882
|
-
mask = series.apply(lambda x: na_(x, errors='X'))
|
|
2882
|
+
mask = series.apply(lambda x: na_(x, errors='X')).isna()
|
|
2883
2883
|
return mask
|
|
2884
2884
|
|
|
2885
2885
|
|
|
@@ -5111,7 +5111,7 @@ class SetToYN(Symbol):
|
|
|
5111
5111
|
args: list,
|
|
5112
5112
|
q: Query,
|
|
5113
5113
|
) -> pd.Series:
|
|
5114
|
-
series_new = series[mask].apply(yn_).
|
|
5114
|
+
series_new = series[mask].apply(yn_).astype('string')
|
|
5115
5115
|
if mask.all():
|
|
5116
5116
|
series = series_new
|
|
5117
5117
|
else:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|