dukit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dukit-0.1.0/PKG-INFO +113 -0
- dukit-0.1.0/README.md +101 -0
- dukit-0.1.0/pyproject.toml +32 -0
- dukit-0.1.0/src/dukit/__init__.py +108 -0
- dukit-0.1.0/src/dukit/_test_utils.py +374 -0
- dukit-0.1.0/src/dukit/diffing.py +1461 -0
- dukit-0.1.0/src/dukit/excel.py +134 -0
- dukit-0.1.0/src/dukit/interfaces.py +160 -0
- dukit-0.1.0/src/dukit/os.py +280 -0
- dukit-0.1.0/src/dukit/pandas.py +563 -0
- dukit-0.1.0/src/dukit/py.typed +0 -0
- dukit-0.1.0/src/dukit/qlang/__init__.py +61 -0
- dukit-0.1.0/src/dukit/qlang/engine.py +447 -0
- dukit-0.1.0/src/dukit/qlang/symbols.py +6867 -0
- dukit-0.1.0/src/dukit/typing.py +683 -0
- dukit-0.1.0/src/dukit/utils.py +288 -0
dukit-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: dukit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: dukit - data utilities kit for exploring, modifying and visualizing small but messy data using pandas
|
|
5
|
+
Author: MartinVoelkl94
|
|
6
|
+
Author-email: MartinVoelkl94 <martinv16248@gmail.com>
|
|
7
|
+
Requires-Dist: ipython>=9.15.0
|
|
8
|
+
Requires-Dist: jinja2>=3.1.6
|
|
9
|
+
Requires-Dist: pandas>=3.0.3
|
|
10
|
+
Requires-Python: >=3.14
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
|
|
13
|
+
# dukit - data utilities kit
|
|
14
|
+
|
|
15
|
+
Provides utilities focused on data exploration, modification and visualization of small but messy datasets in pandas.
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# qlang
|
|
20
|
+
|
|
21
|
+
A small query language.
|
|
22
|
+
It is implemented as a pandas [accessor extension](https://pandas.pydata.org/docs/development/extending.html), meaning it can be called from a dataframe without any further preparation by calling df.dk.qs() with a valid text string.
|
|
23
|
+
|
|
24
|
+
example:
|
|
25
|
+
```python
|
|
26
|
+
from dukit import get_df
|
|
27
|
+
|
|
28
|
+
df = get_df()
|
|
29
|
+
|
|
30
|
+
#show all patients with ID > 20000
|
|
31
|
+
df.dk.qs(r'id >20000')
|
|
32
|
+
|
|
33
|
+
#select all patients whose name contains "john"
|
|
34
|
+
df.dk.qs(r'name ?john')
|
|
35
|
+
|
|
36
|
+
#select all patients whose name contains "j" and "a"
|
|
37
|
+
df.dk.qs(r'name ?j &&?a')
|
|
38
|
+
|
|
39
|
+
#select all patients whose name contains "j" or "a"
|
|
40
|
+
df.dk.qs(r'name ?j //?a')
|
|
41
|
+
|
|
42
|
+
#select ages between 18 and 80 and highlight them green
|
|
43
|
+
df.dk.qs(r'age >18 &&<80 .color(green)')
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Take a look at "interactive_demo.ipynb" in the github repo for a quick syntax introduction and more examples.
|
|
47
|
+
|
|
48
|
+
The query language allows for arbitrary code execution via eval(), please be aware of the risks.
|
|
49
|
+
|
|
50
|
+
Currently published on [testpypi](https://test.pypi.org/project/dukit/).
|
|
51
|
+
|
|
52
|
+
<br>
|
|
53
|
+
<br>
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
# other utilities
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
## logging with dukit.log()
|
|
62
|
+
|
|
63
|
+
A small logger to be used in notebooks or the REPL. Makes it easier to keep track of outputs in large notebooks by providing color coded output. Does not log to file, but instead to a dataframe which can then be viewed at the end of the notebook.
|
|
64
|
+
<br>
|
|
65
|
+
<br>
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
## dukit.diff()
|
|
69
|
+
|
|
70
|
+
creates colored diff output for two dataframes, see interactive_demo.ipynb for examples. Works with pandas dataframes, csv files, excel files and excel files with multiple sheets.
|
|
71
|
+
<br>
|
|
72
|
+
<br>
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
## "bashlike" wrappers
|
|
76
|
+
|
|
77
|
+
While python has functions to achieve similar results as common bash commands, they are often more verbose, less intuitive if you are already used to the bash names and spread out over different modules and different namespaces in those modules (os, os.path, shutil, sys, ...).
|
|
78
|
+
These wrappers use the same names as the bash commands and offer some additional functionality.
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
available wrappers:
|
|
82
|
+
- dukit.ls()
|
|
83
|
+
- dukit.lsr()
|
|
84
|
+
- dukit.pwd()
|
|
85
|
+
- dukit.cd()
|
|
86
|
+
- dukit.cp()
|
|
87
|
+
- dukit.mkdir()
|
|
88
|
+
- dukit.isdir()
|
|
89
|
+
- dukit.isfile()
|
|
90
|
+
- dukit.ispath()
|
|
91
|
+
<br>
|
|
92
|
+
<br>
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
## type conversion
|
|
96
|
+
|
|
97
|
+
Mostly wrappers for pandas functions but with some additional functionality and generally more lenient handling of edge cases.
|
|
98
|
+
|
|
99
|
+
available functions:
|
|
100
|
+
- dukit.int()
|
|
101
|
+
- dukit.float()
|
|
102
|
+
- dukit.num()
|
|
103
|
+
- dukit.bool()
|
|
104
|
+
- dukit.date()
|
|
105
|
+
- dukit.datetime()
|
|
106
|
+
- dukit.na()
|
|
107
|
+
- dukit.nk()
|
|
108
|
+
- dukit.yn()
|
|
109
|
+
- dukit.type()
|
|
110
|
+
- dukit.convert()
|
|
111
|
+
<br>
|
|
112
|
+
<br>
|
|
113
|
+
|
dukit-0.1.0/README.md
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# dukit - data utilities kit
|
|
2
|
+
|
|
3
|
+
Provides utilities focused on data exploration, modification and visualization of small but messy datasets in pandas.
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
# qlang
|
|
8
|
+
|
|
9
|
+
A small query language.
|
|
10
|
+
It is implemented as a pandas [accessor extension](https://pandas.pydata.org/docs/development/extending.html), meaning it can be called from a dataframe without any further preparation by calling df.dk.qs() with a valid text string.
|
|
11
|
+
|
|
12
|
+
example:
|
|
13
|
+
```python
|
|
14
|
+
from dukit import get_df
|
|
15
|
+
|
|
16
|
+
df = get_df()
|
|
17
|
+
|
|
18
|
+
#show all patients with ID > 20000
|
|
19
|
+
df.dk.qs(r'id >20000')
|
|
20
|
+
|
|
21
|
+
#select all patients whose name contains "john"
|
|
22
|
+
df.dk.qs(r'name ?john')
|
|
23
|
+
|
|
24
|
+
#select all patients whose name contains "j" and "a"
|
|
25
|
+
df.dk.qs(r'name ?j &&?a')
|
|
26
|
+
|
|
27
|
+
#select all patients whose name contains "j" or "a"
|
|
28
|
+
df.dk.qs(r'name ?j //?a')
|
|
29
|
+
|
|
30
|
+
#select ages between 18 and 80 and highlight them green
|
|
31
|
+
df.dk.qs(r'age >18 &&<80 .color(green)')
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Take a look at "interactive_demo.ipynb" in the github repo for a quick syntax introduction and more examples.
|
|
35
|
+
|
|
36
|
+
The query language allows for arbitrary code execution via eval(), please be aware of the risks.
|
|
37
|
+
|
|
38
|
+
Currently published on [testpypi](https://test.pypi.org/project/dukit/).
|
|
39
|
+
|
|
40
|
+
<br>
|
|
41
|
+
<br>
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
# other utilities
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
## logging with dukit.log()
|
|
50
|
+
|
|
51
|
+
A small logger to be used in notebooks or the REPL. Makes it easier to keep track of outputs in large notebooks by providing color coded output. Does not log to file, but instead to a dataframe which can then be viewed at the end of the notebook.
|
|
52
|
+
<br>
|
|
53
|
+
<br>
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
## dukit.diff()
|
|
57
|
+
|
|
58
|
+
creates colored diff output for two dataframes, see interactive_demo.ipynb for examples. Works with pandas dataframes, csv files, excel files and excel files with multiple sheets.
|
|
59
|
+
<br>
|
|
60
|
+
<br>
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
## "bashlike" wrappers
|
|
64
|
+
|
|
65
|
+
While python has functions to achieve similar results as common bash commands, they are often more verbose, less intuitive if you are already used to the bash names and spread out over different modules and different namespaces in those modules (os, os.path, shutil, sys, ...).
|
|
66
|
+
These wrappers use the same names as the bash commands and offer some additional functionality.
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
available wrappers:
|
|
70
|
+
- dukit.ls()
|
|
71
|
+
- dukit.lsr()
|
|
72
|
+
- dukit.pwd()
|
|
73
|
+
- dukit.cd()
|
|
74
|
+
- dukit.cp()
|
|
75
|
+
- dukit.mkdir()
|
|
76
|
+
- dukit.isdir()
|
|
77
|
+
- dukit.isfile()
|
|
78
|
+
- dukit.ispath()
|
|
79
|
+
<br>
|
|
80
|
+
<br>
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
## type conversion
|
|
84
|
+
|
|
85
|
+
Mostly wrappers for pandas functions but with some additional functionality and generally more lenient handling of edge cases.
|
|
86
|
+
|
|
87
|
+
available functions:
|
|
88
|
+
- dukit.int()
|
|
89
|
+
- dukit.float()
|
|
90
|
+
- dukit.num()
|
|
91
|
+
- dukit.bool()
|
|
92
|
+
- dukit.date()
|
|
93
|
+
- dukit.datetime()
|
|
94
|
+
- dukit.na()
|
|
95
|
+
- dukit.nk()
|
|
96
|
+
- dukit.yn()
|
|
97
|
+
- dukit.type()
|
|
98
|
+
- dukit.convert()
|
|
99
|
+
<br>
|
|
100
|
+
<br>
|
|
101
|
+
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "dukit"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "dukit - data utilities kit for exploring, modifying and visualizing small but messy data using pandas"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
authors = [
|
|
7
|
+
{ name = "MartinVoelkl94", email = "martinv16248@gmail.com" }
|
|
8
|
+
]
|
|
9
|
+
requires-python = ">=3.14"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"ipython>=9.15.0",
|
|
12
|
+
"jinja2>=3.1.6",
|
|
13
|
+
"pandas>=3.0.3",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
[build-system]
|
|
17
|
+
requires = ["uv_build>=0.10.9,<0.11.0"]
|
|
18
|
+
build-backend = "uv_build"
|
|
19
|
+
|
|
20
|
+
[dependency-groups]
|
|
21
|
+
dev = [
|
|
22
|
+
"faker>=40.28.1",
|
|
23
|
+
"flake8>=7.3.0",
|
|
24
|
+
"ipykernel>=7.3.0",
|
|
25
|
+
"ipywidgets>=8.1.8",
|
|
26
|
+
"openpyxl>=3.1.5",
|
|
27
|
+
"pip>=26.1.2",
|
|
28
|
+
"pylance>=8.0.0",
|
|
29
|
+
"pytest>=9.1.1",
|
|
30
|
+
"pytest-cov>=7.1.0",
|
|
31
|
+
"viztracer>=1.1.1",
|
|
32
|
+
]
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
|
|
2
|
+
from .utils import (
|
|
3
|
+
log,
|
|
4
|
+
)
|
|
5
|
+
from .diffing import (
|
|
6
|
+
diff,
|
|
7
|
+
rediff,
|
|
8
|
+
)
|
|
9
|
+
from . import qlang
|
|
10
|
+
from .qlang import (
|
|
11
|
+
q,
|
|
12
|
+
qr,
|
|
13
|
+
qs,
|
|
14
|
+
)
|
|
15
|
+
from .interfaces import (
|
|
16
|
+
DukitAccessor,
|
|
17
|
+
)
|
|
18
|
+
from .pandas import (
|
|
19
|
+
get_df,
|
|
20
|
+
get_dfs,
|
|
21
|
+
deduplicate,
|
|
22
|
+
flatten,
|
|
23
|
+
stagger,
|
|
24
|
+
embed,
|
|
25
|
+
collapse,
|
|
26
|
+
transpose,
|
|
27
|
+
date_delta,
|
|
28
|
+
)
|
|
29
|
+
from .excel import (
|
|
30
|
+
format,
|
|
31
|
+
save,
|
|
32
|
+
)
|
|
33
|
+
from .typing import (
|
|
34
|
+
Box,
|
|
35
|
+
str_ as str,
|
|
36
|
+
int_ as int,
|
|
37
|
+
float_ as float,
|
|
38
|
+
num_ as num,
|
|
39
|
+
bool_ as bool,
|
|
40
|
+
date_ as date,
|
|
41
|
+
datetime_ as datetime,
|
|
42
|
+
na_ as na,
|
|
43
|
+
nk_ as nk,
|
|
44
|
+
yn_ as yn,
|
|
45
|
+
type_ as type,
|
|
46
|
+
convert_ as convert,
|
|
47
|
+
typeinfo_ as typeinfo,
|
|
48
|
+
typeinfostrict_ as typeinfostrict,
|
|
49
|
+
list_ as list,
|
|
50
|
+
)
|
|
51
|
+
from .os import (
|
|
52
|
+
pwd,
|
|
53
|
+
cd,
|
|
54
|
+
cp,
|
|
55
|
+
mv,
|
|
56
|
+
mkdir,
|
|
57
|
+
isdir,
|
|
58
|
+
isfile,
|
|
59
|
+
ispath,
|
|
60
|
+
fetch,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
__all__ = (
|
|
64
|
+
'log',
|
|
65
|
+
'diff',
|
|
66
|
+
'rediff',
|
|
67
|
+
'qlang',
|
|
68
|
+
'q',
|
|
69
|
+
'qr',
|
|
70
|
+
'qs',
|
|
71
|
+
'DukitAccessor',
|
|
72
|
+
'get_df',
|
|
73
|
+
'get_dfs',
|
|
74
|
+
'deduplicate',
|
|
75
|
+
'flatten',
|
|
76
|
+
'stagger',
|
|
77
|
+
'embed',
|
|
78
|
+
'collapse',
|
|
79
|
+
'transpose',
|
|
80
|
+
'date_delta',
|
|
81
|
+
'format',
|
|
82
|
+
'save',
|
|
83
|
+
'Box',
|
|
84
|
+
'str',
|
|
85
|
+
'int',
|
|
86
|
+
'float',
|
|
87
|
+
'num',
|
|
88
|
+
'bool',
|
|
89
|
+
'date',
|
|
90
|
+
'datetime',
|
|
91
|
+
'na',
|
|
92
|
+
'nk',
|
|
93
|
+
'yn',
|
|
94
|
+
'type',
|
|
95
|
+
'convert',
|
|
96
|
+
'typeinfo',
|
|
97
|
+
'typeinfostrict',
|
|
98
|
+
'list',
|
|
99
|
+
'pwd',
|
|
100
|
+
'cd',
|
|
101
|
+
'cp',
|
|
102
|
+
'mv',
|
|
103
|
+
'mkdir',
|
|
104
|
+
'isdir',
|
|
105
|
+
'isfile',
|
|
106
|
+
'ispath',
|
|
107
|
+
'fetch',
|
|
108
|
+
)
|