dtypeflow 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dtypeflow/__init__.py +5 -0
- dtypeflow/core/__init__.py +1 -0
- dtypeflow/core/candidate/__init__.py +40 -0
- dtypeflow/core/candidate/base.py +37 -0
- dtypeflow/core/candidate/factory.py +258 -0
- dtypeflow/core/candidate/primitive.py +79 -0
- dtypeflow/core/dataset.py +46 -0
- dtypeflow/core/dtype/__init__.py +70 -0
- dtypeflow/core/dtype/base.py +14 -0
- dtypeflow/core/dtype/hierachy.md +51 -0
- dtypeflow/core/dtype/nested/__init__.py +5 -0
- dtypeflow/core/dtype/nested/dictionary.py +14 -0
- dtypeflow/core/dtype/primitive/__init__.py +39 -0
- dtypeflow/core/dtype/primitive/base.py +6 -0
- dtypeflow/core/dtype/primitive/boolean.py +20 -0
- dtypeflow/core/dtype/primitive/float.py +40 -0
- dtypeflow/core/dtype/primitive/integer/__init__.py +31 -0
- dtypeflow/core/dtype/primitive/integer/base.py +17 -0
- dtypeflow/core/dtype/primitive/integer/signed.py +37 -0
- dtypeflow/core/dtype/primitive/integer/unsigned.py +37 -0
- dtypeflow/core/dtype/primitive/string.py +14 -0
- dtypeflow/core/override/__init__.py +41 -0
- dtypeflow/core/override/base.py +22 -0
- dtypeflow/core/override/primitive.py +158 -0
- dtypeflow/core/profile/__init__.py +39 -0
- dtypeflow/core/profile/base.py +50 -0
- dtypeflow/core/profile/primitive.py +166 -0
- dtypeflow/core/profiler.py +169 -0
- dtypeflow/core/schema.py +33 -0
- dtypeflow/core/table.py +29 -0
- dtypeflow/py.typed +0 -0
- dtypeflow/tui/__init__.py +17 -0
- dtypeflow/tui/app.py +747 -0
- dtypeflow/tui/app.tcss +220 -0
- dtypeflow/tui/compatibility.py +38 -0
- dtypeflow/tui/dto.py +44 -0
- dtypeflow/tui/service.py +213 -0
- dtypeflow/utils.py +32 -0
- dtypeflow-1.0.0.dist-info/METADATA +93 -0
- dtypeflow-1.0.0.dist-info/RECORD +43 -0
- dtypeflow-1.0.0.dist-info/WHEEL +4 -0
- dtypeflow-1.0.0.dist-info/entry_points.txt +3 -0
- dtypeflow-1.0.0.dist-info/licenses/LICENSE +21 -0
dtypeflow/tui/app.tcss
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
Screen {
|
|
2
|
+
align: center top;
|
|
3
|
+
}
|
|
4
|
+
|
|
5
|
+
.hero {
|
|
6
|
+
align: center middle;
|
|
7
|
+
height: auto;
|
|
8
|
+
padding: 2 4;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
.app-title {
|
|
12
|
+
text-style: bold;
|
|
13
|
+
color: $accent;
|
|
14
|
+
width: auto;
|
|
15
|
+
content-align: center middle;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
.app-subtitle {
|
|
19
|
+
color: $text-muted;
|
|
20
|
+
width: auto;
|
|
21
|
+
content-align: center middle;
|
|
22
|
+
margin-bottom: 1;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
.panel {
|
|
26
|
+
width: 100%;
|
|
27
|
+
max-width: 90;
|
|
28
|
+
height: auto;
|
|
29
|
+
border: round $panel-lighten-2;
|
|
30
|
+
padding: 1 2;
|
|
31
|
+
margin: 1 0;
|
|
32
|
+
background: $panel;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
.field-row {
|
|
36
|
+
height: auto;
|
|
37
|
+
width: 100%;
|
|
38
|
+
align: center middle;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
.field-row Input {
|
|
42
|
+
width: 1fr;
|
|
43
|
+
margin-right: 1;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
.primary-action {
|
|
47
|
+
width: 100%;
|
|
48
|
+
margin-top: 1;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
.status {
|
|
52
|
+
color: $warning;
|
|
53
|
+
height: auto;
|
|
54
|
+
margin-top: 1;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
.status.-ok {
|
|
58
|
+
color: $success;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
.status.-error {
|
|
62
|
+
color: $error;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
#pick-file-screen {
|
|
66
|
+
align: center middle;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
#configure-header {
|
|
70
|
+
width: 100%;
|
|
71
|
+
height: auto;
|
|
72
|
+
padding: 0 2;
|
|
73
|
+
margin-bottom: 1;
|
|
74
|
+
border-bottom: solid $panel-lighten-2;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
#configure-columns {
|
|
78
|
+
width: 100%;
|
|
79
|
+
height: 1fr;
|
|
80
|
+
padding: 1 2;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
ColumnPanel {
|
|
84
|
+
height: auto;
|
|
85
|
+
width: 100%;
|
|
86
|
+
padding: 0 1;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
ColumnPanel .column-stats {
|
|
90
|
+
height: auto;
|
|
91
|
+
color: $text-muted;
|
|
92
|
+
margin-bottom: 1;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
ColumnPanel .column-title {
|
|
96
|
+
color: $primary;
|
|
97
|
+
text-style: bold;
|
|
98
|
+
margin-bottom: 1;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
ColumnPanel .column-recommended {
|
|
102
|
+
color: $text-muted;
|
|
103
|
+
text-style: bold;
|
|
104
|
+
margin-bottom: 1;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
ColumnPanel #dtype-tabs {
|
|
108
|
+
margin-bottom: 1;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
ColumnPanel .dtype-description {
|
|
112
|
+
color: $text-muted;
|
|
113
|
+
margin: 1 0;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
ColumnPanel #dtype-options {
|
|
117
|
+
height: auto;
|
|
118
|
+
max-height: 16;
|
|
119
|
+
border: round $panel-lighten-2;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
ColumnPanel RadioSet {
|
|
123
|
+
height: auto;
|
|
124
|
+
border: none;
|
|
125
|
+
background: transparent;
|
|
126
|
+
margin: 1 0;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
ColumnPanel .boolean-caption {
|
|
130
|
+
margin-top: 1;
|
|
131
|
+
color: $text-muted;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
.dictionary-box {
|
|
135
|
+
height: auto;
|
|
136
|
+
width: 100%;
|
|
137
|
+
margin-top: 1;
|
|
138
|
+
border: round $panel-lighten-2;
|
|
139
|
+
padding: 1;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
CategoryOrderPicker {
|
|
143
|
+
height: auto;
|
|
144
|
+
width: 100%;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
CategoryOrderPicker OptionList {
|
|
148
|
+
height: auto;
|
|
149
|
+
max-height: 12;
|
|
150
|
+
border: round $panel-lighten-2;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
CategoryOrderPicker .reorder-buttons {
|
|
154
|
+
height: auto;
|
|
155
|
+
width: 100%;
|
|
156
|
+
align: center middle;
|
|
157
|
+
margin-top: 1;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
CategoryOrderPicker .reorder-buttons Button {
|
|
161
|
+
margin: 0 1;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
#configure-footer {
|
|
165
|
+
height: auto;
|
|
166
|
+
width: 100%;
|
|
167
|
+
padding: 1 2;
|
|
168
|
+
border-top: solid $panel-lighten-2;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
#configure-footer-nav {
|
|
172
|
+
height: auto;
|
|
173
|
+
width: 100%;
|
|
174
|
+
align: left middle;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
.footer-counter {
|
|
178
|
+
width: 1fr;
|
|
179
|
+
content-align: center middle;
|
|
180
|
+
color: $text-muted;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
#configure-footer-nav Button {
|
|
184
|
+
width: auto;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
#export-result {
|
|
188
|
+
height: 1fr;
|
|
189
|
+
margin-top: 1;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
.browse-modal {
|
|
193
|
+
align: center middle;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
.browse-dialog {
|
|
197
|
+
width: 80%;
|
|
198
|
+
height: 80%;
|
|
199
|
+
max-width: 100;
|
|
200
|
+
border: thick $primary;
|
|
201
|
+
background: $surface;
|
|
202
|
+
padding: 1 2;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
.browse-dialog DirectoryTree {
|
|
206
|
+
height: 1fr;
|
|
207
|
+
border: round $panel-lighten-2;
|
|
208
|
+
margin: 1 0;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
.browse-buttons {
|
|
212
|
+
height: auto;
|
|
213
|
+
width: 100%;
|
|
214
|
+
align: right middle;
|
|
215
|
+
margin-top: 1;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
.browse-buttons Button {
|
|
219
|
+
margin-left: 1;
|
|
220
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Final
|
|
4
|
+
|
|
5
|
+
from dtypeflow.core.candidate import Compatibility
|
|
6
|
+
|
|
7
|
+
GROUP_SAFE: Final[str] = "Safe"
|
|
8
|
+
GROUP_TOO_SMALL: Final[str] = "Too small"
|
|
9
|
+
GROUP_DISCOURAGED: Final[str] = "Discouraged"
|
|
10
|
+
GROUP_INCOMPATIBLE: Final[str] = "Incompatible"
|
|
11
|
+
GROUP_ORDER: Final[tuple[str, ...]] = (
|
|
12
|
+
GROUP_SAFE,
|
|
13
|
+
GROUP_TOO_SMALL,
|
|
14
|
+
GROUP_DISCOURAGED,
|
|
15
|
+
GROUP_INCOMPATIBLE,
|
|
16
|
+
)
|
|
17
|
+
_GROUP_BY_COMPATIBILITY: Final[dict[Compatibility, str]] = {
|
|
18
|
+
Compatibility.FULL_RANGE_FIT: GROUP_SAFE,
|
|
19
|
+
Compatibility.VALUES_PRESERVED: GROUP_SAFE,
|
|
20
|
+
Compatibility.OBSERVED_ONLY_FIT: GROUP_SAFE,
|
|
21
|
+
Compatibility.BOOLEAN_MAPPING: GROUP_SAFE,
|
|
22
|
+
Compatibility.DICTIONARY_ENCODING: GROUP_SAFE,
|
|
23
|
+
Compatibility.RANGE_NO_FIT: GROUP_TOO_SMALL,
|
|
24
|
+
Compatibility.UNEVALUATED_NO_FIT: GROUP_TOO_SMALL,
|
|
25
|
+
Compatibility.PRECISION_REDUCTION: GROUP_DISCOURAGED,
|
|
26
|
+
Compatibility.DECIMAL_TRUNCATION: GROUP_DISCOURAGED,
|
|
27
|
+
Compatibility.DICTIONARY_INEFFICIENT: GROUP_DISCOURAGED,
|
|
28
|
+
Compatibility.VALUES_NO_FIT: GROUP_INCOMPATIBLE,
|
|
29
|
+
Compatibility.NOT_NUMERIC: GROUP_INCOMPATIBLE,
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def group_of(compatibility: Compatibility, /) -> str:
|
|
34
|
+
return _GROUP_BY_COMPATIBILITY[compatibility]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def is_selectable(compatibility: Compatibility, /) -> bool:
|
|
38
|
+
return group_of(compatibility) != GROUP_INCOMPATIBLE
|
dtypeflow/tui/dto.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass(frozen=True)
|
|
7
|
+
class DtypeOption:
|
|
8
|
+
name: str
|
|
9
|
+
label: str
|
|
10
|
+
description: str
|
|
11
|
+
group: str
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class ColumnReport:
|
|
16
|
+
name: str
|
|
17
|
+
inferred_dtype: str
|
|
18
|
+
description: str
|
|
19
|
+
num_unique: int
|
|
20
|
+
total: int
|
|
21
|
+
samples: tuple[str, ...]
|
|
22
|
+
stats: tuple[tuple[str, str], ...]
|
|
23
|
+
options: tuple[DtypeOption, ...]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class DatasetReport:
|
|
28
|
+
source: str
|
|
29
|
+
columns: tuple[ColumnReport, ...]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class ColumnSelection:
|
|
34
|
+
dtype_name: str
|
|
35
|
+
true_value: str | None = None
|
|
36
|
+
false_value: str | None = None
|
|
37
|
+
ordered: bool = False
|
|
38
|
+
categories: tuple[str, ...] | None = None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class LoadedDataset:
|
|
43
|
+
report: DatasetReport
|
|
44
|
+
dataset_ref: object = field(repr=False)
|
dtypeflow/tui/service.py
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from decimal import Decimal
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import TYPE_CHECKING, Final, cast
|
|
6
|
+
|
|
7
|
+
import pyarrow.parquet as pq
|
|
8
|
+
|
|
9
|
+
from dtypeflow import profile as profile_dataset
|
|
10
|
+
from dtypeflow.core.dtype import (
|
|
11
|
+
ALL_TYPES,
|
|
12
|
+
BooleanDtype,
|
|
13
|
+
DictionaryDtype,
|
|
14
|
+
Dtype,
|
|
15
|
+
Float16Dtype,
|
|
16
|
+
Float32Dtype,
|
|
17
|
+
Float64Dtype,
|
|
18
|
+
FloatDtype,
|
|
19
|
+
Int8Dtype,
|
|
20
|
+
Int16Dtype,
|
|
21
|
+
Int32Dtype,
|
|
22
|
+
Int64Dtype,
|
|
23
|
+
IntegerDtype,
|
|
24
|
+
StringDtype,
|
|
25
|
+
UInt8Dtype,
|
|
26
|
+
UInt16Dtype,
|
|
27
|
+
UInt32Dtype,
|
|
28
|
+
UInt64Dtype,
|
|
29
|
+
)
|
|
30
|
+
from dtypeflow.core.override import (
|
|
31
|
+
BooleanOverride,
|
|
32
|
+
DictionaryOverride,
|
|
33
|
+
Float16Override,
|
|
34
|
+
Float32Override,
|
|
35
|
+
Float64Override,
|
|
36
|
+
Int8Override,
|
|
37
|
+
Int16Override,
|
|
38
|
+
Int32Override,
|
|
39
|
+
Int64Override,
|
|
40
|
+
Override,
|
|
41
|
+
StringOverride,
|
|
42
|
+
UInt8Override,
|
|
43
|
+
UInt16Override,
|
|
44
|
+
UInt32Override,
|
|
45
|
+
UInt64Override,
|
|
46
|
+
)
|
|
47
|
+
from dtypeflow.core.schema import Schema
|
|
48
|
+
|
|
49
|
+
from .compatibility import GROUP_SAFE, group_of, is_selectable
|
|
50
|
+
from .dto import (
|
|
51
|
+
ColumnReport,
|
|
52
|
+
ColumnSelection,
|
|
53
|
+
DatasetReport,
|
|
54
|
+
DtypeOption,
|
|
55
|
+
LoadedDataset,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
if TYPE_CHECKING:
|
|
59
|
+
from collections.abc import Mapping
|
|
60
|
+
|
|
61
|
+
from dtypeflow.core.dataset import Dataset
|
|
62
|
+
from dtypeflow.core.profile import Profile
|
|
63
|
+
_DTYPE_BY_NAME: Final[dict[str, type[Dtype]]] = {
|
|
64
|
+
dtype.__name__: dtype for dtype in ALL_TYPES
|
|
65
|
+
}
|
|
66
|
+
_OVERRIDE_BY_DTYPE: Final[dict[type[Dtype], type[Override]]] = {
|
|
67
|
+
BooleanDtype: BooleanOverride,
|
|
68
|
+
UInt8Dtype: UInt8Override,
|
|
69
|
+
UInt16Dtype: UInt16Override,
|
|
70
|
+
UInt32Dtype: UInt32Override,
|
|
71
|
+
UInt64Dtype: UInt64Override,
|
|
72
|
+
Int8Dtype: Int8Override,
|
|
73
|
+
Int16Dtype: Int16Override,
|
|
74
|
+
Int32Dtype: Int32Override,
|
|
75
|
+
Int64Dtype: Int64Override,
|
|
76
|
+
Float16Dtype: Float16Override,
|
|
77
|
+
Float32Dtype: Float32Override,
|
|
78
|
+
Float64Dtype: Float64Override,
|
|
79
|
+
StringDtype: StringOverride,
|
|
80
|
+
DictionaryDtype: DictionaryOverride,
|
|
81
|
+
}
|
|
82
|
+
DESCRIPTIONS: Final[dict[type[Dtype], str]] = {
|
|
83
|
+
BooleanDtype: f"{BooleanDtype.TRUE} or {BooleanDtype.FALSE}",
|
|
84
|
+
UInt8Dtype: f"Whole non-negative number, {UInt8Dtype.LOWER_BOUND:,} to {UInt8Dtype.HIGHER_BOUND:,}",
|
|
85
|
+
UInt16Dtype: f"Whole non-negative number, {UInt16Dtype.LOWER_BOUND:,} to {UInt16Dtype.HIGHER_BOUND:,}",
|
|
86
|
+
UInt32Dtype: f"Whole non-negative number, {UInt32Dtype.LOWER_BOUND:,} to {float(UInt32Dtype.HIGHER_BOUND):.1E}",
|
|
87
|
+
UInt64Dtype: f"Whole non-negative number, {UInt64Dtype.LOWER_BOUND:,} to {float(UInt64Dtype.HIGHER_BOUND):.1E}",
|
|
88
|
+
Int8Dtype: f"Whole number, {Int8Dtype.LOWER_BOUND:,} to {Int8Dtype.HIGHER_BOUND:,}",
|
|
89
|
+
Int16Dtype: f"Whole number, {Int16Dtype.LOWER_BOUND:,} to {Int16Dtype.HIGHER_BOUND:,}",
|
|
90
|
+
Int32Dtype: f"Whole number, {float(Int32Dtype.LOWER_BOUND):.1E} to {float(Int32Dtype.HIGHER_BOUND):.1E}",
|
|
91
|
+
Int64Dtype: f"Whole number, {float(Int64Dtype.LOWER_BOUND):.1E} to {float(Int64Dtype.HIGHER_BOUND):.1E}",
|
|
92
|
+
Float16Dtype: f"Decimal number, about {Float16Dtype.MAX_SIGNIFICANT_DIGITS} digits of precision, up to {Float16Dtype.MAX_ABSOLUTE_VALUE:.1E}",
|
|
93
|
+
Float32Dtype: f"Decimal number, about {Float32Dtype.MAX_SIGNIFICANT_DIGITS} digits of precision, up to {Float32Dtype.MAX_ABSOLUTE_VALUE:.1E}",
|
|
94
|
+
Float64Dtype: f"Decimal number, about {Float64Dtype.MAX_SIGNIFICANT_DIGITS} digits of precision, up to {Float64Dtype.MAX_ABSOLUTE_VALUE:.1E}",
|
|
95
|
+
StringDtype: "Text",
|
|
96
|
+
DictionaryDtype: "Category, a fixed set of repeating values",
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _label(dtype: type[Dtype], /) -> str:
|
|
101
|
+
return dtype.__name__.removesuffix("Dtype")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _dtype_option(dtype: type[Dtype], group: str, /) -> DtypeOption:
|
|
105
|
+
return DtypeOption(
|
|
106
|
+
name=dtype.__name__,
|
|
107
|
+
label=_label(dtype),
|
|
108
|
+
description=DESCRIPTIONS[dtype],
|
|
109
|
+
group=group,
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _format_sample(dtype: type[Dtype], value: str, /) -> str:
|
|
114
|
+
if issubclass(dtype, IntegerDtype):
|
|
115
|
+
return f"{int(value):,}"
|
|
116
|
+
if issubclass(dtype, FloatDtype):
|
|
117
|
+
return f"{Decimal(value):,}"
|
|
118
|
+
return value
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _stats(dtype: type[Dtype], profile: Profile, /) -> tuple[tuple[str, str], ...]:
|
|
122
|
+
samples = ", ".join(
|
|
123
|
+
_format_sample(dtype, value) for value in profile.uniques(n=5)
|
|
124
|
+
)
|
|
125
|
+
stats: list[tuple[str, str]] = [
|
|
126
|
+
("Unique values", f"{profile.num_unique:,}"),
|
|
127
|
+
("Samples", samples),
|
|
128
|
+
]
|
|
129
|
+
if issubclass(dtype, (FloatDtype, IntegerDtype)):
|
|
130
|
+
stats.append(("Range", f"{profile.min:,} – {profile.max:,}"))
|
|
131
|
+
if issubclass(dtype, FloatDtype):
|
|
132
|
+
stats.append(("Precision", str(profile.precision)))
|
|
133
|
+
stats.append(("Scale", str(profile.scale)))
|
|
134
|
+
elif issubclass(dtype, StringDtype):
|
|
135
|
+
stats.append(
|
|
136
|
+
("Length range", f"{profile.min_length:,} – {profile.max_length:,}"),
|
|
137
|
+
)
|
|
138
|
+
return tuple(stats)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _column_report(column: str, profile: Profile, /) -> ColumnReport:
|
|
142
|
+
dtype: type[Dtype] = type(profile).DTYPE
|
|
143
|
+
options: list[DtypeOption] = [_dtype_option(dtype, GROUP_SAFE)]
|
|
144
|
+
options.extend(
|
|
145
|
+
|
|
146
|
+
_dtype_option(candidate.DTYPE, group_of(candidate.compatibility))
|
|
147
|
+
for candidate in profile.candidates
|
|
148
|
+
if is_selectable(candidate.compatibility)
|
|
149
|
+
|
|
150
|
+
)
|
|
151
|
+
return ColumnReport(
|
|
152
|
+
name=column,
|
|
153
|
+
inferred_dtype=dtype.__name__,
|
|
154
|
+
description=DESCRIPTIONS[dtype],
|
|
155
|
+
num_unique=profile.num_unique,
|
|
156
|
+
total=profile.total,
|
|
157
|
+
samples=profile.uniques(n=5),
|
|
158
|
+
stats=_stats(dtype, profile),
|
|
159
|
+
options=tuple(options),
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _build_override(column: str, selection: ColumnSelection, /) -> Override:
|
|
164
|
+
dtype: type[Dtype] = _DTYPE_BY_NAME[selection.dtype_name]
|
|
165
|
+
override_cls: type[Override] = _OVERRIDE_BY_DTYPE[dtype]
|
|
166
|
+
if dtype is BooleanDtype:
|
|
167
|
+
if selection.true_value is None or selection.false_value is None:
|
|
168
|
+
msg = f"Column {column!r} needs both a true and a false value to become Boolean."
|
|
169
|
+
raise ValueError(msg)
|
|
170
|
+
return BooleanOverride(
|
|
171
|
+
column, true_value=selection.true_value, false_value=selection.false_value,
|
|
172
|
+
)
|
|
173
|
+
if dtype is DictionaryDtype:
|
|
174
|
+
if selection.categories is None:
|
|
175
|
+
msg = f"Column {column!r} needs a category list to become Dictionary."
|
|
176
|
+
raise ValueError(msg)
|
|
177
|
+
return DictionaryOverride(
|
|
178
|
+
column, *selection.categories, ordered=selection.ordered,
|
|
179
|
+
)
|
|
180
|
+
return override_cls(column)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
class DtypeflowService:
|
|
184
|
+
def load_dataset(self, path: str | Path, /) -> LoadedDataset:
|
|
185
|
+
dataset: Dataset = profile_dataset(path)
|
|
186
|
+
columns = tuple(
|
|
187
|
+
_column_report(column, dataset[column]) for column in dataset.columns
|
|
188
|
+
)
|
|
189
|
+
report = DatasetReport(source=str(dataset.source), columns=columns)
|
|
190
|
+
return LoadedDataset(report=report, dataset_ref=dataset)
|
|
191
|
+
|
|
192
|
+
def unique_values(self, loaded: LoadedDataset, column: str, /) -> tuple[str, ...]:
|
|
193
|
+
dataset = cast("Dataset", loaded.dataset_ref)
|
|
194
|
+
return dataset[column].uniques()
|
|
195
|
+
|
|
196
|
+
def export(
|
|
197
|
+
self,
|
|
198
|
+
loaded: LoadedDataset,
|
|
199
|
+
destination: str | Path,
|
|
200
|
+
selections: Mapping[str, ColumnSelection],
|
|
201
|
+
/,
|
|
202
|
+
) -> tuple[tuple[str, str], ...]:
|
|
203
|
+
dataset = cast("Dataset", loaded.dataset_ref)
|
|
204
|
+
overrides = tuple(
|
|
205
|
+
|
|
206
|
+
_build_override(column, selection)
|
|
207
|
+
for (column, selection) in selections.items()
|
|
208
|
+
|
|
209
|
+
)
|
|
210
|
+
destination = Path(destination)
|
|
211
|
+
Schema(*overrides).apply(dataset, destination=destination)
|
|
212
|
+
written = pq.read_table(destination).schema
|
|
213
|
+
return tuple((field.name, str(field.type)) for field in written)
|
dtypeflow/utils.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from csv import DictReader
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from dtypeflow.core.table import Table
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def read_csv(path: str | Path, /, *, delimiter: str = ",", newline: str = "") -> Table:
|
|
10
|
+
if isinstance(path, str):
|
|
11
|
+
if not path:
|
|
12
|
+
msg = "The path to the CSV file cannot be empty."
|
|
13
|
+
raise ValueError(msg)
|
|
14
|
+
path: Path = Path(path)
|
|
15
|
+
if not path.exists():
|
|
16
|
+
msg = f"The CSV file at path '{path}' does not exist."
|
|
17
|
+
raise FileNotFoundError(msg)
|
|
18
|
+
if not path.is_file():
|
|
19
|
+
msg = f"The path '{path}' is not a valid file."
|
|
20
|
+
raise ValueError(msg)
|
|
21
|
+
if path.suffix.lower() != ".csv":
|
|
22
|
+
msg = f"The file '{path}' is not a CSV file."
|
|
23
|
+
raise ValueError(msg)
|
|
24
|
+
with path.open(newline=newline) as file:
|
|
25
|
+
reader: DictReader[str] = DictReader(file, delimiter=delimiter)
|
|
26
|
+
rows: list[dict[str, str]] = list(reader)
|
|
27
|
+
return Table(
|
|
28
|
+
**{
|
|
29
|
+
name: tuple(row[name] for row in rows)
|
|
30
|
+
for name in reader.fieldnames or ()
|
|
31
|
+
},
|
|
32
|
+
)
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dtypeflow
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Terminal UI to infer CSV column dtypes and convert CSV to Parquet, locally.
|
|
5
|
+
Keywords: csv,parquet,dtype,tui,textual,arrow
|
|
6
|
+
Author: Pedro José Mora Gallegos
|
|
7
|
+
Author-email: Pedro José Mora Gallegos <developmentbypedrojose@outlook.de>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Topic :: Utilities
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering
|
|
18
|
+
Requires-Dist: pyarrow>=17.0.0
|
|
19
|
+
Requires-Dist: textual>=0.89.0
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Project-URL: Homepage, https://github.com/pedrojosemoragallegos/dtypeflow
|
|
22
|
+
Project-URL: Repository, https://github.com/pedrojosemoragallegos/dtypeflow
|
|
23
|
+
Project-URL: Issues, https://github.com/pedrojosemoragallegos/dtypeflow/issues
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# dtypeflow
|
|
27
|
+
|
|
28
|
+
`dtypeflow` is a terminal UI to convert a CSV file to Parquet.
|
|
29
|
+
|
|
30
|
+
Point it at a CSV file, and for each column it infers the tightest-fitting
|
|
31
|
+
dtype (`Boolean`, unsigned/signed integers `UInt8`…`UInt64`/`Int8`…`Int64`,
|
|
32
|
+
floats `Float16`/`Float32`/`Float64`, `String`, or `Dictionary`), shows the
|
|
33
|
+
observed statistics, and lets you review or override the choice before
|
|
34
|
+
writing the result to Parquet — all locally, no data leaves your machine.
|
|
35
|
+
|
|
36
|
+
## Installation
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
uv tool install dtypeflow
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Or with `pipx`:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pipx install dtypeflow
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Or with `pip`:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
pip install dtypeflow
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Usage
|
|
55
|
+
|
|
56
|
+
Launch the picker and choose a CSV file interactively:
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
dtypeflow
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Or pass a path directly to skip straight to configuring column types:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
dtypeflow path/to/data.csv
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
The workflow has three steps:
|
|
69
|
+
|
|
70
|
+
1. **Pick a file** — enter or browse for a CSV path.
|
|
71
|
+
2. **Configure types** — for each column, review the recommended dtype, its
|
|
72
|
+
stats (unique values, samples, range/precision/scale), and pick from
|
|
73
|
+
compatible alternatives grouped by safety (Safe / Too small / Discouraged).
|
|
74
|
+
3. **Export** — write the configured dataset to a Parquet file.
|
|
75
|
+
|
|
76
|
+
## Development
|
|
77
|
+
|
|
78
|
+
This project uses [uv](https://docs.astral.sh/uv/) for dependency management.
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
uv sync
|
|
82
|
+
uv run dtypeflow
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Lint:
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
uv run --with ruff ruff check src/
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## License
|
|
92
|
+
|
|
93
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
dtypeflow/__init__.py,sha256=42mpdAvj8VsnaMJYL6O8zTWiwokd80lEQ7LWJENCHaw,114
|
|
2
|
+
dtypeflow/core/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
|
|
3
|
+
dtypeflow/core/candidate/__init__.py,sha256=g7tif4In9KVcvMgeVX_lsoGD3xgM8wtR899gnIu0s2k,996
|
|
4
|
+
dtypeflow/core/candidate/base.py,sha256=jOUkmGwtUPLo_gi-z_lIX_MHvpNqq__YVcV64-TjZnY,972
|
|
5
|
+
dtypeflow/core/candidate/factory.py,sha256=zatB4ve4fVT_o7SW3FmqoMMg4Q08FYD4TrswAzXFl6k,7672
|
|
6
|
+
dtypeflow/core/candidate/primitive.py,sha256=MbPVpyt32OL8DxNVnkLkipW97nHgAwBEhiD7G39u9HI,1377
|
|
7
|
+
dtypeflow/core/dataset.py,sha256=X1PQWRXsyvYuaaVN0iLAb2ffhWSs_a2_WTIOBPfRAmI,1356
|
|
8
|
+
dtypeflow/core/dtype/__init__.py,sha256=NKrfx0kFygJnewD9eMcOdtevKpjD98oEdkmUEa73yRw,1642
|
|
9
|
+
dtypeflow/core/dtype/base.py,sha256=EYcwO5oE2w6E-UCReHczgGmpbCUqYSsRgggdzmXXSJA,255
|
|
10
|
+
dtypeflow/core/dtype/hierachy.md,sha256=Nqpky_L6GdWRNW4fO_pfKaAPfW03eFNtStRK-YwJcD8,2040
|
|
11
|
+
dtypeflow/core/dtype/nested/__init__.py,sha256=hLj8pD6YuGT29_Q1CeFLcTa4SqQKOqI8OG0cRg7axG0,132
|
|
12
|
+
dtypeflow/core/dtype/nested/dictionary.py,sha256=njg9lRN4Pw2Br2yOY04Qas0YJqLe35zk1j7dpdFDn9Y,271
|
|
13
|
+
dtypeflow/core/dtype/primitive/__init__.py,sha256=AvFuCtdt5slYOugbSiucL3aCXrRkSZ_ZhjP_Y4U2qFU,740
|
|
14
|
+
dtypeflow/core/dtype/primitive/base.py,sha256=GqhcoqMKnD9KAW9NrmfjeUdPJD62N2X09HerdXXzw0Y,110
|
|
15
|
+
dtypeflow/core/dtype/primitive/boolean.py,sha256=8RiEDwSyZ42IIHzf5wpaYKxImCkVMgi2JhcMNd_HOtw,432
|
|
16
|
+
dtypeflow/core/dtype/primitive/float.py,sha256=f7f5_VLLSA2fcLD7pfeFa_W-8TngFnGYJuPoMbgbseU,991
|
|
17
|
+
dtypeflow/core/dtype/primitive/integer/__init__.py,sha256=54bdf3_PpdLz9gpm6abmxbXNK2N_DGlWlu1UtUK5-tI,622
|
|
18
|
+
dtypeflow/core/dtype/primitive/integer/base.py,sha256=pGCngH5O4CdBqTFLZYRxKi2fsMVgl0ZvUJnLsFutufc,445
|
|
19
|
+
dtypeflow/core/dtype/primitive/integer/signed.py,sha256=pnNMITMMw-YsORGqSEMGBA3kLTLyPVLOpY9kzIOn76k,768
|
|
20
|
+
dtypeflow/core/dtype/primitive/integer/unsigned.py,sha256=ks0o2Prmq2M3dzWB5Sell_oVTqou8R-lG7Z2ZH7zICw,746
|
|
21
|
+
dtypeflow/core/dtype/primitive/string.py,sha256=tqz8np4VfjKZJqJAHdKYzAZke-3h4YpYONi-Z9ul9AM,255
|
|
22
|
+
dtypeflow/core/override/__init__.py,sha256=TFfzxo_ZNBVFUgiWVe5xRFia7YMLYGQ6dOCqSCYl-p0,970
|
|
23
|
+
dtypeflow/core/override/base.py,sha256=ALzi2yTvqlFQ56-hikd1bD289qn7XN5EFiu1uvHkIIs,507
|
|
24
|
+
dtypeflow/core/override/primitive.py,sha256=9bCP7fXKkmZBY3dDCxrWy1ny-LMMCSuXSXRurl33C2c,5712
|
|
25
|
+
dtypeflow/core/profile/__init__.py,sha256=JqyT6sqFWLnh3fBWfcm1tQjhHT8fQQnlhuTbHkHU3-8,874
|
|
26
|
+
dtypeflow/core/profile/base.py,sha256=qtRfKMqm65JoQR9qufekxesc2Sw_7T8ptfcj-B-_M2U,1489
|
|
27
|
+
dtypeflow/core/profile/primitive.py,sha256=YWap_IB5w2AOQrZ3NvXiSr75JwSB2UdSaj2i-RTAsTc,4130
|
|
28
|
+
dtypeflow/core/profiler.py,sha256=EI3qrR3I5UV-jLJT_Xv8fEFh5G_FPdzpvdhXMYs62vc,5123
|
|
29
|
+
dtypeflow/core/schema.py,sha256=CMfyZbaaLsSITYRRdmYuefaFk_Xj-VxB6oXZ-xQWeC4,1173
|
|
30
|
+
dtypeflow/core/table.py,sha256=07cJKLYz2USEgHqo1o4LN6QwL0MsWtHGnkBbYHhjnLE,833
|
|
31
|
+
dtypeflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
32
|
+
dtypeflow/tui/__init__.py,sha256=fVjm0bBy49pMsLreE2h9NUIPA3l5gr3Nw4S3JdfTLX4,469
|
|
33
|
+
dtypeflow/tui/app.py,sha256=8Lic3uG69phm1tjJugjTCTOpCKQwbmhCAYw1ZMIVKXs,26952
|
|
34
|
+
dtypeflow/tui/app.tcss,sha256=zGCT_-OyzEpAcLwS50gTpc5FGVqeKvvEyr6VPH7xNN4,3216
|
|
35
|
+
dtypeflow/tui/compatibility.py,sha256=t2x3h1uISJq35HfoBZZJnzpME5du9tA-7glap22EvrU,1321
|
|
36
|
+
dtypeflow/tui/dto.py,sha256=eU9pgRkDIC9va6h0q3iSDy3YpwF3sSCv3K5oj_bsMxE,847
|
|
37
|
+
dtypeflow/tui/service.py,sha256=4hlAWNiIMkwkuQ2x-85n6HMUC_GbfwIFL3HVpC7KOwU,7536
|
|
38
|
+
dtypeflow/utils.py,sha256=rBHJy1hMEKGDr08vsQo91PIW8xNf4LAU0LtxHtLcUL4,1095
|
|
39
|
+
dtypeflow-1.0.0.dist-info/licenses/LICENSE,sha256=tr5T8mamK1bEK0fgZrIlLs2qbJDJmL-_zTZ7HpiPvVA,1082
|
|
40
|
+
dtypeflow-1.0.0.dist-info/WHEEL,sha256=mru_b36sH6joUMnwf7IlFCun3RoDjrNg9RcfBmEcqsE,81
|
|
41
|
+
dtypeflow-1.0.0.dist-info/entry_points.txt,sha256=HoEXQsH0UUf7ReHGhDcWliO1uX7VzblTGQoIlk5cOCg,49
|
|
42
|
+
dtypeflow-1.0.0.dist-info/METADATA,sha256=cSNYfEZHgEnuDeFAZjfY8-YgXpkwNj35yO9FztcZVAM,2453
|
|
43
|
+
dtypeflow-1.0.0.dist-info/RECORD,,
|