dtypeflow 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. dtypeflow/__init__.py +5 -0
  2. dtypeflow/core/__init__.py +1 -0
  3. dtypeflow/core/candidate/__init__.py +40 -0
  4. dtypeflow/core/candidate/base.py +37 -0
  5. dtypeflow/core/candidate/factory.py +258 -0
  6. dtypeflow/core/candidate/primitive.py +79 -0
  7. dtypeflow/core/dataset.py +46 -0
  8. dtypeflow/core/dtype/__init__.py +70 -0
  9. dtypeflow/core/dtype/base.py +14 -0
  10. dtypeflow/core/dtype/hierachy.md +51 -0
  11. dtypeflow/core/dtype/nested/__init__.py +5 -0
  12. dtypeflow/core/dtype/nested/dictionary.py +14 -0
  13. dtypeflow/core/dtype/primitive/__init__.py +39 -0
  14. dtypeflow/core/dtype/primitive/base.py +6 -0
  15. dtypeflow/core/dtype/primitive/boolean.py +20 -0
  16. dtypeflow/core/dtype/primitive/float.py +40 -0
  17. dtypeflow/core/dtype/primitive/integer/__init__.py +31 -0
  18. dtypeflow/core/dtype/primitive/integer/base.py +17 -0
  19. dtypeflow/core/dtype/primitive/integer/signed.py +37 -0
  20. dtypeflow/core/dtype/primitive/integer/unsigned.py +37 -0
  21. dtypeflow/core/dtype/primitive/string.py +14 -0
  22. dtypeflow/core/override/__init__.py +41 -0
  23. dtypeflow/core/override/base.py +22 -0
  24. dtypeflow/core/override/primitive.py +158 -0
  25. dtypeflow/core/profile/__init__.py +39 -0
  26. dtypeflow/core/profile/base.py +50 -0
  27. dtypeflow/core/profile/primitive.py +166 -0
  28. dtypeflow/core/profiler.py +169 -0
  29. dtypeflow/core/schema.py +33 -0
  30. dtypeflow/core/table.py +29 -0
  31. dtypeflow/py.typed +0 -0
  32. dtypeflow/tui/__init__.py +17 -0
  33. dtypeflow/tui/app.py +747 -0
  34. dtypeflow/tui/app.tcss +220 -0
  35. dtypeflow/tui/compatibility.py +38 -0
  36. dtypeflow/tui/dto.py +44 -0
  37. dtypeflow/tui/service.py +213 -0
  38. dtypeflow/utils.py +32 -0
  39. dtypeflow-1.0.0.dist-info/METADATA +93 -0
  40. dtypeflow-1.0.0.dist-info/RECORD +43 -0
  41. dtypeflow-1.0.0.dist-info/WHEEL +4 -0
  42. dtypeflow-1.0.0.dist-info/entry_points.txt +3 -0
  43. dtypeflow-1.0.0.dist-info/licenses/LICENSE +21 -0
dtypeflow/tui/app.tcss ADDED
@@ -0,0 +1,220 @@
1
+ Screen {
2
+ align: center top;
3
+ }
4
+
5
+ .hero {
6
+ align: center middle;
7
+ height: auto;
8
+ padding: 2 4;
9
+ }
10
+
11
+ .app-title {
12
+ text-style: bold;
13
+ color: $accent;
14
+ width: auto;
15
+ content-align: center middle;
16
+ }
17
+
18
+ .app-subtitle {
19
+ color: $text-muted;
20
+ width: auto;
21
+ content-align: center middle;
22
+ margin-bottom: 1;
23
+ }
24
+
25
+ .panel {
26
+ width: 100%;
27
+ max-width: 90;
28
+ height: auto;
29
+ border: round $panel-lighten-2;
30
+ padding: 1 2;
31
+ margin: 1 0;
32
+ background: $panel;
33
+ }
34
+
35
+ .field-row {
36
+ height: auto;
37
+ width: 100%;
38
+ align: center middle;
39
+ }
40
+
41
+ .field-row Input {
42
+ width: 1fr;
43
+ margin-right: 1;
44
+ }
45
+
46
+ .primary-action {
47
+ width: 100%;
48
+ margin-top: 1;
49
+ }
50
+
51
+ .status {
52
+ color: $warning;
53
+ height: auto;
54
+ margin-top: 1;
55
+ }
56
+
57
+ .status.-ok {
58
+ color: $success;
59
+ }
60
+
61
+ .status.-error {
62
+ color: $error;
63
+ }
64
+
65
+ #pick-file-screen {
66
+ align: center middle;
67
+ }
68
+
69
+ #configure-header {
70
+ width: 100%;
71
+ height: auto;
72
+ padding: 0 2;
73
+ margin-bottom: 1;
74
+ border-bottom: solid $panel-lighten-2;
75
+ }
76
+
77
+ #configure-columns {
78
+ width: 100%;
79
+ height: 1fr;
80
+ padding: 1 2;
81
+ }
82
+
83
+ ColumnPanel {
84
+ height: auto;
85
+ width: 100%;
86
+ padding: 0 1;
87
+ }
88
+
89
+ ColumnPanel .column-stats {
90
+ height: auto;
91
+ color: $text-muted;
92
+ margin-bottom: 1;
93
+ }
94
+
95
+ ColumnPanel .column-title {
96
+ color: $primary;
97
+ text-style: bold;
98
+ margin-bottom: 1;
99
+ }
100
+
101
+ ColumnPanel .column-recommended {
102
+ color: $text-muted;
103
+ text-style: bold;
104
+ margin-bottom: 1;
105
+ }
106
+
107
+ ColumnPanel #dtype-tabs {
108
+ margin-bottom: 1;
109
+ }
110
+
111
+ ColumnPanel .dtype-description {
112
+ color: $text-muted;
113
+ margin: 1 0;
114
+ }
115
+
116
+ ColumnPanel #dtype-options {
117
+ height: auto;
118
+ max-height: 16;
119
+ border: round $panel-lighten-2;
120
+ }
121
+
122
+ ColumnPanel RadioSet {
123
+ height: auto;
124
+ border: none;
125
+ background: transparent;
126
+ margin: 1 0;
127
+ }
128
+
129
+ ColumnPanel .boolean-caption {
130
+ margin-top: 1;
131
+ color: $text-muted;
132
+ }
133
+
134
+ .dictionary-box {
135
+ height: auto;
136
+ width: 100%;
137
+ margin-top: 1;
138
+ border: round $panel-lighten-2;
139
+ padding: 1;
140
+ }
141
+
142
+ CategoryOrderPicker {
143
+ height: auto;
144
+ width: 100%;
145
+ }
146
+
147
+ CategoryOrderPicker OptionList {
148
+ height: auto;
149
+ max-height: 12;
150
+ border: round $panel-lighten-2;
151
+ }
152
+
153
+ CategoryOrderPicker .reorder-buttons {
154
+ height: auto;
155
+ width: 100%;
156
+ align: center middle;
157
+ margin-top: 1;
158
+ }
159
+
160
+ CategoryOrderPicker .reorder-buttons Button {
161
+ margin: 0 1;
162
+ }
163
+
164
+ #configure-footer {
165
+ height: auto;
166
+ width: 100%;
167
+ padding: 1 2;
168
+ border-top: solid $panel-lighten-2;
169
+ }
170
+
171
+ #configure-footer-nav {
172
+ height: auto;
173
+ width: 100%;
174
+ align: left middle;
175
+ }
176
+
177
+ .footer-counter {
178
+ width: 1fr;
179
+ content-align: center middle;
180
+ color: $text-muted;
181
+ }
182
+
183
+ #configure-footer-nav Button {
184
+ width: auto;
185
+ }
186
+
187
+ #export-result {
188
+ height: 1fr;
189
+ margin-top: 1;
190
+ }
191
+
192
+ .browse-modal {
193
+ align: center middle;
194
+ }
195
+
196
+ .browse-dialog {
197
+ width: 80%;
198
+ height: 80%;
199
+ max-width: 100;
200
+ border: thick $primary;
201
+ background: $surface;
202
+ padding: 1 2;
203
+ }
204
+
205
+ .browse-dialog DirectoryTree {
206
+ height: 1fr;
207
+ border: round $panel-lighten-2;
208
+ margin: 1 0;
209
+ }
210
+
211
+ .browse-buttons {
212
+ height: auto;
213
+ width: 100%;
214
+ align: right middle;
215
+ margin-top: 1;
216
+ }
217
+
218
+ .browse-buttons Button {
219
+ margin-left: 1;
220
+ }
@@ -0,0 +1,38 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Final
4
+
5
+ from dtypeflow.core.candidate import Compatibility
6
+
7
+ GROUP_SAFE: Final[str] = "Safe"
8
+ GROUP_TOO_SMALL: Final[str] = "Too small"
9
+ GROUP_DISCOURAGED: Final[str] = "Discouraged"
10
+ GROUP_INCOMPATIBLE: Final[str] = "Incompatible"
11
+ GROUP_ORDER: Final[tuple[str, ...]] = (
12
+ GROUP_SAFE,
13
+ GROUP_TOO_SMALL,
14
+ GROUP_DISCOURAGED,
15
+ GROUP_INCOMPATIBLE,
16
+ )
17
+ _GROUP_BY_COMPATIBILITY: Final[dict[Compatibility, str]] = {
18
+ Compatibility.FULL_RANGE_FIT: GROUP_SAFE,
19
+ Compatibility.VALUES_PRESERVED: GROUP_SAFE,
20
+ Compatibility.OBSERVED_ONLY_FIT: GROUP_SAFE,
21
+ Compatibility.BOOLEAN_MAPPING: GROUP_SAFE,
22
+ Compatibility.DICTIONARY_ENCODING: GROUP_SAFE,
23
+ Compatibility.RANGE_NO_FIT: GROUP_TOO_SMALL,
24
+ Compatibility.UNEVALUATED_NO_FIT: GROUP_TOO_SMALL,
25
+ Compatibility.PRECISION_REDUCTION: GROUP_DISCOURAGED,
26
+ Compatibility.DECIMAL_TRUNCATION: GROUP_DISCOURAGED,
27
+ Compatibility.DICTIONARY_INEFFICIENT: GROUP_DISCOURAGED,
28
+ Compatibility.VALUES_NO_FIT: GROUP_INCOMPATIBLE,
29
+ Compatibility.NOT_NUMERIC: GROUP_INCOMPATIBLE,
30
+ }
31
+
32
+
33
+ def group_of(compatibility: Compatibility, /) -> str:
34
+ return _GROUP_BY_COMPATIBILITY[compatibility]
35
+
36
+
37
+ def is_selectable(compatibility: Compatibility, /) -> bool:
38
+ return group_of(compatibility) != GROUP_INCOMPATIBLE
dtypeflow/tui/dto.py ADDED
@@ -0,0 +1,44 @@
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass, field
4
+
5
+
6
+ @dataclass(frozen=True)
7
+ class DtypeOption:
8
+ name: str
9
+ label: str
10
+ description: str
11
+ group: str
12
+
13
+
14
+ @dataclass(frozen=True)
15
+ class ColumnReport:
16
+ name: str
17
+ inferred_dtype: str
18
+ description: str
19
+ num_unique: int
20
+ total: int
21
+ samples: tuple[str, ...]
22
+ stats: tuple[tuple[str, str], ...]
23
+ options: tuple[DtypeOption, ...]
24
+
25
+
26
+ @dataclass(frozen=True)
27
+ class DatasetReport:
28
+ source: str
29
+ columns: tuple[ColumnReport, ...]
30
+
31
+
32
+ @dataclass(frozen=True)
33
+ class ColumnSelection:
34
+ dtype_name: str
35
+ true_value: str | None = None
36
+ false_value: str | None = None
37
+ ordered: bool = False
38
+ categories: tuple[str, ...] | None = None
39
+
40
+
41
+ @dataclass
42
+ class LoadedDataset:
43
+ report: DatasetReport
44
+ dataset_ref: object = field(repr=False)
@@ -0,0 +1,213 @@
1
+ from __future__ import annotations
2
+
3
+ from decimal import Decimal
4
+ from pathlib import Path
5
+ from typing import TYPE_CHECKING, Final, cast
6
+
7
+ import pyarrow.parquet as pq
8
+
9
+ from dtypeflow import profile as profile_dataset
10
+ from dtypeflow.core.dtype import (
11
+ ALL_TYPES,
12
+ BooleanDtype,
13
+ DictionaryDtype,
14
+ Dtype,
15
+ Float16Dtype,
16
+ Float32Dtype,
17
+ Float64Dtype,
18
+ FloatDtype,
19
+ Int8Dtype,
20
+ Int16Dtype,
21
+ Int32Dtype,
22
+ Int64Dtype,
23
+ IntegerDtype,
24
+ StringDtype,
25
+ UInt8Dtype,
26
+ UInt16Dtype,
27
+ UInt32Dtype,
28
+ UInt64Dtype,
29
+ )
30
+ from dtypeflow.core.override import (
31
+ BooleanOverride,
32
+ DictionaryOverride,
33
+ Float16Override,
34
+ Float32Override,
35
+ Float64Override,
36
+ Int8Override,
37
+ Int16Override,
38
+ Int32Override,
39
+ Int64Override,
40
+ Override,
41
+ StringOverride,
42
+ UInt8Override,
43
+ UInt16Override,
44
+ UInt32Override,
45
+ UInt64Override,
46
+ )
47
+ from dtypeflow.core.schema import Schema
48
+
49
+ from .compatibility import GROUP_SAFE, group_of, is_selectable
50
+ from .dto import (
51
+ ColumnReport,
52
+ ColumnSelection,
53
+ DatasetReport,
54
+ DtypeOption,
55
+ LoadedDataset,
56
+ )
57
+
58
+ if TYPE_CHECKING:
59
+ from collections.abc import Mapping
60
+
61
+ from dtypeflow.core.dataset import Dataset
62
+ from dtypeflow.core.profile import Profile
63
+ _DTYPE_BY_NAME: Final[dict[str, type[Dtype]]] = {
64
+ dtype.__name__: dtype for dtype in ALL_TYPES
65
+ }
66
+ _OVERRIDE_BY_DTYPE: Final[dict[type[Dtype], type[Override]]] = {
67
+ BooleanDtype: BooleanOverride,
68
+ UInt8Dtype: UInt8Override,
69
+ UInt16Dtype: UInt16Override,
70
+ UInt32Dtype: UInt32Override,
71
+ UInt64Dtype: UInt64Override,
72
+ Int8Dtype: Int8Override,
73
+ Int16Dtype: Int16Override,
74
+ Int32Dtype: Int32Override,
75
+ Int64Dtype: Int64Override,
76
+ Float16Dtype: Float16Override,
77
+ Float32Dtype: Float32Override,
78
+ Float64Dtype: Float64Override,
79
+ StringDtype: StringOverride,
80
+ DictionaryDtype: DictionaryOverride,
81
+ }
82
+ DESCRIPTIONS: Final[dict[type[Dtype], str]] = {
83
+ BooleanDtype: f"{BooleanDtype.TRUE} or {BooleanDtype.FALSE}",
84
+ UInt8Dtype: f"Whole non-negative number, {UInt8Dtype.LOWER_BOUND:,} to {UInt8Dtype.HIGHER_BOUND:,}",
85
+ UInt16Dtype: f"Whole non-negative number, {UInt16Dtype.LOWER_BOUND:,} to {UInt16Dtype.HIGHER_BOUND:,}",
86
+ UInt32Dtype: f"Whole non-negative number, {UInt32Dtype.LOWER_BOUND:,} to {float(UInt32Dtype.HIGHER_BOUND):.1E}",
87
+ UInt64Dtype: f"Whole non-negative number, {UInt64Dtype.LOWER_BOUND:,} to {float(UInt64Dtype.HIGHER_BOUND):.1E}",
88
+ Int8Dtype: f"Whole number, {Int8Dtype.LOWER_BOUND:,} to {Int8Dtype.HIGHER_BOUND:,}",
89
+ Int16Dtype: f"Whole number, {Int16Dtype.LOWER_BOUND:,} to {Int16Dtype.HIGHER_BOUND:,}",
90
+ Int32Dtype: f"Whole number, {float(Int32Dtype.LOWER_BOUND):.1E} to {float(Int32Dtype.HIGHER_BOUND):.1E}",
91
+ Int64Dtype: f"Whole number, {float(Int64Dtype.LOWER_BOUND):.1E} to {float(Int64Dtype.HIGHER_BOUND):.1E}",
92
+ Float16Dtype: f"Decimal number, about {Float16Dtype.MAX_SIGNIFICANT_DIGITS} digits of precision, up to {Float16Dtype.MAX_ABSOLUTE_VALUE:.1E}",
93
+ Float32Dtype: f"Decimal number, about {Float32Dtype.MAX_SIGNIFICANT_DIGITS} digits of precision, up to {Float32Dtype.MAX_ABSOLUTE_VALUE:.1E}",
94
+ Float64Dtype: f"Decimal number, about {Float64Dtype.MAX_SIGNIFICANT_DIGITS} digits of precision, up to {Float64Dtype.MAX_ABSOLUTE_VALUE:.1E}",
95
+ StringDtype: "Text",
96
+ DictionaryDtype: "Category, a fixed set of repeating values",
97
+ }
98
+
99
+
100
+ def _label(dtype: type[Dtype], /) -> str:
101
+ return dtype.__name__.removesuffix("Dtype")
102
+
103
+
104
+ def _dtype_option(dtype: type[Dtype], group: str, /) -> DtypeOption:
105
+ return DtypeOption(
106
+ name=dtype.__name__,
107
+ label=_label(dtype),
108
+ description=DESCRIPTIONS[dtype],
109
+ group=group,
110
+ )
111
+
112
+
113
+ def _format_sample(dtype: type[Dtype], value: str, /) -> str:
114
+ if issubclass(dtype, IntegerDtype):
115
+ return f"{int(value):,}"
116
+ if issubclass(dtype, FloatDtype):
117
+ return f"{Decimal(value):,}"
118
+ return value
119
+
120
+
121
+ def _stats(dtype: type[Dtype], profile: Profile, /) -> tuple[tuple[str, str], ...]:
122
+ samples = ", ".join(
123
+ _format_sample(dtype, value) for value in profile.uniques(n=5)
124
+ )
125
+ stats: list[tuple[str, str]] = [
126
+ ("Unique values", f"{profile.num_unique:,}"),
127
+ ("Samples", samples),
128
+ ]
129
+ if issubclass(dtype, (FloatDtype, IntegerDtype)):
130
+ stats.append(("Range", f"{profile.min:,} – {profile.max:,}"))
131
+ if issubclass(dtype, FloatDtype):
132
+ stats.append(("Precision", str(profile.precision)))
133
+ stats.append(("Scale", str(profile.scale)))
134
+ elif issubclass(dtype, StringDtype):
135
+ stats.append(
136
+ ("Length range", f"{profile.min_length:,} – {profile.max_length:,}"),
137
+ )
138
+ return tuple(stats)
139
+
140
+
141
+ def _column_report(column: str, profile: Profile, /) -> ColumnReport:
142
+ dtype: type[Dtype] = type(profile).DTYPE
143
+ options: list[DtypeOption] = [_dtype_option(dtype, GROUP_SAFE)]
144
+ options.extend(
145
+
146
+ _dtype_option(candidate.DTYPE, group_of(candidate.compatibility))
147
+ for candidate in profile.candidates
148
+ if is_selectable(candidate.compatibility)
149
+
150
+ )
151
+ return ColumnReport(
152
+ name=column,
153
+ inferred_dtype=dtype.__name__,
154
+ description=DESCRIPTIONS[dtype],
155
+ num_unique=profile.num_unique,
156
+ total=profile.total,
157
+ samples=profile.uniques(n=5),
158
+ stats=_stats(dtype, profile),
159
+ options=tuple(options),
160
+ )
161
+
162
+
163
+ def _build_override(column: str, selection: ColumnSelection, /) -> Override:
164
+ dtype: type[Dtype] = _DTYPE_BY_NAME[selection.dtype_name]
165
+ override_cls: type[Override] = _OVERRIDE_BY_DTYPE[dtype]
166
+ if dtype is BooleanDtype:
167
+ if selection.true_value is None or selection.false_value is None:
168
+ msg = f"Column {column!r} needs both a true and a false value to become Boolean."
169
+ raise ValueError(msg)
170
+ return BooleanOverride(
171
+ column, true_value=selection.true_value, false_value=selection.false_value,
172
+ )
173
+ if dtype is DictionaryDtype:
174
+ if selection.categories is None:
175
+ msg = f"Column {column!r} needs a category list to become Dictionary."
176
+ raise ValueError(msg)
177
+ return DictionaryOverride(
178
+ column, *selection.categories, ordered=selection.ordered,
179
+ )
180
+ return override_cls(column)
181
+
182
+
183
+ class DtypeflowService:
184
+ def load_dataset(self, path: str | Path, /) -> LoadedDataset:
185
+ dataset: Dataset = profile_dataset(path)
186
+ columns = tuple(
187
+ _column_report(column, dataset[column]) for column in dataset.columns
188
+ )
189
+ report = DatasetReport(source=str(dataset.source), columns=columns)
190
+ return LoadedDataset(report=report, dataset_ref=dataset)
191
+
192
+ def unique_values(self, loaded: LoadedDataset, column: str, /) -> tuple[str, ...]:
193
+ dataset = cast("Dataset", loaded.dataset_ref)
194
+ return dataset[column].uniques()
195
+
196
+ def export(
197
+ self,
198
+ loaded: LoadedDataset,
199
+ destination: str | Path,
200
+ selections: Mapping[str, ColumnSelection],
201
+ /,
202
+ ) -> tuple[tuple[str, str], ...]:
203
+ dataset = cast("Dataset", loaded.dataset_ref)
204
+ overrides = tuple(
205
+
206
+ _build_override(column, selection)
207
+ for (column, selection) in selections.items()
208
+
209
+ )
210
+ destination = Path(destination)
211
+ Schema(*overrides).apply(dataset, destination=destination)
212
+ written = pq.read_table(destination).schema
213
+ return tuple((field.name, str(field.type)) for field in written)
dtypeflow/utils.py ADDED
@@ -0,0 +1,32 @@
1
+ from __future__ import annotations
2
+
3
+ from csv import DictReader
4
+ from pathlib import Path
5
+
6
+ from dtypeflow.core.table import Table
7
+
8
+
9
+ def read_csv(path: str | Path, /, *, delimiter: str = ",", newline: str = "") -> Table:
10
+ if isinstance(path, str):
11
+ if not path:
12
+ msg = "The path to the CSV file cannot be empty."
13
+ raise ValueError(msg)
14
+ path: Path = Path(path)
15
+ if not path.exists():
16
+ msg = f"The CSV file at path '{path}' does not exist."
17
+ raise FileNotFoundError(msg)
18
+ if not path.is_file():
19
+ msg = f"The path '{path}' is not a valid file."
20
+ raise ValueError(msg)
21
+ if path.suffix.lower() != ".csv":
22
+ msg = f"The file '{path}' is not a CSV file."
23
+ raise ValueError(msg)
24
+ with path.open(newline=newline) as file:
25
+ reader: DictReader[str] = DictReader(file, delimiter=delimiter)
26
+ rows: list[dict[str, str]] = list(reader)
27
+ return Table(
28
+ **{
29
+ name: tuple(row[name] for row in rows)
30
+ for name in reader.fieldnames or ()
31
+ },
32
+ )
@@ -0,0 +1,93 @@
1
+ Metadata-Version: 2.4
2
+ Name: dtypeflow
3
+ Version: 1.0.0
4
+ Summary: Terminal UI to infer CSV column dtypes and convert CSV to Parquet, locally.
5
+ Keywords: csv,parquet,dtype,tui,textual,arrow
6
+ Author: Pedro José Mora Gallegos
7
+ Author-email: Pedro José Mora Gallegos <developmentbypedrojose@outlook.de>
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Environment :: Console
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Topic :: Utilities
17
+ Classifier: Topic :: Scientific/Engineering
18
+ Requires-Dist: pyarrow>=17.0.0
19
+ Requires-Dist: textual>=0.89.0
20
+ Requires-Python: >=3.11
21
+ Project-URL: Homepage, https://github.com/pedrojosemoragallegos/dtypeflow
22
+ Project-URL: Repository, https://github.com/pedrojosemoragallegos/dtypeflow
23
+ Project-URL: Issues, https://github.com/pedrojosemoragallegos/dtypeflow/issues
24
+ Description-Content-Type: text/markdown
25
+
26
+ # dtypeflow
27
+
28
+ `dtypeflow` is a terminal UI to convert a CSV file to Parquet.
29
+
30
+ Point it at a CSV file, and for each column it infers the tightest-fitting
31
+ dtype (`Boolean`, unsigned/signed integers `UInt8`…`UInt64`/`Int8`…`Int64`,
32
+ floats `Float16`/`Float32`/`Float64`, `String`, or `Dictionary`), shows the
33
+ observed statistics, and lets you review or override the choice before
34
+ writing the result to Parquet — all locally, no data leaves your machine.
35
+
36
+ ## Installation
37
+
38
+ ```bash
39
+ uv tool install dtypeflow
40
+ ```
41
+
42
+ Or with `pipx`:
43
+
44
+ ```bash
45
+ pipx install dtypeflow
46
+ ```
47
+
48
+ Or with `pip`:
49
+
50
+ ```bash
51
+ pip install dtypeflow
52
+ ```
53
+
54
+ ## Usage
55
+
56
+ Launch the picker and choose a CSV file interactively:
57
+
58
+ ```bash
59
+ dtypeflow
60
+ ```
61
+
62
+ Or pass a path directly to skip straight to configuring column types:
63
+
64
+ ```bash
65
+ dtypeflow path/to/data.csv
66
+ ```
67
+
68
+ The workflow has three steps:
69
+
70
+ 1. **Pick a file** — enter or browse for a CSV path.
71
+ 2. **Configure types** — for each column, review the recommended dtype, its
72
+ stats (unique values, samples, range/precision/scale), and pick from
73
+ compatible alternatives grouped by safety (Safe / Too small / Discouraged).
74
+ 3. **Export** — write the configured dataset to a Parquet file.
75
+
76
+ ## Development
77
+
78
+ This project uses [uv](https://docs.astral.sh/uv/) for dependency management.
79
+
80
+ ```bash
81
+ uv sync
82
+ uv run dtypeflow
83
+ ```
84
+
85
+ Lint:
86
+
87
+ ```bash
88
+ uv run --with ruff ruff check src/
89
+ ```
90
+
91
+ ## License
92
+
93
+ [MIT](LICENSE)
@@ -0,0 +1,43 @@
1
+ dtypeflow/__init__.py,sha256=42mpdAvj8VsnaMJYL6O8zTWiwokd80lEQ7LWJENCHaw,114
2
+ dtypeflow/core/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
3
+ dtypeflow/core/candidate/__init__.py,sha256=g7tif4In9KVcvMgeVX_lsoGD3xgM8wtR899gnIu0s2k,996
4
+ dtypeflow/core/candidate/base.py,sha256=jOUkmGwtUPLo_gi-z_lIX_MHvpNqq__YVcV64-TjZnY,972
5
+ dtypeflow/core/candidate/factory.py,sha256=zatB4ve4fVT_o7SW3FmqoMMg4Q08FYD4TrswAzXFl6k,7672
6
+ dtypeflow/core/candidate/primitive.py,sha256=MbPVpyt32OL8DxNVnkLkipW97nHgAwBEhiD7G39u9HI,1377
7
+ dtypeflow/core/dataset.py,sha256=X1PQWRXsyvYuaaVN0iLAb2ffhWSs_a2_WTIOBPfRAmI,1356
8
+ dtypeflow/core/dtype/__init__.py,sha256=NKrfx0kFygJnewD9eMcOdtevKpjD98oEdkmUEa73yRw,1642
9
+ dtypeflow/core/dtype/base.py,sha256=EYcwO5oE2w6E-UCReHczgGmpbCUqYSsRgggdzmXXSJA,255
10
+ dtypeflow/core/dtype/hierachy.md,sha256=Nqpky_L6GdWRNW4fO_pfKaAPfW03eFNtStRK-YwJcD8,2040
11
+ dtypeflow/core/dtype/nested/__init__.py,sha256=hLj8pD6YuGT29_Q1CeFLcTa4SqQKOqI8OG0cRg7axG0,132
12
+ dtypeflow/core/dtype/nested/dictionary.py,sha256=njg9lRN4Pw2Br2yOY04Qas0YJqLe35zk1j7dpdFDn9Y,271
13
+ dtypeflow/core/dtype/primitive/__init__.py,sha256=AvFuCtdt5slYOugbSiucL3aCXrRkSZ_ZhjP_Y4U2qFU,740
14
+ dtypeflow/core/dtype/primitive/base.py,sha256=GqhcoqMKnD9KAW9NrmfjeUdPJD62N2X09HerdXXzw0Y,110
15
+ dtypeflow/core/dtype/primitive/boolean.py,sha256=8RiEDwSyZ42IIHzf5wpaYKxImCkVMgi2JhcMNd_HOtw,432
16
+ dtypeflow/core/dtype/primitive/float.py,sha256=f7f5_VLLSA2fcLD7pfeFa_W-8TngFnGYJuPoMbgbseU,991
17
+ dtypeflow/core/dtype/primitive/integer/__init__.py,sha256=54bdf3_PpdLz9gpm6abmxbXNK2N_DGlWlu1UtUK5-tI,622
18
+ dtypeflow/core/dtype/primitive/integer/base.py,sha256=pGCngH5O4CdBqTFLZYRxKi2fsMVgl0ZvUJnLsFutufc,445
19
+ dtypeflow/core/dtype/primitive/integer/signed.py,sha256=pnNMITMMw-YsORGqSEMGBA3kLTLyPVLOpY9kzIOn76k,768
20
+ dtypeflow/core/dtype/primitive/integer/unsigned.py,sha256=ks0o2Prmq2M3dzWB5Sell_oVTqou8R-lG7Z2ZH7zICw,746
21
+ dtypeflow/core/dtype/primitive/string.py,sha256=tqz8np4VfjKZJqJAHdKYzAZke-3h4YpYONi-Z9ul9AM,255
22
+ dtypeflow/core/override/__init__.py,sha256=TFfzxo_ZNBVFUgiWVe5xRFia7YMLYGQ6dOCqSCYl-p0,970
23
+ dtypeflow/core/override/base.py,sha256=ALzi2yTvqlFQ56-hikd1bD289qn7XN5EFiu1uvHkIIs,507
24
+ dtypeflow/core/override/primitive.py,sha256=9bCP7fXKkmZBY3dDCxrWy1ny-LMMCSuXSXRurl33C2c,5712
25
+ dtypeflow/core/profile/__init__.py,sha256=JqyT6sqFWLnh3fBWfcm1tQjhHT8fQQnlhuTbHkHU3-8,874
26
+ dtypeflow/core/profile/base.py,sha256=qtRfKMqm65JoQR9qufekxesc2Sw_7T8ptfcj-B-_M2U,1489
27
+ dtypeflow/core/profile/primitive.py,sha256=YWap_IB5w2AOQrZ3NvXiSr75JwSB2UdSaj2i-RTAsTc,4130
28
+ dtypeflow/core/profiler.py,sha256=EI3qrR3I5UV-jLJT_Xv8fEFh5G_FPdzpvdhXMYs62vc,5123
29
+ dtypeflow/core/schema.py,sha256=CMfyZbaaLsSITYRRdmYuefaFk_Xj-VxB6oXZ-xQWeC4,1173
30
+ dtypeflow/core/table.py,sha256=07cJKLYz2USEgHqo1o4LN6QwL0MsWtHGnkBbYHhjnLE,833
31
+ dtypeflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
32
+ dtypeflow/tui/__init__.py,sha256=fVjm0bBy49pMsLreE2h9NUIPA3l5gr3Nw4S3JdfTLX4,469
33
+ dtypeflow/tui/app.py,sha256=8Lic3uG69phm1tjJugjTCTOpCKQwbmhCAYw1ZMIVKXs,26952
34
+ dtypeflow/tui/app.tcss,sha256=zGCT_-OyzEpAcLwS50gTpc5FGVqeKvvEyr6VPH7xNN4,3216
35
+ dtypeflow/tui/compatibility.py,sha256=t2x3h1uISJq35HfoBZZJnzpME5du9tA-7glap22EvrU,1321
36
+ dtypeflow/tui/dto.py,sha256=eU9pgRkDIC9va6h0q3iSDy3YpwF3sSCv3K5oj_bsMxE,847
37
+ dtypeflow/tui/service.py,sha256=4hlAWNiIMkwkuQ2x-85n6HMUC_GbfwIFL3HVpC7KOwU,7536
38
+ dtypeflow/utils.py,sha256=rBHJy1hMEKGDr08vsQo91PIW8xNf4LAU0LtxHtLcUL4,1095
39
+ dtypeflow-1.0.0.dist-info/licenses/LICENSE,sha256=tr5T8mamK1bEK0fgZrIlLs2qbJDJmL-_zTZ7HpiPvVA,1082
40
+ dtypeflow-1.0.0.dist-info/WHEEL,sha256=mru_b36sH6joUMnwf7IlFCun3RoDjrNg9RcfBmEcqsE,81
41
+ dtypeflow-1.0.0.dist-info/entry_points.txt,sha256=HoEXQsH0UUf7ReHGhDcWliO1uX7VzblTGQoIlk5cOCg,49
42
+ dtypeflow-1.0.0.dist-info/METADATA,sha256=cSNYfEZHgEnuDeFAZjfY8-YgXpkwNj35yO9FztcZVAM,2453
43
+ dtypeflow-1.0.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: uv 0.12.11
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,3 @@
1
+ [console_scripts]
2
+ dtypeflow = dtypeflow.tui:run
3
+