dtypeflow 1.0.0__tar.gz

dtypeflow-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Pedro José Mora Gallegos
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dtypeflow-1.0.0/PKG-INFO

@@ -0,0 +1,93 @@
+Metadata-Version: 2.4
+Name: dtypeflow
+Version: 1.0.0
+Summary: Terminal UI to infer CSV column dtypes and convert CSV to Parquet, locally.
+Keywords: csv,parquet,dtype,tui,textual,arrow
+Author: Pedro José Mora Gallegos
+Author-email: Pedro José Mora Gallegos <developmentbypedrojose@outlook.de>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Utilities
+Classifier: Topic :: Scientific/Engineering
+Requires-Dist: pyarrow>=17.0.0
+Requires-Dist: textual>=0.89.0
+Requires-Python: >=3.11
+Project-URL: Homepage, https://github.com/pedrojosemoragallegos/dtypeflow
+Project-URL: Repository, https://github.com/pedrojosemoragallegos/dtypeflow
+Project-URL: Issues, https://github.com/pedrojosemoragallegos/dtypeflow/issues
+Description-Content-Type: text/markdown
+
+# dtypeflow
+
+`dtypeflow` is a terminal UI to convert a CSV file to Parquet.
+
+Point it at a CSV file, and for each column it infers the tightest-fitting
+dtype (`Boolean`, unsigned/signed integers `UInt8`…`UInt64`/`Int8`…`Int64`,
+floats `Float16`/`Float32`/`Float64`, `String`, or `Dictionary`), shows the
+observed statistics, and lets you review or override the choice before
+writing the result to Parquet — all locally, no data leaves your machine.
+
+## Installation
+
+```bash
+uv tool install dtypeflow
+```
+
+Or with `pipx`:
+
+```bash
+pipx install dtypeflow
+```
+
+Or with `pip`:
+
+```bash
+pip install dtypeflow
+```
+
+## Usage
+
+Launch the picker and choose a CSV file interactively:
+
+```bash
+dtypeflow
+```
+
+Or pass a path directly to skip straight to configuring column types:
+
+```bash
+dtypeflow path/to/data.csv
+```
+
+The workflow has three steps:
+
+1. **Pick a file** — enter or browse for a CSV path.
+2. **Configure types** — for each column, review the recommended dtype, its
+   stats (unique values, samples, range/precision/scale), and pick from
+   compatible alternatives grouped by safety (Safe / Too small / Discouraged).
+3. **Export** — write the configured dataset to a Parquet file.
+
+## Development
+
+This project uses [uv](https://docs.astral.sh/uv/) for dependency management.
+
+```bash
+uv sync
+uv run dtypeflow
+```
+
+Lint:
+
+```bash
+uv run --with ruff ruff check src/
+```
+
+## License
+
+[MIT](LICENSE)

dtypeflow-1.0.0/README.md

@@ -0,0 +1,68 @@
+# dtypeflow
+
+`dtypeflow` is a terminal UI to convert a CSV file to Parquet.
+
+Point it at a CSV file, and for each column it infers the tightest-fitting
+dtype (`Boolean`, unsigned/signed integers `UInt8`…`UInt64`/`Int8`…`Int64`,
+floats `Float16`/`Float32`/`Float64`, `String`, or `Dictionary`), shows the
+observed statistics, and lets you review or override the choice before
+writing the result to Parquet — all locally, no data leaves your machine.
+
+## Installation
+
+```bash
+uv tool install dtypeflow
+```
+
+Or with `pipx`:
+
+```bash
+pipx install dtypeflow
+```
+
+Or with `pip`:
+
+```bash
+pip install dtypeflow
+```
+
+## Usage
+
+Launch the picker and choose a CSV file interactively:
+
+```bash
+dtypeflow
+```
+
+Or pass a path directly to skip straight to configuring column types:
+
+```bash
+dtypeflow path/to/data.csv
+```
+
+The workflow has three steps:
+
+1. **Pick a file** — enter or browse for a CSV path.
+2. **Configure types** — for each column, review the recommended dtype, its
+   stats (unique values, samples, range/precision/scale), and pick from
+   compatible alternatives grouped by safety (Safe / Too small / Discouraged).
+3. **Export** — write the configured dataset to a Parquet file.
+
+## Development
+
+This project uses [uv](https://docs.astral.sh/uv/) for dependency management.
+
+```bash
+uv sync
+uv run dtypeflow
+```
+
+Lint:
+
+```bash
+uv run --with ruff ruff check src/
+```
+
+## License
+
+[MIT](LICENSE)

dtypeflow-1.0.0/pyproject.toml

@@ -0,0 +1,46 @@
+[project]
+name = "dtypeflow"
+version = "1.0.0"
+description = "Terminal UI to infer CSV column dtypes and convert CSV to Parquet, locally."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.11"
+keywords = [
+    "csv",
+    "parquet",
+    "dtype",
+    "tui",
+    "textual",
+    "arrow",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Utilities",
+    "Topic :: Scientific/Engineering",
+]
+dependencies = [
+    "pyarrow>=17.0.0",
+    "textual>=0.89.0",
+]
+
+[[project.authors]]
+name = "Pedro José Mora Gallegos"
+email = "developmentbypedrojose@outlook.de"
+
+[project.urls]
+Homepage = "https://github.com/pedrojosemoragallegos/dtypeflow"
+Repository = "https://github.com/pedrojosemoragallegos/dtypeflow"
+Issues = "https://github.com/pedrojosemoragallegos/dtypeflow/issues"
+
+[project.scripts]
+dtypeflow = "dtypeflow.tui:run"
+
+[build-system]
+requires = ["uv_build>=0.12.9,<0.13.0"]
+build-backend = "uv_build"

dtypeflow-1.0.0/pyproject.toml.orig

@@ -0,0 +1,35 @@
+[project]
+name = "dtypeflow"
+version = "1.0.0"
+description = "Terminal UI to infer CSV column dtypes and convert CSV to Parquet, locally."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [
+    { name = "Pedro José Mora Gallegos", email = "developmentbypedrojose@outlook.de" },
+]
+requires-python = ">=3.11"
+keywords = ["csv", "parquet", "dtype", "tui", "textual", "arrow"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Utilities",
+    "Topic :: Scientific/Engineering",
+]
+dependencies = ["pyarrow>=17.0.0", "textual>=0.89.0"]
+
+[project.urls]
+Homepage = "https://github.com/pedrojosemoragallegos/dtypeflow"
+Repository = "https://github.com/pedrojosemoragallegos/dtypeflow"
+Issues = "https://github.com/pedrojosemoragallegos/dtypeflow/issues"
+
+[project.scripts]
+dtypeflow = "dtypeflow.tui:run"
+
+[build-system]
+requires = ["uv_build>=0.12.9,<0.13.0"]
+build-backend = "uv_build"

dtypeflow-1.0.0/src/dtypeflow/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from dtypeflow.core.profiler import profile
+
+__all__: list[str] = ["profile"]

dtypeflow-1.0.0/src/dtypeflow/core/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

dtypeflow-1.0.0/src/dtypeflow/core/candidate/__init__.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from .base import Candidate, Compatibility
+from .factory import candidate_factory
+from .primitive import (
+    Boolean as BooleanCandidate,
+    Dictionary as DictionaryCandidate,
+    Float16 as Float16Candidate,
+    Float32 as Float32Candidate,
+    Float64 as Float64Candidate,
+    Int8 as Int8Candidate,
+    Int16 as Int16Candidate,
+    Int32 as Int32Candidate,
+    Int64 as Int64Candidate,
+    String as StringCandidate,
+    UInt8 as UInt8Candidate,
+    UInt16 as UInt16Candidate,
+    UInt32 as UInt32Candidate,
+    UInt64 as UInt64Candidate,
+)
+
+__all__: list[str] = [
+    "BooleanCandidate",
+    "Candidate",
+    "Compatibility",
+    "DictionaryCandidate",
+    "Float16Candidate",
+    "Float32Candidate",
+    "Float64Candidate",
+    "Int8Candidate",
+    "Int16Candidate",
+    "Int32Candidate",
+    "Int64Candidate",
+    "StringCandidate",
+    "UInt8Candidate",
+    "UInt16Candidate",
+    "UInt32Candidate",
+    "UInt64Candidate",
+    "candidate_factory",
+]

dtypeflow-1.0.0/src/dtypeflow/core/candidate/base.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from abc import ABC
+from enum import Enum, auto
+from typing import TYPE_CHECKING, ClassVar
+
+if TYPE_CHECKING:
+    from dtypeflow.core.dtype import Dtype
+
+
+class Compatibility(Enum):
+    FULL_RANGE_FIT = auto()
+    VALUES_PRESERVED = auto()
+    VALUES_NO_FIT = auto()
+    RANGE_NO_FIT = auto()
+    UNEVALUATED_NO_FIT = auto()
+    NOT_NUMERIC = auto()
+    OBSERVED_ONLY_FIT = auto()
+    DECIMAL_TRUNCATION = auto()
+    PRECISION_REDUCTION = auto()
+    BOOLEAN_MAPPING = auto()
+    DICTIONARY_ENCODING = auto()
+    DICTIONARY_INEFFICIENT = auto()
+
+
+class Candidate(ABC):
+    DTYPE: ClassVar[type[Dtype]]
+
+    def __init__(self, compatibility: Compatibility, /) -> None:
+        self._compatibility: Compatibility = compatibility
+
+    @property
+    def compatibility(self) -> Compatibility:
+        return self._compatibility
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}(compatibility={self._compatibility.name})"

dtypeflow-1.0.0/src/dtypeflow/core/candidate/factory.py

@@ -0,0 +1,258 @@
+from __future__ import annotations
+
+from decimal import Decimal
+from typing import Final, overload
+
+from dtypeflow.core.dtype import (
+    ALL_TYPES,
+    BooleanDtype,
+    DictionaryDtype,
+    Dtype,
+    Float16Dtype,
+    Float32Dtype,
+    Float64Dtype,
+    FloatDtype,
+    Int8Dtype,
+    Int16Dtype,
+    Int32Dtype,
+    Int64Dtype,
+    IntegerDtype,
+    StringDtype,
+    UInt8Dtype,
+    UInt16Dtype,
+    UInt32Dtype,
+    UInt64Dtype,
+)
+
+from .base import Candidate, Compatibility
+from .primitive import (
+    Boolean as BooleanCandidate,
+    Dictionary as DictionaryCandidate,
+    Float16 as Float16Candidate,
+    Float32 as Float32Candidate,
+    Float64 as Float64Candidate,
+    Int8 as Int8Candidate,
+    Int16 as Int16Candidate,
+    Int32 as Int32Candidate,
+    Int64 as Int64Candidate,
+    String as StringCandidate,
+    UInt8 as UInt8Candidate,
+    UInt16 as UInt16Candidate,
+    UInt32 as UInt32Candidate,
+    UInt64 as UInt64Candidate,
+)
+
+_MAX_BOOLEAN_UNIQUE: Final[int] = 2
+_MAX_DICTIONARY_ABSOLUTE: Final[int] = 20
+_MAX_DICTIONARY_RATIO: Final[float] = 0.05
+_CANDIDATE_MAPPER: Final[dict[type[Dtype], type[Candidate]]] = {
+    BooleanDtype: BooleanCandidate,
+    UInt8Dtype: UInt8Candidate,
+    UInt16Dtype: UInt16Candidate,
+    UInt32Dtype: UInt32Candidate,
+    UInt64Dtype: UInt64Candidate,
+    Int8Dtype: Int8Candidate,
+    Int16Dtype: Int16Candidate,
+    Int32Dtype: Int32Candidate,
+    Int64Dtype: Int64Candidate,
+    Float16Dtype: Float16Candidate,
+    Float32Dtype: Float32Candidate,
+    Float64Dtype: Float64Candidate,
+    StringDtype: StringCandidate,
+    DictionaryDtype: DictionaryCandidate,
+}
+
+
+def _full_bounds(dtype: type[Dtype], /) -> tuple[Decimal, Decimal] | None:
+    if issubclass(dtype, BooleanDtype):
+        return (Decimal(0), Decimal(1))
+    if issubclass(dtype, IntegerDtype):
+        return (Decimal(dtype.LOWER_BOUND), Decimal(dtype.HIGHER_BOUND))
+    if issubclass(dtype, FloatDtype):
+        return (-dtype.MAX_ABSOLUTE_VALUE, dtype.MAX_ABSOLUTE_VALUE)
+    return None
+
+
+def _full_precision(dtype: type[Dtype], /) -> int:
+    if issubclass(dtype, FloatDtype):
+        return dtype.MAX_SIGNIFICANT_DIGITS
+    bounds: tuple[Decimal, Decimal] | None = _full_bounds(dtype)
+    if bounds is None:
+        return 0
+    return len(str(int(max(abs(bounds[0]), abs(bounds[1])))))
+
+
+def _boolean_compatibility(num_unique: int | None, /) -> Compatibility:
+    if num_unique is not None and num_unique <= _MAX_BOOLEAN_UNIQUE:
+        return Compatibility.BOOLEAN_MAPPING
+    return Compatibility.VALUES_NO_FIT
+
+
+def _dictionary_compatibility(
+    num_unique: int | None, total: int | None, /,
+) -> Compatibility:
+    eligible: bool = num_unique is not None and (
+        num_unique <= _MAX_DICTIONARY_ABSOLUTE
+        or (
+            total is not None
+            and total > 0
+            and (num_unique / total <= _MAX_DICTIONARY_RATIO)
+        )
+    )
+    if not eligible:
+        return Compatibility.DICTIONARY_INEFFICIENT
+    return Compatibility.DICTIONARY_ENCODING
+
+
+def _fits_full_range(source: type[Dtype], target: type[Dtype], /) -> bool:
+    source_bounds: tuple[Decimal, Decimal] | None = _full_bounds(source)
+    target_bounds: tuple[Decimal, Decimal] | None = _full_bounds(target)
+    return (
+        source_bounds is not None
+        and target_bounds is not None
+        and (target_bounds[0] <= source_bounds[0])
+        and (source_bounds[1] <= target_bounds[1])
+        and (
+            not issubclass(target, FloatDtype)
+            or _full_precision(source) <= target.MAX_SIGNIFICANT_DIGITS
+        )
+    )
+
+
+def _exceeds_precision(
+    target: type[FloatDtype],
+    min_value: int | Decimal,
+    max_value: int | Decimal,
+    precision: int | None,
+    /,
+) -> bool:
+    if precision is not None:
+        observed_precision: int = precision
+    else:
+        bound: Decimal | int = max(abs(min_value), abs(max_value))
+        observed_precision = len(str(int(bound)))
+    return observed_precision > target.MAX_SIGNIFICANT_DIGITS
+
+
+def _unevaluated_compatibility(*, full_range_fits: bool) -> Compatibility:
+    if full_range_fits:
+        return Compatibility.FULL_RANGE_FIT
+    return Compatibility.UNEVALUATED_NO_FIT
+
+
+def _observed_compatibility(*, full_range_fits: bool) -> Compatibility:
+    if full_range_fits:
+        return Compatibility.FULL_RANGE_FIT
+    return Compatibility.OBSERVED_ONLY_FIT
+
+
+def _range_compatibility(
+    source: type[Dtype],
+    target: type[Dtype],
+    /,
+    *,
+    min_value: int | Decimal | None,
+    max_value: int | Decimal | None,
+    precision: int | None,
+    scale: int | None,
+) -> Compatibility:
+    full_range_fits: bool = _fits_full_range(source, target)
+    target_bounds: tuple[Decimal, Decimal] | None = _full_bounds(target)
+    if min_value is None or max_value is None or target_bounds is None:
+        if issubclass(source, StringDtype):
+            return Compatibility.NOT_NUMERIC
+        return _unevaluated_compatibility(full_range_fits=full_range_fits)
+    if issubclass(target, IntegerDtype) and (scale or 0) > 0:
+        return Compatibility.DECIMAL_TRUNCATION
+    if not (target_bounds[0] <= min_value and max_value <= target_bounds[1]):
+        return Compatibility.RANGE_NO_FIT
+    if issubclass(target, FloatDtype) and _exceeds_precision(
+        target, min_value, max_value, precision,
+    ):
+        return Compatibility.PRECISION_REDUCTION
+    return _observed_compatibility(full_range_fits=full_range_fits)
+
+
+def _compatibility(
+    source: type[Dtype],
+    target: type[Dtype],
+    /,
+    *,
+    num_unique: int | None,
+    total: int | None,
+    min_value: int | Decimal | None,
+    max_value: int | Decimal | None,
+    precision: int | None,
+    scale: int | None,
+) -> Compatibility:
+    if issubclass(target, StringDtype):
+        return Compatibility.VALUES_PRESERVED
+    if issubclass(target, BooleanDtype):
+        return _boolean_compatibility(num_unique)
+    if issubclass(target, DictionaryDtype):
+        return _dictionary_compatibility(num_unique, total)
+    return _range_compatibility(
+        source,
+        target,
+        min_value=min_value,
+        max_value=max_value,
+        precision=precision,
+        scale=scale,
+    )
+
+
+@overload
+def candidate_factory(dtype: type[Dtype], /) -> tuple[Candidate, ...]: ...
+
+
+@overload
+def candidate_factory(
+    dtype: type[Dtype], /, *, num_unique: int, total: int | None = None,
+) -> tuple[Candidate, ...]: ...
+
+
+@overload
+def candidate_factory(
+    dtype: type[Dtype],
+    /,
+    *,
+    num_unique: int,
+    total: int | None = None,
+    min_value: int | Decimal,
+    max_value: int | Decimal,
+    precision: int | None = None,
+    scale: int | None = None,
+) -> tuple[Candidate, ...]: ...
+
+
+def candidate_factory(
+    dtype: type[Dtype],
+    /,
+    *,
+    num_unique: int | None = None,
+    total: int | None = None,
+    min_value: int | Decimal | None = None,
+    max_value: int | Decimal | None = None,
+    precision: int | None = None,
+    scale: int | None = None,
+) -> tuple[Candidate, ...]:
+    if dtype not in ALL_TYPES:
+        msg = f"Unsupported dtype: {dtype}"
+        raise ValueError(msg)
+    candidate_dtypes: tuple[type[Dtype], ...] = tuple(
+        target for target in ALL_TYPES if target is not dtype
+    )
+    candidates: list[Candidate] = []
+    for target in candidate_dtypes:
+        compatibility: Compatibility = _compatibility(
+            dtype,
+            target,
+            num_unique=num_unique,
+            total=total,
+            min_value=min_value,
+            max_value=max_value,
+            precision=precision,
+            scale=scale,
+        )
+        candidates.append(_CANDIDATE_MAPPER[target](compatibility))
+    return tuple(candidates)

dtypeflow-1.0.0/src/dtypeflow/core/candidate/primitive.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from typing import Final
+
+from dtypeflow.core.dtype import (
+    BooleanDtype,
+    DictionaryDtype,
+    Dtype,
+    Float16Dtype,
+    Float32Dtype,
+    Float64Dtype,
+    Int8Dtype,
+    Int16Dtype,
+    Int32Dtype,
+    Int64Dtype,
+    StringDtype,
+    UInt8Dtype,
+    UInt16Dtype,
+    UInt32Dtype,
+    UInt64Dtype,
+)
+
+from .base import Candidate
+
+
+class Boolean(Candidate):
+    DTYPE: Final[type[Dtype]] = BooleanDtype
+
+
+class UInt8(Candidate):
+    DTYPE: Final[type[Dtype]] = UInt8Dtype
+
+
+class UInt16(Candidate):
+    DTYPE: Final[type[Dtype]] = UInt16Dtype
+
+
+class UInt32(Candidate):
+    DTYPE: Final[type[Dtype]] = UInt32Dtype
+
+
+class UInt64(Candidate):
+    DTYPE: Final[type[Dtype]] = UInt64Dtype
+
+
+class Int8(Candidate):
+    DTYPE: Final[type[Dtype]] = Int8Dtype
+
+
+class Int16(Candidate):
+    DTYPE: Final[type[Dtype]] = Int16Dtype
+
+
+class Int32(Candidate):
+    DTYPE: Final[type[Dtype]] = Int32Dtype
+
+
+class Int64(Candidate):
+    DTYPE: Final[type[Dtype]] = Int64Dtype
+
+
+class Float16(Candidate):
+    DTYPE: Final[type[Dtype]] = Float16Dtype
+
+
+class Float32(Candidate):
+    DTYPE: Final[type[Dtype]] = Float32Dtype
+
+
+class Float64(Candidate):
+    DTYPE: Final[type[Dtype]] = Float64Dtype
+
+
+class String(Candidate):
+    DTYPE: Final[type[Dtype]] = StringDtype
+
+
+class Dictionary(Candidate):
+    DTYPE: Final[type[Dtype]] = DictionaryDtype