dtypeflow 1.0.0__py3-none-any.whl

dtypeflow/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from dtypeflow.core.profiler import profile
+
+__all__: list[str] = ["profile"]

dtypeflow/core/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

dtypeflow/core/candidate/__init__.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from .base import Candidate, Compatibility
+from .factory import candidate_factory
+from .primitive import (
+    Boolean as BooleanCandidate,
+    Dictionary as DictionaryCandidate,
+    Float16 as Float16Candidate,
+    Float32 as Float32Candidate,
+    Float64 as Float64Candidate,
+    Int8 as Int8Candidate,
+    Int16 as Int16Candidate,
+    Int32 as Int32Candidate,
+    Int64 as Int64Candidate,
+    String as StringCandidate,
+    UInt8 as UInt8Candidate,
+    UInt16 as UInt16Candidate,
+    UInt32 as UInt32Candidate,
+    UInt64 as UInt64Candidate,
+)
+
+__all__: list[str] = [
+    "BooleanCandidate",
+    "Candidate",
+    "Compatibility",
+    "DictionaryCandidate",
+    "Float16Candidate",
+    "Float32Candidate",
+    "Float64Candidate",
+    "Int8Candidate",
+    "Int16Candidate",
+    "Int32Candidate",
+    "Int64Candidate",
+    "StringCandidate",
+    "UInt8Candidate",
+    "UInt16Candidate",
+    "UInt32Candidate",
+    "UInt64Candidate",
+    "candidate_factory",
+]

dtypeflow/core/candidate/base.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from abc import ABC
+from enum import Enum, auto
+from typing import TYPE_CHECKING, ClassVar
+
+if TYPE_CHECKING:
+    from dtypeflow.core.dtype import Dtype
+
+
+class Compatibility(Enum):
+    FULL_RANGE_FIT = auto()
+    VALUES_PRESERVED = auto()
+    VALUES_NO_FIT = auto()
+    RANGE_NO_FIT = auto()
+    UNEVALUATED_NO_FIT = auto()
+    NOT_NUMERIC = auto()
+    OBSERVED_ONLY_FIT = auto()
+    DECIMAL_TRUNCATION = auto()
+    PRECISION_REDUCTION = auto()
+    BOOLEAN_MAPPING = auto()
+    DICTIONARY_ENCODING = auto()
+    DICTIONARY_INEFFICIENT = auto()
+
+
+class Candidate(ABC):
+    DTYPE: ClassVar[type[Dtype]]
+
+    def __init__(self, compatibility: Compatibility, /) -> None:
+        self._compatibility: Compatibility = compatibility
+
+    @property
+    def compatibility(self) -> Compatibility:
+        return self._compatibility
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}(compatibility={self._compatibility.name})"

dtypeflow/core/candidate/factory.py

@@ -0,0 +1,258 @@
+from __future__ import annotations
+
+from decimal import Decimal
+from typing import Final, overload
+
+from dtypeflow.core.dtype import (
+    ALL_TYPES,
+    BooleanDtype,
+    DictionaryDtype,
+    Dtype,
+    Float16Dtype,
+    Float32Dtype,
+    Float64Dtype,
+    FloatDtype,
+    Int8Dtype,
+    Int16Dtype,
+    Int32Dtype,
+    Int64Dtype,
+    IntegerDtype,
+    StringDtype,
+    UInt8Dtype,
+    UInt16Dtype,
+    UInt32Dtype,
+    UInt64Dtype,
+)
+
+from .base import Candidate, Compatibility
+from .primitive import (
+    Boolean as BooleanCandidate,
+    Dictionary as DictionaryCandidate,
+    Float16 as Float16Candidate,
+    Float32 as Float32Candidate,
+    Float64 as Float64Candidate,
+    Int8 as Int8Candidate,
+    Int16 as Int16Candidate,
+    Int32 as Int32Candidate,
+    Int64 as Int64Candidate,
+    String as StringCandidate,
+    UInt8 as UInt8Candidate,
+    UInt16 as UInt16Candidate,
+    UInt32 as UInt32Candidate,
+    UInt64 as UInt64Candidate,
+)
+
+_MAX_BOOLEAN_UNIQUE: Final[int] = 2
+_MAX_DICTIONARY_ABSOLUTE: Final[int] = 20
+_MAX_DICTIONARY_RATIO: Final[float] = 0.05
+_CANDIDATE_MAPPER: Final[dict[type[Dtype], type[Candidate]]] = {
+    BooleanDtype: BooleanCandidate,
+    UInt8Dtype: UInt8Candidate,
+    UInt16Dtype: UInt16Candidate,
+    UInt32Dtype: UInt32Candidate,
+    UInt64Dtype: UInt64Candidate,
+    Int8Dtype: Int8Candidate,
+    Int16Dtype: Int16Candidate,
+    Int32Dtype: Int32Candidate,
+    Int64Dtype: Int64Candidate,
+    Float16Dtype: Float16Candidate,
+    Float32Dtype: Float32Candidate,
+    Float64Dtype: Float64Candidate,
+    StringDtype: StringCandidate,
+    DictionaryDtype: DictionaryCandidate,
+}
+
+
+def _full_bounds(dtype: type[Dtype], /) -> tuple[Decimal, Decimal] | None:
+    if issubclass(dtype, BooleanDtype):
+        return (Decimal(0), Decimal(1))
+    if issubclass(dtype, IntegerDtype):
+        return (Decimal(dtype.LOWER_BOUND), Decimal(dtype.HIGHER_BOUND))
+    if issubclass(dtype, FloatDtype):
+        return (-dtype.MAX_ABSOLUTE_VALUE, dtype.MAX_ABSOLUTE_VALUE)
+    return None
+
+
+def _full_precision(dtype: type[Dtype], /) -> int:
+    if issubclass(dtype, FloatDtype):
+        return dtype.MAX_SIGNIFICANT_DIGITS
+    bounds: tuple[Decimal, Decimal] | None = _full_bounds(dtype)
+    if bounds is None:
+        return 0
+    return len(str(int(max(abs(bounds[0]), abs(bounds[1])))))
+
+
+def _boolean_compatibility(num_unique: int | None, /) -> Compatibility:
+    if num_unique is not None and num_unique <= _MAX_BOOLEAN_UNIQUE:
+        return Compatibility.BOOLEAN_MAPPING
+    return Compatibility.VALUES_NO_FIT
+
+
+def _dictionary_compatibility(
+    num_unique: int | None, total: int | None, /,
+) -> Compatibility:
+    eligible: bool = num_unique is not None and (
+        num_unique <= _MAX_DICTIONARY_ABSOLUTE
+        or (
+            total is not None
+            and total > 0
+            and (num_unique / total <= _MAX_DICTIONARY_RATIO)
+        )
+    )
+    if not eligible:
+        return Compatibility.DICTIONARY_INEFFICIENT
+    return Compatibility.DICTIONARY_ENCODING
+
+
+def _fits_full_range(source: type[Dtype], target: type[Dtype], /) -> bool:
+    source_bounds: tuple[Decimal, Decimal] | None = _full_bounds(source)
+    target_bounds: tuple[Decimal, Decimal] | None = _full_bounds(target)
+    return (
+        source_bounds is not None
+        and target_bounds is not None
+        and (target_bounds[0] <= source_bounds[0])
+        and (source_bounds[1] <= target_bounds[1])
+        and (
+            not issubclass(target, FloatDtype)
+            or _full_precision(source) <= target.MAX_SIGNIFICANT_DIGITS
+        )
+    )
+
+
+def _exceeds_precision(
+    target: type[FloatDtype],
+    min_value: int | Decimal,
+    max_value: int | Decimal,
+    precision: int | None,
+    /,
+) -> bool:
+    if precision is not None:
+        observed_precision: int = precision
+    else:
+        bound: Decimal | int = max(abs(min_value), abs(max_value))
+        observed_precision = len(str(int(bound)))
+    return observed_precision > target.MAX_SIGNIFICANT_DIGITS
+
+
+def _unevaluated_compatibility(*, full_range_fits: bool) -> Compatibility:
+    if full_range_fits:
+        return Compatibility.FULL_RANGE_FIT
+    return Compatibility.UNEVALUATED_NO_FIT
+
+
+def _observed_compatibility(*, full_range_fits: bool) -> Compatibility:
+    if full_range_fits:
+        return Compatibility.FULL_RANGE_FIT
+    return Compatibility.OBSERVED_ONLY_FIT
+
+
+def _range_compatibility(
+    source: type[Dtype],
+    target: type[Dtype],
+    /,
+    *,
+    min_value: int | Decimal | None,
+    max_value: int | Decimal | None,
+    precision: int | None,
+    scale: int | None,
+) -> Compatibility:
+    full_range_fits: bool = _fits_full_range(source, target)
+    target_bounds: tuple[Decimal, Decimal] | None = _full_bounds(target)
+    if min_value is None or max_value is None or target_bounds is None:
+        if issubclass(source, StringDtype):
+            return Compatibility.NOT_NUMERIC
+        return _unevaluated_compatibility(full_range_fits=full_range_fits)
+    if issubclass(target, IntegerDtype) and (scale or 0) > 0:
+        return Compatibility.DECIMAL_TRUNCATION
+    if not (target_bounds[0] <= min_value and max_value <= target_bounds[1]):
+        return Compatibility.RANGE_NO_FIT
+    if issubclass(target, FloatDtype) and _exceeds_precision(
+        target, min_value, max_value, precision,
+    ):
+        return Compatibility.PRECISION_REDUCTION
+    return _observed_compatibility(full_range_fits=full_range_fits)
+
+
+def _compatibility(
+    source: type[Dtype],
+    target: type[Dtype],
+    /,
+    *,
+    num_unique: int | None,
+    total: int | None,
+    min_value: int | Decimal | None,
+    max_value: int | Decimal | None,
+    precision: int | None,
+    scale: int | None,
+) -> Compatibility:
+    if issubclass(target, StringDtype):
+        return Compatibility.VALUES_PRESERVED
+    if issubclass(target, BooleanDtype):
+        return _boolean_compatibility(num_unique)
+    if issubclass(target, DictionaryDtype):
+        return _dictionary_compatibility(num_unique, total)
+    return _range_compatibility(
+        source,
+        target,
+        min_value=min_value,
+        max_value=max_value,
+        precision=precision,
+        scale=scale,
+    )
+
+
+@overload
+def candidate_factory(dtype: type[Dtype], /) -> tuple[Candidate, ...]: ...
+
+
+@overload
+def candidate_factory(
+    dtype: type[Dtype], /, *, num_unique: int, total: int | None = None,
+) -> tuple[Candidate, ...]: ...
+
+
+@overload
+def candidate_factory(
+    dtype: type[Dtype],
+    /,
+    *,
+    num_unique: int,
+    total: int | None = None,
+    min_value: int | Decimal,
+    max_value: int | Decimal,
+    precision: int | None = None,
+    scale: int | None = None,
+) -> tuple[Candidate, ...]: ...
+
+
+def candidate_factory(
+    dtype: type[Dtype],
+    /,
+    *,
+    num_unique: int | None = None,
+    total: int | None = None,
+    min_value: int | Decimal | None = None,
+    max_value: int | Decimal | None = None,
+    precision: int | None = None,
+    scale: int | None = None,
+) -> tuple[Candidate, ...]:
+    if dtype not in ALL_TYPES:
+        msg = f"Unsupported dtype: {dtype}"
+        raise ValueError(msg)
+    candidate_dtypes: tuple[type[Dtype], ...] = tuple(
+        target for target in ALL_TYPES if target is not dtype
+    )
+    candidates: list[Candidate] = []
+    for target in candidate_dtypes:
+        compatibility: Compatibility = _compatibility(
+            dtype,
+            target,
+            num_unique=num_unique,
+            total=total,
+            min_value=min_value,
+            max_value=max_value,
+            precision=precision,
+            scale=scale,
+        )
+        candidates.append(_CANDIDATE_MAPPER[target](compatibility))
+    return tuple(candidates)

dtypeflow/core/candidate/primitive.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from typing import Final
+
+from dtypeflow.core.dtype import (
+    BooleanDtype,
+    DictionaryDtype,
+    Dtype,
+    Float16Dtype,
+    Float32Dtype,
+    Float64Dtype,
+    Int8Dtype,
+    Int16Dtype,
+    Int32Dtype,
+    Int64Dtype,
+    StringDtype,
+    UInt8Dtype,
+    UInt16Dtype,
+    UInt32Dtype,
+    UInt64Dtype,
+)
+
+from .base import Candidate
+
+
+class Boolean(Candidate):
+    DTYPE: Final[type[Dtype]] = BooleanDtype
+
+
+class UInt8(Candidate):
+    DTYPE: Final[type[Dtype]] = UInt8Dtype
+
+
+class UInt16(Candidate):
+    DTYPE: Final[type[Dtype]] = UInt16Dtype
+
+
+class UInt32(Candidate):
+    DTYPE: Final[type[Dtype]] = UInt32Dtype
+
+
+class UInt64(Candidate):
+    DTYPE: Final[type[Dtype]] = UInt64Dtype
+
+
+class Int8(Candidate):
+    DTYPE: Final[type[Dtype]] = Int8Dtype
+
+
+class Int16(Candidate):
+    DTYPE: Final[type[Dtype]] = Int16Dtype
+
+
+class Int32(Candidate):
+    DTYPE: Final[type[Dtype]] = Int32Dtype
+
+
+class Int64(Candidate):
+    DTYPE: Final[type[Dtype]] = Int64Dtype
+
+
+class Float16(Candidate):
+    DTYPE: Final[type[Dtype]] = Float16Dtype
+
+
+class Float32(Candidate):
+    DTYPE: Final[type[Dtype]] = Float32Dtype
+
+
+class Float64(Candidate):
+    DTYPE: Final[type[Dtype]] = Float64Dtype
+
+
+class String(Candidate):
+    DTYPE: Final[type[Dtype]] = StringDtype
+
+
+class Dictionary(Candidate):
+    DTYPE: Final[type[Dtype]] = DictionaryDtype

dtypeflow/core/dataset.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from collections.abc import Iterator, Mapping
+from pathlib import Path
+from types import MappingProxyType
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from dtypeflow.core.dtype import Dtype
+    from dtypeflow.core.profile import Profile
+
+
+class Dataset(Mapping[str, "Profile"]):
+    def __init__(self, source: str | Path, /, **profiles: Profile) -> None:
+        if isinstance(source, str):
+            source: Path = Path(source)
+        self._source: Path = source
+        self._profiles: MappingProxyType[str, Profile] = MappingProxyType(profiles)
+
+    @property
+    def source(self) -> Path:
+        return self._source
+
+    @property
+    def columns(self) -> tuple[str, ...]:
+        return tuple(self._profiles.keys())
+
+    @property
+    def dtypes(self) -> MappingProxyType[str, type[Dtype]]:
+        return MappingProxyType(
+            {column: profile.DTYPE for (column, profile) in self._profiles.items()},
+        )
+
+    def __getitem__(self, column: str) -> Profile:
+        return self._profiles[column]
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._profiles)
+
+    def __len__(self) -> int:
+        return len(self._profiles)
+
+    def __repr__(self) -> str:
+        return (
+            f"{type(self).__name__}(source={self._source!r}, columns={self.columns!r})"
+        )

dtypeflow/core/dtype/__init__.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from typing import Final
+
+from .base import Dtype
+from .nested import DictionaryDtype
+from .primitive import (
+    BooleanDtype,
+    Float16Dtype,
+    Float32Dtype,
+    Float64Dtype,
+    FloatDtype,
+    Int8Dtype,
+    Int16Dtype,
+    Int32Dtype,
+    Int64Dtype,
+    IntegerDtype,
+    StringDtype,
+    UInt8Dtype,
+    UInt16Dtype,
+    UInt32Dtype,
+    UInt64Dtype,
+)
+from .primitive.integer import SignedDtype, UnsignedDtype
+
+__all__: list[str] = [
+    "ALL_TYPES",
+    "BOOLEAN_TYPES",
+    "FLOAT_TYPES",
+    "INTEGER_TYPES",
+    "SIGNED_INTEGER_TYPES",
+    "STRING_TYPES",
+    "UNSIGNED_INTEGER_TYPES",
+    "BooleanDtype",
+    "DictionaryDtype",
+    "Dtype",
+    "Float16Dtype",
+    "Float32Dtype",
+    "Float64Dtype",
+    "FloatDtype",
+    "Int8Dtype",
+    "Int16Dtype",
+    "Int32Dtype",
+    "Int64Dtype",
+    "IntegerDtype",
+    "StringDtype",
+    "UInt8Dtype",
+    "UInt16Dtype",
+    "UInt32Dtype",
+    "UInt64Dtype",
+]
+BOOLEAN_TYPES: Final[tuple[type[Dtype], ...]] = (BooleanDtype,)
+UNSIGNED_INTEGER_TYPES: Final[tuple[type[IntegerDtype], ...]] = tuple(
+    UnsignedDtype.__subclasses__(),
+)
+SIGNED_INTEGER_TYPES: Final[tuple[type[IntegerDtype], ...]] = tuple(
+    SignedDtype.__subclasses__(),
+)
+INTEGER_TYPES: Final[tuple[type[IntegerDtype], ...]] = (
+    UNSIGNED_INTEGER_TYPES + SIGNED_INTEGER_TYPES
+)
+FLOAT_TYPES: Final[tuple[type[FloatDtype], ...]] = tuple(FloatDtype.__subclasses__())
+STRING_TYPES: Final[tuple[type[Dtype], ...]] = (StringDtype,)
+ALL_TYPES: Final[tuple[type[Dtype], ...]] = (
+    *BOOLEAN_TYPES,
+    DictionaryDtype,
+    *INTEGER_TYPES,
+    *FLOAT_TYPES,
+    *STRING_TYPES,
+)

dtypeflow/core/dtype/base.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Protocol, runtime_checkable
+
+
+class Dtype(ABC):
+    @abstractmethod
+    def __repr__(self) -> str: ...
+
+
+@runtime_checkable
+class Sizeable(Protocol):
+    SIZE: int

dtypeflow/core/dtype/hierachy.md

@@ -0,0 +1,51 @@
+# Hierachie
+
+```markdown
+DataType
+│
+├── Primitive
+│   ├── Boolean                  → 1 bit (stored bitmap; ≈ 0.125 bytes/value)
+│   │
+│   ├── Integer
+│   │   ├── Signed
+│   │   │   ├── Int8             → 1 byte
+│   │   │   ├── Int16            → 2 bytes
+│   │   │   ├── Int32            → 4 bytes
+│   │   │   └── Int64            → 8 bytes
+│   │   │
+│   │   └── Unsigned
+│   │       ├── UInt8            → 1 byte
+│   │       ├── UInt16           → 2 bytes
+│   │       ├── UInt32           → 4 bytes
+│   │       └── UInt64           → 8 bytes
+│   │
+│   ├── Float
+│   │   ├── Float16             → 2 bytes
+│   │   ├── Float32             → 4 bytes
+│   │   └── Float64             → 8 bytes
+│   │
+│   └── Decimal                  → 16 bytes (Decimal128) 32 bytes (Decimal256)
+│
+├── Temporal
+│   ├── Date32                   → 4 bytes
+│   ├── Date64                   → 8 bytes
+│   ├── Time32                   → 4 bytes
+│   ├── Time64                   → 8 bytes
+│   ├── Timestamp                → 8 bytes
+│   └── Duration                 → 8 bytes
+│
+├── Binary
+│   ├── Binary                   → variable-length
+│   ├── LargeBinary              → variable-length
+│   ├── String                   → variable-length (UTF-8)
+│   └── LargeString              → variable-length (UTF-8)
+│
+├── Nested
+│   ├── List                     → variable-length
+│   ├── LargeList                → variable-length
+│   ├── Struct                   → sum of cHIGHER_BOUNDld field SIZEs
+│   ├── Map                      → variable-length
+│   └── Dictionary               → index SIZE (1, 2, 4, or 8 bytes)
+│
+└── Null                         → 0 bytes
+```

dtypeflow/core/dtype/nested/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from .dictionary import Dictionary as DictionaryDtype
+
+__all__: list[str] = ["DictionaryDtype"]

dtypeflow/core/dtype/nested/dictionary.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from typing import final
+
+from typing_extensions import override
+
+from dtypeflow.core.dtype.base import Dtype
+
+
+@final
+class Dictionary(Dtype):
+    @override
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}()"

dtypeflow/core/dtype/primitive/__init__.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from .boolean import Boolean as BooleanDtype
+from .float import (
+    Float as FloatDtype,
+    Float16 as Float16Dtype,
+    Float32 as Float32Dtype,
+    Float64 as Float64Dtype,
+)
+from .integer import (
+    Int8Dtype,
+    Int16Dtype,
+    Int32Dtype,
+    Int64Dtype,
+    IntegerDtype,
+    UInt8Dtype,
+    UInt16Dtype,
+    UInt32Dtype,
+    UInt64Dtype,
+)
+from .string import String as StringDtype
+
+__all__: list[str] = [
+    "BooleanDtype",
+    "Float16Dtype",
+    "Float32Dtype",
+    "Float64Dtype",
+    "FloatDtype",
+    "Int8Dtype",
+    "Int16Dtype",
+    "Int32Dtype",
+    "Int64Dtype",
+    "IntegerDtype",
+    "StringDtype",
+    "UInt8Dtype",
+    "UInt16Dtype",
+    "UInt32Dtype",
+    "UInt64Dtype",
+]

dtypeflow/core/dtype/primitive/base.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from dtypeflow.core.dtype.base import Dtype
+
+
+class Primitive(Dtype): ...

dtypeflow/core/dtype/primitive/boolean.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from typing import Final, final
+
+from typing_extensions import override
+
+from dtypeflow.core.dtype.base import Sizeable
+
+from .base import Primitive
+
+
+@final
+class Boolean(Primitive, Sizeable):
+    SIZE: Final[int] = 1
+    TRUE: Final[bool] = True
+    FALSE: Final[bool] = False
+
+    @override
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.TRUE}, {self.FALSE})"