dtypeflow 1.0.0__py3-none-any.whl

dtypeflow/core/dtype/primitive/float.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from decimal import Decimal
+from typing import ClassVar, Final, final
+
+from typing_extensions import override
+
+from dtypeflow.core.dtype.base import Sizeable
+
+from .base import Primitive
+
+
+class Float(Primitive, Sizeable):
+    MAX_ABSOLUTE_VALUE: ClassVar[Decimal]
+    MAX_SIGNIFICANT_DIGITS: ClassVar[int]
+
+    @override
+    def __repr__(self) -> str:
+        return f"Float(±{self.MAX_ABSOLUTE_VALUE} (≈ {self.MAX_SIGNIFICANT_DIGITS} significant digits))"
+
+
+@final
+class Float16(Float):
+    SIZE: Final[int] = 2
+    MAX_ABSOLUTE_VALUE: Final[Decimal] = Decimal(65504)
+    MAX_SIGNIFICANT_DIGITS: Final[int] = 4
+
+
+@final
+class Float32(Float):
+    SIZE: Final[int] = 4
+    MAX_ABSOLUTE_VALUE: Final[Decimal] = Decimal("3.4028235e38")
+    MAX_SIGNIFICANT_DIGITS: Final[int] = 7
+
+
+@final
+class Float64(Float):
+    SIZE: Final[int] = 8
+    MAX_ABSOLUTE_VALUE: Final[Decimal] = Decimal("1.7976931348623157e308")
+    MAX_SIGNIFICANT_DIGITS: Final[int] = 15

dtypeflow/core/dtype/primitive/integer/__init__.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from .base import Integer as IntegerDtype
+from .signed import (
+    Int8 as Int8Dtype,
+    Int16 as Int16Dtype,
+    Int32 as Int32Dtype,
+    Int64 as Int64Dtype,
+    Signed as SignedDtype,
+)
+from .unsigned import (
+    UInt8 as UInt8Dtype,
+    UInt16 as UInt16Dtype,
+    UInt32 as UInt32Dtype,
+    UInt64 as UInt64Dtype,
+    Unsigned as UnsignedDtype,
+)
+
+__all__: list[str] = [
+    "Int8Dtype",
+    "Int16Dtype",
+    "Int32Dtype",
+    "Int64Dtype",
+    "IntegerDtype",
+    "SignedDtype",
+    "UInt8Dtype",
+    "UInt16Dtype",
+    "UInt32Dtype",
+    "UInt64Dtype",
+    "UnsignedDtype",
+]

dtypeflow/core/dtype/primitive/integer/base.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from typing import ClassVar
+
+from typing_extensions import override
+
+from dtypeflow.core.dtype.base import Sizeable
+from dtypeflow.core.dtype.primitive.base import Primitive
+
+
+class Integer(Primitive, Sizeable):
+    LOWER_BOUND: ClassVar[int]
+    HIGHER_BOUND: ClassVar[int]
+
+    @override
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.LOWER_BOUND} – {self.HIGHER_BOUND})"

dtypeflow/core/dtype/primitive/integer/signed.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import Final, final
+
+from dtypeflow.core.dtype.base import Sizeable
+from dtypeflow.core.dtype.primitive.integer.base import Integer
+
+
+class Signed(Integer, Sizeable): ...
+
+
+@final
+class Int8(Signed):
+    SIZE: Final[int] = 1
+    LOWER_BOUND: Final[int] = -128
+    HIGHER_BOUND: Final[int] = 127
+
+
+@final
+class Int16(Signed):
+    SIZE: Final[int] = 2
+    LOWER_BOUND: Final[int] = -32768
+    HIGHER_BOUND: Final[int] = 32767
+
+
+@final
+class Int32(Signed):
+    SIZE: Final[int] = 4
+    LOWER_BOUND: Final[int] = -2147483648
+    HIGHER_BOUND: Final[int] = 2147483647
+
+
+@final
+class Int64(Signed):
+    SIZE: Final[int] = 8
+    LOWER_BOUND: Final[int] = -9223372036854775808
+    HIGHER_BOUND: Final[int] = 9223372036854775807

dtypeflow/core/dtype/primitive/integer/unsigned.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import Final, final
+
+from dtypeflow.core.dtype.base import Sizeable
+from dtypeflow.core.dtype.primitive.integer.base import Integer
+
+
+class Unsigned(Integer, Sizeable): ...
+
+
+@final
+class UInt8(Unsigned):
+    SIZE: Final[int] = 1
+    LOWER_BOUND: Final[int] = 0
+    HIGHER_BOUND: Final[int] = 255
+
+
+@final
+class UInt16(Unsigned):
+    SIZE: Final[int] = 2
+    LOWER_BOUND: Final[int] = 0
+    HIGHER_BOUND: Final[int] = 65535
+
+
+@final
+class UInt32(Unsigned):
+    SIZE: Final[int] = 4
+    LOWER_BOUND: Final[int] = 0
+    HIGHER_BOUND: Final[int] = 4294967295
+
+
+@final
+class UInt64(Unsigned):
+    SIZE: Final[int] = 8
+    LOWER_BOUND: Final[int] = 0
+    HIGHER_BOUND: Final[int] = 18446744073709551615

dtypeflow/core/dtype/primitive/string.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from typing import final
+
+from typing_extensions import override
+
+from .base import Primitive
+
+
+@final
+class String(Primitive):
+    @override
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}()"

dtypeflow/core/override/__init__.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from .base import Override
+from .primitive import (
+    Boolean as BooleanOverride,
+    Dictionary as DictionaryOverride,
+    Float as FloatOverride,
+    Float16 as Float16Override,
+    Float32 as Float32Override,
+    Float64 as Float64Override,
+    Int8 as Int8Override,
+    Int16 as Int16Override,
+    Int32 as Int32Override,
+    Int64 as Int64Override,
+    Integer as IntegerOverride,
+    String as StringOverride,
+    UInt8 as UInt8Override,
+    UInt16 as UInt16Override,
+    UInt32 as UInt32Override,
+    UInt64 as UInt64Override,
+)
+
+__all__: list[str] = [
+    "BooleanOverride",
+    "DictionaryOverride",
+    "Float16Override",
+    "Float32Override",
+    "Float64Override",
+    "FloatOverride",
+    "Int8Override",
+    "Int16Override",
+    "Int32Override",
+    "Int64Override",
+    "IntegerOverride",
+    "Override",
+    "StringOverride",
+    "UInt8Override",
+    "UInt16Override",
+    "UInt32Override",
+    "UInt64Override",
+]

dtypeflow/core/override/base.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import pyarrow as pa
+
+
+class Override(ABC):
+    def __init__(self, column: str, /) -> None:
+        self._column: str = column
+
+    @property
+    def column(self) -> str:
+        return self._column
+
+    @abstractmethod
+    def transform(self, *values: str | None) -> pa.Array: ...
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}(column={self._column!r})"

dtypeflow/core/override/primitive.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+from decimal import Decimal, InvalidOperation
+
+import pyarrow as pa
+
+from .base import Override
+
+
+class Boolean(Override):
+    def __init__(self, column: str, /, *, true_value: str, false_value: str) -> None:
+        super().__init__(column)
+        if (true_value := true_value.strip().lower()) == (
+            false_value := false_value.strip().lower()
+        ):
+            msg = f"true_value and false_value must differ for column {column!r}: {true_value!r}."
+            raise ValueError(msg)
+        self._true_value: str = true_value
+        self._false_value: str = false_value
+
+    def _coerce(self, value: str | None, /) -> bool | None:
+        if value is None or not value.strip():
+            return None
+        if (value := value.strip().lower()) not in (
+            self._true_value,
+            self._false_value,
+        ):
+            msg = f"{value!r} is neither {self._true_value!r} nor {self._false_value!r} for column {self.column!r}."
+            raise ValueError(msg)
+        return value == self._true_value
+
+    def transform(self, *values: str | None) -> pa.Array:
+        return pa.array([self._coerce(value) for value in values], type=pa.bool_())
+
+
+class Integer(Override):
+    def _coerce(self, value: str | None, /) -> int | None:
+        if value is None or not value.strip():
+            return None
+        stripped = value.strip()
+        try:
+            return int(Decimal(stripped))
+        except InvalidOperation:
+            msg = f"{stripped!r} is not a valid number for column {self.column!r}."
+            raise ValueError(msg) from None
+
+
+class Int8(Integer):
+    def transform(self, *values: str | None) -> pa.Array:
+        return pa.array([self._coerce(value) for value in values], type=pa.int8())
+
+
+class Int16(Integer):
+    def transform(self, *values: str | None) -> pa.Array:
+        return pa.array([self._coerce(value) for value in values], type=pa.int16())
+
+
+class Int32(Integer):
+    def transform(self, *values: str | None) -> pa.Array:
+        return pa.array([self._coerce(value) for value in values], type=pa.int32())
+
+
+class Int64(Integer):
+    def transform(self, *values: str | None) -> pa.Array:
+        return pa.array([self._coerce(value) for value in values], type=pa.int64())
+
+
+class UInt8(Integer):
+    def transform(self, *values: str | None) -> pa.Array:
+        return pa.array([self._coerce(value) for value in values], type=pa.uint8())
+
+
+class UInt16(Integer):
+    def transform(self, *values: str | None) -> pa.Array:
+        return pa.array([self._coerce(value) for value in values], type=pa.uint16())
+
+
+class UInt32(Integer):
+    def transform(self, *values: str | None) -> pa.Array:
+        return pa.array([self._coerce(value) for value in values], type=pa.uint32())
+
+
+class UInt64(Integer):
+    def transform(self, *values: str | None) -> pa.Array:
+        return pa.array([self._coerce(value) for value in values], type=pa.uint64())
+
+
+class Float(Override):
+    def _coerce(self, value: str | None, /) -> float | None:
+        if value is None or not value.strip():
+            return None
+        return float(value.strip())
+
+
+class Float16(Float):
+    def transform(self, *values: str | None) -> pa.Array:
+        return pa.array([self._coerce(value) for value in values], type=pa.float16())
+
+
+class Float32(Float):
+    def transform(self, *values: str | None) -> pa.Array:
+        return pa.array([self._coerce(value) for value in values], type=pa.float32())
+
+
+class Float64(Float):
+    def transform(self, *values: str | None) -> pa.Array:
+        return pa.array([self._coerce(value) for value in values], type=pa.float64())
+
+
+class String(Override):
+    def _coerce(self, value: str | None, /) -> str | None:
+        if value is None or not value.strip():
+            return None
+        return value.strip()
+
+    def transform(self, *values: str | None) -> pa.Array:
+        return pa.array([self._coerce(value) for value in values], type=pa.string())
+
+
+class Dictionary(String):
+    def __init__(self, column: str, /, *categories: str, ordered: bool = False) -> None:
+        super().__init__(column)
+        if not categories:
+            msg = f"categories must be provided for column {column!r}."
+            raise ValueError(msg)
+        if len(set(categories)) != len(categories):
+            msg = f"categories must not contain duplicates for column {column!r}: {categories!r}."
+            raise ValueError(msg)
+        self._ordered: bool = ordered
+        self._categories: tuple[str, ...] = categories
+
+    def _dictionary_index_type(self, num_unique: int, /) -> pa.DataType:
+        if num_unique <= 2**8:
+            return pa.int8()
+        if num_unique <= 2**16:
+            return pa.int16()
+        if num_unique <= 2**32:
+            return pa.int32()
+        return pa.int64()
+
+    def transform(self, *values: str | None) -> pa.Array:
+        coerced: list[str | None] = [self._coerce(value) for value in values]
+        if missing := (
+            {value for value in coerced if value is not None} - set(self._categories)
+        ):
+            msg = f"values {sorted(missing)!r} for column {self.column!r} are not in the provided categories {self._categories!r}."
+            raise ValueError(msg)
+        rank: dict[str, int] = {
+            category: index for (index, category) in enumerate(self._categories)
+        }
+        return pa.DictionaryArray.from_arrays(
+            pa.array(
+                [None if value is None else rank[value] for value in coerced],
+                type=self._dictionary_index_type(len(self._categories)),
+            ),
+            pa.array(self._categories, type=pa.string()),
+            ordered=self._ordered,
+        )

dtypeflow/core/profile/__init__.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from .base import Profile
+from .primitive import (
+    Boolean as BooleanProfile,
+    Float as FloatProfile,
+    Float16 as Float16Profile,
+    Float32 as Float32Profile,
+    Float64 as Float64Profile,
+    Int8 as Int8Profile,
+    Int16 as Int16Profile,
+    Int32 as Int32Profile,
+    Int64 as Int64Profile,
+    Integer as IntegerProfile,
+    String as StringProfile,
+    UInt8 as UInt8Profile,
+    UInt16 as UInt16Profile,
+    UInt32 as UInt32Profile,
+    UInt64 as UInt64Profile,
+)
+
+__all__: list[str] = [
+    "BooleanProfile",
+    "Float16Profile",
+    "Float32Profile",
+    "Float64Profile",
+    "FloatProfile",
+    "Int8Profile",
+    "Int16Profile",
+    "Int32Profile",
+    "Int64Profile",
+    "IntegerProfile",
+    "Profile",
+    "StringProfile",
+    "UInt8Profile",
+    "UInt16Profile",
+    "UInt32Profile",
+    "UInt64Profile",
+]

dtypeflow/core/profile/base.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, ClassVar
+
+if TYPE_CHECKING:
+    from dtypeflow.core.candidate import Candidate
+    from dtypeflow.core.dtype import Dtype
+
+
+class Profile(ABC):
+    DTYPE: ClassVar[type[Dtype]]
+
+    def __init__(
+        self,
+        *values: str | None,
+        uniques: tuple[int, ...],
+        non_nan: tuple[int, ...],
+        nan: tuple[int, ...],
+    ) -> None:
+        self._values: tuple[str | None, ...] = values
+        self._non_nan_indices: tuple[int, ...] = non_nan
+        self._nan_indices: tuple[int, ...] = nan
+        self._uniques: tuple[int, ...] = uniques
+        self._candidates: tuple[Candidate, ...] | None = None
+
+    def uniques(self, *, n: int | None = None) -> tuple[str, ...]:
+        if n is None:
+            return tuple(self._values[index] for index in self._uniques)
+        return tuple(self._values[index] for index in self._uniques[:n])
+
+    @property
+    def num_unique(self) -> int:
+        return len(self._uniques)
+
+    @property
+    def total(self) -> int:
+        return len(self._non_nan_indices)
+
+    @property
+    def candidates(self) -> tuple[Candidate, ...]:
+        if self._candidates is None:
+            self._candidates: tuple[Candidate, ...] = self._compute_candidates()
+        return self._candidates
+
+    @abstractmethod
+    def _compute_candidates(self) -> tuple[Candidate, ...]: ...
+
+    @abstractmethod
+    def __repr__(self) -> str: ...

dtypeflow/core/profile/primitive.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+from decimal import Decimal
+from typing import TYPE_CHECKING, ClassVar, Final, cast
+
+from dtypeflow.core.candidate import candidate_factory
+from dtypeflow.core.dtype import (
+    BooleanDtype,
+    Dtype,
+    Float16Dtype,
+    Float32Dtype,
+    Float64Dtype,
+    FloatDtype,
+    Int8Dtype,
+    Int16Dtype,
+    Int32Dtype,
+    Int64Dtype,
+    IntegerDtype,
+    StringDtype,
+    UInt8Dtype,
+    UInt16Dtype,
+    UInt32Dtype,
+    UInt64Dtype,
+)
+
+from .base import Profile
+
+if TYPE_CHECKING:
+    from dtypeflow.core.candidate import Candidate
+
+
+class Boolean(Profile):
+    DTYPE: Final[type[Dtype]] = BooleanDtype
+
+    def _compute_candidates(self) -> tuple[Candidate, ...]:
+        return candidate_factory(
+            self.DTYPE, num_unique=self.num_unique, total=self.total,
+        )
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}(num_unique={self.num_unique})"
+
+
+class Integer(Profile):
+    DTYPE: ClassVar[type[Dtype]] = IntegerDtype
+
+    @property
+    def min(self) -> int:
+        return min(int(value) for value in self.uniques())
+
+    @property
+    def max(self) -> int:
+        return max(int(value) for value in self.uniques())
+
+    def _compute_candidates(self) -> tuple[Candidate, ...]:
+        return candidate_factory(
+            self.DTYPE,
+            num_unique=self.num_unique,
+            total=self.total,
+            min_value=self.min,
+            max_value=self.max,
+        )
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}(min={self.min}, max={self.max}, num_unique={self.num_unique})"
+
+
+class UInt8(Integer):
+    DTYPE: Final[type[Dtype]] = UInt8Dtype
+
+
+class UInt16(Integer):
+    DTYPE: Final[type[Dtype]] = UInt16Dtype
+
+
+class UInt32(Integer):
+    DTYPE: Final[type[Dtype]] = UInt32Dtype
+
+
+class UInt64(Integer):
+    DTYPE: Final[type[Dtype]] = UInt64Dtype
+
+
+class Int8(Integer):
+    DTYPE: Final[type[Dtype]] = Int8Dtype
+
+
+class Int16(Integer):
+    DTYPE: Final[type[Dtype]] = Int16Dtype
+
+
+class Int32(Integer):
+    DTYPE: Final[type[Dtype]] = Int32Dtype
+
+
+class Int64(Integer):
+    DTYPE: Final[type[Dtype]] = Int64Dtype
+
+
+class Float(Profile):
+    DTYPE: ClassVar[type[Dtype]] = FloatDtype
+
+    @property
+    def min(self) -> Decimal:
+        return min(Decimal(value) for value in self.uniques())
+
+    @property
+    def max(self) -> Decimal:
+        return max(Decimal(value) for value in self.uniques())
+
+    @property
+    def precision(self) -> int:
+        return max(len(Decimal(value).as_tuple().digits) for value in self.uniques())
+
+    @property
+    def scale(self) -> int:
+        exponents: list[int] = [
+            cast("int", Decimal(value).as_tuple().exponent) for value in self.uniques()
+        ]
+        return max([0, *(-exponent for exponent in exponents)])
+
+    def _compute_candidates(self) -> tuple[Candidate, ...]:
+        return candidate_factory(
+            self.DTYPE,
+            num_unique=self.num_unique,
+            total=self.total,
+            min_value=self.min,
+            max_value=self.max,
+            precision=self.precision,
+            scale=self.scale,
+        )
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}(min={self.min}, max={self.max}, precision={self.precision}, scale={self.scale}, num_unique={self.num_unique})"
+
+
+class Float16(Float):
+    DTYPE: Final[type[Dtype]] = Float16Dtype
+
+
+class Float32(Float):
+    DTYPE: Final[type[Dtype]] = Float32Dtype
+
+
+class Float64(Float):
+    DTYPE: Final[type[Dtype]] = Float64Dtype
+
+
+class String(Profile):
+    DTYPE: Final[type[Dtype]] = StringDtype
+
+    @property
+    def min_length(self) -> int:
+        return min(len(value) for value in self.uniques())
+
+    @property
+    def max_length(self) -> int:
+        return max(len(value) for value in self.uniques())
+
+    def _compute_candidates(self) -> tuple[Candidate, ...]:
+        return candidate_factory(
+            self.DTYPE, num_unique=self.num_unique, total=self.total,
+        )
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}(min_length={self.min_length}, max_length={self.max_length}, num_unique={self.num_unique})"