pydiverse-common 0.3.2__tar.gz → 0.3.3__tar.gz

{pydiverse_common-0.3.2 → pydiverse_common-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydiverse-common
-Version: 0.3.2
+Version: 0.3.3
 Summary: Common functionality shared between pydiverse libraries
 Author: QuantCo, Inc.
 Author-email: Martin Trautmann <windiana@users.sf.net>, Finn Rudolph <finn.rudolph@t-online.de>

{pydiverse_common-0.3.2 → pydiverse_common-0.3.3}/docs/source/changelog.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.3.3 (2025-06-08)
+- improved support of None and List type
+- bug fixes in type conversion functions
+
 ## 0.3.2 (2025-06-08)
 - pydiverse.common.__version__ (implemented via importlib.metadata)
 

{pydiverse_common-0.3.2 → pydiverse_common-0.3.3}/docs/source/reference/api.rst

@@ -8,25 +8,48 @@ Public
 .. py:module:: pydiverse.common
 
 .. autoclass:: Dtype
+   :members:
 .. autoclass:: Bool
+   :members:
 .. autoclass:: Date
+   :members:
 .. autoclass:: Datetime
+   :members:
 .. autoclass:: Decimal
+   :members:
 .. autoclass:: Duration
+   :members:
 .. autoclass:: Float
+   :members:
 .. autoclass:: Float32
+   :members:
 .. autoclass:: Float64
+   :members:
 .. autoclass:: Int
+   :members:
 .. autoclass:: Int8
+   :members:
 .. autoclass:: Int16
+   :members:
 .. autoclass:: Int32
+   :members:
 .. autoclass:: Int64
+   :members:
 .. autoclass:: NullType
+   :members:
 .. autoclass:: String
+   :members:
 .. autoclass:: Time
+   :members:
 .. autoclass:: UInt8
+   :members:
 .. autoclass:: UInt16
+   :members:
 .. autoclass:: UInt32
+   :members:
 .. autoclass:: UInt64
+   :members:
 .. autoclass:: List
+   :members:
 .. autoclass:: PandasBackend
+   :members:

{pydiverse_common-0.3.2 → pydiverse_common-0.3.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pydiverse-common"
-version = "0.3.2"
+version = "0.3.3"
 description = "Common functionality shared between pydiverse libraries"
 authors = [
   { name = "QuantCo, Inc." },
@@ -29,14 +29,13 @@ classifiers = [
 packages = ["src/pydiverse"]
 
 [tool.ruff]
-fix = true
 target-version = "py310"
 extend-exclude = ["docs/*"]
+fix = true
 
 [tool.ruff.lint]
 select = ["F", "E", "UP", "W", "I001", "I002", "B", "A"]
-ignore = ["B028"]
-# 'ignore-init-module-imports' is deprecated and removed per warning
+#ignore = ["B028"]
 
 [tool.ruff.lint.per-file-ignores]
 #"__init__.py" = ["F401", "F403"]

{pydiverse_common-0.3.2 → pydiverse_common-0.3.3}/src/pydiverse/common/dtypes.py

@@ -12,24 +12,33 @@ class Dtype:
     """Base class for all data types."""
 
     def __eq__(self, rhs):
+        """Return ``True`` if this dtype is equal to `rhs`."""
         return isinstance(rhs, Dtype) and type(self) is type(rhs)
 
     def __hash__(self):
+        """Return a hash for this dtype."""
         return hash(type(self))
 
     def __repr__(self):
+        """Return a string representation of this dtype."""
         return self.__class__.__name__
 
     @classmethod
     def is_int(cls):
+        """Return ``True`` if this dtype is an integer type."""
         return False
 
     @classmethod
     def is_float(cls):
+        """Return ``True`` if this dtype is a float type."""
         return False
 
     @classmethod
     def is_subtype(cls, rhs):
+        """Return ``True`` if this dtype is a subtype of `rhs`.
+
+        For example, ``Int8.is_subtype(Int())`` is ``True``.
+        """
         rhs_cls = type(rhs)
         return (
             (cls is rhs_cls)
@@ -39,6 +48,7 @@ class Dtype:
 
     @staticmethod
     def from_sql(sql_type) -> "Dtype":
+        """Convert a SQL type to a Dtype."""
         import sqlalchemy as sqa
 
         if isinstance(sql_type, sqa.SmallInteger):
@@ -73,14 +83,15 @@ class Dtype:
         if isinstance(sql_type, sqa.Interval):
             return Duration()
         if isinstance(sql_type, sqa.ARRAY):
-            return List(Dtype.from_sql(sql_type.item_type.from_sql))
-        if isinstance(sql_type, sqa.Null):
+            return List(Dtype.from_sql(sql_type.item_type))
+        if isinstance(sql_type, sqa.types.NullType):
             return NullType()
 
         raise TypeError
 
     @staticmethod
     def from_pandas(pandas_type) -> "Dtype":
+        """Convert a pandas type to a Dtype."""
         import numpy as np
         import pandas as pd
 
@@ -127,13 +138,14 @@ class Dtype:
             return Datetime()
         if pd.api.types.is_timedelta64_dtype(pandas_type):
             return Duration()
-        # we don't know any decimal/time dtypes in pandas if column is not
+        # we don't know any decimal/time/null dtypes in pandas if column is not
         # arrow backed
 
         raise TypeError
 
     @staticmethod
     def from_arrow(arrow_type) -> "Dtype":
+        """Convert a PyArrow type to a Dtype."""
         import pyarrow as pa
 
         if pa.types.is_signed_integer(arrow_type):
@@ -179,10 +191,15 @@ class Dtype:
             return Time()
         if pa.types.is_duration(arrow_type):
             return Duration()
+        if pa.types.is_null(arrow_type):
+            return NullType()
+        if pa.types.is_list(arrow_type):
+            return List(Dtype.from_arrow(arrow_type.value_type))
         raise TypeError
 
     @staticmethod
     def from_polars(polars_type) -> "Dtype":
+        """Convert a Polars type to a Dtype."""
         import polars as pl
 
         if isinstance(polars_type, pl.List):
@@ -211,6 +228,7 @@ class Dtype:
         }[polars_type.base_type()]
 
     def to_sql(self):
+        """Convert this Dtype to a SQL type."""
         import sqlalchemy as sqa
 
         return {
@@ -237,6 +255,7 @@ class Dtype:
         }[self]
 
     def to_pandas(self, backend: PandasBackend = PandasBackend.ARROW):
+        """Convert this Dtype to a pandas type."""
         import pandas as pd
 
         if backend == PandasBackend.NUMPY:
@@ -247,12 +266,32 @@ class Dtype:
             return pd.ArrowDtype(self.to_arrow())
 
     def to_pandas_nullable(self, backend: PandasBackend = PandasBackend.ARROW):
+        """Convert this Dtype to a pandas nullable type.
+
+        Nullable can be either pandas extension types like StringDtype or ArrowDtype.
+
+        Parameters
+        ----------
+        backend : PandasBackend, optional
+            The pandas backend to use. Defaults to ``PandasBackend.ARROW``.
+            If ``PandasBackend.NUMPY`` is selected, this method will attempt
+            to return a NumPy-backed nullable pandas dtype. Note that
+            Time, NullType, and List will raise a TypeError for the
+            NUMPY backend as pandas doesn't have corresponding native
+            nullable dtypes for these.
+        """
         import pandas as pd
 
-        if self == Time():
-            if backend == PandasBackend.ARROW:
-                return pd.ArrowDtype(self.to_arrow())
+        if backend == PandasBackend.ARROW:
+            return pd.ArrowDtype(self.to_arrow())
+
+        # we don't want to produce object columns
+        if isinstance(self, Time):
             raise TypeError("pandas doesn't have a native time dtype")
+        if isinstance(self, NullType):
+            raise TypeError("pandas doesn't have a native null dtype")
+        if isinstance(self, List):
+            raise TypeError("pandas doesn't have a native list dtype")
 
         return {
             Int(): pd.Int64Dtype(),  # we default to 64 bit
@@ -277,6 +316,7 @@ class Dtype:
         }[self]
 
     def to_arrow(self):
+        """Convert this Dtype to a PyArrow type."""
         import pyarrow as pa
 
         return {
@@ -299,9 +339,11 @@ class Dtype:
             Time(): pa.time64("us"),
             Datetime(): pa.timestamp("us"),
             Duration(): pa.duration("us"),
+            NullType(): pa.null(),
         }[self]
 
     def to_polars(self: "Dtype"):
+        """Convert this Dtype to a Polars type."""
         import polars as pl
 
         return {
@@ -416,3 +458,8 @@ class List(Dtype):
         import polars as pl
 
         return pl.List(self.inner.to_polars())
+
+    def to_arrow(self):
+        import pyarrow as pa
+
+        return pa.list_(self.inner.to_arrow())

pydiverse_common-0.3.3/src/pydiverse/common/testing.py

@@ -0,0 +1,14 @@
+# Copyright (c) QuantCo and pydiverse contributors 2025-2025
+# SPDX-License-Identifier: BSD-3-Clause
+
+import inspect
+
+import pydiverse.common.dtypes as dtypes
+
+ALL_TYPES = [
+    getattr(dtypes, c)
+    for c in dir(dtypes)
+    if inspect.isclass(getattr(dtypes, c))
+    and issubclass(getattr(dtypes, c), dtypes.Dtype)
+    and c != "Dtype"
+]

{pydiverse_common-0.3.2 → pydiverse_common-0.3.3}/tests/dtypes/test_dtype_pandas.py

@@ -5,6 +5,9 @@ import types
 
 import pytest
 
+import pydiverse.common as pdc
+from pydiverse.common.testing import ALL_TYPES
+
 try:
     import numpy as np
     import pandas as pd
@@ -141,3 +144,48 @@ def test_dtype_to_pandas_pyarrow():
     assert_conversion(Date(), pa.date32())
     assert_conversion(Time(), pa.time64("us"))
     assert_conversion(Datetime(), pa.timestamp("us"))
+
+
+@pytest.mark.skipif(np is None, reason="requires pandas, numpy, and pyarrow")
+@pytest.mark.parametrize(
+    "type_",
+    ALL_TYPES,
+)
+def test_all_types_numpy(type_):
+    if type_ is pdc.List:
+        type_obj = type_(pdc.Int64())
+    else:
+        type_obj = type_()
+    if type_ is pdc.NullType:
+        with pytest.raises(TypeError, match="pandas doesn't have a native null dtype"):
+            type_obj.to_pandas(PandasBackend.NUMPY)
+    elif type_ is pdc.Time:
+        with pytest.raises(TypeError, match="pandas doesn't have a native time dtype"):
+            type_obj.to_pandas(PandasBackend.NUMPY)
+    elif type_ is pdc.List:
+        with pytest.raises(TypeError, match="pandas doesn't have a native list dtype"):
+            type_obj.to_pandas(PandasBackend.NUMPY)
+    else:
+        dst_type = type_obj.to_pandas(PandasBackend.NUMPY)
+        back_type = Dtype.from_pandas(dst_type)
+        if type_ is pdc.Decimal:
+            assert isinstance(back_type, pdc.Float64)
+        elif type_ is pdc.Date:
+            assert isinstance(back_type, pdc.Datetime)
+        else:
+            assert isinstance(back_type, type_)
+
+
+@pytest.mark.skipif(np is None, reason="requires pandas, numpy, and pyarrow")
+@pytest.mark.parametrize(
+    "type_",
+    ALL_TYPES,
+)
+def test_all_types_arrow(type_):
+    if type_ is pdc.List:
+        type_obj = type_(pdc.Int64())
+    else:
+        type_obj = type_()
+    dst_type = type_obj.to_pandas(PandasBackend.ARROW)
+    back_type = Dtype.from_pandas(dst_type)
+    assert isinstance(back_type, type_)

{pydiverse_common-0.3.2 → pydiverse_common-0.3.3}/tests/dtypes/test_dtype_polars.py

@@ -1,9 +1,9 @@
 # Copyright (c) QuantCo and pydiverse contributors 2025-2025
 # SPDX-License-Identifier: BSD-3-Clause
-from typing import TYPE_CHECKING
 
 import pytest
 
+import pydiverse.common as pdc
 from pydiverse.common import (
     Bool,
     Date,
@@ -22,13 +22,15 @@ from pydiverse.common import (
     UInt32,
     UInt64,
 )
+from pydiverse.common.testing import ALL_TYPES
 
-pl = pytest.importorskip("polars")
-
-if TYPE_CHECKING:
+try:
     import polars as pl
+except ImportError:
+    pl = None
 
 
+@pytest.mark.skipif(pl is None, reason="requires polars")
 def test_dtype_from_polars():
     def assert_conversion(type_, expected):
         assert Dtype.from_polars(type_) == expected
@@ -57,6 +59,7 @@ def test_dtype_from_polars():
     assert_conversion(pl.Datetime("ns"), Datetime())
 
 
+@pytest.mark.skipif(pl is None, reason="requires polars")
 def test_dtype_to_polars():
     def assert_conversion(type_: Dtype, expected):
         assert type_.to_polars() == expected
@@ -80,3 +83,18 @@ def test_dtype_to_polars():
     assert_conversion(Date(), pl.Date)
     assert_conversion(Time(), pl.Time)
     assert_conversion(Datetime(), pl.Datetime("us"))
+
+
+@pytest.mark.skipif(pl is None, reason="requires polars")
+@pytest.mark.parametrize(
+    "type_",
+    ALL_TYPES,
+)
+def test_all_types(type_):
+    if type_ is pdc.List:
+        type_obj = type_(pdc.Int64())
+    else:
+        type_obj = type_()
+    dst_type = type_obj.to_polars()
+    back_type = Dtype.from_polars(dst_type)
+    assert isinstance(back_type, type_)

{pydiverse_common-0.3.2 → pydiverse_common-0.3.3}/tests/dtypes/test_dtype_pyarrow.py

@@ -2,6 +2,9 @@
 # SPDX-License-Identifier: BSD-3-Clause
 import pytest
 
+import pydiverse.common as pdc
+from pydiverse.common.testing import ALL_TYPES
+
 try:
     import pyarrow as pa
 except ImportError:
@@ -87,3 +90,18 @@ def test_dtype_to_pyarrow():
     assert_conversion(Date(), pa.date32())
     assert_conversion(Time(), pa.time64("us"))
     assert_conversion(Datetime(), pa.timestamp("us"))
+
+
+@pytest.mark.skipif(pa is None, reason="requires pandas, numpy, and pyarrow")
+@pytest.mark.parametrize(
+    "type_",
+    ALL_TYPES,
+)
+def test_all_types(type_):
+    if type_ is pdc.List:
+        type_obj = type_(pdc.Int64())
+    else:
+        type_obj = type_()
+    dst_type = type_obj.to_arrow()
+    back_type = Dtype.from_arrow(dst_type)
+    assert isinstance(back_type, type_)

{pydiverse_common-0.3.2 → pydiverse_common-0.3.3}/tests/dtypes/test_dtype_sqlalchemy.py

@@ -2,10 +2,12 @@
 # SPDX-License-Identifier: BSD-3-Clause
 import pytest
 
+import pydiverse.common as pdc
 from pydiverse.common import (
     Bool,
     Date,
     Datetime,
+    Decimal,
     Dtype,
     Float32,
     Float64,
@@ -20,6 +22,7 @@ from pydiverse.common import (
     UInt32,
     UInt64,
 )
+from pydiverse.common.testing import ALL_TYPES
 
 try:
     import sqlalchemy as sa
@@ -77,3 +80,29 @@ def test_dtype_to_sqlalchemy():
     assert_conversion(Date(), sa.Date)
     assert_conversion(Time(), sa.Time)
     assert_conversion(Datetime(), sa.DateTime)
+
+
+@pytest.mark.skipif(sa is None, reason="requires polars")
+@pytest.mark.parametrize(
+    "type_",
+    ALL_TYPES,
+)
+def test_all_types(type_):
+    if type_ is pdc.List:
+        type_obj = type_(pdc.Int64())
+    else:
+        type_obj = type_()
+    dst_type = type_obj.to_sql()
+    back_type = Dtype.from_sql(dst_type)
+    acceptance_map = {
+        # SQL is a bit less strict about integer precisions
+        Int8: Int16,
+        UInt8: Int16,
+        UInt16: Int32,
+        UInt32: Int64,
+        UInt64: Int64,
+        # we intentionally fetch Decimal as Float since Decimal is more a relational
+        # database thing
+        Decimal: Float64,
+    }
+    assert isinstance(back_type, acceptance_map.get(type_, type_))