pydiverse-common 0.3.8__py3-none-any.whl → 0.3.9__py3-none-any.whl

pydiverse/common/dtypes.py

@@ -201,6 +201,11 @@ class Dtype:
             return NullType()
         if pa.types.is_list(arrow_type):
             return List(Dtype.from_arrow(arrow_type.value_type))
+        if pa.types.is_dictionary(arrow_type):
+            raise RuntimeError(
+                "Most likely this is an Enum type. But metadata about categories is "
+                "only in the pyarrow field and not in the pyarrow dtype"
+            )
         raise TypeError
 
     @staticmethod
@@ -268,7 +273,7 @@ class Dtype:
         if backend == PandasBackend.NUMPY:
             return self.to_pandas_nullable(backend)
         if backend == PandasBackend.ARROW:
-            if self == String():
+            if self == String() or isinstance(self, Enum):
                 return pd.StringDtype(storage="pyarrow")
             return pd.ArrowDtype(self.to_arrow())
 
@@ -355,6 +360,12 @@ class Dtype:
             NullType(): pa.null(),
         }[self]
 
+    def to_arrow_field(self, name: str, nullable: bool = True):
+        """Convert this Dtype to a PyArrow Field."""
+        import pyarrow as pa
+
+        return pa.field(name, self.to_arrow(), nullable=nullable)
+
     def to_polars(self: "Dtype"):
         """Convert this Dtype to a Polars type."""
         import polars as pl
@@ -506,6 +517,19 @@ class Enum(String):
     def to_arrow(self):
         import pyarrow as pa
 
-        # There is also pa.dictionary(), which seems to be kind of similar to an enum.
-        # Maybe it is better to convert to this.
+        # enum categories can only be maintained in pyarrow field (see to_arrow_field)
         return pa.string()
+
+    def to_arrow_field(self, name: str, nullable: bool = True):
+        """Convert this Dtype to a PyArrow Field."""
+        import pyarrow as pa
+
+        # try to mimic what polars does
+        return pa.field(
+            name,
+            pa.dictionary(pa.uint32(), pa.large_string()),
+            nullable=nullable,
+            metadata={
+                "_PL_ENUM_VALUES": "".join([f"{len(c)};{c}" for c in self.categories])
+            },
+        )

{pydiverse_common-0.3.8.dist-info → pydiverse_common-0.3.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydiverse-common
-Version: 0.3.8
+Version: 0.3.9
 Summary: Common functionality shared between pydiverse libraries
 Author: QuantCo, Inc.
 Author-email: Martin Trautmann <windiana@users.sf.net>, Finn Rudolph <finn.rudolph@t-online.de>

{pydiverse_common-0.3.8.dist-info → pydiverse_common-0.3.9.dist-info}/RECORD

@@ -1,5 +1,5 @@
 pydiverse/common/__init__.py,sha256=J7b4iStFyaEMYre_jdlZ4l_8dLyrMWCIpQdsMQcB8aI,806
-pydiverse/common/dtypes.py,sha256=LYZKaKYq_4uI4kUhoaCTTo5j1SRurswIOfN11Bkz25A,15986
+pydiverse/common/dtypes.py,sha256=TN3UjI-4uBT_CB_3mLowOYq5GPcyvSbI3N6zHpxxV3c,16885
 pydiverse/common/testing.py,sha256=FcivI5wn0X3gzJhwnysKvCOgjSTTXaN6FtSFJ72jfSg,341
 pydiverse/common/version.py,sha256=1IU_m4r76_Qq0u-Tyo2_bERZFOkh0ZFueVzDqcCfLO0,336
 pydiverse/common/errors/__init__.py,sha256=FNeEfVbUa23b9sHkFsmxHYhY6sRgjaZysPQmlovpJrI,262
@@ -11,7 +11,7 @@ pydiverse/common/util/disposable.py,sha256=4XoGz70YRWA9TAqnUBvRCTAdsOGBviFN0gzxU
 pydiverse/common/util/hashing.py,sha256=8Z1NybJ_zd3ONpn5annHGjowwArWkd2ZkCtlb3dtz_Q,4576
 pydiverse/common/util/import_.py,sha256=K7dSgz4YyrqEvqhoOzbwgD7D8HScMoO5XoSWtjbaoUs,4056
 pydiverse/common/util/structlog.py,sha256=xxhauxMuyxcKXTVg1MiPTkuvPBj8Zcr4o_v8Bq59Nig,3778
-pydiverse_common-0.3.8.dist-info/METADATA,sha256=ptAGp299BY9NSaM-XEaojLzhL_KVc0SEY-MFqqqAwL0,3399
-pydiverse_common-0.3.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-pydiverse_common-0.3.8.dist-info/licenses/LICENSE,sha256=AcE6SDVuAq6v9ZLE_8eOCe_NvSE0rAPR3NR7lSowYh4,1517
-pydiverse_common-0.3.8.dist-info/RECORD,,
+pydiverse_common-0.3.9.dist-info/METADATA,sha256=_P0mMant1LZMCa4p5tFBnpr0cvLIYXZXfu5xmYuih9Q,3399
+pydiverse_common-0.3.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pydiverse_common-0.3.9.dist-info/licenses/LICENSE,sha256=AcE6SDVuAq6v9ZLE_8eOCe_NvSE0rAPR3NR7lSowYh4,1517
+pydiverse_common-0.3.9.dist-info/RECORD,,