atdata 0.1.1a2__py3-none-any.whl → 0.1.2a1__py3-none-any.whl

atdata/__init__.py

@@ -1,2 +1,14 @@
-def main() -> None:
-    print("Hello from ekumen!")
+"""A loose federation of distributed, typed datasets"""
+
+##
+# Expose components
+
+from .dataset import (
+    PackableSample,
+    SampleBatch,
+    Dataset,
+    packable,
+)
+
+
+#

atdata/_helpers.py

@@ -1,30 +1,22 @@
-"""Assorted helper methods for `ekumen`"""
+"""Assorted helper methods for `atdata`"""
 
 ##
 # Imports
 
 from io import BytesIO
-import ormsgpack as omp
 
 import numpy as np
 
 
 ##
-# 
 
-def pack_instance( x ) -> bytes:
-    return omp.packb( x )
-
-def unpack( bs: bytes ):
-    return omp.unpackb( bs )
-
-##
-
-def array_to_bytes(x: np.ndarray) -> bytes:
+def array_to_bytes( x: np.ndarray ) -> bytes:
+    """Convert `numpy` array to a format suitable for packing"""
     np_bytes = BytesIO()
-    np.save(np_bytes, x, allow_pickle=True)
+    np.save( np_bytes, x, allow_pickle = True )
     return np_bytes.getvalue()
 
-def bytes_to_array(b: bytes) -> np.ndarray:
-    np_bytes = BytesIO(b)
-    return np.load(np_bytes, allow_pickle=True)
+def bytes_to_array( b: bytes ) -> np.ndarray:
+    """Convert packed bytes back to a `numpy` array"""
+    np_bytes = BytesIO( b )
+    return np.load( np_bytes, allow_pickle = True )

atdata/dataset.py

@@ -5,6 +5,7 @@
 
 import webdataset as wds
 
+import functools
 from dataclasses import dataclass
 import uuid
 
@@ -57,38 +58,38 @@ DT = TypeVar( 'DT' )
 
 MsgpackRawSample: TypeAlias = Dict[str, Any]
 
-@dataclass
-class ArrayBytes:
-    """Annotates bytes that should be interpreted as the raw contents of a
-    numpy NDArray"""
+# @dataclass
+# class ArrayBytes:
+#     """Annotates bytes that should be interpreted as the raw contents of a
+#     numpy NDArray"""
     
-    raw_bytes: bytes
-    """The raw bytes of the corresponding NDArray"""
-
-    def __init__( self,
-            array: Optional[ArrayLike] = None,  
-            raw: Optional[bytes] = None,
-        ):
-        """TODO"""
-
-        if array is not None:
-            array = np.array( array )
-            self.raw_bytes = eh.array_to_bytes( array )
+#     raw_bytes: bytes
+#     """The raw bytes of the corresponding NDArray"""
+
+#     def __init__( self,
+#             array: Optional[ArrayLike] = None,  
+#             raw: Optional[bytes] = None,
+#         ):
+#         """TODO"""
+
+#         if array is not None:
+#             array = np.array( array )
+#             self.raw_bytes = eh.array_to_bytes( array )
         
-        elif raw is not None:
-            self.raw_bytes = raw
+#         elif raw is not None:
+#             self.raw_bytes = raw
         
-        else:
-            raise ValueError( 'Must provide either `array` or `raw` bytes' )
+#         else:
+#             raise ValueError( 'Must provide either `array` or `raw` bytes' )
 
-    @property
-    def to_numpy( self ) -> NDArray:
-        """Return the `raw_bytes` data as an NDArray"""
-        return eh.bytes_to_array( self.raw_bytes )
+#     @property
+#     def to_numpy( self ) -> NDArray:
+#         """Return the `raw_bytes` data as an NDArray"""
+#         return eh.bytes_to_array( self.raw_bytes )
 
 def _make_packable( x ):
-    if isinstance( x, ArrayBytes ):
-        return x.raw_bytes
+    # if isinstance( x, ArrayBytes ):
+    #     return x.raw_bytes
     if isinstance( x, np.ndarray ):
         return eh.array_to_bytes( x )
     return x
@@ -114,8 +115,8 @@ class PackableSample( ABC ):
                     # we're good!
                     pass
 
-                elif isinstance( var_cur_value, ArrayBytes ):
-                    setattr( self, var_name, var_cur_value.to_numpy )
+                # elif isinstance( var_cur_value, ArrayBytes ):
+                #     setattr( self, var_name, var_cur_value.to_numpy )
 
                 elif isinstance( var_cur_value, bytes ):
                     setattr( self, var_name, eh.bytes_to_array( var_cur_value ) )
@@ -172,7 +173,7 @@ def _batch_aggregate( xs: Sequence ):
 
     return list( xs )
 
-class SamlpeBatch( Generic[DT] ):
+class SampleBatch( Generic[DT] ):
 
     def __init__( self, samples: Sequence[DT] ):
         """TODO"""
@@ -233,7 +234,7 @@ class Dataset( Generic[ST] ):
     def batch_type( self ) -> Type:
         """The type of a batch built from `sample_class`"""
         # return self.__orig_class__.__args__[1]
-        return SamlpeBatch[self.sample_type]
+        return SampleBatch[self.sample_type]
 
 
     # _schema_registry_sample: dict[str, Type]
@@ -396,7 +397,7 @@ class Dataset( Generic[ST] ):
                 value = sample,
             )
 
-    def wrap_batch( self, batch: WDSRawBatch ) -> SamlpeBatch[ST]:
+    def wrap_batch( self, batch: WDSRawBatch ) -> SampleBatch[ST]:
         """Wrap a `batch` of samples into the appropriate dataset-specific type
        
         This default implementation simply creates a list one sample at a time
@@ -405,7 +406,7 @@ class Dataset( Generic[ST] ):
         assert 'msgpack' in batch
         batch_unpacked = [ self.sample_type.from_bytes( bs )
                            for bs in batch['msgpack'] ]
-        return SamlpeBatch[self.sample_type]( batch_unpacked )
+        return SampleBatch[self.sample_type]( batch_unpacked )
 
 
     # # @classmethod
@@ -415,4 +416,40 @@ class Dataset( Generic[ST] ):
     #     This default implementation simply creates a list one sample at a time
     #     """
     #     assert cls.batch_class is not None, 'No batch class specified'
-    #     return cls.batch_class( **batch )
+    #     return cls.batch_class( **batch )
+
+
+##
+# Shortcut decorators
+
+# def packable( cls ):
+#     """TODO"""
+
+#     def decorator( cls ):
+#         # Create a new class dynamically
+#         # The new class inherits from the new_parent_class first, then the original cls
+#         new_bases = (PackableSample,) + cls.__bases__
+#         new_cls = type(cls.__name__, new_bases, dict(cls.__dict__))
+
+#         # Optionally, update __module__ and __qualname__ for better introspection
+#         new_cls.__module__ = cls.__module__
+#         new_cls.__qualname__ = cls.__qualname__
+
+#         return new_cls
+#     return decorator
+
+def packable( cls ):
+    """TODO"""
+    
+    ##
+
+    as_dataclass = dataclass( cls )
+
+    class as_packable( as_dataclass, PackableSample ):
+        pass
+    
+    as_packable.__name__ = cls.__name__
+
+    ##
+
+    return as_packable

{atdata-0.1.1a2.dist-info → atdata-0.1.2a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atdata
-Version: 0.1.1a2
+Version: 0.1.2a1
 Summary: A loose federation of distributed, typed datasets
 Author-email: Maxine Levesque <hello@maxine.science>
 License-File: LICENSE

atdata-0.1.2a1.dist-info/RECORD

@@ -0,0 +1,8 @@
+atdata/__init__.py,sha256=jPZVd_6UIo0DSbCnXAnYZ2eMwHYzOk--5vtEDTZvwqw,173
+atdata/_helpers.py,sha256=R63JhXewAKZYnZ9Th7R6yZh0IOUPYGBsth3FpRUMD-U,503
+atdata/dataset.py,sha256=xPxDkQk1fBkU3sbLMT_Rm4CmvoBXIpEpmRNxrir4nis,13045
+atdata-0.1.2a1.dist-info/METADATA,sha256=8aK-P0A7YZ2Cl6r_GC3Gi1huZRDrcR9zME2gMdT-fFc,434
+atdata-0.1.2a1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+atdata-0.1.2a1.dist-info/entry_points.txt,sha256=6-iQr1veSTq-ac94bLyfcyGHprrZWevPEd12BWX37tQ,39
+atdata-0.1.2a1.dist-info/licenses/LICENSE,sha256=Pz2eACSxkhsGfW9_iN60pgy-enjnbGTj8df8O3ebnQQ,16726
+atdata-0.1.2a1.dist-info/RECORD,,

atdata-0.1.2a1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+atdata = atdata:main

atdata-0.1.1a2.dist-info/RECORD

@@ -1,8 +0,0 @@
-atdata/__init__.py,sha256=yN07kW_3UcMlYZrM_Jrpy6DMCzTp9kvu2ICcU7n1-5w,52
-atdata/_helpers.py,sha256=CjIvLruNOhHRl1Arse5SahGTmI0Et3BoNqsWC9b8noE,515
-atdata/dataset.py,sha256=mvmCYtL6wD9961qq4lprZSkone56ubTKp3vDgEnWdPI,12158
-atdata-0.1.1a2.dist-info/METADATA,sha256=Mf75Ai8KKLhm7r2m65dP_QV6e9dkE7mTHtGSASGEKBE,434
-atdata-0.1.1a2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-atdata-0.1.1a2.dist-info/entry_points.txt,sha256=KuQtj4ZAwWLSyJUxhpQEHYfwSG-0ZXuj5hcZ1uAgGRQ,39
-atdata-0.1.1a2.dist-info/licenses/LICENSE,sha256=Pz2eACSxkhsGfW9_iN60pgy-enjnbGTj8df8O3ebnQQ,16726
-atdata-0.1.1a2.dist-info/RECORD,,

atdata-0.1.1a2.dist-info/entry_points.txt

@@ -1,2 +0,0 @@
-[console_scripts]
-ekumen = atdata:main