atdata 0.1.2a1__tar.gz → 0.1.2a4__tar.gz

{atdata-0.1.2a1 → atdata-0.1.2a4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atdata
-Version: 0.1.2a1
+Version: 0.1.2a4
 Summary: A loose federation of distributed, typed datasets
 Author-email: Maxine Levesque <hello@maxine.science>
 License-File: LICENSE

{atdata-0.1.2a1 → atdata-0.1.2a4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "atdata"
-version = "0.1.2a1"
+version = "0.1.2a4"
 description = "A loose federation of distributed, typed datasets"
 readme = "README.md"
 authors = [

{atdata-0.1.2a1 → atdata-0.1.2a4}/src/atdata/dataset.py

@@ -97,7 +97,8 @@ def _make_packable( x ):
 class PackableSample( ABC ):
     """A sample that can be packed and unpacked with msgpack"""
 
-    def __post_init__( self ):
+    def _ensure_good( self ):
+        """TODO Stupid kludge because of __post_init__ nonsense for wrapped classes"""
 
         # Auto-convert known types when annotated
         for var_name, var_type in vars( self.__class__ )['__annotations__'].items():
@@ -121,12 +122,17 @@ class PackableSample( ABC ):
                 elif isinstance( var_cur_value, bytes ):
                     setattr( self, var_name, eh.bytes_to_array( var_cur_value ) )
 
+    def __post_init__( self ):
+        self._ensure_good()
+
     ##
 
     @classmethod
     def from_data( cls, data: MsgpackRawSample ) -> Self:
         """Create a sample instance from unpacked msgpack data"""
-        return cls( **data )
+        ret = cls( **data )
+        ret._ensure_good()
+        return ret
     
     @classmethod
     def from_bytes( cls, bs: bytes ) -> Self:
@@ -443,12 +449,17 @@ def packable( cls ):
     
     ##
 
+    # Add in dataclass niceness to original class
     as_dataclass = dataclass( cls )
 
+    # This triggers a bunch of behind-the-scenes stuff for the newly annotated class
+    @dataclass
     class as_packable( as_dataclass, PackableSample ):
-        pass
+        def __post_init__( self ):
+            return PackableSample.__post_init__( self )
     
     as_packable.__name__ = cls.__name__
+    as_packable.__annotations__ = cls.__annotations__
 
     ##
 

{atdata-0.1.2a1 → atdata-0.1.2a4}/tests/test_dataset.py

@@ -39,6 +39,17 @@ class NumpyTestSample( atdata.PackableSample ):
     label: int
     image: NDArray
 
+@atdata.packable
+class BasicTestSampleDecorated:
+    name: str
+    position: int
+    value: float
+
+@atdata.packable
+class NumpyTestSampleDecorated:
+    label: int
+    image: NDArray
+
 test_cases = [
     {
         'SampleType': BasicTestSample,
@@ -58,6 +69,24 @@ test_cases = [
         },
         'sample_wds_stem': 'numpy_test',
     },
+    {
+        'SampleType': BasicTestSampleDecorated,
+        'sample_data': {
+            'name': 'Hello, world!',
+            'position': 42,
+            'value': 1024.768,
+        },
+        'sample_wds_stem': 'basic_test_decorated',
+    },
+    {
+        'SampleType': NumpyTestSampleDecorated,
+        'sample_data':
+        {
+            'label': 9_001,
+            'image': np.random.randn( 1024, 1024 ),
+        },
+        'sample_wds_stem': 'numpy_test_decorated',
+    },
 ]
 
 
@@ -89,32 +118,32 @@ def test_create_sample(
 
 #
 
-def test_decorator_syntax():
-    """Test use of decorator syntax for sample types"""
+# def test_decorator_syntax():
+#     """Test use of decorator syntax for sample types"""
     
-    @atdata.packable
-    class BasicTestSampleDecorated:
-        name: str
-        position: int
-        value: float
-
-    @atdata.packable
-    class NumpyTestSampleDecorated:
-        label: int
-        image: NDArray
+#     @atdata.packable
+#     class BasicTestSampleDecorated:
+#         name: str
+#         position: int
+#         value: float
+
+#     @atdata.packable
+#     class NumpyTestSampleDecorated:
+#         label: int
+#         image: NDArray
     
-    ##
+#     ##
     
-    test_create_sample( BasicTestSampleDecorated, {
-        'name': 'Hello, world!',
-        'position': 42,
-        'value': 1024.768,
-    } )
+#     test_create_sample( BasicTestSampleDecorated, {
+#         'name': 'Hello, world!',
+#         'position': 42,
+#         'value': 1024.768,
+#     } )
 
-    test_create_sample( NumpyTestSampleDecorated, {
-        'label': 9_001,
-        'image': np.random.randn( 1024, 1024 ),
-    } )
+#     test_create_sample( NumpyTestSampleDecorated, {
+#         'label': 9_001,
+#         'image': np.random.randn( 1024, 1024 ),
+#     } )
 
 #
 
@@ -138,7 +167,6 @@ def test_wds(
     batch_size = 4
     n_iterate = 10
 
-
     ## Write sharded dataset
 
     file_pattern = (
@@ -169,7 +197,7 @@ def test_wds(
 
     iterations_run = 0
     for i_iterate, cur_sample in enumerate( dataset.ordered( batch_size = None ) ):
-        
+
         assert isinstance( cur_sample, SampleType ), \
             f'Single sample for {SampleType} written to `wds` is of wrong type'
         
@@ -181,7 +209,7 @@ def test_wds(
             else:
                 is_correct = getattr( cur_sample, k ) == v
             assert is_correct, \
-                f'{SampleType}: Incorrect sample value found for {k}'
+                f'{SampleType}: Incorrect sample value found for {k} - {type( getattr( cur_sample, k ) )}'
 
         iterations_run += 1
         if iterations_run >= n_iterate:
@@ -195,7 +223,6 @@ def test_wds(
     start_id = f'{0:06d}'
     end_id = f'{9:06d}'
     first_filename = file_pattern.format( shard_id = '{' + start_id + '..' + end_id + '}' )
-    print( first_filename )
     dataset = atdata.Dataset[SampleType]( first_filename )
 
     iterations_run = 0
@@ -270,7 +297,6 @@ def test_wds(
     start_id = f'{0:06d}'
     end_id = f'{9:06d}'
     first_filename = file_pattern.format( shard_id = '{' + start_id + '..' + end_id + '}' )
-    print( first_filename )
     dataset = atdata.Dataset[SampleType]( first_filename )
 
     iterations_run = 0