atdata 0.1.2b1__py3-none-any.whl → 0.1.3a2__py3-none-any.whl

atdata/__init__.py

@@ -12,6 +12,7 @@ from .dataset import (
 
 from .lens import (
     Lens,
+    LensNetwork,
     lens,
 )
 

atdata/dataset.py

@@ -48,6 +48,7 @@ from numpy.typing import (
 import msgpack
 import ormsgpack
 from . import _helpers as eh
+from .lens import Lens, LensNetwork
 
 
 ##
@@ -231,6 +232,8 @@ class SampleBatch( Generic[DT] ):
 ST = TypeVar( 'ST', bound = PackableSample )
 # BT = TypeVar( 'BT' )
 
+RT = TypeVar( 'RT', bound = PackableSample )
+
 # TODO For python 3.13
 # BT = TypeVar( 'BT', default = None )
 # IT = TypeVar( 'IT', default = Any )
@@ -268,6 +271,17 @@ class Dataset( Generic[ST] ):
         super().__init__()
         self.url = url
 
+        # Allow addition of automatic transformation of raw underlying data
+        self._output_lens: Lens | None = None
+
+    def as_type( self, other: Type[RT] ) -> 'Dataset[RT]':
+        """TODO"""
+        ret = Dataset[other]( self.url )
+        # Get the singleton lens registry
+        lenses = LensNetwork()
+        ret._output_lens = lenses.transform( self.sample_type, ret.sample_type )
+        return ret
+
     # @classmethod
     # def register( cls, uri: str,
     #             sample_class: Type,
@@ -293,9 +307,9 @@ class Dataset( Generic[ST] ):
             A full (non-lazy) list of the individual ``tar`` files within the
             source WebDataset.
         """
-        pipe = wds.DataPipeline(
-            wds.SimpleShardList( self.url ),
-            wds.map( lambda x: x['url'] )
+        pipe = wds.pipeline.DataPipeline(
+            wds.shardlists.SimpleShardList( self.url ),
+            wds.filters.map( lambda x: x['url'] )
         )
         return list( pipe )
     
@@ -317,23 +331,23 @@ class Dataset( Generic[ST] ):
 
         if batch_size is None:
             # TODO Duplication here
-            return wds.DataPipeline(
-                wds.SimpleShardList( self.url ),
-                wds.split_by_worker,
+            return wds.pipeline.DataPipeline(
+                wds.shardlists.SimpleShardList( self.url ),
+                wds.shardlists.split_by_worker,
                 #
-                wds.tarfile_to_samples(),
+                wds.tariterators.tarfile_to_samples(),
                 # wds.map( self.preprocess ),
-                wds.map( self.wrap ),
+                wds.filters.map( self.wrap ),
             )
 
-        return wds.DataPipeline(
-            wds.SimpleShardList( self.url ),
-            wds.split_by_worker,
+        return wds.pipeline.DataPipeline(
+            wds.shardlists.SimpleShardList( self.url ),
+            wds.shardlists.split_by_worker,
             #
-            wds.tarfile_to_samples(),
+            wds.tariterators.tarfile_to_samples(),
             # wds.map( self.preprocess ),
-            wds.batched( batch_size ),
-            wds.map( self.wrap_batch ),
+            wds.filters.batched( batch_size ),
+            wds.filters.map( self.wrap_batch ),
         )
 
     def shuffled( self,
@@ -461,7 +475,11 @@ class Dataset( Generic[ST] ):
         assert 'msgpack' in sample
         assert type( sample['msgpack'] ) == bytes
         
-        return self.sample_type.from_bytes( sample['msgpack'] )
+        if self._output_lens is None:
+            return self.sample_type.from_bytes( sample['msgpack'] )
+
+        source_sample = self._output_lens.source_type.from_bytes( sample['msgpack'] )
+        return self._output_lens( source_sample )
     
         # try:
         #     assert type( sample ) == dict

atdata/lens.py

@@ -3,8 +3,6 @@
 ##
 # Imports
 
-from .dataset import PackableSample
-
 import functools
 import inspect
 
@@ -17,17 +15,22 @@ from typing import (
     Callable,
     Optional,
     Generic,
+    #
+    TYPE_CHECKING
 )
 
+if TYPE_CHECKING:
+    from .dataset import PackableSample
+
 
 ##
 # Typing helpers
 
-DatasetType: TypeAlias = Type[PackableSample]
+DatasetType: TypeAlias = Type['PackableSample']
 LensSignature: TypeAlias = Tuple[DatasetType, DatasetType]
 
-S = TypeVar( 'S', bound = PackableSample )
-V = TypeVar( 'V', bound = PackableSample )
+S = TypeVar( 'S', bound = 'PackableSample' )
+V = TypeVar( 'V', bound = 'PackableSample' )
 type LensGetter[S, V] = Callable[[S], V]
 type LensPutter[S, V] = Callable[[V, S], S]
 
@@ -38,15 +41,37 @@ type LensPutter[S, V] = Callable[[V, S], S]
 class Lens( Generic[S, V] ):
     """TODO"""
 
+    # @property
+    # def source_type( self ) -> Type[S]:
+    #     """The source type (S) for the lens; what is put to"""
+    #     # TODO Figure out why linting fails here
+    #     return self.__orig_class__.__args__[0]
+
+    # @property
+    # def view_type( self ) -> Type[V]:
+    #     """The view type (V) for the lens; what is get'd from"""
+    #     # TODO FIgure out why linting fails here
+    #     return self.__orig_class__.__args__[1]
+
     def __init__( self, get: LensGetter[S, V],
                 put: Optional[LensPutter[S, V]] = None
             ) -> None:
         """TODO"""
         ##
 
-        # Update
+        # Check argument validity
+
+        sig = inspect.signature( get )
+        input_types = list( sig.parameters.values() )
+        assert len( input_types ) == 1, \
+            'Wrong number of input args for lens: should only have one'
+
+        # Update function details for this object as returned by annotation
         functools.update_wrapper( self, get )
 
+        self.source_type: Type[PackableSample] = input_types[0].annotation
+        self.view_type = sig.return_annotation
+
         # Store the getter
         self._getter = get
         
@@ -56,21 +81,7 @@ class Lens( Generic[S, V] ):
             def _trivial_put( v: V, s: S ) -> S:
                 return s
             put = _trivial_put
-        
         self._putter = put
-
-        # Register this lens for this type signature
-    
-        sig = inspect.signature( get )
-        input_types = list( sig.parameters.values() )
-        assert len( input_types ) == 1, \
-            'Wrong number of input args for lens: should only have one'
-        
-        input_type = input_types[0].annotation
-        output_type = sig.return_annotation
-
-        _registered_lenses[(input_type, output_type)] = self
-        print( _registered_lenses )
     
     #
 
@@ -80,6 +91,8 @@ class Lens( Generic[S, V] ):
         self._putter = put
         return put
     
+    # Methods to actually execute transformations
+
     def put( self, v: V, s: S ) -> S:
         """TODO"""
         return self._putter( v, s )
@@ -88,21 +101,86 @@ class Lens( Generic[S, V] ):
         """TODO"""
         return self( s )
 
-    #
+    # Convenience to enable calling the lens as its getter
     
     def __call__( self, s: S ) -> V:
         return self._getter( s )
 
-def lens( f: LensGetter[S, V] ) -> Lens[S, V]:
-    """Register the annotated function `f` as the getter of a sample lens"""
-    return Lens[S, V]( f )
+# TODO Figure out how to properly parameterize this
+# def _lens_factory[S, V]( register: bool = True ):
+#     """Register the annotated function `f` as the getter of a sample lens"""
+
+#     # The actual lens decorator taking a lens getter function to a lens object
+#     def _decorator( f: LensGetter[S, V] ) -> Lens[S, V]:
+#         ret = Lens[S, V]( f )
+#         if register:
+#             _network.register( ret )
+#         return ret
+    
+#     # Return the lens decorator
+#     return _decorator
+
+# # For convenience
+# lens = _lens_factory
+
+def lens(  f: LensGetter[S, V] ) -> Lens[S, V]:
+    ret = Lens[S, V]( f )
+    _network.register( ret )
+    return ret
 
 
 ##
-# Global registration of used lenses
+# Global registry of used lenses
+
+# _registered_lenses: Dict[LensSignature, Lens] = dict()
+# """TODO"""
+
+class LensNetwork:
+    """TODO"""
+
+    _instance = None
+    """The singleton instance"""
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            # If no instance exists, create a new one
+            cls._instance = super().__new__(cls)
+        return cls._instance  # Return the existing (or newly created) instance
+
+    def __init__(self):
+        if not hasattr(self, '_initialized'):  # Check if already initialized
+            self._registry: Dict[LensSignature, Lens] = dict()
+            self._initialized = True
+    
+    def register( self, _lens: Lens ):
+        """Set `lens` as the canonical view between its source and view types"""
+    
+        # sig = inspect.signature( _lens.get )
+        # input_types = list( sig.parameters.values() )
+        # assert len( input_types ) == 1, \
+        #     'Wrong number of input args for lens: should only have one'
+        
+        # input_type = input_types[0].annotation
+        # print( input_type )
+        # output_type = sig.return_annotation
+
+        # self._registry[input_type, output_type] = _lens
+        print( _lens.source_type )
+        self._registry[_lens.source_type, _lens.view_type] = _lens
+    
+    def transform( self, source: DatasetType, view: DatasetType ) -> Lens:
+        """TODO"""
+
+        # TODO Handle compositional closure
+        ret = self._registry.get( (source, view), None )
+        if ret is None:
+            raise ValueError( f'No registered lens from source {source} to view {view}' )
+        
+        return ret
+
 
-_registered_lenses: Dict[LensSignature, Lens] = dict()
-"""TODO"""
+# Create global singleton registry instance
+_network = LensNetwork()
 
 # def lens( f: LensPutter ) -> Lens:
 #     """Register the annotated function `f` as a sample lens"""

{atdata-0.1.2b1.dist-info → atdata-0.1.3a2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atdata
-Version: 0.1.2b1
+Version: 0.1.3a2
 Summary: A loose federation of distributed, typed datasets
 Author-email: Maxine Levesque <hello@maxine.science>
 License-File: LICENSE

atdata-0.1.3a2.dist-info/RECORD

@@ -0,0 +1,9 @@
+atdata/__init__.py,sha256=V2qBg7i2mfCNG9nww6Gi_fDp7iwolDMrNzhmNO6VA7M,233
+atdata/_helpers.py,sha256=R63JhXewAKZYnZ9Th7R6yZh0IOUPYGBsth3FpRUMD-U,503
+atdata/dataset.py,sha256=brNKGMkA_au2nLF5oUmjwub1E08DVwBKl9PnzPV6rPM,16722
+atdata/lens.py,sha256=bGlxQ6PEnLj5poQ41DHj1LfpsmI5fELnjnUf4qXOsCo,5304
+atdata-0.1.3a2.dist-info/METADATA,sha256=pYLvssfCzaiqer4yZQM3vh7nRlP8ZcEutXA8po7GZC0,529
+atdata-0.1.3a2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+atdata-0.1.3a2.dist-info/entry_points.txt,sha256=6-iQr1veSTq-ac94bLyfcyGHprrZWevPEd12BWX37tQ,39
+atdata-0.1.3a2.dist-info/licenses/LICENSE,sha256=Pz2eACSxkhsGfW9_iN60pgy-enjnbGTj8df8O3ebnQQ,16726
+atdata-0.1.3a2.dist-info/RECORD,,

atdata-0.1.2b1.dist-info/RECORD

@@ -1,9 +0,0 @@
-atdata/__init__.py,sha256=YnlohxQwTUK6V84XHm2gdeCQH5sIrTHVLSApB-nt_z8,216
-atdata/_helpers.py,sha256=R63JhXewAKZYnZ9Th7R6yZh0IOUPYGBsth3FpRUMD-U,503
-atdata/dataset.py,sha256=pBaND2D33JiJoiL9CtCTBa3octtifa21P19K076sW3Q,15905
-atdata/lens.py,sha256=ikExMWdGP3QH-bEuUDNAYO_ZjeaKJTfL9lpaN9CrRB4,2624
-atdata-0.1.2b1.dist-info/METADATA,sha256=WtHM3N0kMxJKwPXl5cluRNnvk49WU_e72lVhmcInraY,529
-atdata-0.1.2b1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-atdata-0.1.2b1.dist-info/entry_points.txt,sha256=6-iQr1veSTq-ac94bLyfcyGHprrZWevPEd12BWX37tQ,39
-atdata-0.1.2b1.dist-info/licenses/LICENSE,sha256=Pz2eACSxkhsGfW9_iN60pgy-enjnbGTj8df8O3ebnQQ,16726
-atdata-0.1.2b1.dist-info/RECORD,,