datachain 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl

datachain/catalog/catalog.py

@@ -1609,13 +1609,22 @@ class Catalog:
                 ...
             }
         """
-        from datachain.lib.signal_schema import SignalSchema
+        from datachain.lib.signal_schema import DEFAULT_DELIMITER, SignalSchema
 
         version = self.get_dataset(dataset_name).get_version(dataset_version)
 
-        file_signals_values = SignalSchema.deserialize(
-            version.feature_schema
-        ).get_file_signals_values(row)
+        file_signals_values = {}
+
+        schema = SignalSchema.deserialize(version.feature_schema)
+        for file_signals in schema.get_file_signals():
+            prefix = file_signals.replace(".", DEFAULT_DELIMITER) + DEFAULT_DELIMITER
+            file_signals_values[file_signals] = {
+                c_name.removeprefix(prefix): c_value
+                for c_name, c_value in row.items()
+                if c_name.startswith(prefix)
+                and DEFAULT_DELIMITER not in c_name.removeprefix(prefix)
+            }
+
         if not file_signals_values:
             return None
 

datachain/lib/signal_schema.py

@@ -13,17 +13,13 @@ from datachain.lib.feature import (
     convert_type_to_datachain,
 )
 from datachain.lib.feature_registry import Registry
-from datachain.lib.file import File, IndexedFile, TextFile
-from datachain.lib.image import ImageFile
+from datachain.lib.file import File
 from datachain.lib.utils import DataChainParamsError
-from datachain.lib.webdataset import TarStream, WDSAllFile, WDSBasic
-from datachain.lib.webdataset_laion import Laion, WDSLaion
 
 if TYPE_CHECKING:
     from datachain.catalog import Catalog
 
 
-# TODO fix hardcoded Feature class names with://github.com/iterative/dvcx/issues/1625
 NAMES_TO_TYPES = {
     "int": int,
     "str": str,
@@ -33,15 +29,6 @@ NAMES_TO_TYPES = {
     "dict": dict,
     "bytes": bytes,
     "datetime": datetime,
-    "WDSLaion": WDSLaion,
-    "Laion": Laion,
-    "Source": IndexedFile,
-    "File": File,
-    "ImageFile": ImageFile,
-    "TextFile": TextFile,
-    "TarStream": TarStream,
-    "WDSBasic": WDSBasic,
-    "WDSAllFile": WDSAllFile,
 }
 
 
@@ -239,37 +226,6 @@ class SignalSchema:
             if has_subtree and issubclass(type_, File):
                 yield ".".join(path)
 
-    def get_file_signals_values(self, row: dict[str, Any]) -> dict[str, Any]:
-        """
-        Method that returns values with clean field names (without prefix) for
-        all file signals found in this schema for some row
-        Output example:
-        {
-            laion.file: {
-                "source": "s3://ldb-public",
-                "name": "dog.jpg",
-                ...
-            },
-            meta.file: {
-                "source": "s3://datacomp",
-                "name": "cat.jpg",
-                ...
-            }
-        }
-        """
-        res = {}
-
-        for file_signals in self.get_file_signals():
-            prefix = file_signals.replace(".", DEFAULT_DELIMITER) + DEFAULT_DELIMITER
-            res[file_signals] = {
-                c_name.removeprefix(prefix): c_value
-                for c_name, c_value in row.items()
-                if c_name.startswith(prefix)
-                and DEFAULT_DELIMITER not in c_name.removeprefix(prefix)
-            }
-
-        return res
-
     def create_model(self, name: str) -> type[Feature]:
         fields = {key: (value, None) for key, value in self.values.items()}
 

{datachain-0.2.2.dist-info → datachain-0.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.2.2
+Version: 0.2.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.2.2.dist-info → datachain-0.2.4.dist-info}/RECORD

@@ -16,7 +16,7 @@ datachain/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/storage.py,sha256=RiSJLYdHUjnrEWkLBKPcETHpAxld_B2WxLg711t0aZI,3733
 datachain/utils.py,sha256=12yQAV8tfyCHqp_xJcJBeNnr1L_BO8e2bOPyXdM68gs,10759
 datachain/catalog/__init__.py,sha256=g2iAAFx_gEIrqshXlhSEbrc8qDaEH11cjU40n3CHDz4,409
-datachain/catalog/catalog.py,sha256=JbrISLLWVCqqHMgiOI2sTFLeRyCrtwukFvaN73PFHr4,79161
+datachain/catalog/catalog.py,sha256=pulKGJgAmxqSmFqBhA-J0wCKdBqGX4vqpV0cAvV6vUw,79578
 datachain/catalog/datasource.py,sha256=D-VWIVDCM10A8sQavLhRXdYSCG7F4o4ifswEF80_NAQ,1412
 datachain/catalog/loader.py,sha256=GJ8zhEYkC7TuaPzCsjJQ4LtTdECu-wwYzC12MikPOMQ,7307
 datachain/catalog/subclass.py,sha256=B5R0qxeTYEyVAAPM1RutBPSoXZc8L5mVVZeSGXki9Sw,2096
@@ -55,7 +55,7 @@ datachain/lib/iptc_exif_xmp.py,sha256=xrbxFeY-wRP6T5JsUgE3EXfTxKvZVymRaRD_VIfxD0
 datachain/lib/meta_formats.py,sha256=wIVVLRLp45Zk4vjZRd_P1UtD24vpDCb-vILWtcsACwk,6630
 datachain/lib/pytorch.py,sha256=Z7iZCsqJzUT0PynVo23Xu4Fx7qIuuEZyH83R1tR5mfI,5561
 datachain/lib/settings.py,sha256=6Nkoh8riETrftYwDp3aniK53Dsjc07MdztL8N0cW1D8,2849
-datachain/lib/signal_schema.py,sha256=6YOWWzmaL0PvruTym7Xdq2ZQuhaDdpzV2hdjT3uHvmo,11669
+datachain/lib/signal_schema.py,sha256=KTegbx-yMvtaKEoUxLgDx5MxMA8De-nmdtqnV1932N8,10151
 datachain/lib/text.py,sha256=PUT1O0jNJoQGsuhff2LgDpzTWk2eMdwIKqEDBrE448M,1307
 datachain/lib/udf.py,sha256=kMlOsHCVybnnq4AMtYqjylZH7x2tGE62FsDPOu9qhWM,6612
 datachain/lib/udf_signature.py,sha256=CUKgoVpM_N8CgvMncpAw2RYchoiJdAGdDSdluoP0hIk,7161
@@ -92,9 +92,9 @@ datachain/sql/sqlite/__init__.py,sha256=TAdJX0Bg28XdqPO-QwUVKy8rg78cgMileHvMNot7
 datachain/sql/sqlite/base.py,sha256=nPMF6_FF04hclDNZev_YfxMgbJAsWEdF-rU2pUhqBtc,12048
 datachain/sql/sqlite/types.py,sha256=oP93nLfTBaYnN0z_4Dsv-HZm8j9rrUf1esMM-z3JLbg,1754
 datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR0,469
-datachain-0.2.2.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.2.2.dist-info/METADATA,sha256=0zLcpMCLlgU7bAxHYFmXH4ewJlxqxxWcdlcOIlv6Skg,14399
-datachain-0.2.2.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
-datachain-0.2.2.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.2.2.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.2.2.dist-info/RECORD,,
+datachain-0.2.4.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.2.4.dist-info/METADATA,sha256=rWswQ1xeEbhMXJ3xRTiFhWjgEqjhSX4ay-ashGNxf8o,14399
+datachain-0.2.4.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
+datachain-0.2.4.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.2.4.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.2.4.dist-info/RECORD,,