datachain 0.2.2__tar.gz → 0.2.4__tar.gz

{datachain-0.2.2/src/datachain.egg-info → datachain-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.2.2
+Version: 0.2.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.2.2 → datachain-0.2.4}/src/datachain/catalog/catalog.py

@@ -1609,13 +1609,22 @@ class Catalog:
                 ...
             }
         """
-        from datachain.lib.signal_schema import SignalSchema
+        from datachain.lib.signal_schema import DEFAULT_DELIMITER, SignalSchema
 
         version = self.get_dataset(dataset_name).get_version(dataset_version)
 
-        file_signals_values = SignalSchema.deserialize(
-            version.feature_schema
-        ).get_file_signals_values(row)
+        file_signals_values = {}
+
+        schema = SignalSchema.deserialize(version.feature_schema)
+        for file_signals in schema.get_file_signals():
+            prefix = file_signals.replace(".", DEFAULT_DELIMITER) + DEFAULT_DELIMITER
+            file_signals_values[file_signals] = {
+                c_name.removeprefix(prefix): c_value
+                for c_name, c_value in row.items()
+                if c_name.startswith(prefix)
+                and DEFAULT_DELIMITER not in c_name.removeprefix(prefix)
+            }
+
         if not file_signals_values:
             return None
 

{datachain-0.2.2 → datachain-0.2.4}/src/datachain/lib/signal_schema.py

@@ -13,17 +13,13 @@ from datachain.lib.feature import (
     convert_type_to_datachain,
 )
 from datachain.lib.feature_registry import Registry
-from datachain.lib.file import File, IndexedFile, TextFile
-from datachain.lib.image import ImageFile
+from datachain.lib.file import File
 from datachain.lib.utils import DataChainParamsError
-from datachain.lib.webdataset import TarStream, WDSAllFile, WDSBasic
-from datachain.lib.webdataset_laion import Laion, WDSLaion
 
 if TYPE_CHECKING:
     from datachain.catalog import Catalog
 
 
-# TODO fix hardcoded Feature class names with://github.com/iterative/dvcx/issues/1625
 NAMES_TO_TYPES = {
     "int": int,
     "str": str,
@@ -33,15 +29,6 @@ NAMES_TO_TYPES = {
     "dict": dict,
     "bytes": bytes,
     "datetime": datetime,
-    "WDSLaion": WDSLaion,
-    "Laion": Laion,
-    "Source": IndexedFile,
-    "File": File,
-    "ImageFile": ImageFile,
-    "TextFile": TextFile,
-    "TarStream": TarStream,
-    "WDSBasic": WDSBasic,
-    "WDSAllFile": WDSAllFile,
 }
 
 
@@ -239,37 +226,6 @@ class SignalSchema:
             if has_subtree and issubclass(type_, File):
                 yield ".".join(path)
 
-    def get_file_signals_values(self, row: dict[str, Any]) -> dict[str, Any]:
-        """
-        Method that returns values with clean field names (without prefix) for
-        all file signals found in this schema for some row
-        Output example:
-        {
-            laion.file: {
-                "source": "s3://ldb-public",
-                "name": "dog.jpg",
-                ...
-            },
-            meta.file: {
-                "source": "s3://datacomp",
-                "name": "cat.jpg",
-                ...
-            }
-        }
-        """
-        res = {}
-
-        for file_signals in self.get_file_signals():
-            prefix = file_signals.replace(".", DEFAULT_DELIMITER) + DEFAULT_DELIMITER
-            res[file_signals] = {
-                c_name.removeprefix(prefix): c_value
-                for c_name, c_value in row.items()
-                if c_name.startswith(prefix)
-                and DEFAULT_DELIMITER not in c_name.removeprefix(prefix)
-            }
-
-        return res
-
     def create_model(self, name: str) -> type[Feature]:
         fields = {key: (value, None) for key, value in self.values.items()}
 

{datachain-0.2.2 → datachain-0.2.4/src/datachain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.2.2
+Version: 0.2.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.2.2 → datachain-0.2.4}/tests/unit/lib/test_datachain.py

@@ -767,7 +767,7 @@ def test_parse_parquet_filter_partitions(tmp_dir, catalog):
 
 
 @pytest.mark.parametrize("processes", [False, 2, True])
-def test_parallel(processes):
+def test_parallel(processes, catalog):
     prefix = "t & "
     vals = ["a", "b", "c", "d", "e", "f", "g", "h", "i"]
 

{datachain-0.2.2 → datachain-0.2.4}/tests/unit/lib/test_datachain_bootstrap.py

@@ -24,7 +24,7 @@ class MyMapper(Mapper):
         self.value = MyMapper.TEARDOWN_VALUE
 
 
-def test_udf():
+def test_udf(catalog):
     vals = ["a", "b", "c", "d", "e", "f"]
     chain = DataChain.from_features(key=vals)
 
@@ -36,7 +36,7 @@ def test_udf():
 
 
 @pytest.mark.skip(reason="Skip until tests module will be importer for unit-tests")
-def test_udf_parallel():
+def test_udf_parallel(catalog):
     vals = ["a", "b", "c", "d", "e", "f"]
     chain = DataChain.from_features(key=vals)
 
@@ -45,7 +45,7 @@ def test_udf_parallel():
     assert res == [MyMapper.BOOTSTRAP_VALUE] * len(vals)
 
 
-def test_no_bootstrap_for_callable():
+def test_no_bootstrap_for_callable(catalog):
     class MyMapper:
         def __init__(self):
             self._had_bootstrap = False

{datachain-0.2.2 → datachain-0.2.4}/tests/unit/lib/test_signal_schema.py

@@ -197,36 +197,6 @@ def test_get_file_signals_nested(nested_file_schema):
     assert files == ["f", "my_f", "my_f.nested_file"]
 
 
-def test_get_file_signals_values(nested_file_schema):
-    row = {
-        "name": "Jon",
-        "age": 25,
-        "f__source": "s3://first_bucket",
-        "f__name": "image1.jpeg",
-        "my_f__source": "s3://second_bucket",
-        "my_f__name": "image2.jpeg",
-        "my_f__ref": "reference",
-        "my_f__nested_file__source": "s3://third_bucket",
-        "my_f__nested_file__name": "image3.jpeg",
-    }
-
-    assert nested_file_schema.get_file_signals_values(row) == {
-        "f": {"source": "s3://first_bucket", "name": "image1.jpeg"},
-        "my_f": {
-            "source": "s3://second_bucket",
-            "name": "image2.jpeg",
-            "ref": "reference",
-        },
-        "my_f.nested_file": {"source": "s3://third_bucket", "name": "image3.jpeg"},
-    }
-
-
-def test_get_file_signals_values_no_files():
-    schema = {"name": str, "age": float}
-    row = {"name": "Jon", "age": 25}
-    assert SignalSchema(schema).get_file_signals_values(row) == {}
-
-
 def test_create_model():
     class MyFr(Feature):
         count: int