datachain 0.2.4__py3-none-any.whl → 0.2.5__py3-none-any.whl

datachain/lib/dc.py

@@ -137,6 +137,7 @@ class DataChain(DatasetQuery):
             indexing_column_types=File._datachain_column_types,
         )
         self._settings = Settings()
+        self._setup = {}
 
         if self.feature_schema:
             self.signals_schema = SignalSchema.deserialize(self.feature_schema)
@@ -536,9 +537,9 @@ class DataChain(DatasetQuery):
         name = self.name or ""
 
         sign = UdfSignature.parse(name, signal_map, func, params, output, is_generator)
-        params_schema = self.signals_schema.slice(sign.params)
+        params_schema = self.signals_schema.slice(sign.params, self._setup)
 
-        return UDFBase._create(target_class, sign, params_schema, self.catalog)
+        return UDFBase._create(target_class, sign, params_schema)
 
     def _extend_features(self, method_name, *args, **kwargs):
         super_func = getattr(super(), method_name)
@@ -569,18 +570,6 @@ class DataChain(DatasetQuery):
         chain.signals_schema = new_schema
         return chain
 
-    def get_values(self, *cols: str) -> Iterator[list]:
-        """Iterate over rows, getting feature values and applying reader calls.
-
-        If columns are specified - limit them to specified columns.
-        """
-        for features in self.iterate(*cols):
-            yield [fr.get_value() if isinstance(fr, Feature) else fr for fr in features]  # type: ignore[union-attr,call-arg]
-
-    def get_one_value(self, col: str) -> Iterator:
-        for item in self.get_values(col):
-            yield item[0]
-
     def iterate(self, *cols: str) -> Iterator[list[FeatureType]]:
         """Iterate over rows.
 
@@ -604,12 +593,6 @@ class DataChain(DatasetQuery):
     def collect_one(self, col: str) -> list[FeatureType]:
         return list(self.iterate_one(col))
 
-    def collect_values(self, *cols: str) -> list[list]:
-        return list(self.get_values(*cols))
-
-    def collect_one_value(self, col: str) -> list:
-        return list(self.get_one_value(col))
-
     def to_pytorch(self, **kwargs):
         """Convert to pytorch dataset format."""
 
@@ -931,3 +914,12 @@ class DataChain(DatasetQuery):
 
     def max(self, fr: FeatureType):  # type: ignore[override]
         return self._extend_features("max", fr)
+
+    def setup(self, **kwargs) -> "Self":
+        intersection = set(self._setup.keys()) & set(kwargs.keys())
+        if intersection:
+            keys = ", ".join(intersection)
+            raise DatasetPrepareError(self.name, f"this value(s) already setup: {keys}")
+
+        self._setup = self._setup | kwargs
+        return self

datachain/lib/feature_utils.py

@@ -12,9 +12,6 @@ from datachain.lib.feature import (
     convert_type_to_datachain,
 )
 from datachain.lib.utils import DataChainParamsError
-from datachain.query.schema import Column
-
-FeatureLike = Union[type["Feature"], Column, str]
 
 AUTO_FEATURE_PREFIX = "_auto_fr"
 SUFFIX_SYMBOLS = string.digits + string.ascii_lowercase

datachain/lib/file.py

@@ -238,9 +238,6 @@ class File(FileFeature):
         return self._catalog.get_client(self.source).fs
 
 
-BinaryFile = File
-
-
 class TextFile(File):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)

datachain/lib/pytorch.py

@@ -8,6 +8,7 @@ from torch.utils.data import IterableDataset, get_worker_info
 
 from datachain.catalog import Catalog, get_catalog
 from datachain.lib.dc import DataChain
+from datachain.lib.feature import Feature
 from datachain.lib.text import convert_text
 
 if TYPE_CHECKING:
@@ -100,8 +101,14 @@ class PytorchDataset(IterableDataset):
         if self.num_samples > 0:
             ds = ds.sample(self.num_samples)
         ds = ds.chunk(total_rank, total_workers)
-        stream = ds.get_values()
-        for row in stream:
+        stream = ds.iterate()
+        for row_features in stream:
+            row = []
+            for fr in row_features:
+                if isinstance(fr, Feature):
+                    row.append(fr.get_value())  # type: ignore[unreachable]
+                else:
+                    row.append(fr)
             # Apply transforms
             if self.transform:
                 try:

datachain/lib/signal_schema.py

@@ -1,7 +1,7 @@
 import copy
 from collections.abc import Iterator, Sequence
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Optional, Union, get_args, get_origin
+from typing import TYPE_CHECKING, Any, Callable, Optional, Union, get_args, get_origin
 
 from pydantic import create_model
 
@@ -42,6 +42,11 @@ class SignalResolvingError(SignalSchemaError):
         super().__init__(f"cannot resolve signal name{name}: {msg}")
 
 
+class SetupError(SignalSchemaError):
+    def __init__(self, name: str, msg: str):
+        super().__init__(f"cannot setup value '{name}': {msg}")
+
+
 class SignalResolvingTypeError(SignalResolvingError):
     def __init__(self, method: str, field):
         super().__init__(
@@ -52,9 +57,31 @@ class SignalResolvingTypeError(SignalResolvingError):
 
 
 class SignalSchema:
-    def __init__(self, values: dict[str, FeatureType]):
+    def __init__(
+        self,
+        values: dict[str, FeatureType],
+        setup: Optional[dict[str, Callable]] = None,
+    ):
         self.values = values
-        self.tree = self._build_tree()
+        self.tree = self._build_tree(values)
+
+        self.setup_func = setup or {}
+        self.setup_values = None
+        for key, func in self.setup_func.items():
+            if not callable(func):
+                raise SetupError(key, "value must be function or callable class")
+
+    def _init_setup_values(self):
+        if self.setup_values is not None:
+            return self.setup_values
+
+        res = {}
+        for key, func in self.setup_func.items():
+            try:
+                res[key] = func()
+            except Exception as ex:
+                raise SetupError(key, f"error when call function: '{ex}'") from ex
+        self.setup_values = res
 
     @staticmethod
     def from_column_types(col_types: dict[str, Any]) -> "SignalSchema":
@@ -111,16 +138,22 @@ class SignalSchema:
     def to_udf_spec(self) -> dict[str, Any]:
         res = {}
         for path, type_, has_subtree, _ in self.get_flat_tree():
+            if path[0] in self.setup_func:
+                continue
             if not has_subtree:
                 db_name = DEFAULT_DELIMITER.join(path)
                 res[db_name] = convert_type_to_datachain(type_)
         return res
 
     def row_to_objs(self, row: Sequence[Any]) -> list[FeatureType]:
+        self._init_setup_values()
+
         objs = []
         pos = 0
-        for fr_type in self.values.values():
-            if Feature.is_feature(fr_type):
+        for name, fr_type in self.values.items():
+            if val := self.setup_values.get(name, None):  # type: ignore[attr-defined]
+                objs.append(val)
+            elif Feature.is_feature(fr_type):
                 j, pos = fr_type._unflatten_to_json_pos(row, pos)  # type: ignore[union-attr]
                 objs.append(fr_type(**j))
             else:
@@ -135,8 +168,14 @@ class SignalSchema:
             if Feature.is_feature(fr)
         )
 
-    def slice(self, keys: Sequence[str]) -> "SignalSchema":
-        return SignalSchema({k: self.values[k] for k in keys if k in self.values})
+    def slice(
+        self, keys: Sequence[str], setup: Optional[dict[str, Callable]] = None
+    ) -> "SignalSchema":
+        setup = setup or {}
+        setup_no_types = dict.fromkeys(setup.keys(), str)
+        union = self.values | setup_no_types
+        schema = {k: union[k] for k in keys if k in union}
+        return SignalSchema(schema, setup)
 
     def row_to_features(self, row: Sequence, catalog: "Catalog") -> list[FeatureType]:
         res = []
@@ -235,10 +274,11 @@ class SignalSchema:
             **fields,
         )
 
-    def _build_tree(self) -> dict[str, Any]:
+    @staticmethod
+    def _build_tree(values: dict[str, FeatureType]) -> dict[str, Any]:
         res = {}
 
-        for name, val in self.values.items():
+        for name, val in values.items():
             subtree = val.build_tree() if Feature.is_feature(val) else None  # type: ignore[union-attr]
             res[name] = (val, subtree)
 

datachain/lib/udf.py

@@ -64,7 +64,6 @@ class UDFBase(AbstractUDF):
         target_class: type["UDFBase"],
         sign: UdfSignature,
         params: SignalSchema,
-        catalog,
     ) -> "UDFBase":
         if isinstance(sign.func, AbstractUDF):
             if not isinstance(sign.func, target_class):  # type: ignore[unreachable]

{datachain-0.2.4.dist-info → datachain-0.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.2.4
+Version: 0.2.5
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -59,12 +59,11 @@ Requires-Dist: mkdocs-section-index >=0.3.6 ; extra == 'docs'
 Requires-Dist: mkdocstrings-python >=1.6.3 ; extra == 'docs'
 Requires-Dist: mkdocs-literate-nav >=0.6.1 ; extra == 'docs'
 Provides-Extra: remote
-Requires-Dist: datachain[pandas] ; extra == 'remote'
 Requires-Dist: lz4 ; extra == 'remote'
 Requires-Dist: msgpack <2,>=1.0.4 ; extra == 'remote'
 Requires-Dist: requests >=2.22.0 ; extra == 'remote'
 Provides-Extra: tests
-Requires-Dist: datachain[cv,pandas,remote,vector] ; extra == 'tests'
+Requires-Dist: datachain[cv,remote,vector] ; extra == 'tests'
 Requires-Dist: pytest <9,>=8 ; extra == 'tests'
 Requires-Dist: pytest-sugar >=0.9.6 ; extra == 'tests'
 Requires-Dist: pytest-cov >=4.1.0 ; extra == 'tests'
@@ -332,6 +331,12 @@ Datasets can be exported to CSV or webdataset formats. However, a much better wa
                  parallel=2,
              )
 
+Tutorials
+------------------
+
+* `Computer Vision <examples/computer_vision/fashion_product_images/1-quick-start.ipynb>`_ (try in `Colab <https://colab.research.google.com/github/iterative/dvcx/blob/main/examples/computer_vision/fashion_product_images/1-quick-start.ipynb>`__)
+* `Multimodal <examples/multimodal/clip_fine_tuning.ipynb>`_ (try in `Colab <https://colab.research.google.com/github/iterative/dvclive/blob/main/examples/multimodal/clip_fine_tuning.ipynb>`__)
+
 💻  More examples
 ------------------
 

{datachain-0.2.4.dist-info → datachain-0.2.5.dist-info}/RECORD

@@ -41,11 +41,11 @@ datachain/lib/arrow.py,sha256=FF3WWUOjB6Prw8ygfiLsrVfrdob0S01lPzEazuGqoO8,2556
 datachain/lib/cached_stream.py,sha256=t2ifK0hZVZiVn0MQ8D3FaFK1-qK84TwJW2Dw1SRsw9g,1066
 datachain/lib/claude.py,sha256=iAauA1zNVNONpLzUo1t0QN5PZ5Ot6cZkfib7Ka_c638,1969
 datachain/lib/clip.py,sha256=rDeZlFGs0DXBlpmh5ZQJhR9Sz13bWAZGQjfYm1hsUI4,5388
-datachain/lib/dc.py,sha256=Sf99R0oOqf7tlS2gieaG56z3bF7YVcMjhJOZrFRfFs8,34778
+datachain/lib/dc.py,sha256=D3cgib-U0Mo0x5wEK1_NfgymAldHqCvooZwtyohi53Q,34426
 datachain/lib/feature.py,sha256=QDloA9HE7URf9J_veKrguYBvSg-0cbXZFTswNxrKsB8,12135
 datachain/lib/feature_registry.py,sha256=K3jGQzBp2HZDjR9hdGe1BZaXOAne8RpkCRRQdTVjkTs,1622
-datachain/lib/feature_utils.py,sha256=F4ZENO6tTQvd36a-O1AurYjFSUpoyZaT4qgXsKjQDts,4650
-datachain/lib/file.py,sha256=TdhsPYmG0Atkd_QAO997oA8AuM854wNbjjLLT1uiD2M,8346
+datachain/lib/feature_utils.py,sha256=oqRO_Mu3epOr1HPTxAJ8TxsJshUfKJQtulCDgHtInMI,4557
+datachain/lib/file.py,sha256=LGBwC7tFU7VcSWk5kjPpEWPBQas5me69L2uTDNvYXGM,8326
 datachain/lib/gpt4_vision.py,sha256=idyXVZVWzltstGaVIu5RYE5UNbdqcPEjIWy81O1MwkM,2922
 datachain/lib/hf_image_to_text.py,sha256=HiPSWzJRDT-vnz9DXJbJBNCMNl9wmpxiSS3PbbVz8SE,3310
 datachain/lib/hf_pipeline.py,sha256=f0AH_XCziOF1OKN3d1w1swTBLaeajMJ8xgdsX37i5-o,2287
@@ -53,11 +53,11 @@ datachain/lib/image.py,sha256=ZYfDqr9p-RRmWBeWFQwXLS1J3vQS616ykfMUvQVpqBY,2717
 datachain/lib/image_transform.py,sha256=NXWtnVOcofWBgl_YMxb4ABpaT7JTBMx7tLKvErH1IC4,3024
 datachain/lib/iptc_exif_xmp.py,sha256=xrbxFeY-wRP6T5JsUgE3EXfTxKvZVymRaRD_VIfxD0A,2236
 datachain/lib/meta_formats.py,sha256=wIVVLRLp45Zk4vjZRd_P1UtD24vpDCb-vILWtcsACwk,6630
-datachain/lib/pytorch.py,sha256=Z7iZCsqJzUT0PynVo23Xu4Fx7qIuuEZyH83R1tR5mfI,5561
+datachain/lib/pytorch.py,sha256=Ea1sXhborF6zcywQjLpXgKnbr1lTez4Bfu3m0Gr78FI,5843
 datachain/lib/settings.py,sha256=6Nkoh8riETrftYwDp3aniK53Dsjc07MdztL8N0cW1D8,2849
-datachain/lib/signal_schema.py,sha256=KTegbx-yMvtaKEoUxLgDx5MxMA8De-nmdtqnV1932N8,10151
+datachain/lib/signal_schema.py,sha256=hD56hyO1H3A5H2oyTUwPcNu6UOQ_XY0DeA0nrXBqFaU,11492
 datachain/lib/text.py,sha256=PUT1O0jNJoQGsuhff2LgDpzTWk2eMdwIKqEDBrE448M,1307
-datachain/lib/udf.py,sha256=kMlOsHCVybnnq4AMtYqjylZH7x2tGE62FsDPOu9qhWM,6612
+datachain/lib/udf.py,sha256=axMvqYz4tdyg_C3nyuOcDsu3Aqr19jWv2vl54U_8LQM,6595
 datachain/lib/udf_signature.py,sha256=CUKgoVpM_N8CgvMncpAw2RYchoiJdAGdDSdluoP0hIk,7161
 datachain/lib/unstructured.py,sha256=9Y6rAelXdYqkNbPaqz6DhXjhS8d6qXcP0ieIsWkzvkk,1143
 datachain/lib/utils.py,sha256=5-kJlAZE0D9nXXweAjo7-SP_AWGo28feaDByONYaooQ,463
@@ -92,9 +92,9 @@ datachain/sql/sqlite/__init__.py,sha256=TAdJX0Bg28XdqPO-QwUVKy8rg78cgMileHvMNot7
 datachain/sql/sqlite/base.py,sha256=nPMF6_FF04hclDNZev_YfxMgbJAsWEdF-rU2pUhqBtc,12048
 datachain/sql/sqlite/types.py,sha256=oP93nLfTBaYnN0z_4Dsv-HZm8j9rrUf1esMM-z3JLbg,1754
 datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR0,469
-datachain-0.2.4.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.2.4.dist-info/METADATA,sha256=rWswQ1xeEbhMXJ3xRTiFhWjgEqjhSX4ay-ashGNxf8o,14399
-datachain-0.2.4.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
-datachain-0.2.4.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.2.4.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.2.4.dist-info/RECORD,,
+datachain-0.2.5.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.2.5.dist-info/METADATA,sha256=VDc20_FTRJRF63521iwyb67LMCYVXn2BqeER4IVc840,14810
+datachain-0.2.5.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
+datachain-0.2.5.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.2.5.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.2.5.dist-info/RECORD,,