datachain 0.3.8__py3-none-any.whl → 0.3.9__py3-none-any.whl

datachain/catalog/catalog.py

@@ -1560,17 +1560,8 @@ class Catalog:
         version = self.get_dataset(dataset_name).get_version(dataset_version)
 
         file_signals_values = {}
-        file_schemas = {}
-        # TODO: To remove after we properly fix deserialization
-        for signal, type_name in version.feature_schema.items():
-            from datachain.lib.model_store import ModelStore
 
-            type_name_parsed, v = ModelStore.parse_name_version(type_name)
-            fr = ModelStore.get(type_name_parsed, v)
-            if fr and issubclass(fr, File):
-                file_schemas[signal] = type_name
-
-        schema = SignalSchema.deserialize(file_schemas)
+        schema = SignalSchema.deserialize(version.feature_schema)
         for file_signals in schema.get_signals(File):
             prefix = file_signals.replace(".", DEFAULT_DELIMITER) + DEFAULT_DELIMITER
             file_signals_values[file_signals] = {
@@ -1916,7 +1907,7 @@ class Catalog:
         """
         from datachain.query.dataset import ExecutionResult
 
-        feature_file = tempfile.NamedTemporaryFile(
+        feature_file = tempfile.NamedTemporaryFile(  # noqa: SIM115
             dir=os.getcwd(), suffix=".py", delete=False
         )
         _, feature_module = os.path.split(feature_file.name)

datachain/lib/arrow.py

@@ -131,7 +131,7 @@ def arrow_type_mapper(col_type: pa.DataType) -> type:  # noqa: PLR0911
 
 
 def _nrows_file(file: File, nrows: int) -> str:
-    tf = NamedTemporaryFile(delete=False)
+    tf = NamedTemporaryFile(delete=False)  # noqa: SIM115
     with file.open(mode="r") as reader:
         with open(tf.name, "a") as writer:
             for row, line in enumerate(reader):

datachain/lib/dc.py

@@ -1153,17 +1153,35 @@ class DataChain(DatasetQuery):
         self,
         other: "DataChain",
         on: Optional[Union[str, Sequence[str]]] = None,
+        right_on: Optional[Union[str, Sequence[str]]] = None,
     ) -> "Self":
         """Remove rows that appear in another chain.
 
         Parameters:
             other: chain whose rows will be removed from `self`
-            on: columns to consider for determining row equality. If unspecified,
-                defaults to all common columns between `self` and `other`.
+            on: columns to consider for determining row equality in `self`.
+                If unspecified, defaults to all common columns
+                between `self` and `other`.
+            right_on: columns to consider for determining row equality in `other`.
+                If unspecified, defaults to the same values as `on`.
         """
         if isinstance(on, str):
+            if not on:
+                raise DataChainParamsError("'on' cannot be an empty string")
             on = [on]
-        if on is None:
+        elif isinstance(on, Sequence):
+            if not on or any(not col for col in on):
+                raise DataChainParamsError("'on' cannot contain empty strings")
+
+        if isinstance(right_on, str):
+            if not right_on:
+                raise DataChainParamsError("'right_on' cannot be an empty string")
+            right_on = [right_on]
+        elif isinstance(right_on, Sequence):
+            if not right_on or any(not col for col in right_on):
+                raise DataChainParamsError("'right_on' cannot contain empty strings")
+
+        if on is None and right_on is None:
             other_columns = set(other._effective_signals_schema.db_signals())
             signals = [
                 c
@@ -1172,16 +1190,29 @@ class DataChain(DatasetQuery):
             ]
             if not signals:
                 raise DataChainParamsError("subtract(): no common columns")
-        elif not isinstance(on, Sequence):
-            raise TypeError(
-                f"'on' must be 'str' or 'Sequence' object but got type '{type(on)}'",
-            )
-        elif not on:
+        elif on is not None and right_on is None:
+            right_on = on
+            signals = list(self.signals_schema.resolve(*on).db_signals())
+        elif on is None and right_on is not None:
             raise DataChainParamsError(
-                "'on' cannot be empty",
+                "'on' must be specified when 'right_on' is provided"
             )
         else:
-            signals = self.signals_schema.resolve(*on).db_signals()  # type: ignore[assignment]
+            if not isinstance(on, Sequence) or not isinstance(right_on, Sequence):
+                raise TypeError(
+                    "'on' and 'right_on' must be 'str' or 'Sequence' object"
+                )
+            if len(on) != len(right_on):
+                raise DataChainParamsError(
+                    "'on' and 'right_on' must have the same length"
+                )
+            signals = list(
+                zip(
+                    self.signals_schema.resolve(*on).db_signals(),
+                    other.signals_schema.resolve(*right_on).db_signals(),
+                )  # type: ignore[arg-type]
+            )
+
         return super()._subtract(other, signals)  # type: ignore[arg-type]
 
     @classmethod

datachain/lib/webdataset.py

@@ -222,7 +222,7 @@ class TarStream(File):
         self._tar = None
 
     def open(self):
-        self._tar = tarfile.open(fileobj=super().open())
+        self._tar = tarfile.open(fileobj=super().open())  # noqa: SIM115
         return self
 
     def getmembers(self) -> list[tarfile.TarInfo]:

datachain/query/dataset.py

@@ -296,15 +296,23 @@ class DatasetDiffOperation(Step):
 
 @frozen
 class Subtract(DatasetDiffOperation):
-    on: Sequence[str]
+    on: Sequence[tuple[str, str]]
 
     def query(self, source_query: Select, target_query: Select) -> sa.Selectable:
         sq = source_query.alias("source_query")
         tq = target_query.alias("target_query")
         where_clause = sa.and_(
-            getattr(sq.c, col_name).is_not_distinct_from(getattr(tq.c, col_name))
-            for col_name in self.on
-        )  # type: ignore[arg-type]
+            *[
+                getattr(
+                    sq.c, col_name[0] if isinstance(col_name, tuple) else col_name
+                ).is_not_distinct_from(
+                    getattr(
+                        tq.c, col_name[1] if isinstance(col_name, tuple) else col_name
+                    )
+                )
+                for col_name in self.on
+            ]
+        )
         return sq.select().except_(sq.select().where(where_clause))
 
 
@@ -1571,10 +1579,10 @@ class DatasetQuery:
 
     @detach
     def subtract(self, dq: "DatasetQuery") -> "Self":
-        return self._subtract(dq, on=["source", "path"])
+        return self._subtract(dq, on=[("source", "source"), ("path", "path")])
 
     @detach
-    def _subtract(self, dq: "DatasetQuery", on: Sequence[str]) -> "Self":
+    def _subtract(self, dq: "DatasetQuery", on: Sequence[tuple[str, str]]) -> "Self":
         query = self.clone()
         query.steps.append(Subtract(dq, self.catalog, on=on))
         return query

{datachain-0.3.8.dist-info → datachain-0.3.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.3.8
+Version: 0.3.9
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0
@@ -115,31 +115,30 @@ AI 🔗 DataChain
 
 DataChain is a modern Pythonic data-frame library designed for artificial intelligence.
 It is made to organize your unstructured data into datasets and wrangle it at scale on
-your local machine.
+your local machine. Datachain does not abstract or hide the AI models and API calls, but helps to integrate them into the postmodern data stack.
 
 Key Features
 ============
 
 📂 **Storage as a Source of Truth.**
-   - Process unstructured data without redundant copies: S3, GCP, Azure, and local
+   - Process unstructured data without redundant copies from S3, GCP, Azure, and local
      file systems.
-   - Multimodal data: images, video, text, PDFs, JSONs, CSVs, parquet.
-   - Join files and metadata together into persistent, versioned, columnar datasets.
+   - Multimodal data support: images, video, text, PDFs, JSONs, CSVs, parquet.
+   - Unite files and metadata together into persistent, versioned, columnar datasets.
 
 🐍 **Python-friendly data pipelines.**
    - Operate on Python objects and object fields.
-   - Built-in parallelization and out-of-memory compute without a need in SQL or
-     Spark jobs.
+   - Built-in parallelization and out-of-memory compute without SQL or Spark.
 
 🧠 **Data Enrichment and Processing.**
-   - Generate metadata columns using local AI models and LLM APIs.
-   - Filter, join, and group by AI metadata. Vector similarity search.
-   - Pass datasets to Pytorch and Tensorflow, or export back into storage.
+   - Generate metadata using local AI models and LLM APIs.
+   - Filter, join, and group by metadata. Search by vector embeddings.
+   - Pass datasets to Pytorch and Tensorflow, or export them back into storage.
 
 🚀 **Efficiency.**
    - Parallelization, out-of-memory workloads and data caching.
    - Vectorized operations on Python object fields: sum, count, avg, etc.
-   - Vector search on embeddings.
+   - Optimized vector search.
 
 
 Quick Start
@@ -164,7 +163,7 @@ where each image has a matching JSON file like `cat.1009.json`:
         "inference": {"class": "dog", "confidence": 0.68}
     }
 
-Example of downloading only high-confidence cat images using JSON metadata:
+Example of downloading only "high-confidence cat" inferred images using JSON metadata:
 
 
 .. code:: py
@@ -234,7 +233,7 @@ detected are then copied to the local directory.
 LLM judging chatbots
 =============================
 
-LLMs can work as efficient universal classifiers. In the example below,
+LLMs can work as universal classifiers. In the example below,
 we employ a free API from Mistral to judge the `publicly available`_ chatbot dialogs. Please get a free
 Mistral API key at https://console.mistral.ai
 

{datachain-0.3.8.dist-info → datachain-0.3.9.dist-info}/RECORD

@@ -17,7 +17,7 @@ datachain/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/storage.py,sha256=RiSJLYdHUjnrEWkLBKPcETHpAxld_B2WxLg711t0aZI,3733
 datachain/utils.py,sha256=ROVCLwb37VmFRzgTlSGUDw4eJNgYGiQ4yMX581HfUX8,12988
 datachain/catalog/__init__.py,sha256=g2iAAFx_gEIrqshXlhSEbrc8qDaEH11cjU40n3CHDz4,409
-datachain/catalog/catalog.py,sha256=6S4AnDos4sGYGhy4wNSyV2pKPQNXvo819cd3Dl8Htgg,78271
+datachain/catalog/catalog.py,sha256=kGpp9IEyr1YS7QFWjLYprRT1gp7freyt-WLaLNzqUZg,77859
 datachain/catalog/datasource.py,sha256=D-VWIVDCM10A8sQavLhRXdYSCG7F4o4ifswEF80_NAQ,1412
 datachain/catalog/loader.py,sha256=-6VelNfXUdgUnwInVyA8g86Boxv2xqhTh9xNS-Zlwig,8242
 datachain/catalog/subclass.py,sha256=B5R0qxeTYEyVAAPM1RutBPSoXZc8L5mVVZeSGXki9Sw,2096
@@ -38,11 +38,11 @@ datachain/data_storage/serializer.py,sha256=6G2YtOFqqDzJf1KbvZraKGXl2XHZyVml2kru
 datachain/data_storage/sqlite.py,sha256=jLgkvikYkENQUO_ykoNFfsBc2ofZXwFHLMa1nyWP3aw,28316
 datachain/data_storage/warehouse.py,sha256=cvlfa-nyIxqrrpSRtCdeVjlTwhn7rcIoWjOq91HhItU,33668
 datachain/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datachain/lib/arrow.py,sha256=W8bIxMIe_b3dqMFYKGWmfbC_7Xe0gV3UiJjQ2i4EYLA,4925
+datachain/lib/arrow.py,sha256=17-jHLdYhsSdO5kfKWpBS5OAWbMjNi5r8ao0zGXUBoA,4941
 datachain/lib/clip.py,sha256=33RL11OIqfbwyhvBgiMGM8rDAnZx1IRmxk9dY89ls3Q,6130
 datachain/lib/data_model.py,sha256=gHIjlow84GMRDa78yLL1Ud-N18or21fnTyPEwsatpXY,2045
 datachain/lib/dataset_info.py,sha256=lONGr71ozo1DS4CQEhnpKORaU4qFb6Ketv8Xm8CVm2U,2188
-datachain/lib/dc.py,sha256=wdMzFLglOhwWKHwh4qcLA0ezMrjuRJq2il2WnkHjyag,62490
+datachain/lib/dc.py,sha256=tY_ccOsv9njsXF23cwoZ7tSTCDKCfakyRvsIBLKE0SE,63976
 datachain/lib/file.py,sha256=ZHpdilDPYCob8uqtwUPtBvBNxVvQRq4AC_0IGg5m-G4,12003
 datachain/lib/hf.py,sha256=mYaHFPS4CW2-stRZHBMWW-NKN4dhrnhjZobBgRocnvo,5317
 datachain/lib/image.py,sha256=WbcwSaFzuyqjg4x4hH5CUogeUQjkZFjQHqw_oDEV1nA,2655
@@ -57,7 +57,7 @@ datachain/lib/udf.py,sha256=nG7DDuPgZ5ZuijwvDoCq-OZMxlDM8vFNzyxMmik0Y1c,11716
 datachain/lib/udf_signature.py,sha256=gMStcEeYJka5M6cg50Z9orC6y6HzCAJ3MkFqqn1fjZg,7137
 datachain/lib/utils.py,sha256=5-kJlAZE0D9nXXweAjo7-SP_AWGo28feaDByONYaooQ,463
 datachain/lib/vfile.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datachain/lib/webdataset.py,sha256=SsjCKLSKEkHRRfeTHQhjoGqNPqIWw_SCWQcUwgUWWP0,8282
+datachain/lib/webdataset.py,sha256=Q3UlCk66341sq-nvFbBCX4Cv3cYXBK9n12ejG4axPXE,8298
 datachain/lib/webdataset_laion.py,sha256=PQP6tQmUP7Xu9fPuAGK1JDBYA6T5UufYMUTGaxgspJA,2118
 datachain/lib/convert/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/lib/convert/flatten.py,sha256=Uebc5CeqCsacp-nr6IG9i6OGuUavXqdqnoGctZBk3RQ,1384
@@ -68,7 +68,7 @@ datachain/lib/convert/values_to_tuples.py,sha256=YOdbjzHq-uj6-cV2Qq43G72eN2avMND
 datachain/query/__init__.py,sha256=tv-spkjUCYamMN9ys_90scYrZ8kJ7C7d1MTYVmxGtk4,325
 datachain/query/batch.py,sha256=-vlpINJiertlnaoUVv1C95RatU0F6zuhpIYRufJRo1M,3660
 datachain/query/builtins.py,sha256=EmKPYsoQ46zwdyOn54MuCzvYFmfsBn5F8zyF7UBUfrc,2550
-datachain/query/dataset.py,sha256=G6xA3ItIGUJTXhizdAb6S3L1zFwTf8I0w0jHa1A6F4A,61103
+datachain/query/dataset.py,sha256=mHqSyovJlCQ7pKVMQKKKCiTJs3bP1GDXLKpOSpzVxx8,61378
 datachain/query/dispatch.py,sha256=GBh3EZHDp5AaXxrjOpfrpfsuy7Umnqxu-MAXcK9X3gc,12945
 datachain/query/metrics.py,sha256=vsECqbZfoSDBnvC3GQlziKXmISVYDLgHP1fMPEOtKyo,640
 datachain/query/params.py,sha256=O_j89mjYRLOwWNhYZl-z7mi-rkdP7WyFmaDufsdTryE,863
@@ -95,9 +95,9 @@ datachain/sql/sqlite/base.py,sha256=WLPHBhZbXbiqPoRV1VgDrXJqku4UuvJpBhYeQ0k5rI8,
 datachain/sql/sqlite/types.py,sha256=yzvp0sXSEoEYXs6zaYC_2YubarQoZH-MiUNXcpuEP4s,1573
 datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR0,469
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.3.8.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.3.8.dist-info/METADATA,sha256=ivteXQrJgp8dKgIO2pdwUj6Qdg96rbI3Gq0kx5fyxtk,16903
-datachain-0.3.8.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
-datachain-0.3.8.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.3.8.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.3.8.dist-info/RECORD,,
+datachain-0.3.9.dist-info/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.3.9.dist-info/METADATA,sha256=r5uNlVdal7YrsX7nYE56c_Ak8YZIgXqCiSwNJF5KjlY,17015
+datachain-0.3.9.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
+datachain-0.3.9.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.3.9.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.3.9.dist-info/RECORD,,