datachain 0.2.15__tar.gz → 0.2.16__tar.gz

{datachain-0.2.15/src/datachain.egg-info → datachain-0.2.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.2.15
+Version: 0.2.16
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.2.15 → datachain-0.2.16}/examples/computer_vision/iptc_exif_xmp_lib.py

@@ -1,3 +1,4 @@
+# pip install defusedxml
 import json
 
 from PIL import (
@@ -63,7 +64,7 @@ if __name__ == "__main__":
         DataChain.from_storage(source, type="image")
         .settings(parallel=-1)
         .filter(C("file.name").glob("*.jpg"))
-        .limit(10000)
+        .limit(5000)
         .map(
             image_description,
             params=["file"],

{datachain-0.2.15 → datachain-0.2.16}/examples/computer_vision/openimage-detect.py

@@ -48,7 +48,7 @@ def openimage_detect(args):
         yield fstream, bbox
 
 
-source = "gs://datachain-demo/openimages-v6-test-jsonpairs"
+source = "gs://datachain-demo/openimages-v6-test-jsonpairs/"
 
 (
     DataChain.from_storage(source)

{datachain-0.2.15 → datachain-0.2.16}/examples/get_started/json-csv-reader.py

@@ -36,7 +36,7 @@ def main():
     print("========================================================================")
     uri = "gs://datachain-demo/jsonl/object.jsonl"
     jsonl_ds = DataChain.from_json(uri, meta_type="jsonl", print_schema=True)
-    print(jsonl_ds.to_pandas())
+    jsonl_ds.show()
 
     print()
     print("========================================================================")
@@ -49,8 +49,7 @@ def main():
     json_pairs_ds = DataChain.from_json(
         uri, schema_from=schema_uri, jmespath="@", model_name="OpenImage"
     )
-    print(json_pairs_ds.to_pandas())
-    # print(list(json_pairs_ds.collect())[0])
+    json_pairs_ds.show()
 
     uri = "gs://datachain-demo/coco2017/annotations_captions/"
 
@@ -72,7 +71,7 @@ def main():
     static_json_ds = DataChain.from_json(
         uri, jmespath="licenses", spec=LicenseFeature, nrows=3
     )
-    print(static_json_ds.to_pandas())
+    static_json_ds.show()
 
     print()
     print("========================================================================")
@@ -88,16 +87,7 @@ def main():
     print("========================================================================")
     static_csv_ds = DataChain.from_csv(uri, output=ChatDialog, object_name="chat")
     static_csv_ds.print_schema()
-    print(static_csv_ds.to_pandas())
-
-    uri = "gs://datachain-demo/laion-aesthetics-csv"
-    print()
-    print("========================================================================")
-    print("dynamic CSV with header schema test parsing 3/3M objects")
-    print("========================================================================")
-    dynamic_csv_ds = DataChain.from_csv(uri, object_name="laion", nrows=3)
-    dynamic_csv_ds.print_schema()
-    print(dynamic_csv_ds.to_pandas())
+    static_csv_ds.show()
 
 
 if __name__ == "__main__":

{datachain-0.2.15 → datachain-0.2.16}/examples/get_started/torch-loader.py

@@ -64,7 +64,7 @@ if __name__ == "__main__":
     optimizer = optim.Adam(model.parameters(), lr=0.001)
 
     # Train the model
-    num_epochs = 10
+    num_epochs = 3
     for epoch in range(num_epochs):
         for i, data in enumerate(train_loader):
             inputs, labels = data

{datachain-0.2.15 → datachain-0.2.16}/examples/multimodal/wds.py

@@ -1,5 +1,3 @@
-import pandas as pd
-
 from datachain import C, DataChain
 from datachain.lib.webdataset import process_webdataset
 from datachain.lib.webdataset_laion import WDSLaion, process_laion_meta
@@ -9,25 +7,36 @@ wds = (
     .filter(C("file.name").glob("00000000.tar"))
     .settings(cache=True)
     .gen(laion=process_webdataset(spec=WDSLaion), params="file")
+    .save()  # materialize chain to avoid downloading data multiple times
+)
+
+meta_pq = (
+    DataChain.from_parquet("gs://datachain-demo/datacomp-small/metadata/0020f*.parquet")
+    .filter(
+        C("uid").in_(values[0] for values in wds.select("laion.json.uid").collect())
+    )
+    .map(stem=lambda file: file.get_file_stem(), params=["source.file"], output=str)
+    .save()
 )
 
 meta_emd = (
-    DataChain.from_storage("gs://datachain-demo/datacomp-small/metadata")
-    .filter(C("file.name").glob("0020f*.npz"))
+    DataChain.from_storage("gs://datachain-demo/datacomp-small/metadata/0020f*.npz")
     .gen(emd=process_laion_meta)
+    .filter(
+        C("emd.index").in_(
+            values[0] for values in meta_pq.select("source.index").collect()
+        )
+    )
     .map(stem=lambda file: file.get_file_stem(), params=["emd.file"], output=str)
 )
 
-meta_pq = DataChain.from_parquet(
-    "gs://datachain-demo/datacomp-small/metadata/0020f*.parquet"
-).map(stem=lambda file: file.get_file_stem(), params=["source.file"], output=str)
 
 meta = meta_emd.merge(
-    meta_pq, on=["stem", "emd.index"], right_on=["stem", "source.index"]
+    meta_pq,
+    on=["stem", "emd.index"],
+    right_on=["stem", "source.index"],
 )
 
 res = wds.merge(meta, on="laion.json.uid", right_on="uid")
 
-df = res.limit(10).to_pandas()
-with pd.option_context("display.max_columns", None):
-    print(df)
+res.show(3)

{datachain-0.2.15 → datachain-0.2.16}/examples/multimodal/wds_filtered.py

@@ -31,6 +31,7 @@ filtered = (
         / least(C("laion.json.original_width"), C("laion.json.original_height"))
         < 3.0
     )
+    .save()
 )
 filtered.show(3)
 

{datachain-0.2.15 → datachain-0.2.16}/src/datachain/catalog/catalog.py

@@ -1217,16 +1217,14 @@ class Catalog:
     def get_temp_table_names(self) -> list[str]:
         return self.warehouse.get_temp_table_names()
 
-    def cleanup_temp_tables(self, names: Iterable[str]) -> None:
+    def cleanup_tables(self, names: Iterable[str]) -> None:
         """
-        Drop tables created temporarily when processing datasets.
+        Drop tables passed.
 
-        This should be implemented even if temporary tables are used to
-        ensure that they are cleaned up as soon as they are no longer
-        needed. When running the same `DatasetQuery` multiple times we
-        may use the same temporary table names.
+        This should be implemented to ensure that the provided tables
+        are cleaned up as soon as they are no longer needed.
         """
-        self.warehouse.cleanup_temp_tables(names)
+        self.warehouse.cleanup_tables(names)
         self.id_generator.delete_uris(names)
 
     def create_dataset_from_sources(

{datachain-0.2.15 → datachain-0.2.16}/src/datachain/cli.py

@@ -910,7 +910,7 @@ def garbage_collect(catalog: "Catalog"):
         print("Nothing to clean up.")
     else:
         print(f"Garbage collecting {len(temp_tables)} tables.")
-        catalog.cleanup_temp_tables(temp_tables)
+        catalog.cleanup_tables(temp_tables)
 
 
 def completion(shell: str) -> str:

{datachain-0.2.15 → datachain-0.2.16}/src/datachain/data_storage/metastore.py

@@ -97,7 +97,7 @@ class AbstractMetastore(ABC, Serializable):
     def close(self) -> None:
         """Closes any active database or HTTP connections."""
 
-    def cleanup_temp_tables(self, temp_table_names: list[str]) -> None:
+    def cleanup_tables(self, temp_table_names: list[str]) -> None:
         """Cleanup temp tables."""
 
     def cleanup_for_tests(self) -> None:
@@ -457,7 +457,7 @@ class AbstractDBMetastore(AbstractMetastore):
         """Closes any active database connections."""
         self.db.close()
 
-    def cleanup_temp_tables(self, temp_table_names: list[str]) -> None:
+    def cleanup_tables(self, temp_table_names: list[str]) -> None:
         """Cleanup temp tables."""
         self.id_generator.delete_uris(temp_table_names)
 

{datachain-0.2.15 → datachain-0.2.16}/src/datachain/data_storage/warehouse.py

@@ -915,14 +915,12 @@ class AbstractWarehouse(ABC, Serializable):
             if self.is_temp_table_name(t)
         ]
 
-    def cleanup_temp_tables(self, names: Iterable[str]) -> None:
+    def cleanup_tables(self, names: Iterable[str]) -> None:
         """
-        Drop tables created temporarily when processing datasets.
+        Drop tables passed.
 
-        This should be implemented even if temporary tables are used to
-        ensure that they are cleaned up as soon as they are no longer
-        needed. When running the same `DatasetQuery` multiple times we
-        may use the same temporary table names.
+        This should be implemented to ensure that the provided tables
+        are cleaned up as soon as they are no longer needed.
         """
         for name in names:
             self.db.drop_table(Table(name, self.db.metadata), if_exists=True)

{datachain-0.2.15 → datachain-0.2.16}/src/datachain/lib/dc.py

@@ -193,8 +193,6 @@ class DataChain(DatasetQuery):
         ```
     """
 
-    max_row_count: Optional[int] = None
-
     DEFAULT_FILE_RECORD: ClassVar[dict] = {
         "source": "",
         "name": "",
@@ -1124,7 +1122,7 @@ class DataChain(DatasetQuery):
         def _func_fr() -> Iterator[tuple_type]:  # type: ignore[valid-type]
             yield from tuples
 
-        chain = DataChain.create_empty(DataChain.DEFAULT_FILE_RECORD, session=session)
+        chain = DataChain.from_records(DataChain.DEFAULT_FILE_RECORD, session=session)
         if object_name:
             output = {object_name: DataChain._dict_to_data_model(object_name, output)}  # type: ignore[arg-type]
         return chain.gen(_func_fr, output=output)
@@ -1441,13 +1439,14 @@ class DataChain(DatasetQuery):
         )
 
     @classmethod
-    def create_empty(
+    def from_records(
         cls,
         to_insert: Optional[Union[dict, list[dict]]],
         session: Optional[Session] = None,
     ) -> "DataChain":
-        """Create empty chain. Returns a chain. This method is used for programmatically
-        generating a chains in contrast of reading data from storages or other sources.
+        """Create a DataChain from the provided records. This method can be used for
+        programmatically generating a chain in contrast of reading data from storages
+        or other sources.
 
         Parameters:
             to_insert : records (or a single record) to insert. Each record is
@@ -1455,8 +1454,8 @@ class DataChain(DatasetQuery):
 
         Example:
             ```py
-            empty = DataChain.create_empty()
-            single_record = DataChain.create_empty(DataChain.DEFAULT_FILE_RECORD)
+            empty = DataChain.from_records()
+            single_record = DataChain.from_records(DataChain.DEFAULT_FILE_RECORD)
             ```
         """
         session = Session.get(session)
@@ -1602,18 +1601,7 @@ class DataChain(DatasetQuery):
     @detach
     def limit(self, n: int) -> "Self":
         """Return the first n rows of the chain."""
-        n = max(n, 0)
-
-        if self.max_row_count is None:
-            self.max_row_count = n
-            return super().limit(n)
-
-        limit = min(n, self.max_row_count)
-        if limit == self.max_row_count:
-            return self
-
-        self.max_row_count = limit
-        return super().limit(self.max_row_count)
+        return super().limit(n)
 
     @detach
     def offset(self, offset: int) -> "Self":

{datachain-0.2.15 → datachain-0.2.16}/src/datachain/lib/signal_schema.py

@@ -243,8 +243,11 @@ class SignalSchema:
         curr_type = None
         i = 0
         while curr_tree is not None and i < len(path):
-            if val := curr_tree.get(path[i], None):
+            if val := curr_tree.get(path[i]):
                 curr_type, curr_tree = val
+            elif i == 0 and len(path) > 1 and (val := curr_tree.get(".".join(path))):
+                curr_type, curr_tree = val
+                break
             else:
                 curr_type = None
             i += 1

{datachain-0.2.15 → datachain-0.2.16}/src/datachain/query/dataset.py

@@ -1201,10 +1201,10 @@ class DatasetQuery:
         # implementations, as errors may close or render unusable the existing
         # connections.
         metastore = self.catalog.metastore.clone(use_new_connection=True)
-        metastore.cleanup_temp_tables(self.temp_table_names)
+        metastore.cleanup_tables(self.temp_table_names)
         metastore.close()
         warehouse = self.catalog.warehouse.clone(use_new_connection=True)
-        warehouse.cleanup_temp_tables(self.temp_table_names)
+        warehouse.cleanup_tables(self.temp_table_names)
         warehouse.close()
         self.temp_table_names = []
 
@@ -1383,6 +1383,9 @@ class DatasetQuery:
     @detach
     def limit(self, n: int) -> "Self":
         query = self.clone(new_table=False)
+        for step in query.steps:
+            if isinstance(step, SQLLimit) and step.n < n:
+                return query
         query.steps.append(SQLLimit(n))
         return query
 

{datachain-0.2.15 → datachain-0.2.16/src/datachain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datachain
-Version: 0.2.15
+Version: 0.2.16
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License: Apache-2.0

{datachain-0.2.15 → datachain-0.2.16}/tests/func/test_catalog.py

@@ -1120,7 +1120,7 @@ def test_garbage_collect(cloud_test_catalog, from_cli, capsys):
         captured = capsys.readouterr()
         assert captured.out == "Garbage collecting 4 tables.\n"
     else:
-        catalog.cleanup_temp_tables(temp_tables)
+        catalog.cleanup_tables(temp_tables)
     assert catalog.get_temp_table_names() == []
 
 

{datachain-0.2.15 → datachain-0.2.16}/tests/unit/lib/test_datachain.py

@@ -95,7 +95,7 @@ def test_pandas_incorrect_column_names(catalog):
 
 
 def test_from_features_basic(catalog):
-    ds = DataChain.create_empty(DataChain.DEFAULT_FILE_RECORD)
+    ds = DataChain.from_records(DataChain.DEFAULT_FILE_RECORD)
     ds = ds.gen(lambda prm: [File(name="")] * 5, params="parent", output={"file": File})
 
     ds_name = "my_ds"
@@ -109,7 +109,7 @@ def test_from_features_basic(catalog):
 
 
 def test_from_features(catalog):
-    ds = DataChain.create_empty(DataChain.DEFAULT_FILE_RECORD)
+    ds = DataChain.from_records(DataChain.DEFAULT_FILE_RECORD)
     ds = ds.gen(
         lambda prm: list(zip([File(name="")] * len(features), features)),
         params="parent",
@@ -138,7 +138,7 @@ def test_datasets(catalog):
 
 
 def test_preserve_feature_schema(catalog):
-    ds = DataChain.create_empty(DataChain.DEFAULT_FILE_RECORD)
+    ds = DataChain.from_records(DataChain.DEFAULT_FILE_RECORD)
     ds = ds.gen(
         lambda prm: list(zip([File(name="")] * len(features), features, features)),
         params="parent",
@@ -612,6 +612,41 @@ def test_select_restore_from_saving(catalog):
     assert n == len(features_nested)
 
 
+def test_select_distinct(catalog):
+    class Embedding(BaseModel):
+        id: int
+        filename: str
+        values: list[float]
+
+    expected = [
+        [0.1, 0.3],
+        [0.1, 0.4],
+        [0.1, 0.5],
+        [0.1, 0.6],
+    ]
+
+    actual = (
+        DataChain.from_values(
+            embedding=[
+                Embedding(id=1, filename="a.jpg", values=expected[0]),
+                Embedding(id=2, filename="b.jpg", values=expected[2]),
+                Embedding(id=3, filename="c.jpg", values=expected[1]),
+                Embedding(id=4, filename="d.jpg", values=expected[1]),
+                Embedding(id=5, filename="e.jpg", values=expected[3]),
+            ],
+        )
+        .select("embedding.values", "embedding.filename")
+        .distinct("embedding.values")
+        .order_by("embedding.values")
+        .collect()
+    )
+
+    actual = [emb[0] for emb in actual]
+    assert len(actual) == 4
+    for i in [0, 1]:
+        assert np.allclose([emb[i] for emb in actual], [emp[i] for emp in expected])
+
+
 def test_from_dataset_name_version(catalog):
     name = "test-version"
     DataChain.from_values(
@@ -1193,3 +1228,33 @@ def test_custom_model_with_nested_lists():
             traces_double=[[{"x": 0.5, "y": 0.5}], [{"x": 0.5, "y": 0.5}]],
         )
     ]
+
+
+def test_min_limit():
+    dc = DataChain.from_values(a=[1, 2, 3, 4, 5])
+    assert dc.count() == 5
+    assert dc.limit(4).count() == 4
+    assert dc.count() == 5
+    assert dc.limit(1).count() == 1
+    assert dc.count() == 5
+    assert dc.limit(2).limit(3).count() == 2
+    assert dc.count() == 5
+    assert dc.limit(3).limit(2).count() == 2
+    assert dc.count() == 5
+
+
+def test_show_limit():
+    dc = DataChain.from_values(a=[1, 2, 3, 4, 5])
+    assert dc.count() == 5
+    assert dc.limit(4).count() == 4
+    dc.show(1)
+    assert dc.count() == 5
+    assert dc.limit(1).count() == 1
+    dc.show(1)
+    assert dc.count() == 5
+    assert dc.limit(2).limit(3).count() == 2
+    dc.show(1)
+    assert dc.count() == 5
+    assert dc.limit(3).limit(2).count() == 2
+    dc.show(1)
+    assert dc.count() == 5