datachain 0.29.0__tar.gz → 0.29.1__tar.gz

{datachain-0.29.0 → datachain-0.29.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.29.0
+Version: 0.29.1
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.29.0 → datachain-0.29.1}/src/datachain/query/utils.py

@@ -2,12 +2,12 @@ from typing import Optional, Union
 
 import sqlalchemy as sa
 
-ColT = Union[sa.Column, sa.ColumnElement, sa.TextClause]
+ColT = Union[sa.Column, sa.ColumnElement, sa.TextClause, sa.Label]
 
 
 def column_name(col: ColT) -> str:
     """Returns column name from column element."""
-    return col.name if isinstance(col, sa.Column) else str(col)
+    return col.name if isinstance(col, (sa.Column, sa.Label)) else str(col)
 
 
 def get_query_column(query: sa.Select, name: str) -> Optional[ColT]:

{datachain-0.29.0 → datachain-0.29.1}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.29.0
+Version: 0.29.1
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

datachain-0.29.1/tests/func/test_warehouse.py

@@ -0,0 +1,35 @@
+from types import GeneratorType
+
+import sqlalchemy as sa
+
+import datachain as dc
+
+
+def test_dataset_stats_no_table(cloud_test_catalog, dogs_dataset):
+    catalog = cloud_test_catalog.catalog
+    catalog.warehouse.drop_dataset_rows_table(dogs_dataset, version="1.0.0")
+    num_objects, size = catalog.warehouse.dataset_stats(dogs_dataset, version="1.0.0")
+    assert num_objects is None
+    assert size is None
+
+
+def test_dataset_select_paginated_dataset_larger_than_batch_size(test_session):
+    catalog = test_session.catalog
+    warehouse = catalog.warehouse
+    chain = dc.read_values(value=list(range(10_000)), session=test_session).save(
+        "large"
+    )
+    table = warehouse.get_table(
+        warehouse.dataset_table_name(chain.dataset, chain.dataset.latest_version)
+    )
+    db_values = chain.to_values("value")
+
+    rows = warehouse.dataset_select_paginated(
+        sa.select(table.c.value).order_by(table.c.value), page_size=1000
+    )
+    assert isinstance(rows, GeneratorType)
+
+    rows = list(rows)
+    assert len(rows) == 10_000
+    (values,) = zip(*rows)
+    assert set(values) == set(db_values)

{datachain-0.29.0 → datachain-0.29.1}/tests/unit/test_utils.py

@@ -4,6 +4,7 @@ import pytest
 
 from datachain.utils import (
     batched,
+    batched_it,
     datachain_paths_join,
     determine_processes,
     determine_workers,
@@ -272,3 +273,27 @@ def test_batched_row_limit():
     assert batches[1] == (4, 5, 6, 7)
     assert batches[2] == (8, 9, 10, 11)
     assert batches[3] == (12, 13, 14)
+
+
+@pytest.mark.parametrize(
+    "num_rows,batch_size",
+    (
+        (300_000, 500),
+        (100_000, 500),
+        (100_000, 100_000),
+        (1, 1),
+    ),
+)
+def test_batched_it(num_rows, batch_size):
+    rows = iter(list(range(num_rows)))
+    batches = batched_it(iter(rows), batch_size)
+    num_batches = 0
+    uniq_data = set()
+    for batch in batches:
+        num_batches += 1
+        batch_l = list(batch)
+        assert len(batch_l) == batch_size
+        uniq_data.update(batch_l)
+
+    assert num_batches == num_rows / batch_size
+    assert len(uniq_data) == num_rows

datachain-0.29.0/tests/func/test_warehouse.py

@@ -1,6 +0,0 @@
-def test_dataset_stats_no_table(cloud_test_catalog, dogs_dataset):
-    catalog = cloud_test_catalog.catalog
-    catalog.warehouse.drop_dataset_rows_table(dogs_dataset, version="1.0.0")
-    num_objects, size = catalog.warehouse.dataset_stats(dogs_dataset, version="1.0.0")
-    assert num_objects is None
-    assert size is None