datachain 0.18.5__tar.gz → 0.18.7__tar.gz

{datachain-0.18.5/src/datachain.egg-info → datachain-0.18.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.18.5
+Version: 0.18.7
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.18.5 → datachain-0.18.7}/docs/references/remotes.md

@@ -86,7 +86,7 @@ DataChain uses [s3fs](https://s3fs.readthedocs.io/en/latest/) to interact with A
     if no caching is desired. See fsspec's documentation for other available
     `cache_type` values. Default cache_type is `"readahead"`.
 
-- `version_aware`: `bool` (default: `False`)
+- `version_aware`: `bool` (default: `True`)
 
     Whether to support bucket versioning. If enable this will require the
     user to have the necessary IAM permissions for dealing with versioned

{datachain-0.18.5 → datachain-0.18.7}/src/datachain/client/azure.py

@@ -15,7 +15,7 @@ class AzureClient(Client):
     protocol = "az"
 
     def info_to_file(self, v: dict[str, Any], path: str) -> File:
-        version_id = v.get("version_id")
+        version_id = v.get("version_id") if self._is_version_aware() else None
         return File(
             source=self.uri,
             path=path,

{datachain-0.18.5 → datachain-0.18.7}/src/datachain/client/fsspec.py

@@ -208,7 +208,7 @@ class Client(ABC):
 
     async def get_current_etag(self, file: "File") -> str:
         kwargs = {}
-        if getattr(self.fs, "version_aware", False):
+        if self._is_version_aware():
             kwargs["version_id"] = file.version
         info = await self.fs._info(
             self.get_full_path(file.path, file.version), **kwargs
@@ -326,8 +326,11 @@ class Client(ABC):
         """
         return not (key.startswith("/") or key.endswith("/") or "//" in key)
 
+    def _is_version_aware(self) -> bool:
+        return getattr(self.fs, "version_aware", False)
+
     async def ls_dir(self, path):
-        if getattr(self.fs, "version_aware", False):
+        if self._is_version_aware():
             kwargs = {"versions": True}
         return await self.fs._ls(path, detail=True, **kwargs)
 

{datachain-0.18.5 → datachain-0.18.7}/src/datachain/client/gcs.py

@@ -115,7 +115,7 @@ class GCSClient(Client):
                     maxResults=page_size,
                     pageToken=next_page_token,
                     json_out=True,
-                    versions="true",
+                    versions="true" if self._is_version_aware() else "false",
                 )
                 assert page["kind"] == "storage#objects"
                 await page_queue.put(page.get("items", []))
@@ -134,7 +134,7 @@ class GCSClient(Client):
             source=self.uri,
             path=path,
             etag=v.get("etag", ""),
-            version=v.get("generation", ""),
+            version=v.get("generation", "") if self._is_version_aware() else "",
             is_latest=not v.get("timeDeleted"),
             last_modified=self.parse_timestamp(v["updated"]),
             size=v.get("size", ""),

{datachain-0.18.5 → datachain-0.18.7}/src/datachain/client/s3.py

@@ -101,7 +101,7 @@ class ClientS3(Client):
             prefix = start_prefix
             if prefix:
                 prefix = prefix.lstrip(DELIMITER) + DELIMITER
-            versions = True
+            versions = self._is_version_aware()
             fs = self.fs
             await fs.set_session()
             s3 = await fs.get_s3(self.name)
@@ -139,7 +139,9 @@ class ClientS3(Client):
             source=self.uri,
             path=v["Key"],
             etag=v.get("ETag", "").strip('"'),
-            version=ClientS3.clean_s3_version(v.get("VersionId", "")),
+            version=(
+                ClientS3.clean_s3_version(v.get("VersionId", "")) if versions else ""
+            ),
             is_latest=v.get("IsLatest", True),
             last_modified=v.get("LastModified", ""),
             size=v["Size"],
@@ -193,7 +195,11 @@ class ClientS3(Client):
             source=self.uri,
             path=path,
             size=v["size"],
-            version=ClientS3.clean_s3_version(v.get("VersionId", "")),
+            version=(
+                ClientS3.clean_s3_version(v.get("VersionId", ""))
+                if self._is_version_aware()
+                else ""
+            ),
             etag=v.get("ETag", "").strip('"'),
             is_latest=v.get("IsLatest", True),
             last_modified=v.get("LastModified", ""),

{datachain-0.18.5 → datachain-0.18.7}/src/datachain/query/dispatch.py

@@ -264,9 +264,10 @@ class UDFDispatcher:
             # Will be set to True when the input is exhausted
             input_finished = False
 
-            if not self.is_batching:
-                batch_size = self.input_batch_size(n_workers)
-                input_rows = batched(flatten(input_rows), batch_size)
+            input_rows = batched(
+                input_rows if self.is_batching else flatten(input_rows),
+                self.input_batch_size(n_workers),
+            )
 
             # Stop all workers after the input rows have finished processing
             input_data = chain(input_rows, [STOP_SIGNAL] * n_workers)

{datachain-0.18.5 → datachain-0.18.7/src/datachain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.18.5
+Version: 0.18.7
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.18.5 → datachain-0.18.7}/tests/func/test_datachain.py

@@ -8,7 +8,7 @@ import re
 import uuid
 from collections.abc import Iterator
 from datetime import datetime, timedelta, timezone
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from unittest.mock import patch
 
 import numpy as np
@@ -2235,3 +2235,44 @@ def test_datachain_functional_after_exceptions(test_session):
     for _ in range(4):
         with pytest.raises(Exception, match="Test Error!"):
             chain.map(res=func).exec()
+
+
+@pytest.mark.parametrize("parallel", [1, 2])
+def test_agg(catalog_tmpfile, parallel):
+    from datachain import func
+
+    session = catalog_tmpfile.session
+
+    def process(files: list[str]) -> Iterator[tuple[str, int]]:
+        yield str(PurePosixPath(files[0]).parent), len(files)
+
+    ds = (
+        dc.read_values(
+            filename=(
+                "cats/cat1",
+                "cats/cat2",
+                "dogs/dog1",
+                "dogs/dog2",
+                "dogs/dog3",
+                "dogs/others/dog4",
+            ),
+            session=session,
+        )
+        .settings(parallel=parallel)
+        .agg(
+            process,
+            params=["filename"],
+            output={"parent": str, "count": int},
+            partition_by=func.path.parent("filename"),
+        )
+        .save("my-ds")
+    )
+
+    assert sorted_dicts(ds.to_records(), "parent") == sorted_dicts(
+        [
+            {"parent": "cats", "count": 2},
+            {"parent": "dogs", "count": 3},
+            {"parent": "dogs/others", "count": 1},
+        ],
+        "parent",
+    )

{datachain-0.18.5 → datachain-0.18.7}/tests/func/test_listing.py

@@ -25,6 +25,12 @@ def test_listing_generator(cloud_test_catalog, cloud_type):
         assert cat_file.source == ctc.src_uri
         assert cat_file.path == cat_entry.path
         assert cat_file.size == cat_entry.size
+        assert cat_file.etag is not None
+        # If version_aware is not passed it is enforced to be True internally
+        if catalog.client_config.get("version_aware", True):
+            assert cat_file.version is not None
+        else:
+            assert cat_file.version == ""
         assert cat_file.is_latest == cat_entry.is_latest
         assert cat_file.location is None
 

{datachain-0.18.5 → datachain-0.18.7}/tests/unit/lib/test_datachain.py

@@ -681,7 +681,7 @@ def test_agg_two_params(test_session):
     assert list(ds.order_by("x.cnt").collect("x.cnt")) == [7, 20]
 
 
-def test_agg_simple_iiterator(test_session):
+def test_agg_simple_iterator(test_session):
     def func(key, val) -> Iterator[tuple[File, str]]:
         for i in range(val):
             yield File(path=""), f"{key}_{i}"