datachain 0.18.5__py3-none-any.whl → 0.18.7__py3-none-any.whl

datachain/client/azure.py

@@ -15,7 +15,7 @@ class AzureClient(Client):
     protocol = "az"
 
     def info_to_file(self, v: dict[str, Any], path: str) -> File:
-        version_id = v.get("version_id")
+        version_id = v.get("version_id") if self._is_version_aware() else None
         return File(
             source=self.uri,
             path=path,

datachain/client/fsspec.py

@@ -208,7 +208,7 @@ class Client(ABC):
 
     async def get_current_etag(self, file: "File") -> str:
         kwargs = {}
-        if getattr(self.fs, "version_aware", False):
+        if self._is_version_aware():
             kwargs["version_id"] = file.version
         info = await self.fs._info(
             self.get_full_path(file.path, file.version), **kwargs
@@ -326,8 +326,11 @@ class Client(ABC):
         """
         return not (key.startswith("/") or key.endswith("/") or "//" in key)
 
+    def _is_version_aware(self) -> bool:
+        return getattr(self.fs, "version_aware", False)
+
     async def ls_dir(self, path):
-        if getattr(self.fs, "version_aware", False):
+        if self._is_version_aware():
             kwargs = {"versions": True}
         return await self.fs._ls(path, detail=True, **kwargs)
 

datachain/client/gcs.py

@@ -115,7 +115,7 @@ class GCSClient(Client):
                     maxResults=page_size,
                     pageToken=next_page_token,
                     json_out=True,
-                    versions="true",
+                    versions="true" if self._is_version_aware() else "false",
                 )
                 assert page["kind"] == "storage#objects"
                 await page_queue.put(page.get("items", []))
@@ -134,7 +134,7 @@ class GCSClient(Client):
             source=self.uri,
             path=path,
             etag=v.get("etag", ""),
-            version=v.get("generation", ""),
+            version=v.get("generation", "") if self._is_version_aware() else "",
             is_latest=not v.get("timeDeleted"),
             last_modified=self.parse_timestamp(v["updated"]),
             size=v.get("size", ""),

datachain/client/s3.py

@@ -101,7 +101,7 @@ class ClientS3(Client):
             prefix = start_prefix
             if prefix:
                 prefix = prefix.lstrip(DELIMITER) + DELIMITER
-            versions = True
+            versions = self._is_version_aware()
             fs = self.fs
             await fs.set_session()
             s3 = await fs.get_s3(self.name)
@@ -139,7 +139,9 @@ class ClientS3(Client):
             source=self.uri,
             path=v["Key"],
             etag=v.get("ETag", "").strip('"'),
-            version=ClientS3.clean_s3_version(v.get("VersionId", "")),
+            version=(
+                ClientS3.clean_s3_version(v.get("VersionId", "")) if versions else ""
+            ),
             is_latest=v.get("IsLatest", True),
             last_modified=v.get("LastModified", ""),
             size=v["Size"],
@@ -193,7 +195,11 @@ class ClientS3(Client):
             source=self.uri,
             path=path,
             size=v["size"],
-            version=ClientS3.clean_s3_version(v.get("VersionId", "")),
+            version=(
+                ClientS3.clean_s3_version(v.get("VersionId", ""))
+                if self._is_version_aware()
+                else ""
+            ),
             etag=v.get("ETag", "").strip('"'),
             is_latest=v.get("IsLatest", True),
             last_modified=v.get("LastModified", ""),

datachain/query/dispatch.py

@@ -264,9 +264,10 @@ class UDFDispatcher:
             # Will be set to True when the input is exhausted
             input_finished = False
 
-            if not self.is_batching:
-                batch_size = self.input_batch_size(n_workers)
-                input_rows = batched(flatten(input_rows), batch_size)
+            input_rows = batched(
+                input_rows if self.is_batching else flatten(input_rows),
+                self.input_batch_size(n_workers),
+            )
 
             # Stop all workers after the input rows have finished processing
             input_data = chain(input_rows, [STOP_SIGNAL] * n_workers)

{datachain-0.18.5.dist-info → datachain-0.18.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.18.5
+Version: 0.18.7
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.18.5.dist-info → datachain-0.18.7.dist-info}/RECORD

@@ -37,13 +37,13 @@ datachain/cli/parser/job.py,sha256=acdVYuTsqluRDI_FYhZ1ohjQcVtBj-taUm8y9tGb0_0,4
 datachain/cli/parser/studio.py,sha256=Y-1OlQGecLVi9QofvWUfSlPd2ISyaESf7QFGZqGsrdw,3609
 datachain/cli/parser/utils.py,sha256=rETdD-9Hq9A4OolgfT7jQw4aoawtbfmkdtH6E7nkhpI,2888
 datachain/client/__init__.py,sha256=1kDpCPoibMXi1gExR4lTLc5pi-k6M5TANiwtXkPoLhU,49
-datachain/client/azure.py,sha256=dM9rGHHmi40iT3FOcLP5iNo_1nS2exjmJH9U4PElW5A,3232
+datachain/client/azure.py,sha256=7yyAgANHfu9Kfh187MKNTT1guvu9Q-WYsi4vYoY3aew,3270
 datachain/client/fileslice.py,sha256=bT7TYco1Qe3bqoc8aUkUZcPdPofJDHlryL5BsTn9xsY,3021
-datachain/client/fsspec.py,sha256=UJ7PDq1F11gf7OMjfXYqzrS1GHL3FZctOwXI0S_LU74,13852
-datachain/client/gcs.py,sha256=ckdP7utmBfSDkj0GOhV_34itw5pN3DOD74hrWNKIcQg,5127
+datachain/client/fsspec.py,sha256=c8oRBUMo31k8bMB_mIA60PDfna4nYTdslzHqmqL2Uvg,13918
+datachain/client/gcs.py,sha256=8hcFhEHp8qGRsJoyfCoawfuwb1Et-MSkyQoM9AnNuXI,5204
 datachain/client/hf.py,sha256=posnI5WOKOMG1yY_ZiV9Orcd24QsUPKZlOXgJVLxxrM,1558
 datachain/client/local.py,sha256=cGoCYflribzexiOe-Y1qbaE2fJRh-_EgQrfCSa0yK_E,4568
-datachain/client/s3.py,sha256=bhlQGOVi0smKmLkkyO-Y1YR95u-aPim4bdkEj4wyhzU,7343
+datachain/client/s3.py,sha256=6DNVGLg-woPS1DVlYVX2rIlunNblsuxyOnI1rSzhW3k,7515
 datachain/data_storage/__init__.py,sha256=9Wit-oe5P46V7CJQTD0BJ5MhOa2Y9h3ddJ4VWTe-Lec,273
 datachain/data_storage/db_engine.py,sha256=n8ojCbvVMPY2e3SG8fUaaD0b9GkVfpl_Naa_6EiHfWg,3788
 datachain/data_storage/job.py,sha256=9r0OGwh22bHNIvLHqg8_-eJSP1YYB-BN5HOla5TdCxw,402
@@ -122,7 +122,7 @@ datachain/model/ultralytics/segment.py,sha256=63bDCj43E6iZ0hFI5J6uQfksdCmjEp6sEm
 datachain/query/__init__.py,sha256=7DhEIjAA8uZJfejruAVMZVcGFmvUpffuZJwgRqNwe-c,263
 datachain/query/batch.py,sha256=-goxLpE0EUvaDHu66rstj53UnfHpYfBUGux8GSpJ93k,4306
 datachain/query/dataset.py,sha256=3c3MAiIl7ZnCii_0dZA-Om73ornNMSKkna32JX3H05E,60587
-datachain/query/dispatch.py,sha256=15M3zlTUFKM6D2ijITX4o5QxCkRe2klkODsIDi3aQOg,15544
+datachain/query/dispatch.py,sha256=A0nPxn6mEN5d9dDo6S8m16Ji_9IvJLXrgF2kqXdi4fs,15546
 datachain/query/metrics.py,sha256=DOK5HdNVaRugYPjl8qnBONvTkwjMloLqAr7Mi3TjCO0,858
 datachain/query/params.py,sha256=O_j89mjYRLOwWNhYZl-z7mi-rkdP7WyFmaDufsdTryE,863
 datachain/query/queue.py,sha256=v0UeK4ilmdiRoJ5OdjB5qpnHTYDxRP4vhVp5Iw_toaI,3512
@@ -153,9 +153,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=ktGWzY4kyzjWyR86dhvzw-Zhl0lVk_LOX3NciTac6qo,2914
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.18.5.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.18.5.dist-info/METADATA,sha256=nRjXtvxIMAGDnI5sLMewoVcqsRbn9jGWCZnmu2wzk1I,11319
-datachain-0.18.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-datachain-0.18.5.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.18.5.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.18.5.dist-info/RECORD,,
+datachain-0.18.7.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.18.7.dist-info/METADATA,sha256=OXGuP0EbV6ZC57NPhtyse2-6OP2pDKbhJkmcDfHp1mU,11319
+datachain-0.18.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datachain-0.18.7.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.18.7.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.18.7.dist-info/RECORD,,