datachain 0.16.4__py3-none-any.whl → 0.16.5__py3-none-any.whl

datachain/query/dispatch.py

@@ -3,7 +3,6 @@ from collections.abc import Iterable, Sequence
 from itertools import chain
 from multiprocessing import cpu_count
 from sys import stdin
-from threading import Timer
 from typing import TYPE_CHECKING, Literal, Optional
 
 import multiprocess
@@ -15,7 +14,6 @@ from datachain.catalog import Catalog
 from datachain.catalog.catalog import clone_catalog_with_cache
 from datachain.catalog.loader import DISTRIBUTED_IMPORT_PATH, get_udf_distributor_class
 from datachain.lib.udf import _get_cache
-from datachain.query.batch import RowsOutput, RowsOutputBatch
 from datachain.query.dataset import (
     get_download_callback,
     get_generated_callback,
@@ -32,6 +30,7 @@ if TYPE_CHECKING:
 
     from datachain.data_storage import AbstractMetastore, AbstractWarehouse
     from datachain.lib.udf import UDFAdapter
+    from datachain.query.batch import RowsOutput
 
 DEFAULT_BATCH_SIZE = 10000
 STOP_SIGNAL = "STOP"
@@ -50,34 +49,30 @@ def get_n_workers_from_arg(n_workers: Optional[int] = None) -> int:
 
 
 def udf_entrypoint() -> int:
+    """Parallel processing (faster for more CPU-heavy UDFs)."""
     # Load UDF info from stdin
     udf_info: UdfInfo = load(stdin.buffer)
 
-    # Parallel processing (faster for more CPU-heavy UDFs)
-    dispatch = UDFDispatcher(udf_info)
-
     query = udf_info["query"]
-    rows_total = udf_info["rows_total"]
     batching = udf_info["batching"]
     is_generator = udf_info["is_generator"]
-    n_workers = udf_info["processes"]
-    if n_workers is True:
-        n_workers = None  # Use default number of CPUs (cores)
+
+    download_cb = get_download_callback()
+    processed_cb = get_processed_callback()
+    generated_cb = get_generated_callback(is_generator)
 
     wh_cls, wh_args, wh_kwargs = udf_info["warehouse_clone_params"]
     warehouse: AbstractWarehouse = wh_cls(*wh_args, **wh_kwargs)
 
+    id_col = get_query_id_column(query)
+
     with contextlib.closing(
-        batching(warehouse.dataset_select_paginated, query, ids_only=True)
+        batching(warehouse.dataset_select_paginated, query, id_col=id_col)
     ) as udf_inputs:
-        download_cb = get_download_callback()
-        processed_cb = get_processed_callback()
-        generated_cb = get_generated_callback(is_generator)
         try:
-            dispatch.run_udf_parallel(
+            UDFDispatcher(udf_info).run_udf(
                 udf_inputs,
-                rows_total=rows_total,
-                n_workers=n_workers,
+                ids_only=id_col is not None,
                 download_cb=download_cb,
                 processed_cb=processed_cb,
                 generated_cb=generated_cb,
@@ -90,17 +85,18 @@ def udf_entrypoint() -> int:
     return 0
 
 
-def udf_worker_entrypoint() -> int:
+def udf_worker_entrypoint(fd: Optional[int] = None) -> int:
     if not (udf_distributor_class := get_udf_distributor_class()):
         raise RuntimeError(
             f"{DISTRIBUTED_IMPORT_PATH} import path is required "
             "for distributed UDF processing."
         )
-    return udf_distributor_class.run_worker()
+
+    return udf_distributor_class.run_udf(fd)
 
 
 class UDFDispatcher:
-    catalog: Optional[Catalog] = None
+    _catalog: Optional[Catalog] = None
     task_queue: Optional[multiprocess.Queue] = None
     done_queue: Optional[multiprocess.Queue] = None
 
@@ -115,77 +111,147 @@ class UDFDispatcher:
         self.cache = udf_info["cache"]
         self.is_generator = udf_info["is_generator"]
         self.is_batching = udf_info["batching"].is_batching
+        self.processes = udf_info["processes"]
+        self.rows_total = udf_info["rows_total"]
         self.buffer_size = buffer_size
-        self.catalog = None
         self.task_queue = None
         self.done_queue = None
         self.ctx = get_context("spawn")
 
-    def _create_worker(self) -> "UDFWorker":
-        if not self.catalog:
+    @property
+    def catalog(self) -> "Catalog":
+        if not self._catalog:
             ms_cls, ms_args, ms_kwargs = self.metastore_clone_params
             metastore: AbstractMetastore = ms_cls(*ms_args, **ms_kwargs)
             ws_cls, ws_args, ws_kwargs = self.warehouse_clone_params
             warehouse: AbstractWarehouse = ws_cls(*ws_args, **ws_kwargs)
-            self.catalog = Catalog(metastore, warehouse, **self.catalog_init_params)
-        self.udf = loads(self.udf_data)
+            self._catalog = Catalog(metastore, warehouse, **self.catalog_init_params)
+        return self._catalog
+
+    def _create_worker(self) -> "UDFWorker":
+        udf: UDFAdapter = loads(self.udf_data)
         return UDFWorker(
             self.catalog,
-            self.udf,
+            udf,
             self.task_queue,
             self.done_queue,
             self.query,
             self.table,
-            self.is_batching,
             self.cache,
+            self.is_batching,
             self.udf_fields,
         )
 
-    def _run_worker(self) -> None:
+    def _run_worker(self, ids_only: bool) -> None:
         try:
             worker = self._create_worker()
-            worker.run()
+            worker.run(ids_only)
         except (Exception, KeyboardInterrupt) as e:
             if self.done_queue:
                 put_into_queue(
                     self.done_queue,
                     {"status": FAILED_STATUS, "exception": e},
                 )
+            if isinstance(e, KeyboardInterrupt):
+                return
             raise
 
-    @staticmethod
-    def send_stop_signal_to_workers(task_queue, n_workers: Optional[int] = None):
+    def run_udf(
+        self,
+        input_rows: Iterable["RowsOutput"],
+        ids_only: bool,
+        download_cb: Callback = DEFAULT_CALLBACK,
+        processed_cb: Callback = DEFAULT_CALLBACK,
+        generated_cb: Callback = DEFAULT_CALLBACK,
+    ) -> None:
+        n_workers = self.processes
+        if n_workers is True:
+            n_workers = None  # Use default number of CPUs (cores)
+        elif not n_workers or n_workers < 1:
+            n_workers = 1  # Single-threaded (on this worker)
         n_workers = get_n_workers_from_arg(n_workers)
-        for _ in range(n_workers):
-            put_into_queue(task_queue, STOP_SIGNAL)
 
-    def run_udf_parallel(  # noqa: C901, PLR0912
+        if n_workers == 1:
+            # no need to spawn worker processes if we are running in a single process
+            self.run_udf_single(
+                input_rows, ids_only, download_cb, processed_cb, generated_cb
+            )
+        else:
+            if self.buffer_size < n_workers:
+                raise RuntimeError(
+                    "Parallel run error: buffer size is smaller than "
+                    f"number of workers: {self.buffer_size} < {n_workers}"
+                )
+
+            self.run_udf_parallel(
+                n_workers, input_rows, ids_only, download_cb, processed_cb, generated_cb
+            )
+
+    def run_udf_single(
         self,
-        input_rows: Iterable[RowsOutput],
-        rows_total: int,
-        n_workers: Optional[int] = None,
+        input_rows: Iterable["RowsOutput"],
+        ids_only: bool,
         download_cb: Callback = DEFAULT_CALLBACK,
         processed_cb: Callback = DEFAULT_CALLBACK,
         generated_cb: Callback = DEFAULT_CALLBACK,
     ) -> None:
-        n_workers = get_n_workers_from_arg(n_workers)
+        udf: UDFAdapter = loads(self.udf_data)
+
+        if ids_only and not self.is_batching:
+            input_rows = flatten(input_rows)
+
+        def get_inputs() -> Iterable["RowsOutput"]:
+            warehouse = self.catalog.warehouse.clone()
+            if ids_only:
+                yield from warehouse.dataset_rows_select_from_ids(
+                    self.query, input_rows, self.is_batching
+                )
+            else:
+                yield from input_rows
+
+        prefetch = udf.prefetch
+        with _get_cache(self.catalog.cache, prefetch, use_cache=self.cache) as _cache:
+            udf_results = udf.run(
+                self.udf_fields,
+                get_inputs(),
+                self.catalog,
+                self.cache,
+                download_cb=download_cb,
+                processed_cb=processed_cb,
+            )
+            with safe_closing(udf_results):
+                process_udf_outputs(
+                    self.catalog.warehouse.clone(),
+                    self.table,
+                    udf_results,
+                    udf,
+                    cb=generated_cb,
+                )
 
-        input_batch_size = rows_total // n_workers
+    def input_batch_size(self, n_workers: int) -> int:
+        input_batch_size = self.rows_total // n_workers
         if input_batch_size == 0:
             input_batch_size = 1
         elif input_batch_size > DEFAULT_BATCH_SIZE:
             input_batch_size = DEFAULT_BATCH_SIZE
+        return input_batch_size
 
-        if self.buffer_size < n_workers:
-            raise RuntimeError(
-                "Parallel run error: buffer size is smaller than "
-                f"number of workers: {self.buffer_size} < {n_workers}"
-            )
-
+    def run_udf_parallel(  # noqa: C901, PLR0912
+        self,
+        n_workers: int,
+        input_rows: Iterable["RowsOutput"],
+        ids_only: bool,
+        download_cb: Callback = DEFAULT_CALLBACK,
+        processed_cb: Callback = DEFAULT_CALLBACK,
+        generated_cb: Callback = DEFAULT_CALLBACK,
+    ) -> None:
         self.task_queue = self.ctx.Queue()
         self.done_queue = self.ctx.Queue()
+
         pool = [
-            self.ctx.Process(name=f"Worker-UDF-{i}", target=self._run_worker)
+            self.ctx.Process(
+                name=f"Worker-UDF-{i}", target=self._run_worker, args=[ids_only]
+            )
             for i in range(n_workers)
         ]
         for p in pool:
@@ -198,7 +264,8 @@ class UDFDispatcher:
             input_finished = False
 
             if not self.is_batching:
-                input_rows = batched(flatten(input_rows), input_batch_size)
+                batch_size = self.input_batch_size(n_workers)
+                input_rows = batched(flatten(input_rows), batch_size)
 
             # Stop all workers after the input rows have finished processing
             input_data = chain(input_rows, [STOP_SIGNAL] * n_workers)
@@ -213,10 +280,15 @@ class UDFDispatcher:
 
             # Process all tasks
             while n_workers > 0:
-                result = get_from_queue(self.done_queue)
+                try:
+                    result = get_from_queue(self.done_queue)
+                except KeyboardInterrupt:
+                    break
 
+                if bytes_downloaded := result.get("bytes_downloaded"):
+                    download_cb.relative_update(bytes_downloaded)
                 if downloaded := result.get("downloaded"):
-                    download_cb.relative_update(downloaded)
+                    download_cb.increment_file_count(downloaded)
                 if processed := result.get("processed"):
                     processed_cb.relative_update(processed)
                 if generated := result.get("generated"):
@@ -246,13 +318,12 @@ class UDFDispatcher:
                 # Stop all workers if there is an unexpected exception
                 for _ in pool:
                     put_into_queue(self.task_queue, STOP_SIGNAL)
-                self.task_queue.close()
 
                 # This allows workers (and this process) to exit without
                 # consuming any remaining data in the queues.
                 # (If they exit due to an exception.)
-                self.task_queue.cancel_join_thread()
-                self.done_queue.cancel_join_thread()
+                self.task_queue.close()
+                self.task_queue.join_thread()
 
                 # Flush all items from the done queue.
                 # This is needed if any workers are still running.
@@ -262,6 +333,9 @@ class UDFDispatcher:
                     if status != OK_STATUS:
                         n_workers -= 1
 
+                self.done_queue.close()
+                self.done_queue.join_thread()
+
             # Wait for workers to stop
             for p in pool:
                 p.join()
@@ -273,8 +347,7 @@ class DownloadCallback(Callback):
         super().__init__()
 
     def relative_update(self, inc: int = 1) -> None:
-        # This callback is used to notify the size of the downloaded files
-        pass
+        put_into_queue(self.queue, {"status": NOTIFY_STATUS, "bytes_downloaded": inc})
 
     def increment_file_count(self, inc: int = 1) -> None:
         put_into_queue(self.queue, {"status": NOTIFY_STATUS, "downloaded": inc})
@@ -303,8 +376,8 @@ class UDFWorker:
         done_queue: "multiprocess.Queue",
         query: "Select",
         table: "Table",
-        is_batching: bool,
         cache: bool,
+        is_batching: bool,
         udf_fields: Sequence[str],
     ) -> None:
         self.catalog = catalog
@@ -313,21 +386,21 @@ class UDFWorker:
         self.done_queue = done_queue
         self.query = query
         self.table = table
-        self.is_batching = is_batching
         self.cache = cache
+        self.is_batching = is_batching
         self.udf_fields = udf_fields
 
         self.download_cb = DownloadCallback(self.done_queue)
         self.processed_cb = ProcessedCallback("processed", self.done_queue)
         self.generated_cb = ProcessedCallback("generated", self.done_queue)
 
-    def run(self) -> None:
+    def run(self, ids_only: bool) -> None:
         prefetch = self.udf.prefetch
         with _get_cache(self.catalog.cache, prefetch, use_cache=self.cache) as _cache:
             catalog = clone_catalog_with_cache(self.catalog, _cache)
             udf_results = self.udf.run(
                 self.udf_fields,
-                self.get_inputs(),
+                self.get_inputs(ids_only),
                 catalog,
                 self.cache,
                 download_cb=self.download_cb,
@@ -348,23 +421,12 @@ class UDFWorker:
             put_into_queue(self.done_queue, {"status": OK_STATUS})
             yield row
 
-    def get_inputs(self):
+    def get_inputs(self, ids_only: bool) -> Iterable["RowsOutput"]:
         warehouse = self.catalog.warehouse.clone()
-        col_id = get_query_id_column(self.query)
-
-        if self.is_batching:
-            while (batch := get_from_queue(self.task_queue)) != STOP_SIGNAL:
-                ids = [row[0] for row in batch.rows]
-                rows = warehouse.dataset_rows_select(self.query.where(col_id.in_(ids)))
-                yield RowsOutputBatch(list(rows))
-        else:
-            while (batch := get_from_queue(self.task_queue)) != STOP_SIGNAL:
-                yield from warehouse.dataset_rows_select(
-                    self.query.where(col_id.in_(batch))
+        while (batch := get_from_queue(self.task_queue)) != STOP_SIGNAL:
+            if ids_only:
+                yield from warehouse.dataset_rows_select_from_ids(
+                    self.query, batch, self.is_batching
                 )
-
-
-class RepeatTimer(Timer):
-    def run(self):
-        while not self.finished.wait(self.interval):
-            self.function(*self.args, **self.kwargs)
+            else:
+                yield from batch

datachain/query/metrics.py

@@ -15,11 +15,10 @@ def set(key: str, value: Union[str, int, float, bool, None]) -> None:  # noqa: P
     metrics[key] = value
 
     if job_id := os.getenv("DATACHAIN_JOB_ID"):
-        from datachain.data_storage.job import JobStatus
         from datachain.query.session import Session
 
         metastore = Session.get().catalog.metastore
-        metastore.set_job_status(job_id, JobStatus.RUNNING, metrics=metrics)
+        metastore.update_job(job_id, metrics=metrics)
 
 
 def get(key: str) -> Optional[Union[str, int, float, bool]]:

datachain/query/queue.py

@@ -7,7 +7,7 @@ from typing import Any
 
 import msgpack
 
-from datachain.query.batch import RowsOutput, RowsOutputBatch
+from datachain.query.batch import RowsOutput
 
 DEFAULT_BATCH_SIZE = 10000
 STOP_SIGNAL = "STOP"
@@ -56,7 +56,6 @@ def put_into_queue(queue: Queue, item: Any) -> None:
 
 
 MSGPACK_EXT_TYPE_DATETIME = 42
-MSGPACK_EXT_TYPE_ROWS_INPUT_BATCH = 43
 
 
 def _msgpack_pack_extended_types(obj: Any) -> msgpack.ExtType:
@@ -70,12 +69,6 @@ def _msgpack_pack_extended_types(obj: Any) -> msgpack.ExtType:
         data = (obj.timestamp(),)  # type: ignore   # noqa: PGH003
         return msgpack.ExtType(MSGPACK_EXT_TYPE_DATETIME, pack("!d", *data))
 
-    if isinstance(obj, RowsOutputBatch):
-        return msgpack.ExtType(
-            MSGPACK_EXT_TYPE_ROWS_INPUT_BATCH,
-            msgpack_pack(obj.rows),
-        )
-
     raise TypeError(f"Unknown type: {obj}")
 
 
@@ -100,9 +93,6 @@ def _msgpack_unpack_extended_types(code: int, data: bytes) -> Any:
             tz_info = datetime.timezone(datetime.timedelta(seconds=timezone_offset))
         return datetime.datetime.fromtimestamp(timestamp, tz=tz_info)
 
-    if code == MSGPACK_EXT_TYPE_ROWS_INPUT_BATCH:
-        return RowsOutputBatch(msgpack_unpack(data))
-
     return msgpack.ExtType(code, data)
 
 

datachain/query/udf.py

@@ -46,4 +46,4 @@ class AbstractUDFDistributor(ABC):
 
     @staticmethod
     @abstractmethod
-    def run_worker() -> int: ...
+    def run_udf(fd: Optional[int] = None) -> int: ...

datachain/query/utils.py

@@ -1,33 +1,27 @@
-from typing import TYPE_CHECKING, Optional, Union
+from typing import Optional, Union
 
-from sqlalchemy import Column
+import sqlalchemy as sa
 
-if TYPE_CHECKING:
-    from sqlalchemy import ColumnElement, Select, TextClause
-
-
-ColT = Union[Column, "ColumnElement", "TextClause"]
+ColT = Union[sa.Column, sa.ColumnElement, sa.TextClause]
 
 
 def column_name(col: ColT) -> str:
     """Returns column name from column element."""
-    return col.name if isinstance(col, Column) else str(col)
+    return col.name if isinstance(col, sa.Column) else str(col)
 
 
-def get_query_column(query: "Select", name: str) -> Optional[ColT]:
+def get_query_column(query: sa.Select, name: str) -> Optional[ColT]:
     """Returns column element from query by name or None if column not found."""
     return next((col for col in query.inner_columns if column_name(col) == name), None)
 
 
-def get_query_id_column(query: "Select") -> ColT:
+def get_query_id_column(query: sa.Select) -> Optional[sa.ColumnElement]:
     """Returns ID column element from query or None if column not found."""
     col = get_query_column(query, "sys__id")
-    if col is None:
-        raise RuntimeError("sys__id column not found in query")
-    return col
+    return col if col is not None and isinstance(col, sa.ColumnElement) else None
 
 
-def select_only_columns(query: "Select", *names: str) -> "Select":
+def select_only_columns(query: sa.Select, *names: str) -> sa.Select:
     """Returns query selecting defined columns only."""
     if not names:
         return query

datachain/utils.py

@@ -323,6 +323,9 @@ def determine_processes(
         return True
     if parallel < 0:
         return True
+    if parallel == 1:
+        # Disable parallel processing if only one process is requested.
+        return False
     return parallel
 
 

{datachain-0.16.4.dist-info → datachain-0.16.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.16.4
+Version: 0.16.5
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.16.4.dist-info → datachain-0.16.5.dist-info}/RECORD

@@ -15,12 +15,12 @@ datachain/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/script_meta.py,sha256=V-LaFOZG84pD0Zc0NvejYdzwDgzITv6yHvAHggDCnuY,4978
 datachain/studio.py,sha256=HULrTH3lSpzoH8JBBDrXnOBlSdxjG6pf0M8bSHOakng,10113
 datachain/telemetry.py,sha256=0A4IOPPp9VlP5pyW9eBfaTK3YhHGzHl7dQudQjUAx9A,994
-datachain/utils.py,sha256=8Qz8lRrX0bUTGvwYd-OR-l6ElVRsQBdBO5QMvwt56T4,15190
+datachain/utils.py,sha256=DNqOi-Ydb7InyWvD9m7_yailxz6-YGpZzh00biQaHNo,15305
 datachain/catalog/__init__.py,sha256=cMZzSz3VoUi-6qXSVaHYN-agxQuAcz2XSqnEPZ55crE,353
-datachain/catalog/catalog.py,sha256=drCemStFXk2MZgexbUsSIBJuUvn0YwL1tJO69KrWeeg,61004
+datachain/catalog/catalog.py,sha256=tvPnX9eNd986hxBOtntckVNQNzw3AGWsgPJ7tRVALXU,61155
 datachain/catalog/datasource.py,sha256=IkGMh0Ttg6Q-9DWfU_H05WUnZepbGa28HYleECi6K7I,1353
 datachain/catalog/loader.py,sha256=UXjYD6BNRoupPvkiz3-b04jepXhtLHCA4gzKFnXxOtQ,5987
-datachain/cli/__init__.py,sha256=i40xHzVZP3iZFBw3UixQ2OU-s_GQq6OyvQ-_6opwIYc,8333
+datachain/cli/__init__.py,sha256=eKCyqT05OMESHXCC93iQdqzusBdk1ptqZbBeaEghkgo,8344
 datachain/cli/utils.py,sha256=wrLnAh7Wx8O_ojZE8AE4Lxn5WoxHbOj7as8NWlLAA74,3036
 datachain/cli/commands/__init__.py,sha256=zp3bYIioO60x_X04A4-IpZqSYVnpwOa1AdERQaRlIhI,493
 datachain/cli/commands/datasets.py,sha256=sQ83zxHLuP04cXqBYD3iVcsr49LHA3lnjYxdL142HMk,5793
@@ -28,9 +28,9 @@ datachain/cli/commands/du.py,sha256=9edEzDEs98K2VYk8Wf-ZMpUzALcgm9uD6YtoqbvtUGU,
 datachain/cli/commands/index.py,sha256=eglNaIe1yyIadUHHumjtNbgIjht6kme7SS7xE3YHR88,198
 datachain/cli/commands/ls.py,sha256=dSD2_MHng4t9HRFJZWMOCjPL4XU3qaBV3piNl8UXP08,5275
 datachain/cli/commands/misc.py,sha256=c0DmkOLwcDI2YhA8ArOuLJk6aGzSMZCiKL_E2JGibVE,600
-datachain/cli/commands/query.py,sha256=2S7hQxialt1fkbocxi6JXZI6jS5QnFrD1aOjKgZkzfI,1471
+datachain/cli/commands/query.py,sha256=Xzfgh14nPVH-sclqX1tpZqgfdTugw5s_44v0D33z6FA,1505
 datachain/cli/commands/show.py,sha256=K__cCLDJLTRt-sBTMxDID0A_4dFgRRMvjDrrVWcbMUQ,1606
-datachain/cli/parser/__init__.py,sha256=SKB94ZS9kRHV7UOrQcIXsSQ7BOFlp4U2To4wseXXcaI,15724
+datachain/cli/parser/__init__.py,sha256=0N3QZKJPBhcxCt5Ky83_Oa-p9RMwBz6KfMxSSMtMbz0,15913
 datachain/cli/parser/job.py,sha256=1TfM8qkGn0e-zixnn40Z5-Pv9h_mZncPaDAOyzjIf5k,3601
 datachain/cli/parser/studio.py,sha256=Y-1OlQGecLVi9QofvWUfSlPd2ISyaESf7QFGZqGsrdw,3609
 datachain/cli/parser/utils.py,sha256=rETdD-9Hq9A4OolgfT7jQw4aoawtbfmkdtH6E7nkhpI,2888
@@ -44,12 +44,12 @@ datachain/client/local.py,sha256=cGoCYflribzexiOe-Y1qbaE2fJRh-_EgQrfCSa0yK_E,456
 datachain/client/s3.py,sha256=YCtDhKVO_jGsMPeyqe3xk5QsF5lqMabqkt0tPFWUHOM,7286
 datachain/data_storage/__init__.py,sha256=9Wit-oe5P46V7CJQTD0BJ5MhOa2Y9h3ddJ4VWTe-Lec,273
 datachain/data_storage/db_engine.py,sha256=n8ojCbvVMPY2e3SG8fUaaD0b9GkVfpl_Naa_6EiHfWg,3788
-datachain/data_storage/job.py,sha256=w-7spowjkOa1P5fUVtJou3OltT0L48P0RYWZ9rSJ9-s,383
-datachain/data_storage/metastore.py,sha256=bhfAaijM7p_D5ltMWg-CVEv9lTflL3bGUWqAmJ8qFbc,37774
+datachain/data_storage/job.py,sha256=9r0OGwh22bHNIvLHqg8_-eJSP1YYB-BN5HOla5TdCxw,402
+datachain/data_storage/metastore.py,sha256=NZwV4LaCwCtPzwmzFV28FqnkV9g2MUvRwY06qzS7eAc,38295
 datachain/data_storage/schema.py,sha256=asZYz1cg_WKfe2Q-k5W51E2z2CzHU5B4QEDZDMFr8yo,9346
 datachain/data_storage/serializer.py,sha256=6G2YtOFqqDzJf1KbvZraKGXl2XHZyVml2krunWUum5o,927
 datachain/data_storage/sqlite.py,sha256=f4tvq0gzYQP7aYGnfL3j4IBUNvctpBxI_ioFU-B1LFc,24540
-datachain/data_storage/warehouse.py,sha256=0GQdf7BwGav783zhB-RBKwjUqkLf2JW2lS-KXTqimnQ,31083
+datachain/data_storage/warehouse.py,sha256=HYFj3HAMFtsWsmen4Xg5KTEkk5fXVT0pDWWiFW0l0UU,31516
 datachain/diff/__init__.py,sha256=YkGdiDbZIMhAZ2SJ4eSe00HU67VP1P6SL2L_t0ODYMs,9425
 datachain/fs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/fs/reference.py,sha256=A8McpXF0CqbXPqanXuvpKu50YLB3a2ZXA3YAPxtBXSM,914
@@ -66,14 +66,14 @@ datachain/func/random.py,sha256=pENOLj9rSmWfGCnOsUIaCsVC5486zQb66qfQvXaz9Z4,452
 datachain/func/string.py,sha256=8az3BTeezlaZt6NW-54GWX7WSosAOVMbTr6bXIYyJq4,5958
 datachain/func/window.py,sha256=0MB1yjpVbwOrl_WNLZ8V3jkJz3o0XlYinpAcZQJuxiA,1688
 datachain/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datachain/lib/arrow.py,sha256=9UBCF-lftQaz0yxdsjbLKbyzVSmrF_QSWdhp2oBDPqs,9486
+datachain/lib/arrow.py,sha256=mFO_6wRqzpEzBhXf7Xn1aeLUvaiHcC6XQ-8as9sbcgY,10253
 datachain/lib/clip.py,sha256=lm5CzVi4Cj1jVLEKvERKArb-egb9j1Ls-fwTItT6vlI,6150
 datachain/lib/data_model.py,sha256=ZwBXELtqROEdLL4DmxTipnwUZmhQvMz_UVDzyf7nQ9Y,2899
 datachain/lib/dataset_info.py,sha256=Mmo3r_MWRb-47H4QueSaUqgeENJiJZmjkTYBMpRuKM8,3128
 datachain/lib/file.py,sha256=HLQXS_WULm7Y-fkHMy0WpibVAcrkLPRS6CrZy6rwFe0,30450
 datachain/lib/hf.py,sha256=gjxuStZBlKtNk3-4yYSlWZDv9zBGblOdvEy_Lwap5hA,5882
 datachain/lib/image.py,sha256=butvUY_33PVEYPKX2nVCPeJjJVcBaptZwsE9REQsTS8,3247
-datachain/lib/listing.py,sha256=qmie3z8UR5bdXiZnMo3Qy5IpT76vFu5gn0x2ksMi-y8,7036
+datachain/lib/listing.py,sha256=lFG1Ms6jwm_uqlOcsBUpkmyeSO9TZdtNd820PEpAHP4,7077
 datachain/lib/listing_info.py,sha256=9ua40Hw0aiQByUw3oAEeNzMavJYfW0Uhe8YdCTK-m_g,1110
 datachain/lib/meta_formats.py,sha256=Epydbdch1g4CojK8wd_ePzmwmljC4fVWlJtZ16jsX-A,6349
 datachain/lib/model_store.py,sha256=DNIv8Y6Jtk1_idNLzIpsThOsdW2BMAudyUCbPUcgcxk,2515
@@ -82,7 +82,7 @@ datachain/lib/settings.py,sha256=ZELRCTLbi5vzRPiDX6cQ9LLg9TefJ_A05gIGni0lll8,253
 datachain/lib/signal_schema.py,sha256=uIBHYXtu_XpLbOUVC-kq-GduEOCfz9hQORi9ZG3JFqo,35820
 datachain/lib/tar.py,sha256=3WIzao6yD5fbLqXLTt9GhPGNonbFIs_fDRu-9vgLgsA,1038
 datachain/lib/text.py,sha256=UNHm8fhidk7wdrWqacEWaA6I9ykfYqarQ2URby7jc7M,1261
-datachain/lib/udf.py,sha256=zCdO5__gLMCgrdHmOvIa0eoWKCDAU1uO-MMAu_EU13o,16228
+datachain/lib/udf.py,sha256=FWqA476ygdk4MU-0qehYKxvnt8Tekh21Cyf3RgddD1k,16674
 datachain/lib/udf_signature.py,sha256=2EtsOPDNSPqcOlYwqbCdy6RF5MldI-7smii8aLy8p7Y,7543
 datachain/lib/utils.py,sha256=QrjVs_oLRXEotOPUYurBJypBFi_ReTJmxcnJeH4j2Uk,1596
 datachain/lib/video.py,sha256=suH_8Mi8VYk4-IVb1vjSduF_njs64ji1WGKHxDLnGYw,6629
@@ -95,9 +95,9 @@ datachain/lib/convert/sql_to_python.py,sha256=XXCBYDQFUXJIBNWkjEP944cnCfJ8GF2Tji
 datachain/lib/convert/unflatten.py,sha256=ysMkstwJzPMWUlnxn-Z-tXJR3wmhjHeSN_P-sDcLS6s,2010
 datachain/lib/convert/values_to_tuples.py,sha256=j5yZMrVUH6W7b-7yUvdCTGI7JCUAYUOzHUGPoyZXAB0,4360
 datachain/lib/dc/__init__.py,sha256=HD0NYrdy44u6kkpvgGjJcvGz-UGTHui2azghcT8ZUg0,838
-datachain/lib/dc/csv.py,sha256=asWPAxhMgIoLAdD2dObDlnGL8CTSD3TAuFuM4ci89bQ,4374
+datachain/lib/dc/csv.py,sha256=q6a9BpapGwP6nwy6c5cklxQumep2fUp9l2LAjtTJr6s,4411
 datachain/lib/dc/database.py,sha256=g5M6NjYR1T0vKte-abV-3Ejnm-HqxTIMir5cRi_SziE,6051
-datachain/lib/dc/datachain.py,sha256=36J8QIB04hKKumQgLvHNTC94Pd7G2yE4slZ9RfwI9zw,76980
+datachain/lib/dc/datachain.py,sha256=vIGbRi1f9pwk1fi-Dg90cdDVWvHfR2Gi22F7-E7GJ1U,77362
 datachain/lib/dc/datasets.py,sha256=u6hlz0Eodh_s39TOW6kz0VIL3nGfadqu8FLoWqDxSJs,6890
 datachain/lib/dc/hf.py,sha256=PJl2wiLjdRsMz0SYbLT-6H8b-D5i2WjeH7li8HHOk_0,2145
 datachain/lib/dc/json.py,sha256=ZUThPDAaP2gBFIL5vsQTwKBcuN_dhvC_O44wdDv0jEc,2683
@@ -118,16 +118,16 @@ datachain/model/ultralytics/bbox.py,sha256=1wnu3tXQn3i-047pncfHyCdKxHdP7MJDI1DXq
 datachain/model/ultralytics/pose.py,sha256=gXAWfAk4OWZl93hKcQPKZvqJa3nIrECB4RM8K8w8Og0,3109
 datachain/model/ultralytics/segment.py,sha256=koq1HASo29isf0in6oSlzmU4IzsmOXe87F1ajQQVfh4,2911
 datachain/query/__init__.py,sha256=7DhEIjAA8uZJfejruAVMZVcGFmvUpffuZJwgRqNwe-c,263
-datachain/query/batch.py,sha256=6w8gzLTmLeylststu-gT5jIqEfi4-djS7_yTYyeo-fw,4190
-datachain/query/dataset.py,sha256=8O9TFOBLyh_ylqY4gZ7MRLziwAQaU6YdDR_SfBSK65c,58806
-datachain/query/dispatch.py,sha256=5p_jXxKJVCfIA4jLSQ0tAY1IhZUS3oJvyQXUH0Dk3bc,13215
-datachain/query/metrics.py,sha256=r5b0ygYhokbXp8Mg3kCH8iFSRw0jxzyeBe-C-J_bKFc,938
+datachain/query/batch.py,sha256=-goxLpE0EUvaDHu66rstj53UnfHpYfBUGux8GSpJ93k,4306
+datachain/query/dataset.py,sha256=O1sO169hiXWdf3uqigT3AJUPvjQDL3LBXXJ-0OC3SIw,59102
+datachain/query/dispatch.py,sha256=xNAZ5HNx48vY8Azhh3j-H_WSeAlRTQKNWKJbOAEhyPY,15398
+datachain/query/metrics.py,sha256=DOK5HdNVaRugYPjl8qnBONvTkwjMloLqAr7Mi3TjCO0,858
 datachain/query/params.py,sha256=O_j89mjYRLOwWNhYZl-z7mi-rkdP7WyFmaDufsdTryE,863
-datachain/query/queue.py,sha256=waqM_KzavU8C-G95-4211Nd4GXna_u2747Chgwtgz2w,3839
+datachain/query/queue.py,sha256=v0UeK4ilmdiRoJ5OdjB5qpnHTYDxRP4vhVp5Iw_toaI,3512
 datachain/query/schema.py,sha256=b_KnVy6B26Ol4nYG0LqNNpeQ1QYPk95YRGUjXfdaQWs,6606
 datachain/query/session.py,sha256=wNdOHAi4HrsEihfzdcTlfB5i1xyj0dw6rlUz84StOoU,6512
-datachain/query/udf.py,sha256=ljAYaF-J77t7iS4zc1-g1ssYd4c6Q-ccKGEc3VQQmeM,1322
-datachain/query/utils.py,sha256=u0A_BwG9PNs0DxoDcvSWgWLpj3ByTUv8CqH13CIuGag,1293
+datachain/query/udf.py,sha256=e753bDJzTNjGFQn1WGTvOAWSwjDbrFI1-_DDWkWN2ls,1343
+datachain/query/utils.py,sha256=HaSDNH_XGvp_NIcXjcB7j4vJRPi4_tbztDWclYelHY4,1208
 datachain/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datachain/remote/studio.py,sha256=Lz8C4aps7-4dUiRKWNHNfChJ7YFVQAS0zp14ienThOc,13187
 datachain/sql/__init__.py,sha256=6SQRdbljO3d2hx3EAVXEZrHQKv5jth0Jh98PogT59No,262
@@ -151,9 +151,9 @@ datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR
 datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
 datachain/toolkit/split.py,sha256=ktGWzY4kyzjWyR86dhvzw-Zhl0lVk_LOX3NciTac6qo,2914
 datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
-datachain-0.16.4.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
-datachain-0.16.4.dist-info/METADATA,sha256=-Dc2kwVGq9FvhINK3PwvBVQs3dcMw3KU9qzZmHz8Htc,11331
-datachain-0.16.4.dist-info/WHEEL,sha256=wXxTzcEDnjrTwFYjLPcsW_7_XihufBwmpiBeiXNBGEA,91
-datachain-0.16.4.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
-datachain-0.16.4.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
-datachain-0.16.4.dist-info/RECORD,,
+datachain-0.16.5.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.16.5.dist-info/METADATA,sha256=gSZHMVWTIzRdW_LPQZuY73OVmkRpXnQfEiEjwA3-xfg,11331
+datachain-0.16.5.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
+datachain-0.16.5.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.16.5.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.16.5.dist-info/RECORD,,

{datachain-0.16.4.dist-info → datachain-0.16.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.1.0)
+Generator: setuptools (80.3.1)
 Root-Is-Purelib: true
 Tag: py3-none-any