datachain 0.30.5__py3-none-any.whl → 0.39.0__py3-none-any.whl

datachain/utils.py

@@ -1,6 +1,5 @@
 import glob
 import io
-import json
 import logging
 import os
 import os.path as osp
@@ -10,9 +9,8 @@ import sys
 import time
 from collections.abc import Iterable, Iterator, Sequence
 from contextlib import contextmanager
-from datetime import date, datetime, timezone
-from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
-from uuid import UUID
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any, TypeVar
 
 import cloudpickle
 import platformdirs
@@ -25,7 +23,7 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
 
-DEFAULT_CHUNK_ROWS = 2000
+DEFAULT_BATCH_SIZE = 2000
 
 logger = logging.getLogger("datachain")
 
@@ -53,11 +51,11 @@ class DataChainDir:
 
     def __init__(
         self,
-        root: Optional[str] = None,
-        cache: Optional[str] = None,
-        tmp: Optional[str] = None,
-        db: Optional[str] = None,
-        config: Optional[str] = None,
+        root: str | None = None,
+        cache: str | None = None,
+        tmp: str | None = None,
+        db: str | None = None,
+        config: str | None = None,
     ) -> None:
         self.root = osp.abspath(root) if root is not None else self.default_root()
         self.cache = (
@@ -122,7 +120,7 @@ def global_config_dir():
     )
 
 
-def human_time_to_int(time: str) -> Optional[int]:
+def human_time_to_int(time: str) -> int | None:
     if not time:
         return None
 
@@ -146,7 +144,7 @@ def time_to_str(dt):
     return dt.strftime("%Y-%m-%d %H:%M:%S")
 
 
-def time_to_local(dt: Union[datetime, str]) -> datetime:
+def time_to_local(dt: datetime | str) -> datetime:
     # TODO check usage
     if isinstance(dt, str):
         dt = isoparse(dt)
@@ -156,11 +154,11 @@ def time_to_local(dt: Union[datetime, str]) -> datetime:
         return dt
 
 
-def time_to_local_str(dt: Union[datetime, str]) -> str:
+def time_to_local_str(dt: datetime | str) -> str:
     return time_to_str(time_to_local(dt))
 
 
-def is_expired(expires: Optional[Union[datetime, str]]):
+def is_expired(expires: datetime | str | None):
     if expires:
         return time_to_local(expires) < time_to_local(datetime.now())  # noqa: DTZ005
 
@@ -228,7 +226,7 @@ _T_co = TypeVar("_T_co", covariant=True)
 
 def _dynamic_batched_core(
     iterable: Iterable[_T_co],
-    batch_rows: int,
+    batch_size: int,
 ) -> Iterator[list[_T_co]]:
     """Core batching logic that yields lists."""
 
@@ -236,7 +234,7 @@ def _dynamic_batched_core(
 
     for item in iterable:
         # Check if adding this item would exceed limits
-        if len(batch) >= batch_rows and batch:  # Yield current batch if we have one
+        if len(batch) >= batch_size and batch:  # Yield current batch if we have one
             yield batch
             batch = []
 
@@ -247,23 +245,22 @@ def _dynamic_batched_core(
         yield batch
 
 
-def batched(iterable: Iterable[_T_co], batch_rows: int) -> Iterator[tuple[_T_co, ...]]:
+def batched(iterable: Iterable[_T_co], batch_size: int) -> Iterator[tuple[_T_co, ...]]:
     """
-    Batch data into tuples of length batch_rows .
+    Batch data into tuples of length batch_size.
     The last batch may be shorter.
     """
-    yield from (tuple(batch) for batch in _dynamic_batched_core(iterable, batch_rows))
+    yield from (tuple(batch) for batch in _dynamic_batched_core(iterable, batch_size))
 
 
 def batched_it(
     iterable: Iterable[_T_co],
-    batch_rows: int = DEFAULT_CHUNK_ROWS,
+    batch_size: int = DEFAULT_BATCH_SIZE,
 ) -> Iterator[Iterator[_T_co]]:
     """
-    Batch data into iterators with dynamic sizing
-    based on row count and memory usage.
+    Batch data into iterators with dynamic sizing based on row count and memory usage.
     """
-    yield from (iter(batch) for batch in _dynamic_batched_core(iterable, batch_rows))
+    yield from (iter(batch) for batch in _dynamic_batched_core(iterable, batch_size))
 
 
 def flatten(items):
@@ -302,9 +299,9 @@ def retry_with_backoff(retries=5, backoff_sec=1, errors=(Exception,)):
 
 
 def determine_workers(
-    workers: Union[bool, int],
-    rows_total: Optional[int] = None,
-) -> Union[bool, int]:
+    workers: bool | int,
+    rows_total: int | None = None,
+) -> bool | int:
     """Determine the number of workers to use for distributed processing."""
     if rows_total is not None and rows_total <= 1:
         # Disable distributed processing if there is no rows or only one row.
@@ -323,9 +320,9 @@ def determine_workers(
 
 
 def determine_processes(
-    parallel: Optional[Union[bool, int]] = None,
-    rows_total: Optional[int] = None,
-) -> Union[bool, int]:
+    parallel: bool | int | None = None,
+    rows_total: int | None = None,
+) -> bool | int:
     """Determine the number of processes to use for parallel processing."""
     if rows_total is not None and rows_total <= 1:
         # Disable parallel processing if there is no rows or only one row.
@@ -345,8 +342,8 @@ def determine_processes(
 
 
 def get_env_list(
-    key: str, default: Optional[Sequence] = None, sep: str = ","
-) -> Optional[Sequence[str]]:
+    key: str, default: Sequence | None = None, sep: str = ","
+) -> Sequence[str] | None:
     try:
         str_val = os.environ[key]
     except KeyError:
@@ -387,10 +384,10 @@ def show_df(
 
 
 def show_records(
-    records: Optional[list[dict]],
+    records: list[dict] | None,
     collapse_columns: bool = False,
     system_columns: bool = False,
-    hidden_fields: Optional[list[str]] = None,
+    hidden_fields: list[str] | None = None,
 ) -> None:
     import pandas as pd
 
@@ -403,18 +400,6 @@ def show_records(
     return show_df(df, collapse_columns=collapse_columns, system_columns=system_columns)
 
 
-class JSONSerialize(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, bytes):
-            return list(obj[:1024])
-        if isinstance(obj, (datetime, date)):
-            return obj.isoformat()
-        if isinstance(obj, UUID):
-            return str(obj)
-
-        return super().default(obj)
-
-
 def inside_colab() -> bool:
     try:
         from google import colab  # type: ignore[attr-defined]  # noqa: F401
@@ -434,7 +419,7 @@ def inside_notebook() -> bool:
 
     if shell == "ZMQInteractiveShell":
         try:
-            import IPython
+            import IPython  # type: ignore[import-not-found]
 
             return IPython.__version__ >= "6.0.0"
         except ImportError:
@@ -519,7 +504,7 @@ def row_to_nested_dict(
 ) -> dict[str, Any]:
     """Converts a row to a nested dict based on the provided headers."""
     result: dict[str, Any] = {}
-    for h, v in zip(headers, row):
+    for h, v in zip(headers, row, strict=False):
         nested_dict_path_set(result, h, v)
     return result
 
@@ -530,7 +515,7 @@ def safe_closing(thing: T) -> Iterator[T]:
         yield thing
     finally:
         if hasattr(thing, "close"):
-            thing.close()
+            thing.close()  # type: ignore[attr-defined]
 
 
 def getenv_bool(name: str, default: bool = False) -> bool:
@@ -538,3 +523,9 @@ def getenv_bool(name: str, default: bool = False) -> bool:
     if val is None:
         return default
     return val.lower() in ("1", "true", "yes", "on")
+
+
+def ensure_sequence(x) -> Sequence:
+    if isinstance(x, Sequence) and not isinstance(x, (str, bytes)):
+        return x
+    return [x]

{datachain-0.30.5.dist-info → datachain-0.39.0.dist-info}/METADATA

@@ -1,20 +1,19 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.30.5
+Version: 0.39.0
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0
 Project-URL: Documentation, https://datachain.dvc.ai
-Project-URL: Issues, https://github.com/iterative/datachain/issues
-Project-URL: Source, https://github.com/iterative/datachain
+Project-URL: Issues, https://github.com/datachain-ai/datachain/issues
+Project-URL: Source, https://github.com/datachain-ai/datachain
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Development Status :: 2 - Pre-Alpha
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 Requires-Dist: pyyaml
@@ -42,7 +41,7 @@ Requires-Dist: cloudpickle
 Requires-Dist: pydantic
 Requires-Dist: jmespath>=1.0
 Requires-Dist: datamodel-code-generator>=0.25
-Requires-Dist: Pillow<12,>=10.0.0
+Requires-Dist: Pillow<13,>=10.0.0
 Requires-Dist: msgpack<2,>=1.0.4
 Requires-Dist: psutil
 Requires-Dist: huggingface_hub
@@ -56,16 +55,15 @@ Provides-Extra: docs
 Requires-Dist: mkdocs>=1.5.2; extra == "docs"
 Requires-Dist: mkdocs-gen-files>=0.5.0; extra == "docs"
 Requires-Dist: mkdocs-material==9.5.22; extra == "docs"
-Requires-Dist: mkdocs-section-index>=0.3.6; extra == "docs"
 Requires-Dist: mkdocstrings-python>=1.6.3; extra == "docs"
 Requires-Dist: mkdocs-literate-nav>=0.6.1; extra == "docs"
+Requires-Dist: mkdocs-section-index>=0.3.10; extra == "docs"
 Requires-Dist: eval-type-backport; extra == "docs"
 Provides-Extra: torch
 Requires-Dist: torch>=2.1.0; extra == "torch"
 Requires-Dist: torchvision; extra == "torch"
 Requires-Dist: transformers>=4.36.0; extra == "torch"
 Provides-Extra: audio
-Requires-Dist: torchaudio; extra == "audio"
 Requires-Dist: soundfile; extra == "audio"
 Provides-Extra: remote
 Requires-Dist: lz4; extra == "remote"
@@ -85,7 +83,8 @@ Provides-Extra: postgres
 Requires-Dist: psycopg2-binary>=2.9.0; extra == "postgres"
 Provides-Extra: tests
 Requires-Dist: datachain[audio,hf,postgres,remote,torch,vector,video]; extra == "tests"
-Requires-Dist: pytest<9,>=8; extra == "tests"
+Requires-Dist: pytest<10,>=8; extra == "tests"
+Requires-Dist: pytest-asyncio; extra == "tests"
 Requires-Dist: pytest-sugar>=0.9.6; extra == "tests"
 Requires-Dist: pytest-cov>=4.1.0; extra == "tests"
 Requires-Dist: pytest-mock>=3.12.0; extra == "tests"
@@ -102,7 +101,7 @@ Requires-Dist: scipy; extra == "tests"
 Requires-Dist: ultralytics; extra == "tests"
 Provides-Extra: dev
 Requires-Dist: datachain[docs,tests]; extra == "dev"
-Requires-Dist: mypy==1.17.0; extra == "dev"
+Requires-Dist: mypy==1.19.0; extra == "dev"
 Requires-Dist: types-python-dateutil; extra == "dev"
 Requires-Dist: types-dateparser; extra == "dev"
 Requires-Dist: types-pytz; extra == "dev"
@@ -117,6 +116,7 @@ Requires-Dist: huggingface_hub[hf_transfer]; extra == "examples"
 Requires-Dist: ultralytics; extra == "examples"
 Requires-Dist: open_clip_torch; extra == "examples"
 Requires-Dist: openai; extra == "examples"
+Requires-Dist: torchaudio; extra == "examples"
 Dynamic: license-file
 
 ================
@@ -133,14 +133,14 @@ Dynamic: license-file
 .. |Python Version| image:: https://img.shields.io/pypi/pyversions/datachain
    :target: https://pypi.org/project/datachain
    :alt: Python Version
-.. |Codecov| image:: https://codecov.io/gh/iterative/datachain/graph/badge.svg?token=byliXGGyGB
-   :target: https://codecov.io/gh/iterative/datachain
+.. |Codecov| image:: https://codecov.io/gh/datachain-ai/datachain/graph/badge.svg?token=byliXGGyGB
+   :target: https://codecov.io/gh/datachain-ai/datachain
    :alt: Codecov
-.. |Tests| image:: https://github.com/iterative/datachain/actions/workflows/tests.yml/badge.svg
-   :target: https://github.com/iterative/datachain/actions/workflows/tests.yml
+.. |Tests| image:: https://github.com/datachain-ai/datachain/actions/workflows/tests.yml/badge.svg
+   :target: https://github.com/datachain-ai/datachain/actions/workflows/tests.yml
    :alt: Tests
 .. |DeepWiki| image:: https://deepwiki.com/badge.svg
-   :target: https://deepwiki.com/iterative/datachain
+   :target: https://deepwiki.com/datachain-ai/datachain
    :alt: DeepWiki
 
 DataChain is a Python-based AI-data warehouse for transforming and analyzing unstructured
@@ -210,45 +210,33 @@ datasets that evolve over time and may occasionally have processing errors.
 .. code:: py
 
     import datachain as dc
-    from datachain import C, File
 
-    def process_file(file: File):
-        """Process a file, which may occasionally fail."""
+    def process_file(file: dc.File) -> tuple[str, str, str]:
+        """Analyze a file, may occasionally fail."""
         try:
             # Your processing logic here
             content = file.read_text()
-            result = analyze_content(content)
-            return {
-                "content": content,
-                "result": result,
-                "error": None  # No error
-            }
+            result = content.upper()
+            return content, result, ""  # No error
         except Exception as e:
             # Return an error that will trigger reprocessing next time
-            return {
-                "content": None,
-                "result": None,
-                "error": str(e)  # Error field will trigger retry
-            }
+            return "", "", str(e)  # Error field will trigger retry
 
     # Process files efficiently with delta and retry
+    # Run it many times, keep adding files, to see delta and retry in action
     chain = (
         dc.read_storage(
             "data/",
             update=True,
             delta=True,              # Process only new/changed files
             delta_on="file.path",    # Identify files by path
-            retry_on="error"         # Field that indicates errors
+            delta_retry="error",     # Process files with error again
         )
-        .map(processed_result=process_file)
-        .mutate(
-            content=C("processed_result.content"),
-            result=C("processed_result.result"),
-            error=C("processed_result.error")
-        )
-        .save(name="processed_data")
+        .map(process_file, output=("content", "result", "error"))
+        .save("processed-data")
     )
 
+
 Example: LLM based text-file evaluation
 ---------------------------------------
 
@@ -355,7 +343,7 @@ DataChain Studio Platform
 - **Access control** including SSO and team based collaboration.
 
 .. _PyPI: https://pypi.org/
-.. _file an issue: https://github.com/iterative/datachain/issues
+.. _file an issue: https://github.com/datachain-ai/datachain/issues
 .. github-only
 .. _Contributor Guide: https://docs.datachain.ai/contributing
 .. _Pydantic: https://github.com/pydantic/pydantic

datachain-0.39.0.dist-info/RECORD

@@ -0,0 +1,173 @@
+datachain/__init__.py,sha256=KVwlU1kC6qWTFQD1DxQs1M_4rh0GC9ZUG3FXJZmQjh4,1852
+datachain/__main__.py,sha256=hG3Y4ARGEqe1AWwNMd259rBlqtphx1Wk39YbueQ0yV8,91
+datachain/asyn.py,sha256=3xSQkc2AMABDZcbDdrbJq6QxVWH3yIaY0pzCiXRqR0U,9634
+datachain/cache.py,sha256=Klkc7iL_KvryeZk-UNjtByTFk7URbpb60XblalqHoYI,3604
+datachain/checkpoint.py,sha256=AOMqN_2fNuEBJDAsmc-P4L7FU444eQxTU4MCgr-XEH8,1121
+datachain/config.py,sha256=KPXef6P4NAZiEbSDMUcFwuNVTul2fZBs5xrCbyRl6Tg,4193
+datachain/dataset.py,sha256=U3UAbElJEWND0_-aO8JPdNlAL-BDyOa6D_fVbvs5c1U,25978
+datachain/delta.py,sha256=2c4XO-nKKYD1wMOW_zhEfUceAhH_8nISfnKVpZbScDg,12194
+datachain/error.py,sha256=5Qkd55kl4NIG33iwOeY1Qa6SM8FLftIZ8vcf663r-9Y,1961
+datachain/hash_utils.py,sha256=uU7xa-XQRC_8zmUxHg9h__Kts4TxW566311d1LRTz-Q,4510
+datachain/job.py,sha256=mx4Cs-qX8DLyaie5JbZMSYYQSRspbQruM7L1s5kg1ws,1374
+datachain/json.py,sha256=yK6ID17dP0vd0NIhSHAa3u2ym4N0jjtDEpBZ5PUuRlE,3673
+datachain/listing.py,sha256=yIZYiCLVHQirGsXs52Ssph2j-woCIRnagqLJIRT1l7U,7406
+datachain/namespace.py,sha256=YhxHdmCekWH4l-ZayNHGiPy5KAz_5LGqviivFYu0u9U,2337
+datachain/node.py,sha256=gBLCoh-3nyaCDnMPt3gLS_t3m7qL0_JDiN02a9DH_kY,5552
+datachain/nodes_fetcher.py,sha256=_wgaKyqEjkqdwJ_Hj6D8vUYz7hnU7g6xhm0H6ZnYxmE,1095
+datachain/nodes_thread_pool.py,sha256=Fh4YZlwNzZMJxoUpuLGPKAnsvTGn8zy6xfrxBOeySfA,3971
+datachain/plugins.py,sha256=QRpM-xcDAyGGOGAO6K3Hk4rGb78plgQ-k0iVWU8tn-I,693
+datachain/progress.py,sha256=lRzxoYP4Qv2XBwD78sOkmYRzHFpZ2ExVNJF8wAeICtY,770
+datachain/project.py,sha256=2_RAUmkDpbk5q5Iab9H6_k3S8Cju7dFRofk3OaaaQYo,2261
+datachain/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datachain/script_meta.py,sha256=vQOnPnqsNEDGtCHfy2DozcOdPwjGQ-dwDrrgHGpw90s,4935
+datachain/semver.py,sha256=UB8GHPBtAP3UJGeiuJoInD7SK-DnB93_Xd1qy_CQ9cU,2074
+datachain/studio.py,sha256=seWeSACPVgUI23Z8UbgjeJQIZWVLu45aVG4p7kMDj5E,15611
+datachain/telemetry.py,sha256=0A4IOPPp9VlP5pyW9eBfaTK3YhHGzHl7dQudQjUAx9A,994
+datachain/utils.py,sha256=qizzUaT3SOoO7cWeZImWAMG00sMx69_j6EmcaiqO_1s,15414
+datachain/catalog/__init__.py,sha256=r5aZompzY1jKJGt9PmvMTrObx1rQDiG5PsRp_Da3yLs,299
+datachain/catalog/catalog.py,sha256=pM1lXssrI4PdnGms6QJDVMGjBJrwZyKd5djskbndG54,64739
+datachain/catalog/datasource.py,sha256=IkGMh0Ttg6Q-9DWfU_H05WUnZepbGa28HYleECi6K7I,1353
+datachain/catalog/dependency.py,sha256=EHuu_Ox76sEhy71NXjFJiHxQVTz19KecqBcrjwFCa7M,5280
+datachain/catalog/loader.py,sha256=VTaGPc4ASNdUdr7Elobp8qcXUOHwd0oqQcnk3LUwtF0,6244
+datachain/cli/__init__.py,sha256=DrJ9hABuo7ujKwoqt66wiH5vLNYFw_3yz6KJ9khXYdk,8133
+datachain/cli/utils.py,sha256=v4__FknyILQejFE-GdqvBamOaXhfdeHxmJ9hBmEbwFE,2471
+datachain/cli/commands/__init__.py,sha256=vH52z3H9wmfMJRiaFY-_Gk8tIzK3x6A_26oE79RROY8,434
+datachain/cli/commands/datasets.py,sha256=S6sMuG9MvC2svQzfVZ9DKkLqMONWWJ1P5k1fywkrBr8,6915
+datachain/cli/commands/du.py,sha256=9edEzDEs98K2VYk8Wf-ZMpUzALcgm9uD6YtoqbvtUGU,391
+datachain/cli/commands/index.py,sha256=eglNaIe1yyIadUHHumjtNbgIjht6kme7SS7xE3YHR88,198
+datachain/cli/commands/ls.py,sha256=0QyE1dSUES4EmhG_qcisfdDdr8m2G6i5nmQS7sjGeVU,5293
+datachain/cli/commands/misc.py,sha256=c0DmkOLwcDI2YhA8ArOuLJk6aGzSMZCiKL_E2JGibVE,600
+datachain/cli/commands/show.py,sha256=zkgBEVW04j41ewNnyD0nUY-NcCBANoEpQ8dpIrKof5U,1593
+datachain/cli/parser/__init__.py,sha256=ub3Y7AM0stEKmW3QZPcXhO_Pefy98GEMAgKjuAcwmt0,14019
+datachain/cli/parser/job.py,sha256=f6IQrFUyXCSseOYxlDFPs414MsJszA8YWxEYAv530r0,6105
+datachain/cli/parser/studio.py,sha256=Bo__LKM7qhJGgkyX8M_bCvgZ2Gvqq6r_X4t1NdtaBIY,3881
+datachain/cli/parser/utils.py,sha256=WbuKIDTTAPItgbMoxyz-DBInHKxOkNfgW7L2V0hcBxs,2857
+datachain/client/__init__.py,sha256=1kDpCPoibMXi1gExR4lTLc5pi-k6M5TANiwtXkPoLhU,49
+datachain/client/azure.py,sha256=cknEJaVrSGQNApdcDKrq9HykJJWyzEPetfs_sYZuahE,3257
+datachain/client/fileslice.py,sha256=bT7TYco1Qe3bqoc8aUkUZcPdPofJDHlryL5BsTn9xsY,3021
+datachain/client/fsspec.py,sha256=MZ1vmCIUIIxFCI9mlp_Fbd0KuqTL49vf-qQ5OMj5mFg,14603
+datachain/client/gcs.py,sha256=Qy2mqYJnZCbTR80D2ZLygLjaVkmzjkYJ7L3LeV3449M,5188
+datachain/client/hf.py,sha256=n5xJZdvNLS-SqokxuBCIPfGbhIeC_XfLm_BNYtEVvg4,2677
+datachain/client/http.py,sha256=ebeXD0xy73VFmxuAr9Dk79KpyvCk2TL8Qx-3JTK02rY,5152
+datachain/client/local.py,sha256=slWi7dB77A8XsNcwDFMyKzaDyUONwaCVwRAiMkLpBV8,4739
+datachain/client/s3.py,sha256=KS9o0jxXJRFp7Isdibz366VaWrULmpegzfYdurJpAl0,7499
+datachain/data_storage/__init__.py,sha256=9Wit-oe5P46V7CJQTD0BJ5MhOa2Y9h3ddJ4VWTe-Lec,273
+datachain/data_storage/db_engine.py,sha256=-XHNnmLtg8_oqmlYB1aQ2swVB7G5P3vllNPmH0mWQCQ,3831
+datachain/data_storage/job.py,sha256=KTNKarEq1p7sXEsfV3oeQHXWVy6stv4wmD3qv4nF4F4,465
+datachain/data_storage/metastore.py,sha256=kYSqEx1ZvcnqyGqLOsykD0owjiCsvCuGyD14dkYwz10,74006
+datachain/data_storage/schema.py,sha256=3fAgiE11TIDYCW7EbTdiOm61SErRitvsLr7YPnUlVm0,9801
+datachain/data_storage/serializer.py,sha256=lhWm_N1F97S8wJW647sJcdOZQSX25XVERL5xcw-XrWI,3877
+datachain/data_storage/sqlite.py,sha256=qFmf0YQKNsPmlkD1wiVYIo5n5QodksjzQZsx0gJgw28,30107
+datachain/data_storage/warehouse.py,sha256=AI281_7_VEkP7lgh5_vplow8ue_cB6XquoaSOuj8Rtc,35972
+datachain/diff/__init__.py,sha256=lGrygGzdWSSYJ1DgX4h2q_ko5QINEW8PKfxOwE9ZFnI,9394
+datachain/fs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datachain/fs/reference.py,sha256=A8McpXF0CqbXPqanXuvpKu50YLB3a2ZXA3YAPxtBXSM,914
+datachain/fs/utils.py,sha256=s-FkTOCGBk-b6TT3toQH51s9608pofoFjUSTc1yy7oE,825
+datachain/func/__init__.py,sha256=9K2MEC1NclY_zWuqevfEUOcrSE26cXDVnGqhNTj4lF8,1288
+datachain/func/aggregate.py,sha256=jtyzePQ_vRDZuEeLM7YZw5PzekOU86c4HDR5qeIWLwY,12363
+datachain/func/array.py,sha256=EpGSeBnnwmX16XqL8v7aqkJ8PAqkMZKLcnA0Auke0VI,13565
+datachain/func/base.py,sha256=ykhGUBlMGWSNN1fRDlPSyDJVf8DV7k_r_tyNVm_o7ZY,590
+datachain/func/conditional.py,sha256=vpBLKqwIA0OKrAClvQhnN6NiplCitXMOAim-enyrjUk,10076
+datachain/func/func.py,sha256=Au3B51AiJSh7OC2LANNBv220L6vhYu1CTgrEz0Xxkqg,18092
+datachain/func/numeric.py,sha256=L2fq6DgZLTEsqVPrPIf4b84WRxidZvQcBZLMd-kOBPo,6932
+datachain/func/path.py,sha256=9Jas35QhEtRai4l54hMqVvuJsqxHvOx88oo4vym1H_I,4077
+datachain/func/random.py,sha256=t7jwXsI8-hy0qAdvjAntgzy-AHtTAfozlZ1CpKR-QZE,458
+datachain/func/string.py,sha256=kXkPHimtA__EVg_Th1yldGaLJpw4HYVhIeYtKy3DuyQ,7406
+datachain/func/window.py,sha256=ImyRpc1QI8QUSPO7KdD60e_DPVo7Ja0G5kcm6BlyMcw,1584
+datachain/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datachain/lib/arrow.py,sha256=EoGpA9Lqv5xUmc5P0SQYfjBQ-wSNHN16PvkZ6exMcsI,10821
+datachain/lib/audio.py,sha256=hHG29vqrV389im152wCjh80d0xqXGGvFnUpUwkzZejQ,7385
+datachain/lib/clip.py,sha256=nF8-N6Uz0MbAsPJBY2iXEYa3DPLo80OOer5SRNAtcGM,6149
+datachain/lib/data_model.py,sha256=srz0pfFohSXwFnt5OMi1fNjSbKkFq8vzkcO0n4PHxlQ,3904
+datachain/lib/dataset_info.py,sha256=Yl11SqW47Uf-WkuB9zTVUS17ALGTQiWkvxzKLN2GWQ4,3288
+datachain/lib/file.py,sha256=-iLKN2lnn4fHxDGk91b4FDjTgQFUvk3KCX-ZgxvKh-k,49277
+datachain/lib/hf.py,sha256=jmyqRDXdksojUJCiU_2XFSIoMzzDJAZQs9xr-sEwEJc,7281
+datachain/lib/image.py,sha256=xKyVsFKi1Shji7oluvd4Ibr3Atiz-Q0MNJhIsXeGcMI,3197
+datachain/lib/listing.py,sha256=pXRzHCUxX0b1sZrFWPN77bHY69Hrn6rFwr5IzSxuhvI,7060
+datachain/lib/listing_info.py,sha256=lnl5oQpkt7kVHyb06TdWhy5Fi2A0gBVIbnECgDtvLec,1079
+datachain/lib/meta_formats.py,sha256=HO8WxPeQLdVSNh2sOy0XSc66yRhBjFOjNgkzIVFIFBk,6345
+datachain/lib/model_store.py,sha256=GQoqNTPNwizX1hB4peKWs-h0VovJxX8dt1MdnTPI7bQ,3713
+datachain/lib/namespaces.py,sha256=d4Zt2mYdGFctkA20SkB1woUxrNI4JwSxruxUGKwfauc,3731
+datachain/lib/projects.py,sha256=FfBfGoWvy1SccCQW2ITKdDA6V03FbnRCusOeHdPHr6Y,4059
+datachain/lib/pytorch.py,sha256=Ux_Gpl0dUJuME7d8qJSpgVQL0kQv6-lnq22KXQCy1NE,8069
+datachain/lib/settings.py,sha256=maMtywOUetJvEApDiMVfTTq-oaRNvUIfDCrqZwFL2GE,7559
+datachain/lib/signal_schema.py,sha256=Pi5HMYisCio4dq3zmKn1ijDf6-9vHvZONVfa9kVWSu0,47623
+datachain/lib/tar.py,sha256=MLcVjzIgBqRuJacCNpZ6kwSZNq1i2tLyROc8PVprHsA,999
+datachain/lib/text.py,sha256=uZom8qXfrv9QYvuDrvd0PuvPmj6qCsjVUwZSNr60BI4,1242
+datachain/lib/udf.py,sha256=CUodkCS3GCndDSjFwRh4m9zB9SsaPpOHk_zVaxJdm28,19356
+datachain/lib/udf_signature.py,sha256=C3J0E_01elO81swP7Sy6Ne5P__BbHTfWc_ena-CCcmQ,8563
+datachain/lib/utils.py,sha256=506ULwIQYwT1DWPyI3WwWS10qQ-qgUf39GDDoVcT2G0,5220
+datachain/lib/video.py,sha256=7Q4oWvf4_HMX2QrkjZwTE0e8mZpoQlBxOxV9k2ubRag,6804
+datachain/lib/webdataset.py,sha256=GKq_R9wDIH8Ckn9Hh-ipDu9X6eIsCPvcR95nCqSZ9yA,7315
+datachain/lib/webdataset_laion.py,sha256=3vMWW_MrMvTA4FvwJVRHHL-Hxf9bq0wmLzefXyNW2bM,2486
+datachain/lib/convert/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datachain/lib/convert/flatten.py,sha256=_5rjGFnN6t1KCX5ftL5rG7tiiNat7j0SdNqajO15KUY,1539
+datachain/lib/convert/python_to_sql.py,sha256=wfnqJ2vRL5UydNPQHshd82hUONsDBa4XyobCSTGqcEo,3187
+datachain/lib/convert/sql_to_python.py,sha256=Gxc4FylWC_Pvvuawuc2MKZIiuAWI7wje8pyeN1MxRrU,670
+datachain/lib/convert/unflatten.py,sha256=ysMkstwJzPMWUlnxn-Z-tXJR3wmhjHeSN_P-sDcLS6s,2010
+datachain/lib/convert/values_to_tuples.py,sha256=25VmP7b9Fa3BPAlSFFo54O9FAgEAqicmE44R4pPsLcM,7313
+datachain/lib/dc/__init__.py,sha256=xguTjbh3iMDTUjfggCDGYZNA0_1iOVvkc-53f6cUpXc,926
+datachain/lib/dc/csv.py,sha256=fIfj5-2Ix4z5D5yZueagd5WUWw86pusJ9JJKD-U3KGg,4407
+datachain/lib/dc/database.py,sha256=RNXpTUdEyBAtlMJIesVfEzZee5peTDPszZPwFQtqEYA,14817
+datachain/lib/dc/datachain.py,sha256=iYScoUXH1QeqffcoLBtj_RnZwcw9MSH-qF7IogK0b3g,107239
+datachain/lib/dc/datasets.py,sha256=oY1t8QBAaZdhjwR439zZT74hMOspewVCrgdwy6juXng,15321
+datachain/lib/dc/hf.py,sha256=FeruEO176L2qQ1Mnx0QmK4kV0GuQ4xtj717N8fGJrBI,2849
+datachain/lib/dc/json.py,sha256=iJ6G0jwTKz8xtfh1eICShnWk_bAMWjF5bFnOXLHaTlw,2683
+datachain/lib/dc/listings.py,sha256=0XTZERQZ2ErP3LSVg9lF9i3alKebqA1Kip2Zf15unUM,4507
+datachain/lib/dc/pandas.py,sha256=o9rTcZf27-3mCEaDdX1ZzM0I4bSOsu-4mA2zK6rWoS4,1460
+datachain/lib/dc/parquet.py,sha256=wa_VazXotY5RZ8ypC0_M9Qo30tamzXmYeVE6P-NcQ1Y,2375
+datachain/lib/dc/records.py,sha256=WvbaLhMqM9e54gJLLeG54QX5ZXkkBIK3FokojLTSbZc,2974
+datachain/lib/dc/storage.py,sha256=zfVMkYqwmhI4bnOqyO6bW5gg_DfdYPM7ltWLTHDjGZo,9737
+datachain/lib/dc/storage_pattern.py,sha256=TqaDb5yq050W9IxpESz9iotjs0R__i5ngRtVo5BmJ-8,7645
+datachain/lib/dc/utils.py,sha256=Ya2rqVj0UHzY_gxdZrIKnmoPZaKgLs397Y1o6M6kGQE,4273
+datachain/lib/dc/values.py,sha256=-EI3xYUNzfwzogbW8WdHX0XbWev-je6_5-CnDsLRcF4,1399
+datachain/model/__init__.py,sha256=R9faX5OHV1xh2EW-g2MPedwbtEqt3LodJRyluB-QylI,189
+datachain/model/bbox.py,sha256=7RT-xsKL8Rywy7l_R9DhzsTF4eQx35VCNdbaQEFDcVc,9362
+datachain/model/pose.py,sha256=rjquA6M-I-Y30Xm6YSkGv1OY52hJZmR2AuxbIpE5uD0,3865
+datachain/model/segment.py,sha256=NhcEYB_KVa0aLQYiZ4jEwkylH9QBLd8fZhmg6PVnx1Y,1967
+datachain/model/utils.py,sha256=5elwCKleOO6CZM0IuWjFykPekrhc5m7V4jSIOcgGMms,6733
+datachain/model/ultralytics/__init__.py,sha256=EvcNX9qUyxKXXlKCPpsXeRrabyXk5E9EkN-tyiYkfS4,750
+datachain/model/ultralytics/bbox.py,sha256=C-aDiBhVa_ML2oERWvksRkyMU1XuYSpb6eItHB5q0qc,4764
+datachain/model/ultralytics/pose.py,sha256=pvoXrWWUSWT_UBaMwUb5MBHAY57Co2HFDPigFYNZWUA,3392
+datachain/model/ultralytics/segment.py,sha256=v9_xDxd5zw_I8rXsbl7yQXgEdTs2T38zyY_Y4XGN8ok,3194
+datachain/query/__init__.py,sha256=7DhEIjAA8uZJfejruAVMZVcGFmvUpffuZJwgRqNwe-c,263
+datachain/query/batch.py,sha256=ugTlSFqh_kxMcG6vJ5XrEzG9jBXRdb7KRAEEsFWiPew,4190
+datachain/query/dataset.py,sha256=CxHOFJwMhlmZUHhMKVw22HAuMOgORiQJ-25qJWCjk7g,69548
+datachain/query/dispatch.py,sha256=VOq0Lzp-Dmy4dkVIeqL5ekYIe5QqcN4o0_xtPJGI-X4,16907
+datachain/query/metrics.py,sha256=qOMHiYPTMtVs2zI-mUSy8OPAVwrg4oJtVF85B9tdQyM,810
+datachain/query/params.py,sha256=JkVz6IKUIpF58JZRkUXFT8DAHX2yfaULbhVaGmHKFLc,826
+datachain/query/queue.py,sha256=kCetMG6y7_ynV_jJDAXkLsf8WsVZCEk1fAuQGd7yTOo,3543
+datachain/query/schema.py,sha256=Cn1keXjktptAbEDbHlxSzdoCu5H6h_Vzp_DtNpMSr5w,6697
+datachain/query/session.py,sha256=IN0ruNMxEU5K5xp_cL4-zMOxeYD0EnCAUunHa0zNdJ4,12793
+datachain/query/udf.py,sha256=SsVHb_TksVygWsqKX7SX8SOnjOyyd8n5NPF7i-EKb2Q,1364
+datachain/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datachain/remote/studio.py,sha256=4voPFVDXAU6BSBHDAvB_LTYiCACA6Zr0IfYnDjrnN6s,16737
+datachain/sql/__init__.py,sha256=8D2omsBiATt8bjLjGo6jBEtaKEkOlnlNFWhVryHMDv0,388
+datachain/sql/postgresql_dialect.py,sha256=pDTfH8xaXz5xZsq8O1aQUvWLRIv_ogYeAqtmKlPp3Rw,280
+datachain/sql/postgresql_types.py,sha256=ryb_0lzuA9UOJ_B6nW9Yb8nJjzeSmEItAL_Ceue65lc,627
+datachain/sql/selectable.py,sha256=cTc60qVoAwqqss0Vop8Lt5Z-ROnM1XrQmL_GLjRxhXs,1765
+datachain/sql/types.py,sha256=u0WNBjZafuG54lHQUtSSUOd14vHASQ1Nk3bKYjgPpJs,16152
+datachain/sql/utils.py,sha256=rzlJw08etivdrcuQPqNVvVWhuVSyUPUQEEc6DOhu258,818
+datachain/sql/default/__init__.py,sha256=XQ2cEZpzWiABqjV-6yYHUBGI9vN_UHxbxZENESmVAWw,45
+datachain/sql/default/base.py,sha256=QD-31C6JnyOXzogyDx90sUhm7QvgXIYpeHEASH84igU,628
+datachain/sql/functions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datachain/sql/functions/aggregate.py,sha256=3AQdA8YHPFdtCEfwZKQXTT8SlQWdG9gD5PBtGN3Odqs,944
+datachain/sql/functions/array.py,sha256=eRWpDRItwIG87-AU7jb8WuiR-MGuhklVxWwR7t97GvY,2050
+datachain/sql/functions/conditional.py,sha256=q7YUKfunXeEldXaxgT-p5pUTcOEVU_tcQ2BJlquTRPs,207
+datachain/sql/functions/numeric.py,sha256=BK2KCiPSgM2IveCq-9M_PG3CtPBlztaS9TTn1LGzyLs,1250
+datachain/sql/functions/path.py,sha256=zixpERotTFP6LZ7I4TiGtyRA8kXOoZmH1yzH9oRW0mg,1294
+datachain/sql/functions/random.py,sha256=vBwEEj98VH4LjWixUCygQ5Bz1mv1nohsCG0-ZTELlVg,271
+datachain/sql/functions/string.py,sha256=E-T9OIzUR-GKaLgjZsEtg5CJrY_sLf1lt1awTvY7w2w,1426
+datachain/sql/sqlite/__init__.py,sha256=PsLaDSij9a03VxGSpagpNl7NQsGtgm72ArUeALZONoc,183
+datachain/sql/sqlite/base.py,sha256=llTx_wzuXo7pCM95eq8xvZ1iFOeKlcN7WVyJyHlBrW8,21840
+datachain/sql/sqlite/types.py,sha256=EDmvEJz8oBQnv0r_ST-M1jQVPC39ceAC5pgzSB9vnkg,1821
+datachain/sql/sqlite/vector.py,sha256=ncW4eu2FlJhrP_CIpsvtkUabZlQdl2D5Lgwy_cbfqR0,469
+datachain/toolkit/__init__.py,sha256=eQ58Q5Yf_Fgv1ZG0IO5dpB4jmP90rk8YxUWmPc1M2Bo,68
+datachain/toolkit/split.py,sha256=9HHZl0fGs5Zj8b9l2L3IKf0AiiVNL9SnWbc2rfDiXRA,3710
+datachain/torch/__init__.py,sha256=gIS74PoEPy4TB3X6vx9nLO0Y3sLJzsA8ckn8pRWihJM,579
+datachain-0.39.0.dist-info/licenses/LICENSE,sha256=8DnqK5yoPI_E50bEg_zsHKZHY2HqPy4rYN338BHQaRA,11344
+datachain-0.39.0.dist-info/METADATA,sha256=0rSXP0DTd51AK74TiFsQL1ftNgY6huoeHUu8IvqWEHI,13635
+datachain-0.39.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+datachain-0.39.0.dist-info/entry_points.txt,sha256=0GMJS6B_KWq0m3VT98vQI2YZodAMkn4uReZ_okga9R4,49
+datachain-0.39.0.dist-info/top_level.txt,sha256=lZPpdU_2jJABLNIg2kvEOBi8PtsYikbN1OdMLHk8bTg,10
+datachain-0.39.0.dist-info/RECORD,,

datachain/cli/commands/query.py

@@ -1,54 +0,0 @@
-import os
-import sys
-import traceback
-from typing import TYPE_CHECKING, Optional
-
-if TYPE_CHECKING:
-    from datachain.catalog import Catalog
-
-
-def query(
-    catalog: "Catalog",
-    script: str,
-    parallel: Optional[int] = None,
-    params: Optional[dict[str, str]] = None,
-) -> None:
-    from datachain.data_storage import JobQueryType, JobStatus
-
-    with open(script, encoding="utf-8") as f:
-        script_content = f.read()
-
-    if parallel is not None:
-        # This also sets this environment variable for any subprocesses
-        os.environ["DATACHAIN_SETTINGS_PARALLEL"] = str(parallel)
-
-    python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
-    python_executable = sys.executable
-
-    job_id = catalog.metastore.create_job(
-        name=os.path.basename(script),
-        query=script_content,
-        query_type=JobQueryType.PYTHON,
-        status=JobStatus.RUNNING,
-        python_version=python_version,
-        params=params,
-    )
-
-    try:
-        catalog.query(
-            script_content,
-            python_executable=python_executable,
-            params=params,
-            job_id=job_id,
-        )
-    except Exception as e:
-        error_message = str(e)
-        error_stack = traceback.format_exc()
-        catalog.metastore.set_job_status(
-            job_id,
-            JobStatus.FAILED,
-            error_message=error_message,
-            error_stack=error_stack,
-        )
-        raise
-    catalog.metastore.set_job_status(job_id, JobStatus.COMPLETE)

datachain/query/utils.py

@@ -1,36 +0,0 @@
-from typing import Optional, Union
-
-import sqlalchemy as sa
-
-ColT = Union[sa.Column, sa.ColumnElement, sa.TextClause, sa.Label]
-
-
-def column_name(col: ColT) -> str:
-    """Returns column name from column element."""
-    return col.name if isinstance(col, (sa.Column, sa.Label)) else str(col)
-
-
-def get_query_column(query: sa.Select, name: str) -> Optional[ColT]:
-    """Returns column element from query by name or None if column not found."""
-    return next((col for col in query.inner_columns if column_name(col) == name), None)
-
-
-def get_query_id_column(query: sa.Select) -> Optional[sa.ColumnElement]:
-    """Returns ID column element from query or None if column not found."""
-    col = get_query_column(query, "sys__id")
-    return col if col is not None and isinstance(col, sa.ColumnElement) else None
-
-
-def select_only_columns(query: sa.Select, *names: str) -> sa.Select:
-    """Returns query selecting defined columns only."""
-    if not names:
-        return query
-
-    cols: list[ColT] = []
-    for name in names:
-        col = get_query_column(query, name)
-        if col is None:
-            raise ValueError(f"Column '{name}' not found in query")
-        cols.append(col)
-
-    return query.with_only_columns(*cols)